From 3142a2f6541641488d572da453efba728a6c0539 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 11:10:37 -0400 Subject: [PATCH 01/75] fix: fail denied input downloads once per batch (#177) --- api/src/download.test.ts | 70 +++++++++++++++++++++++++++++++++++++++- api/src/job.ts | 40 +++++++++++++++++++---- 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/api/src/download.test.ts b/api/src/download.test.ts index 5372a40c..2c7248e8 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll } from 'bun:test'; +import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, spyOn } from 'bun:test'; import * as fsp from 'fs/promises'; import * as path from 'path'; import * as os from 'os'; @@ -439,6 +439,74 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(contents).toBe('hi'); }); + it.each([401, 403])('does not retry an HTTP %i authorization denial', async status => { + const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; + let requests = 0; + routes.set('/sessions/previous/objects/denied', { + status, + onRequest: () => { requests++; }, + }); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + + await expect(job.downloadAndWriteFile(file, 5, 1)).rejects.toThrow(`HTTP error: ${status}`); + expect(requests).toBe(1); + expect(await fsp.readdir(tmpDir)).toEqual([]); + }); + + it.each([404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + const file: TFile = { id: 'transient', storage_session_id: 'previous', name: 'ready.txt' }; + let requests = 0; + const route: Route = { + status, + body: 'ready', + onRequest: () => { if (++requests === 2) route.status = 200; }, + }; + routes.set('/sessions/previous/objects/transient', route); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + + await expect(job.downloadAndWriteFile(file, 5, 1)).resolves.toBe('ready.txt'); + expect(requests).toBe(2); + expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); + }); + + it('accounts for a denied 240-file batch once and stops queued downloads', async () => { + const files: TFile[] = Array.from({ length: 240 }, (_, index) => ({ + id: `file-${index}`, storage_session_id: 'previous', name: `file-${index}.txt`, + })); + let requests = 0; + routes.set('/sessions/previous/objects', { status: 200, body: '[]' }); + for (const file of files) { + routes.set(`/sessions/previous/objects/${file.id}`, { + status: 403, + delayMs: file.id === 'file-0' ? 0 : 30, + onRequest: () => { requests++; }, + }); + } + let dirty = false; + const job = makeJob(files, sessionWorkspaceAt(tmpDir, 'batch-test', () => { dirty = true; })); + const log = (job as unknown as { log: import('pino').Logger }).log; + const errorLog = spyOn(log, 'error'); + const originalConcurrency = config.prime_concurrency; + config.prime_concurrency = 8; + try { + await expect(job.prime()).rejects.toBeInstanceOf(SessionWorkspaceDirtyError); + expect(dirty).toBe(true); + expect(requests).toBeGreaterThan(0); + expect(requests).toBeLessThanOrEqual(8); + expect(errorLog).toHaveBeenCalledTimes(1); + expect(errorLog).toHaveBeenCalledWith(expect.objectContaining({ + inputCount: 240, completed: 0, failed: 1, cancelled: 7, notStarted: 232, + }), 'Input preparation batch failed'); + expect(await fsp.readdir(tmpDir)).toEqual([]); + } finally { + errorLog.mockRestore(); + config.prime_concurrency = originalConcurrency; + await job.cleanup(); + } + }); + it('fails when the server keeps 404-ing past the retry cap (no phantom write)', async () => { const file: TFile = { id: 'missing-id', diff --git a/api/src/job.ts b/api/src/job.ts index 610b748b..8121f51d 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -60,6 +60,14 @@ export { const AUTO_LOAD_DIRKEEP_TIMEOUT_MS = 10000; const AUTO_LOAD_DIRKEEP_RETRIES = 2; +/** Replaying the same sealed grant cannot repair an authorization denial. */ +class InputAuthorizationError extends Error { + constructor(status: number) { + super(`HTTP error: ${status}`); + this.name = 'InputAuthorizationError'; + } +} + /** * Bridges a `fetch` response body to a Node-stream Readable. The types at the * module boundary (Node's `stream/web` vs. lib.dom) don't overlap cleanly, @@ -993,11 +1001,18 @@ export class Job { submissionDir: this.submissionDir, identity: this.jobIdentity, }; + const startedAt = performance.now(); + let started = 0; + let completed = 0; + let cancelled = 0; let firstFailure: { error: unknown } | undefined; const runFileOperation = async (operation: () => Promise): Promise => { + started++; try { await operation(); + completed++; } catch (error) { + if (firstFailure) cancelled++; if (!firstFailure) { firstFailure = { error }; controller.abort(error); @@ -1031,6 +1046,15 @@ export class Job { Array.from({ length: workerCount }, () => runPrimeWorker()), ); if (firstFailure) { + this.log.error({ + inputCount: fileOps.length, + completed, + failed: 1, + cancelled, + notStarted: fileOps.length - started, + durationMs: Math.round(performance.now() - startedAt), + err: firstFailure.error, + }, 'Input preparation batch failed'); if (this.session) { /* A sibling may already have atomically replaced its destination. The * workspace now matches neither the previous checkpoint nor the full @@ -1302,6 +1326,9 @@ export class Job { if (!response.ok) { await response.body?.cancel().catch(() => {}); + if (response.status === 401 || response.status === 403) { + throw new InputAuthorizationError(response.status); + } throw new Error(`HTTP error: ${response.status}`); } @@ -1366,11 +1393,10 @@ export class Job { try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw abortReason(operation.signal); } - /* ValidationError is deterministic — a bad Content-Disposition - * filename will fail identically on every retry. Abort fast - * (cleanup + rethrow) instead of burning ~7.5s on exponential - * backoff and surfacing the error as a generic download failure. */ - if (error instanceof ValidationError) { + /* Invalid filenames and authorization denials cannot recover by + * replaying the same request. Abort the batch before exponential + * backoff amplifies the failure across its remaining files. */ + if (error instanceof ValidationError || error instanceof InputAuthorizationError) { try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw error; } @@ -1383,7 +1409,9 @@ export class Job { } } - this.log.error({ fileId: file.id, maxRetries, err: lastError }, 'Failed to download file'); + if (!context?.signal) { + this.log.error({ fileId: file.id, maxRetries, err: lastError }, 'Failed to download file'); + } try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw lastError ?? new Error(`Failed to download input ${file.id}`); } From 794df9e444acc2a8a2c6a0fbb65ddc0976c8ef41 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 15:47:10 -0400 Subject: [PATCH 02/75] fix: preserve retries for transient egress ledger conflicts (#179) * fix: distinguish retryable ledger contention from scope denials * fix: preserve error classification through marker discovery and relay * test: exercise classified denials through gateway configuration * fix: honor bounded gateway retry hints during object downloads --- api/src/download.test.ts | 79 +++++++++++++++++++++++++++++- api/src/egress.ts | 1 + api/src/job.ts | 23 +++++++-- packages/code/src/relay.test.ts | 35 +++++++++++++ packages/code/src/relay.ts | 6 +++ service/src/egress-gateway.test.ts | 36 +++++++++++++- service/src/egress-gateway.ts | 4 ++ service/src/egress-grant.ts | 4 +- service/src/egress-ledger.ts | 2 +- 9 files changed, 182 insertions(+), 8 deletions(-) diff --git a/api/src/download.test.ts b/api/src/download.test.ts index 2c7248e8..bb8ec7e9 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -55,6 +55,7 @@ function makeRuntime(): Runtime { function makeJob(files: TFile[] = [], session?: SessionWorkspace): Job { return new Job({ session_id: 'test-session', + egress_grant: 'test-grant', runtime: makeRuntime(), files, args: [], @@ -440,10 +441,12 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { }); it.each([401, 403])('does not retry an HTTP %i authorization denial', async status => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; let requests = 0; routes.set('/sessions/previous/objects/denied', { status, + headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' }, onRequest: () => { requests++; }, }); const job = makeJob([file]); @@ -454,7 +457,8 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.readdir(tmpDir)).toEqual([]); }); - it.each([404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + it.each([403, 404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const file: TFile = { id: 'transient', storage_session_id: 'previous', name: 'ready.txt' }; let requests = 0; const route: Route = { @@ -471,7 +475,79 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); }); + it.each([false, true])('honors conflict retry hints with cancellation=%s', async cancel => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; + const controller = new AbortController(); + const timestamps: number[] = []; + let timer: ReturnType | undefined; + const route: Route = { + status: 503, body: 'ready', + headers: { 'X-CodeAPI-Error-Code': 'ledger_conflict', 'Retry-After': '1' }, + onRequest: () => { + timestamps.push(performance.now()); + if (timestamps.length === 2) route.status = 200; + else if (cancel) timer = setTimeout(() => controller.abort(new Error('cancelled retry')), 25); + }, + }; + routes.set('/sessions/previous/objects/retry-hint', route); + const file: TFile = { id: 'retry-hint', storage_session_id: 'previous', name: 'ready.txt' }; + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + try { + const result = job.downloadAndWriteFile(file, 5, 1, { + submissionDir: tmpDir, identity: fallbackSandboxIdentity(), signal: controller.signal, + }); + if (cancel) { + await expect(result).rejects.toThrow('cancelled retry'); + expect(timestamps).toHaveLength(1); + expect(await fsp.readdir(tmpDir)).toEqual([]); + } else { + await expect(result).resolves.toBe('ready.txt'); + expect(timestamps).toHaveLength(2); + expect(timestamps[1] - timestamps[0]).toBeGreaterThanOrEqual(900); + } + } finally { + clearTimeout(timer); + } + }); + + it('does not retry an unclassified direct file-server denial', async () => { + config.egress_gateway_url = ''; + let requests = 0; + const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; + routes.set('/sessions/previous/objects/denied', { + status: 403, onRequest: () => { requests++; }, + }); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + await expect(job.downloadAndWriteFile(file, 5, 1)).rejects.toThrow('HTTP error: 403'); + expect(requests).toBe(1); + }); + + it.each(['legacy', 'classified', 'direct'])('handles %s marker denials before priming', async mode => { + config.egress_gateway_url = mode === 'direct' ? '' : `http://127.0.0.1:${serverPort}`; + let requests = 0; + const route: Route = { + status: 403, body: '[]', + headers: mode === 'classified' ? { 'X-CodeAPI-Error-Code': 'scope_mismatch' } : {}, + onRequest: () => { if (++requests === 2) route.status = 200; }, + }; + routes.set('/sessions/previous/objects', route); + const file: TFile = { id: 'ready', storage_session_id: 'previous', name: 'ready.txt' }; + routes.set('/sessions/previous/objects/ready', { status: 200, body: 'ready' }); + const job = makeJob([file], sessionWorkspaceAt(tmpDir, 'marker-retry')); + if (mode === 'legacy') { + await job.prime(); + expect(requests).toBe(2); + expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); + } else { + await expect(job.prime()).rejects.toThrow('HTTP error loading .dirkeep markers: 403'); + expect(requests).toBe(1); + } + }); + it('accounts for a denied 240-file batch once and stops queued downloads', async () => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const files: TFile[] = Array.from({ length: 240 }, (_, index) => ({ id: `file-${index}`, storage_session_id: 'previous', name: `file-${index}.txt`, })); @@ -480,6 +556,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { for (const file of files) { routes.set(`/sessions/previous/objects/${file.id}`, { status: 403, + headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' }, delayMs: file.id === 'file-0' ? 0 : 30, onRequest: () => { requests++; }, }); diff --git a/api/src/egress.ts b/api/src/egress.ts index 442872ab..69ecf6db 100644 --- a/api/src/egress.ts +++ b/api/src/egress.ts @@ -1 +1,2 @@ export const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +export const EGRESS_ERROR_CODE_HEADER = 'X-CodeAPI-Error-Code'; diff --git a/api/src/job.ts b/api/src/job.ts index 8121f51d..bdcd5061 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -15,7 +15,7 @@ import { getRuntimes } from './runtime'; import { execute } from './nsjail'; import { config } from './config'; import { internalServiceHeaders } from './internal-service-auth'; -import { EGRESS_GRANT_HEADER } from './egress'; +import { EGRESS_GRANT_HEADER, EGRESS_ERROR_CODE_HEADER } from './egress'; import { injectTraceHeaders } from './telemetry'; import { applyReadOnlyInputPermissions, @@ -1196,6 +1196,11 @@ export class Job { } } + private isLegacyGatewayDenial(response: Response): boolean { + return !!config.egress_gateway_url && response.status === 403 && + !response.headers.has(EGRESS_ERROR_CODE_HEADER); + } + /** * Fetches normalized objects for one inherited session and returns the * `.dirkeep` markers belonging to exactly that session. Guards against: @@ -1219,7 +1224,7 @@ export class Job { signal: controller.signal, }, ); - if (res.status === 503 && attempt < AUTO_LOAD_DIRKEEP_RETRIES) { + if ((res.status === 503 || this.isLegacyGatewayDenial(res)) && attempt < AUTO_LOAD_DIRKEEP_RETRIES) { await res.body?.cancel().catch(() => {}); const retryAfterSeconds = Number(res.headers.get('retry-after')); await sleep( @@ -1326,7 +1331,10 @@ export class Job { if (!response.ok) { await response.body?.cancel().catch(() => {}); - if (response.status === 401 || response.status === 403) { + /* Older gateways also used 403 for transient ledger contention. + * Only classify 403 as permanent when the gateway distinguishes it. */ + if (response.status === 401 || + (response.status === 403 && !this.isLegacyGatewayDenial(response))) { throw new InputAuthorizationError(response.status); } throw new Error(`HTTP error: ${response.status}`); @@ -1402,7 +1410,14 @@ export class Job { } lastError = error instanceof Error ? error : new Error(String(error)); if (attempt < maxRetries) { - const delay = retryDelay * Math.pow(2, attempt - 1); + const backoff = retryDelay * Math.pow(2, attempt - 1); + const retryAfterSeconds = response?.status === 503 + ? Number(response.headers.get('retry-after')) : NaN; + /* Use the same bounded retry hint as marker discovery, without + * shortening exponential backoff or bypassing batch cancellation. */ + const delay = Number.isFinite(retryAfterSeconds) + ? Math.max(backoff, Math.min(1000, Math.max(0, retryAfterSeconds * 1000))) + : backoff; this.log.warn({ fileId: file.id, attempt, maxRetries, delay, err: lastError }, 'Download failed, retrying'); await sleep(delay, operation.signal); } diff --git a/packages/code/src/relay.test.ts b/packages/code/src/relay.test.ts index c0814408..a9b81fd0 100644 --- a/packages/code/src/relay.test.ts +++ b/packages/code/src/relay.test.ts @@ -381,3 +381,38 @@ test('file relay rejects plaintext remote upstreams', async () => { /HTTPS unless it is a local development host/, ); }); + +for (const [status, reason] of [[403, 'scope_mismatch'], [503, 'ledger_conflict']] as const) { + test(`file relay preserves ${reason} classification`, async () => { + const upstream = createServer((_req, res) => { + res.writeHead(status, { + 'X-CodeAPI-Error-Code': reason, + 'Retry-After': '1', + 'X-Internal-Secret': 'must-not-forward', + }).end('rejected'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', + maxBytes: 1024, timeoutMs: 1000, + }); + try { + const response = await fetch(`${relay.url}/sessions/storage-1/objects/file-1`, { + headers: { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-1', + }, + }); + assert.equal(response.status, status); + assert.equal(response.headers.get('x-codeapi-error-code'), reason); + assert.equal(response.headers.get('retry-after'), '1'); + assert.equal(response.headers.get('x-internal-secret'), null); + assert.equal(await response.text(), 'rejected'); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close(error => error ? reject(error) : resolve()), + ); + } + }); +} diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts index 255845e9..14e91280 100644 --- a/packages/code/src/relay.ts +++ b/packages/code/src/relay.ts @@ -240,6 +240,12 @@ export async function startFileRelay( )!, } : {}), + ...(upstreamResponse.headers.has('x-codeapi-error-code') + ? { 'X-CodeAPI-Error-Code': upstreamResponse.headers.get('x-codeapi-error-code')! } + : {}), + ...(upstreamResponse.headers.has('retry-after') + ? { 'Retry-After': upstreamResponse.headers.get('retry-after')! } + : {}), 'Content-Length': String(body.length), }); response.end(body); diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index 99303850..717309d7 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -1,6 +1,6 @@ process.env.CODEAPI_EGRESS_GATEWAY_AUTOSTART = 'false'; -import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import { afterAll, beforeAll, beforeEach, describe, expect, test, spyOn } from 'bun:test'; import crypto from 'crypto'; import RedisMock from 'ioredis-mock'; import type { Server } from 'http'; @@ -435,6 +435,39 @@ describe('egress gateway routes', () => { } }); + test('reports exhausted ledger conflicts as retryable without forwarding the read', async () => { + const redis = new RedisMock(); + env.EGRESS_LEDGER_REQUIRED = true; + setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); + const duplicate = redis.duplicate.bind(redis); + const duplication = spyOn(redis, 'duplicate').mockImplementation(() => { + const connection = duplicate(); + const transaction = { + set: () => transaction, + exec: async () => null, + }; + spyOn(connection, 'multi').mockImplementation(() => transaction as never); + return connection; + }); + try { + await createEgressLedger(claims()); + const readSession = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const response = await gatewayFetch(`/sessions/${readSession}/objects?detail=normalized`, { + headers: grantHeader(), + }); + expect(response.status).toBe(503); + expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('ledger_conflict'); + expect(response.headers.get('Retry-After')).toBe('1'); + expect(upstreamCalls).toHaveLength(0); + expect((await assertEgressGrantActive(claims())).request_count).toBe(0); + } finally { + duplication.mockRestore(); + setEgressLedgerRedisForTest(null); + redis.disconnect(); + env.EGRESS_LEDGER_REQUIRED = false; + } + }); + test('lists only scoped objects and injects internal credentials', async () => { upstreamResponse = Response.json([ { id: 'file_123', name: 'inputs/data.csv', storage_session_id: 'sess_input' }, @@ -557,6 +590,7 @@ describe('egress gateway routes', () => { }); expect(response.status).toBe(403); + expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('scope_mismatch'); expect(upstreamCalls).toHaveLength(0); } finally { await redis.disconnect(); diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index d4d3e713..13a6de95 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -6,6 +6,7 @@ import { Readable } from 'stream'; import { env } from './config'; import { EGRESS_GRANT_HEADER, + EGRESS_ERROR_CODE_HEADER, EgressGrantError, egressGrantFromExecutionClaims, openEgressGrant, @@ -157,6 +158,7 @@ app.use((req: Request, res: Response, next: NextFunction) => { function errorStatus(error: EgressGrantError): number { if (error.reason === 'missing_secret' || error.reason === 'weak_secret') return 500; + if (error.reason === 'ledger_conflict') return 503; if (error.reason === 'malformed') return 400; if (error.reason === 'expired') return 401; return 403; @@ -165,6 +167,8 @@ function errorStatus(error: EgressGrantError): number { function sendEgressError(req: Request, res: Response, error: unknown): Response { if (error instanceof EgressGrantError) { const statusCode = errorStatus(error); + res.setHeader(EGRESS_ERROR_CODE_HEADER, error.reason); + if (error.reason === 'ledger_conflict') res.setHeader('Retry-After', '1'); logger.warn('Rejected egress gateway request', { requestId: requestId(res), reason: error.reason, diff --git a/service/src/egress-grant.ts b/service/src/egress-grant.ts index 8d8b1352..8148bec7 100644 --- a/service/src/egress-grant.ts +++ b/service/src/egress-grant.ts @@ -3,6 +3,7 @@ import type { ExecutionManifestClaims, ExecutionManifestInputFile } from './exec import type * as t from './types'; export const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +export const EGRESS_ERROR_CODE_HEADER = 'X-CodeAPI-Error-Code'; export const EGRESS_GRANT_VERSION = 1; const TOKEN_PREFIX = 'ceg1'; @@ -19,7 +20,8 @@ export type EgressGrantErrorReason = | 'malformed' | 'expired' | 'wrong_type' - | 'scope_mismatch'; + | 'scope_mismatch' + | 'ledger_conflict'; export class EgressGrantError extends Error { readonly reason: EgressGrantErrorReason; diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index 24dda87c..b6bbf4fb 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -264,7 +264,7 @@ async function mutateRecord( }); releaseMutationConnection(client); } - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger update conflicted'); + throw new EgressGrantError('ledger_conflict', 'Egress grant ledger update conflicted'); } export async function assertEgressGrantActive(grant: EgressGrantClaims): Promise { From dead07bd466a17dab4bc5b2b3520312fa52b0e03 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 16:36:29 -0400 Subject: [PATCH 03/75] perf: reuse authorized input versions and make egress accounting atomic (#180) * perf: reuse authorized input versions and make egress accounting atomic * perf: resolve authorized input manifests once per execution * test: preserve fetch signature in revocation fixture * fix: isolate shared-grant failures and prevent ledger replay --- .github/workflows/ci.yml | 3 + api/src/config.ts | 3 + api/src/download.test.ts | 44 +++ api/src/http-input-cache.test.ts | 163 +++++++++++ api/src/http-input-cache.ts | 177 +++++++++++ api/src/input-manifest.test.ts | 124 ++++++++ api/src/job.ts | 55 ++++ api/src/metrics.ts | 6 + api/src/session-inputs.ts | 36 ++- docs/INPUT_REUSE.md | 91 ++++++ .../templates/egress-gateway-deployment.yaml | 8 + .../templates/file-server-deployment.yaml | 4 + .../templates/worker-sandbox-deployment.yaml | 8 + helm/codeapi/values.yaml | 11 + launcher/src/main.rs | 4 + packages/code/src/relay.test.ts | 65 +++++ packages/code/src/relay.ts | 16 +- service/src/config.ts | 6 + service/src/egress-gateway.test.ts | 207 ++++++++++--- service/src/egress-gateway.ts | 97 +++++- service/src/egress-ledger-reconnect.test.ts | 63 ++++ service/src/egress-ledger-script.ts | 123 ++++++++ service/src/egress-ledger.test.ts | 81 ++++- service/src/egress-ledger.ts | 276 ++++-------------- service/src/file-download.test.ts | 49 ++++ service/src/file-download.ts | 27 ++ service/src/file-object-resolver.test.ts | 50 ++++ service/src/file-object-resolver.ts | 75 +++++ service/src/file-server.ts | 160 ++++------ service/src/test/redis.ts | 45 +++ 30 files changed, 1690 insertions(+), 387 deletions(-) create mode 100644 api/src/http-input-cache.test.ts create mode 100644 api/src/http-input-cache.ts create mode 100644 api/src/input-manifest.test.ts create mode 100644 docs/INPUT_REUSE.md create mode 100644 service/src/egress-ledger-reconnect.test.ts create mode 100644 service/src/egress-ledger-script.ts create mode 100644 service/src/file-download.test.ts create mode 100644 service/src/file-download.ts create mode 100644 service/src/file-object-resolver.test.ts create mode 100644 service/src/file-object-resolver.ts create mode 100644 service/src/test/redis.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71536caa..f041e2d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,9 @@ jobs: - name: Install dependencies run: bun ci + - name: Install Redis for ledger integration tests + run: sudo apt-get update && sudo apt-get install -y redis-server + - name: Build service run: bun run build diff --git a/api/src/config.ts b/api/src/config.ts index bbc2c184..f3b935ff 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -122,6 +122,9 @@ export const config = { /* Ceiling for the pushed input cache (session-inputs.ts). Eviction is * always safe — a miss simply re-pushes on the next probe — so this is a * disk guard, not a correctness knob. */ + http_input_cache_enabled: process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED === 'true', + http_input_cache_max_objects: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS, 4096), + http_input_cache_max_inflight: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT, 16), input_cache_max_bytes: safeInt( process.env.SANDBOX_INPUT_CACHE_MAX_BYTES, 512 * 1024 * 1024, diff --git a/api/src/download.test.ts b/api/src/download.test.ts index bb8ec7e9..da877dad 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -2,6 +2,8 @@ import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, spyOn import * as fsp from 'fs/promises'; import * as path from 'path'; import * as os from 'os'; +import { createHash, randomUUID } from 'node:crypto'; +import { SESSION_INPUT_CACHE_DIR } from './session-inputs'; import * as semver from 'semver'; import { Job, SessionWorkspaceDirtyError, type TFile } from './job'; import type { Runtime } from './runtime'; @@ -511,6 +513,48 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { } }); + it('reuses versioned bytes in fresh workspaces without bypassing a later denial', async () => { + const previousCache = config.http_input_cache_enabled; + const version = randomUUID(); + const cacheKey = createHash('sha256').update(version).digest('hex'); + const otherDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'codeapi-cache-second-')); + config.http_input_cache_enabled = true; + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; + let reads = 0; + let checks = 0; + const meta: Route = { status: 200, body: JSON.stringify({ cacheable: true, cacheKey, version, size: 8, readOnly: false }), + onRequest: () => { checks++; }, + }; + routes.set('/sessions/previous/objects/cached/metadata', meta); + routes.set('/sessions/previous/objects/cached', { status: 200, body: 'original', + headers: { 'X-CodeAPI-Input-Version': version }, + onRequest: request => { reads++; expect(request.headers.get('x-codeapi-input-version')).toBe(version); }, + }); + const file: TFile = { id: 'cached', storage_session_id: 'previous', name: 'data.txt', input_cache_key: cacheKey }; + try { + const first = makeJob([file]); + asInternals(first).submissionDir = tmpDir; + await first.downloadAndWriteFile(file); + await fsp.writeFile(path.join(tmpDir, 'data.txt'), 'sandbox changed this'); + const second = makeJob([file]); + asInternals(second).submissionDir = otherDir; + await second.downloadAndWriteFile(file); + expect(await fsp.readFile(path.join(otherDir, 'data.txt'), 'utf8')).toBe('original'); + expect(reads).toBe(1); + expect(checks).toBe(2); + meta.status = 403; + meta.headers = { 'X-CodeAPI-Error-Code': 'scope_mismatch' }; + await expect(second.downloadAndWriteFile(file)).rejects.toThrow('HTTP error: 403'); + expect(checks).toBe(3); + expect(reads).toBe(1); + } finally { + config.http_input_cache_enabled = previousCache; + await fsp.rm(otherDir, { recursive: true, force: true }); + await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, cacheKey), { force: true }); + await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, `${cacheKey}.json`), { force: true }); + } + }); + it('does not retry an unclassified direct file-server denial', async () => { config.egress_gateway_url = ''; let requests = 0; diff --git a/api/src/http-input-cache.test.ts b/api/src/http-input-cache.test.ts new file mode 100644 index 00000000..674a03ab --- /dev/null +++ b/api/src/http-input-cache.test.ts @@ -0,0 +1,163 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash, randomUUID } from 'node:crypto'; +import { rm } from 'node:fs/promises'; +import path from 'node:path'; +import { fetchCachedHttpInput } from './http-input-cache'; +import { hasCachedInput, SESSION_INPUT_CACHE_DIR } from './session-inputs'; + +const keys = new Set(); +afterEach(async () => { + for (const key of keys) { + await rm(path.join(SESSION_INPUT_CACHE_DIR, key), { force: true }); + await rm(path.join(SESSION_INPUT_CACHE_DIR, `${key}.json`), { force: true }); + } + keys.clear(); +}); +function fixture(body = 'input', principal = 'tenant/user') { + const version = randomUUID(); + const cacheKey = createHash('sha256').update(principal + version).digest('hex'); + keys.add(cacheKey); + const meta = { cacheable: true, version, cacheKey, size: Buffer.byteLength(body), readOnly: false, name: 'input.txt' }; + let reads = 0; + let authorizations = 0; + return { + meta, + counts: () => ({ reads, authorizations }), + args: { + maxBytes: 8192, maxObjects: 2, maxFileBytes: 8192, maxInflight: 4, + metadata: async () => { authorizations++; return Response.json(meta); }, + download: async (expected: string, _signal: AbortSignal) => { + reads++; + expect(expected).toBe(version); + return new Response(body, { headers: { 'X-CodeAPI-Input-Version': version } }); + }, + }, + }; +} + +function gate() { + let release!: () => void; + const promise = new Promise(resolve => { release = resolve; }); + return { promise, release }; +} + +describe('authorized HTTP input cache', () => { + test('fresh executions reuse bytes but authorize every hit', async () => { + const f = fixture(); + for (let i = 0; i < 3; i++) { + const response = await fetchCachedHttpInput(f.args); + expect(await response?.text()).toBe('input'); + expect(response?.headers.get('content-disposition')).toContain('input.txt'); + } + expect(f.counts()).toEqual({ reads: 1, authorizations: 3 }); + expect(await hasCachedInput('', '', f.meta.cacheKey)).toBe(false); // Cannot bypass preflight using a pushed key. + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(true); + const denied = await fetchCachedHttpInput({ ...f.args, + metadata: async () => new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' } }), + }); + expect(denied?.status).toBe(403); + expect(f.counts().reads).toBe(1); + }); + + test('new versions and principals never reuse an existing version key', async () => { + for (const [body, principal] of [['old', 'tenant/user'], ['new', 'tenant/user'], ['private', 'another-tenant/user']]) { + const f = fixture(body, principal); + expect(await (await fetchCachedHttpInput(f.args))?.text()).toBe(body); + expect(f.counts().reads).toBe(1); + } + }); + + test('coalesces misses while one cancelled caller leaves the remaining reader intact', async () => { + const f = fixture(); + const started = gate(); + const finish = gate(); + let downloads = 0; + let sharedSignal: AbortSignal | undefined; + const args = { ...f.args, download: async (version: string, signal: AbortSignal) => { + downloads++; sharedSignal = signal; started.release(); + await finish.promise; + return f.args.download(version, signal); + } }; + const controller = new AbortController(); + const first = fetchCachedHttpInput({ ...args, signal: controller.signal }); + const second = fetchCachedHttpInput(args); + await started.promise; + await new Promise(resolve => setTimeout(resolve, 10)); + controller.abort(new Error('first cancelled')); + await expect(first).rejects.toThrow('first cancelled'); + expect(sharedSignal?.aborted).toBe(false); + finish.release(); + expect(await (await second)?.text()).toBe('input'); + expect(downloads).toBe(1); + expect(f.counts().authorizations).toBe(2); + }); + + test('a shared fill does not propagate its creator grant denial to a valid waiter', async () => { + const f = fixture(); + const started = gate(); + const finish = gate(); + const denied = fetchCachedHttpInput({ ...f.args, download: async () => { + started.release(); + await finish.promise; + return new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'request_budget_exceeded' } }); + } }); + await started.promise; + const valid = fetchCachedHttpInput(f.args); + await Bun.sleep(10); + finish.release(); + expect((await denied)?.status).toBe(403); + // Job.fetchInputObject uses the waiter's own normal download on undefined. + expect(await valid).toBeUndefined(); + expect(await (await f.args.download(f.meta.version, new AbortController().signal)).text()).toBe('input'); + expect(f.counts().authorizations).toBe(2); + }); + + test('last-reader cancellation aborts the upstream fill without publishing', async () => { + const f = fixture(); + const started = gate(); + const aborted = gate(); + const controller = new AbortController(); + const pending = fetchCachedHttpInput({ ...f.args, signal: controller.signal, + download: async (_version, signal) => { + started.release(); + return new Promise((_resolve, reject) => signal.addEventListener('abort', () => { + aborted.release(); reject(signal.reason); + }, { once: true })); + }, + }); + await started.promise; + controller.abort(new Error('cancel fill')); + await expect(pending).rejects.toThrow('cancel fill'); + await aborted.promise; + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + }); + + test('changed-version and oversized responses are never published', async () => { + const f = fixture(); + const changed = await fetchCachedHttpInput({ ...f.args, download: async () => new Response('changed', { + headers: { 'X-CodeAPI-Input-Version': randomUUID() }, + }) }); + expect(changed).toBeUndefined(); + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + await expect(fetchCachedHttpInput({ ...f.args, download: async () => new Response('too many bytes', { + headers: { 'X-CodeAPI-Input-Version': f.meta.version }, + }) })).rejects.toThrow(); + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + }); + + test('cache quotas evict old entries and preserve an already-open reader', async () => { + const first = fixture('a'.repeat(4000)); + const second = fixture('b'.repeat(4000)); + const response = await fetchCachedHttpInput({ ...first.args, maxObjects: 1 }); + expect(await (await fetchCachedHttpInput({ ...second.args, maxObjects: 1 }))?.text()).toBe('b'.repeat(4000)); + expect(await hasCachedInput('', '', first.meta.cacheKey, 'http')).toBe(false); + expect(await response?.text()).toBe('a'.repeat(4000)); + }); + + test('legacy metadata protocols fall back without a cache read or fill', async () => { + const f = fixture(); + expect(await fetchCachedHttpInput({ ...f.args, metadata: async () => new Response(null, { status: 404 }) })).toBeUndefined(); + expect(await fetchCachedHttpInput({ ...f.args, metadata: async () => Response.json({ cacheable: false }) })).toBeUndefined(); + expect(f.counts().reads).toBe(0); + }); +}); diff --git a/api/src/http-input-cache.ts b/api/src/http-input-cache.ts new file mode 100644 index 00000000..c785bdf0 --- /dev/null +++ b/api/src/http-input-cache.ts @@ -0,0 +1,177 @@ +import { Readable } from 'node:stream'; +import { createGzip } from 'node:zlib'; +import { httpInputCacheEvents } from './metrics'; +import { cachedInputResponse, openCachedInput, storeCachedInputs } from './session-inputs'; + +type Metadata = { cacheable: true; cacheKey: string; version: string; size: number; name?: string; readOnly: boolean }; +type FillResult = { stored: boolean; status?: number; headers?: Headers }; +type Fill = { controller: AbortController; users: number; result: Promise }; +const fills = new Map(); + +function validMetadata(value: unknown, maxBytes: number): value is Metadata { + if (!value || typeof value !== 'object') return false; + const m = value as Metadata; + return m.cacheable === true && typeof m.cacheKey === 'string' && /^[0-9a-f]{64}$/.test(m.cacheKey) && + typeof m.version === 'string' && /^[0-9a-f-]{36}$/.test(m.version) && + Number.isSafeInteger(m.size) && m.size >= 0 && m.size + 1024 <= maxBytes && + typeof m.readOnly === 'boolean' && (m.name === undefined || (typeof m.name === 'string' && m.name.length <= 4096)); +} + +function tarHeader(name: string, bytes: number): Buffer { + const header = Buffer.alloc(512); + header.write(name, 0, 100, 'utf8'); + header.write('0000600\0', 100, 8, 'ascii'); + header.write(bytes.toString(8).padStart(11, '0') + '\0', 124, 12, 'ascii'); + header.fill(32, 148, 156); + header[156] = 48; + header.write('ustar\0', 257, 6, 'ascii'); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + header.write(checksum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii'); + return header; +} + +/** Reuse the pushed-cache writer's staging, quota, no-follow and atomic commit + * rules. No workspace pathname or sandbox-visible file is used as cache input. */ +async function fillCache(response: Response, meta: Metadata, maxBytes: number, maxObjects: number): Promise { + if (!response.body) throw new Error('Input response has no body'); + const body = response.body; + const sidecar = Buffer.from(JSON.stringify({ readOnly: meta.readOnly, source: 'http' })); + async function* archive(): AsyncGenerator { + yield tarHeader(meta.cacheKey, meta.size); + const reader = body.getReader(); + let bytes = 0; + try { + for (;;) { + const part = await reader.read(); + if (part.done) break; + bytes += part.value.byteLength; + if (bytes > meta.size) throw new Error('Input exceeded its authorized metadata size'); + yield Buffer.from(part.value); + } + if (bytes !== meta.size) throw new Error('Input size changed during preparation'); + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } + yield Buffer.alloc((512 - meta.size % 512) % 512); + yield tarHeader(`${meta.cacheKey}.json`, sidecar.length); + yield sidecar; + yield Buffer.alloc((512 - sidecar.length % 512) % 512); + yield Buffer.alloc(1024); + } + const source = Readable.from(archive()); + const compressed = createGzip(); + compressed.on('error', () => {}); // Queue admission checks an already-failed stream. + source.on('error', error => compressed.destroy(error)); + source.pipe(compressed); + try { + await storeCachedInputs(compressed, maxBytes, meta.size + sidecar.length, maxObjects); + } finally { + source.destroy(); + compressed.destroy(); + await body.cancel().catch(() => {}); + } +} + +async function waitForFill(fill: Fill, signal?: AbortSignal): Promise { + if (signal?.aborted) { + if (fill.users === 0) fill.controller.abort(signal.reason); + signal.throwIfAborted(); + } + fill.users++; + let abort: (() => void) | undefined; + try { + const cancelled = new Promise((_resolve, reject) => { + abort = () => reject(signal?.reason ?? new Error('Input preparation cancelled')); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + }); + return await Promise.race([fill.result, cancelled]); + } finally { + if (abort) signal?.removeEventListener('abort', abort); + if (--fill.users === 0) fill.controller.abort(new Error('No input-cache consumers remain')); + } +} + +/** Every caller performs its own scoped preflight, even for hits or shared fills. + * Only opaque version keys returned by that authorized gateway enter this cache. */ +export async function fetchCachedHttpInput(args: { + metadata(): Promise; + download(version: string, signal: AbortSignal): Promise; + signal?: AbortSignal; + maxBytes: number; + maxFileBytes: number; + maxInflight: number; + maxObjects: number; +}): Promise { + args.signal?.throwIfAborted(); + const preflight = await args.metadata(); + if (preflight.status === 404 || preflight.status === 405) { + await preflight.body?.cancel(); + httpInputCacheEvents.inc({ event: 'legacy_bypass' }); + return undefined; // Older gateway/relay: retain the uncached protocol. + } + if (!preflight.ok) { httpInputCacheEvents.inc({ event: 'preflight_failure' }); return preflight; } + const value: unknown = await preflight.json(); + if (!validMetadata(value, args.maxBytes) || value.size > args.maxFileBytes) { + httpInputCacheEvents.inc({ event: 'uncacheable' }); + return undefined; + } + const meta = value; + args.signal?.throwIfAborted(); + let cached = await openCachedInput('', '', meta.cacheKey, 'http'); + if (cached) httpInputCacheEvents.inc({ event: 'hit' }); + if (!cached) { + let fill = fills.get(meta.cacheKey); + const joinedExistingFill = fill !== undefined; + if (fill) httpInputCacheEvents.inc({ event: 'coalesced' }); + if (!fill) { + if (fills.size >= args.maxInflight) { + httpInputCacheEvents.inc({ event: 'capacity_bypass' }); + return undefined; + } + httpInputCacheEvents.inc({ event: 'fill' }); + const controller = new AbortController(); + fill = { controller, users: 0, result: Promise.resolve({ stored: false }) }; + const ownFill = fill; + fills.set(meta.cacheKey, ownFill); + ownFill.result = (async (): Promise => { + const response = await args.download(meta.version, controller.signal); + if (!response.ok) { + await response.body?.cancel(); + return { stored: false, status: response.status, headers: response.headers }; + } + if (response.headers.get('x-codeapi-input-version') !== meta.version || + (response.headers.get('x-read-only')?.toLowerCase() === 'true') !== meta.readOnly) { + await response.body?.cancel(); + return { stored: false }; // Old file server or inconsistent metadata: never publish. + } + await fillCache(response, meta, args.maxBytes, args.maxObjects); + return { stored: true }; + })().finally(() => { + if (fills.get(meta.cacheKey) === ownFill) fills.delete(meta.cacheKey); + }); + // A caller can cancel between admission and waiting; avoid an unhandled rejection. + void ownFill.result.catch(() => {}); + } + const result = await waitForFill(fill, args.signal); + if (result.status) { + // The transfer used the creator's grant. Its denial/budget must not reject + // another caller whose own preflight succeeded; use that caller's fetch. + if (joinedExistingFill) return undefined; + const headers = new Headers(result.headers); + headers.delete('content-length'); + return new Response(null, { status: result.status, headers }); + } + if (!result.stored) return undefined; + cached = await openCachedInput('', '', meta.cacheKey, 'http'); + } + if (!cached) return undefined; // Evicted between commit and open: normal download remains correct. + if (args.signal?.aborted) { + await cached.handle.close(); + args.signal.throwIfAborted(); + } + const response = cachedInputResponse(cached); + if (meta.name) response.headers.set('content-disposition', `attachment; filename*=UTF-8''${encodeURIComponent(meta.name)}`); + return response; +} diff --git a/api/src/input-manifest.test.ts b/api/src/input-manifest.test.ts new file mode 100644 index 00000000..db55ca4a --- /dev/null +++ b/api/src/input-manifest.test.ts @@ -0,0 +1,124 @@ +import { afterEach, expect, test } from 'bun:test'; +import { createHash, randomUUID } from 'node:crypto'; +import * as fsp from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { config } from './config'; +import { Job } from './job'; +import { SESSION_INPUT_CACHE_DIR } from './session-inputs'; +import { fallbackSandboxIdentity } from './workspace-isolation'; + +const originalFetch = globalThis.fetch; +const originalConfig = { http_input_cache_enabled: config.http_input_cache_enabled, egress_gateway_url: config.egress_gateway_url }; +const dirs: string[] = []; +const keys: string[] = []; +afterEach(async () => { + globalThis.fetch = originalFetch; + Object.assign(config, originalConfig); + await Promise.all(dirs.splice(0).map(dir => fsp.rm(dir, { recursive: true, force: true }))); + await Promise.all(keys.splice(0).flatMap(key => [key, `${key}.json`]).map(key => fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, key), { force: true }))); +}); + +async function fixture(count: number, mode: 'batch' | 'legacy' | 'race' = 'batch') { + config.http_input_cache_enabled = true; + config.egress_gateway_url = 'http://manifest.test'; + let manifests = 0, singlePreflights = 0, downloads = 0; + const version = randomUUID(); + const freshVersion = randomUUID(); + const metadata = (id: string, v = version) => { + const key = createHash('sha256').update(id + v).digest('hex'); + keys.push(key); + return { cacheable: true, cacheKey: key, version: v, size: 5, readOnly: false }; + }; + globalThis.fetch = (async (url: RequestInfo | URL, init?: RequestInit) => { + const pathname = new URL(String(url)).pathname; + if (pathname.endsWith('/objects')) return Response.json([]); + if (pathname === '/input-manifest') { + manifests++; + if (mode === 'legacy') return new Response(null, { status: 404 }); + const request = JSON.parse(init!.body as string) as { files: { objectHandle: string }[] }; + expect(request.files).toHaveLength(count); + return Response.json({ files: request.files.map(file => metadata(file.objectHandle)) }); + } + if (pathname.endsWith('/metadata')) { + singlePreflights++; + return Response.json(metadata(pathname.split('/').slice(-2)[0], mode === 'race' ? freshVersion : version)); + } + downloads++; + if (mode === 'race' && new Headers(init?.headers).get('X-CodeAPI-Input-Version') === version) { + return new Response(null, { status: 409 }); + } + return new Response('bytes', { headers: { 'X-CodeAPI-Input-Version': mode === 'race' ? freshVersion : version } }); + }) as typeof fetch; + async function prime(egressGrant = 'test-grant') { + const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'manifest-prime-')); + dirs.push(dir); + const session = { + runtimeSessionId: 'test', acquire: async () => ({ dir, workspaceId: 'test', identity: fallbackSandboxIdentity() }), + primedInputId: () => undefined, markPrimed: () => {}, markDirty: () => {}, + }; + const job = new Job({ + session_id: 'test', egress_grant: egressGrant, runtime: { language: 'bash', version: '5.0.0', aliases: [] }, + files: Array.from({ length: count }, (_, i) => ({ id: `f${i}`, storage_session_id: 's', name: `file${i}.txt` })), + args: [], stdin: '', timeouts: { run: 5000, compile: 5000 }, cpu_times: { run: 5000, compile: 5000 }, + memory_limits: { run: 128e6, compile: 128e6 }, session, + } as never); + (job as unknown as { log: { level: string } }).log.level = 'silent'; + await job.prime(); + expect(await fsp.readFile(path.join(dir, 'file0.txt'), 'utf8')).toBe('bytes'); + } + return { prime, counts: () => ({ manifests, singlePreflights, downloads }) }; +} + +test('240 inputs use one authorized manifest per fresh workspace and reuse only content', async () => { + const f = await fixture(240); + await f.prime(); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 2, singlePreflights: 0, downloads: 240 }); +}, 30000); + +test('an older gateway falls back to independently authorized preflights', async () => { + const f = await fixture(2, 'legacy'); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 1, singlePreflights: 2, downloads: 2 }); +}); + +test('a raced version consumes the batch entry and retries against fresh metadata', async () => { + const f = await fixture(1, 'race'); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 1, singlePreflights: 1, downloads: 2 }); +}); + + +test('one execution grant denial cannot fail a coalesced execution with its own valid grant', async () => { + const f = await fixture(1); + const underlying = globalThis.fetch; + let release!: () => void; + let started!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const creatorStarted = new Promise(resolve => { started = resolve; }); + let deniedDownloads = 0, validDownloads = 0; + globalThis.fetch = (async (url: RequestInfo | URL, init?: RequestInit) => { + if (String(url).endsWith('/objects/f0')) { + if (new Headers(init?.headers).get('X-CodeAPI-Egress-Grant') === 'denied') { + deniedDownloads++; + started(); await blocked; + return new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' } }); + } + validDownloads++; + } + return underlying(url, init); + }) as typeof fetch; + const creator = f.prime('denied'); + void creator.catch(() => {}); + await creatorStarted; + const waiter = f.prime('valid'); + try { + while (f.counts().manifests < 2) await Bun.sleep(1); + await Bun.sleep(20); + } finally { release(); } + const result = await Promise.allSettled([creator, waiter]); + expect(result.map(item => item.status)).toEqual(['rejected', 'fulfilled']); + expect(deniedDownloads).toBe(1); + expect(validDownloads).toBe(1); +}); diff --git a/api/src/job.ts b/api/src/job.ts index bdcd5061..d2504de9 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -42,6 +42,7 @@ import { validateFilePath, isValidFilePath, } from './validation'; +import { fetchCachedHttpInput } from './http-input-cache'; import { cachedInputResponse, inputCacheKey, openCachedInput } from './session-inputs'; export { @@ -733,6 +734,7 @@ export class Job { private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; private inputFileHashes = new Map(); + private inputManifest = new Map(); private inputDestinations = new Map(); private entryPointName: string | undefined; private chmoddedDirs = new Set(); @@ -938,6 +940,7 @@ export class Job { async prime(): Promise { this.inputDestinations.clear(); + this.inputManifest.clear(); const requestedDestinations = new Map(); for (const file of this.files) { validateFilePath(file.name, '/tmp/codeapi-request-validation'); @@ -990,6 +993,8 @@ export class Job { await this.autoLoadDirkeep(); } + await this.prepareInputManifest(); + /* Promise.all rejects as soon as one operation fails, while its siblings * keep running. The route's finally then calls cleanup(), which clears the * session path/identity. A delayed sibling used to resume afterward and @@ -1431,6 +1436,38 @@ export class Job { throw lastError ?? new Error(`Failed to download input ${file.id}`); } + private async prepareInputManifest(): Promise { + if (!config.http_input_cache_enabled || !config.egress_gateway_url) return; + const files = this.files.filter(file => file.id && file.storage_session_id); + if (!files.length) return; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), AUTO_LOAD_DIRKEEP_TIMEOUT_MS); + try { + const response = await fetch(`${this.fileEgressBaseUrl()}/input-manifest`, { + method: 'POST', headers: this.fileEgressHeaders({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ files: files.map(file => ({ + sessionHandle: file.storage_session_id, objectHandle: file.id, + })) }), signal: controller.signal, + }); + if (!response.ok) { + await response.body?.cancel(); + return; // Older gateways/relays and transient failures use per-file preflight. + } + const manifest = await response.json() as { files?: unknown[] }; + if (!Array.isArray(manifest.files) || manifest.files.length !== files.length) return; + manifest.files.forEach((metadata, index) => { + if (metadata && typeof metadata === 'object' && !('retry' in metadata)) { + this.inputManifest.set(files[index], metadata); + } + }); + } catch { + // This is an optimization. Individual reads still authorize and report failures. + } finally { + clearTimeout(timeout); + controller.abort(); + } + } + /** * Resolves an input object's bytes, preferring the runner-local cache the * control plane pushes into on backends whose sandbox cannot reach the file @@ -1462,6 +1499,24 @@ export class Job { `Input ${file.id} was not delivered to the sandbox and no file server is reachable`, ); } + if (config.http_input_cache_enabled && config.egress_gateway_url) { + const response = await fetchCachedHttpInput({ + metadata: () => { + const metadata = this.inputManifest.get(file); + // A version-race retry must obtain a new authorized storage version. + this.inputManifest.delete(file); + return metadata === undefined + ? fetch(`${this.buildDownloadUrl(file)}/metadata`, { headers: this.fileEgressHeaders(), signal }) + : Promise.resolve(Response.json(metadata)); + }, + download: (version, sharedSignal) => fetch(this.buildDownloadUrl(file), { + headers: this.fileEgressHeaders({ 'X-CodeAPI-Input-Version': version }), signal: sharedSignal, + }), + signal, maxBytes: config.input_cache_max_bytes, maxFileBytes: config.max_file_size, + maxInflight: config.http_input_cache_max_inflight, maxObjects: config.http_input_cache_max_objects, + }); + if (response) return response; + } return fetch(this.buildDownloadUrl(file), { headers: this.fileEgressHeaders(), signal, diff --git a/api/src/metrics.ts b/api/src/metrics.ts index 2da33b27..328b4871 100644 --- a/api/src/metrics.ts +++ b/api/src/metrics.ts @@ -16,6 +16,12 @@ const httpRequestDuration = new Histogram({ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], }); +export const httpInputCacheEvents = new Counter({ + name: 'codeapi_sandbox_http_input_cache_events_total', + help: 'Authorized HTTP input cache events; fills and failures may both occur for one input', + labelNames: ['event'] as const, +}); + export const sandboxExecutions = new Counter({ name: 'codeapi_sandbox_executions_total', help: 'Total number of sandbox execution attempts by outcome', diff --git a/api/src/session-inputs.ts b/api/src/session-inputs.ts index 0f1b7f3a..cc9f4b5a 100644 --- a/api/src/session-inputs.ts +++ b/api/src/session-inputs.ts @@ -147,6 +147,8 @@ export interface CachedInputMeta { * ref resolve to the first ref's path — which then overwrote a file the * sandbox had edited. */ readOnly: boolean; + /** HTTP entries require a fresh gateway preflight and cannot satisfy pushed-input probes. */ + source?: 'http'; } export interface CachedInput { @@ -176,7 +178,9 @@ function parseCachedInputMeta(raw: string): CachedInputMeta | null { ) { return null; } - return { readOnly: (parsed as { readOnly: boolean }).readOnly }; + const source = (parsed as { source?: unknown }).source; + if (source !== undefined && source !== 'http') return null; + return { readOnly: (parsed as { readOnly: boolean }).readOnly, ...(source === 'http' ? { source } : {}) }; } catch { return null; } @@ -186,8 +190,9 @@ export async function hasCachedInput( storageSessionId: string, id: string, cacheKey?: string, + source: 'push' | 'http' = 'push', ): Promise { - const opened = await openCachedInput(storageSessionId, id, cacheKey); + const opened = await openCachedInput(storageSessionId, id, cacheKey, source); if (!opened) return false; await opened.handle.close(); return true; @@ -197,6 +202,7 @@ export async function openCachedInput( storageSessionId: string, id: string, cacheKey?: string, + source: 'push' | 'http' = 'push', ): Promise { const key = cacheKey ?? inputCacheKey(storageSessionId, id); if (!/^[0-9a-f]{64}$/.test(key)) return null; @@ -229,7 +235,7 @@ export async function openCachedInput( await metaHandle?.close().catch(() => {}); } const meta = raw === null ? null : parseCachedInputMeta(raw); - if (!meta) { + if (!meta || (meta.source ?? 'push') !== source) { logger.warn({ key }, 'Ignoring session input with missing or invalid metadata'); await handle.close(); return null; @@ -325,7 +331,13 @@ async function extractInputArchive( }; compressedGuard.on('error', forwardCompressedError); body.once('error', forwardBodyError); - body.pipe(compressedGuard).pipe(gunzip); + const sourceState = body as NodeJS.ReadableStream & { errored?: Error; destroyed?: boolean }; + compressedGuard.pipe(gunzip); + if (sourceState.errored || sourceState.destroyed) { + forwardBodyError(sourceState.errored ?? new Error('Input stream was cancelled before extraction')); + } else { + body.pipe(compressedGuard); + } let buffered = Buffer.alloc(0); let current: @@ -517,7 +529,10 @@ async function storeCachedInputsOnce( body: NodeJS.ReadableStream, maxBytes = Number.MAX_SAFE_INTEGER, expectedBytes?: number, + maxObjects = Number.MAX_SAFE_INTEGER, ): Promise { + const readable = body as NodeJS.ReadableStream & { errored?: Error; destroyed?: boolean }; + if (readable.errored || readable.destroyed) throw readable.errored ?? new Error('Input stream was cancelled before cache admission'); await fsp.mkdir(SESSION_INPUT_CACHE_DIR, { recursive: true, mode: 0o700 }); const cacheStat = await fsp.lstat(SESSION_INPUT_CACHE_DIR); if (!cacheStat.isDirectory() || cacheStat.isSymbolicLink()) { @@ -571,6 +586,8 @@ async function storeCachedInputsOnce( } } + if (keys.length > maxObjects) throw new Error('Input batch exceeds cache object limit'); + await pruneInputCache(Math.max(0, maxBytes - stagedBytes), maxObjects - keys.length); let stored = 0; /* Commit sidecars before data. A new key remains a probe miss until both * exist; replacing an immutable key can only expose its new validated @@ -596,6 +613,7 @@ export async function storeCachedInputs( body: NodeJS.ReadableStream, maxBytes = Number.MAX_SAFE_INTEGER, expectedBytes?: number, + maxObjects = Number.MAX_SAFE_INTEGER, ): Promise { /* Concurrent pushes otherwise each budget only its own staging tree and can * collectively recreate the same transient disk spike. Queue extraction; @@ -607,7 +625,7 @@ export async function storeCachedInputs( }); await previous; try { - return await storeCachedInputsOnce(body, maxBytes, expectedBytes); + return await storeCachedInputsOnce(body, maxBytes, expectedBytes, maxObjects); } finally { release(); } @@ -615,7 +633,7 @@ export async function storeCachedInputs( /** Drops least-recently-used entries until the cache fits `maxBytes`. Eviction * is always safe: a miss simply re-pushes on the next probe. */ -export async function pruneInputCache(maxBytes: number): Promise { +export async function pruneInputCache(maxBytes: number, maxObjects = Number.MAX_SAFE_INTEGER): Promise { const names = await fsp.readdir(SESSION_INPUT_CACHE_DIR).catch(() => [] as string[]); const nameSet = new Set(names.filter(name => ENTRY_PATTERN.test(name))); const pairs: Array<{ key: string; size: number; atime: number }> = []; @@ -644,14 +662,16 @@ export async function pruneInputCache(maxBytes: number): Promise { await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, orphan), { force: true }).catch(() => {}); } } - if (total <= maxBytes) return; + let objects = pairs.length; + if (total <= maxBytes && objects <= maxObjects) return; pairs.sort((a, b) => a.atime - b.atime); for (const pair of pairs) { - if (total <= maxBytes) break; + if (total <= maxBytes && objects <= maxObjects) break; await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, pair.key), { force: true }).catch(() => {}); await fsp .rm(path.join(SESSION_INPUT_CACHE_DIR, `${pair.key}${META_SUFFIX}`), { force: true }) .catch(() => {}); total -= pair.size; + objects -= 1; } } diff --git a/docs/INPUT_REUSE.md b/docs/INPUT_REUSE.md new file mode 100644 index 00000000..40e277c4 --- /dev/null +++ b/docs/INPUT_REUSE.md @@ -0,0 +1,91 @@ +# Bounded input reuse for stateless executions + +Fresh execution workspaces can reuse input contents without retaining a mutable conversation sandbox. Each reader first authorizes a metadata request through the egress gateway. New file-server uploads carry a random `codeapi-version` metadata value that changes on every PUT, including overwrites with identical contents. The gateway binds a cache key to that version, storage identity, tenant, user, size, filename, and read-only flag. + +```mermaid +sequenceDiagram + participant R as Runner + participant G as Egress gateway + participant F as File server + participant C as Protected input cache + R->>G: POST bounded input manifest (one per execution) + G->>G: Verify grant, scope, expiry, revocation, budget + G->>F: Resolve current metadata with bounded concurrency + F-->>G: Current upload version and metadata + G-->>R: Ordered principal-scoped version keys + R->>C: Open authorized version + alt Cache miss + R->>G: Download with expected version + G->>G: Authorize and account download + G->>F: Forward expected version + F-->>R: Exact GET metadata and bytes, or 409 if changed + R->>C: Stage, validate size, atomically publish + end + R->>R: Copy into fresh workspace using existing priming rules +``` + +## Invariants + +- A cache hit never authorizes an input. Every execution performs its own preflight, including readers joining a shared fill. Denied or revoked grants cannot use cached data. +- Every manifest handle is scope-checked before storage access. Revocation is checked again before returning resolved metadata. A deadline and disconnect cancel storage work; failures and older gateways fall back to independently authorized per-file preflights. A version-race retry discards its manifest entry. +- HTTP entries are marked separately from pushed inputs. Supplying an HTTP key in `input_cache_key` cannot bypass preflight through the older pushed-cache path. +- Cache files stay outside execution workspaces and sandbox mounts. Priming copies bytes; it never hard-links a writable workspace to trusted cache contents. Existing no-follow, read-only, hashing, atomic rename, and descriptor-pinning behavior remains in use. +- Concurrent authorized misses for the same version can share one download. Cancelling one reader does not cancel remaining readers; cancelling the last reader aborts the shared request. The number of fills, cached bytes, and object count are bounded. +- The downloader checks the version from the **actual GET**, rather than labeling bytes with metadata from an earlier HEAD. A raced overwrite returns 409 and preparation retries from current metadata. Legacy objects without a version use the uncached path. +- All writers of input objects must assign a fresh version on every overwrite. The file server does so for both upload routes. Checkpoint storage uses a separate path. Direct bucket writes that preserve an old version marker are outside this protocol. +- The optional Redis object-key index stores only a locator hint. It is not an authorization or metadata cache. Preflights still read current storage metadata; indexed keys must match the exact session and object identity. +- Shared download errors belong to the initiating grant. A coalesced caller falls back to its own authorized download rather than inheriting that grant's denial or exhausted budget. +- Redis reconnects never replay unfulfilled ledger mutations. A lost reply fails closed and may leave a conservatively charged counter/reservation until grant expiry; automatically refunding an ambiguous mutation could over-credit its budget. +- Full grant policy is no longer returned to the gateway for each authorization check. Atomic Redis scripts serialize accounting with revocation. Duplicate releases cannot repeatedly refund unrelated counters. Newly created compact ledgers keep immutable policy separate from mutable counters. + +## Configuration + +| Helm value | Environment variable | Default | +|---|---|---| +| `egressGrant.ledgerCompact` | `CODEAPI_EGRESS_LEDGER_COMPACT` | `false` | +| `egressGrant.inputManifestMaxFiles` | `CODEAPI_INPUT_MANIFEST_MAX_FILES` | `512` | +| `egressGrant.inputManifestConcurrency` | `CODEAPI_INPUT_MANIFEST_CONCURRENCY` | `8` | +| `egressGrant.inputManifestTimeoutMs` | `CODEAPI_INPUT_MANIFEST_TIMEOUT_MS` | `10000` | +| `fileServer.objectIndexEnabled` | `CODEAPI_FILE_OBJECT_INDEX_ENABLED` | `false` | +| `fileServer.metadataConcurrency` | `CODEAPI_FILE_METADATA_CONCURRENCY` | `1` | +| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `false` | +| `workerSandbox.sandbox.httpInputCacheMaxInflight` | `SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT` | `16` | +| `workerSandbox.sandbox.httpInputCacheMaxObjects` | `SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS` | `4096` | +| `workerSandbox.sandbox.inputCacheMaxBytes` | `SANDBOX_INPUT_CACHE_MAX_BYTES` | `536870912` | + +HTTP reuse requires a configured egress gateway. Cacheable objects are also bounded by the existing runner maximum file size. Cache capacity is local to each runner; eviction, restart, or routing to another runner causes a safe cache miss. The cache does not require persistent-session affinity. + +The manifest accepts at most 512 entries and a 4 MiB JSON body (protocol safety ceilings), with configured concurrency capped at 64. The runner bounds its opportunistic manifest request to 10 seconds, matching directory preparation, then uses per-file authorization if it cannot obtain a complete response. Oversized batches also fall back. Manifest requests remove repeated grant-header transfer and decoding, but still read current storage metadata for each file. + +Metadata listing concurrency preserves order and is capped at 64. A canary can use 8 after measuring storage load. This applies to directory-marker preparation as well; marker listings still happen and are not a retained conversation manifest. + +## Rollout and rollback + +1. Deploy the new binaries with feature flags off. Atomic accounting supports existing JSON ledgers, and legacy downloads retain metadata compatibility. The new file server stamps future uploads with versions. +2. Update **all** egress-gateway replicas before enabling compact ledgers. New binaries read both formats regardless of the creation flag. Older binaries cannot read compact hashes. To roll back to an older binary, disable compact creation, drain active grants, and wait their maximum TTL plus grace; never delete active ledgers to force a rollback. +3. Update all file-server writers before enabling the object-key index. Otherwise an older writer can change a locator without updating the index. Keep file-server replicas consistent during an indexed rollout. +4. Update the gateway, relay, runner, and launcher before enabling HTTP reuse on a small runner canary. Older gateway/relay metadata routes return 404/405 and fall back safely. Older files return `cacheable: false`. Keep the feature disabled for storage adapters that cannot return user metadata on GET. +5. Observe `codeapi_sandbox_http_input_cache_events_total` (bounded event labels, no identities), cold and warm preparation latency, storage/Redis operations, admission fairness, request budgets, and memory/disk pressure before widening the rollout. A successful manifest consumes one read request for the batch, matching the existing list-request accounting unit. Per-file compatibility preflights each consume a read request; each cold miss consumes an additional download request. Do not disable budget enforcement to accommodate a workload. +6. Disable HTTP reuse to return to normal downloads immediately. Cached files can age out normally; no workspace deletion or migration is needed. + +The creation flags default off. No deployment or object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. + +## Validation + +Ledger tests use an isolated real `redis-server` on a Unix socket with persistence disabled. Install Redis before running service tests. They cover legacy/compact formats, 240 concurrent reads against a strict budget, revocation, expiry, rejected uploads, duplicate releases, and format changes without resetting state. + +Focused commands: + +```sh +cd api +bun test src/input-manifest.test.ts src/http-input-cache.test.ts src/session-inputs.test.ts src/session-inputs.prime.test.ts src/download.test.ts src/inline-prime-atomicity.test.ts src/job-cleanup.test.ts +npx tsc --noEmit +``` + +```sh +cd service +bun test src/egress-ledger.test.ts src/egress-ledger-reconnect.test.ts src/egress-gateway.test.ts src/file-object-resolver.test.ts src/file-download.test.ts src/file-metadata.test.ts +npx tsc --noEmit +``` + +Run the code-package tests with Node (its supported test runner), plus launcher and deployment checks in CI. Cache regressions cover fresh-workspace reuse, cross-principal/version separation, denied preflights, pushed-key bypass prevention, coalesced cancellation, changed or oversized responses, and descriptor-safe eviction. Production latency targets must be validated with real regional storage latency and workload sizes; local synthetic results are not a production SLO. diff --git a/helm/codeapi/templates/egress-gateway-deployment.yaml b/helm/codeapi/templates/egress-gateway-deployment.yaml index 6e394909..49c865f0 100644 --- a/helm/codeapi/templates/egress-gateway-deployment.yaml +++ b/helm/codeapi/templates/egress-gateway-deployment.yaml @@ -40,6 +40,14 @@ spec: value: {{ .Values.hardenedSandboxMode | quote }} - name: CODEAPI_EGRESS_LEDGER_REQUIRED value: {{ .Values.egressGrant.ledgerRequired | quote }} + - name: CODEAPI_INPUT_MANIFEST_MAX_FILES + value: {{ .Values.egressGrant.inputManifestMaxFiles | quote }} + - name: CODEAPI_INPUT_MANIFEST_CONCURRENCY + value: {{ .Values.egressGrant.inputManifestConcurrency | quote }} + - name: CODEAPI_INPUT_MANIFEST_TIMEOUT_MS + value: {{ .Values.egressGrant.inputManifestTimeoutMs | quote }} + - name: CODEAPI_EGRESS_LEDGER_COMPACT + value: {{ .Values.egressGrant.ledgerCompact | quote }} - name: CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS value: {{ .Values.egressGrant.ledgerTtlGraceSeconds | quote }} - name: EGRESS_GATEWAY_PORT diff --git a/helm/codeapi/templates/file-server-deployment.yaml b/helm/codeapi/templates/file-server-deployment.yaml index 1dd95166..1a9e8531 100644 --- a/helm/codeapi/templates/file-server-deployment.yaml +++ b/helm/codeapi/templates/file-server-deployment.yaml @@ -51,6 +51,10 @@ spec: containerPort: {{ .Values.fileServer.service.port }} protocol: TCP env: + - name: CODEAPI_FILE_METADATA_CONCURRENCY + value: {{ .Values.fileServer.metadataConcurrency | quote }} + - name: CODEAPI_FILE_OBJECT_INDEX_ENABLED + value: {{ .Values.fileServer.objectIndexEnabled | quote }} {{ include "codeapi.otel.env" (dict "root" . "serviceName" "aiml-codeapi-file-server") | nindent 12 }} {{- if $useS3 }} # AWS S3 configuration (IRSA or static credentials) diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml index 39c2e197..2819d998 100644 --- a/helm/codeapi/templates/worker-sandbox-deployment.yaml +++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml @@ -314,6 +314,14 @@ spec: value: {{ (.Values.workerSandbox.sandbox.jobUidCount | default (.Values.workerSandbox.sandbox.maxConcurrentJobs | default (mul (.Values.workerSandbox.launcher.vcpus | default 2) 4))) | quote }} - name: SANDBOX_WORKSPACE_REAPER_MAX_AGE_SECONDS value: {{ (.Values.workerSandbox.sandbox.workspaceReaperMaxAgeSeconds | default 3600) | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_ENABLED + value: {{ .Values.workerSandbox.sandbox.httpInputCacheEnabled | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT + value: {{ .Values.workerSandbox.sandbox.httpInputCacheMaxInflight | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS + value: {{ .Values.workerSandbox.sandbox.httpInputCacheMaxObjects | quote }} + - name: SANDBOX_INPUT_CACHE_MAX_BYTES + value: {{ .Values.workerSandbox.sandbox.inputCacheMaxBytes | quote }} - name: SANDBOX_EXECUTE_BODY_LIMIT value: {{ .Values.workerSandbox.sandbox.executeBodyLimit | quote }} - name: SANDBOX_DISABLE_NETWORKING diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index ee997ce8..3ecfbef8 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -46,6 +46,11 @@ egressGrant: ttlSeconds: 900 ledgerRequired: true ledgerTtlGraceSeconds: 300 + # Enable only after all gateway replicas support compact ledgers. + ledgerCompact: false + inputManifestMaxFiles: 512 + inputManifestConcurrency: 8 + inputManifestTimeoutMs: 10000 # Worker signs sandbox execute requests with this private key; sandbox-runner # receives only the public verifier so a runner compromise cannot mint new @@ -268,6 +273,10 @@ workerSandbox: # Defaults to maxConcurrentJobs when unset. jobUidCount: null workspaceReaperMaxAgeSeconds: 3600 + httpInputCacheEnabled: false + httpInputCacheMaxInflight: 16 + httpInputCacheMaxObjects: 4096 + inputCacheMaxBytes: 536870912 # Language runtime package delivery packages: @@ -327,6 +336,8 @@ workerSandbox: # FILE SERVER (S3/MinIO integration, stateless) # ============================================================================= fileServer: + objectIndexEnabled: false + metadataConcurrency: 1 enabled: true replicaCount: 1 diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 02d26418..7771b9b9 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -421,6 +421,10 @@ fn is_allowed_guest_env_key(key: &str, egress_gateway_enabled: bool) -> bool { "SANDBOX_COMPILE_TIMEOUT", "SANDBOX_DATA_DIRECTORY", "SANDBOX_DISABLE_NETWORKING", + "SANDBOX_HTTP_INPUT_CACHE_ENABLED", + "SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT", + "SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS", + "SANDBOX_INPUT_CACHE_MAX_BYTES", "SANDBOX_EXECUTE_BODY_LIMIT", "SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY", "SANDBOX_FORWARD_TARGET", diff --git a/packages/code/src/relay.test.ts b/packages/code/src/relay.test.ts index a9b81fd0..c4d92288 100644 --- a/packages/code/src/relay.test.ts +++ b/packages/code/src/relay.test.ts @@ -416,3 +416,68 @@ for (const [status, reason] of [[403, 'scope_mismatch'], [503, 'ledger_conflict' } }); } + +test('file relay carries version preflights and download preconditions without opening metadata writes', async () => { + let requests = 0; + const upstream = createServer((req, res) => { + requests++; + if (req.url?.endsWith('/metadata')) { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ cacheable: true, version: 'opaque-version' })); + } else { + assert.equal(req.headers['x-codeapi-input-version'], 'opaque-version'); + res.writeHead(200, { 'X-CodeAPI-Input-Version': 'opaque-version' }).end('bytes'); + } + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', maxBytes: 1024, timeoutMs: 1000 }); + const headers = { 'X-LibreChat-Code-Relay-Token': 'relay-secret', 'X-CodeAPI-Egress-Grant': 'grant' }; + try { + const metadata = await fetch(`${relay.url}/sessions/s/objects/o/metadata`, { headers }); + assert.equal(metadata.status, 200); + assert.equal((await metadata.json() as { version: string }).version, 'opaque-version'); + const input = await fetch(`${relay.url}/sessions/s/objects/o`, { headers: { ...headers, 'X-CodeAPI-Input-Version': 'opaque-version' } }); + assert.equal(input.headers.get('x-codeapi-input-version'), 'opaque-version'); + assert.equal(await input.text(), 'bytes'); + const denied = await fetch(`${relay.url}/sessions/s/objects/o/metadata`, { method: 'PUT', headers, body: '' }); + assert.equal(denied.status, 404); + assert.equal(requests, 2); + } finally { + await relay.close(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); + + +test('file relay forwards bounded manifest POSTs and rejects alternate manifest methods', async () => { + let requests = 0; + const upstream = createServer(async (req, res) => { + requests++; + assert.equal(req.method, 'POST'); + assert.equal(req.url, '/input-manifest'); + assert.equal(req.headers['x-codeapi-egress-grant'], 'grant'); + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + assert.deepEqual(JSON.parse(Buffer.concat(chunks).toString()), { files: [] }); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ files: [] })); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', maxBytes: 128, timeoutMs: 1000 }); + const headers = { 'X-LibreChat-Code-Relay-Token': 'relay-secret', 'X-CodeAPI-Egress-Grant': 'grant', 'Content-Type': 'application/json' }; + try { + const response = await fetch(`${relay.url}/input-manifest`, { method: 'POST', headers, body: JSON.stringify({ files: [] }) }); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { files: [] }); + for (const method of ['GET', 'PUT']) { + const denied = await fetch(`${relay.url}/input-manifest`, { method, headers }); + assert.equal(denied.status, 404); + } + const oversized = await fetch(`${relay.url}/input-manifest`, { method: 'POST', headers, body: 'x'.repeat(129) }); + assert.equal(oversized.status, 413); + assert.equal(requests, 1); + } finally { + await relay.close(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts index 14e91280..07aa8714 100644 --- a/packages/code/src/relay.ts +++ b/packages/code/src/relay.ts @@ -20,6 +20,7 @@ export interface FileRelayHandle { } const OBJECT_PATH = /^\/sessions\/[^/]+\/objects\/[^/]+$/; +const OBJECT_METADATA_PATH = /^\/sessions\/[^/]+\/objects\/[^/]+\/metadata$/; const OBJECT_LIST_PATH = /^\/sessions\/[^/]+\/objects$/; const MAX_RELAY_HEADER_BYTES = 512 * 1024; const LOCAL_HTTP_HOSTS = new Set([ @@ -160,16 +161,19 @@ export async function startFileRelay( response.end('{"status":"ok"}'); return; } + const manifestRequest = request.method === 'POST' && requestUrl.pathname === '/input-manifest' && requestUrl.search.length === 0; const objectRequest = OBJECT_PATH.test(requestUrl.pathname) && requestUrl.search.length === 0; + const metadataRequest = request.method === 'GET' && + OBJECT_METADATA_PATH.test(requestUrl.pathname) && requestUrl.search.length === 0; const normalizedListRequest = request.method === 'GET' && OBJECT_LIST_PATH.test(requestUrl.pathname) && requestUrl.searchParams.size === 1 && requestUrl.searchParams.get('detail') === 'normalized'; if ( - (request.method !== 'GET' && request.method !== 'PUT') || - (!objectRequest && !normalizedListRequest) + (request.method !== 'GET' && request.method !== 'PUT' && !manifestRequest) || + (!objectRequest && !normalizedListRequest && !metadataRequest && !manifestRequest) ) { response.writeHead(404).end(); return; @@ -191,7 +195,7 @@ export async function startFileRelay( }`; target.search = requestUrl.search; const requestBody = - request.method === 'PUT' + (request.method === 'PUT' || manifestRequest) ? await readRequestBody(request, options.maxBytes) : undefined; const upstreamResponse = await fetch(target, { @@ -200,7 +204,9 @@ export async function startFileRelay( ...(typeof grant === 'string' ? { 'X-CodeAPI-Egress-Grant': grant } : {}), - ...(request.method === 'PUT' + ...(typeof request.headers['x-codeapi-input-version'] === 'string' + ? { 'X-CodeAPI-Input-Version': request.headers['x-codeapi-input-version'] } : {}), + ...((request.method === 'PUT' || manifestRequest) ? { 'Content-Length': String(requestBody?.length ?? 0), ...(typeof request.headers['content-type'] === 'string' @@ -246,6 +252,8 @@ export async function startFileRelay( ...(upstreamResponse.headers.has('retry-after') ? { 'Retry-After': upstreamResponse.headers.get('retry-after')! } : {}), + ...(upstreamResponse.headers.has('x-codeapi-input-version') + ? { 'X-CodeAPI-Input-Version': upstreamResponse.headers.get('x-codeapi-input-version')! } : {}), 'Content-Length': String(body.length), }); response.end(body); diff --git a/service/src/config.ts b/service/src/config.ts index 94daf5a3..90df6b58 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -318,6 +318,12 @@ export const env = { EGRESS_GATEWAY_REQUEST_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS) || 30_000, EGRESS_GATEWAY_REVOKE_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS) || 5_000, EGRESS_LEDGER_REQUIRED: process.env.CODEAPI_EGRESS_LEDGER_REQUIRED === 'true' || process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', + FILE_METADATA_CONCURRENCY: Math.min(64, Math.max(1, Math.floor(Number(process.env.CODEAPI_FILE_METADATA_CONCURRENCY) || 1))), + FILE_OBJECT_INDEX_ENABLED: process.env.CODEAPI_FILE_OBJECT_INDEX_ENABLED === 'true', + INPUT_MANIFEST_MAX_FILES: Math.min(512, Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_MAX_FILES) || 512))), + INPUT_MANIFEST_CONCURRENCY: Math.min(64, Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_CONCURRENCY) || 8))), + INPUT_MANIFEST_TIMEOUT_MS: Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_TIMEOUT_MS) || 10000)), + EGRESS_LEDGER_COMPACT: process.env.CODEAPI_EGRESS_LEDGER_COMPACT === 'true', EGRESS_LEDGER_TTL_GRACE_SECONDS: Number(process.env.CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS) || 300, EGRESS_GRANT_SECRET: process.env.CODEAPI_EGRESS_GRANT_SECRET ?? '', EGRESS_GRANT_TTL_SECONDS: resolveEgressGrantTtlSeconds(process.env.EGRESS_GRANT_TTL_SECONDS, defaultJobTimeoutMs), diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index 717309d7..c3cd4da4 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -1,14 +1,15 @@ process.env.CODEAPI_EGRESS_GATEWAY_AUTOSTART = 'false'; -import { afterAll, beforeAll, beforeEach, describe, expect, test, spyOn } from 'bun:test'; +import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; import crypto from 'crypto'; -import RedisMock from 'ioredis-mock'; +import { startTestRedis } from './test/redis'; import type { Server } from 'http'; import type { AddressInfo } from 'net'; import { env } from './config'; import { assertEgressGrantActive, createEgressLedger, + revokeEgressLedger, setEgressLedgerRedisForTest, } from './egress-ledger'; import { @@ -435,39 +436,171 @@ describe('egress gateway routes', () => { } }); - test('reports exhausted ledger conflicts as retryable without forwarding the read', async () => { - const redis = new RedisMock(); + test('batch preflight scopes every entry before reading and preserves order under a concurrency bound', async () => { + const files = Array.from({ length: 17 }, (_, i) => ({ id: `file_${i}`, session_id: 'sess_input', name: `file_${i}.csv` })); + const grant = claims({ input_files: files }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const body = { files: files.map(file => ({ sessionHandle: sid, objectHandle: objectHandle({ fileId: file.id, name: file.name }) })) }; + let active = 0, peak = 0, calls = 0; + const width = env.INPUT_MANIFEST_CONCURRENCY; + env.INPUT_MANIFEST_CONCURRENCY = 3; + globalThis.fetch = (async (input: RequestInfo | URL) => { + calls++; active++; peak = Math.max(peak, active); + await Bun.sleep(5); + active--; + const id = String(input).split('/').at(-2)!; + return Response.json({ version: crypto.randomUUID(), size: 5, originalFilename: `${id}.csv` }); + }) as typeof fetch; + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); env.EGRESS_LEDGER_REQUIRED = true; - setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); - const duplicate = redis.duplicate.bind(redis); - const duplication = spyOn(redis, 'duplicate').mockImplementation(() => { - const connection = duplicate(); - const transaction = { - set: () => transaction, - exec: async () => null, - }; - spyOn(connection, 'multi').mockImplementation(() => transaction as never); - return connection; + try { + await createEgressLedger(grant); + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, body: JSON.stringify(body), + }); + expect(response.status).toBe(200); + const result = await response.json() as { files: { cacheKey: string; name: string }[] }; + expect(result.files.map(file => file.name)).toEqual(files.map(file => file.name)); + expect(result.files.every(file => /^[0-9a-f]{64}$/.test(file.cacheKey))).toBe(true); + expect(calls).toBe(17); + expect(peak).toBe(3); + expect((await assertEgressGrantActive(grant)).request_count).toBe(1); + body.files.push({ sessionHandle: sid, objectHandle: objectHandle({ fileId: 'outside_scope' }) }); + const denied = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, body: JSON.stringify(body), + }); + expect(denied.status).toBe(403); + expect(calls).toBe(17); + } finally { + env.INPUT_MANIFEST_CONCURRENCY = width; + env.EGRESS_LEDGER_REQUIRED = false; + setEgressLedgerRedisForTest(null); + await redis.closeTestServer(); + } + }); + + test('batch preflight withholds resolved metadata if the grant is revoked during storage access', async () => { + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); + env.EGRESS_LEDGER_REQUIRED = true; + try { + const grant = claims(); + await createEgressLedger(grant); + globalThis.fetch = (async (_input: RequestInfo | URL) => { + await revokeEgressLedger(grant.grant_id!, 'test revocation'); + return Response.json({ version: crypto.randomUUID(), size: 5 }); + }) as typeof fetch; + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, + body: JSON.stringify({ files: [{ sessionHandle: sessionHandle({ dir: 'read', sessionId: 'sess_input' }), objectHandle: objectHandle({}) }] }), + }); + expect(response.status).toBe(403); + expect(await response.text()).not.toContain('cacheKey'); + } finally { + env.EGRESS_LEDGER_REQUIRED = false; + setEgressLedgerRedisForTest(null); + await redis.closeTestServer(); + } + }); + + test('batch deadline aborts in-flight storage requests without starting queued inputs', async () => { + const timeout = env.INPUT_MANIFEST_TIMEOUT_MS; + const width = env.INPUT_MANIFEST_CONCURRENCY; + env.INPUT_MANIFEST_TIMEOUT_MS = 20; + env.INPUT_MANIFEST_CONCURRENCY = 1; + let started = 0, aborted = 0; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + started++; + return new Promise((_resolve, reject) => { + init!.signal!.addEventListener('abort', () => { aborted++; reject(init!.signal!.reason); }, { once: true }); + }); + }) as typeof fetch; + try { + const file = { sessionHandle: sessionHandle({ dir: 'read', sessionId: 'sess_input' }), objectHandle: objectHandle({}) }; + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(), 'Content-Type': 'application/json' }, + body: JSON.stringify({ files: [file, file] }), + }); + expect(response.ok).toBe(false); + expect(started).toBe(1); + expect(aborted).toBe(1); + } finally { + env.INPUT_MANIFEST_TIMEOUT_MS = timeout; + env.INPUT_MANIFEST_CONCURRENCY = width; + } + }); + + test('batch preflight rejects oversized and malformed manifests before storage access', async () => { + for (const files of [Array.from({ length: env.INPUT_MANIFEST_MAX_FILES + 1 }, () => ({})), [null], [{}]]) { + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(), 'Content-Type': 'application/json' }, body: JSON.stringify({ files }), + }); + expect(response.status).toBe(400); + } + expect(upstreamCalls).toHaveLength(0); + }); + + test('preflight authorizes scope and returns version keys scoped to the principal', async () => { + const version = crypto.randomUUID(); + upstreamResponse = Response.json({ version, size: 5, originalFilename: 'inputs/data.csv', readOnly: true }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const object = objectHandle({}); + const response = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(response.status).toBe(200); + const metadata = await response.json() as { cacheKey: string; version: string; readOnly: boolean }; + expect(metadata.cacheKey).toMatch(/^[0-9a-f]{64}$/); + expect(metadata.version).toBe(version); + expect(metadata.readOnly).toBe(true); + expect(upstreamCalls[0].url).toEndWith('/sessions/sess_input/objects/file_123/metadata'); + const second = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { + headers: grantHeader(claims({ tenant_id: 'another_tenant' })), }); + expect((await second.json() as { cacheKey: string }).cacheKey).not.toBe(metadata.cacheKey); + const before = upstreamCalls.length; + const denied = await gatewayFetch(`/sessions/${sid}/objects/${objectHandle({ fileId: 'outside_scope' })}/metadata`, { + headers: grantHeader(), + }); + expect(denied.status).toBe(403); + expect(upstreamCalls).toHaveLength(before); + }); + + test('preflight denies revoked grants and old metadata stays uncached', async () => { + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const object = objectHandle({}); + upstreamResponse = Response.json({ size: 5 }); + const legacy = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(await legacy.json()).toEqual({ cacheable: false }); + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); + env.EGRESS_LEDGER_REQUIRED = true; try { await createEgressLedger(claims()); - const readSession = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); - const response = await gatewayFetch(`/sessions/${readSession}/objects?detail=normalized`, { - headers: grantHeader(), - }); - expect(response.status).toBe(503); - expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('ledger_conflict'); - expect(response.headers.get('Retry-After')).toBe('1'); - expect(upstreamCalls).toHaveLength(0); - expect((await assertEgressGrantActive(claims())).request_count).toBe(0); + await redis.del(`codeapi:egress:grant:${claims().grant_id}`); + const before = upstreamCalls.length; + const denied = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(denied.status).toBe(403); + expect(upstreamCalls).toHaveLength(before); } finally { - duplication.mockRestore(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); - redis.disconnect(); env.EGRESS_LEDGER_REQUIRED = false; } }); + test('forwards the authorized input-version precondition on downloads', async () => { + const version = crypto.randomUUID(); + upstreamResponse = new Response('bytes', { headers: { 'X-CodeAPI-Input-Version': version } }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const response = await gatewayFetch(`/sessions/${sid}/objects/${objectHandle({})}`, { + headers: { ...grantHeader(), 'X-CodeAPI-Input-Version': version }, + }); + expect(response.status).toBe(200); + expect(response.headers.get('x-codeapi-input-version')).toBe(version); + expect(new Headers(upstreamCalls[0].init.headers).get('x-codeapi-input-version')).toBe(version); + await response.text(); + }); + test('lists only scoped objects and injects internal credentials', async () => { upstreamResponse = Response.json([ { id: 'file_123', name: 'inputs/data.csv', storage_session_id: 'sess_input' }, @@ -495,7 +628,7 @@ describe('egress gateway routes', () => { }); test('accepts legacy rollout grants and handles while ledger-required mode is enabled', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -526,14 +659,14 @@ describe('egress gateway routes', () => { expect(record.max_output_files).toBe(50); expect(record.max_requests).toBe(1000); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('restores token-only legacy grants and creates ledger state before returning handles', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -569,14 +702,14 @@ describe('egress gateway routes', () => { expect(record.grant_id).toBe(legacyGrant.grant_id); expect(record.exec_id).toBe('exec_123'); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('rejects grantless handles for non-legacy grants in ledger-required mode', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -593,7 +726,7 @@ describe('egress gateway routes', () => { expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('scope_mismatch'); expect(upstreamCalls).toHaveLength(0); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } @@ -849,7 +982,7 @@ describe('egress gateway routes', () => { }); test('rolls back upload reservations when upstream PUT throws', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); const grant = claims({ max_output_files: 1, max_requests: 3 }); @@ -894,14 +1027,14 @@ describe('egress gateway routes', () => { expect(retried.status).toBe(201); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('does not roll back ledger state when upload reservation is rejected', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { @@ -933,14 +1066,14 @@ describe('egress gateway routes', () => { expect(await upload('bbbbbbbbbbbbbbbbbbbbb')).toBe(403); expect(upstreamCalls).toHaveLength(1); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('enforces output budgets per turn when grants reuse an output session', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { @@ -989,7 +1122,7 @@ describe('egress gateway routes', () => { expect(await upload(secondTurn, 'ddddddddddddddddddddd')).toBe(201); expect(await upload(secondTurn, 'eeeeeeeeeeeeeeeeeeeee')).toBe(403); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 13a6de95..f86ce656 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -26,7 +26,7 @@ import { isSyntheticInternalRequestHeader, } from './internal-synthetic'; import { - assertEgressGrantActive, + checkEgressGrantActive, createEgressLedger, ensureEgressLedger, pingEgressLedger, @@ -44,6 +44,7 @@ import logger from './logger'; import { parseBoundedContentLength } from './http-limits'; import { validateEgressGatewayHardenedConfig } from './secure-startup'; import { isOpaqueObjectContentDisposition } from './file-metadata'; +import { mapObjectDetails } from './file-object-resolver'; export const app: Express = express(); app.disable('x-powered-by'); @@ -89,6 +90,8 @@ function routeFamily(req: Request): string { if (req.path === '/tool-call') return 'ptc-tool-call'; if (req.path.startsWith('/sessions/')) { if (req.method === 'PUT') return 'file-upload'; + if (req.method === 'POST' && req.path === '/input-manifest') return 'input-manifest'; + if (req.method === 'GET' && req.path.endsWith('/metadata')) return 'input-metadata'; if (req.method === 'GET' && req.path.includes('/objects/')) return 'file-download'; if (req.method === 'GET' && req.path.endsWith('/objects')) return 'file-list'; return 'file-unknown'; @@ -199,7 +202,7 @@ async function getGrant(req: Request, res: Response): Promise if (grant.legacy_grant) { await ensureEgressLedger(grant); } - await assertEgressGrantActive(grant); + await checkEgressGrantActive(grant); return grant; } @@ -438,7 +441,7 @@ async function restoreInternalSandboxResult(args: { if (grant.legacy_grant) { await ensureEgressLedger(grant); } - await assertEgressGrantActive(grant); + await checkEgressGrantActive(grant); const restored = restoreSandboxExecuteResult( args.result as Parameters[0], args.egressGrantToken, @@ -624,6 +627,88 @@ app.get('/sessions/:sessionHandle/objects', async (req, res) => { } }); +function inputMetadata( + metadata: { version?: unknown; size?: unknown; originalFilename?: unknown; readOnly?: unknown }, + grant: EgressGrantClaims, sessionId: string, objectId: string, +) { + if (typeof metadata.version !== 'string' || !/^[0-9a-f-]{36}$/.test(metadata.version) || + !Number.isSafeInteger(metadata.size) || (metadata.size as number) < 0) { + return { cacheable: false }; + } + const name = typeof metadata.originalFilename === 'string' ? metadata.originalFilename : undefined; + const readOnly = metadata.readOnly === true; + const cacheKey = crypto.createHash('sha256').update(JSON.stringify([ + 'authorized-http-input-v1', grant.tenant_id, grant.user_id, sessionId, objectId, + metadata.version, metadata.size, name, readOnly, + ])).digest('hex'); + return { cacheable: true, cacheKey, version: metadata.version, size: metadata.size, name, readOnly }; +} + +/** One budgeted HTTP read, with every handle checked before any storage access. + * Results belong only to this execution; the manifest is not an auth token. */ +app.post('/input-manifest', express.json({ limit: '4mb' }), async (req, res) => { + const controller = new AbortController(); + const cancel = () => controller.abort(); + const timeout = setTimeout(cancel, env.INPUT_MANIFEST_TIMEOUT_MS); + res.once('close', cancel); + try { + if (Object.keys(req.query).length || !Array.isArray(req.body?.files) || + req.body.files.length > env.INPUT_MANIFEST_MAX_FILES || + req.body.files.some((file: unknown) => !file || typeof file !== 'object' || + typeof (file as { sessionHandle?: unknown }).sessionHandle !== 'string' || + typeof (file as { objectHandle?: unknown }).objectHandle !== 'string')) { + return res.status(400).json({ error: 'Invalid input manifest' }); + } + const grant = await getGrant(req, res); + const files: Array<{ sessionId: string; objectId: string }> = req.body.files.map((file: { sessionHandle: string; objectHandle: string }) => { + const sessionId = openSessionParam(file.sessionHandle, grant, 'read'); + const object = openObjectParam(file.objectHandle, grant, sessionId); + return { sessionId, objectId: object.id }; + }); + await recordEgressRead(grant); + async function* inputs() { yield* files; } + const metadata = await mapObjectDetails(inputs(), async ({ sessionId, objectId }) => { + controller.signal.throwIfAborted(); + const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, + `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(objectId)}/metadata`), + { headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); + if (!upstream.ok) { + await upstream.body?.cancel(); + // Recheck failures individually through the existing retry/classification path. + return { cacheable: false, retry: true }; + } + return inputMetadata(await upstream.json(), grant, sessionId, objectId); + }, env.INPUT_MANIFEST_CONCURRENCY); + // Do not publish a manifest after revocation/expiry during storage resolution. + await checkEgressGrantActive(grant); + return res.json({ files: metadata }); + } catch (error) { + return sendEgressError(req, res, error); + } finally { + clearTimeout(timeout); + res.removeListener('close', cancel); + controller.abort(); + } +}); + +/** Cache preflight is a scoped, budgeted read, never a reusable authorization grant. */ +app.get('/sessions/:sessionHandle/objects/:objectHandle/metadata', async (req, res) => { + try { + if (Object.keys(req.query).length) return res.status(400).json({ error: 'Metadata query parameters are not supported' }); + const grant = await getGrant(req, res); + const sessionId = openSessionParam(req.params.sessionHandle, grant, 'read'); + const object = openObjectParam(req.params.objectHandle, grant, sessionId); + await recordEgressRead(grant); + const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, + `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}/metadata`), + { headers: injectTraceHeaders(internalServiceHeaders()) }); + if (!upstream.ok) return pipeFetchResponse(upstream, res); + return res.json(inputMetadata(await upstream.json(), grant, sessionId, object.id)); + } catch (error) { + return sendEgressError(req, res, error); + } +}); + app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { try { if (Object.keys(req.query).length > 0) { @@ -633,12 +718,16 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { const sessionId = openSessionParam(req.params.sessionHandle, grant, 'read'); const object = openObjectParam(req.params.objectHandle, grant, sessionId); await recordEgressRead(grant); + const expectedVersion = req.header('x-codeapi-input-version'); + if (expectedVersion && !/^[0-9a-f-]{36}$/.test(expectedVersion)) { + return res.status(400).json({ error: 'Invalid input version' }); + } const upstream = await fetch( forwardUrl( env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}`, ), - { headers: injectTraceHeaders(internalServiceHeaders()) }, + { headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, ); const headerOverrides = isOpaqueObjectContentDisposition( upstream.headers.get('content-disposition'), diff --git a/service/src/egress-ledger-reconnect.test.ts b/service/src/egress-ledger-reconnect.test.ts new file mode 100644 index 00000000..0cf4bb78 --- /dev/null +++ b/service/src/egress-ledger-reconnect.test.ts @@ -0,0 +1,63 @@ +import { expect, test } from 'bun:test'; +import { createConnection, createServer, type Socket, type AddressInfo } from 'node:net'; +import IORedis from 'ioredis'; +import { env } from './config'; +import { startTestRedis } from './test/redis'; +import { + EGRESS_LEDGER_REDIS_RETRY_OPTIONS, createEgressLedger, recordEgressRead, + assertEgressGrantActive, setEgressLedgerRedisForTest, +} from './egress-ledger'; +import type { EgressGrantClaims } from './egress-grant'; + +test('a lost Redis mutation reply rejects without replaying its applied counter after reconnect', async () => { + const redis = await startTestRedis(); + const sockets = new Set(); + let dropReply = false; + const proxy = createServer(downstream => { + const upstream = createConnection(redis.options.path!); + sockets.add(downstream); sockets.add(upstream); + downstream.pipe(upstream); + upstream.on('data', data => { + if (dropReply) { + dropReply = false; + downstream.destroy(); upstream.destroy(); + } else downstream.write(data); + }); + downstream.on('error', () => {}); + upstream.on('error', () => downstream.destroy()); + downstream.on('close', () => { sockets.delete(downstream); upstream.destroy(); }); + upstream.on('close', () => { sockets.delete(upstream); downstream.destroy(); }); + }); + await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve)); + const client = new IORedis({ host: '127.0.0.1', port: (proxy.address() as AddressInfo).port, + ...EGRESS_LEDGER_REDIS_RETRY_OPTIONS, retryStrategy: () => 10, lazyConnect: true }); + client.on('error', () => {}); + const required = env.EGRESS_LEDGER_REQUIRED; + const compact = env.EGRESS_LEDGER_COMPACT; + env.EGRESS_LEDGER_REQUIRED = true; + env.EGRESS_LEDGER_COMPACT = true; + setEgressLedgerRedisForTest(client); + try { + await client.connect(); + const now = Math.floor(Date.now() / 1000); + const grant: EgressGrantClaims = { v: 1, typ: 'grant', grant_id: 'reconnect', exec_id: 'exec', + tenant_id: 'tenant', user_id: 'user', session_key: 'session', input_files: [], read_sessions: [], + output_session_id: 'output', max_upload_bytes: 100, max_output_files: 10, max_requests: 10, iat: now, exp: now + 300 }; + await createEgressLedger(grant); + const reconnected = new Promise(resolve => client.once('ready', resolve)); + dropReply = true; + await expect(recordEgressRead(grant)).rejects.toThrow(); + await reconnected; + expect((await assertEgressGrantActive(grant)).request_count).toBe(1); + await recordEgressRead(grant); + expect((await assertEgressGrantActive(grant)).request_count).toBe(2); + } finally { + env.EGRESS_LEDGER_REQUIRED = required; + env.EGRESS_LEDGER_COMPACT = compact; + setEgressLedgerRedisForTest(null); + client.disconnect(); + for (const socket of sockets) socket.destroy(); + await new Promise(resolve => proxy.close(() => resolve())); + await redis.closeTestServer(); + } +}, 10000); diff --git a/service/src/egress-ledger-script.ts b/service/src/egress-ledger-script.ts new file mode 100644 index 00000000..c63346ad --- /dev/null +++ b/service/src/egress-ledger-script.ts @@ -0,0 +1,123 @@ +/** One-key transactions work on Redis Cluster and serialize admission with revocation. + * Legacy JSON is retained for mixed-version rollout; compact hashes avoid decoding + * the immutable input policy on the hot path. Never retry an ambiguous EVAL result: + * the operation may already have consumed its budget. */ +export const EGRESS_LEDGER_SCRIPT = ` +local key = KEYS[1] +local op = ARGV[1] +local kind = redis.call('TYPE', key) +if type(kind) == 'table' then kind = kind.ok end +local now = tonumber(ARGV[3]) +local function denied(message) return {'error', 'scope_mismatch', message} end +if op == 'create' then + if kind ~= 'none' then return {'ok'} end + local policy = cjson.decode(ARGV[4]) + if ARGV[5] == 'compact' then + redis.call('HSET', key, 'policy', ARGV[4], 'status', 'active', + 'exec_id', policy.exec_id, 'exp', policy.exp, + 'max_requests', policy.max_requests, 'max_upload_bytes', policy.max_upload_bytes, + 'max_output_files', policy.max_output_files, + 'request_count', 0, 'read_count', 0, 'upload_count', 0, 'tool_call_count', 0, 'uploaded_bytes', 0) + else + redis.call('SET', key, ARGV[4]) + end + redis.call('EXPIRE', key, tonumber(ARGV[6])) + return {'ok'} +end +if kind == 'none' then + if op == 'revoke' then return {'ok'} end + return denied('Egress grant ledger record is missing') +end +if kind ~= 'hash' and kind ~= 'string' then return denied('Invalid egress ledger representation') end +local compact = kind == 'hash' +local record = nil +if not compact then record = cjson.decode(redis.call('GET', key)) end +local function get(field) + if compact then return redis.call('HGET', key, field) end + return record[field] +end +local function number(field) return tonumber(get(field)) end +local function put(field, value) + if compact then redis.call('HSET', key, field, value) else record[field] = value end +end +local function add(field, value) + if compact then redis.call('HINCRBY', key, field, value) else record[field] = record[field] + value end +end +local function encodeRecord(value) + local encoded = cjson.encode(value) + -- Preserve the array contract for old gateways when cjson sees empty lists. + for _, field in ipairs({'input_files', 'read_sessions', 'output_file_ids'}) do + encoded = string.gsub(encoded, '"' .. field .. '":{}', '"' .. field .. '":[]') + end + return encoded +end +local function save() + if not compact then + -- Keep the original expiration, including a revocation tombstone's lifetime. + local ttl = redis.call('PTTL', key) + redis.call('SET', key, encodeRecord(record)) + if ttl >= 0 then redis.call('PEXPIRE', key, math.max(1, ttl)) end + end +end +if op == 'revoke' then + put('status', 'revoked') + put('revoked_at', now) + put('revoke_reason', ARGV[4]) + save() + return {'ok'} +end +if get('exec_id') ~= ARGV[2] then return denied('Egress grant ledger record does not match token') end +if get('status') ~= 'active' then return denied('Egress grant has been revoked') end +if number('exp') <= now then return {'error', 'expired', 'Egress grant is expired'} end +if op == 'check' then return {'ok'} end +if op == 'snapshot' then + if compact then + record = cjson.decode(redis.call('HGET', key, 'policy')) + for _, field in ipairs({'request_count', 'read_count', 'upload_count', 'tool_call_count', 'uploaded_bytes'}) do + record[field] = number(field) + end + record.output_file_ids = {} + local fields = redis.call('HKEYS', key) + for _, field in ipairs(fields) do + if string.sub(field, 1, 7) == 'output:' then table.insert(record.output_file_ids, string.sub(field, 8)) end + end + end + return {'ok', encodeRecord(record)} +end +local file = ARGV[4] +local bytes = tonumber(ARGV[5]) +local outputField = 'output:' .. file +local outputIndex = nil +if not compact and (op == 'reserve' or op == 'release') then + for i, id in ipairs(record.output_file_ids) do if id == file then outputIndex = i end end +end +local existing = compact and redis.call('HGET', key, outputField) or outputIndex +if op == 'release' then + -- A retried release must not refund another operation's request or byte budget. + if not existing then return {'ok'} end + local reservedBytes = compact and tonumber(existing) or bytes + if compact and reservedBytes ~= bytes then return denied('Upload release does not match reservation') end + add('uploaded_bytes', -math.min(number('uploaded_bytes'), reservedBytes)) + add('upload_count', -1) + add('request_count', -1) + if compact then redis.call('HDEL', key, outputField) else table.remove(record.output_file_ids, outputIndex) end +elseif op == 'reserve' or op == 'read' or op == 'tool' then + if number('request_count') >= number('max_requests') then return denied('Egress grant request budget exceeded') end + if op == 'reserve' then + local maxBytes = math.min(number('max_upload_bytes'), tonumber(ARGV[6])) + if not bytes or bytes < 0 or bytes ~= math.floor(bytes) or bytes > maxBytes then + return denied('Upload exceeds per-file egress byte limit') + end + if existing then return denied('Output file id has already been used for this grant') end + if number('upload_count') >= number('max_output_files') then return denied('Output file count budget exceeded') end + if number('uploaded_bytes') + bytes > maxBytes * number('max_output_files') then return denied('Aggregate upload byte budget exceeded') end + add('uploaded_bytes', bytes) + add('upload_count', 1) + if compact then redis.call('HSET', key, outputField, bytes) else table.insert(record.output_file_ids, file) end + elseif op == 'read' then add('read_count', 1) + else add('tool_call_count', 1) end + add('request_count', 1) +else return denied('Unknown egress ledger operation') end +save() +return {'ok'} +`; diff --git a/service/src/egress-ledger.test.ts b/service/src/egress-ledger.test.ts index 48b69d71..87fbec8d 100644 --- a/service/src/egress-ledger.test.ts +++ b/service/src/egress-ledger.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; -import RedisMock from 'ioredis-mock'; +import { startTestRedis } from './test/redis'; import { env } from './config'; import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; @@ -9,6 +9,9 @@ import { ensureEgressLedger, releaseEgressUpload, reserveEgressUpload, + recordEgressRead, + recordEgressToolCall, + checkEgressGrantActive, revokeEgressLedger, setEgressLedgerRedisForTest, } from './egress-ledger'; @@ -49,26 +52,30 @@ function expectEgressError(fn: () => Promise, reason: EgressGrantError[ ); } -describe('egress Redis ledger', () => { - let redis: InstanceType; +describe.each([false, true])('egress Redis ledger compact=%s', compact => { + let redis: Awaited>; let previousRequired: boolean; + let previousCompact: boolean; let previousMaxFileBytes: number; let previousTtlGraceSeconds: number; - beforeEach(() => { + beforeEach(async () => { previousRequired = env.EGRESS_LEDGER_REQUIRED; + previousCompact = env.EGRESS_LEDGER_COMPACT; + env.EGRESS_LEDGER_COMPACT = compact; previousMaxFileBytes = env.EGRESS_GATEWAY_MAX_FILE_BYTES; previousTtlGraceSeconds = env.EGRESS_LEDGER_TTL_GRACE_SECONDS; env.EGRESS_LEDGER_REQUIRED = true; env.EGRESS_GATEWAY_MAX_FILE_BYTES = 10; - redis = new RedisMock(); + redis = await startTestRedis(); setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); }); afterEach(async () => { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = previousRequired; + env.EGRESS_LEDGER_COMPACT = previousCompact; env.EGRESS_GATEWAY_MAX_FILE_BYTES = previousMaxFileBytes; env.EGRESS_LEDGER_TTL_GRACE_SECONDS = previousTtlGraceSeconds; }); @@ -103,7 +110,7 @@ describe('egress Redis ledger', () => { ); }); - test('clears Redis WATCH after rejected mutations so later valid updates can proceed', async () => { + test('leaves counters unchanged after a rejected mutation', async () => { const claims = grant({ max_output_files: 2, max_requests: 5 }); await createEgressLedger(claims); @@ -143,7 +150,7 @@ describe('egress Redis ledger', () => { await expectEgressError(() => assertEgressGrantActive(claims), 'scope_mismatch'); }); - test('keeps concurrent WATCH mutations isolated on dedicated Redis connections', async () => { + test('accounts concurrent operations without WATCH connections', async () => { const claims = grant({ max_output_files: 16, max_requests: 16, @@ -170,7 +177,7 @@ describe('egress Redis ledger', () => { ); const record = await assertEgressGrantActive(claims); - expect(duplicateCount).toBe(8); + expect(duplicateCount).toBe(0); expect(record.request_count).toBe(12); expect(record.upload_count).toBe(12); expect(record.uploaded_bytes).toBe(12); @@ -184,4 +191,60 @@ describe('egress Redis ledger', () => { redis.duplicate = duplicate as typeof redis.duplicate; } }); + test('admits exactly the request budget under a 240-file burst', async () => { + const claims = grant({ max_requests: 100, input_files: Array.from({ length: 240 }, (_, i) => ({ + id: `file_${i}`, session_id: 'inputs', name: `${i}.txt`, + })) }); + await createEgressLedger(claims); + const results = await Promise.allSettled(Array.from({ length: 240 }, () => recordEgressRead(claims))); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(100); + expect((await assertEgressGrantActive(claims)).request_count).toBe(100); + expect((await assertEgressGrantActive(claims)).read_count).toBe(100); + }); + + test('rejects wrong execution, expired grants and mutations after revocation', async () => { + const claims = grant({ max_requests: 1000 }); + await createEgressLedger(claims); + await expectEgressError(() => recordEgressRead({ ...claims, exec_id: 'wrong' }), 'scope_mismatch'); + await recordEgressToolCall(claims.grant_id, claims.exec_id); + await Promise.all([ + ...Array.from({ length: 40 }, () => recordEgressRead(claims).catch(() => {})), + revokeEgressLedger(claims.grant_id, 'done'), + ]); + await createEgressLedger(claims); + await expectEgressError(() => checkEgressGrantActive(claims), 'scope_mismatch'); + await expectEgressError(() => recordEgressRead(claims), 'scope_mismatch'); + const expired = grant({ grant_id: 'expired', exp: nowSeconds() - 1 }); + await createEgressLedger(expired); + await expectEgressError(() => checkEgressGrantActive(expired), 'expired'); + }); + + test('duplicate releases cannot refund another upload or read', async () => { + const claims = grant({ max_requests: 10, max_output_files: 2 }); + await createEgressLedger(claims); + await recordEgressRead(claims); + await reserveEgressUpload({ grant: claims, fileId: 'a', bytes: 3 }); + await reserveEgressUpload({ grant: claims, fileId: 'b', bytes: 4 }); + await Promise.all(Array.from({ length: 10 }, () => releaseEgressUpload({ grant: claims, fileId: 'a', bytes: 3 }))); + expect(await assertEgressGrantActive(claims)).toMatchObject({ + request_count: 2, upload_count: 1, uploaded_bytes: 4, output_file_ids: ['b'], + }); + }); + + test('format selection affects new grants only and never resets existing state', async () => { + const claims = grant(); + await createEgressLedger(claims); + await recordEgressRead(claims); + env.EGRESS_LEDGER_COMPACT = !compact; + await ensureEgressLedger(claims); + await recordEgressRead(claims); + expect(await redis.type(`codeapi:egress:grant:${claims.grant_id}`)).toBe(compact ? 'hash' : 'string'); + expect((await assertEgressGrantActive(claims)).request_count).toBe(2); + if (!compact) { + const legacy = JSON.parse((await redis.get(`codeapi:egress:grant:${claims.grant_id}`))!); + expect(legacy.output_file_ids).toEqual([]); + expect(Array.isArray(legacy.input_files)).toBe(true); + } + }); + }); diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index b6bbf4fb..c635dcc4 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -6,6 +6,7 @@ import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; import logger from './logger'; import { redisKeepAliveOptions } from './redis-options'; +import { EGRESS_LEDGER_SCRIPT } from './egress-ledger-script'; type LedgerStatus = 'active' | 'revoked'; @@ -30,21 +31,17 @@ export interface EgressLedgerRecord { output_file_ids: string[]; } -let redis: IORedis | null = null; -const LEDGER_MUTATION_ATTEMPTS = 32; -const LEDGER_MUTATION_POOL_SIZE = Math.max(1, Number(process.env.CODEAPI_EGRESS_LEDGER_MUTATION_CONNECTIONS) || 32); - -type MutationConnectionWaiter = { - resolve: (client: IORedis) => void; - reject: (error: Error) => void; -}; - -const mutationConnections = new Set(); -let idleMutationConnections: IORedis[] = []; -let mutationConnectionWaiters: MutationConnectionWaiter[] = []; +/** A lost reply is ambiguous: never replay a possibly applied mutation. + * maxRetries=0 also rejects the pending promise on disconnect instead of leaving + * it unresolved when ioredis discards its unfulfilled-command queue. */ +export const EGRESS_LEDGER_REDIS_RETRY_OPTIONS = { + autoResendUnfulfilledCommands: false, + maxRetriesPerRequest: 0, +} as const; +let redis: IORedis | null = null; +const scriptClients = new WeakSet(); export function setEgressLedgerRedisForTest(client: IORedis | null): void { - resetMutationConnections(); redis = client; } @@ -66,7 +63,7 @@ function redisConnection(): IORedis { host: process.env.REDIS_HOST ?? 'redis', port: Number(process.env.REDIS_PORT) || 6379, password: process.env.REDIS_PASSWORD, - maxRetriesPerRequest: 1, + ...EGRESS_LEDGER_REDIS_RETRY_OPTIONS, retryStrategy, enableReadyCheck: true, connectTimeout: 10000, @@ -82,71 +79,6 @@ function redisConnection(): IORedis { return redis; } -function resetMutationConnections(): void { - const resetError = new Error('Egress ledger Redis connection reset'); - for (const waiter of mutationConnectionWaiters) { - waiter.reject(resetError); - } - mutationConnectionWaiters = []; - idleMutationConnections = []; - for (const client of mutationConnections) { - client.disconnect(); - } - mutationConnections.clear(); -} - -async function dedicatedMutationConnection(): Promise { - while (idleMutationConnections.length > 0) { - const client = idleMutationConnections.pop()!; - if (client.status !== 'end') { - return client; - } - mutationConnections.delete(client); - } - - if (mutationConnections.size < LEDGER_MUTATION_POOL_SIZE) { - return createMutationConnection(); - } - - return new Promise((resolve, reject) => { - mutationConnectionWaiters.push({ resolve, reject }); - }); -} - -function createMutationConnection(): IORedis { - const client = redisConnection().duplicate(); - mutationConnections.add(client); - client.on('error', error => logger.error('Egress ledger mutation Redis error', { error })); - return client; -} - -function releaseMutationConnection(client: IORedis): void { - if (!mutationConnections.has(client) || client.status === 'end') { - mutationConnections.delete(client); - const waiter = mutationConnectionWaiters.shift(); - if (waiter) { - try { - waiter.resolve(createMutationConnection()); - } catch (error) { - waiter.reject(error instanceof Error ? error : new Error(String(error))); - } - } - return; - } - - const waiter = mutationConnectionWaiters.shift(); - if (waiter) { - waiter.resolve(client); - return; - } - - idleMutationConnections.push(client); -} - -function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} - export async function pingEgressLedger(): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; await redisConnection().ping(); @@ -173,171 +105,79 @@ function recordFromGrant(grant: EgressGrantClaims): EgressLedgerRecord { }; } -export async function createEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); +async function executeLedger( + operation: string, + grantId: string, + executionId = '', + extra: Array = [], +): Promise { + const client = redisConnection() as IORedis & { + executeEgressLedger: (...args: Array) => Promise; + }; + if (!scriptClients.has(client)) { + client.defineCommand('executeEgressLedger', { numberOfKeys: 1, lua: EGRESS_LEDGER_SCRIPT }); + scriptClients.add(client); } - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - ); + const result = await client.executeEgressLedger( + ledgerKey(grantId), operation, executionId, + Math.floor(Date.now() / 1000), ...extra, + ) as string[]; + if (result[0] === 'error') { + throw new EgressGrantError(result[1] as EgressGrantError['reason'], result[2]); + } + return result[1]; } -export async function ensureEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); - } +export async function createEgressLedger(grant: EgressGrantClaims): Promise { + if (!grant.grant_id) throw new EgressGrantError('malformed', 'Egress grant id is required'); if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - 'NX', - ); + await executeLedger('create', grant.grant_id, grant.exec_id, [ + JSON.stringify(recordFromGrant(grant)), env.EGRESS_LEDGER_COMPACT ? 'compact' : 'legacy', ttlSeconds(grant.exp), + ]); } -async function loadRecord(grantId: string): Promise { - const raw = await redisConnection().get(ledgerKey(grantId)); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - return JSON.parse(raw) as EgressLedgerRecord; -} +/** Admission is idempotent: neither replay nor rolling deployment resets budgets or revocation. */ +export const ensureEgressLedger = createEgressLedger; -function assertActive(record: EgressLedgerRecord, grant: Pick): void { - if (record.grant_id !== grant.grant_id || record.exec_id !== grant.exec_id) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record does not match token'); - } - if (record.status !== 'active') { - throw new EgressGrantError('scope_mismatch', 'Egress grant has been revoked'); - } - if (record.exp <= Math.floor(Date.now() / 1000)) { - throw new EgressGrantError('expired', 'Egress grant is expired'); - } -} - -async function mutateRecord( - grant: EgressGrantClaims, - mutate: (record: EgressLedgerRecord) => void, -): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) { - return recordFromGrant(grant); - } - const client = await dedicatedMutationConnection(); - const key = ledgerKey(grant.grant_id); - try { - for (let i = 0; i < LEDGER_MUTATION_ATTEMPTS; i++) { - await client.watch(key); - let record: EgressLedgerRecord; - try { - const raw = await client.get(key); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - record = JSON.parse(raw) as EgressLedgerRecord; - assertActive(record, grant); - mutate(record); - if (record.request_count > record.max_requests) { - throw new EgressGrantError('scope_mismatch', 'Egress grant request budget exceeded'); - } - } catch (error) { - await client.unwatch().catch(unwatchError => { - logger.warn('Failed to clear egress ledger WATCH after rejected mutation', { error: unwatchError }); - }); - throw error; - } - const result = await client.multi() - .set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)) - .exec(); - if (result) return record; - if (i < LEDGER_MUTATION_ATTEMPTS - 1) { - await sleep(Math.min(25, i + 1)); - } - } - } finally { - await client.unwatch().catch(error => { - logger.warn('Failed to clear egress ledger WATCH before returning mutation connection', { error }); - }); - releaseMutationConnection(client); - } - throw new EgressGrantError('ledger_conflict', 'Egress grant ledger update conflicted'); +/** Authorization hot path deliberately does not return the potentially large input policy. */ +export async function checkEgressGrantActive(grant: Pick): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return; + await executeLedger('check', grant.grant_id, grant.exec_id); } export async function assertEgressGrantActive(grant: EgressGrantClaims): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return recordFromGrant(grant); - const record = await loadRecord(grant.grant_id); - assertActive(record, grant); + const record = JSON.parse((await executeLedger('snapshot', grant.grant_id, grant.exec_id))!) as EgressLedgerRecord; + // Redis cjson represents empty Lua arrays as objects. + if (!Array.isArray(record.output_file_ids)) record.output_file_ids = []; + if (!Array.isArray(record.input_files)) record.input_files = []; + if (!Array.isArray(record.read_sessions)) record.read_sessions = []; return record; } export async function recordEgressRead(grant: EgressGrantClaims): Promise { - await mutateRecord(grant, record => { - record.request_count += 1; - record.read_count += 1; - }); + if (!env.EGRESS_LEDGER_REQUIRED) return; + await executeLedger('read', grant.grant_id, grant.exec_id, ['', 0]); } -export async function reserveEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; -}): Promise { - await mutateRecord(args.grant, record => { - if (args.bytes > Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES)) { - throw new EgressGrantError('scope_mismatch', 'Upload exceeds per-file egress byte limit'); - } - if (record.output_file_ids.includes(args.fileId)) { - throw new EgressGrantError('scope_mismatch', 'Output file id has already been used for this grant'); - } - if (record.output_file_ids.length >= record.max_output_files) { - throw new EgressGrantError('scope_mismatch', 'Output file count budget exceeded'); - } - const aggregateLimit = Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES) * record.max_output_files; - if (record.uploaded_bytes + args.bytes > aggregateLimit) { - throw new EgressGrantError('scope_mismatch', 'Aggregate upload byte budget exceeded'); - } - record.request_count += 1; - record.upload_count += 1; - record.uploaded_bytes += args.bytes; - record.output_file_ids.push(args.fileId); - }); +export async function reserveEgressUpload(args: { grant: EgressGrantClaims; fileId: string; bytes: number }): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return; + if (!Number.isSafeInteger(args.bytes) || args.bytes < 0) throw new EgressGrantError('scope_mismatch', 'Invalid upload byte count'); + await executeLedger('reserve', args.grant.grant_id, args.grant.exec_id, [args.fileId, args.bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES]); } -export async function releaseEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; -}): Promise { +export async function releaseEgressUpload(args: { grant: EgressGrantClaims; fileId: string; bytes: number }): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; - await mutateRecord(args.grant, record => { - record.uploaded_bytes = Math.max(0, record.uploaded_bytes - args.bytes); - record.upload_count = Math.max(0, record.upload_count - 1); - record.request_count = Math.max(0, record.request_count - 1); - record.output_file_ids = record.output_file_ids.filter(id => id !== args.fileId); - }); + if (!Number.isSafeInteger(args.bytes) || args.bytes < 0) throw new EgressGrantError('scope_mismatch', 'Invalid upload byte count'); + await executeLedger('release', args.grant.grant_id, args.grant.exec_id, [args.fileId, args.bytes]); } export async function recordEgressToolCall(grantId: string | undefined, executionId: string): Promise { if (!env.EGRESS_LEDGER_REQUIRED || !grantId) return; - const grant = { grant_id: grantId, exec_id: executionId } as EgressGrantClaims; - await mutateRecord(grant, record => { - record.request_count += 1; - record.tool_call_count += 1; - }); + await executeLedger('tool', grantId, executionId, ['', 0]); } export async function revokeEgressLedger(grantId: string, reason: string): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; - const key = ledgerKey(grantId); - const raw = await redisConnection().get(key); - if (!raw) return; - const record = JSON.parse(raw) as EgressLedgerRecord; - record.status = 'revoked'; - record.revoked_at = Math.floor(Date.now() / 1000); - record.revoke_reason = reason; - await redisConnection().set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)); + await executeLedger('revoke', grantId, '', [reason]); } diff --git a/service/src/file-download.test.ts b/service/src/file-download.test.ts new file mode 100644 index 00000000..cb32d771 --- /dev/null +++ b/service/src/file-download.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from 'bun:test'; +import { Readable, Writable } from 'node:stream'; +import { createServer } from 'node:http'; +import express from 'express'; +import { sendFileDownload } from './file-download'; + +async function serverFor(stream: Readable) { + const app = express(); + app.get('/', (req, res) => { void sendFileDownload(stream, res, req.header('x-codeapi-input-version')).catch(() => res.destroy()); }); + const server = createServer(app); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as { port: number }; + return { url: `http://127.0.0.1:${address.port}`, close: () => new Promise(resolve => server.close(() => resolve())) }; +} + +test('serves metadata from the downloaded version and rejects a stale preflight', async () => { + for (const expected of ['current', 'stale']) { + const stream = Object.assign(Readable.from(['bytes']), { headers: { + 'x-amz-meta-codeapi-version': 'current', 'x-amz-meta-read-only': 'true', + 'x-amz-meta-original-filename': 'verified.txt', + } }); + const server = await serverFor(stream); + try { + const response = await fetch(server.url, { headers: { 'X-CodeAPI-Input-Version': expected } }); + expect(response.status).toBe(expected === 'current' ? 200 : 409); + if (expected === 'current') { + expect(response.headers.get('x-read-only')).toBe('true'); + expect(response.headers.get('content-disposition')).toContain('verified.txt'); + expect(await response.text()).toBe('bytes'); + } else await response.text(); + } finally { await server.close(); } + } +}); + +test('downstream cancellation stops the storage stream under backpressure', async () => { + let produced = 0; + const stream = new Readable({ read() { if (++produced <= 1000) this.push(Buffer.alloc(64 * 1024)); else this.push(null); } }); + const response = Object.assign(new Writable({ + highWaterMark: 1, + write(_chunk, _encoding, callback) { setTimeout(callback, 10); }, + }), { setHeader() {} }); + const transfer = sendFileDownload(stream, response as unknown as express.Response); + const timer = setTimeout(() => response.destroy(), 25); + try { + await expect(transfer).rejects.toThrow(); + expect(stream.destroyed).toBe(true); + expect(produced).toBeLessThan(1000); + } finally { clearTimeout(timer); } +}); diff --git a/service/src/file-download.ts b/service/src/file-download.ts new file mode 100644 index 00000000..c46f80a1 --- /dev/null +++ b/service/src/file-download.ts @@ -0,0 +1,27 @@ +import type { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import type { Response } from 'express'; +import { contentDispositionForOriginalFilename, originalFilenameFromMetadata } from './file-metadata'; + +export async function sendFileDownload(dataStream: Readable, res: Response, expectedVersion?: string): Promise { + // MinIO returns the HTTP response stream. Read metadata from this exact GET, + // avoiding both a redundant HEAD and metadata/content races on overwrite. + const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; + if (expectedVersion && headers['x-amz-meta-codeapi-version'] !== expectedVersion) { + dataStream.destroy(); + res.status(409).json({ error: 'Input changed during preparation; retry with current metadata' }); + return; + } + const metadata: Record = {}; + for (const [key, value] of Object.entries(headers)) { + if (key.startsWith('x-amz-meta-')) metadata[key.slice(11)] = value; + } + res.setHeader('Content-Disposition', contentDispositionForOriginalFilename(originalFilenameFromMetadata(metadata))); + if (headers['content-type']) res.setHeader('Content-Type', headers['content-type']); + if (headers['content-length']) res.setHeader('Content-Length', headers['content-length']); + if (metadata['read-only'] === 'true') res.setHeader('X-Read-Only', 'true'); + if (metadata['codeapi-version']) res.setHeader('X-CodeAPI-Input-Version', metadata['codeapi-version']); + const cancel = (): void => { if (!res.writableFinished) dataStream.destroy(new Error('Download client disconnected')); }; + res.once('close', cancel); + try { await pipeline(dataStream, res); } finally { res.off('close', cancel); } +} diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts new file mode 100644 index 00000000..deeb3265 --- /dev/null +++ b/service/src/file-object-resolver.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from 'bun:test'; +import { FileObjectResolver, mapObjectDetails } from './file-object-resolver'; +import type { BucketItemStat } from 'minio'; + +describe('storage object resolution', () => { + test('indexes exact identities while reading fresh version metadata on every request', async () => { + const index = new Map(); + let lists = 0; + let heads = 0; + let version = 'first'; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { lists++; yield { name: 's/identifier.txt' }; yield { name: 's/id.txt' }; }, + stat: async key => { heads++; expect(key).toBe('s/id.txt'); return { size: 5, etag: 'etag', lastModified: new Date(), metaData: { 'codeapi-version': version } } as BucketItemStat; }, + index: { get: async k => index.get(k) ?? null, set: async (k, v) => index.set(k, v), forget: async k => index.delete(k) }, + }); + expect((await resolver.metadata('s', 'id'))?.stat.metaData['codeapi-version']).toBe('first'); + version = 'second'; + expect((await resolver.metadata('s', 'id'))?.stat.metaData['codeapi-version']).toBe('second'); + expect(lists).toBe(1); + expect(heads).toBe(2); + }); + + test('ignores foreign-session index entries and does not cache absence', async () => { + let present = false; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's2/id.txt' }; if (present) yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { get: async () => 's2/id.txt', set: async () => {}, forget: async () => {} }, + }); + expect(await resolver.resolve('s', 'id')).toBeUndefined(); + present = true; + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + }); +}); + +test('metadata listing stays bounded and ordered across 240 objects', async () => { + let active = 0; + let maximum = 0; + async function* objects() { for (let i = 0; i < 240; i++) yield i; } + const result = await mapObjectDetails(objects(), async value => { + maximum = Math.max(maximum, ++active); + await new Promise(resolve => setTimeout(resolve, value % 3)); + active--; + return value; + }, 8); + expect(maximum).toBe(8); + expect(result).toEqual(Array.from({ length: 240 }, (_, i) => i)); +}); diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts new file mode 100644 index 00000000..a41e0dfc --- /dev/null +++ b/service/src/file-object-resolver.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import type { BucketItemStat } from 'minio'; + +export interface ObjectResolverDependencies { + bucket: string; + list(prefix: string): AsyncIterable<{ name?: string }>; + stat(key: string): Promise; + index?: { + get(key: string): Promise; + set(key: string, value: string, replace: boolean): Promise; + forget(key: string, value: string): Promise; + }; +} + +/** Storage-key index is a hint, never metadata or authorization. A fresh HEAD + * proves existence and supplies the current version even on index/cache hits. */ +export class FileObjectResolver { + constructor(private readonly deps: ObjectResolverDependencies) {} + + private indexKey(session: string, id: string): string { + return `codeapi:file-key:${createHash('sha256').update(JSON.stringify([this.deps.bucket, session, id])).digest('hex')}`; + } + + private matches(key: string, session: string, id: string): boolean { + return path.posix.dirname(key) === session && + (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); + } + + async remember(session: string, id: string, key: string, replace = true): Promise { + if (!this.matches(key, session, id)) throw new Error('Object key does not match storage identity'); + await this.deps.index?.set(this.indexKey(session, id), key, replace); + } + + async resolve(session: string, id: string): Promise { + const cached = await this.deps.index?.get(this.indexKey(session, id)); + if (cached && this.matches(cached, session, id)) return cached; + for await (const object of this.deps.list(`${session}/${id}`)) { + if (object.name && this.matches(object.name, session, id)) { + await this.remember(session, id, object.name, false); + return object.name; + } + } + return undefined; + } + + async metadata(session: string, id: string): Promise<{ key: string; stat: BucketItemStat } | undefined> { + const key = await this.resolve(session, id); + if (!key) return undefined; + try { + return { key, stat: await this.deps.stat(key) }; + } catch (error) { + if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; + await this.deps.index?.forget(this.indexKey(session, id), key); + // Do not cache absence: a later upload can publish this identity again. + return undefined; + } + } +} + +/** Bound storage metadata requests while preserving listing order. */ +export async function mapObjectDetails(objects: AsyncIterable, describe: (object: T) => Promise, concurrency: number): Promise { + const results: R[] = []; + const batch: T[] = []; + const width = Math.max(1, Math.min(64, Math.floor(concurrency) || 1)); + for await (const object of objects) { + batch.push(object); + if (batch.length === width) { + results.push(...await Promise.all(batch.map(describe))); + batch.length = 0; + } + } + results.push(...await Promise.all(batch.map(describe))); + return results; +} diff --git a/service/src/file-server.ts b/service/src/file-server.ts index f9293e48..9b326ca2 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,4 +1,8 @@ import b from 'busboy'; +import { randomUUID } from 'node:crypto'; +import { mapObjectDetails } from './file-object-resolver'; +import { sendFileDownload } from './file-download'; +import { FileObjectResolver } from './file-object-resolver'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; @@ -18,7 +22,6 @@ import logger from './fileServerLogger'; import { env } from './config'; import { redisKeepAliveOptions } from './redis-options'; import { - contentDispositionForOriginalFilename, decodeOriginalFilename, originalFilenameFromMetadata, } from './file-metadata'; @@ -144,6 +147,21 @@ redisClient.on('ready', () => { logger.info('Redis Client Ready'); }); +const objectResolver = new FileObjectResolver({ + bucket: bucketName, + list: prefix => minioClient.listObjects(bucketName, prefix, true), + stat: key => minioClient.statObject(bucketName, key), + ...(env.FILE_OBJECT_INDEX_ENABLED ? { index: { + get: (key: string) => redisClient.get(key), + set: (key: string, value: string, replace: boolean) => replace + ? redisClient.set(key, value, 'EX', env.SESSION_CACHE_TTL) + : redisClient.set(key, value, 'EX', env.SESSION_CACHE_TTL, 'NX'), + forget: (key: string, value: string) => redisClient.eval( + "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end return 0", 1, key, value, + ), + } } : {}), +}); + const minioRegion = process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; async function ensureBucketExists(retries = 10, delay = 1000): Promise { @@ -250,6 +268,8 @@ async function uploadFile( * `getObject` / `statObject` without a separate Redis lookup. */ const metaData: Record = { 'Content-Type': mimetype, + // New marker on every PUT, including same-ID overwrites and metadata changes. + 'X-Amz-Meta-Codeapi-Version': randomUUID(), 'X-Amz-Meta-Original-Filename': encodedFilename, 'X-Amz-Meta-Original-Filename-Encoded': 'base64', }; @@ -267,6 +287,7 @@ async function uploadFile( } else { await minioClient.putObject(bucketName, objectName, peeked.body, undefined, metaData); } + await objectResolver.remember(session_id, fileId, objectName); logger.info(`[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`); await redisClient.set(`upload:${sessionKey}${session_id}${fileId}`, 'true', 'EX', env.SESSION_CACHE_TTL); fileUploads.inc(); @@ -443,30 +464,14 @@ app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => const { session_id, objectId } = req.params; try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - }); - } - - const stat: Partial = await minioClient.statObject(bucketName, objectName); + const resolved = await objectResolver.metadata(session_id, objectId); + if (!resolved) return res.status(404).json({ error: 'File not found' }); + const { key: objectName, stat } = resolved; const originalFilename = originalFilenameFromMetadata(stat.metaData); return res.status(200).json({ name: objectName, + version: stat.metaData?.['codeapi-version'], ...(originalFilename ? { originalFilename } : {}), size: stat.size, lastModified: stat.lastModified, @@ -487,87 +492,33 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { const { session_id, objectId } = req.params; try { - // List objects to find the correct file with extension - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - logger.warn('File not found', { session_id, objectId, bucketName }); - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - bucketName - }); - } - - logger.info(`[${INSTANCE_ID}] Attempting to download: ${objectName}`); - - const stat: Partial = await minioClient.statObject(bucketName, objectName); - - const originalFilename = originalFilenameFromMetadata(stat.metaData); - - logger.info(`[${INSTANCE_ID}] File found: ${objectName}`); - - // Explicitly remove problematic headers that might be duplicated - res.removeHeader('Transfer-Encoding'); - res.removeHeader('Date'); - - /* An object-key basename is only a storage identifier, not an original - * filename. If an S3-compatible backend drops user metadata, retain - * attachment semantics but omit the filename so the runner uses its - * caller-supplied destination. */ - res.setHeader('Content-Disposition', contentDispositionForOriginalFilename(originalFilename)); - if (stat.metaData?.['content-type'] != null) { - res.setHeader('Content-Type', stat.metaData['content-type']); - } - /* Surface the read-only flag on download so the sandbox can plumb it - * onto its in-memory file metadata without a separate metadata fetch. - * MinIO normalizes `X-Amz-Meta-Read-Only` to `read-only` in stat.metaData. */ - if (stat.metaData?.['read-only'] === 'true') { - res.setHeader('X-Read-Only', 'true'); - } - + const objectName = await objectResolver.resolve(session_id, objectId); + if (!objectName) return res.status(404).json({ error: 'File not found' }); const dataStream = await minioClient.getObject(bucketName, objectName); - fileDownloads.inc(); - - dataStream.on('data', (chunk) => { - res.write(chunk); - }); - - dataStream.on('end', () => { - res.end(); - }); - - dataStream.on('error', (err) => { - logger.error('Error streaming file:', { error: err, session_id, objectId, bucketName }); - // Only send error if headers haven't been sent yet - if (!res.headersSent) { - res.status(500).json({ - error: 'Error streaming file', - details: err.message - }); - } else { - res.end(); + try { + const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; + if (!headers['x-amz-meta-codeapi-version'] || !headers['x-amz-meta-original-filename']) { + // Preserve legacy/S3-compatible metadata behavior without promoting a + // later HEAD's version marker onto bytes from an earlier GET. + const stat = await minioClient.statObject(bucketName, objectName); + if (headers.etag?.replace(/^"|"$/g, '') !== stat.etag) { + return res.status(409).json({ error: 'Input changed during metadata lookup' }); + } + for (const [key, value] of Object.entries(stat.metaData ?? {})) { + if (key !== 'codeapi-version') headers[`x-amz-meta-${key}`] ??= value; + } } - }); + fileDownloads.inc(); + await sendFileDownload(dataStream, res, req.header('x-codeapi-input-version')); + } finally { + dataStream.destroy(); + } } catch (err) { - logger.error('Error downloading file:', { error: err, session_id, objectId, bucketName }); - return res.status(500).json({ - error: 'Error downloading file', - details: (err as Error | undefined)?.message, - session_id, - objectId, - bucketName - }); + logger.error('Error downloading file', { error: err, session_id, objectId }); + if (!res.headersSent && !res.destroyed) { + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); + return res.status(missing ? 404 : 500).json({ error: 'Error downloading file' }); + } } }); @@ -585,7 +536,7 @@ function parseObjectName(objectName: string | undefined): { session_id: string; return { session_id, file_id }; } -const detailLevels: Record Promise> | undefined> = { +const detailLevels: Record Promise>> = { simple: async (obj: BucketItem): Promise> => obj.name ?? '', summary: async (obj: BucketItem): Promise> => ({ name: obj.name, @@ -639,14 +590,9 @@ app.get('/sessions/:session_id/objects', async (req, res) => { const { detail = 'simple' } = req.query; try { - const stream = minioClient.listObjects(bucketName, session_id, true); - const objects: (t.ObjectTypes | Partial | undefined)[] = []; - + const stream = minioClient.listObjects(bucketName, `${session_id}/`, true); const getDetail = detailLevels[detail as string] ?? detailLevels.simple; - - for await (const obj of stream) { - objects.push(await getDetail(obj)); - } + const objects = await mapObjectDetails(stream, getDetail, env.FILE_METADATA_CONCURRENCY); res.json(objects); } catch (err) { diff --git a/service/src/test/redis.ts b/service/src/test/redis.ts new file mode 100644 index 00000000..f243f657 --- /dev/null +++ b/service/src/test/redis.ts @@ -0,0 +1,45 @@ +import { spawn } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import IORedis from 'ioredis'; + +/** Real Lua semantics, isolated Unix socket, no TCP listener or durable data. */ +export async function startTestRedis(): Promise }> { + const dir = await mkdtemp(path.join(tmpdir(), 'codeapi-redis-')); + const socket = path.join(dir, 'redis.sock'); + const process = spawn('redis-server', [ + '--port', '0', '--unixsocket', socket, '--unixsocketperm', '700', + '--save', '', '--appendonly', 'no', '--dir', dir, + ], { stdio: 'ignore' }); + let failure: Error | undefined; + process.on('error', error => { failure = error; }); + const exited = new Promise(resolve => { + process.once('exit', () => resolve()); + process.once('error', () => resolve()); + }); + const client = new IORedis(socket, { lazyConnect: true, retryStrategy: () => null, maxRetriesPerRequest: 0 }); + client.on('error', () => {}); + const closeTestServer = async (): Promise => { + client.disconnect(); + process.kill('SIGTERM'); + await exited; + await rm(dir, { recursive: true, force: true }); + }; + try { + for (let attempt = 0; attempt < 100; attempt++) { + if (failure) throw failure; + try { + await client.connect(); + await client.ping(); + return Object.assign(client, { closeTestServer }); + } catch { + await new Promise(resolve => setTimeout(resolve, 20)); + } + } + throw new Error('Test Redis did not start; install redis-server'); + } catch (error) { + await closeTestServer(); + throw error; + } +} From a992ec0b4bf4c4a46caf9ef38b49f0a07010eb6b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:45:37 -0400 Subject: [PATCH 04/75] =?UTF-8?q?=F0=9F=A7=B9=20fix:=20Evict=20Stale=20Fil?= =?UTF-8?q?e-Object=20Index=20Entries=20(#182)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: evict stale file-object index entries Forget cached locators after successful deletion and missing-object downloads so replacement keys resolve immediately. Reuse exact resolver matching in the delete route to avoid prefix collisions. Fixes #181 * fix: keep file-object deletion storage-authoritative * fix: retire superseded upload objects * fix: canonicalize replacement object keys * fix: collapse legacy object-key siblings * fix: recover reads from stale locators * fix: namespace canonical object identities --- service/src/file-object-resolver.test.ts | 143 ++++++++++++++++++++++- service/src/file-object-resolver.ts | 124 +++++++++++++++++--- service/src/file-server.ts | 74 +++++++++--- 3 files changed, 306 insertions(+), 35 deletions(-) diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts index deeb3265..2342d87a 100644 --- a/service/src/file-object-resolver.test.ts +++ b/service/src/file-object-resolver.test.ts @@ -1,8 +1,27 @@ import { describe, expect, test } from 'bun:test'; -import { FileObjectResolver, mapObjectDetails } from './file-object-resolver'; +import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; import type { BucketItemStat } from 'minio'; describe('storage object resolution', () => { + test('replacement uploads converge on one stable object key', () => { + expect(storageKeyForUpload('s', 'id', '.csv', true)).toBe('s/.codeapi-objects/aWQ'); + expect(storageKeyForUpload('s', 'id', '.pdf', true)).toBe('s/.codeapi-objects/aWQ'); + expect(canonicalObjectId(storageKeyForUpload('s', 'report.csv', '', true))).toBe('report.csv'); + expect(storageKeyForUpload('s', 'generated', '.csv', false)).toBe('s/generated.csv'); + }); + + test('canonical dotted identities cannot match another legacy identity', async () => { + const dottedKey = storageKeyForUpload('s', 'report.csv', '', true); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: dottedKey }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + }); + + expect(await resolver.listFresh('s', 'report.csv')).toEqual([dottedKey]); + expect(await resolver.listFresh('s', 'report')).toEqual([]); + }); + test('indexes exact identities while reading fresh version metadata on every request', async () => { const index = new Map(); let lists = 0; @@ -33,6 +52,128 @@ describe('storage object resolution', () => { present = true; expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); }); + + test('falls back to storage when the advisory index is unavailable', async () => { + const failures: string[] = []; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + onIndexError: operation => failures.push(operation), + index: { + get: async () => { throw new Error('redis unavailable'); }, + set: async () => { throw new Error('redis unavailable'); }, + forget: async () => { throw new Error('redis unavailable'); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + expect(await resolver.listFresh('s', 'id')).toEqual(['s/id.txt']); + expect(failures).toEqual(['get', 'set']); + }); + + test('fresh listing ignores a stale locator and returns every exact sibling', async () => { + const index = new Map([['locator', 's/id.txt']]); + let lists = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { + lists++; + yield { name: 's/identifier.txt' }; + yield { name: 's/id.csv' }; + yield { name: 's/id.pdf' }; + }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async () => index.get('locator') ?? null, + set: async (_key, value) => index.set('locator', value), + forget: async (_key, value) => { if (index.get('locator') === value) index.delete('locator'); }, + }, + }); + + expect(await resolver.listFresh('s', 'id')).toEqual(['s/id.csv', 's/id.pdf']); + expect(index.get('locator')).toBe('s/id.txt'); + expect(lists).toBe(2); + }); + + test('metadata re-resolves storage after a cached locator is missing', async () => { + const index = new Map([['locator', 's/id.txt']]); + let heads = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's/.codeapi-objects/aWQ' }; }, + stat: async key => { + heads++; + if (key === 's/id.txt') throw Object.assign(new Error('missing'), { code: 'NoSuchKey' }); + return { + size: 1, + etag: 'current', + lastModified: new Date(), + metaData: { 'codeapi-version': 'current' }, + } as BucketItemStat; + }, + index: { + get: async () => index.get('locator') ?? null, + set: async (_key, value) => index.set('locator', value), + forget: async (_key, value) => { if (index.get('locator') === value) index.delete('locator'); }, + }, + }); + + const metadata = await resolver.metadata('s', 'id'); + expect(metadata?.key).toBe('s/.codeapi-objects/aWQ'); + expect(metadata?.stat.metaData['codeapi-version']).toBe('current'); + expect(index.get('locator')).toBe('s/.codeapi-objects/aWQ'); + expect(heads).toBe(2); + }); + + test('forgets a deleted locator so a replacement key for the same identity resolves', async () => { + const index = new Map(); + const stored = new Set(['s/id.txt']); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* (prefix) { for (const name of stored) if (name.startsWith(prefix)) yield { name }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async k => index.get(k) ?? null, + set: async (k, v, replace) => { if (replace || !index.has(k)) index.set(k, v); }, + forget: async (k, v) => { if (index.get(k) === v) index.delete(k); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + stored.delete('s/id.txt'); + await resolver.forget('s', 'id', 's/id.txt'); + expect(index.size).toBe(0); + + stored.add('s/id.csv'); + expect(await resolver.resolve('s', 'id')).toBe('s/id.csv'); + }); + + test('eviction is scoped to the identity and never drops a newer cached key', async () => { + const index = new Map(); + let lists = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { lists++; yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async k => index.get(k) ?? null, + set: async (k, v, replace) => { if (replace || !index.has(k)) index.set(k, v); }, + forget: async (k, v) => { if (index.get(k) === v) index.delete(k); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + // A concurrent upload republished the identity before the delete evicted it. + await resolver.remember('s', 'id', 's/id.csv'); + await resolver.forget('s', 'id', 's/id.txt'); + // Keys outside the identity can never reach its entry. + await resolver.forget('s', 'id', 's2/id.txt'); + await resolver.forget('s', 'id', 's/other.txt'); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.csv'); + expect(lists).toBe(1); + }); }); test('metadata listing stays bounded and ordered across 240 objects', async () => { diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts index a41e0dfc..bc4b4ce5 100644 --- a/service/src/file-object-resolver.ts +++ b/service/src/file-object-resolver.ts @@ -2,10 +2,28 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import type { BucketItemStat } from 'minio'; +const CANONICAL_OBJECT_DIRECTORY = '.codeapi-objects'; + +export function canonicalObjectKey(session: string, id: string): string { + return `${session}/${CANONICAL_OBJECT_DIRECTORY}/${Buffer.from(id, 'utf8').toString('base64url')}`; +} + +export function canonicalObjectId(key: string): string | undefined { + const parts = key.split('/'); + if (parts.length !== 3 || parts[1] !== CANONICAL_OBJECT_DIRECTORY || parts[2] === '') return undefined; + try { + const id = Buffer.from(parts[2], 'base64url').toString('utf8'); + return canonicalObjectKey(parts[0], id) === key ? id : undefined; + } catch { + return undefined; + } +} + export interface ObjectResolverDependencies { bucket: string; list(prefix: string): AsyncIterable<{ name?: string }>; stat(key: string): Promise; + onIndexError?(operation: 'get' | 'set' | 'forget', error: unknown): void; index?: { get(key: string): Promise; set(key: string, value: string, replace: boolean): Promise; @@ -13,48 +31,122 @@ export interface ObjectResolverDependencies { }; } +/** Caller-supplied identities keep one stable storage key across replacement + * filenames. This gives concurrent PUTs one last-writer-wins S3 object without + * requiring a distributed lock or leaving extension-keyed siblings behind. */ +export function storageKeyForUpload( + session: string, + id: string, + extension: string, + replacing: boolean, +): string { + return replacing ? canonicalObjectKey(session, id) : `${session}/${id}${extension}`; +} + /** Storage-key index is a hint, never metadata or authorization. A fresh HEAD * proves existence and supplies the current version even on index/cache hits. */ export class FileObjectResolver { constructor(private readonly deps: ObjectResolverDependencies) {} + private reportIndexError(operation: 'get' | 'set' | 'forget', error: unknown): void { + this.deps.onIndexError?.(operation, error); + } + private indexKey(session: string, id: string): string { return `codeapi:file-key:${createHash('sha256').update(JSON.stringify([this.deps.bucket, session, id])).digest('hex')}`; } private matches(key: string, session: string, id: string): boolean { + if (key === canonicalObjectKey(session, id)) return true; return path.posix.dirname(key) === session && (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); } async remember(session: string, id: string, key: string, replace = true): Promise { if (!this.matches(key, session, id)) throw new Error('Object key does not match storage identity'); - await this.deps.index?.set(this.indexKey(session, id), key, replace); + try { + await this.deps.index?.set(this.indexKey(session, id), key, replace); + } catch (error) { + this.reportIndexError('set', error); + } } - async resolve(session: string, id: string): Promise { - const cached = await this.deps.index?.get(this.indexKey(session, id)); - if (cached && this.matches(cached, session, id)) return cached; - for await (const object of this.deps.list(`${session}/${id}`)) { - if (object.name && this.matches(object.name, session, id)) { - await this.remember(session, id, object.name, false); - return object.name; + /** Evict a cached locator once its object is known to be gone. Scoped to the + * requested identity, and conditional on the stored value so a replacement + * key published concurrently for the same identity is never dropped. */ + async forget(session: string, id: string, key: string): Promise { + if (!this.matches(key, session, id)) return; + try { + await this.deps.index?.forget(this.indexKey(session, id), key); + } catch (error) { + this.reportIndexError('forget', error); + } + } + + private async cached(session: string, id: string): Promise { + try { + const key = await this.deps.index?.get(this.indexKey(session, id)); + return key && this.matches(key, session, id) ? key : undefined; + } catch (error) { + this.reportIndexError('get', error); + return undefined; + } + } + + private async findInStorage(session: string, id: string, replaceIndex: boolean): Promise { + for (const prefix of [canonicalObjectKey(session, id), `${session}/${id}`]) { + for await (const object of this.deps.list(prefix)) { + if (object.name && this.matches(object.name, session, id)) { + await this.remember(session, id, object.name, replaceIndex); + return object.name; + } } } return undefined; } + /** List every exact storage key for an identity without consulting its + * locator. Used to collapse legacy siblings and delete the whole identity. */ + async listFresh(session: string, id: string): Promise { + const keys = new Set(); + for (const prefix of [canonicalObjectKey(session, id), `${session}/${id}`]) { + for await (const object of this.deps.list(prefix)) { + if (object.name && this.matches(object.name, session, id)) keys.add(object.name); + } + } + return [...keys]; + } + + async resolve(session: string, id: string): Promise { + return await this.cached(session, id) ?? await this.findInStorage(session, id, false); + } + + /** Recover once a cached key is proven missing. Eviction and publication are + * advisory; the authoritative storage listing determines the replacement. */ + async recover(session: string, id: string, missingKey: string): Promise { + await this.forget(session, id, missingKey); + const [current] = await this.listFresh(session, id); + if (current) await this.remember(session, id, current); + return current; + } + async metadata(session: string, id: string): Promise<{ key: string; stat: BucketItemStat } | undefined> { - const key = await this.resolve(session, id); + let key = await this.resolve(session, id); if (!key) return undefined; - try { - return { key, stat: await this.deps.stat(key) }; - } catch (error) { - if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; - await this.deps.index?.forget(this.indexKey(session, id), key); - // Do not cache absence: a later upload can publish this identity again. - return undefined; + for (let attempt = 0; attempt < 2; attempt++) { + try { + return { key, stat: await this.deps.stat(key) }; + } catch (error) { + if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; + if (attempt === 1) { + await this.forget(session, id, key); + return undefined; + } + key = await this.recover(session, id, key); + if (!key) return undefined; + } } + return undefined; } } diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 9b326ca2..3dfa0a10 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,8 +1,7 @@ import b from 'busboy'; import { randomUUID } from 'node:crypto'; -import { mapObjectDetails } from './file-object-resolver'; +import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; import { sendFileDownload } from './file-download'; -import { FileObjectResolver } from './file-object-resolver'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; @@ -151,6 +150,7 @@ const objectResolver = new FileObjectResolver({ bucket: bucketName, list: prefix => minioClient.listObjects(bucketName, prefix, true), stat: key => minioClient.statObject(bucketName, key), + onIndexError: (operation, error) => logger.warn('File-object index operation failed', { operation, error }), ...(env.FILE_OBJECT_INDEX_ENABLED ? { index: { get: (key: string) => redisClient.get(key), set: (key: string, value: string, replace: boolean) => replace @@ -162,6 +162,16 @@ const objectResolver = new FileObjectResolver({ } } : {}), }); +/** Index eviction is best effort: the index is only a hint, so a Redis failure + * must never turn a completed delete or a missing-object 404 into a 500. */ +async function forgetObjectKey(session_id: string, objectId: string, objectName: string): Promise { + try { + await objectResolver.forget(session_id, objectId, objectName); + } catch (error) { + logger.warn('Failed to evict file-object index entry', { error, session_id, objectId, objectName }); + } +} + const minioRegion = process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; async function ensureBucketExists(retries = 10, delay = 1000): Promise { @@ -257,7 +267,14 @@ async function uploadFile( ): Promise { const fileId = existingFileId ?? nanoid(); const fileExtension = path.extname(filename); - const objectName = `${session_id}/${fileId}${fileExtension}`; + // Caller-supplied identities use one canonical key, so concurrent writers + // converge on S3's last-writer semantics regardless of filename extension. + const objectName = storageKeyForUpload( + session_id, + fileId, + fileExtension, + existingFileId != null, + ); const encodedFilename = Buffer.from(filename).toString('base64'); @@ -287,6 +304,15 @@ async function uploadFile( } else { await minioClient.putObject(bucketName, objectName, peeked.body, undefined, metaData); } + if (existingFileId != null) { + // Retire every extension-keyed sibling left by older replacement behavior. + // Concurrent replacement writers share objectName and never delete it. + for (const sibling of await objectResolver.listFresh(session_id, fileId)) { + if (sibling === objectName) continue; + await minioClient.removeObject(bucketName, sibling); + await objectResolver.forget(session_id, fileId, sibling); + } + } await objectResolver.remember(session_id, fileId, objectName); logger.info(`[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`); await redisClient.set(`upload:${sessionKey}${session_id}${fileId}`, 'true', 'EX', env.SESSION_CACHE_TTL); @@ -490,11 +516,21 @@ app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { const { session_id, objectId } = req.params; + let objectName: string | undefined; try { - const objectName = await objectResolver.resolve(session_id, objectId); + objectName = await objectResolver.resolve(session_id, objectId); if (!objectName) return res.status(404).json({ error: 'File not found' }); - const dataStream = await minioClient.getObject(bucketName, objectName); + let dataStream: Readable; + try { + dataStream = await minioClient.getObject(bucketName, objectName); + } catch (error) { + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? ''); + if (!missing) throw error; + objectName = await objectResolver.recover(session_id, objectId, objectName); + if (!objectName) return res.status(404).json({ error: 'File not found' }); + dataStream = await minioClient.getObject(bucketName, objectName); + } try { const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; if (!headers['x-amz-meta-codeapi-version'] || !headers['x-amz-meta-original-filename']) { @@ -515,8 +551,11 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { } } catch (err) { logger.error('Error downloading file', { error: err, session_id, objectId }); + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); + // A locator that no longer names bytes must not shadow a replacement object + // published for the same identity until the index TTL expires. + if (missing && objectName) await forgetObjectKey(session_id, objectId, objectName); if (!res.headersSent && !res.destroyed) { - const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); return res.status(missing ? 404 : 500).json({ error: 'Error downloading file' }); } } @@ -527,6 +566,10 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { */ function parseObjectName(objectName: string | undefined): { session_id: string; file_id: string } | null { if (objectName == null || objectName === '') return null; + const canonicalId = canonicalObjectId(objectName); + if (canonicalId != null) { + return { session_id: objectName.split('/', 1)[0], file_id: canonicalId }; + } const parts = objectName.split('/'); if (parts.length < 2) return null; const session_id = parts[0]; @@ -605,17 +648,9 @@ app.delete('/sessions/:session_id/objects/:fileId', async (req, res) => { const { session_id, fileId } = req.params; try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${fileId}`, true); - let objectName = ''; + const objectNames = await objectResolver.listFresh(session_id, fileId); - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${fileId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { + if (objectNames.length === 0) { logger.warn('File not found for deletion', { session_id, fileId, bucketName }); return res.status(404).json({ error: 'File not found', @@ -626,8 +661,11 @@ app.delete('/sessions/:session_id/objects/:fileId', async (req, res) => { }); } - await minioClient.removeObject(bucketName, objectName); - logger.info(`[${INSTANCE_ID}] File deleted successfully: ${objectName}`); + for (const objectName of objectNames) { + await minioClient.removeObject(bucketName, objectName); + await forgetObjectKey(session_id, fileId, objectName); + } + logger.info(`[${INSTANCE_ID}] File identity deleted successfully`, { session_id, fileId, objectNames }); return res.status(200).json({ message: 'File deleted successfully', session_id, From c3fd558195c8c4513977b83963c6aad495a4874d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:45:48 -0400 Subject: [PATCH 05/75] perf: enable authorized input reuse by default (#183) --- api/src/config-defaults.test.ts | 21 +++++++++++++++++++++ api/src/config.ts | 3 ++- docs/INPUT_REUSE.md | 8 ++++---- helm/codeapi/values.yaml | 2 +- 4 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 api/src/config-defaults.test.ts diff --git a/api/src/config-defaults.test.ts b/api/src/config-defaults.test.ts new file mode 100644 index 00000000..281f5679 --- /dev/null +++ b/api/src/config-defaults.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from 'bun:test'; +import path from 'node:path'; + +function readHttpInputCacheDefault(value?: string): boolean { + const env = { ...process.env }; + if (value === undefined) delete env.SANDBOX_HTTP_INPUT_CACHE_ENABLED; + else env.SANDBOX_HTTP_INPUT_CACHE_ENABLED = value; + const script = [ + `const { config } = await import(${JSON.stringify(path.resolve(process.cwd(), 'src/config.ts'))})`, + 'process.stdout.write(JSON.stringify(config.http_input_cache_enabled))', + ].join(';'); + const result = Bun.spawnSync({ cmd: [process.execPath, '--eval', script], env }); + expect(result.exitCode).toBe(0); + return JSON.parse(result.stdout.toString()) as boolean; +} + +test('authorized HTTP input reuse defaults on and retains an explicit rollback switch', () => { + expect(readHttpInputCacheDefault()).toBe(true); + expect(readHttpInputCacheDefault('false')).toBe(false); + expect(readHttpInputCacheDefault('true')).toBe(true); +}); diff --git a/api/src/config.ts b/api/src/config.ts index f3b935ff..a5d0c993 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -122,7 +122,8 @@ export const config = { /* Ceiling for the pushed input cache (session-inputs.ts). Eviction is * always safe — a miss simply re-pushes on the next probe — so this is a * disk guard, not a correctness knob. */ - http_input_cache_enabled: process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED === 'true', + http_input_cache_enabled: + (process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED ?? 'true') === 'true', http_input_cache_max_objects: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS, 4096), http_input_cache_max_inflight: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT, 16), input_cache_max_bytes: safeInt( diff --git a/docs/INPUT_REUSE.md b/docs/INPUT_REUSE.md index 40e277c4..f87afa89 100644 --- a/docs/INPUT_REUSE.md +++ b/docs/INPUT_REUSE.md @@ -48,7 +48,7 @@ sequenceDiagram | `egressGrant.inputManifestTimeoutMs` | `CODEAPI_INPUT_MANIFEST_TIMEOUT_MS` | `10000` | | `fileServer.objectIndexEnabled` | `CODEAPI_FILE_OBJECT_INDEX_ENABLED` | `false` | | `fileServer.metadataConcurrency` | `CODEAPI_FILE_METADATA_CONCURRENCY` | `1` | -| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `false` | +| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `true` | | `workerSandbox.sandbox.httpInputCacheMaxInflight` | `SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT` | `16` | | `workerSandbox.sandbox.httpInputCacheMaxObjects` | `SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS` | `4096` | | `workerSandbox.sandbox.inputCacheMaxBytes` | `SANDBOX_INPUT_CACHE_MAX_BYTES` | `536870912` | @@ -61,14 +61,14 @@ Metadata listing concurrency preserves order and is capped at 64. A canary can u ## Rollout and rollback -1. Deploy the new binaries with feature flags off. Atomic accounting supports existing JSON ledgers, and legacy downloads retain metadata compatibility. The new file server stamps future uploads with versions. +1. Deploy the new binaries with HTTP input reuse enabled by default. Mixed-version requests remain compatible: older gateways and relays fall back to normal downloads, while older unversioned objects return `cacheable: false`. The new file server stamps future uploads with versions. Set `workerSandbox.sandbox.httpInputCacheEnabled=false` only when a staged rollout requires the immediate rollback path. 2. Update **all** egress-gateway replicas before enabling compact ledgers. New binaries read both formats regardless of the creation flag. Older binaries cannot read compact hashes. To roll back to an older binary, disable compact creation, drain active grants, and wait their maximum TTL plus grace; never delete active ledgers to force a rollback. 3. Update all file-server writers before enabling the object-key index. Otherwise an older writer can change a locator without updating the index. Keep file-server replicas consistent during an indexed rollout. -4. Update the gateway, relay, runner, and launcher before enabling HTTP reuse on a small runner canary. Older gateway/relay metadata routes return 404/405 and fall back safely. Older files return `cacheable: false`. Keep the feature disabled for storage adapters that cannot return user metadata on GET. +4. Canary the default-on HTTP reuse path after updating the gateway, relay, runner, and launcher. Keep the feature explicitly disabled for storage adapters that cannot return user metadata on GET. 5. Observe `codeapi_sandbox_http_input_cache_events_total` (bounded event labels, no identities), cold and warm preparation latency, storage/Redis operations, admission fairness, request budgets, and memory/disk pressure before widening the rollout. A successful manifest consumes one read request for the batch, matching the existing list-request accounting unit. Per-file compatibility preflights each consume a read request; each cold miss consumes an additional download request. Do not disable budget enforcement to accommodate a workload. 6. Disable HTTP reuse to return to normal downloads immediately. Cached files can age out normally; no workspace deletion or migration is needed. -The creation flags default off. No deployment or object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. +HTTP input reuse defaults on. Compact-ledger creation and the object-key index remain off until their mixed-version rollout requirements are satisfied. No object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. ## Validation diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 3ecfbef8..bafed305 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -273,7 +273,7 @@ workerSandbox: # Defaults to maxConcurrentJobs when unset. jobUidCount: null workspaceReaperMaxAgeSeconds: 3600 - httpInputCacheEnabled: false + httpInputCacheEnabled: true httpInputCacheMaxInflight: 16 httpInputCacheMaxObjects: 4096 inputCacheMaxBytes: 536870912 From 76129e15e5f5a62d193074524459c80708c0d741 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:58:40 -0400 Subject: [PATCH 06/75] fix: honor requested input destinations (#184) --- api/openapi.yaml | 1 + api/src/download.test.ts | 111 +++++++++-------------- api/src/job-helpers.test.ts | 169 +++--------------------------------- api/src/job.ts | 86 ++++-------------- 4 files changed, 67 insertions(+), 300 deletions(-) diff --git a/api/openapi.yaml b/api/openapi.yaml index e4c04c14..23e1224c 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -67,6 +67,7 @@ components: properties: name: type: string + description: Relative path where the file is mounted in the sandbox. id: type: string content: diff --git a/api/src/download.test.ts b/api/src/download.test.ts index da877dad..d0b6ed6e 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -18,12 +18,8 @@ import { /** * Integration tests for `Job.downloadAndWriteFile` against a real HTTP - * listener. These exercise the cross-repo round-trip: the file-server - * (codeapi/service) emits `Content-Disposition: attachment; - * filename*=UTF-8''` for nested artifacts, and the - * sandbox-side parser must recover the path so the file lands at the same - * nested location on the next prime(). Hitting a real listener (not a - * mocked Response) catches anything fetch-level that a unit test would miss. + * listener. Hitting a real listener verifies that response metadata cannot + * redirect a caller-validated sandbox destination. */ interface DownloadInternals { @@ -176,21 +172,16 @@ afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); }); -describe('downloadAndWriteFile / RFC 5987 round-trip', () => { - it('writes a nested-path artifact at the encoded location', async () => { - /* Simulates the matplotlib-bug shape: codeapi previously returned a - * flat `file.name` and the original path was carried only by the - * server's `filename*=` header. The fix is: parser recovers the path, - * `mkdir { recursive: true }` creates the parent dir, file ends up - * where the user expects to `cat` it on the next turn. */ +describe('downloadAndWriteFile destinations', () => { + it('writes a nested-path artifact at the requested location', async () => { const file: TFile = { id: 'nested-id', storage_session_id: 'prev-session', - name: 'flat-fallback.txt', + name: 'proj/notes.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, - contentDisposition: "attachment; filename*=UTF-8''proj%2Fnotes.txt", + contentDisposition: "attachment; filename*=UTF-8''stored-original.txt", body: 'hello from a nested artifact\n', }); @@ -212,7 +203,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { const file: TFile = { id: 'opaque-object-handle', storage_session_id: 'opaque-session-handle', - name: 'gateway-fallback.txt', + name: 'gateway.txt', }; let sawGrantHeader = false; let sawRelayToken = false; @@ -275,14 +266,11 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect((await fsp.stat(path.join(tmpDir, 'readonly.txt'))).mode & 0o777).toBe(SANDBOX_READONLY_FILE_MODE); }); - it('falls back to the legacy filename= form when filename*= is absent', async () => { - /* Backwards-compat: older file-servers (or proxies that strip RFC - * 5987 extended-form headers) still send the legacy quoted form. The - * parser must still find a name and write the file. */ + it('downloads when a legacy filename header matches the requested name', async () => { const file: TFile = { id: 'legacy-id', storage_session_id: 'prev-session', - name: 'ignored.txt', + name: 'legacy.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, @@ -323,7 +311,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.stat(path.join(tmpDir, 'opaque-storage-id.xlsx')).catch(() => null)).toBeNull(); }); - it('resolves concurrent header destinations without provisional-name false conflicts', async () => { + it('keeps concurrent inputs at their requested destinations', async () => { const renamed: TFile = { id: 'renamed-id', storage_session_id: 'prev-session', @@ -338,8 +326,6 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { status: 200, contentDisposition: 'attachment; filename="actual.txt"', body: 'renamed bytes', - /* Make the other ref resolve `vacated.txt` while this ref's requested - * name would still be provisional under the old reservation scheme. */ delayMs: 75, }); routes.set(`/sessions/${encodeURIComponent(replacement.storage_session_id!)}/objects/${encodeURIComponent(replacement.id!)}`, { @@ -356,75 +342,62 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { await job.prime(); const submissionDir = asInternals(job).submissionDir; expect(await fsp.readFile(path.join(submissionDir, 'actual.txt'), 'utf8')) - .toBe('renamed bytes'); - expect(await fsp.readFile(path.join(submissionDir, 'vacated.txt'), 'utf8')) .toBe('replacement bytes'); + expect(await fsp.readFile(path.join(submissionDir, 'vacated.txt'), 'utf8')) + .toBe('renamed bytes'); } finally { config.prime_concurrency = originalPrimeConcurrency; await job.cleanup(); } }); - it('rejects concurrent refs that resolve to the same destination before either can overwrite', async () => { - const slower: TFile = { - id: 'same-slower-id', + it('keeps distinct requested names when stored objects share an original filename', async () => { + const original: TFile = { + id: 'original-id', storage_session_id: 'prev-session', - name: 'slower-fallback.txt', + name: 'data.xlsx', }; - const faster: TFile = { - id: 'same-faster-id', + const aliased: TFile = { + id: 'aliased-id', storage_session_id: 'prev-session', - name: 'faster-fallback.txt', + name: 'data-3f9a2c.xlsx', }; - routes.set(`/sessions/${encodeURIComponent(slower.storage_session_id!)}/objects/${encodeURIComponent(slower.id!)}`, { + routes.set(`/sessions/${encodeURIComponent(original.storage_session_id!)}/objects/${encodeURIComponent(original.id!)}`, { status: 200, - contentDisposition: 'attachment; filename="same.txt"', - body: 'slower bytes', + contentDisposition: 'attachment; filename="data.xlsx"', + body: 'original bytes', delayMs: 75, }); - routes.set(`/sessions/${encodeURIComponent(faster.storage_session_id!)}/objects/${encodeURIComponent(faster.id!)}`, { + routes.set(`/sessions/${encodeURIComponent(aliased.storage_session_id!)}/objects/${encodeURIComponent(aliased.id!)}`, { status: 200, - contentDisposition: 'attachment; filename="same.txt"', - body: 'faster bytes', + contentDisposition: 'attachment; filename="data.xlsx"', + body: 'aliased bytes', }); - let dirty = false; const job = makeJob( - [slower, faster], - sessionWorkspaceAt(tmpDir, 'rt_concurrent_same_destination', () => { dirty = true; }), + [original, aliased], + sessionWorkspaceAt(tmpDir, 'rt_shared_original_filename'), ); const originalPrimeConcurrency = config.prime_concurrency; config.prime_concurrency = 2; - let deadlockTimer: ReturnType | undefined; try { - const outcome = await Promise.race([ - job.prime().then( - () => ({ status: 'fulfilled' as const }), - error => ({ status: 'rejected' as const, error }), - ), - new Promise<{ status: 'timeout' }>(resolve => { - deadlockTimer = setTimeout(() => resolve({ status: 'timeout' }), 2_000); - }), - ]); - if (deadlockTimer) clearTimeout(deadlockTimer); - expect(outcome.status).toBe('rejected'); - if (outcome.status === 'rejected') { - expect(outcome.error).toBeInstanceOf(SessionWorkspaceDirtyError); - } - expect(dirty).toBe(true); - expect(await fsp.readFile(path.join(tmpDir, 'same.txt'), 'utf8')).toBe('faster bytes'); + await job.prime(); + const submissionDir = asInternals(job).submissionDir; + expect(await fsp.readFile(path.join(submissionDir, 'data.xlsx'), 'utf8')) + .toBe('original bytes'); + expect(await fsp.readFile(path.join(submissionDir, 'data-3f9a2c.xlsx'), 'utf8')) + .toBe('aliased bytes'); } finally { - if (deadlockTimer) clearTimeout(deadlockTimer); config.prime_concurrency = originalPrimeConcurrency; await job.cleanup(); } }); - it('decodes UTF-8 percent-encoded names with non-ASCII characters', async () => { + it('keeps a Unicode requested name when the header is percent encoded', async () => { const file: TFile = { id: 'utf8-id', storage_session_id: 'prev-session', - name: 'ignored.txt', + name: '你好.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, @@ -645,11 +618,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { await expect(fsp.access(path.join(tmpDir, 'should-not-exist.txt'))).rejects.toThrow(); }); - it('rejects a server-supplied filename that escapes the submission dir', async () => { - /* Companion guarantee for the path-preserving sanitizer on the - * LibreChat side: if a malicious / misconfigured server tries to - * smuggle a `..` traversal via Content-Disposition, the codeapi-side - * `validateFilePath` aborts before any write happens. */ + it('ignores a server-supplied filename that escapes the submission dir', async () => { const file: TFile = { id: 'evil-id', storage_session_id: 'prev-session', @@ -658,16 +627,14 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, contentDisposition: "attachment; filename*=UTF-8''..%2F..%2Fescape.txt", - body: 'should never be written', + body: 'safe bytes', }); const job = makeJob([file]); asInternals(job).submissionDir = tmpDir; - /* downloadAndWriteFile rethrows ValidationError fast (no retries) so - * `expect(...).rejects` is the right assertion. */ - await expect(job.downloadAndWriteFile(file)).rejects.toThrow(); - /* Defensive: nothing escaped to a parent dir. */ + await expect(job.downloadAndWriteFile(file)).resolves.toBe('innocent.txt'); + expect(await fsp.readFile(path.join(tmpDir, 'innocent.txt'), 'utf8')).toBe('safe bytes'); const parent = path.dirname(tmpDir); await expect(fsp.access(path.join(parent, 'escape.txt'))).rejects.toThrow(); }); diff --git a/api/src/job-helpers.test.ts b/api/src/job-helpers.test.ts index 0c31f9d3..460902da 100644 --- a/api/src/job-helpers.test.ts +++ b/api/src/job-helpers.test.ts @@ -3,7 +3,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { - resolveOriginalName, + resolveInputDestination, isNormalizedObjectForSession, markerConflictsWithExplicitFile, aggregateBashExtras, @@ -41,168 +41,19 @@ function makeRuntime(overrides: Partial & { language: string; pkgdir: s }; } -describe('resolveOriginalName', () => { - function responseWithHeader(value?: string): Response { - const headers = new Headers(); - if (value !== undefined) headers.set('content-disposition', value); - return new Response(null, { headers }); - } - - it('returns file.name when no Content-Disposition is present', () => { - expect( - resolveOriginalName(responseWithHeader(), { name: 'script.py', id: 'abc' }), - ).toBe('script.py'); - }); - - it('extracts quoted filename from Content-Disposition', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="server-name.py"'), - { name: 'client-name.py', id: 'abc' }, - ), - ).toBe('server-name.py'); - }); - - it('extracts unquoted filename from Content-Disposition', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=plain.txt'), - { name: 'ignored.txt', id: 'abc' }, - ), - ).toBe('plain.txt'); - }); - - it('falls back to file.id when file.name is empty and no header exists', () => { - expect( - resolveOriginalName(responseWithHeader(), { name: '', id: 'file-id-123' }), - ).toBe('file-id-123'); - }); - - it('falls back to file.name when header is malformed (no filename token)', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment'), - { name: 'fallback.py', id: 'abc' }, - ), - ).toBe('fallback.py'); - }); - - it('stops at the closing quote when the quoted filename is followed by more params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="foo.txt"; size=123'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('stops at a semicolon when the unquoted filename is followed by more params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=foo.txt; size=123'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('stops at whitespace when the unquoted filename is followed by whitespace-separated params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=foo.txt extra'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('returns empty string when both name and id are absent', () => { - expect(resolveOriginalName(responseWithHeader(), { name: '' })).toBe(''); +describe('resolveInputDestination', () => { + it('uses the caller-requested sandbox path', () => { + expect(resolveInputDestination({ name: 'nested/script.py', id: 'abc' })) + .toBe('nested/script.py'); }); - it('returns empty string when name is empty, id is absent, and header is malformed', () => { - expect( - resolveOriginalName(responseWithHeader('attachment'), { name: '' }), - ).toBe(''); - }); - - it('decodes RFC 5987 filename*= preserving slashes for nested artifact paths', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''test_folder%2Ftest_file.txt"), - { name: 'test_file.txt', id: 'abc' }, - ), - ).toBe('test_folder/test_file.txt'); - }); - - it('decodes RFC 5987 filename*= with a UTF-8 charset that includes a language tag', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8'en'foo%20bar.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo bar.txt'); - }); - - it('decodes RFC 5987 filename*= with non-ASCII characters', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''%E4%BD%A0%E5%A5%BD.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('你好.txt'); - }); - - it('tolerates a filename*= form missing the UTF-8 prefix', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename*=plain.txt'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('plain.txt'); + it('falls back to the object id when no name exists', () => { + expect(resolveInputDestination({ name: '', id: 'file-id-123' })) + .toBe('file-id-123'); }); - it('falls through to legacy filename= when filename*= is malformed', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''bad%ZZ; filename=\"legacy.txt\""), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('legacy.txt'); - }); - - it('prefers filename*= over a legacy filename= present in the same header', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename=\"legacy.txt\"; filename*=UTF-8''nested%2Ffile.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('nested/file.txt'); - }); - - it('keeps the requested name when an old file server advertises the opaque object basename', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''storage-id.xlsx"), - { name: 'Sample_-_Superstore.xlsx', id: 'storage-id' }, - ), - ).toBe('Sample_-_Superstore.xlsx'); - }); - - it('keeps the requested name for a legacy opaque filename header', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="storage-id.csv"'), - { name: 'original.csv', id: 'storage-id' }, - ), - ).toBe('original.csv'); - }); - - it('keeps an authoritative nested filename even when its basename matches the object id', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''exports%2Fstorage-id.csv"), - { name: 'original.csv', id: 'storage-id' }, - ), - ).toBe('exports/storage-id.csv'); + it('returns an empty path when both name and id are absent', () => { + expect(resolveInputDestination({ name: '' })).toBe(''); }); }); diff --git a/api/src/job.ts b/api/src/job.ts index d2504de9..b7b82148 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -173,52 +173,12 @@ export function ensureNodeModulesSymlink( } /** - * Extracts the on-disk filename from a Content-Disposition response header, - * falling back to the request-supplied `file.name` (or `file.id` if no name - * was provided). Pure; exported for unit testing. - * - * Matches RFC 5987 / 8187 `filename*=UTF-8''` first because - * the file server emits that form for UTF-8-safe transport of arbitrary - * names — including paths with `/` separators that the legacy `filename=` - * form would mangle. Falls back to the legacy quoted (`filename="..."`) or - * unquoted (`filename=...`) forms, each stopping at the closing quote or - * the first whitespace/semicolon so trailing params like - * `attachment; filename="foo.txt"; size=123` correctly yield `foo.txt`. + * Resolves the on-disk destination for a by-reference input. The request owns + * the sandbox path; object response metadata must not redirect the write. + * Pure; exported for unit testing. */ -export function resolveOriginalName(response: Response, file: TFile): string { - const fallback = file.name || (file.id ?? ''); - const header = response.headers.get('content-disposition'); - if (!header) return fallback; - - const preferRequestedName = (candidate: string): string => { - /* Older file servers advertised path.basename(objectName) when an - * S3-compatible backend omitted original-filename user metadata. That - * basename is ``, so it is a storage identifier rather - * than an authoritative destination. Preserve the caller's requested name - * during rolling upgrades instead of exposing the opaque id in /mnt/data. */ - const opaqueStem = path.basename(candidate, path.extname(candidate)); - const isFlatObjectBasename = candidate === path.basename(candidate); - return file.name && file.id && isFlatObjectBasename && opaqueStem === file.id - ? file.name - : candidate; - }; - - const star = header.match(/filename\*=(?:UTF-8'[^']*')?([^;]+)/i); - if (star) { - const raw = star[1].trim(); - try { - return preferRequestedName(decodeURIComponent(raw)); - } catch { - /* Malformed percent-encoding (e.g. `%ZZ`) — fall through to the legacy - * forms. The same header may emit both `filename*=` and a legacy - * `filename=` per RFC 5987 §4.3, so a corrupt extended form should - * not poison a valid fallback. */ - } - } - - const match = header.match(/filename="([^"]+)"/i) - ?? header.match(/filename=([^\s;]+)/i); - return match ? preferRequestedName(match[1]) : fallback; +export function resolveInputDestination(file: TFile): string { + return file.name || (file.id ?? ''); } /** @@ -959,12 +919,9 @@ export class Job { ); } requestedDestinations.set(file.name, file); - /* Inline destinations are final, so keep them reserved while reference - * downloads resolve their authoritative Content-Disposition names. - * A ref's requested name is only a fallback, not a real destination yet: - * reserving every ref here makes concurrent swaps/order-dependent - * renames falsely conflict before the owning response has resolved. */ - if (!file.id) this.inputDestinations.set(file.name, file); + /* The request owns every sandbox destination. Reserve it before parallel + * priming begins so object metadata cannot redirect a later write. */ + this.inputDestinations.set(file.name, file); } if (this.session) { @@ -1083,10 +1040,8 @@ export class Job { ): Promise { throwIfAborted(context.signal); if (this.session && file.id && (await this.reusePrimedInput(file, context))) { - /* Reuse has no response header to pass through downloadAndWriteFile, so - * its requested name becomes authoritative only after the on-disk copy - * has been verified. Reserve it before another concurrent ref can claim - * and overwrite that path. */ + /* Inherited markers are registered after prime's initial reservation + * pass, so reserve the verified requested path here as well. */ this.reserveInputDestination(file, file.name); return; } @@ -1345,10 +1300,10 @@ export class Job { throw new Error(`HTTP error: ${response.status}`); } - const originalName = resolveOriginalName(response, file); - validateFilePath(originalName, operation.submissionDir); - this.reserveInputDestination(file, originalName); - const finalPath = path.join(operation.submissionDir, originalName); + const destination = resolveInputDestination(file); + validateFilePath(destination, operation.submissionDir); + this.reserveInputDestination(file, destination); + const finalPath = path.join(operation.submissionDir, destination); const finalParent = path.dirname(finalPath); /* Persistent-session workspaces can hold a prior turn's symlink, so build * ancestors no-follow; a fresh per-job workspace can use plain mkdir -p. */ @@ -1376,7 +1331,7 @@ export class Job { operation.signal, ); const readOnly = response.headers.get('x-read-only')?.toLowerCase() === 'true'; - this.inputFileHashes.set(originalName, { + this.inputFileHashes.set(destination, { originalId: file.id, originalSessionId: file.storage_session_id!, hash, @@ -1389,15 +1344,8 @@ export class Job { await applyReadOnlyInputPermissions(finalPath); } - /* Keep the in-memory TFile in sync with the on-disk name so that - * inputByName lookups in handleSessionFiles match walkDir's - * path.relative() output. Otherwise a Content-Disposition override - * would leave file.name pointing at the client-submitted name while - * the file lives under originalName on disk. */ - if (originalName !== file.name) file.name = originalName; - - this.log.info({ file: originalName, hash: hash.substring(0, 8) }, 'Downloaded file'); - return originalName; + this.log.info({ file: destination, hash: hash.substring(0, 8) }, 'Downloaded file'); + return destination; } catch (error: unknown) { if (response?.body && !response.bodyUsed) { await response.body.cancel().catch(() => {}); From 9d3936fa73e1b447f2e9a3a3961cdef8b411da18 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 08:39:20 -0400 Subject: [PATCH 07/75] fix: disambiguate legacy dotted object identities (#186) --- service/src/file-object-resolver.test.ts | 37 +++++++++++++++++++++++- service/src/file-object-resolver.ts | 15 ++++++++-- service/src/file-server.ts | 13 ++++++--- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts index 2342d87a..69ad3c2f 100644 --- a/service/src/file-object-resolver.test.ts +++ b/service/src/file-object-resolver.test.ts @@ -1,5 +1,12 @@ import { describe, expect, test } from 'bun:test'; -import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; +import { + canonicalObjectId, + canonicalObjectKey, + FileObjectResolver, + legacyObjectId, + mapObjectDetails, + storageKeyForUpload, +} from './file-object-resolver'; import type { BucketItemStat } from 'minio'; describe('storage object resolution', () => { @@ -22,6 +29,34 @@ describe('storage object resolution', () => { expect(await resolver.listFresh('s', 'report')).toEqual([]); }); + test('legacy extension keys map to exactly one dotted or undotted identity', async () => { + const canonicalDotted = canonicalObjectKey('s', 'report.csv'); + const objects = new Set([ + 's/report.csv', + 's/report.csv.txt', + canonicalDotted, + ]); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* (prefix) { + for (const name of objects) if (name.startsWith(prefix)) yield { name }; + }, + stat: async () => ({ metaData: {} } as BucketItemStat), + }); + + expect(legacyObjectId('s/report.csv', 's')).toBe('report'); + expect(legacyObjectId('s/report.csv.txt', 's')).toBe('report.csv'); + expect(legacyObjectId('s/report', 's')).toBe('report'); + expect(legacyObjectId('other/report.csv', 's')).toBeUndefined(); + await expect(resolver.remember('s', 'report.csv', 's/report.csv')) + .rejects.toThrow('Object key does not match storage identity'); + expect(await resolver.listFresh('s', 'report')).toEqual(['s/report.csv']); + expect(await resolver.listFresh('s', 'report.csv')).toEqual([ + canonicalDotted, + 's/report.csv.txt', + ]); + }); + test('indexes exact identities while reading fresh version metadata on every request', async () => { const index = new Map(); let lists = 0; diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts index bc4b4ce5..1665941a 100644 --- a/service/src/file-object-resolver.ts +++ b/service/src/file-object-resolver.ts @@ -19,6 +19,18 @@ export function canonicalObjectId(key: string): string | undefined { } } +/** Legacy objects were stored as `/`. + * Derive exactly one identity by removing that final extension when present. + * In particular, `session/report.csv` belongs to `report`, while a legacy + * `report.csv` identity would be stored as e.g. `session/report.csv.txt`. + * Dotted identities without a filename extension use canonical storage. */ +export function legacyObjectId(key: string, session: string): string | undefined { + if (path.posix.dirname(key) !== session) return undefined; + const basename = path.posix.basename(key); + const extension = path.posix.extname(basename); + return extension === '' ? basename : basename.slice(0, -extension.length); +} + export interface ObjectResolverDependencies { bucket: string; list(prefix: string): AsyncIterable<{ name?: string }>; @@ -58,8 +70,7 @@ export class FileObjectResolver { private matches(key: string, session: string, id: string): boolean { if (key === canonicalObjectKey(session, id)) return true; - return path.posix.dirname(key) === session && - (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); + return legacyObjectId(key, session) === id; } async remember(session: string, id: string, key: string, replace = true): Promise { diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 3dfa0a10..22302042 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,6 +1,12 @@ import b from 'busboy'; import { randomUUID } from 'node:crypto'; -import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; +import { + canonicalObjectId, + FileObjectResolver, + legacyObjectId, + mapObjectDetails, + storageKeyForUpload, +} from './file-object-resolver'; import { sendFileDownload } from './file-download'; import path from 'path'; import IORedis from 'ioredis'; @@ -573,9 +579,8 @@ function parseObjectName(objectName: string | undefined): { session_id: string; const parts = objectName.split('/'); if (parts.length < 2) return null; const session_id = parts[0]; - const fileNameWithExt = parts[1]; - // Remove extension to get file_id - const file_id = fileNameWithExt.replace(/\.[^.]+$/, ''); + const file_id = legacyObjectId(objectName, session_id); + if (file_id == null) return null; return { session_id, file_id }; } From 3946ffb6e8664c152a589f37dd8c0feb4dccb6f8 Mon Sep 17 00:00:00 2001 From: "Ignaz \"Ian\" Kraft" Date: Sat, 12 Sep 2026 20:26:48 +0200 Subject: [PATCH 08/75] fix: helm egress deployment getting stuck on install (#176) * fix: helm egress deployment getting stuck on install * only wait for redis if the ledger is required * Update helm/codeapi/templates/egress-gateway-deployment.yaml Co-authored-by: Danny Avila --------- Co-authored-by: Danny Avila --- helm/codeapi/templates/egress-gateway-deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helm/codeapi/templates/egress-gateway-deployment.yaml b/helm/codeapi/templates/egress-gateway-deployment.yaml index 49c865f0..941844ce 100644 --- a/helm/codeapi/templates/egress-gateway-deployment.yaml +++ b/helm/codeapi/templates/egress-gateway-deployment.yaml @@ -24,6 +24,12 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- if or .Values.egressGrant.ledgerRequired .Values.hardenedSandboxMode }} + initContainers: + - name: wait-for-redis + image: busybox:1.36.1 + command: ['sh', '-c', 'until nc -z {{ include "codeapi.redis.host" . }} {{ include "codeapi.redis.port" . }}; do echo waiting for redis; sleep 2; done'] + {{- end }} containers: - name: egress-gateway image: "{{ .Values.egressGateway.image.repository }}:{{ .Values.egressGateway.image.tag }}" From 8a90f6d1ba01c97ed5c4e01f835a783958e581b7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 16:17:36 -0400 Subject: [PATCH 09/75] feat: Add Trusted VM Command Policy (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🛰️ feat: Add trusted VM command policy * docs: clarify trusted VM socket boundary --- docs/adr/001-stateful-code-environments.md | 13 +++- packages/code/README.md | 35 +++++++++ packages/code/package.json | 4 + packages/code/src/cli.test.ts | 40 ++++++++++ packages/code/src/cli.ts | 24 +++++- packages/code/src/index.ts | 1 + packages/code/src/native-policy.test.ts | 63 ++++++++++++++++ packages/code/src/native-policy.ts | 86 ++++++++++++++++++++++ packages/code/src/native-process.test.ts | 30 ++++++++ packages/code/src/native-process.ts | 2 + packages/code/src/native-sandbox.test.ts | 50 ++++++++++++- packages/code/src/native-sandbox.ts | 29 ++++++-- 12 files changed, 364 insertions(+), 13 deletions(-) create mode 100644 packages/code/src/native-policy.test.ts create mode 100644 packages/code/src/native-policy.ts diff --git a/docs/adr/001-stateful-code-environments.md b/docs/adr/001-stateful-code-environments.md index 8b9830df..cfa44bf4 100644 --- a/docs/adr/001-stateful-code-environments.md +++ b/docs/adr/001-stateful-code-environments.md @@ -56,8 +56,14 @@ worker replacement; the UI and operator documentation must not imply otherwise. revocation. - Pairing codes and credentials are stored by digest where lookup permits. - One configured worker has at most one active fenced assignment. -- Sandbox isolation and default-deny egress remain mandatory; pairing secures - the transport identity but does not make the host a sandbox. +- Sandbox isolation and default-deny egress remain the mandatory default; + pairing secures the transport identity but does not make the host a sandbox. + An operator may explicitly delegate network and local-socket restrictions to + an approved outer VM boundary through a named, digested worker policy. That + delegation retains direct workspace filesystem rules, cancellation, and + resource limits. The operator is responsible for preventing permitted host + services (for example, a privileged container socket) from bypassing those + rules and exposing worker identity or credential material. - A compromised worker can lie about advertised capabilities. Capability labels and policy digests are audit signals until enforcement is coupled to an attested sandbox or trusted host policy. @@ -65,7 +71,8 @@ worker replacement; the UI and operator documentation must not imply otherwise. ## Consequences - `@librechat/code` owns the provider-neutral protocol, identity handling, and - worker CLI; Code API owns enrollment, scheduling, and execution policy. + worker CLI, including machine-local execution-policy presets; Code API owns + enrollment, scheduling, and execution policy. - LibreChat owns environment persistence, ownership, RBAC, and user experience. - The Agents SDK keeps only its adapter until a second concrete consumer proves which coding-tool abstractions are genuinely provider neutral. diff --git a/packages/code/README.md b/packages/code/README.md index 819d9f5c..e46d6fd6 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -211,6 +211,41 @@ LIBRECHAT_CODE_COMMAND_SANDBOX=native-srt librechat-code run \ --worker-dir /path/to/project --allow-workspace-commands ``` +### Trusted VM command policy + +The native SRT backend can be made intentionally permissive when the selected +machine already supplies an administrator-approved outer security boundary. +The `trusted-vm` preset keeps SRT's direct filesystem rules, credential +masking, private scratch storage, cancellation, time limits, and output limits, +while allowing unmatched outbound destinations, local port binding, and Unix +sockets: + +```bash +librechat-code run \ + --worker-dir /home/ubuntu/src \ + --allow-workspace-writes \ + --allow-workspace-commands \ + --command-policy-preset trusted-vm +``` + +`LIBRECHAT_CODE_COMMAND_POLICY_PRESET=trusted-vm` is the environment equivalent. +The default is `restricted`, which preserves the default-deny network policy. +The preset configures `native-srt`; it is not an unsandboxed host-shell +backend. It is rejected unless native workspace commands are enabled. Its +normalized effective controls are included in the worker policy digest, and +the worker advertises `anthropic-srt:trusted-vm` unless an operator supplied a +custom sandbox profile label. + +Treat this preset as delegation to the machine's outer security controls. Any +outbound destination can receive workspace data, local listeners can accept +connections reachable under host policy, and Unix socket access may expose +powerful host services such as a container daemon. A socket that grants host +privilege can bypass SRT's filesystem rules and reach worker or GitHub identity +material; the outer VM boundary must prevent that path or explicitly accept +that trust. Register only the intended source root. Worker identity, +mutation-quarantine state, and configured GitHub App key files must remain +outside it. + ## Docker runtime supervisor (optional hardened adapter) `DockerRuntimeSupervisor` is the first self-contained local OCI adapter. It diff --git a/packages/code/package.json b/packages/code/package.json index a819af9d..452a8be7 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -35,6 +35,10 @@ "types": "./dist/native-sandbox.d.ts", "import": "./dist/native-sandbox.js" }, + "./native-policy": { + "types": "./dist/native-policy.d.ts", + "import": "./dist/native-policy.js" + }, "./github": { "types": "./dist/github.d.ts", "import": "./dist/github.js" diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f978fce5..18456072 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -94,6 +94,46 @@ test('CLI rejects an unknown command sandbox before entering the run loop', () = ); }); +test('CLI rejects an unknown native SRT command policy preset', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'host-shell', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /must be restricted or trusted-vm/); +}); + +test('CLI refuses a permissive policy when native commands are unavailable', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires native-srt workspace commands/); +}); + test('CLI rejects incomplete GitHub App authentication before worker registration', () => { const result = spawnSync( process.execPath, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 27a8d2d9..fdd28e9f 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -29,6 +29,10 @@ import { import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { + resolveNativeSrtCommandPolicy, + serializeNativeSrtCommandPolicy, +} from './native-policy.js'; import { workspaceMutationGuard } from './workspace-guards.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; import type { LocalWorkspaceConfig } from './workspace.js'; @@ -404,6 +408,19 @@ async function run( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + const commandPolicy = resolveNativeSrtCommandPolicy( + option(args, '--command-policy-preset') ?? + process.env.LIBRECHAT_CODE_COMMAND_POLICY_PRESET?.trim().toLowerCase() ?? + 'restricted', + ); + if ( + commandPolicy.preset !== 'restricted' && + (!allowWorkspaceCommands || commandSandboxMode !== 'native-srt') + ) { + throw new Error( + 'A permissive command policy preset requires native-srt workspace commands', + ); + } const github = runtimeSessionId == null ? githubCredentials() @@ -756,6 +773,7 @@ async function run( }); const nativeOptions: NativeProcessSandboxOptions = { workspaceRoot: canonicalWorkerDirectory!, + commandPolicy, protectedPaths: [ identityPath, ...rootQuarantinePaths.values(), @@ -820,7 +838,9 @@ async function run( sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? (allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? 'anthropic-srt' + ? commandPolicy.preset === 'restricted' + ? 'anthropic-srt' + : `anthropic-srt:${commandPolicy.preset}` : runtimeMode.startsWith('docker') ? 'oci-docker' : 'nsjail'), @@ -829,7 +849,7 @@ async function run( .update(policy) .update( allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? `\0native-srt\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` + ? `\0native-srt\0${serializeNativeSrtCommandPolicy(commandPolicy)}\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` : '', ) .digest('hex'), diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 712e7487..5363f0c0 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -5,6 +5,7 @@ export * from './storage.js'; export * from './runtime.js'; export * from './workspace.js'; export * from './workspace-runtime.js'; +export * from './native-policy.js'; export * from './native-sandbox.js'; export * from './native-process.js'; export * from './github.js'; diff --git a/packages/code/src/native-policy.test.ts b/packages/code/src/native-policy.test.ts new file mode 100644 index 00000000..4e9f4691 --- /dev/null +++ b/packages/code/src/native-policy.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + normalizeNativeSrtCommandPolicy, + resolveNativeSrtCommandPolicy, + serializeNativeSrtCommandPolicy, +} from './native-policy.js'; + +test('restricted remains the default native SRT command policy', () => { + assert.deepEqual(resolveNativeSrtCommandPolicy(), { + version: 1, + preset: 'restricted', + network: { + outbound: 'allowlist', + allowLocalBinding: false, + allowAllUnixSockets: false, + }, + }); +}); + +test('trusted-vm resolves to explicit permissive network controls', () => { + assert.deepEqual(resolveNativeSrtCommandPolicy('trusted-vm'), { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }); +}); + +test('unknown and forged native policies fail closed', () => { + assert.throws( + () => resolveNativeSrtCommandPolicy('host-shell'), + /must be restricted or trusted-vm/, + ); + assert.throws( + () => + normalizeNativeSrtCommandPolicy({ + ...resolveNativeSrtCommandPolicy('restricted'), + network: { + ...resolveNativeSrtCommandPolicy('restricted').network, + allowAllUnixSockets: true, + }, + }), + /does not match its preset/, + ); +}); + +test('serialized policy is stable and includes effective controls', () => { + const first = serializeNativeSrtCommandPolicy( + resolveNativeSrtCommandPolicy('trusted-vm'), + ); + const second = serializeNativeSrtCommandPolicy( + resolveNativeSrtCommandPolicy('trusted-vm'), + ); + assert.equal(first, second); + assert.match(first, /"outbound":"unrestricted"/); + assert.match(first, /"allowLocalBinding":true/); + assert.match(first, /"allowAllUnixSockets":true/); +}); diff --git a/packages/code/src/native-policy.ts b/packages/code/src/native-policy.ts new file mode 100644 index 00000000..5d289bfb --- /dev/null +++ b/packages/code/src/native-policy.ts @@ -0,0 +1,86 @@ +export const NATIVE_SRT_COMMAND_POLICY_PRESETS = [ + 'restricted', + 'trusted-vm', +] as const; + +export type NativeSrtCommandPolicyPreset = + typeof NATIVE_SRT_COMMAND_POLICY_PRESETS[number]; + +export interface NativeSrtCommandPolicy { + version: 1; + preset: NativeSrtCommandPolicyPreset; + network: { + outbound: 'allowlist' | 'unrestricted'; + allowLocalBinding: boolean; + allowAllUnixSockets: boolean; + }; +} + +const PRESETS: Record = { + restricted: { + version: 1, + preset: 'restricted', + network: { + outbound: 'allowlist', + allowLocalBinding: false, + allowAllUnixSockets: false, + }, + }, + 'trusted-vm': { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, +}; + +function isPreset(value: unknown): value is NativeSrtCommandPolicyPreset { + return ( + typeof value === 'string' && + NATIVE_SRT_COMMAND_POLICY_PRESETS.some((preset) => preset === value) + ); +} + +/** Resolve a named convenience preset into the explicit policy SRT enforces. */ +export function resolveNativeSrtCommandPolicy( + preset: unknown = 'restricted', +): NativeSrtCommandPolicy { + if (!isPreset(preset)) { + throw new Error( + 'Native SRT command policy preset must be restricted or trusted-vm', + ); + } + const policy = PRESETS[preset]; + return { ...policy, network: { ...policy.network } }; +} + +/** Validate a programmatic policy and return canonical preset-owned values. */ +export function normalizeNativeSrtCommandPolicy( + policy?: NativeSrtCommandPolicy, +): NativeSrtCommandPolicy { + const normalized = resolveNativeSrtCommandPolicy( + policy?.preset ?? 'restricted', + ); + if ( + policy !== undefined && + (policy.version !== normalized.version || + policy.network?.outbound !== normalized.network.outbound || + policy.network?.allowLocalBinding !== + normalized.network.allowLocalBinding || + policy.network?.allowAllUnixSockets !== + normalized.network.allowAllUnixSockets) + ) { + throw new Error('Native SRT command policy does not match its preset'); + } + return normalized; +} + +/** Stable policy material used in the bridge capability digest. */ +export function serializeNativeSrtCommandPolicy( + policy: NativeSrtCommandPolicy, +): string { + return JSON.stringify(normalizeNativeSrtCommandPolicy(policy)); +} diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index d77ec559..8dcfd4c1 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -108,6 +108,36 @@ test('executor bootstrap excludes bridge credentials and Node injection variable await sandbox.close(); }); +test('executor forwards the resolved command policy without worker credentials', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + commandPolicy: { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, + }, + fake.fork, + ); + await sandbox.prepare(); + assert.deepEqual(fake.messages[0].options.commandPolicy, { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }); + await sandbox.close(); +}); + test('executor hands credentials over IPC only for the current command', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index fe4e1aa8..96d89355 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -182,6 +182,7 @@ export class NativeProcessWorkspaceCommandSandbox child.on('disconnect', lost); const { workspaceRoot, + commandPolicy, protectedPaths, allowedDomains, homeDirectory, @@ -192,6 +193,7 @@ export class NativeProcessWorkspaceCommandSandbox { options: { workspaceRoot, + commandPolicy, protectedPaths, allowedDomains, homeDirectory, diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index e2dc4bda..cf790965 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -19,7 +19,10 @@ import { join } from 'node:path'; import { PassThrough } from 'node:stream'; import test from 'node:test'; -import type { SandboxRuntimeConfig } from '@anthropic-ai/sandbox-runtime'; +import type { + SandboxAskCallback, + SandboxRuntimeConfig, +} from '@anthropic-ai/sandbox-runtime'; import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; @@ -46,6 +49,7 @@ function fakeManager( } = {}, ) { let config: SandboxRuntimeConfig | undefined; + let askCallback: SandboxAskCallback | undefined; let reset = false; let credentialSeenDuringWrap: string | undefined; let gitLfsRequiredSeenDuringWrap: string | undefined; @@ -55,8 +59,12 @@ function fakeManager( async checkDependenciesAsync() { return { warnings: [], errors: options.dependencyErrors ?? [] }; }, - async initialize(value: SandboxRuntimeConfig) { + async initialize( + value: SandboxRuntimeConfig, + callback?: SandboxAskCallback, + ) { config = value; + askCallback = callback; if (options.initializeError) throw options.initializeError; }, async wrapWithSandboxArgv(command: string) { @@ -107,6 +115,9 @@ function fakeManager( get config() { return config; }, + get askCallback() { + return askCallback; + }, get reset() { return reset; }, @@ -276,6 +287,8 @@ test('initializes SRT with a default-deny network and scrubbed worker credential assert.deepEqual(fake.config?.network.allowedDomains, []); assert.equal(fake.config?.network.strictAllowlist, true); assert.equal(fake.config?.network.allowAllUnixSockets, false); + assert.equal(fake.config?.network.allowLocalBinding, false); + assert.equal(fake.askCallback, undefined); assert.deepEqual(fake.config?.filesystem.allowRead, [ canonicalRoot, scratchDirectory, @@ -305,6 +318,39 @@ test('initializes SRT with a default-deny network and scrubbed worker credential await assert.rejects(access(scratchDirectory!)); }); +test('trusted-vm permits unmatched egress and local development sockets', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + + await sandbox.prepare(); + + assert.equal(fake.config?.network.strictAllowlist, false); + assert.equal(fake.config?.network.allowLocalBinding, true); + assert.equal(fake.config?.network.allowAllUnixSockets, true); + assert.equal( + await fake.askCallback?.({ host: 'packages.example', port: 443 }), + true, + ); + assert.deepEqual(fake.config?.filesystem.allowWrite.slice(0, 1), [ + await realpath(root), + ]); +}); + test('provides an isolated scratch directory to commands and restores the host environment', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 139a10d6..171ea663 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -41,7 +41,12 @@ import type { ChildProcessWithoutNullStreams, SpawnOptionsWithoutStdio, } from 'node:child_process'; -import type { SandboxRuntimeConfig } from '@anthropic-ai/sandbox-runtime'; +import type { + SandboxAskCallback, + SandboxRuntimeConfig, +} from '@anthropic-ai/sandbox-runtime'; +import { normalizeNativeSrtCommandPolicy } from './native-policy.js'; +import type { NativeSrtCommandPolicy } from './native-policy.js'; import type { WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, @@ -124,7 +129,10 @@ const HOST_TEMPORARY_ROOT = tmpdir(); interface NativeSandboxManager { isSupportedPlatform(): boolean; checkDependenciesAsync(): Promise<{ warnings: string[]; errors: string[] }>; - initialize(config: SandboxRuntimeConfig): Promise; + initialize( + config: SandboxRuntimeConfig, + sandboxAskCallback?: SandboxAskCallback, + ): Promise; wrapWithSandboxArgv( command: string, binShell?: string, @@ -153,6 +161,7 @@ type SpawnCommand = ( export interface NativeSrtWorkspaceCommandSandboxOptions { workspaceRoot: string; + commandPolicy?: NativeSrtCommandPolicy; /** Trusted worker files that must never become workspace-readable or writable. */ protectedPaths?: string[]; allowedDomains?: string[]; @@ -369,13 +378,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'REGISTRATION_INVALID', ); } + const commandPolicy = normalizeNativeSrtCommandPolicy( + this.options.commandPolicy, + ); + const unrestrictedNetwork = + commandPolicy.network.outbound === 'unrestricted'; const config: SandboxRuntimeConfig = { network: { allowedDomains: [...(this.options.allowedDomains ?? [])], deniedDomains: [], - strictAllowlist: true, - allowAllUnixSockets: false, - allowLocalBinding: false, + strictAllowlist: !unrestrictedNetwork, + allowAllUnixSockets: commandPolicy.network.allowAllUnixSockets, + allowLocalBinding: commandPolicy.network.allowLocalBinding, ...(this.options.maskedEnvironment ? { tlsTerminate: {} } : {}), }, filesystem: { @@ -439,7 +453,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox enableWeakerNetworkIsolation: false, git: { safeDirectories: [root] }, }; - await this.manager.initialize(config); + await this.manager.initialize( + config, + unrestrictedNetwork ? async () => true : undefined, + ); this.canonicalRoot = root; } From 31def177aa3ca2b4651bc413f96cf63d6522d707 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 20:31:55 -0400 Subject: [PATCH 10/75] fix: Retry Clean Cancelled BYOM Settlements Through Stop Grace (#188) A clean atomic workspace mutation rejection was settled with retries cut off at the original execution deadline. When Stop arrived near that deadline, process-tree termination finished after it, so the first settlement attempt was aborted immediately while Code API was still draining the cancellation. The client received ASSIGNMENT_EXPIRED and the durable mutation guard stayed armed. Route clean workspace mutation rejections through the known-clean rejection recovery path: a transient-retrying heartbeat and settlement retries through the rejection acknowledgement grace, floored at the bridge cancellation settlement grace. Worker shutdown still fails closed. Share the grace constant from the protocol module so the bridge and worker stay aligned. Closes #173 --- packages/code/src/protocol.ts | 2 + packages/code/src/worker.ts | 26 ++- packages/code/src/workspace-worker.test.ts | 202 +++++++++++++++++++++ service/src/bridge/store.ts | 4 +- 4 files changed, 227 insertions(+), 7 deletions(-) diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 44cfdfd0..c1dad949 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -20,6 +20,8 @@ export const BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS = 5 * 60_000; export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; +/** How long Code API drains a clean rejection after Stop cancels a workspace mutation. */ +export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index fde84768..19f219da 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -1,6 +1,7 @@ import { randomBytes } from 'node:crypto'; import { + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, BRIDGE_PROTOCOL_VERSION, BridgeProtocolError, bridgeWorkerPath, @@ -1619,7 +1620,13 @@ export class BridgeWorker { assignment.runtimeSessionId != null && settlement.status === 'rejected' && (!sandboxStarted || sandboxRejectedExecution); - if (knownCleanStatefulRejection) { + // An armed mutation reaches settlement as rejected only after an atomic + // failure that does not require quarantine. Code API accepts that + // rejection after expiry and drains it for its own grace after Stop, so a + // Stop near the deadline must not cut off retries at the deadline. + const knownCleanWorkspaceRejection = + workspaceMutationArmed && settlement.status === 'rejected'; + if (knownCleanStatefulRejection || knownCleanWorkspaceRejection) { heartbeatController.abort(); await heartbeat; const recoveryHeartbeatController = new AbortController(); @@ -1627,15 +1634,24 @@ export class BridgeWorker { recoveryHeartbeatController.signal, true, ).catch(() => undefined); + const rejectionAckGraceMs = Math.max( + 0, + this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, + ); try { await this.settleWithRetry( assignment, settlement, localDeadlineAtMs + - Math.max( - 0, - this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, - ), + (knownCleanStatefulRejection + ? rejectionAckGraceMs + : Math.max( + rejectionAckGraceMs, + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, + )), + // Stateful rejections outlive shutdown; workspace guards still + // fail closed when the worker itself stops. + knownCleanStatefulRejection ? undefined : signal, ); } finally { recoveryHeartbeatController.abort(); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 14bf2d61..8f9fcd45 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1480,6 +1480,208 @@ test('worker clears quarantine after a command cancellation confirms process ter assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); }); +test('worker retries a clean Stop rejection near its deadline through the cancellation grace', async () => { + const lifecycle: string[] = []; + const settlements: Array> = []; + const remainingMs = 100; + const startedAt = Date.now(); + const baseCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', operations: ['read_file' as const] }], + }; + const workspaceTools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: baseCapabilities, + mutationFailuresAreAtomic: true, + async execute() { throw new Error('base executor must not run'); }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute(_request, signal) { + lifecycle.push('execute'); + await new Promise((resolve) => { + if (signal?.aborted) return resolve(); + signal?.addEventListener('abort', () => resolve(), { once: true }); + }); + lifecycle.push('stop'); + // Process-group termination is confirmed after the original deadline. + await new Promise((resolve) => setTimeout(resolve, remainingMs)); + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceTools.capabilities, + }, + workspaceTools, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + cancellationPollIntervalMs: 5, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/cancellation')) { + return Response.json({ + protocolVersion: 1, + cancelled: Date.now() >= startedAt + remainingMs / 2, + }); + } + if (!String(input).endsWith('/settle')) { + return Response.json({ protocolVersion: 1, accepted: true }); + } + lifecycle.push('settle'); + settlements.push({ + ...(JSON.parse(String(init?.body)) as Record), + attemptedAt: Date.now(), + }); + // Settlement delivery takes a real transport turn and honors its deadline. + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, 10); + init?.signal?.addEventListener( + 'abort', + () => { + clearTimeout(timer); + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }); + if (settlements.length === 1) { + return Response.json( + { error: 'Bridge settlement temporarily unavailable' }, + { status: 503 }, + ); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-command-stopped-near-deadline', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(startedAt + remainingMs).toISOString(), + remainingMs, + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'sleep 30; touch delayed.txt', + }, + }); + + assert.deepEqual(lifecycle, [ + 'arm', + 'execute', + 'stop', + 'settle', + 'settle', + 'clear', + ]); + assert.ok(Number(settlements[0]?.attemptedAt) > startedAt + remainingMs); + assert.equal(settlements[1]?.status, 'rejected'); + assert.equal(settlements[1]?.errorCode, 'EXECUTION_ABORTED'); +}); + +test('worker keeps quarantine armed when shutdown interrupts a clean command rejection', async () => { + const lifecycle: string[] = []; + const controller = new AbortController(); + const baseCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', operations: ['read_file' as const] }], + }; + const workspaceTools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: baseCapabilities, + mutationFailuresAreAtomic: true, + async execute() { throw new Error('base executor must not run'); }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute() { + lifecycle.push('execute'); + controller.abort(new Error('shutdown')); + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceTools.capabilities, + }, + workspaceTools, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await assert.rejects( + worker.executeAndSettle( + { + protocolVersion: 1, + assignmentId: 'assignment-command-shutdown-cleanly', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'sleep 30', + }, + }, + controller.signal, + ), + /shutdown/, + ); + assert.deepEqual(lifecycle, ['arm', 'execute']); +}); + test('worker retains quarantine when an atomic executor cannot confirm durability', async () => { const lifecycle: string[] = []; const workspaceCapabilities = { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 32205571..5e4249a9 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -11,6 +11,7 @@ import type { } from '../../../packages/code/src/protocol'; import { + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, @@ -23,7 +24,6 @@ import { BridgeWorkspaceSlots } from './slots'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; -const CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; const DEFAULT_WORKER_TTL_SECONDS = 60; const DEFAULT_REDIS_COMMAND_TIMEOUT_MS = 1_000; @@ -1924,7 +1924,7 @@ export class RedisBridgeStore { // Give Stop its own grace so a near-timeout cancellation is not // misclassified as an ambiguous timeout. const cancellationDeadlineAtMs = - Date.now() + CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS; + Date.now() + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS; let cancellationPollMs = POLL_INTERVAL_MS; while (Date.now() < cancellationDeadlineAtMs) { const raw = await boundedCommand( From 118eb9b3ad704e752acc6ffa4b50fb9f1b3bb0ee Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 21:15:39 -0400 Subject: [PATCH 11/75] fix: Release Unassigned Workspace Slots When Dispatch Cleanup Fails (#189) Closes #170 --- service/src/bridge/concurrent-store.test.ts | 40 +++++++++++++++++++++ service/src/bridge/store.ts | 40 +++++++++++++++------ 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index fe53c44f..736b7216 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -384,6 +384,46 @@ test('queued cancellation never leases and does not block another root', async ( ).toBeUndefined(); }); +test('unassigned slot releases even when dispatch cleanup fails', async () => { + await register(); + const originalIncr = redis.incr.bind(redis); + const originalSet = redis.set.bind(redis); + const set = originalSet as (...args: unknown[]) => unknown; + redis.incr = ((key: string) => + key.endsWith(':generation') + ? Promise.reject(new Error('injected generation outage')) + : originalIncr(key)) as typeof redis.incr; + redis.set = ((key: string, ...args: unknown[]) => + key.endsWith(':cancelled') + ? Promise.reject(new Error('injected cancellation outage')) + : set(key, ...args)) as typeof redis.set; + try { + // The reservation succeeds, then dispatch fails before storing an assignment. + await expect(dispatch('a')).rejects.toThrow('injected cancellation outage'); + } finally { + redis.incr = originalIncr; + redis.set = originalSet; + } + expect( + await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`), + ).toBe(0); + expect( + await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`), + ).toBeNull(); + const next = dispatch('a'); + const assignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + expect(assignment.request).toMatchObject({ workspaceId: 'a' }); + await settle(assignment); + await expect(next).resolves.toMatchObject({ status: 'rejected' }); +}); + test('late quarantine releases its slot after caller cancellation and retains only its root fence', async () => { await register(); const controller = new AbortController(); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 5e4249a9..b09d3792 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -1060,20 +1060,15 @@ export class RedisBridgeStore { // already committed result rather than turning cleanup availability // into a client-visible failure that could prompt duplicate work. } + } else if (workspaceSlots != null && assignment == null) { + await this.cleanupUnassignedSlot( + args.workerId, + lockIncarnationId, + assignmentId, + ); } else { await this.cleanupDispatch(args.workerId, assignmentId, assignment); } - if (workspaceSlots != null && assignment == null) { - await boundedCommand( - workspaceSlots.release( - args.workerId, - lockIncarnationId, - assignmentId, - ), - this.redisCommandTimeoutMs, - 'Bridge unassigned slot cleanup', - ); - } } } @@ -2138,6 +2133,29 @@ export class RedisBridgeStore { ]); } + private async cleanupUnassignedSlot( + workerId: string, + incarnationId: string, + assignmentId: string, + ): Promise { + // No stored assignment owns this reservation, so a cancellation outage + // must not leave the slot and its root busy until TTL expiry. + const [cleanup, release] = await Promise.allSettled([ + this.cleanupDispatch(workerId, assignmentId, undefined), + boundedCommand( + new BridgeWorkspaceSlots(this.redis).release( + workerId, + incarnationId, + assignmentId, + ), + this.redisCommandTimeoutMs, + 'Bridge unassigned slot cleanup', + ), + ]); + if (cleanup.status === 'rejected') throw cleanup.reason; + if (release.status === 'rejected') throw release.reason; + } + private async commitPendingWorkspace( assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, From 8764d019ecb4a503d6be34d8b8234f031b1f77e5 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 13 Sep 2026 04:05:29 -0400 Subject: [PATCH 12/75] fix: Exit Cleanly When Native Executor Shuts Down Concurrently (#191) * fix: Exit Cleanly When Native Executor Shuts Down Concurrently A native BYOM worker under systemd KillMode=control-group receives SIGTERM at the same time as its forked SRT executor. The child ignores IPC once it is shutting down, so the parent's close handshake is left pending until the child exits, which rejects it with 'Native executor is unavailable'. That rejection escaped the CLI finally block and turned an idle administrative stop into exit status 1. Treat the close handshake as best-effort: the executor is terminated in finally regardless, and the active command has already drained, so a lost or stalled reply carries no mutation risk. Also make the child report exit status 0 when its own SRT teardown succeeded. Closes #190 * fix: Surface Explicit Executor Cleanup Failures During Close Only a lost, refused, or stalled close handshake is benign at shutdown. A negative close reply from the executor is a real cleanup failure and still rejects so pool shutdown can aggregate it. --- .../code/src/native-process-child.test.ts | 2 +- packages/code/src/native-process-child.ts | 5 +- packages/code/src/native-process.test.ts | 70 +++++++++++++++++++ packages/code/src/native-process.ts | 22 ++++-- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/packages/code/src/native-process-child.test.ts b/packages/code/src/native-process-child.test.ts index 1d4866ae..8aca3652 100644 --- a/packages/code/src/native-process-child.test.ts +++ b/packages/code/src/native-process-child.test.ts @@ -33,7 +33,7 @@ for (const signal of ['SIGINT', 'SIGHUP', 'SIGTERM'] as const) { // installation finished without requiring platform SRT dependencies. child.once('message', () => child.kill(signal)); child.send({ id: 'startup-probe', type: 'probe' }); - assert.deepEqual(await exited, { code: 1, signal: null }); + assert.deepEqual(await exited, { code: 0, signal: null }); }, ); } diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index b5221731..05ffedfe 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -25,7 +25,10 @@ const shutdown = () => { if (shuttingDown) return; shuttingDown = true; active?.controller.abort(); - void (sandbox?.close() ?? Promise.resolve()).finally(() => process.exit(1)); + void (sandbox?.close() ?? Promise.resolve()).then( + () => process.exit(0), + () => process.exit(1), + ); setTimeout(() => process.exit(1), 5000); }; process.on('disconnect', shutdown); diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 8dcfd4c1..800dcf11 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -301,6 +301,76 @@ test('executor close drains an active command before closing IPC', async () => { await assert.rejects(sandbox.execute(request), /unavailable/); }); +test('executor close resolves when the child exits during the close handshake', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { + send(message: Record, callback: (error: null) => void) { + fake.messages.push(message); + callback(null); + queueMicrotask(() => { + Object.assign(fake.child, { connected: false }); + fake.child.emit('exit', 1, null); + fake.child.emit('disconnect'); + }); + return true; + }, + }); + await sandbox.close(); + assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + +test('executor close still reports a cleanup failure the child replies with', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { + send(message: Record, callback: (error: null) => void) { + fake.messages.push(message); + callback(null); + queueMicrotask(() => + fake.child.emit('message', { + id: message.id, + ok: false, + code: 'COMMAND_UNAVAILABLE', + errorMessage: 'scratch cleanup failed', + mutation: false, + requiresQuarantine: false, + }), + ); + return true; + }, + }); + await assert.rejects(sandbox.close(), /scratch cleanup failed/); + assert.equal(fake.killCalls, 1); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + +test('executor close skips the handshake once the child is already lost', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { connected: false }); + fake.child.emit('exit', 1, null); + await sandbox.close(); + assert.equal( + fake.messages.some((m) => m.type === 'close'), + false, + ); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + test('executor startup loss is not reported as an applied mutation', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 96d89355..e1a02498 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -73,6 +73,15 @@ export function nativeExecutorEnvironment( ); } +/** The executor process was lost, refused a send, or stalled past its + * deadline, as opposed to a failure the executor reported explicitly. */ +class NativeExecutorUnavailableError extends WorkspaceToolError { + constructor(mutation: boolean) { + super('Native executor is unavailable', 'COMMAND_UNAVAILABLE', mutation); + this.name = 'NativeExecutorUnavailableError'; + } +} + /** One persistent, process-isolated SRT manager per workspace. No automatic * restart/replay: losing IPC after execution starts is an ambiguous mutation. */ export class NativeProcessWorkspaceCommandSandbox @@ -109,11 +118,7 @@ export class NativeProcessWorkspaceCommandSandbox } private unavailable(mutation: boolean): WorkspaceToolError { - return new WorkspaceToolError( - 'Native executor is unavailable', - 'COMMAND_UNAVAILABLE', - mutation, - ); + return new NativeExecutorUnavailableError(mutation); } private async start(): Promise { @@ -340,12 +345,17 @@ export class NativeProcessWorkspaceCommandSandbox return this.closing; } + /** An executor that exits, disconnects, or stalls while closing is + * terminated in `finally` regardless, and the active command has already + * drained, so only a failure the executor reports explicitly is surfaced. */ private async stop(): Promise { await this.active?.catch(() => undefined); await this.ready?.catch(() => undefined); try { if (this.child?.connected && !this.failed) - await this.rpc('close', {}, 10_000, false); + await this.rpc('close', {}, 10_000, false).catch((error: unknown) => { + if (!(error instanceof NativeExecutorUnavailableError)) throw error; + }); } finally { this.failed = true; this.terminate(); From 25f3841c64422c289f30743520cb3ca88cc471fb Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 13 Sep 2026 23:21:49 -0400 Subject: [PATCH 13/75] fix: authenticate GitHub App Git operations (#192) --- packages/code/src/github.test.ts | 9 +++++++-- packages/code/src/github.ts | 9 ++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index bd62acfd..78d9751d 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -137,12 +137,17 @@ test('builds process-scoped Git HTTPS authorization without embedding credential const provider = new StaticGitHubCredentialProvider( 'github_pat_abcdefghijklmnopqrstuvwxyz', ); + const encodedCredential = Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'); assert.deepEqual( gitHubCredentialEnvironment(await provider.getCredential()), { - [GITHUB_CREDENTIAL_ENV_NAME]: 'github_pat_abcdefghijklmnopqrstuvwxyz', + [GITHUB_CREDENTIAL_ENV_NAME]: encodedCredential, }, ); + assert.ok(!encodedCredential.includes('github_pat_')); }); test('composes the masked credential with SRT Git configuration inside the sandbox', () => { @@ -155,7 +160,7 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); assert.match(wrapped, /\$\{LIBRECHAT_CODE_GITHUB_AUTHORIZATION\}/); assert.match(wrapped, /unset LIBRECHAT_CODE_GITHUB_AUTHORIZATION/); - assert.equal(wrapped.match(/Authorization: Bearer/g)?.length, 1); + assert.equal(wrapped.match(/Authorization: Basic/g)?.length, 1); assert.ok(!wrapped.includes('github_pat_')); }); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 7b1f12b1..b8046897 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -197,7 +197,10 @@ export function gitHubCredentialEnvironment( credential: GitHubCredential, ): Record { return { - [GITHUB_CREDENTIAL_ENV_NAME]: credential.value, + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + `x-access-token:${credential.value}`, + 'utf8', + ).toString('base64'), }; } @@ -250,14 +253,14 @@ export function wrapGitHubCredentialCommand( return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', - `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Bearer %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, + `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, command, ].join(' && '); } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', - `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Bearer \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, + `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, command, ].join(';\n'); From 1c7af888c774a52d3a8d4170590c6c409b03ae6e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 01:17:13 -0400 Subject: [PATCH 14/75] fix: isolate file deletion rate limits (#193) --- service/src/config.ts | 6 +++ service/src/middleware/limits.test.ts | 56 ++++++++++++++++++++++++ service/src/middleware/limits.ts | 11 +++++ service/src/service/exec-timeout.test.ts | 2 +- service/src/service/router.ts | 12 +++-- 5 files changed, 83 insertions(+), 4 deletions(-) diff --git a/service/src/config.ts b/service/src/config.ts index 90df6b58..d025831e 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -346,6 +346,12 @@ export const env = { // Files List Rate Limits FETCH_LIMIT_WINDOW: Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, // 1 minute FETCH_MAX_REQUESTS: Number(process.env.FETCH_MAX_REQUESTS) || 120, // 120 requests per minute + // File Delete Rate Limits. Fall back to the fetch settings so existing + // deployments keep their current limits while using an independent bucket. + DELETE_LIMIT_WINDOW: + Number(process.env.DELETE_LIMIT_WINDOW) || Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, + DELETE_MAX_REQUESTS: + Number(process.env.DELETE_MAX_REQUESTS) || Number(process.env.FETCH_MAX_REQUESTS) || 120, // Redis Key Cache Config SESSION_CACHE_TTL: Number(process.env.SESSION_CACHE_TTL) || 86400, /** TTL for the durable `session-owner:` record that backs diff --git a/service/src/middleware/limits.test.ts b/service/src/middleware/limits.test.ts index e7c1c46f..12f3daf4 100644 --- a/service/src/middleware/limits.test.ts +++ b/service/src/middleware/limits.test.ts @@ -111,6 +111,40 @@ async function startRateLimitedApp(max: number, windowMs: number): Promise { + const redis = new TestRedisRateLimitStore(); + setRateLimitRedisForTests(redis); + + const app = express(); + app.use((req, _res, next) => { + applyPrincipal(req as AuthenticatedRequest, { + userId: 'user-a', + tenantId: 'tenant-a', + principalSource: 'librechat_jwt', + }); + next(); + }); + app.get( + '/v1/files/session-a', + createRateLimiter('test-fetch', windowMs, max, { message: 'Too many file list requests.' }), + (_req, res) => res.status(200).json({ ok: true }), + ); + app.delete( + '/v1/files/session-a/file-a', + createRateLimiter('test-delete', windowMs, max, { + message: 'Too many file deletion requests.', + structuredBody: true, + }), + (_req, res) => res.status(200).json({ ok: true }), + ); + + const server = app.listen(0, '127.0.0.1'); + servers.push(server); + await once(server, 'listening'); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + function postExec(url: string, headers: Record = {}): Promise { return fetch(`${url}/v1/exec`, { method: 'POST', @@ -228,3 +262,25 @@ describe('execution rate limiting', () => { expect((await postExec(url)).status).toBe(200); }); }); + +describe('file operation rate limiting', () => { + test('keeps deletion traffic out of the file-list bucket and returns structured retry guidance', async () => { + const url = await startIndependentFileLimiterApp(1, 30_000); + + expect((await fetch(`${url}/v1/files/session-a`)).status).toBe(200); + expect((await fetch(`${url}/v1/files/session-a/file-a`, { method: 'DELETE' })).status).toBe(200); + + const rejectedDelete = await fetch(`${url}/v1/files/session-a/file-a`, { method: 'DELETE' }); + const body = await rejectedDelete.json() as ReturnType; + expect(rejectedDelete.status).toBe(429); + expect(rejectedDelete.headers.get('retry-after')).not.toBeNull(); + expect(body.error).toBe('rate_limited'); + expect(body.message).toContain('Too many file deletion requests.'); + + const rejectedList = await fetch(`${url}/v1/files/session-a`); + expect(rejectedList.status).toBe(429); + expect(await rejectedList.json()).toEqual({ + error: expect.stringContaining('Too many file list requests.'), + }); + }); +}); diff --git a/service/src/middleware/limits.ts b/service/src/middleware/limits.ts index b16ed196..099261a1 100644 --- a/service/src/middleware/limits.ts +++ b/service/src/middleware/limits.ts @@ -202,3 +202,14 @@ export const fetchLimiter = createRateLimiter( env.FETCH_MAX_REQUESTS, { message: 'Too many file list requests.' } ); + +export const deleteLimiter = createRateLimiter( + 'delete', + env.DELETE_LIMIT_WINDOW, + env.DELETE_MAX_REQUESTS, + { + message: 'Too many file deletion requests.', + structuredBody: true, + logRejections: true, + } +); diff --git a/service/src/service/exec-timeout.test.ts b/service/src/service/exec-timeout.test.ts index 70e00d17..b10a94c6 100644 --- a/service/src/service/exec-timeout.test.ts +++ b/service/src/service/exec-timeout.test.ts @@ -11,7 +11,7 @@ test('/exec validates timeout before enqueue and forwards its cap to both langua mock.module('./src/middleware/auth', () => ({ sessionAuth: passthrough })); mock.module('./src/middleware/limits', () => ({ executionLimiter: passthrough, uploadLimiter: passthrough, - downloadLimiter: passthrough, fetchLimiter: passthrough, + downloadLimiter: passthrough, fetchLimiter: passthrough, deleteLimiter: passthrough, })); mock.module('./src/lifecycle', () => ({ checkServiceStartUp: () => false, checkServiceShutDown: () => false, diff --git a/service/src/service/router.ts b/service/src/service/router.ts index 2c42f60c..43d0ff4b 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -7,7 +7,13 @@ import { Readable } from 'stream'; import type * as t from '../types'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; import { sessionAuth } from '../middleware/auth'; -import { executionLimiter, uploadLimiter, downloadLimiter, fetchLimiter } from '../middleware/limits'; +import { + executionLimiter, + uploadLimiter, + downloadLimiter, + fetchLimiter, + deleteLimiter, +} from '../middleware/limits'; import { internalServiceHeaders } from '../internal-service-auth'; import { resolveSessionKey, resolveOutputBucketSessionKey, SessionKeyResolutionError, parseUploadSessionKeyInput, type SessionKeyInput } from '../session-key'; import { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, queueNames, connection } from '../queue'; @@ -997,7 +1003,7 @@ const deleteSessionObject = async (req: t.AuthenticatedRequest, res: Response) = } }; -router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, deleteSessionObject); +router.delete('/files/:session_id/:fileId', deleteLimiter, sessionAuth, deleteSessionObject); /** * Alias of the route above, on the path LibreChat's `deleteCodeEnvFile` @@ -1013,6 +1019,6 @@ router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, deleteSes * * GET on this same path is the metadata proxy above. */ -router.delete('/sessions/:session_id/objects/:fileId', fetchLimiter, sessionAuth, deleteSessionObject); +router.delete('/sessions/:session_id/objects/:fileId', deleteLimiter, sessionAuth, deleteSessionObject); export default router; From 737f498ebedc3b9b26da3e5f206d7e0c49b4e901 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 01:27:39 -0400 Subject: [PATCH 15/75] feat: broker GitHub CLI authentication (#194) --- packages/code/README.md | 19 +++++---- packages/code/src/cli.ts | 15 +++---- packages/code/src/github.test.ts | 73 ++++++++++++++++++++++++++++++++ packages/code/src/github.ts | 40 +++++++++++++++++ 4 files changed, 129 insertions(+), 18 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index e46d6fd6..d06fbf9a 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -180,14 +180,17 @@ verification are implemented; use macOS, Linux, or WSL2. This also applies to Gi private keys. Git receives authentication through process-scoped `GIT_CONFIG_*` variables. -The same isolated config supplies the standard Git LFS filters; hosts using LFS -must install `git-lfs`, and checkout fails instead of silently leaving pointer -files when it is unavailable. -SRT replaces only the bearer-token portion with a sentinel inside the sandbox -and substitutes the real value in its host proxy only for `github.com` HTTPS -traffic. TLS termination is enabled for that substitution. The worker restores -the parent environment immediately after constructing the sandbox command; it -never writes credentials into the repository, a remote URL, or Git config. +When the GitHub CLI is installed, `gh api`, pull-request, issue, and workflow +commands receive the same installation scope through `GH_TOKEN` (or +`GH_ENTERPRISE_TOKEN` for GHES). The same isolated Git config supplies the +standard Git LFS filters; hosts using LFS must install `git-lfs`, and checkout +fails instead of silently leaving pointer files when it is unavailable. +SRT replaces each real credential with a sentinel inside the sandbox and +substitutes the real value in its host proxy only for the corresponding Git or +GitHub API host. TLS termination is enabled for that substitution. The worker +restores the parent environment immediately after constructing the sandbox +command; it never writes credentials into the repository, a remote URL, Git +config, or the GitHub CLI credential store. GitHub's required domains are added to the command egress allowlist only when authentication is configured. The worker identity, GitHub App key path, token source variables, and mutation-quarantine record remain denied to sandboxed diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index fdd28e9f..a87f1a5a 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -38,11 +38,11 @@ import type { NativeProcessSandboxOptions } from './native-process.js'; import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, - GITHUB_CREDENTIAL_ENV_NAME, GitHubAppCredentialProvider, + gitHubCommandCredentialEnvironment, + gitHubMaskedCredentialVariables, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, - gitHubCredentialEnvironment, normalizeGitHubHost, wrapGitHubCredentialCommand, } from './github.js'; @@ -783,16 +783,11 @@ async function run( ...(github.provider ? { maskedEnvironment: { - variables: [ - { - name: GITHUB_CREDENTIAL_ENV_NAME, - extract: '^(.+)$', - injectHosts: [github.host], - }, - ], + variables: gitHubMaskedCredentialVariables(github.host), async resolve(signal?: AbortSignal) { - return gitHubCredentialEnvironment( + return gitHubCommandCredentialEnvironment( await github.provider!.getCredential(signal), + github.host, ); }, wrapCommand(command: string, platform: NodeJS.Platform) { diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 78d9751d..4b028239 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -17,6 +17,9 @@ import { GitHubAppCredentialProvider, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, + gitHubCliTokenEnvironmentName, + gitHubCommandCredentialEnvironment, + gitHubMaskedCredentialVariables, GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, normalizeGitHubHost, @@ -150,6 +153,65 @@ test('builds process-scoped Git HTTPS authorization without embedding credential assert.ok(!encodedCredential.includes('github_pat_')); }); +test('adds a GitHub CLI token only to the command-sandbox credential bundle', async () => { + const provider = new StaticGitHubCredentialProvider( + 'github_pat_abcdefghijklmnopqrstuvwxyz', + ); + const credential = await provider.getCredential(); + assert.deepEqual(gitHubCommandCredentialEnvironment(credential), { + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'), + GH_TOKEN: 'github_pat_abcdefghijklmnopqrstuvwxyz', + }); + assert.deepEqual( + gitHubCommandCredentialEnvironment(credential, 'github.example.test'), + { + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'), + GH_ENTERPRISE_TOKEN: 'github_pat_abcdefghijklmnopqrstuvwxyz', + }, + ); +}); + +test('selects the GitHub CLI token variable for public and enterprise hosts', () => { + assert.equal(gitHubCliTokenEnvironmentName('github.com'), 'GH_TOKEN'); + assert.equal( + gitHubCliTokenEnvironmentName('github.example.test'), + 'GH_ENTERPRISE_TOKEN', + ); +}); + +test('restricts Git and GitHub CLI credential substitution to their respective hosts', () => { + assert.deepEqual(gitHubMaskedCredentialVariables('github.com'), [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: ['github.com'], + }, + { + name: 'GH_TOKEN', + extract: '^(.+)$', + injectHosts: ['api.github.com'], + }, + ]); + assert.deepEqual(gitHubMaskedCredentialVariables('github.example.test'), [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: ['github.example.test'], + }, + { + name: 'GH_ENTERPRISE_TOKEN', + extract: '^(.+)$', + injectHosts: ['github.example.test'], + }, + ]); +}); + test('composes the masked credential with SRT Git configuration inside the sandbox', () => { const wrapped = wrapGitHubCredentialCommand( 'git push', @@ -160,10 +222,21 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); assert.match(wrapped, /\$\{LIBRECHAT_CODE_GITHUB_AUTHORIZATION\}/); assert.match(wrapped, /unset LIBRECHAT_CODE_GITHUB_AUTHORIZATION/); + assert.doesNotMatch(wrapped, /unset GH_TOKEN/); assert.equal(wrapped.match(/Authorization: Basic/g)?.length, 1); assert.ok(!wrapped.includes('github_pat_')); }); +test('targets GitHub CLI at an enterprise host without exposing its token', () => { + const wrapped = wrapGitHubCredentialCommand( + 'gh pr create', + 'github.example.test', + 'linux', + ); + assert.match(wrapped, /GH_HOST=github\.example\.test/); + assert.doesNotMatch(wrapped, /GH_ENTERPRISE_TOKEN=/); +}); + test('rejects an insecure GitHub App API endpoint before reading the private key', () => { assert.throws( () => diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index b8046897..c727e70d 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -204,6 +204,43 @@ export function gitHubCredentialEnvironment( }; } +export function gitHubCommandCredentialEnvironment( + credential: GitHubCredential, + host = 'github.com', +): Record { + return { + ...gitHubCredentialEnvironment(credential), + [gitHubCliTokenEnvironmentName(host)]: credential.value, + }; +} + +export function gitHubCliTokenEnvironmentName(host: string): string { + return host === 'github.com' ? 'GH_TOKEN' : 'GH_ENTERPRISE_TOKEN'; +} + +export function gitHubApiHost(host: string): string { + return host === 'github.com' ? 'api.github.com' : host; +} + +export function gitHubMaskedCredentialVariables(host: string): Array<{ + name: string; + injectHosts: string[]; + extract: string; +}> { + return [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: [host], + }, + { + name: gitHubCliTokenEnvironmentName(host), + extract: '^(.+)$', + injectHosts: [gitHubApiHost(host)], + }, + ]; +} + export function gitHubAuthenticationPolicyIdentity(options: { mode?: 'app' | 'token'; host: string; @@ -249,10 +286,12 @@ export function wrapGitHubCredentialCommand( platform: NodeJS.Platform = process.platform, ): string { const key = `http.https://${host}/.extraheader`; + const cliHost = host === 'github.com' ? undefined : host; if (platform === 'win32') { return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', + ...(cliHost ? [`set "GH_HOST=${cliHost}"`] : []), `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, command, @@ -260,6 +299,7 @@ export function wrapGitHubCredentialCommand( } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', + ...(cliHost ? [`export GH_HOST=${cliHost}`] : []), `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, command, From 03e2fc11951fa3faf2fb5943afc445346177434e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 13:58:14 -0400 Subject: [PATCH 16/75] feat: Run PTC in Selected BYOM Workspaces (#195) * feat: run PTC in selected BYOM workspaces * fix: harden native workspace PTC replay * fix: preserve replay isolation and bridge limits * test: tolerate hosts without filesystem cloning * test: surface copy-on-write clone faults * fix: harden native workspace PTC admission * fix: close native replay effect and finalization boundaries --- docs/remote-bridge/README.md | 19 +- packages/code/README.md | 38 +- packages/code/src/cli.ts | 14 + packages/code/src/index.ts | 1 + packages/code/src/native-pool.test.ts | 25 + packages/code/src/native-pool.ts | 76 +- packages/code/src/native-process-child.ts | 35 +- packages/code/src/native-process.test.ts | 166 +++- packages/code/src/native-process.ts | 249 +++++- .../code/src/native-programmatic-live.test.ts | 41 + packages/code/src/native-programmatic.test.ts | 608 +++++++++++++ packages/code/src/native-programmatic.ts | 816 ++++++++++++++++++ packages/code/src/native-sandbox.test.ts | 357 ++++++-- packages/code/src/native-sandbox.ts | 448 ++++++++-- packages/code/src/protocol.test.ts | 184 ++++ packages/code/src/protocol.ts | 454 ++++++++-- packages/code/src/worker-slots.test.ts | 48 ++ packages/code/src/worker.ts | 159 +++- packages/code/src/workspace-worker.test.ts | 82 ++ packages/code/src/workspace.ts | 5 + service/src/bridge/router.ts | 1 + service/src/bridge/selection.ts | 1 + service/src/bridge/store.ts | 112 ++- service/src/bridge/workspace-store.test.ts | 117 +++ service/src/egress-gateway.ts | 37 +- service/src/egress-grant.test.ts | 8 + service/src/preamble-bash.test.ts | 226 ++++- service/src/preamble-bash.ts | 125 ++- service/src/preamble.test.ts | 46 +- service/src/preamble.ts | 276 ++++-- service/src/ptc-constants.test.ts | 9 + service/src/ptc-constants.ts | 7 +- .../src/sandbox-backend/remote-bridge.test.ts | 28 + service/src/sandbox-backend/remote-bridge.ts | 1 + service/src/sandbox-backend/types.ts | 3 + service/src/sandbox-dispatch.test.ts | 67 +- service/src/sandbox-dispatch.ts | 33 +- service/src/sandbox-egress.ts | 29 + service/src/service/programmatic-router.ts | 477 +++++++--- .../src/service/programmatic-state.test.ts | 2 + service/src/service/programmatic-state.ts | 2 + service/src/service/replay-state.ts | 2 + service/src/types/service.ts | 29 +- service/src/workers.ts | 11 + 44 files changed, 4850 insertions(+), 624 deletions(-) create mode 100644 packages/code/src/native-programmatic-live.test.ts create mode 100644 packages/code/src/native-programmatic.test.ts create mode 100644 packages/code/src/native-programmatic.ts create mode 100644 service/src/ptc-constants.test.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index ab7e2f64..99a0dece 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -149,6 +149,17 @@ implementation and an allowlist of workspace IDs, preserves per-workspace operation restrictions, validates bounded results, and treats an unknown command failure as an uncertain mutation. +Selected attached workspaces on macOS, Linux, and WSL2 can also advertise Bash +Programmatic Tool Calling. Native Windows workers do not advertise Bash PTC. +Code API then runs each replay iteration through the same workspace-scoped +native SRT executor. Source code operates in the selected local root, while +replay metadata, injected skills and attachments, and generated artifacts are +staged in an execution-private data directory and removed after settlement. +Only authorized file references and returned artifacts cross the relay; the +repository is never uploaded to Code API. This capability is advertised only +when native SRT commands and a file-relay upstream are both configured, so +older or partially configured workers continue to fail closed. + Native SRT is the MVP and default command backend on a user's chosen laptop or VM. It uses Seatbelt on macOS, bubblewrap/seccomp on Linux, and the SRT restricted-account helper on Windows. It confines writes to the registered @@ -157,9 +168,11 @@ credentials, and denies network egress by default. Startup fails closed when the platform dependencies are unavailable; there is no unsandboxed fallback. Use `LIBRECHAT_CODE_COMMAND_ALLOWED_DOMAINS` for an explicit comma-separated egress allowlist. -Linux hosts must provide Bash at `/bin/bash`, `bubblewrap`, `socat`, and -`ripgrep`; macOS uses system facilities. Windows requires SRT's one-time -restricted-account setup. +Linux hosts must provide `bubblewrap`, `socat`, and `ripgrep`; macOS uses +system facilities. Bash Programmatic Tool Calling additionally requires Bash +5.2 or newer and `jq` on `PATH` on macOS, Linux, and WSL2. The worker resolves +the compatible shell from `PATH` rather than assuming `/bin/bash`. Windows +requires SRT's one-time restricted-account setup. The optional `docker-nsjail` adapter enables a stronger container boundary with `--allow-workspace-commands` (or diff --git a/packages/code/README.md b/packages/code/README.md index d06fbf9a..87d9dded 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -143,14 +143,48 @@ policy. This matches the personal-machine SRT trust model; use the Docker/NsJail backend or a dedicated VM boundary when hard teardown of adversarial process trees is required. -Linux hosts need Bash at `/bin/bash`, `bubblewrap`, `socat`, and `ripgrep`; macOS uses system -facilities. Follow SRT's one-time restricted-account setup when using Windows. +Linux hosts need `bubblewrap`, `socat`, and `ripgrep`; macOS uses system +facilities. Bash Programmatic Tool Calling additionally requires Bash 5.2 or +newer and `jq` on `PATH` on macOS, Linux, and WSL2. The worker resolves that +shell explicitly instead of assuming `/bin/bash`, which remains Bash 3.2 on +many macOS hosts. Follow SRT's one-time restricted-account setup when using Windows. An operator may allow explicit egress destinations with the comma-separated `LIBRECHAT_CODE_COMMAND_ALLOWED_DOMAINS` setting. Treat that as a security policy: an allowed destination can receive workspace data. The normalized allowlist is included in the worker policy digest. Tool approval hooks remain the user-facing allow/deny boundary for each invocation. +When Code API negotiates `bash` programmatic execution for a selected +workspace, the same native SRT executor also supports replay-mode Programmatic +Tool Calling on macOS, Linux, and WSL2 workers. Native Windows does not +advertise this Bash capability. The repository remains the command working directory. Generated +PTC scripts, replay history, skill files, chat attachments, and returned +artifacts use an owner-only per-execution directory under the worker's private +SRT scratch root, exposed to code as `LIBRECHAT_CODE_DATA_DIR`. That directory +is removed after every iteration and is never placed in the repository. + +Replay probes run against a disposable copy-on-write snapshot with network and +socket access denied, including under `trusted-vm`. External effects must not +repeat while discovering pending tools. Use registered tools for network-dependent +replay control flow; the final commit pass runs once under the configured policy. +Each probe's SRT proxy session is revoked before restoring the commit policy; +per-command network overrides alone do not restrict SRT's session-level proxies. +Probe failures do not quarantine the real workspace. Once the commit pass starts, +its fence remains until result restoration succeeds; uncertain finalization +quarantines only that workspace. + +Reference inputs and artifact outputs travel only through the configured +`LIBRECHAT_CODE_FILE_RELAY_UPSTREAM`, using Code API's execution-scoped opaque +egress grant. The worker rejects redirects and bounds each transfer to 10 MiB, +each execution to 100 files and 100 MiB total, and transfer concurrency to four. +Caller inputs are limited to 98 files, reserving two for the script and replay +history. Code API reserves one third of the job budget for all transfer batches +and negotiates each transfer's deadline before signing the request. +Its parent process keeps a 64-entry/32-MiB LRU input cache keyed by a stable, +Code-API-authorized digest; sandboxed commands cannot read that cache. Requests +against one workspace remain serialized, while negotiated lease slots allow +different registered roots to execute concurrently. + The native sandbox preserves standard `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and `NO_PROXY` names (including lowercase forms), plus Windows process and profile variables on Windows. SRT remains responsible for the final sandbox environment diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index a87f1a5a..289b0dc4 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -408,6 +408,11 @@ async function run( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + const nativeProgrammaticEnabled = + allowWorkspaceCommands && + commandSandboxMode === 'native-srt' && + process.platform !== 'win32' && + (fileRelayUpstream?.length ?? 0) > 0; const commandPolicy = resolveNativeSrtCommandPolicy( option(args, '--command-policy-preset') ?? process.env.LIBRECHAT_CODE_COMMAND_POLICY_PRESET?.trim().toLowerCase() ?? @@ -780,6 +785,9 @@ async function run( github.privateKeyPath, ].filter((path): path is string => path != null), allowedDomains: commandAllowedDomains, + ...(nativeProgrammaticEnabled + ? { programmaticFileUpstream: fileRelayUpstream } + : {}), ...(github.provider ? { maskedEnvironment: { @@ -819,6 +827,9 @@ async function run( workspaceTools = new SandboxWorkspaceTools({ workspaceTools, commandWorkspaces: roots.map((root) => root.id), + ...(nativeProgrammaticEnabled + ? { programmaticLanguages: ['bash'] } + : {}), commandSandbox: nativeCommandSandbox ?? new RuntimeWorkspaceCommandSandbox({ @@ -878,6 +889,9 @@ async function run( runtimeSupervisor, capabilities, workspaceTools, + ...(nativeProgrammaticEnabled && nativeCommandSandbox + ? { workspaceProgrammatic: nativeCommandSandbox } + : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 5363f0c0..6f94b190 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -7,6 +7,7 @@ export * from './workspace.js'; export * from './workspace-runtime.js'; export * from './native-policy.js'; export * from './native-sandbox.js'; +export * from './native-programmatic.js'; export * from './native-process.js'; export * from './github.js'; export * from './worker.js'; diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index 3a24db34..cad30a52 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -13,6 +13,31 @@ const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ workspaceId, command: 'fixture', }); + +test('native pool preflights every registered root with bounded concurrency', async () => { + const prepared: string[] = []; + let active = 0; + let peak = 0; + const pool = new NativeWorkspaceCommandPool(roots, 2, (options) => ({ + async prepare() { + active += 1; + peak = Math.max(peak, active); + await new Promise(resolve => setTimeout(resolve, 5)); + prepared.push(options.workspaceRoot); + active -= 1; + }, + async close() {}, + async execute() { + throw new Error('unreachable'); + }, + })); + + await pool.prepare(); + assert.deepEqual(prepared.sort(), ['/fixture/a', '/fixture/b', '/fixture/c']); + assert.equal(peak, 2); + await pool.close(); +}); + test('a known-clean executor failure is retired without replaying the command', async () => { let created = 0; let executed = 0; diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 28f155e2..766409b1 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -2,6 +2,7 @@ import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; import type { + BridgeWorkspaceProgrammaticRequest, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, } from './protocol.js'; @@ -10,7 +11,10 @@ interface Entry { sandbox: Pick< NativeProcessWorkspaceCommandSandbox, 'prepare' | 'execute' | 'close' - >; + > & + Partial< + Pick + >; busy: boolean; } @@ -92,12 +96,25 @@ export class NativeWorkspaceCommandPool { } async prepare(): Promise { - const entry = await this.allocate(this.roots.keys().next().value!); - try { - await entry.sandbox.prepare(); - } finally { - entry.busy = false; - } + const workspaceIds = [...this.roots.keys()]; + let next = 0; + await Promise.all( + Array.from( + { length: Math.min(this.capacity, workspaceIds.length) }, + async () => { + for (;;) { + const index = next++; + if (index >= workspaceIds.length) return; + const entry = await this.allocate(workspaceIds[index]!); + try { + await entry.sandbox.prepare(); + } finally { + entry.busy = false; + } + } + }, + ), + ); } async execute( @@ -136,6 +153,51 @@ export class NativeWorkspaceCommandPool { } } + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const entry = await this.allocate(workspaceId); + let enteredExecutor = false; + try { + if (signal?.aborted) + throw new WorkspaceToolError( + 'Programmatic execution cancelled before dispatch', + 'EXECUTION_ABORTED', + ); + enteredExecutor = true; + if (!entry.sandbox.executeProgrammatic) { + throw new WorkspaceToolError( + 'Native programmatic executor is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + return await entry.sandbox.executeProgrammatic( + workspaceId, + request, + signal, + ); + } catch (error) { + if ( + enteredExecutor && + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted + ) { + try { + await entry.sandbox.close(); + if (this.entries.get(workspaceId) === entry) + this.entries.delete(workspaceId); + } catch { + /* Retain ownership for subsequent cleanup/shutdown. */ + } + } + throw error; + } finally { + entry.busy = false; + } + } + async close(): Promise { this.closing = true; await this.allocation; diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index 05ffedfe..2e8beb38 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -1,11 +1,16 @@ import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { NativeWorkspaceProgrammaticExecutor } from './native-programmatic.js'; import { WorkspaceToolError } from './workspace.js'; import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; -import type { WorkspaceExecuteCommandRequest } from './protocol.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, +} from './protocol.js'; // This entrypoint is private to a forked trusted executor. No HTTP listener, // argv credentials, bridge token, or persisted pairing material is required. let sandbox: NativeSrtWorkspaceCommandSandbox | undefined; +let programmaticExecutor: NativeWorkspaceProgrammaticExecutor | undefined; let active: { id: string; controller: AbortController } | undefined; let busy = false; let credentials: Record = {}; @@ -44,11 +49,14 @@ process.on('message', async (raw: unknown) => { NativeSrtWorkspaceCommandSandboxOptions, 'maskedEnvironment' > & { + programmaticFileUpstream?: string; variables?: NonNullable< NativeSrtWorkspaceCommandSandboxOptions['maskedEnvironment'] >['variables']; }; request: WorkspaceExecuteCommandRequest; + programmaticRequest?: BridgeWorkspaceProgrammaticRequest; + workspaceId?: string; credentials?: Record; wrappedCommand?: string; }; @@ -62,7 +70,8 @@ process.on('message', async (raw: unknown) => { try { let result: unknown; if (message.type === 'prepare' && !sandbox) { - const { variables, ...options } = message.options; + const { variables, programmaticFileUpstream, ...options } = + message.options; sandbox = new NativeSrtWorkspaceCommandSandbox({ ...options, ...(variables @@ -80,11 +89,33 @@ process.on('message', async (raw: unknown) => { : {}), }); await sandbox.prepare(); + programmaticExecutor = programmaticFileUpstream + ? new NativeWorkspaceProgrammaticExecutor({ + sandbox, + upstreamUrl: programmaticFileUpstream, + }) + : undefined; + await programmaticExecutor?.prepare(); } else if (message.type === 'execute' && sandbox) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; result = await sandbox.execute(message.request, active.controller.signal); + } else if ( + message.type === 'programmatic' && + sandbox && + programmaticExecutor && + message.programmaticRequest && + typeof message.workspaceId === 'string' + ) { + active = { id: message.id, controller: new AbortController() }; + credentials = message.credentials ?? {}; + wrappedCommand = message.wrappedCommand; + result = await programmaticExecutor.execute( + message.programmaticRequest, + message.workspaceId, + active.controller.signal, + ); } else if (message.type === 'close' && sandbox) { await sandbox.close(); } else throw new Error('Invalid executor state'); diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 800dcf11..e96f1e65 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -1,13 +1,29 @@ import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; import test from 'node:test'; +import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import { NativeProcessWorkspaceCommandSandbox, nativeExecutorEnvironment, + trustedProgrammaticExecutable, } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; +test('preflight rejects relative and workspace-controlled executables including symlinks', async t => { + const root = await mkdtemp(join(tmpdir(), 'native-ptc-path-')); + const outside = await mkdtemp(join(tmpdir(), 'native-ptc-link-')); + t.after(async () => { await rm(root, { recursive: true, force: true }); await rm(outside, { recursive: true, force: true }); }); + const executable = join(root, 'bash'); + await writeFile(executable, '#!/bin/sh\nexit 0\n', { mode: 0o700 }); + await symlink(executable, join(outside, 'bash')); + await assert.rejects(trustedProgrammaticExecutable('./bash', root), /absolute/); + await assert.rejects(trustedProgrammaticExecutable(executable, root), /outside the workspace/); + await assert.rejects(trustedProgrammaticExecutable(join(outside, 'bash'), root), /outside the workspace/); +}); + const request = { protocolVersion: 1 as const, operation: 'execute_command' as const, @@ -43,13 +59,19 @@ function fixture( queueMicrotask(() => { if (message.type === 'prepare' && prepare) return prepare(child, message); - if (message.type === 'execute' && execute) + if ( + (message.type === 'execute' || message.type === 'programmatic') && + execute + ) return execute(child, message); if (message.type === 'cancel') return; child.emit('message', { id: message.id, ok: true, - ...(message.type === 'execute' ? { result } : {}), + ...(message.type === 'execute' || + message.type === 'programmatic' + ? { result } + : {}), }); }); return true; @@ -171,8 +193,88 @@ test('executor hands credentials over IPC only for the current command', async ( await sandbox.close(); }); +test('programmatic executor resolves and scopes credentials to its command', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + maskedEnvironment: { + variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], + async resolve() { + return { TOKEN: 'per-programmatic-secret' }; + }, + wrapCommand(command) { + return `wrapped ${command}`; + }, + }, + }, + fake.fork, + ); + const programmaticRequest = { + headers: {}, + body: { + language: 'bash' as const, + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'git status' }], + }, + }; + await sandbox.executeProgrammatic('primary', programmaticRequest); + assert.equal( + JSON.stringify(fake.options).includes('per-programmatic-secret'), + false, + ); + const message = fake.messages.find( + candidate => candidate.type === 'programmatic', + )!; + assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); + assert.equal( + message.wrappedCommand, + 'wrapped exec "$LIBRECHAT_CODE_BASH_PATH" "$LIBRECHAT_CODE_DATA_DIR/main.sh"', + ); + await sandbox.close(); +}); + +test('programmatic executor preserves a child-reported pre-dispatch failure', async () => { + const fake = fixture((child, message) => + child.emit('message', { + id: message.id, + ok: false, + code: 'COMMAND_UNAVAILABLE', + errorMessage: 'Programmatic input download failed', + mutation: false, + requiresQuarantine: false, + }), + ); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, + ); + await sandbox.close(); +}); + test('executor loss after dispatch is an uncertain mutation and is never replayed', async () => { - const fake = fixture((child) => child.emit('exit', 1)); + const fake = fixture(child => child.emit('exit', 1)); const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: '/workspace' }, fake.fork, @@ -180,16 +282,17 @@ test('executor loss after dispatch is an uncertain mutation and is never replaye await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted, ); await assert.rejects(sandbox.execute(request), /unavailable/); - assert.equal(fake.messages.filter((m) => m.type === 'execute').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'execute').length, 1); await sandbox.close(); }); test('executor cancellation targets the active request and preserves mutation certainty', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -202,7 +305,7 @@ test('executor cancellation targets the active request and preserves mutation ce await dispatch; await assert.rejects(sandbox.execute(request), /unavailable/); controller.abort(); - const command = fake.messages.find((m) => m.type === 'execute')!; + const command = fake.messages.find(m => m.type === 'execute')!; assert.deepEqual(fake.messages.at(-1), { type: 'cancel', id: command.id }); fake.child.emit('message', { id: command.id, @@ -224,7 +327,7 @@ test('executor cancellation targets the active request and preserves mutation ce test('executor ignores a cleanup exemption on non-cancellation failures', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -234,7 +337,7 @@ test('executor ignores a cleanup exemption on non-cancellation failures', async ); const execution = sandbox.execute(request); await dispatch; - const command = fake.messages.find((message) => message.type === 'execute')!; + const command = fake.messages.find(message => message.type === 'execute')!; fake.child.emit('message', { id: command.id, ok: false, @@ -269,7 +372,8 @@ test('executor rejects mismatched results as uncertain and fences subsequent com await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted, ); await assert.rejects(sandbox.execute(request), /unavailable/); await sandbox.close(); @@ -277,7 +381,7 @@ test('executor rejects mismatched results as uncertain and fences subsequent com test('executor close drains an active command before closing IPC', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -288,16 +392,16 @@ test('executor close drains an active command before closing IPC', async () => { const execution = sandbox.execute(request); await dispatch; const closing = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal( - fake.messages.some((m) => m.type === 'close'), + fake.messages.some(m => m.type === 'close'), false, ); - const command = fake.messages.find((m) => m.type === 'execute')!; + const command = fake.messages.find(m => m.type === 'execute')!; fake.child.emit('message', { id: command.id, ok: true, result }); assert.deepEqual(await execution, result); await closing; - assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'close').length, 1); await assert.rejects(sandbox.execute(request), /unavailable/); }); @@ -321,7 +425,7 @@ test('executor close resolves when the child exits during the close handshake', }, }); await sandbox.close(); - assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'close').length, 1); await assert.rejects(sandbox.execute(request), /unavailable/); }); @@ -365,7 +469,7 @@ test('executor close skips the handshake once the child is already lost', async fake.child.emit('exit', 1, null); await sandbox.close(); assert.equal( - fake.messages.some((m) => m.type === 'close'), + fake.messages.some(m => m.type === 'close'), false, ); await assert.rejects(sandbox.execute(request), /unavailable/); @@ -377,17 +481,20 @@ test('executor startup loss is not reported as an applied mutation', async () => { workspaceRoot: '/workspace' }, (path, args, options) => { const child = fake.fork(path, args, options); - queueMicrotask(() => child.emit('error', new Error('startup failed'))); + queueMicrotask(() => + child.emit('error', new Error('startup failed')), + ); return child; }, ); await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && !error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted, ); assert.equal( - fake.messages.some((m) => m.type === 'execute'), + fake.messages.some(m => m.type === 'execute'), false, ); await sandbox.close(); @@ -409,7 +516,7 @@ test('executor shutdown receipt fences reuse before the OS exit event', async () ); await assert.rejects(sandbox.execute(request)); await assert.rejects(sandbox.execute(request), /unavailable/); - assert.equal(fake.messages.filter((m) => m.type === 'execute').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'execute').length, 1); await sandbox.close(); }); @@ -431,7 +538,8 @@ test('executor preserves bounded startup diagnostics and conventional host setti ok: false, mutation: false, code: 'COMMAND_UNAVAILABLE', - errorMessage: 'Native sandbox dependencies are unavailable: bubblewrap', + errorMessage: + 'Native sandbox dependencies are unavailable: bubblewrap', }), ); const sandbox = new NativeProcessWorkspaceCommandSandbox( @@ -465,7 +573,10 @@ test('executor matches POSIX names exactly and folds names only on Windows', () https_proxy: 'http://proxy:8080', }); assert.deepEqual( - nativeExecutorEnvironment({ Path: 'C:\\bin', Temp: 'C:\\temp' }, 'win32'), + nativeExecutorEnvironment( + { Path: 'C:\\bin', Temp: 'C:\\temp' }, + 'win32', + ), { Path: 'C:\\bin', Temp: 'C:\\temp' }, ); }); @@ -486,7 +597,8 @@ test('executor classifies every pre-dispatch setup failure as mutation-atomic', return {}; }, wrapCommand(command) { - if (failure === 'wrapper') throw new Error('wrapper failed'); + if (failure === 'wrapper') + throw new Error('wrapper failed'); return command; }, }, @@ -503,10 +615,12 @@ test('executor classifies every pre-dispatch setup failure as mutation-atomic', error instanceof WorkspaceToolError && !error.mutationMayHaveCommitted && error.code === - (failure === 'abort' ? 'EXECUTION_ABORTED' : 'COMMAND_UNAVAILABLE'), + (failure === 'abort' + ? 'EXECUTION_ABORTED' + : 'COMMAND_UNAVAILABLE'), ); assert.equal( - fake.messages.some((m) => m.type === 'execute'), + fake.messages.some(m => m.type === 'execute'), false, ); await sandbox.close(); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index e1a02498..d2ea0d72 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -1,11 +1,21 @@ -import { fork } from 'node:child_process'; +import { execFile, fork } from 'node:child_process'; import { randomUUID } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { access, realpath } from 'node:fs/promises'; +import { isAbsolute, join, relative, sep } from 'node:path'; +import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; -import { isWorkspaceToolRequest, isWorkspaceToolResult } from './protocol.js'; +import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + isWorkspaceToolRequest, + isWorkspaceToolResult, +} from './protocol.js'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; import type { WorkspaceCommandSandbox } from './workspace.js'; import type { + BridgeWorkspaceProgrammaticRequest, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, } from './protocol.js'; @@ -13,7 +23,84 @@ import type { export type NativeProcessSandboxOptions = Omit< NativeSrtWorkspaceCommandSandboxOptions, 'manager' | 'spawnCommand' | 'platform' ->; +> & { + /** Hardened Code API egress gateway used for execution-scoped files. */ + programmaticFileUpstream?: string; +}; + +const execFileAsync = promisify(execFile); + +async function systemProgrammaticExecutable( + name: string, + workspaceRoot: string, +): Promise { + // Preflight runs outside SRT. Never execute a workspace-controlled PATH + // entry (including cwd, node_modules/.bin, or a symlink to another root). + for (const directory of ['/opt/homebrew/bin', '/usr/local/bin', '/usr/bin', '/bin', '/home/linuxbrew/.linuxbrew/bin']) { + const candidate = join(directory, name); + try { + const canonical = await trustedProgrammaticExecutable(candidate, workspaceRoot); + if (!['/opt/homebrew/', '/usr/local/', '/usr/bin/', '/bin/', '/home/linuxbrew/.linuxbrew/'].some(root => canonical.startsWith(root))) continue; + return canonical; + } catch { + // Continue through the bounded PATH entries. + } + } +} + +export async function trustedProgrammaticExecutable(candidate: string, workspaceRoot: string): Promise { + if (!isAbsolute(candidate)) throw new Error('Programmatic executable must be absolute'); + const [canonical, root] = await Promise.all([realpath(candidate), realpath(workspaceRoot)]); + const path = relative(root, canonical); + if (path === '' || (!isAbsolute(path) && path !== '..' && !path.startsWith(`..${sep}`))) { + throw new Error('Programmatic executable must be outside the workspace'); + } + await access(canonical, fsConstants.X_OK); + return canonical; +} + +async function resolveProgrammaticShell( + options: NativeProcessSandboxOptions, +): Promise { + const environment = options.environment ?? process.env; + const shellPath = + options.shellPath != null + ? await trustedProgrammaticExecutable(options.shellPath, options.workspaceRoot) + : await systemProgrammaticExecutable('bash', options.workspaceRoot); + const jqPath = await systemProgrammaticExecutable('jq', options.workspaceRoot); + if (!shellPath || !jqPath) { + throw new WorkspaceToolError( + 'Native programmatic execution requires trusted host installations of Bash 5.2 or newer and jq', + 'COMMAND_UNAVAILABLE', + ); + } + try { + const [{ stdout: bashVersion }] = await Promise.all([ + execFileAsync(shellPath, ['--version'], { + env: nativeExecutorEnvironment(environment), + timeout: 5_000, + }), + execFileAsync(jqPath, ['--version'], { + env: nativeExecutorEnvironment(environment), + timeout: 5_000, + }), + ]); + const match = /version\s+(\d+)\.(\d+)/i.exec(bashVersion); + if ( + !match || + Number(match[1]) < 5 || + (Number(match[1]) === 5 && Number(match[2]) < 2) + ) { + throw new Error('unsupported Bash version'); + } + } catch { + throw new WorkspaceToolError( + 'Native programmatic execution requires trusted host installations of Bash 5.2 or newer and jq', + 'COMMAND_UNAVAILABLE', + ); + } + return shellPath; +} /** Only OS discovery and conventional proxy settings cross into the executor. * In particular, never inherit NODE_OPTIONS, bridge identity, or app secrets. */ @@ -77,20 +164,22 @@ export function nativeExecutorEnvironment( * deadline, as opposed to a failure the executor reported explicitly. */ class NativeExecutorUnavailableError extends WorkspaceToolError { constructor(mutation: boolean) { - super('Native executor is unavailable', 'COMMAND_UNAVAILABLE', mutation); + super( + 'Native executor is unavailable', + 'COMMAND_UNAVAILABLE', + mutation, + ); this.name = 'NativeExecutorUnavailableError'; } } /** One persistent, process-isolated SRT manager per workspace. No automatic * restart/replay: losing IPC after execution starts is an ambiguous mutation. */ -export class NativeProcessWorkspaceCommandSandbox - implements WorkspaceCommandSandbox -{ +export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSandbox { readonly mutationFailuresAreAtomic = true as const; private child?: ChildProcess; private ready?: Promise; - private active?: Promise; + private active?: Promise; private closing?: Promise; private failed = false; private terminationTimer?: ReturnType; @@ -122,12 +211,17 @@ export class NativeProcessWorkspaceCommandSandbox } private async start(): Promise { + const programmaticShellPath = this.options.programmaticFileUpstream + ? await resolveProgrammaticShell(this.options) + : this.options.shellPath; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], { execArgv: [], - env: nativeExecutorEnvironment(this.options.environment ?? process.env), + env: nativeExecutorEnvironment( + this.options.environment ?? process.env, + ), stdio: ['ignore', 'ignore', 'ignore', 'ipc'], serialization: 'json', }, @@ -165,6 +259,8 @@ export class NativeProcessWorkspaceCommandSandbox const processTerminationConfirmed = code === 'EXECUTION_ABORTED' && message.requiresQuarantine === false; + const mutationMayHaveCommitted = + pending.mutation && message.mutation !== false; pending.reject( new WorkspaceToolError( typeof message.errorMessage === 'string' && @@ -172,8 +268,8 @@ export class NativeProcessWorkspaceCommandSandbox ? message.errorMessage : 'Native executor request failed', code, - pending.mutation && message.mutation !== false, - pending.mutation && !processTerminationConfirmed, + mutationMayHaveCommitted, + mutationMayHaveCommitted && !processTerminationConfirmed, ), ); } @@ -192,6 +288,7 @@ export class NativeProcessWorkspaceCommandSandbox allowedDomains, homeDirectory, shellPath, + programmaticFileUpstream, } = this.options; await this.rpc( 'prepare', @@ -202,13 +299,14 @@ export class NativeProcessWorkspaceCommandSandbox protectedPaths, allowedDomains, homeDirectory, - shellPath, + shellPath: programmaticShellPath ?? shellPath, + programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, }, 30_000, false, - ).catch((error) => { + ).catch(error => { this.failed = true; this.terminate(); throw error; @@ -223,7 +321,10 @@ export class NativeProcessWorkspaceCommandSandbox !isWorkspaceToolRequest(request) || request.operation !== 'execute_command' ) { - throw new WorkspaceToolError('Invalid native command', 'INVALID_REQUEST'); + throw new WorkspaceToolError( + 'Invalid native command', + 'INVALID_REQUEST', + ); } if (this.active || this.closing || this.failed) throw this.unavailable(false); @@ -236,12 +337,114 @@ export class NativeProcessWorkspaceCommandSandbox } } + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + if (this.active || this.closing || this.failed) + throw this.unavailable(false); + if (!this.options.programmaticFileUpstream) { + throw new WorkspaceToolError( + 'Native programmatic file transport is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const active = this.executeProgrammaticOnce( + request, + workspaceId, + signal, + ); + this.active = active; + try { + return await active; + } finally { + this.active = undefined; + } + } + + private async executeProgrammaticOnce( + request: BridgeWorkspaceProgrammaticRequest, + workspaceId: string, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + let credentials: Record | undefined; + let wrappedCommand: string | undefined; + try { + await this.prepare(); + if (signal?.aborted) throw new Error('aborted'); + credentials = await this.options.maskedEnvironment?.resolve(signal); + if (signal?.aborted) throw new Error('aborted'); + wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( + NATIVE_PROGRAMMATIC_COMMAND, + process.platform, + ); + if (signal?.aborted) throw new Error('aborted'); + } catch (error) { + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + throw error instanceof WorkspaceToolError + ? new WorkspaceToolError(error.message, error.code, false) + : new WorkspaceToolError( + 'Native programmatic executor setup failed before dispatch', + 'COMMAND_UNAVAILABLE', + ); + } + const result = await this.rpc( + 'programmatic', + { + programmaticRequest: request, + workspaceId, + credentials, + wrappedCommand, + }, + (request.body.run_timeout ?? 30_000) * + ((request.body.replay_tool_count ?? 0) > 0 ? 2 : 1) + + (Math.ceil( + request.body.files.filter(file => 'id' in file).length / 4, + ) + + Math.ceil( + (request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, + )) * + (request.body.transfer_timeout_ms ?? 30_000) + + 5_000, + true, + signal, + ); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + true, + ); + } + if (typeof result !== 'object' || result === null) { + this.failed = true; + this.terminate(); + throw this.unavailable(true); + } + return result; + } + private async executeOnce( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, ): Promise { if (signal?.aborted) - throw new WorkspaceToolError('Command aborted', 'EXECUTION_ABORTED'); + throw new WorkspaceToolError( + 'Command aborted', + 'EXECUTION_ABORTED', + ); let credentials: Record | undefined; let wrappedCommand: string | undefined; try { @@ -258,7 +461,10 @@ export class NativeProcessWorkspaceCommandSandbox // No execute RPC has been sent: setup, token refresh and wrapping cannot // have mutated the workspace. Do not quarantine it for setup failures. if (signal?.aborted) - throw new WorkspaceToolError('Command aborted', 'EXECUTION_ABORTED'); + throw new WorkspaceToolError( + 'Command aborted', + 'EXECUTION_ABORTED', + ); throw error instanceof WorkspaceToolError ? new WorkspaceToolError(error.message, error.code, false) : new WorkspaceToolError( @@ -324,7 +530,7 @@ export class NativeProcessWorkspaceCommandSandbox reject(this.unavailable(mutation)); }; try { - child.send({ type, id, ...payload }, (error) => { + child.send({ type, id, ...payload }, error => { if (error) sendFailed(); }); } catch { @@ -353,9 +559,12 @@ export class NativeProcessWorkspaceCommandSandbox await this.ready?.catch(() => undefined); try { if (this.child?.connected && !this.failed) - await this.rpc('close', {}, 10_000, false).catch((error: unknown) => { - if (!(error instanceof NativeExecutorUnavailableError)) throw error; - }); + await this.rpc('close', {}, 10_000, false).catch( + (error: unknown) => { + if (!(error instanceof NativeExecutorUnavailableError)) + throw error; + }, + ); } finally { this.failed = true; this.terminate(); diff --git a/packages/code/src/native-programmatic-live.test.ts b/packages/code/src/native-programmatic-live.test.ts new file mode 100644 index 00000000..2bc22356 --- /dev/null +++ b/packages/code/src/native-programmatic-live.test.ts @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import type { AddressInfo } from 'node:net'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +test('real SRT prevents speculative network effects under trusted-vm', { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, +}, async () => { + const root = await mkdtemp(join(tmpdir(), 'native-ptc-effects-')); + let effects = 0; + const server = createServer((_req, res) => { effects += 1; res.end('ok'); }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as AddressInfo).port; + const executor = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + programmaticFileUpstream: `http://127.0.0.1:${port}`, + }); + try { + await executor.prepare(); + const result = await executor.executeProgrammatic('primary', { headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'isolated-canary', replay_tool_count: 1, + run_timeout: 5000, + files: [{ name: 'main.sh', content: `curl --noproxy '*' --connect-timeout 1 --max-time 2 -s -X POST http://127.0.0.1:${port}/effect >/dev/null\nprintf once >> commit.txt\n` }], + } }) as { run: { code: number } }; + assert.equal(result.run.code, 0); + assert.equal(effects, 1, 'probe must not emit a network effect'); + assert.equal(await readFile(join(root, 'commit.txt'), 'utf8'), 'once'); + } finally { + try { await executor.close(); } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(root, { recursive: true, force: true }); + } + } +}); diff --git a/packages/code/src/native-programmatic.test.ts b/packages/code/src/native-programmatic.test.ts new file mode 100644 index 00000000..82650f95 --- /dev/null +++ b/packages/code/src/native-programmatic.test.ts @@ -0,0 +1,608 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { NativeWorkspaceProgrammaticExecutor } from './native-programmatic.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { AddressInfo } from 'node:net'; +import type { BridgeWorkspaceProgrammaticRequest } from './protocol.js'; + +test('stages skill files privately and returns generated artifacts', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-test-')); + const uploads = new Map(); + let downloadCount = 0; + const server = createServer(async (req, res) => { + assert.equal(req.headers['x-codeapi-egress-grant'], 'grant'); + if (req.method === 'GET') { + downloadCount += 1; + assert.match( + req.url ?? '', + /\/sessions\/input-session\/objects\/skill-file$/, + ); + res.end('skill-value'); + return; + } + assert.equal(req.method, 'PUT'); + assert.equal(req.headers['content-type'], 'text/plain'); + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + uploads.set( + decodeURIComponent(req.headers['x-original-filename'] as string), + Buffer.concat(chunks), + ); + res.statusCode = 200; + res.end(); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + let observedDataDirectory = ''; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + observedDataDirectory = dataDirectory; + assert.equal( + await readFile( + join(dataDirectory, 'skills/example/reference.txt'), + 'utf8', + ), + 'skill-value', + ); + await writeFile(join(dataDirectory, 'result.txt'), 'artifact'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + const request: BridgeWorkspaceProgrammaticRequest = { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'execution-one', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'printf done' }, + { name: '_ptc_history.json', content: '{}' }, + { + name: 'skills/example/reference.txt', + id: 'skill-file', + storage_session_id: 'input-session', + input_cache_key: createHash('sha256') + .update('stable-authorized-input-identity') + .digest('hex'), + }, + ], + }, + }; + try { + const result = await executor.execute(request, 'primary'); + const replay = await executor.execute(request, 'primary'); + assert.equal(result.run.stdout, 'done\n'); + assert.equal(replay.run.stdout, 'done\n'); + assert.equal(result.session_id, 'output-session'); + assert.equal(downloadCount, 1); + await executor.execute( + { + ...request, + body: { ...request.body, execution_id: 'execution-two' }, + }, + 'primary', + ); + assert.equal(downloadCount, 2); + assert.equal(result.files.length, 1); + assert.equal(result.files[0]?.name, 'result.txt'); + assert.equal(uploads.get('result.txt')?.toString(), 'artifact'); + assert.deepEqual( + await readdir(observedDataDirectory).catch(() => []), + [], + ); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('reports unsupported and rejected artifacts without invalidating a completed command', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-test-')); + const uploads = new Map(); + const server = createServer(async (req, res) => { + const name = decodeURIComponent(req.headers['x-original-filename'] as string); + uploads.set(name, req.headers['content-type']); + for await (const _chunk of req) { + // Drain the bounded request body before responding. + } + res.statusCode = name === 'image.png' ? 503 : 200; + res.end(); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(_request, dataDirectory) { + await writeFile(join(dataDirectory, '_ptc_report.csv'), 'a,b\n1,2\n'); + await writeFile(join(dataDirectory, 'image.png'), 'not-a-real-png'); + await writeFile(join(dataDirectory, 'model.bin'), 'unsupported'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ); + assert.equal(result.run.code, 0); + assert.deepEqual(result.files.map(file => file.name), ['_ptc_report.csv']); + assert.deepEqual(result.artifact_delivery, { + code: 'artifact_delivery_failed', + status: 'partial', + attempted: 3, + delivered: 1, + failed: 2, + }); + assert.equal(uploads.get('_ptc_report.csv'), 'text/csv'); + assert.equal(uploads.get('image.png'), 'image/png'); + assert.equal(uploads.has('model.bin'), false); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('reports artifact transport failure without quarantining a completed command', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-transport-test-')); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + fetchImpl: async () => { + throw new TypeError('transport unavailable'); + }, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(_request, dataDirectory) { + await writeFile(join(dataDirectory, 'result.txt'), 'artifact'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ); + assert.equal(result.run.code, 0); + assert.deepEqual(result.files, []); + assert.deepEqual(result.artifact_delivery, { + code: 'artifact_delivery_failed', + status: 'failed', + attempted: 1, + delivered: 0, + failed: 1, + }); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('preflights copy-on-write isolation and removes its private snapshot', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-preflight-test-')); + let executionDirectory = ''; + let probes = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + executionDirectory = await mkdtemp(join(scratch, 'execution-')); + return executionDirectory; + }, + async createProgrammaticProbeWorkspace(directory) { + probes += 1; + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + try { + await executor.prepare(); + assert.equal(probes, 1); + assert.deepEqual(await readdir(executionDirectory).catch(() => []), []); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('keeps replay probes read-only and commits the script exactly once', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-probe-test-')); + const phases: boolean[] = []; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async createProgrammaticProbeWorkspace(executionDirectory) { + const workspace = join(executionDirectory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic( + _request, + dataDirectory, + _signal, + options, + ) { + phases.push(options?.probe === true); + if (options?.probe === true) { + assert.match(options.workspaceRoot ?? '', /\/workspace$/); + } + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: options?.probe ? 1 : 0, + stdout: options?.probe ? 'probe\n' : 'commit\n', + stderr: options?.probe ? 'expected probe denial\n' : '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'probe-then-commit', + replay_tool_count: 1, + session_id: 'execution-session', + files: [ + { name: 'main.sh', content: 'printf done' }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }, + 'primary', + ); + assert.deepEqual(phases, [true, false]); + assert.equal(result.run.stdout, 'commit\n'); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('returns pending calls from the private control file even when stdout truncates', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-control-test-')); + let phases = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async createProgrammaticProbeWorkspace(executionDirectory) { + const workspace = join(executionDirectory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic( + _request, + dataDirectory, + _signal, + options, + ) { + assert.equal(options?.probe, true); + phases += 1; + await writeFile( + join(dataDirectory, '_ptc_pending_result.json'), + JSON.stringify({ + pending: [ + { + call_id: 'call_001', + tool_name: 'lookup', + input: {}, + }, + ], + }), + ); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'x'.repeat(256 * 1024), + stderr: '', + truncated: true, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'truncated-control', + replay_tool_count: 1, + session_id: 'execution-session', + files: [ + { name: 'main.sh', content: 'lookup "{}"' }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }, + 'primary', + ); + assert.equal(phases, 1); + assert.equal(result.run.stdout, ''); + assert.equal(result.run.stderr, ''); + assert.deepEqual(JSON.parse(result.pending_tool_calls_payload ?? ''), { + pending: [{ call_id: 'call_001', tool_name: 'lookup', input: {} }], + }); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('rejects traversal before creating execution state', async () => { + let allocated = false; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + allocated = true; + return '/unused'; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + await assert.rejects( + executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + files: [{ name: '../main.sh', content: 'echo unsafe' }], + }, + }, + 'primary', + ), + /Invalid selected-workspace programmatic request/, + ); + assert.equal(allocated, false); +}); + +test('rejects artifacts above the negotiated byte ceiling before upload', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-output-limit-')); + let uploads = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + fetchImpl: async () => { + uploads += 1; + return new Response(); + }, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await writeFile(join(dataDirectory, 'artifact.txt'), 'too large'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + await assert.rejects( + executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + max_output_file_bytes: 4, + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ), + /exceeds the file limit/, + ); + assert.equal(uploads, 0); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +for (const failure of ['truncated', 'process-error']) test(`a failed speculative probe does not quarantine the real workspace (${failure})`, async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-failed-probe-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async createProgrammaticProbeWorkspace(directory) { const root = join(directory, 'workspace'); await mkdir(root); return root; }, + async executeProgrammatic(request, _directory, _signal, options) { + assert.equal(options?.probe, true); + if (failure === 'process-error') throw new WorkspaceToolError('probe output exceeded its limit', 'COMMAND_UNAVAILABLE', true, true); + return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, + exitCode: 0, stdout: '', stderr: '', truncated: true, timedOut: false }; + }, + }, + }); + await assert.rejects(executor.execute({ headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'session', replay_tool_count: 1, + files: [{ name: 'main.sh', content: 'true' }], + } }, 'primary'), (error: unknown) => { + assert.match(String(error), /probe output exceeded/); + assert.equal((error as { requiresQuarantine: boolean }).requiresQuarantine, false); + assert.equal((error as { mutationMayHaveCommitted: boolean }).mutationMayHaveCommitted, false); + return true; + }); +}); + +test('unchanged inputs do not consume the negotiated output budget', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-unchanged-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async executeProgrammatic(request) { + return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, + exitCode: 0, stdout: '', stderr: '', truncated: false, timedOut: false }; + }, + }, + }); + const result = await executor.execute({ headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'session', max_output_file_bytes: 1, + files: [{ name: 'main.sh', content: 'true' }, { name: 'input.txt', content: 'unchanged input' }], + } }, 'primary'); + assert.deepEqual(result.files, []); +}); + +test('stops admitting downloads and drains in-flight transfers before cleanup', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-transfer-test-')); + let executionDirectory = ''; + let requestCount = 0; + const server = createServer((req, res) => { + requestCount += 1; + if (requestCount === 1) { + res.statusCode = 503; + res.end(); + return; + } + setTimeout(() => res.end('in-flight'), 25); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + executionDirectory = await mkdtemp(join(scratch, 'execution-')); + return executionDirectory; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + const request: BridgeWorkspaceProgrammaticRequest = { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [ + ...Array.from({ length: 8 }, (_, index) => ({ + name: `inputs/${index}.txt`, + id: `input-${index}`, + storage_session_id: 'input-session', + })), + { name: 'main.sh', content: 'printf done' }, + ], + }, + }; + try { + const startedAt = performance.now(); + await assert.rejects( + executor.execute(request, 'primary'), + /Programmatic input download failed with HTTP 503/, + ); + assert.ok(performance.now() - startedAt >= 20); + assert.ok(requestCount <= 4); + assert.deepEqual(await readdir(executionDirectory).catch(() => []), []); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts new file mode 100644 index 00000000..42974b79 --- /dev/null +++ b/packages/code/src/native-programmatic.ts @@ -0,0 +1,816 @@ +import { createHash, randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; +import { cp, mkdir, open, readdir, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, relative, sep } from 'node:path'; + +import { + BRIDGE_PROTOCOL_VERSION, + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES, + bridgeArtifactMediaType, + isBridgeWorkspaceProgrammaticRequest, + isSafePortableRelativePath, + isSupportedBridgeArtifactName, +} from './protocol.js'; +import { validateFileRelayUpstream } from './relay.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { + BridgeProgrammaticPayloadFile, + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandResult, +} from './protocol.js'; +import type { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; + +const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +const EXECUTION_MAIN_FILE = 'main.sh'; +const EXECUTION_HISTORY_FILE = '_ptc_history.json'; +const EXECUTION_CONTROL_FILE = '_ptc_pending_result.json'; +export const NATIVE_PROGRAMMATIC_COMMAND = + 'exec "$LIBRECHAT_CODE_BASH_PATH" "$LIBRECHAT_CODE_DATA_DIR/main.sh"'; +const TRANSFER_TIMEOUT_MS = 30_000; +const TRANSFER_CONCURRENCY = 4; +const MAX_WALK_ENTRIES = 2_000; +const INPUT_CACHE_MAX_ENTRIES = 64; +const INPUT_CACHE_MAX_BYTES = 32 * 1024 * 1024; +const CONTROL_PAYLOAD_MAX_BYTES = 512 * 1024; + +type ProgrammaticFileResult = { + id: string; + name: string; + storage_session_id: string; + modified_from?: { id: string; storage_session_id: string }; +}; + +type ProgrammaticResult = { + language: 'bash'; + version: string; + session_id: string; + files: ProgrammaticFileResult[]; + artifact_delivery?: { + code: 'artifact_delivery_failed'; + status: 'partial' | 'failed'; + attempted: number; + delivered: number; + failed: number; + }; + pending_tool_calls_payload?: string; + run: { + stdout: string; + stderr: string; + code: number | null; + signal: string | null; + output: string; + memory: null; + message: string | null; + status: string | null; + cpu_time: null; + wall_time: number; + }; +}; + +type InputBaseline = { + sha256: string; + source?: { id: string; storage_session_id: string }; +}; + +function sha256(value: Uint8Array): string { + return createHash('sha256').update(value).digest('hex'); +} + +function outputFileId(): string { + return randomBytes(18).toString('base64url').slice(0, 21); +} + +function localPath(root: string, name: string): string { + if (!isSafePortableRelativePath(name)) { + throw new WorkspaceToolError( + 'Invalid programmatic file path', + 'INVALID_PATH', + ); + } + const path = join(root, ...name.split('/')); + const child = relative(root, path); + if (child === '' || child === '..' || child.startsWith(`..${sep}`)) { + throw new WorkspaceToolError( + 'Invalid programmatic file path', + 'INVALID_PATH', + ); + } + return path; +} + +async function readBoundedResponse( + response: Response, + signal: AbortSignal, +): Promise { + const declaredLength = response.headers.get('content-length'); + if ( + declaredLength != null && + (!/^\d+$/.test(declaredLength) || + Number(declaredLength) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) + ) { + await response.body?.cancel(); + throw new WorkspaceToolError( + 'Programmatic input exceeds the file limit', + 'READ_LIMIT_EXCEEDED', + ); + } + if (!response.body) return Buffer.alloc(0); + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let bytes = 0; + try { + for (;;) { + if (signal.aborted) throw signal.reason; + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) { + throw new WorkspaceToolError( + 'Programmatic input exceeds the file limit', + 'READ_LIMIT_EXCEEDED', + ); + } + chunks.push(Buffer.from(value)); + } + } finally { + await reader.cancel().catch(() => undefined); + } + return Buffer.concat(chunks, bytes); +} + +async function mapConcurrent( + values: readonly T[], + concurrency: number, + action: (value: T) => Promise, +): Promise { + const results = new Array(values.length); + let next = 0; + let failed = false; + let failure: unknown; + await Promise.all( + Array.from( + { length: Math.min(concurrency, values.length) }, + async () => { + for (;;) { + if (failed) return; + const index = next++; + if (index >= values.length) return; + try { + results[index] = await action(values[index]!); + } catch (error) { + if (!failed) { + failed = true; + failure = error; + } + return; + } + } + }, + ), + ); + if (failed) throw failure; + return results; +} + +async function listRegularFiles(root: string): Promise { + const files: string[] = []; + const pending = ['']; + let entries = 0; + while (pending.length > 0) { + const directory = pending.pop()!; + for (const entry of await readdir(join(root, directory), { + withFileTypes: true, + })) { + if (++entries > MAX_WALK_ENTRIES) { + throw new WorkspaceToolError( + 'Programmatic output contains too many entries', + 'WRITE_LIMIT_EXCEEDED', + ); + } + const name = directory ? `${directory}/${entry.name}` : entry.name; + if (!isSafePortableRelativePath(name)) continue; + if (entry.isSymbolicLink()) continue; + if (entry.isDirectory()) pending.push(name); + else if (entry.isFile()) files.push(name); + } + } + return files.sort(); +} + +export interface NativeWorkspaceProgrammaticOptions { + sandbox: Pick< + NativeSrtWorkspaceCommandSandbox, + 'createExecutionDirectory' | 'executeProgrammatic' + > & + Partial< + Pick + >; + upstreamUrl: string; + fetchImpl?: typeof fetch; +} + +/** + * Executes one replay-mode Bash PTC iteration in an attached workspace. + * Program code and injected files live in a private SRT scratch directory; + * the selected repository remains the command cwd and is never used as a + * transport cache. + */ +export class NativeWorkspaceProgrammaticExecutor { + private readonly upstream: URL; + private readonly fetchImpl: typeof fetch; + /** Parent-process cache: sandboxed children cannot inspect this memory. */ + private readonly inputCache = new Map< + string, + { bytes: Buffer; lastUsed: number } + >(); + private inputCacheBytes = 0; + + constructor(private readonly options: NativeWorkspaceProgrammaticOptions) { + this.upstream = validateFileRelayUpstream(options.upstreamUrl); + this.fetchImpl = options.fetchImpl ?? fetch; + } + + /** + * Prove copy-on-write isolation before the worker advertises Bash PTC. + * The probe uses the exact registered root and private scratch path that a + * real replay will use, then removes the snapshot before registration. + */ + async prepare(signal?: AbortSignal): Promise { + const createProbeWorkspace = + this.options.sandbox.createProgrammaticProbeWorkspace; + if (createProbeWorkspace == null) { + throw new WorkspaceToolError( + 'Selected-workspace PTC probe isolation is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const executionDirectory = + await this.options.sandbox.createExecutionDirectory(); + try { + await createProbeWorkspace.call( + this.options.sandbox, + executionDirectory, + signal, + ); + } finally { + await rm(executionDirectory, { recursive: true, force: true }); + } + } + + private cacheKey( + executionId: string | undefined, + file: Extract, + ): string | undefined { + return executionId && file.input_cache_key + ? `${executionId}:${file.input_cache_key}` + : undefined; + } + + private cachedInput(key: string): Buffer | undefined { + const cached = this.inputCache.get(key); + if (!cached) return undefined; + cached.lastUsed = Date.now(); + return cached.bytes; + } + + private cacheInput(key: string, bytes: Buffer): void { + if (bytes.byteLength > INPUT_CACHE_MAX_BYTES) return; + const existing = this.inputCache.get(key); + if (existing) this.inputCacheBytes -= existing.bytes.byteLength; + while ( + this.inputCache.size >= INPUT_CACHE_MAX_ENTRIES || + this.inputCacheBytes + bytes.byteLength > INPUT_CACHE_MAX_BYTES + ) { + let oldestKey: string | undefined; + let oldestAt = Number.POSITIVE_INFINITY; + for (const [candidate, value] of this.inputCache) { + if (value.lastUsed < oldestAt) { + oldestAt = value.lastUsed; + oldestKey = candidate; + } + } + if (!oldestKey) break; + this.inputCacheBytes -= + this.inputCache.get(oldestKey)!.bytes.byteLength; + this.inputCache.delete(oldestKey); + } + this.inputCache.set(key, { bytes, lastUsed: Date.now() }); + this.inputCacheBytes += bytes.byteLength; + } + + private async downloadInput( + file: Extract, + grant: string, + executionId: string | undefined, + signal?: AbortSignal, + transferTimeoutMs = TRANSFER_TIMEOUT_MS, + ): Promise { + const key = this.cacheKey(executionId, file); + const cached = key ? this.cachedInput(key) : undefined; + if (cached) return cached; + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout(() => controller.abort(), transferTimeoutMs); + try { + const response = await this.fetchImpl( + new URL( + `sessions/${encodeURIComponent(file.storage_session_id)}/objects/${encodeURIComponent(file.id)}`, + `${this.upstream.toString().replace(/\/+$/, '')}/`, + ), + { + headers: { [EGRESS_GRANT_HEADER]: grant }, + redirect: 'error', + signal: controller.signal, + }, + ); + if (!response.ok) { + await response.body?.cancel(); + throw new WorkspaceToolError( + `Programmatic input download failed with HTTP ${response.status}`, + 'COMMAND_UNAVAILABLE', + ); + } + const bytes = await readBoundedResponse( + response, + controller.signal, + ); + if (key) this.cacheInput(key, bytes); + return bytes; + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } + } + + async execute( + request: BridgeWorkspaceProgrammaticRequest, + workspaceId: string, + signal?: AbortSignal, + ): Promise { + if (!isBridgeWorkspaceProgrammaticRequest(request)) { + throw new WorkspaceToolError( + 'Invalid selected-workspace programmatic request', + 'INVALID_REQUEST', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + const grant = request.body.egress_grant; + const refFiles = request.body.files.filter( + ( + file, + ): file is Extract => + 'id' in file, + ); + if (refFiles.length > 0 && !grant) { + throw new WorkspaceToolError( + 'Programmatic input grant is unavailable', + 'INVALID_REQUEST', + ); + } + const executionDirectory = + await this.options.sandbox.createExecutionDirectory(); + const inputDirectory = join(executionDirectory, 'inputs'); + let dataDirectory = join(executionDirectory, 'final'); + const baselines = new Map(); + let totalInputBytes = 0; + const startedAt = performance.now(); + let commandDispatched = false; + try { + await mkdir(inputDirectory, { mode: 0o700 }); + await mapConcurrent( + request.body.files, + TRANSFER_CONCURRENCY, + async (file): Promise => { + const bytes = + 'content' in file + ? Buffer.from(file.content) + : await this.downloadInput( + file, + grant!, + request.body.execution_id, + signal, + request.body.transfer_timeout_ms, + ); + totalInputBytes += bytes.byteLength; + if ( + totalInputBytes > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ) { + throw new WorkspaceToolError( + 'Programmatic inputs exceed the total byte limit', + 'READ_LIMIT_EXCEEDED', + ); + } + const path = localPath(inputDirectory, file.name); + await mkdir(dirname(path), { + recursive: true, + mode: 0o700, + }); + await writeFile(path, bytes, { flag: 'wx', mode: 0o600 }); + baselines.set(file.name, { + sha256: sha256(bytes), + ...('id' in file + ? { + source: { + id: file.id, + storage_session_id: + file.storage_session_id, + }, + } + : {}), + }); + }, + ); + + const run = async ( + directory: string, + probe: boolean, + workspaceRoot?: string, + ): Promise => { + await cp(inputDirectory, directory, { + recursive: true, + force: false, + errorOnExist: true, + mode: constants.COPYFILE_FICLONE, + }); + if (!probe) commandDispatched = true; + return await this.options.sandbox.executeProgrammatic( + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId, + command: NATIVE_PROGRAMMATIC_COMMAND, + timeoutMs: Math.min( + request.body.run_timeout ?? + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ), + maxOutputBytes: + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES, + }, + directory, + signal, + { probe, workspaceRoot }, + ); + }; + + const readPending = async ( + directory: string, + ): Promise => { + try { + const path = join(directory, EXECUTION_CONTROL_FILE); + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat(); + if ( + !metadata.isFile() || + metadata.size === 0 || + metadata.size > CONTROL_PAYLOAD_MAX_BYTES + ) { + throw new WorkspaceToolError( + 'Native programmatic control frame is invalid', + 'COMMAND_UNAVAILABLE', + ); + } + return await handle.readFile('utf8'); + } finally { + await handle.close(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') + return; + throw error; + } + }; + + if ((request.body.replay_tool_count ?? 0) > 0) { + const probeDirectory = join(executionDirectory, 'probe'); + const createProbeWorkspace = + this.options.sandbox.createProgrammaticProbeWorkspace; + if (createProbeWorkspace == null) { + throw new WorkspaceToolError( + 'Selected-workspace PTC probe isolation is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const probeWorkspace = await createProbeWorkspace.call( + this.options.sandbox, + executionDirectory, + signal, + ); + const probeResult = await run( + probeDirectory, + true, + probeWorkspace, + ); + const pending = await readPending(probeDirectory); + if (pending) { + return this.result( + request, + { + ...probeResult, + /** Probe output is speculative and the script will + * run once under its real policy after tool + * resolution. Never duplicate it or expose + * expected read-only policy denials to callers. */ + stdout: '', + stderr: '', + }, + [], + performance.now() - startedAt, + pending, + ); + } + if (probeResult.truncated) { + throw new WorkspaceToolError( + 'Native programmatic probe output exceeded its limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + if ( + probeResult.timedOut || + probeResult.signal + ) { + return this.result( + request, + probeResult, + [], + performance.now() - startedAt, + ); + } + /** A read-only probe commonly exits non-zero after it reaches + * an intentional workspace write denial. With no pending call, + * run the script once under its real policy so ordinary writes + * and their resulting exit status are evaluated exactly once. */ + } + + const commandResult = await run(dataDirectory, false); + if (await readPending(dataDirectory)) { + throw new WorkspaceToolError( + 'Native programmatic commit pass requested an unexpected replay tool', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + } + if (commandResult.truncated) { + throw new WorkspaceToolError( + 'Native programmatic output exceeded its limit', + 'WRITE_LIMIT_EXCEEDED', + true, + true, + ); + } + + const outputSessionId = request.body.output_session_id; + const outputNames = (await listRegularFiles(dataDirectory)).filter( + name => + name !== EXECUTION_MAIN_FILE && + name !== EXECUTION_HISTORY_FILE && + name !== EXECUTION_CONTROL_FILE && + !name.startsWith('skills/'), + ); + const changed: Array<{ + name: string; + bytes: Buffer; + source?: { id: string; storage_session_id: string }; + }> = []; + let totalOutputBytes = 0; + for (const name of outputNames) { + const path = localPath(dataDirectory, name); + const baseline = baselines.get(name); + const maxOutputFileBytes = Math.min( + request.body.max_output_file_bytes ?? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + ); + let bytes: Buffer; + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat(); + if (!metadata.isFile()) continue; + // An unchanged input is not an output. It may legitimately + // exceed the negotiated output ceiling, but never the + // protocol's bounded input limit. + if (metadata.size > (baseline ? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES : maxOutputFileBytes)) { + throw new WorkspaceToolError( + 'Programmatic output exceeds the file limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + bytes = await handle.readFile(); + } finally { + await handle.close(); + } + if (baseline?.sha256 === sha256(bytes)) continue; + if (bytes.byteLength > maxOutputFileBytes) { + throw new WorkspaceToolError( + 'Programmatic output exceeds the file limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + totalOutputBytes += bytes.byteLength; + if ( + totalOutputBytes > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ) { + throw new WorkspaceToolError( + 'Programmatic outputs exceed the total byte limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + changed.push({ name, bytes, source: baseline?.source }); + } + const maxOutputFiles = Math.min( + request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + ); + if (changed.length > maxOutputFiles) { + throw new WorkspaceToolError( + 'Programmatic output contains too many files', + 'WRITE_LIMIT_EXCEEDED', + ); + } + const uploadable = changed.filter(({ name }) => + isSupportedBridgeArtifactName(name), + ); + if (uploadable.length > 0 && (!grant || !outputSessionId)) { + throw new WorkspaceToolError( + 'Programmatic output grant is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const uploadResults = await mapConcurrent( + uploadable, + TRANSFER_CONCURRENCY, + async ({ name, bytes, source }): Promise => { + const id = outputFileId(); + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout( + () => controller.abort(), + request.body.transfer_timeout_ms ?? TRANSFER_TIMEOUT_MS, + ); + try { + let response: Response; + try { + response = await this.fetchImpl( + new URL( + `sessions/${encodeURIComponent(outputSessionId!)}/objects/${id}`, + `${this.upstream.toString().replace(/\/+$/, '')}/`, + ), + { + method: 'PUT', + headers: { + [EGRESS_GRANT_HEADER]: grant!, + 'Content-Type': bridgeArtifactMediaType(name), + 'Content-Length': String(bytes.byteLength), + 'X-Original-Filename': encodeURIComponent(name), + }, + body: new Uint8Array(bytes), + redirect: 'error', + signal: controller.signal, + }, + ); + } catch (error) { + if (signal?.aborted) throw error; + return undefined; + } + await response.body?.cancel(); + if (!response.ok) { + return undefined; + } + return { + id, + name, + storage_session_id: outputSessionId!, + ...(source ? { modified_from: source } : {}), + }; + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } + }, + ); + const files = uploadResults.filter( + (file): file is ProgrammaticFileResult => file != null, + ); + const artifactDelivery = + files.length < changed.length + ? { + code: 'artifact_delivery_failed' as const, + status: files.length > 0 ? ('partial' as const) : ('failed' as const), + attempted: changed.length, + delivered: files.length, + failed: changed.length - files.length, + } + : undefined; + return this.result( + request, + commandResult, + files, + performance.now() - startedAt, + undefined, + artifactDelivery, + ); + } catch (error) { + if (!commandDispatched) { + // The low-level command runner classifies any launched process as a + // possible mutation. A probe can only mutate its disposable snapshot, + // so translate that classification at this ownership boundary. + throw new WorkspaceToolError( + error instanceof Error ? error.message : 'Programmatic preparation failed', + error instanceof WorkspaceToolError ? error.code : 'COMMAND_UNAVAILABLE', + false, + false, + ); + } + if (error instanceof WorkspaceToolError) { + if ( + error.mutationMayHaveCommitted || + error.requiresQuarantine + ) { + throw error; + } + throw new WorkspaceToolError( + error.message, + error.code, + true, + true, + ); + } + throw new WorkspaceToolError( + 'Native programmatic execution failed after dispatch', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + } finally { + try { + await rm(executionDirectory, { recursive: true, force: true }); + } catch { + throw new WorkspaceToolError( + 'Native programmatic execution cleanup failed', + 'COMMAND_UNAVAILABLE', + commandDispatched, + commandDispatched, + ); + } + } + } + + private result( + request: BridgeWorkspaceProgrammaticRequest, + command: WorkspaceExecuteCommandResult, + files: ProgrammaticFileResult[], + elapsedMs: number, + pendingToolCallsPayload?: string, + artifactDelivery?: ProgrammaticResult['artifact_delivery'], + ): ProgrammaticResult { + return { + language: 'bash', + version: request.body.version, + // Code API masks the execution session separately from the writable + // output bucket. Sandbox results must identify the output bucket so the + // gateway can restore it to the caller-owned session after upload. + session_id: + request.body.output_session_id ?? request.body.session_id, + files, + ...(artifactDelivery ? { artifact_delivery: artifactDelivery } : {}), + ...(pendingToolCallsPayload + ? { pending_tool_calls_payload: pendingToolCallsPayload } + : {}), + run: { + stdout: command.stdout, + stderr: command.stderr, + code: command.exitCode, + signal: command.signal ?? null, + output: `${command.stdout}${command.stderr}`, + memory: null, + message: command.timedOut ? 'Execution timed out' : null, + status: command.timedOut ? 'timeout' : null, + cpu_time: null, + wall_time: elapsedMs / 1000, + }, + }; + } +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index cf790965..3252ee50 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -7,6 +7,7 @@ import { mkdtemp, mkdir, open, + readFile, realpath, rename, rm, @@ -25,7 +26,10 @@ import type { } from '@anthropic-ai/sandbox-runtime'; import type { ChildProcessWithoutNullStreams } from 'node:child_process'; -import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { + CopyOnWriteCloneUnavailableError, + NativeSrtWorkspaceCommandSandbox, +} from './native-sandbox.js'; import { restoreScratchTraversal } from './native-scratch.js'; import { WorkspaceToolError } from './workspace.js'; @@ -54,6 +58,8 @@ function fakeManager( let credentialSeenDuringWrap: string | undefined; let gitLfsRequiredSeenDuringWrap: string | undefined; let scratchSelectorSeenDuringWrap: string | undefined; + let networkSeenDuringWrap: SandboxRuntimeConfig['network'] | undefined; + let customConfigSeenDuringWrap: Partial | undefined; const manager = { isSupportedPlatform: () => true, async checkDependenciesAsync() { @@ -67,19 +73,30 @@ function fakeManager( askCallback = callback; if (options.initializeError) throw options.initializeError; }, - async wrapWithSandboxArgv(command: string) { + updateConfig(value: SandboxRuntimeConfig) { config = value; }, + async wrapWithSandboxArgv( + command: string, + _binShell?: string, + customConfig?: Partial, + ) { await options.beforeWrap?.(); - credentialSeenDuringWrap = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + networkSeenDuringWrap = config?.network; + customConfigSeenDuringWrap = customConfig; + credentialSeenDuringWrap = + process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; gitLfsRequiredSeenDuringWrap = process.env.GIT_CONFIG_VALUE_3; scratchSelectorSeenDuringWrap = process.env.CLAUDE_CODE_TMPDIR; const ambientGitEnvironment = Object.fromEntries( Object.entries(process.env).filter( - ([name, value]) => name.startsWith('GIT_CONFIG_') && value != null, + ([name, value]) => + name.startsWith('GIT_CONFIG_') && value != null, ), ); let gitEnvironment = ambientGitEnvironment; if (options.appendGitSafeDirectory) { - const index = Number(ambientGitEnvironment.GIT_CONFIG_COUNT ?? '0'); + const index = Number( + ambientGitEnvironment.GIT_CONFIG_COUNT ?? '0', + ); gitEnvironment = { ...(options.inheritedGitEnvironment ?? {}), GIT_CONFIG_COUNT: String(index + 1), @@ -130,10 +147,118 @@ function fakeManager( get scratchSelectorSeenDuringWrap() { return scratchSelectorSeenDuringWrap; }, + get networkSeenDuringWrap() { return networkSeenDuringWrap; }, + get customConfigSeenDuringWrap() { + return customConfigSeenDuringWrap; + }, }; } -test('exclusive lifecycle rejects a second workspace sharing an SRT manager', async (t) => { +for (const trustedVm of [false, true]) test(`programmatic probe denies real-workspace writes and external effects (trusted=${trustedVm})`, async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + allowedDomains: ['api.example.com'], + ...(trustedVm ? { commandPolicy: { version: 1 as const, preset: 'trusted-vm' as const, + network: { outbound: 'unrestricted' as const, allowLocalBinding: true, allowAllUnixSockets: true }, + } } : {}), + }); + const dataDirectory = await sandbox.createExecutionDirectory(); + await sandbox.executeProgrammatic(request, dataDirectory, undefined, { + probe: true, + }); + assert.deepEqual(fake.customConfigSeenDuringWrap?.network, { + allowedDomains: [], + deniedDomains: [], + strictAllowlist: true, + allowUnixSockets: [], + allowAllUnixSockets: false, + allowLocalBinding: false, + }); + assert.deepEqual(fake.networkSeenDuringWrap, fake.customConfigSeenDuringWrap?.network); + assert.equal(fake.config?.network.strictAllowlist, !trustedVm); + assert.equal(fake.reset, true, 'probe proxy session must be revoked before restoring policy'); + assert.deepEqual(fake.customConfigSeenDuringWrap?.filesystem?.allowWrite, [ + await realpath(dataDirectory), + ]); + assert.equal( + fake.scratchSelectorSeenDuringWrap, + await realpath(dataDirectory), + ); + assert.ok( + fake.customConfigSeenDuringWrap?.filesystem?.denyWrite?.includes( + await realpath(root), + ), + ); + await sandbox.close(); +}); + +test('probe network cleanup failure fences executor reuse', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-probe-cleanup-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ workspaceRoot: root, manager: fake.manager }); + const directory = await sandbox.createExecutionDirectory(); + const reset = fake.manager.reset; + fake.manager.reset = async () => { throw new Error('proxy shutdown failed'); }; + await assert.rejects(sandbox.executeProgrammatic(request, directory, undefined, { probe: true }), /probe network cleanup failed/); + await assert.rejects(sandbox.execute(request)); + fake.manager.reset = reset; + await sandbox.close(); +}); + +test('programmatic probes use a copy-on-write workspace without mutating the project', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'state.txt'), 'original'); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + const executionDirectory = await sandbox.createExecutionDirectory(); + let snapshot: string; + try { + snapshot = await sandbox.createProgrammaticProbeWorkspace(executionDirectory); + } catch (error) { + if (error instanceof CopyOnWriteCloneUnavailableError) { + t.skip('host filesystem does not support copy-on-write cloning'); + return; + } + throw error; + } + await writeFile(join(snapshot, 'state.txt'), 'probe-only'); + assert.equal(await readFile(join(root, 'state.txt'), 'utf8'), 'original'); + assert.equal(await readFile(join(snapshot, 'state.txt'), 'utf8'), 'probe-only'); +}); + +test('programmatic probes do not hide clone implementation failures as unsupported filesystems', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + spawnCommand() { + throw Object.assign(new Error('spawn /bin/cp ENOENT'), { code: 'ENOENT' }); + }, + }); + t.after(() => sandbox.close()); + const executionDirectory = await sandbox.createExecutionDirectory(); + + await assert.rejects( + sandbox.createProgrammaticProbeWorkspace(executionDirectory), + (error: unknown) => + error instanceof WorkspaceToolError && + !(error instanceof CopyOnWriteCloneUnavailableError) && + error.message === 'Copy-on-write workspace clone failed unexpectedly', + ); +}); + +test('exclusive lifecycle rejects a second workspace sharing an SRT manager', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -164,15 +289,15 @@ test('exclusive lifecycle rejects a second workspace sharing an SRT manager', as assert.equal((await second.execute(request)).stdout, 'hello'); }); -test('exclusive lifecycle rejects overlapping commands and waits before resetting', async (t) => { +test('exclusive lifecycle rejects overlapping commands and waits before resetting', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); let entered!: () => void; - const wrapping = new Promise((resolve) => { + const wrapping = new Promise(resolve => { entered = resolve; }); let release!: () => void; - const gate = new Promise((resolve) => { + const gate = new Promise(resolve => { release = resolve; }); const fake = fakeManager({ @@ -191,7 +316,7 @@ test('exclusive lifecycle rejects overlapping commands and waits before resettin await assert.rejects(sandbox.execute(request), /active command/); const closing = sandbox.close(); const secondClose = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(fake.reset, false); await assert.rejects(sandbox.prepare(), /closing/); release(); @@ -200,7 +325,7 @@ test('exclusive lifecycle rejects overlapping commands and waits before resettin assert.equal(fake.reset, true); }); -test('exclusive lifecycle retains ownership after a failed reset until cleanup succeeds', async (t) => { +test('exclusive lifecycle retains ownership after a failed reset until cleanup succeeds', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -226,16 +351,16 @@ test('exclusive lifecycle retains ownership after a failed reset until cleanup s await second.close(); }); -test('exclusive lifecycle waits for initialization before resetting the manager', async (t) => { +test('exclusive lifecycle waits for initialization before resetting the manager', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); let entered!: () => void; - const initializing = new Promise((resolve) => { + const initializing = new Promise(resolve => { entered = resolve; }); let release!: () => void; - const gate = new Promise((resolve) => { + const gate = new Promise(resolve => { release = resolve; }); fake.manager.initialize = async () => { @@ -249,7 +374,7 @@ test('exclusive lifecycle waits for initialization before resetting the manager' const preparing = sandbox.prepare(); await initializing; const closing = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(fake.reset, false); release(); await preparing; @@ -257,7 +382,7 @@ test('exclusive lifecycle waits for initialization before resetting the manager' assert.equal(fake.reset, true); }); -test('initializes SRT with a default-deny network and scrubbed worker credentials', async (t) => { +test('initializes SRT with a default-deny network and scrubbed worker credentials', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const identity = join(tmpdir(), 'librechat-code-identity.json'); t.after(() => rm(root, { recursive: true, force: true })); @@ -301,7 +426,7 @@ test('initializes SRT with a default-deny network and scrubbed worker credential assert.ok(fake.config?.filesystem.denyRead.includes(canonicalHome)); assert.ok(fake.config?.filesystem.denyWrite.includes(canonicalIdentity)); assert.ok( - fake.config?.filesystem.denyWrite.some((path) => + fake.config?.filesystem.denyWrite.some(path => path.endsWith('/tmp/claude'), ), ); @@ -318,7 +443,7 @@ test('initializes SRT with a default-deny network and scrubbed worker credential await assert.rejects(access(scratchDirectory!)); }); -test('trusted-vm permits unmatched egress and local development sockets', async (t) => { +test('trusted-vm permits unmatched egress and local development sockets', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -351,7 +476,7 @@ test('trusted-vm permits unmatched egress and local development sockets', async ]); }); -test('provides an isolated scratch directory to commands and restores the host environment', async (t) => { +test('provides an isolated scratch directory to commands and restores the host environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const originalTmpdir = process.env.TMPDIR; @@ -380,7 +505,7 @@ test('provides an isolated scratch directory to commands and restores the host e await assert.rejects(access(result.stdout)); }); -test('removes scratch storage when SRT initialization fails', async (t) => { +test('removes scratch storage when SRT initialization fails', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ initializeError: new Error('init failed') }); @@ -396,7 +521,7 @@ test('removes scratch storage when SRT initialization fails', async (t) => { assert.equal(fake.reset, true); }); -test('rejects workspaces nested inside SRT shared scratch storage', async (t) => { +test('rejects workspaces nested inside SRT shared scratch storage', async t => { if (process.platform === 'win32') return; const sharedRoot = '/tmp/claude'; await mkdir(sharedRoot, { recursive: true }); @@ -431,17 +556,17 @@ test('rejects a workspace that contains worker scratch storage', async () => { await sandbox.close(); }); -test('keeps concurrent sandbox scratch directories independent', async (t) => { +test('keeps concurrent sandbox scratch directories independent', async t => { const firstRoot = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const secondRoot = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(firstRoot, { recursive: true, force: true })); t.after(() => rm(secondRoot, { recursive: true, force: true })); let releaseWrap!: () => void; let wrapStarted!: () => void; - const wrapStartedPromise = new Promise((resolve) => { + const wrapStartedPromise = new Promise(resolve => { wrapStarted = resolve; }); - const holdWrap = new Promise((resolve) => { + const holdWrap = new Promise(resolve => { releaseWrap = resolve; }); const firstFake = fakeManager({ @@ -478,7 +603,7 @@ test('keeps concurrent sandbox scratch directories independent', async (t) => { await secondSandbox.close(); }); -test('removes scratch storage after a command revokes traversal permissions', async (t) => { +test('removes scratch storage after a command revokes traversal permissions', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -498,7 +623,7 @@ test('removes scratch storage after a command revokes traversal permissions', as await assert.rejects(access(result.stdout)); }); -test('scratch traversal never follows a descendant replaced after inspection', async (t) => { +test('scratch traversal never follows a descendant replaced after inspection', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-race-')); const outside = await mkdtemp(join(tmpdir(), 'librechat-code-outside-')); @@ -529,7 +654,7 @@ test('scratch traversal never follows a descendant replaced after inspection', a assert.equal((await stat(outsideChild)).mode & 0o777, 0o711); }); -test('scratch traversal removes command-created Darwin ACLs', async (t) => { +test('scratch traversal removes command-created Darwin ACLs', async t => { if (process.platform !== 'darwin') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -548,7 +673,7 @@ test('scratch traversal removes command-created Darwin ACLs', async (t) => { await assert.rejects(access(result.stdout)); }); -test('scratch traversal bounds descriptors and work across a deep tree', async (t) => { +test('scratch traversal bounds descriptors and work across a deep tree', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-depth-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -565,12 +690,17 @@ test('scratch traversal bounds descriptors and work across a deep tree', async ( await restoreScratchTraversal(rootHandle); - assert.equal((await stat(directories[directories.length - 1])).mode & 0o777, 0o700); + assert.equal( + (await stat(directories[directories.length - 1])).mode & 0o777, + 0o700, + ); }); -test('scratch traversal rejects trees beyond its recovery depth limit', async (t) => { +test('scratch traversal rejects trees beyond its recovery depth limit', async t => { if (process.platform === 'win32') return; - const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-depth-limit-')); + const root = await mkdtemp( + join(tmpdir(), 'librechat-code-scratch-depth-limit-'), + ); t.after(() => rm(root, { recursive: true, force: true })); let directory = root; for (let depth = 0; depth < 129; depth += 1) { @@ -586,7 +716,7 @@ test('scratch traversal rejects trees beyond its recovery depth limit', async (t ); }); -test('does not replace scratch state while cleanup remains pending', async (t) => { +test('does not replace scratch state while cleanup remains pending', async t => { if (process.platform === 'win32') return; const workspace = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const retained = await mkdtemp(join(tmpdir(), 'librechat-code-retained-')); @@ -642,7 +772,7 @@ const windowsEnvironment = { }; for (const platform of ['darwin', 'linux', 'win32'] as const) { - test(`preserves required ${platform} environment names without allowing credentials`, async (t) => { + test(`preserves required ${platform} environment names without allowing credentials`, async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -658,23 +788,43 @@ for (const platform of ['darwin', 'linux', 'win32'] as const) { LD_PRELOAD: '/host/private.so', }; const sandbox = new NativeSrtWorkspaceCommandSandbox({ - workspaceRoot: root, platform, allowedDomains: ['github.com'], + workspaceRoot: root, + platform, + allowedDomains: ['github.com'], environment: { - ...proxyEnvironment, ...windowsEnvironment, ...credentials, + ...proxyEnvironment, + ...windowsEnvironment, + ...credentials, HtTp_PrOxY: 'http://mixed-case.invalid:8080', - PATH: '/usr/bin', LC_ALL: 'C.UTF-8', + PATH: '/usr/bin', + LC_ALL: 'C.UTF-8', }, manager: fake.manager, }); t.after(() => sandbox.close()); await sandbox.prepare(); - const denied = new Set(fake.config?.credentials?.envVars - ?.filter(({ mode }) => mode === 'deny').map(({ name }) => name)); - for (const name of [...Object.keys(proxyEnvironment), 'PATH', 'LC_ALL']) { - assert.equal(denied.has(name), false, `${name} must remain available`); + const denied = new Set( + fake.config?.credentials?.envVars + ?.filter(({ mode }) => mode === 'deny') + .map(({ name }) => name), + ); + for (const name of [ + ...Object.keys(proxyEnvironment), + 'PATH', + 'LC_ALL', + ]) { + assert.equal( + denied.has(name), + false, + `${name} must remain available`, + ); } for (const name of Object.keys(windowsEnvironment)) { - assert.equal(denied.has(name), platform !== 'win32', `${name} must be platform-specific`); + assert.equal( + denied.has(name), + platform !== 'win32', + `${name} must be platform-specific`, + ); } assert.equal(denied.has('HtTp_PrOxY'), platform !== 'win32'); for (const name of Object.keys(credentials)) { @@ -685,12 +835,14 @@ for (const platform of ['darwin', 'linux', 'win32'] as const) { }); } -test('uses SRT proxy values without restoring inherited proxies or credentials', async (t) => { +test('uses SRT proxy values without restoring inherited proxies or credentials', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const wrappedEnvironment = { - HTTP_PROXY: 'http://localhost:3128', HTTPS_PROXY: 'http://localhost:3128', - ALL_PROXY: 'http://localhost:3128', NO_PROXY: 'localhost', + HTTP_PROXY: 'http://localhost:3128', + HTTPS_PROXY: 'http://localhost:3128', + ALL_PROXY: 'http://localhost:3128', + NO_PROXY: 'localhost', }; const fake = fakeManager({ wrappedEnvironment }); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -700,14 +852,19 @@ test('uses SRT proxy values without restoring inherited proxies or credentials', }); t.after(() => sandbox.close()); const result = await sandbox.execute({ - ...request, maxOutputBytes: 256, - command: 'printf "%s|%s|%s|%s|%s" "$HTTP_PROXY" "$HTTPS_PROXY" "$ALL_PROXY" "$NO_PROXY" "${GITHUB_TOKEN-unset}"', + ...request, + maxOutputBytes: 256, + command: + 'printf "%s|%s|%s|%s|%s" "$HTTP_PROXY" "$HTTPS_PROXY" "$ALL_PROXY" "$NO_PROXY" "${GITHUB_TOKEN-unset}"', }); assert.equal(result.exitCode, 0); - assert.equal(result.stdout, `${Object.values(wrappedEnvironment).join('|')}|unset`); + assert.equal( + result.stdout, + `${Object.values(wrappedEnvironment).join('|')}|unset`, + ); }); -test('masks a host credential for only its injection host and restores the parent environment', async (t) => { +test('masks a host credential for only its injection host and restores the parent environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -731,7 +888,8 @@ test('masks a host credential for only its injection host and restores the paren ], async resolve() { return { - LIBRECHAT_CODE_TEST_CREDENTIAL: 'Authorization: Bearer real-secret', + LIBRECHAT_CODE_TEST_CREDENTIAL: + 'Authorization: Bearer real-secret', }; }, }, @@ -759,7 +917,7 @@ test('masks a host credential for only its injection host and restores the paren }); }); -test('serializes credential handoff across concurrent sandbox instances', async (t) => { +test('serializes credential handoff across concurrent sandbox instances', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const original = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; @@ -770,11 +928,11 @@ test('serializes credential handoff across concurrent sandbox instances', async else process.env.LIBRECHAT_CODE_TEST_CREDENTIAL = original; }); let firstEntered!: () => void; - const firstEnteredPromise = new Promise((resolve) => { + const firstEnteredPromise = new Promise(resolve => { firstEntered = resolve; }); let releaseFirst!: () => void; - const firstGate = new Promise((resolve) => { + const firstGate = new Promise(resolve => { releaseFirst = resolve; }); let secondEntered = false; @@ -817,7 +975,7 @@ test('serializes credential handoff across concurrent sandbox instances', async const secondExecution = sandbox(second.manager, 'second-secret').execute( request, ); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(secondEntered, false); releaseFirst(); await firstExecution; @@ -828,7 +986,7 @@ test('serializes credential handoff across concurrent sandbox instances', async assert.equal(process.env.LIBRECHAT_CODE_TEST_CREDENTIAL, undefined); }); -test('isolates Git from host-level global and system configuration', async (t) => { +test('isolates Git from host-level global and system configuration', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -844,7 +1002,7 @@ test('isolates Git from host-level global and system configuration', async (t) = assert.equal(result.stdout, '/dev/null|1'); }); -test('restores trusted Git LFS filters without reading host Git configuration', async (t) => { +test('restores trusted Git LFS filters without reading host Git configuration', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ @@ -884,7 +1042,7 @@ test('restores trusted Git LFS filters without reading host Git configuration', assert.ok(!denied?.includes('GIT_CONFIG_VALUE_0')); }); -test('filters environment names case-insensitively only on Windows', async (t) => { +test('filters environment names case-insensitively only on Windows', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -923,7 +1081,7 @@ test('filters environment names case-insensitively only on Windows', async (t) = assert.ok(!denied?.includes('git_config_count')); }); -test('fails closed when the configured POSIX shell is unavailable', async (t) => { +test('fails closed when the configured POSIX shell is unavailable', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -942,7 +1100,7 @@ test('fails closed when the configured POSIX shell is unavailable', async (t) => ); }); -test('fails closed when SRT dependencies are unavailable', async (t) => { +test('fails closed when SRT dependencies are unavailable', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ dependencyErrors: ['bubblewrap missing'] }); @@ -960,7 +1118,7 @@ test('fails closed when SRT dependencies are unavailable', async (t) => { ); }); -test('refuses workspace roots that expose worker home or control files', async (t) => { +test('refuses workspace roots that expose worker home or control files', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const controlDirectory = join(root, '.control'); await mkdir(controlDirectory); @@ -985,7 +1143,7 @@ test('refuses workspace roots that expose worker home or control files', async ( ); }); -test('executes in the canonical workspace and bounds aggregate output', async (t) => { +test('executes in the canonical workspace and bounds aggregate output', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); await mkdir(join(root, 'src')); t.after(() => rm(root, { recursive: true, force: true })); @@ -1015,7 +1173,7 @@ test('executes in the canonical workspace and bounds aggregate output', async (t ); }); -test('rejects an escaping or unavailable command working directory', async (t) => { +test('rejects an escaping or unavailable command working directory', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1026,11 +1184,12 @@ test('rejects an escaping or unavailable command working directory', async (t) = await assert.rejects( sandbox.execute({ ...request, cwd: '..' }), (error: unknown) => - error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + error instanceof WorkspaceToolError && + error.code === 'INVALID_REQUEST', ); }); -test('terminates detached command descendants before returning', async (t) => { +test('terminates detached command descendants before returning', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1043,15 +1202,15 @@ test('terminates detached command descendants before returning', async (t) => { command: '(sleep 0.2; printf late > late.txt) >/dev/null 2>&1 &', }); assert.equal(result.exitCode, 0); - await new Promise((resolve) => setTimeout(resolve, 350)); + await new Promise(resolve => setTimeout(resolve, 350)); await assert.rejects(access(join(root, 'late.txt'))); }); -test('reports cancellation after command start as a potentially committed mutation', async (t) => { +test('reports cancellation after command start as a potentially committed mutation', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); let commandStarted!: () => void; - const commandStartedPromise = new Promise((resolve) => { + const commandStartedPromise = new Promise(resolve => { commandStarted = resolve; }); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1081,7 +1240,7 @@ test('reports cancellation after command start as a potentially committed mutati ); }); -test('closes stdin immediately when the command protocol provides no input', async (t) => { +test('closes stdin immediately when the command protocol provides no input', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1098,7 +1257,7 @@ test('closes stdin immediately when the command protocol provides no input', asy assert.equal(result.timedOut, false); }); -test('maps platform-native exit statuses into the bridge protocol range', async (t) => { +test('maps platform-native exit statuses into the bridge protocol range', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const spawnCommand = () => { @@ -1123,7 +1282,7 @@ test('maps platform-native exit statuses into the bridge protocol range', async assert.equal(result.exitCode, 1); }); -test('cleans allocated command state exactly once on every execution exit', async (t) => { +test('cleans allocated command state exactly once on every execution exit', async t => { for (const outcome of [ 'abort-before-spawn', 'spawn-throw', @@ -1134,8 +1293,12 @@ test('cleans allocated command state exactly once on every execution exit', asyn 'wrap-throw', ] as const) { for (const cleanupThrows of [false, true]) { - await t.test(`${outcome}, cleanup throws: ${cleanupThrows}`, async (t) => { - const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + await t.test( + `${outcome}, cleanup throws: ${cleanupThrows}`, + async t => { + const root = await mkdtemp( + join(tmpdir(), 'librechat-code-native-'), + ); t.after(() => rm(root, { recursive: true, force: true })); const controller = new AbortController(); let cleanupCalls = 0; @@ -1143,9 +1306,11 @@ test('cleans allocated command state exactly once on every execution exit', asyn let allocated = false; const fake = fakeManager({ async beforeWrap() { - if (outcome === 'wrap-throw') throw new Error('wrap failed'); + if (outcome === 'wrap-throw') + throw new Error('wrap failed'); allocated = true; - if (outcome === 'abort-before-spawn') controller.abort(); + if (outcome === 'abort-before-spawn') + controller.abort(); }, }); fake.manager.cleanupAfterCommand = () => { @@ -1160,13 +1325,17 @@ test('cleans allocated command state exactly once on every execution exit', asyn spawnCommand() { spawnCalls += 1; assert.equal(allocated, true); - if (outcome === 'spawn-throw') throw new Error('spawn failed'); - const child = new EventEmitter() as ChildProcessWithoutNullStreams; + if (outcome === 'spawn-throw') + throw new Error('spawn failed'); + const child = + new EventEmitter() as ChildProcessWithoutNullStreams; let closeQueued = false; const close = () => { if (!closeQueued) { closeQueued = true; - queueMicrotask(() => child.emit('close', null, 'SIGKILL')); + queueMicrotask(() => + child.emit('close', null, 'SIGKILL'), + ); } return true; }; @@ -1180,7 +1349,10 @@ test('cleans allocated command state exactly once on every execution exit', asyn queueMicrotask(() => { assert.equal(cleanupCalls, 0); if (outcome === 'error') { - child.emit('error', new Error('spawn failed')); + child.emit( + 'error', + new Error('spawn failed'), + ); } else if (outcome === 'abort-after-spawn') { controller.abort(); } else if (outcome === 'close') { @@ -1196,28 +1368,43 @@ test('cleans allocated command state exactly once on every execution exit', asyn ); if (outcome === 'close' || outcome === 'timeout') { const result = await execution; - assert.equal(result.exitCode, outcome === 'close' ? 0 : null); + assert.equal( + result.exitCode, + outcome === 'close' ? 0 : null, + ); assert.equal(result.timedOut, outcome === 'timeout'); } else { - await assert.rejects(execution, (error: unknown) => + await assert.rejects( + execution, + (error: unknown) => error instanceof WorkspaceToolError && - error.code === (outcome.startsWith('abort') + error.code === + (outcome.startsWith('abort') ? 'EXECUTION_ABORTED' : 'COMMAND_UNAVAILABLE') && - error.mutationMayHaveCommitted === (outcome === 'abort-after-spawn') && + error.mutationMayHaveCommitted === + (outcome === 'abort-after-spawn') && error.requiresQuarantine === - (outcome === 'abort-after-spawn' && process.platform === 'win32'), + (outcome === 'abort-after-spawn' && + process.platform === 'win32'), ); } assert.equal( spawnCalls, - outcome === 'abort-before-spawn' || outcome === 'wrap-throw' ? 0 : 1, + outcome === 'abort-before-spawn' || + outcome === 'wrap-throw' + ? 0 + : 1, + ); + assert.equal( + cleanupCalls, + outcome === 'wrap-throw' ? 0 : 1, ); - assert.equal(cleanupCalls, outcome === 'wrap-throw' ? 0 : 1); assert.equal(allocated, false); await sandbox.close(); assert.equal(fake.reset, true); - }); + }, + ); } } }); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 171ea663..8d150d9f 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -11,14 +11,7 @@ import { sep, } from 'node:path'; import { constants as fsConstants } from 'node:fs'; -import { - access, - mkdtemp, - open, - realpath, - rm, - stat, -} from 'node:fs/promises'; +import { access, mkdtemp, open, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; @@ -141,8 +134,12 @@ interface NativeSandboxManager { cwd?: string, options?: { commandId?: string; commandText?: string }, ): Promise<{ argv: string[]; env: NodeJS.ProcessEnv }>; - annotateStderrWithSandboxFailures(commandId: string, stderr: string): string; + annotateStderrWithSandboxFailures( + commandId: string, + stderr: string, + ): string; cleanupAfterCommand(): void; + updateConfig?(config: SandboxRuntimeConfig): void; reset(): Promise; } @@ -224,13 +221,16 @@ function deniedEnvironmentNames( platform: NodeJS.Platform, ): string[] { return Object.keys(environment) - .filter((name) => { + .filter(name => { const normalized = platform === 'win32' ? name.toUpperCase() : name; return ( normalized.startsWith('LIBRECHAT_CODE_') || (!SAFE_CHILD_ENV_NAMES.has(normalized) && !PROXY_CHILD_ENV_NAMES.has(normalized) && - !(platform === 'win32' && WINDOWS_CHILD_ENV_NAMES.has(normalized)) && + !( + platform === 'win32' && + WINDOWS_CHILD_ENV_NAMES.has(normalized) + ) && !normalized.startsWith('LC_')) ); }) @@ -244,6 +244,36 @@ function normalizedEnvironmentName( return platform === 'win32' ? name.toUpperCase() : name; } +/** Distinguishes an unsupported host filesystem from an implementation fault. */ +export class CopyOnWriteCloneUnavailableError extends WorkspaceToolError { + constructor() { + super( + 'Selected-workspace PTC requires copy-on-write filesystem cloning', + 'COMMAND_UNAVAILABLE', + ); + this.name = 'CopyOnWriteCloneUnavailableError'; + } +} + +function isCopyOnWriteUnsupported( + error: unknown, + platform: NodeJS.Platform, +): boolean { + if (platform === 'win32') return true; + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error.code === 'ENOTSUP' || error.code === 'EOPNOTSUPP') + ) { + return true; + } + return ( + error instanceof Error && + error.message.toLowerCase().includes('operation not supported') + ); +} + export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { readonly mutationFailuresAreAtomic = true as const; private readonly manager: NativeSandboxManager; @@ -252,6 +282,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private readonly platform: NodeJS.Platform; private initialized?: Promise; private canonicalRoot?: string; + private runtimeConfig?: SandboxRuntimeConfig; + private denyReadPaths: string[] = []; + private denyWritePaths: string[] = []; private scratchDirectory?: string; private scratchHandle?: FileHandle; private execution?: Promise; @@ -288,7 +321,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } managerOwners.set(this.manager, this); - this.initialized = this.initializeOnce().catch(async (error) => { + this.initialized = this.initializeOnce().catch(async error => { await this.manager.reset().catch(() => { this.resetFailed = true; }); @@ -314,7 +347,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'COMMAND_UNAVAILABLE', ); } - const home = await canonicalPath(this.options.homeDirectory ?? homedir()); + const home = await canonicalPath( + this.options.homeDirectory ?? homedir(), + ); if (isWithin(root, home)) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain the worker home directory', @@ -324,7 +359,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const protectedPaths = await Promise.all( (this.options.protectedPaths ?? []).map(canonicalPath), ); - if (protectedPaths.some((path) => isWithin(root, path))) { + if (protectedPaths.some(path => isWithin(root, path))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain worker control files', 'REGISTRATION_INVALID', @@ -338,13 +373,16 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const inheritedWritablePaths = [ ...sharedScratchPaths, ...(await Promise.all( - [join(home, '.npm', '_logs'), join(home, '.claude', 'debug')].map( - canonicalPath, - ), + [ + join(home, '.npm', '_logs'), + join(home, '.claude', 'debug'), + ].map(canonicalPath), )), ]; - const deniedInheritedWritablePaths = [...new Set(inheritedWritablePaths)]; - if (deniedInheritedWritablePaths.some((path) => isWithin(path, root))) { + const deniedInheritedWritablePaths = [ + ...new Set(inheritedWritablePaths), + ]; + if (deniedInheritedWritablePaths.some(path => isWithin(path, root))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot be inside an inherited writable path', 'REGISTRATION_INVALID', @@ -359,7 +397,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } if (this.platform !== 'win32') { try { - await access(this.options.shellPath ?? '/bin/bash', fsConstants.X_OK); + await access( + this.options.shellPath ?? '/bin/bash', + fsConstants.X_OK, + ); } catch { throw new WorkspaceToolError( `Native sandbox shell is unavailable: ${this.options.shellPath ?? '/bin/bash'}`, @@ -383,35 +424,40 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); const unrestrictedNetwork = commandPolicy.network.outbound === 'unrestricted'; - const config: SandboxRuntimeConfig = { - network: { + const network: SandboxRuntimeConfig['network'] = { allowedDomains: [...(this.options.allowedDomains ?? [])], deniedDomains: [], strictAllowlist: !unrestrictedNetwork, allowAllUnixSockets: commandPolicy.network.allowAllUnixSockets, allowLocalBinding: commandPolicy.network.allowLocalBinding, ...(this.options.maskedEnvironment ? { tlsTerminate: {} } : {}), - }, + }; + const config: SandboxRuntimeConfig = { + network, filesystem: { denyRead: [ home, - ...sharedScratchPaths.filter((path) => + ...sharedScratchPaths.filter(path => deniedInheritedWritablePaths.includes(path), ), ], allowRead: [ root, - ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), + ...(canonicalScratchDirectory + ? [canonicalScratchDirectory] + : []), ], allowWrite: [ root, - ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), + ...(canonicalScratchDirectory + ? [canonicalScratchDirectory] + : []), ], denyWrite: [...protectedPaths, ...deniedInheritedWritablePaths], allowGitConfig: false, }, credentials: { - files: protectedPaths.map((path) => ({ + files: protectedPaths.map(path => ({ path, mode: 'deny' as const, })), @@ -424,15 +470,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox }, this.platform, ) - .filter((name) => { + .filter(name => { const normalized = normalizedEnvironmentName( name, this.platform, ); return ( - !Object.hasOwn(TRUSTED_GIT_ENVIRONMENT, normalized) && + !Object.hasOwn( + TRUSTED_GIT_ENVIRONMENT, + normalized, + ) && !this.options.maskedEnvironment?.variables.some( - (variable) => + variable => normalizedEnvironmentName( variable.name, this.platform, @@ -440,12 +489,16 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ) ); }) - .map((name) => ({ name, mode: 'deny' as const })), - ...(this.options.maskedEnvironment?.variables.map((variable) => ({ + .map(name => ({ name, mode: 'deny' as const })), + ...(this.options.maskedEnvironment?.variables.map( + variable => ({ ...variable, mode: 'mask' as const, - ...(variable.extract ? { onExtractNoMatch: 'error' as const } : {}), - })) ?? []), + ...(variable.extract + ? { onExtractNoMatch: 'error' as const } + : {}), + }), + ) ?? []), ], }, allowAppleEvents: false, @@ -458,6 +511,17 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox unrestrictedNetwork ? async () => true : undefined, ); this.canonicalRoot = root; + this.runtimeConfig = config; + this.denyReadPaths = [ + home, + ...sharedScratchPaths.filter(path => + deniedInheritedWritablePaths.includes(path), + ), + ]; + this.denyWritePaths = [ + ...protectedPaths, + ...deniedInheritedWritablePaths, + ]; } async execute( @@ -479,9 +543,248 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } } + /** + * Allocate an owner-only execution directory that is already inside this + * sandbox's allowlist. The caller must remove the returned directory after + * the execution settles. It is intentionally unavailable on native Windows + * until the restricted-account TEMP directory can be opened and verified by + * the trusted parent process. + */ + async createExecutionDirectory(): Promise { + await this.initialize(); + if (!this.scratchDirectory || this.platform === 'win32') { + throw new WorkspaceToolError( + 'Native programmatic execution storage is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + return await mkdtemp(join(this.scratchDirectory, 'execution-')); + } + + /** + * Clone the current workspace into private scratch for a side-effect- + * equivalent replay probe. Platform clone flags are intentionally strict: + * silently falling back to a byte copy would make every tool-bearing run + * consume time and disk proportional to the repository size. + */ + async createProgrammaticProbeWorkspace( + executionDirectory: string, + signal?: AbortSignal, + ): Promise { + await this.initialize(); + const scratchDirectory = this.scratchDirectory; + const root = this.canonicalRoot; + let parent: string; + try { + parent = await realpath(executionDirectory); + if ( + !scratchDirectory || + !root || + !isWithin(scratchDirectory, parent) || + !(await stat(parent)).isDirectory() + ) { + throw new Error('invalid execution directory'); + } + } catch { + throw new WorkspaceToolError( + 'Programmatic execution directory is unavailable', + 'INVALID_PATH', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + const destination = join(parent, 'workspace'); + try { + if (this.platform === 'win32') { + throw new Error('copy-on-write cloning is unavailable on Windows'); + } + const args = + this.platform === 'darwin' + ? ['-cR', root, destination] + : ['--archive', '--reflink=always', root, destination]; + await new Promise((resolveCopy, rejectCopy) => { + const child = this.spawnCommand('/bin/cp', args, { + env: { + PATH: this.environment.PATH, + LANG: this.environment.LANG, + LC_ALL: this.environment.LC_ALL, + }, + signal, + }); + let stderr = Buffer.alloc(0); + child.stderr.on('data', (chunk: Buffer) => { + if (stderr.byteLength < 4_096) { + stderr = Buffer.concat([stderr, chunk]).subarray(0, 4_096); + } + }); + child.once('error', rejectCopy); + child.once('close', code => { + if (code === 0) resolveCopy(); + else { + rejectCopy( + new Error( + `copy-on-write clone failed (${code ?? 'signal'}): ${boundedUtf8(stderr, 4_096)}`, + ), + ); + } + }); + }); + return await realpath(destination); + } catch (error) { + await rm(destination, { recursive: true, force: true }).catch( + () => undefined, + ); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (isCopyOnWriteUnsupported(error, this.platform)) { + throw new CopyOnWriteCloneUnavailableError(); + } + throw new WorkspaceToolError( + 'Copy-on-write workspace clone failed unexpectedly', + 'COMMAND_UNAVAILABLE', + ); + } + } + + /** Run a generated program from a verified private execution directory. */ + async executeProgrammatic( + request: WorkspaceExecuteCommandRequest, + dataDirectory: string, + signal?: AbortSignal, + options?: { probe?: boolean; workspaceRoot?: string }, + ): Promise { + if (this.execution || this.closing) { + throw new WorkspaceToolError( + 'Native sandbox already has an active command or is closing', + 'COMMAND_UNAVAILABLE', + ); + } + await this.initialize(); + const scratchDirectory = this.scratchDirectory; + let canonicalDataDirectory: string; + let canonicalWorkspaceRoot: string | undefined; + try { + canonicalDataDirectory = await realpath(dataDirectory); + canonicalWorkspaceRoot = options?.workspaceRoot + ? await realpath(options.workspaceRoot) + : undefined; + if ( + !scratchDirectory || + !isWithin(scratchDirectory, canonicalDataDirectory) || + !(await stat(canonicalDataDirectory)).isDirectory() || + (canonicalWorkspaceRoot != null && + (!isWithin(scratchDirectory, canonicalWorkspaceRoot) || + !(await stat(canonicalWorkspaceRoot)).isDirectory())) + ) { + throw new Error('invalid execution directory'); + } + } catch { + throw new WorkspaceToolError( + 'Programmatic execution directory is unavailable', + 'INVALID_PATH', + ); + } + const execute = () => this.executeExclusive( + request, + signal, + { + LIBRECHAT_CODE_DATA_DIR: canonicalDataDirectory, + LIBRECHAT_CODE_CONTROL_PATH: join( + canonicalDataDirectory, + '_ptc_pending_result.json', + ), + LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', + PTC_HISTORY_PATH: join( + canonicalDataDirectory, + '_ptc_history.json', + ), + TMPDIR: canonicalDataDirectory, + }, + options?.probe + ? { + filesystem: { + allowRead: [ + canonicalWorkspaceRoot ?? this.canonicalRoot!, + canonicalDataDirectory, + ], + allowWrite: [ + ...(canonicalWorkspaceRoot != null + ? [canonicalWorkspaceRoot] + : []), + canonicalDataDirectory, + ], + denyRead: this.denyReadPaths, + denyWrite: [ + this.canonicalRoot!, + ...this.denyWritePaths, + ], + }, + network: { + // A probe is speculative, even on a trusted VM. + // Copy-on-write protects files, not remote mutations. + allowedDomains: [], + deniedDomains: [], + strictAllowlist: true, + allowUnixSockets: [], + allowAllUnixSockets: false, + allowLocalBinding: false, + }, + } + : undefined, + canonicalDataDirectory, + canonicalWorkspaceRoot, + ); + const execution = options?.probe ? this.withProbeNetwork(execute) : execute(); + this.execution = execution; + try { + return await execution; + } finally { + this.execution = undefined; + } + } + + private async withProbeNetwork(execute: () => Promise): Promise { + const config = this.runtimeConfig; + if (!config || !this.manager.updateConfig) { + throw new WorkspaceToolError('Native probe network isolation is unavailable', 'COMMAND_UNAVAILABLE'); + } + // SRT's proxies and Unix/local socket rules read session configuration, + // not wrapWithSandboxArgv's per-command override. + this.manager.updateConfig({ ...config, network: { + allowedDomains: [], deniedDomains: [], strictAllowlist: true, + allowUnixSockets: [], allowAllUnixSockets: false, allowLocalBinding: false, + } }); + try { + return await execute(); + } finally { + try { + // Revoke the probe's proxy endpoints and credentials before restoring + // network access. A lingering probe must never inherit the commit's + // permissive proxy session through a live updateConfig. + await this.manager.reset(); + await this.manager.initialize(config, config.network.strictAllowlist ? undefined : async () => true); + } catch { + this.resetFailed = true; + throw new WorkspaceToolError('Native probe network cleanup failed', 'COMMAND_UNAVAILABLE'); + } + } + } + private async executeExclusive( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, + trustedEnvironment?: NodeJS.ProcessEnv, + customConfig?: Partial, + sandboxScratchDirectory?: string, + workspaceRoot?: string, ): Promise { if ( !isWorkspaceToolRequest(request) || @@ -499,7 +802,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } await this.initialize(); - const root = this.canonicalRoot!; + const root = workspaceRoot ?? this.canonicalRoot!; let cwd: string; try { cwd = await realpath(resolve(root, request.cwd ?? '.')); @@ -528,7 +831,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { ...TRUSTED_GIT_ENVIRONMENT, ...(credentialEnvironment ?? {}), - ...this.scratchSelectorEnvironment(), + ...this.scratchSelectorEnvironment(sandboxScratchDirectory), }, () => this.manager.wrapWithSandboxArgv( @@ -536,7 +839,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.platform === 'win32' ? undefined : (this.options.shellPath ?? '/bin/bash'), - undefined, + customConfig, signal, cwd, { commandId, commandText: request.command }, @@ -561,7 +864,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'EXECUTION_ABORTED', ); } - return await this.runWrapped(request, wrapped, cwd, commandId, signal); + return await this.runWrapped( + request, + wrapped, + cwd, + commandId, + signal, + trustedEnvironment, + ); } finally { // A successful wrap owns command state even when no child is spawned. try { @@ -578,7 +888,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ): Promise { const previousMutation = hostEnvironmentMutationQueue; let releaseMutation!: () => void; - hostEnvironmentMutationQueue = new Promise((resolve) => { + hostEnvironmentMutationQueue = new Promise(resolve => { releaseMutation = resolve; }); await previousMutation; @@ -604,31 +914,41 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox cwd: string, commandId: string, signal?: AbortSignal, + trustedEnvironment?: NodeJS.ProcessEnv, ): Promise { const outputLimit = - request.maxOutputBytes ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; + request.maxOutputBytes ?? + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; const timeoutMs = request.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS; return await new Promise( (resolvePromise, reject) => { let child: ChildProcessWithoutNullStreams; try { - child = this.spawnCommand(wrapped.argv[0], wrapped.argv.slice(1), { + child = this.spawnCommand( + wrapped.argv[0], + wrapped.argv.slice(1), + { cwd, env: { ...wrapped.env, ...this.scratchEnvironment(), + ...trustedEnvironment, ...TRUSTED_GIT_CONFIG_ENTRIES, GIT_CONFIG_COUNT: - wrapped.env.GIT_CONFIG_COUNT ?? TRUSTED_GIT_CONFIG_COUNT, + wrapped.env.GIT_CONFIG_COUNT ?? + TRUSTED_GIT_CONFIG_COUNT, GIT_CONFIG_GLOBAL: - this.platform === 'win32' ? 'NUL' : '/dev/null', + this.platform === 'win32' + ? 'NUL' + : '/dev/null', GIT_CONFIG_NOSYSTEM: '1', }, detached: this.platform !== 'win32', shell: false, windowsHide: true, - }); + }, + ); child.stdin.end(); } catch { reject( @@ -654,10 +974,15 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const accepted = chunk.subarray(0, remaining); target.push(accepted); outputBytes += accepted.byteLength; - if (accepted.byteLength !== chunk.byteLength) truncated = true; + if (accepted.byteLength !== chunk.byteLength) + truncated = true; }; - child.stdout.on('data', (chunk: Buffer) => append(stdout, chunk)); - child.stderr.on('data', (chunk: Buffer) => append(stderr, chunk)); + child.stdout.on('data', (chunk: Buffer) => + append(stdout, chunk), + ); + child.stderr.on('data', (chunk: Buffer) => + append(stderr, chunk), + ); const abort = (): void => { if (settled) return; this.killCommandTree(child); @@ -706,7 +1031,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); return; } - const stdoutValue = boundedUtf8(Buffer.concat(stdout), outputLimit); + const stdoutValue = boundedUtf8( + Buffer.concat(stdout), + outputLimit, + ); const stderrBudget = Math.max( 0, outputLimit - Buffer.byteLength(stdoutValue), @@ -714,7 +1042,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const rawStderr = Buffer.concat(stderr).toString('utf8'); let annotatedStderr = rawStderr; try { - annotatedStderr = this.manager.annotateStderrWithSandboxFailures( + annotatedStderr = + this.manager.annotateStderrWithSandboxFailures( commandId, rawStderr, ); @@ -730,12 +1059,15 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox operation: 'execute_command', workspaceId: request.workspaceId, exitCode: - timedOut || childSignal ? null : this.protocolExitCode(code), + timedOut || childSignal + ? null + : this.protocolExitCode(code), ...(childSignal ? { signal: childSignal } : {}), stdout: stdoutValue, stderr: stderrValue, truncated: - truncated || Buffer.byteLength(annotatedStderr) > stderrBudget, + truncated || + Buffer.byteLength(annotatedStderr) > stderrBudget, timedOut, }); }); @@ -775,7 +1107,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const canonicalTemporaryRoot = await canonicalPath(HOST_TEMPORARY_ROOT); - const sharedScratchRoot = sharedScratchPaths.find((path) => + const sharedScratchRoot = sharedScratchPaths.find(path => isWithin(path, canonicalTemporaryRoot), ); const scratchDirectory = await mkdtemp( @@ -798,7 +1130,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox true, ); if (((await scratchHandle.stat()).mode & 0o777) !== 0o700) { - throw new Error('Native sandbox scratch directory is not private'); + throw new Error( + 'Native sandbox scratch directory is not private', + ); } this.scratchHandle = scratchHandle; } catch (error) { @@ -829,11 +1163,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : { TMPDIR: scratchDirectory }; } - private scratchSelectorEnvironment(): NodeJS.ProcessEnv { - const scratchDirectory = this.scratchDirectory; + private scratchSelectorEnvironment( + selectedDirectory = this.scratchDirectory, + ): NodeJS.ProcessEnv { + const scratchDirectory = selectedDirectory; if (!scratchDirectory) return {}; return Object.fromEntries( - SRT_SCRATCH_SELECTOR_NAMES.map((name) => [name, scratchDirectory]), + SRT_SCRATCH_SELECTOR_NAMES.map(name => [name, scratchDirectory]), ); } diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index fd426783..08cba1c7 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -1,8 +1,11 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + bridgeArtifactMediaType, bridgeWorkerPath, comparePortableRelativePaths, + isBridgeWorkspaceProgrammaticRequest, + isSupportedBridgeArtifactName, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, isWorkspaceToolRequest, @@ -13,6 +16,23 @@ import type { WorkspacePreviewEditRequest, } from './protocol.js'; +test('accepts gateway directory markers as artifacts', () => { + assert.equal(isSupportedBridgeArtifactName('.dirkeep'), true); + assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep'), true); + assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep.exe'), false); +}); + +test('rejects caller-supplied programmatic control payloads', () => { + for (const name of ['_ptc_pending_result.json', '_PTC_PENDING_RESULT.JSON', 'nested/_ptc_pending_result.json']) { + assert.equal(isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { language: 'bash', version: '5.2.0', session_id: 'session', files: [ + { name: 'main.sh', content: 'true' }, { name, content: '{}' }, + ] }, + }), false); + } +}); + const validSingleEditRequest: WorkspaceEditFileRequest = { protocolVersion: 1, operation: 'edit_file', @@ -592,3 +612,167 @@ test('workspace capabilities allow per-workspace operation restrictions', () => false, ); }); + +test('workspace programmatic capability is closed to Bash command roots', () => { + const workspaceTools = { + protocolVersion: 1, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'project-a' }], + }; + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools, + }), + true, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { ...workspaceTools, operations: ['read_file'] }, + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { ...workspaceTools, programmaticLanguages: ['python'] }, + }), + false, + ); +}); + +test('workspace programmatic requests accept only stable input cache identities', () => { + const request = { + headers: {}, + body: { + language: 'bash', + version: '5.2', + execution_id: 'execution_1', + replay_tool_count: 2, + max_output_files: 50, + max_output_file_bytes: 10_000_000, + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { + name: 'skills/example.txt', + id: 'file-1', + storage_session_id: 'storage-1', + input_cache_key: 'a'.repeat(64), + }, + ], + }, + }; + assert.equal(isBridgeWorkspaceProgrammaticRequest(request), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + ...request, + body: { + ...request.body, + files: [request.body.files[0], { ...request.body.files[1], input_cache_key: '../cache' }], + }, + }), + false, + ); + for (const body of [ + { ...request.body, execution_id: '../execution' }, + { ...request.body, replay_tool_count: -1 }, + { ...request.body, replay_tool_count: 257 }, + { ...request.body, max_output_files: -1 }, + { ...request.body, max_output_files: 101 }, + { ...request.body, max_output_file_bytes: 0 }, + { ...request.body, max_output_file_bytes: 10 * 1024 * 1024 + 1 }, + ]) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ ...request, body }), + false, + ); + } +}); + +test('workspace programmatic history can use the bounded replay aggregate budget', () => { + const history = 'h'.repeat(10 * 1024 * 1024 + 1); + const body = { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { name: '_ptc_history.json', content: history }, + ], + }; + assert.equal(isBridgeWorkspaceProgrammaticRequest({ headers: {}, body }), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + ...body, + files: [ + { name: 'main.sh', content: history }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }), + false, + ); +}); + +test('workspace programmatic requests reject non-canonical file paths', () => { + for (const name of ['./main.sh', 'scripts//main.sh', 'scripts/./main.sh', '.']) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { name, content: 'data' }, + ], + }, + }), + false, + name, + ); + } +}); + +test('workspace programmatic requests reject ancestor-descendant input conflicts', () => { + for (const names of [ + ['main.sh', 'main.sh/data.txt'], + ['main.sh', 'assets', 'assets/logo.png'], + ['main.sh', 'deep/path/file.txt', 'deep'], + ]) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: names.map(name => ({ name, content: 'data' })), + }, + }), + false, + names.join(', '), + ); + } +}); + +test('bridge artifact policy and media types match the hardened gateway contract', () => { + assert.equal(isSupportedBridgeArtifactName('reports/result.json'), true); + assert.equal(isSupportedBridgeArtifactName('preview.png'), true); + assert.equal(isSupportedBridgeArtifactName('model.bin'), false); + assert.equal(bridgeArtifactMediaType('preview.png'), 'image/png'); + assert.equal(bridgeArtifactMediaType('reports/result.json'), 'application/json'); + assert.equal(bridgeArtifactMediaType('Dockerfile'), 'application/octet-stream'); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c1dad949..b92d21ac 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -20,9 +20,144 @@ export const BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS = 5 * 60_000; export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES = 100; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY = 4; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; + +/** Reserve a bounded share for all input/output batches, not per-file grants. */ +export function programmaticTransferReserveMs(jobTimeoutMs: number): number { + return Math.max(1, Math.floor(jobTimeoutMs / 3)); +} +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES = 10 * 1024 * 1024; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_HISTORY_BYTES = 40_000_000; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES = 100 * 1024 * 1024; /** How long Code API drains a clean rejection after Stop cancels a workspace mutation. */ export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; +/** + * Artifact names accepted by the hardened egress gateway. Keep this policy in + * the bridge protocol package so a remote worker can reject unsupported output + * locally instead of discovering the mismatch only after mutating a workspace. + */ +const BRIDGE_ARTIFACT_EXTENSIONS = new Set([ + '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', + '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', + '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', + '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', + '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', + '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', + '.xml', '.yaml', '.yml', + '.ics', '.ical', '.ifb', '.icalendar', + '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', + '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', + '.odt', '.ods', '.odp', '.rtf', + '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', + '.tif', '.tiff', '.webp', + '.eot', '.ttf', '.woff', '.woff2', + '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', + '.tf', '.tfvars', '.tfstate', '.hcl', + '.dockerfile', '.Dockerfile', '.dockerignore', + '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', + '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', + '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', +]); + +function portableBasename(name: string): string { + return name.slice(name.lastIndexOf('/') + 1); +} + +/** Apply the gateway's extension allowlist without importing service code. */ +export function isSupportedBridgeArtifactName(name: string): boolean { + const basename = portableBasename(name); + if (basename === '.dirkeep') return true; + const dot = basename.lastIndexOf('.'); + const extension = dot > 0 ? basename.slice(dot).toLowerCase() : ''; + const dottedBasename = `.${basename}`; + return ( + (extension !== '' && BRIDGE_ARTIFACT_EXTENSIONS.has(extension)) || + BRIDGE_ARTIFACT_EXTENSIONS.has(basename) || + BRIDGE_ARTIFACT_EXTENSIONS.has(basename.toLowerCase()) || + (extension === '' && + (BRIDGE_ARTIFACT_EXTENSIONS.has(dottedBasename) || + BRIDGE_ARTIFACT_EXTENSIONS.has(dottedBasename.toLowerCase()))) + ); +} + +const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { + '.avif': 'image/avif', + '.bmp': 'image/bmp', + '.bz2': 'application/x-bzip2', + '.c': 'text/x-c', + '.conf': 'text/plain', + '.cpp': 'text/x-c++src', + '.css': 'text/css', + '.csv': 'text/csv', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.gif': 'image/gif', + '.gz': 'application/gzip', + '.gzip': 'application/gzip', + '.htm': 'text/html', + '.html': 'text/html', + '.ico': 'image/x-icon', + '.ics': 'text/calendar', + '.ifb': 'text/calendar', + '.ical': 'text/calendar', + '.icalendar': 'text/calendar', + '.ini': 'text/plain', + '.java': 'text/x-java-source', + '.jpeg': 'image/jpeg', + '.jpg': 'image/jpeg', + '.js': 'text/javascript', + '.json': 'application/json', + '.json5': 'application/json5', + '.jsonl': 'application/x-ndjson', + '.jsx': 'text/jsx', + '.log': 'text/plain', + '.md': 'text/markdown', + '.odt': 'application/vnd.oasis.opendocument.text', + '.ods': 'application/vnd.oasis.opendocument.spreadsheet', + '.odp': 'application/vnd.oasis.opendocument.presentation', + '.parquet': 'application/vnd.apache.parquet', + '.pdf': 'application/pdf', + '.png': 'image/png', + '.ppt': 'application/vnd.ms-powerpoint', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.py': 'text/x-python', + '.rst': 'text/x-rst', + '.rtf': 'application/rtf', + '.sh': 'application/x-sh', + '.sql': 'application/sql', + '.svg': 'image/svg+xml', + '.tar': 'application/x-tar', + '.tex': 'application/x-tex', + '.tif': 'image/tiff', + '.tiff': 'image/tiff', + '.toml': 'application/toml', + '.ts': 'text/typescript', + '.tsx': 'text/tsx', + '.tsv': 'text/tab-separated-values', + '.txt': 'text/plain', + '.webp': 'image/webp', + '.woff': 'font/woff', + '.woff2': 'font/woff2', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.xml': 'application/xml', + '.yaml': 'application/yaml', + '.yml': 'application/yaml', + '.zip': 'application/zip', +}; + +/** Infer a safe response media type from an already-validated artifact name. */ +export function bridgeArtifactMediaType(name: string): string { + const basename = portableBasename(name).toLowerCase(); + const dot = basename.lastIndexOf('.'); + const extension = dot > 0 ? basename.slice(dot) : basename; + return BRIDGE_ARTIFACT_MEDIA_TYPES[extension] ?? 'application/octet-stream'; +} + export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; export type BridgeWorkspaceToolOperation = @@ -38,6 +173,7 @@ export type WorkspaceWriteFileMode = 'replace' | 'create'; export type WorkspaceEditFileMode = 'single' | 'batch'; export type WorkspaceEditFileFeature = 'expected_base_sha256'; export type WorkspaceListFileFeature = 'after_path'; +export type WorkspaceProgrammaticLanguage = 'bash'; export interface BridgeWorkspaceDescriptor { id: string; @@ -58,6 +194,8 @@ export interface BridgeWorkspaceToolCapabilities { editFileFeatures?: WorkspaceEditFileFeature[]; /** Omitted by workers that cannot continue a bounded file listing. */ listFileFeatures?: WorkspaceListFileFeature[]; + /** Languages that can execute PTC replay inside a selected workspace. */ + programmaticLanguages?: WorkspaceProgrammaticLanguage[]; } export interface WorkspaceReadFileRequest { @@ -153,8 +291,7 @@ interface WorkspaceEditFileRequestBase { expectedBaseSha256?: string; } -export interface WorkspaceSingleEditFileRequest - extends WorkspaceEditFileRequestBase { +export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequestBase { /** Legacy single-edit form. */ oldText: string; /** Legacy single-edit form. */ @@ -162,8 +299,7 @@ export interface WorkspaceSingleEditFileRequest edits?: never; } -export interface WorkspaceBatchEditFileRequest - extends WorkspaceEditFileRequestBase { +export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestBase { /** Ordered exact replacements applied atomically as one file mutation. */ edits: WorkspaceTextEdit[]; oldText?: never; @@ -171,8 +307,7 @@ export interface WorkspaceBatchEditFileRequest } export type WorkspaceEditFileRequest = - | WorkspaceSingleEditFileRequest - | WorkspaceBatchEditFileRequest; + WorkspaceSingleEditFileRequest | WorkspaceBatchEditFileRequest; export interface WorkspaceTextEdit { oldText: string; @@ -195,23 +330,20 @@ interface WorkspacePreviewEditRequestBase { path: string; } -export interface WorkspaceSinglePreviewEditRequest - extends WorkspacePreviewEditRequestBase { +export interface WorkspaceSinglePreviewEditRequest extends WorkspacePreviewEditRequestBase { oldText: string; newText: string; edits?: never; } -export interface WorkspaceBatchPreviewEditRequest - extends WorkspacePreviewEditRequestBase { +export interface WorkspaceBatchPreviewEditRequest extends WorkspacePreviewEditRequestBase { edits: WorkspaceTextEdit[]; oldText?: never; newText?: never; } export type WorkspacePreviewEditRequest = - | WorkspaceSinglePreviewEditRequest - | WorkspaceBatchPreviewEditRequest; + WorkspaceSinglePreviewEditRequest | WorkspaceBatchPreviewEditRequest; export interface WorkspacePreviewEditResult { protocolVersion: BridgeProtocolVersion; @@ -392,12 +524,7 @@ const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'truncated', 'timedOut', ]); -const WORKSPACE_SEARCH_MATCH_KEYS = new Set([ - 'path', - 'line', - 'column', - 'text', -]); +const WORKSPACE_SEARCH_MATCH_KEYS = new Set(['path', 'line', 'column', 'text']); export interface BridgeWorkerCapabilities { /** Opt-in protocol: maximum concurrently leased independent workspace roots. */ @@ -437,6 +564,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceEditFileFeatures?: WorkspaceEditFileFeature[]; /** Listing features this Code API can safely route to a capability-aware worker. */ supportedWorkspaceListFileFeatures?: WorkspaceListFileFeature[]; + /** PTC languages this Code API can safely route into a selected workspace. */ + supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -469,6 +598,37 @@ export interface BridgeSandboxRequest { headers: Record; } +export type BridgeProgrammaticPayloadFile = + | { name: string; content: string } + | { + name: string; + id: string; + storage_session_id: string; + input_cache_key?: string; + }; + +export interface BridgeWorkspaceProgrammaticBody { + language: 'bash'; + version: string; + /** Stable identity shared by every replay iteration of one execution. */ + execution_id?: string; + /** Declared replay tools; zero allows the worker to skip the probe pass. */ + replay_tool_count?: number; + run_timeout?: number; + transfer_timeout_ms?: number; + /** Manifest-bound upload ceiling negotiated by Code API. */ + max_output_files?: number; + /** Effective per-file upload ceiling negotiated by Code API. */ + max_output_file_bytes?: number; + files: BridgeProgrammaticPayloadFile[]; + session_id: string; + output_session_id?: string; + egress_grant?: string; +} + +export type BridgeWorkspaceProgrammaticRequest = + BridgeSandboxRequest; + export interface BridgeAssignment { workspaceLeaseSlot?: number; protocolVersion: BridgeProtocolVersion; @@ -481,7 +641,9 @@ export interface BridgeAssignment { /** Server-calculated execution budget at lease time; avoids VM clock skew. */ remainingMs?: number; runtimeSessionId?: string; - executionKind?: 'sandbox' | 'workspace_tool'; + executionKind?: 'sandbox' | 'workspace_tool' | 'workspace_programmatic'; + /** Selected workspace for workspace-scoped programmatic execution. */ + workspaceId?: string; request: BridgeSandboxRequest | WorkspaceToolRequest; } @@ -589,6 +751,140 @@ export function isValidBridgeWorkerId(workerId: string): boolean { return BRIDGE_WORKER_ID_PATTERN.test(workerId); } +export function isBridgeWorkspaceProgrammaticRequest( + value: unknown, +): value is BridgeWorkspaceProgrammaticRequest { + if (typeof value !== 'object' || value === null) return false; + const request = value as Record; + if ( + typeof request.headers !== 'object' || + request.headers === null || + !Object.values(request.headers).every( + entry => typeof entry === 'string', + ) || + typeof request.body !== 'object' || + request.body === null + ) { + return false; + } + const body = request.body as Record; + if ( + body.language !== 'bash' || + typeof body.version !== 'string' || + body.version.length === 0 || + body.version.length > BRIDGE_RUNTIME_MAX_LENGTH || + (body.execution_id !== undefined && + (typeof body.execution_id !== 'string' || + !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || + (body.replay_tool_count !== undefined && + (!Number.isSafeInteger(body.replay_tool_count) || + Number(body.replay_tool_count) < 0 || + Number(body.replay_tool_count) > 256)) || + (body.max_output_files !== undefined && + (!Number.isSafeInteger(body.max_output_files) || + Number(body.max_output_files) < 0 || + Number(body.max_output_files) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES)) || + (body.max_output_file_bytes !== undefined && + (!Number.isSafeInteger(body.max_output_file_bytes) || + Number(body.max_output_file_bytes) < 1 || + Number(body.max_output_file_bytes) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES)) || + typeof body.session_id !== 'string' || + body.session_id.length === 0 || + body.session_id.length > 32_768 || + /[\0\r\n]/.test(body.session_id) || + (body.output_session_id !== undefined && + (typeof body.output_session_id !== 'string' || + body.output_session_id.length === 0 || + body.output_session_id.length > 32_768 || + /[\0\r\n]/.test(body.output_session_id))) || + (body.egress_grant !== undefined && + (typeof body.egress_grant !== 'string' || + body.egress_grant.length === 0 || + body.egress_grant.length > 256 * 1024)) || + (body.transfer_timeout_ms !== undefined && + (!Number.isSafeInteger(body.transfer_timeout_ms) || + Number(body.transfer_timeout_ms) < 1 || + Number(body.transfer_timeout_ms) > BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || + (body.run_timeout !== undefined && + (!Number.isSafeInteger(body.run_timeout) || + Number(body.run_timeout) < 1 || + Number(body.run_timeout) > + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS)) || + !Array.isArray(body.files) || + body.files.length < 1 || + body.files.length > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES + ) { + return false; + } + let inlineBytes = 0; + const names = new Set(); + for (const rawFile of body.files) { + if (typeof rawFile !== 'object' || rawFile === null) return false; + const file = rawFile as Record; + if ( + !isSafePortableRelativePath(file.name) || + file.name === '.' || + portableBasename(file.name).toLowerCase() === '_ptc_pending_result.json' || + normalizePortableRelativePath(file.name) !== file.name || + names.has(file.name) + ) { + return false; + } + names.add(file.name); + if (typeof file.content === 'string') { + inlineBytes += Buffer.byteLength(file.content); + if ( + Object.keys(file).some( + key => key !== 'name' && key !== 'content', + ) || + Buffer.byteLength(file.content) > + (file.name === '_ptc_history.json' + ? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_HISTORY_BYTES + : BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) + ) { + return false; + } + continue; + } + if ( + typeof file.id !== 'string' || + file.id.length === 0 || + file.id.length > 32_768 || + /[\0\r\n]/.test(file.id) || + typeof file.storage_session_id !== 'string' || + file.storage_session_id.length === 0 || + file.storage_session_id.length > 32_768 || + /[\0\r\n]/.test(file.storage_session_id) || + Object.keys(file).some( + key => + key !== 'name' && + key !== 'id' && + key !== 'storage_session_id' && + key !== 'input_cache_key', + ) || + (file.input_cache_key !== undefined && + (typeof file.input_cache_key !== 'string' || + !/^[a-f0-9]{64}$/.test(file.input_cache_key))) + ) { + return false; + } + } + for (const name of names) { + const segments = name.split('/'); + let ancestor = ''; + for (let index = 0; index < segments.length - 1; index += 1) { + ancestor = ancestor ? `${ancestor}/${segments[index]}` : segments[index]!; + if (names.has(ancestor)) return false; + } + } + return ( + names.has('main.sh') && + inlineBytes <= BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ); +} + export function isSafePortableRelativePath(value: unknown): value is string { if ( typeof value !== 'string' || @@ -602,20 +898,23 @@ export function isSafePortableRelativePath(value: unknown): value is string { ) { return false; } - return value.split('/').every((segment) => segment !== '..'); + return value.split('/').every(segment => segment !== '..'); } function normalizePortableRelativePath(value: string): string { return ( value .split('/') - .filter((segment) => segment.length > 0 && segment !== '.') + .filter(segment => segment.length > 0 && segment !== '.') .join('/') || '.' ); } /** Compare path segments in ripgrep's sorted, depth-first traversal order. */ -export function comparePortableRelativePaths(left: string, right: string): number { +export function comparePortableRelativePaths( + left: string, + right: string, +): number { const encoder = new TextEncoder(); const leftSegments = left.split('/'); const rightSegments = right.split('/'); @@ -645,9 +944,14 @@ function isWithinRequestedPath(candidate: string, requested?: string): boolean { ); } -function isValidWorkspaceEditRequest(request: Record): boolean { +function isValidWorkspaceEditRequest( + request: Record, +): boolean { const hasBatch = request.edits !== undefined; - if (hasBatch && (request.oldText !== undefined || request.newText !== undefined)) { + if ( + hasBatch && + (request.oldText !== undefined || request.newText !== undefined) + ) { return false; } const edits = hasBatch @@ -665,7 +969,10 @@ function isValidWorkspaceEditRequest(request: Record): boolean if ( typeof edit !== 'object' || edit === null || - !hasOnlyKeys(edit as Record, WORKSPACE_TEXT_EDIT_KEYS) + !hasOnlyKeys( + edit as Record, + WORKSPACE_TEXT_EDIT_KEYS, + ) ) { return false; } @@ -673,9 +980,11 @@ function isValidWorkspaceEditRequest(request: Record): boolean if ( typeof candidate.oldText !== 'string' || candidate.oldText.length === 0 || - Buffer.from(candidate.oldText).toString('utf8') !== candidate.oldText || + Buffer.from(candidate.oldText).toString('utf8') !== + candidate.oldText || typeof candidate.newText !== 'string' || - Buffer.from(candidate.newText).toString('utf8') !== candidate.newText + Buffer.from(candidate.newText).toString('utf8') !== + candidate.newText ) { return false; } @@ -698,7 +1007,7 @@ function hasOnlyKeys( value: Record, allowed: ReadonlySet, ): boolean { - return Object.keys(value).every((key) => allowed.has(key)); + return Object.keys(value).every(key => allowed.has(key)); } export function isWorkspaceToolRequest( @@ -723,7 +1032,8 @@ export function isWorkspaceToolRequest( (request.maxLines === undefined || (Number.isSafeInteger(request.maxLines) && Number(request.maxLines) >= 1 && - Number(request.maxLines) <= BRIDGE_WORKSPACE_READ_MAX_LINES)) + Number(request.maxLines) <= + BRIDGE_WORKSPACE_READ_MAX_LINES)) ); } if (request.operation === 'search_text') { @@ -743,7 +1053,8 @@ export function isWorkspaceToolRequest( (request.maxResults === undefined || (Number.isSafeInteger(request.maxResults) && Number(request.maxResults) >= 1 && - Number(request.maxResults) <= BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS)) + Number(request.maxResults) <= + BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS)) ); } if (request.operation === 'list_files') { @@ -753,12 +1064,14 @@ export function isWorkspaceToolRequest( isSafePortableRelativePath(request.path)) && (request.afterPath === undefined || (isSafePortableRelativePath(request.afterPath) && - normalizePortableRelativePath(request.afterPath) === request.afterPath && + normalizePortableRelativePath(request.afterPath) === + request.afterPath && isWithinRequestedPath(request.afterPath, request.path))) && (request.maxResults === undefined || (Number.isSafeInteger(request.maxResults) && Number(request.maxResults) >= 1 && - Number(request.maxResults) <= BRIDGE_WORKSPACE_LIST_MAX_RESULTS)) + Number(request.maxResults) <= + BRIDGE_WORKSPACE_LIST_MAX_RESULTS)) ); } if (request.operation === 'write_file') { @@ -799,7 +1112,8 @@ export function isWorkspaceToolRequest( !request.command.includes('\0') && new TextEncoder().encode(request.command).byteLength <= BRIDGE_WORKSPACE_COMMAND_MAX_BYTES && - (request.cwd === undefined || isSafePortableRelativePath(request.cwd)) && + (request.cwd === undefined || + isSafePortableRelativePath(request.cwd)) && (request.timeoutMs === undefined || (Number.isSafeInteger(request.timeoutMs) && Number(request.timeoutMs) >= 1 && @@ -838,13 +1152,19 @@ export function isWorkspaceToolResult( if (request.operation === 'read_file') { const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; - const content = typeof result.content === 'string' ? result.content : null; + const content = + typeof result.content === 'string' ? result.content : null; const reportedLineCount = - Number.isSafeInteger(result.endLine) && Number(result.endLine) >= startLine - 1 + Number.isSafeInteger(result.endLine) && + Number(result.endLine) >= startLine - 1 ? Number(result.endLine) - startLine + 1 : -1; const actualLineCount = - content === null ? -1 : content.length === 0 ? reportedLineCount : content.split('\n').length; + content === null + ? -1 + : content.length === 0 + ? reportedLineCount + : content.split('\n').length; return ( hasOnlyKeys(result, WORKSPACE_READ_RESULT_KEYS) && result.path === request.path && @@ -899,7 +1219,10 @@ export function isWorkspaceToolResult( (enforcesPaginationContract && (normalizedPath !== path || (previousPath !== undefined && - comparePortableRelativePaths(normalizedPath, previousPath) <= 0))) + comparePortableRelativePaths( + normalizedPath, + previousPath, + ) <= 0))) ) { return false; } @@ -909,7 +1232,8 @@ export function isWorkspaceToolResult( if (!enforcesPaginationContract) { return result.nextAfterPath === undefined; } - if (result.truncated !== true) return result.nextAfterPath === undefined; + if (result.truncated !== true) + return result.nextAfterPath === undefined; return ( result.paths.length > 0 && result.nextAfterPath === result.paths[result.paths.length - 1] @@ -942,7 +1266,8 @@ export function isWorkspaceToolResult( if (request.operation === 'preview_edit') { const replacements = request.edits?.length ?? 1; - const content = typeof result.content === 'string' ? result.content : null; + const content = + typeof result.content === 'string' ? result.content : null; return ( hasOnlyKeys(result, WORKSPACE_PREVIEW_EDIT_RESULT_KEYS) && result.path === request.path && @@ -964,7 +1289,8 @@ export function isWorkspaceToolResult( const stdout = typeof result.stdout === 'string' ? result.stdout : null; const stderr = typeof result.stderr === 'string' ? result.stderr : null; const outputLimit = - request.maxOutputBytes ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; + request.maxOutputBytes ?? + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; return ( hasOnlyKeys(result, WORKSPACE_COMMAND_RESULT_KEYS) && stdout !== null && @@ -980,7 +1306,8 @@ export function isWorkspaceToolResult( Number(result.exitCode) <= 255)) && (result.signal === undefined || (typeof result.signal === 'string' && - result.signal.length <= BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH && + result.signal.length <= + BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH && /^SIG[A-Z0-9]+$/.test(result.signal))) && typeof result.truncated === 'boolean' && typeof result.timedOut === 'boolean' && @@ -995,7 +1322,7 @@ export function isWorkspaceToolResult( return ( hasOnlyKeys(result, WORKSPACE_SEARCH_RESULT_KEYS) && result.matches.length <= maxResults && - result.matches.every((match) => { + result.matches.every(match => { if (typeof match !== 'object' || match === null) return false; const candidate = match as Record; return ( @@ -1007,7 +1334,8 @@ export function isWorkspaceToolResult( Number.isSafeInteger(candidate.column) && Number(candidate.column) >= 1 && typeof candidate.text === 'string' && - candidate.text.length <= BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH && + candidate.text.length <= + BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH && candidate.text.includes(request.query) ); }) @@ -1025,7 +1353,7 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.operations.length < 1 || capabilities.operations.length > 7 || !capabilities.operations.every( - (operation) => + operation => operation === 'read_file' || operation === 'search_text' || operation === 'list_files' || @@ -1034,7 +1362,8 @@ export function isValidBridgeWorkspaceToolCapabilities( operation === 'edit_file' || operation === 'execute_command', ) || - new Set(capabilities.operations).size !== capabilities.operations.length || + new Set(capabilities.operations).size !== + capabilities.operations.length || !Array.isArray(capabilities.workspaces) || capabilities.workspaces.length < 1 || capabilities.workspaces.length > BRIDGE_WORKSPACE_MAX_COUNT @@ -1049,7 +1378,7 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.writeFileModes.length > 2 || !capabilities.operations.includes('write_file') || !capabilities.writeFileModes.every( - (mode) => mode === 'replace' || mode === 'create', + mode => mode === 'replace' || mode === 'create', ) || new Set(capabilities.writeFileModes).size !== capabilities.writeFileModes.length) @@ -1065,7 +1394,7 @@ export function isValidBridgeWorkspaceToolCapabilities( (!capabilities.operations.includes('edit_file') && !capabilities.operations.includes('preview_edit')) || !capabilities.editFileModes.every( - (mode) => mode === 'single' || mode === 'batch', + mode => mode === 'single' || mode === 'batch', ) || new Set(capabilities.editFileModes).size !== capabilities.editFileModes.length) @@ -1093,13 +1422,23 @@ export function isValidBridgeWorkspaceToolCapabilities( return false; } + if ( + capabilities.programmaticLanguages !== undefined && + (!Array.isArray(capabilities.programmaticLanguages) || + capabilities.programmaticLanguages.length !== 1 || + !capabilities.operations.includes('execute_command') || + capabilities.programmaticLanguages[0] !== 'bash') + ) { + return false; + } + const workspaceIds = new Set(); - return capabilities.workspaces.every((workspace) => { + return capabilities.workspaces.every(workspace => { if (typeof workspace !== 'object' || workspace === null) return false; const descriptor = workspace as Record; if ( Object.keys(descriptor).some( - (key) => key !== 'id' && key !== 'name' && key !== 'operations', + key => key !== 'id' && key !== 'name' && key !== 'operations', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || @@ -1107,17 +1446,21 @@ export function isValidBridgeWorkspaceToolCapabilities( (descriptor.name !== undefined && (typeof descriptor.name !== 'string' || descriptor.name.trim().length === 0 || - descriptor.name.length > BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) || + descriptor.name.length > + BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) || (descriptor.operations !== undefined && (!Array.isArray(descriptor.operations) || descriptor.operations.length < 1 || descriptor.operations.length > (capabilities.operations as unknown[]).length || descriptor.operations.some( - (operation) => - !(capabilities.operations as unknown[]).includes(operation), + operation => + !(capabilities.operations as unknown[]).includes( + operation, + ), ) || - new Set(descriptor.operations).size !== descriptor.operations.length)) + new Set(descriptor.operations).size !== + descriptor.operations.length)) ) { return false; } @@ -1139,11 +1482,12 @@ export function isValidBridgeWorkerCapabilities( typeof capabilities.statefulWorkspace === 'boolean' && typeof capabilities.sandboxProfile === 'string' && capabilities.sandboxProfile.trim().length > 0 && - capabilities.sandboxProfile.length <= BRIDGE_SANDBOX_PROFILE_MAX_LENGTH && + capabilities.sandboxProfile.length <= + BRIDGE_SANDBOX_PROFILE_MAX_LENGTH && Array.isArray(capabilities.runtimes) && capabilities.runtimes.length <= BRIDGE_RUNTIME_MAX_COUNT && capabilities.runtimes.every( - (runtime) => + runtime => typeof runtime === 'string' && runtime.length > 0 && runtime.length <= BRIDGE_RUNTIME_MAX_LENGTH, diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index ba65a3cf..6e8cc8b1 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -312,3 +312,51 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b assert.equal(executed, true); assert.equal(internals.activeWorkspaceAssignments.size, 0); }); + +test('programmatic work on an independent workspace bypasses another root cleanup', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + const internals = worker as unknown as { + activeWorkspaceAssignments: Map< + string, + { id: string; done: Promise } + >; + executeOwned: (assignment: BridgeAssignment) => Promise; + }; + internals.activeWorkspaceAssignments.set('a', { + id: 'previous', + done: new Promise(() => {}), + }); + let executed = false; + internals.executeOwned = async () => { + executed = true; + assert.equal(internals.activeWorkspaceAssignments.get('b')?.id, 'next'); + }; + await worker.executeAndSettle({ + assignmentId: 'next', + executionKind: 'workspace_programmatic', + workspaceId: 'b', + remainingMs: 1_000, + request: { + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }, + } as BridgeAssignment); + assert.equal(executed, true); + assert.equal(internals.activeWorkspaceAssignments.has('b'), false); + assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 19f219da..a0517bd9 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -5,6 +5,7 @@ import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError, bridgeWorkerPath, + isBridgeWorkspaceProgrammaticRequest, isWorkspaceToolResult, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; @@ -21,6 +22,7 @@ import type { BridgeWorkerCredentialResponse, BridgeWorkerRegistrationResponse, BridgeWorkspaceToolOperation, + BridgeWorkspaceProgrammaticRequest, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -35,6 +37,13 @@ export interface BridgeWorkerOptions { runtimeSupervisor?: RuntimeSupervisor; capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; + workspaceProgrammatic?: { + executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise; + }; workspaceMutationQuarantine?: WorkspaceMutationQuarantine; /** Required per-root durable guards when opting into concurrent workspace leases. */ workspaceQuarantines?: ReadonlyMap; @@ -164,6 +173,11 @@ function workspaceCapabilitiesMatch( (advertised.listFileFeatures?.every( (feature, index) => feature === executor.listFileFeatures?.[index], ) ?? executor.listFileFeatures == null) && + advertised.programmaticLanguages?.length === + executor.programmaticLanguages?.length && + (advertised.programmaticLanguages?.every( + (language, index) => language === executor.programmaticLanguages?.[index], + ) ?? executor.programmaticLanguages == null) && advertised.workspaces.length === executor.workspaces.length && advertised.workspaces.every( (workspace, index) => @@ -223,6 +237,7 @@ function registrationCompatibleCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + programmaticLanguages: _programmaticLanguages, ...compatibleWorkspaceTools } = workspaceTools; return { @@ -302,11 +317,16 @@ function supportedWorkspaceCapabilities( const listFileFeatures = desired.listFileFeatures?.filter((feature) => registration.supportedWorkspaceListFileFeatures?.includes(feature), ); + const programmaticLanguages = desired.programmaticLanguages?.filter( + (language) => + registration.supportedWorkspaceProgrammaticLanguages?.includes(language), + ); const { writeFileModes: _writeFileModes, editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + programmaticLanguages: _programmaticLanguages, ...compatibleDesired } = desired; return { @@ -327,6 +347,10 @@ function supportedWorkspaceCapabilities( ...(operations.includes('list_files') && listFileFeatures?.length ? { listFileFeatures } : {}), + ...(operations.includes('execute_command') && + programmaticLanguages?.length + ? { programmaticLanguages } + : {}), }, }; } @@ -406,6 +430,16 @@ export class BridgeWorker { 'Workspace tool capabilities require a matching executor', ); } + if ( + (options.workspaceProgrammatic != null) !== + (options.capabilities.workspaceTools?.programmaticLanguages?.includes( + 'bash', + ) === true) + ) { + throw new BridgeProtocolError( + 'Workspace programmatic capability requires a matching executor', + ); + } if ( options.capabilities.workspaceTools?.operations.some( (operation) => @@ -1130,11 +1164,7 @@ export class BridgeWorker { assignment: BridgeAssignment, signal?: AbortSignal, ): Promise { - const root = - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ? assignment.request.workspaceId - : undefined; + const root = this.assignmentWorkspaceId(assignment); const waitingAt = Date.now(); while (root != null && this.activeWorkspaceAssignments.has(root)) { const active = this.activeWorkspaceAssignments.get(root)!; @@ -1212,14 +1242,31 @@ export class BridgeWorker { private workspaceGuard( assignment: BridgeAssignment, ): WorkspaceMutationQuarantine | undefined { - return assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ? (this.options.workspaceQuarantines?.get( - assignment.request.workspaceId, - ) ?? this.options.workspaceMutationQuarantine) + const workspaceId = this.assignmentWorkspaceId(assignment); + return workspaceId != null + ? (this.options.workspaceQuarantines?.get(workspaceId) ?? + this.options.workspaceMutationQuarantine) : this.options.workspaceMutationQuarantine; } + private assignmentWorkspaceId( + assignment: BridgeAssignment, + ): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.workspaceId; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + typeof assignment.workspaceId === 'string' + ) { + return assignment.workspaceId; + } + return undefined; + } + private async executeOwned( assignment: BridgeAssignment, signal?: AbortSignal, @@ -1472,6 +1519,76 @@ export class BridgeWorker { 'Bridge assignment expired during workspace execution', ); } + } else if (assignment.executionKind === 'workspace_programmatic') { + const workspaceId = assignment.workspaceId; + if ( + workspaceId == null || + this.options.workspaceProgrammatic == null || + !isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + throw new BridgeProtocolError( + 'Worker does not provide valid selected-workspace programmatic execution', + ); + } + try { + if (this.quarantinedWorkspaces.has(workspaceId)) { + throw new Error('Workspace requires an explicit quarantine reset'); + } + if (this.options.workspaceQuarantines != null) + await guard?.assertAvailable(); + } catch (error) { + throw new BridgeWorkspaceQuarantinedError( + 'Workspace is quarantined', + error, + ); + } + const advertised = this.activeCapabilities.workspaceTools; + const workspace = advertised?.workspaces.find( + (candidate) => candidate.id === workspaceId, + ); + if ( + workspace == null || + !advertised?.operations.includes('execute_command') || + (workspace.operations != null && + !workspace.operations.includes('execute_command')) || + !advertised.programmaticLanguages?.includes('bash') + ) { + throw new BridgeProtocolError( + 'Selected-workspace programmatic execution is not advertised', + ); + } + this.mutationGuardArmed = true; + try { + this.armedWorkspaces.add(workspaceId); + await guard!.arm( + 'Workspace programmatic execution is pending settlement', + assignment.assignmentId, + ); + workspaceMutationArmed = true; + } catch (error) { + this.mutationGuardArmed = false; + throw new BridgeWorkspaceQuarantinedError( + 'Workspace mutation quarantine could not be armed before execution', + error, + ); + } + payload = await this.options.workspaceProgrammatic.executeProgrammatic( + workspaceId, + assignment.request, + executionController.signal, + ); + workspaceMutationApplied = true; + if (executionController.signal.aborted) { + throw ( + executionController.signal.reason ?? + new DOMException('aborted', 'AbortError') + ); + } + if (Date.now() >= localDeadlineAtMs) { + throw new BridgeProtocolError( + 'Bridge assignment expired during programmatic execution', + ); + } } else { runtimeLease = await this.runtimeSupervisor.acquire( assignment, @@ -1581,7 +1698,8 @@ export class BridgeWorker { leaseToken: assignment.leaseToken, incarnationId: this.incarnationId, status: 'rejected', - ...(assignment.executionKind === 'workspace_tool' && + ...((assignment.executionKind === 'workspace_tool' || + assignment.executionKind === 'workspace_programmatic') && error instanceof WorkspaceToolError ? { errorCode: error.code } : {}), @@ -1693,12 +1811,8 @@ export class BridgeWorker { clearTimeout(timer); } } - if ( - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ) { - this.armedWorkspaces.delete(assignment.request.workspaceId); - } + const workspaceId = this.assignmentWorkspaceId(assignment); + if (workspaceId != null) this.armedWorkspaces.delete(workspaceId); this.mutationGuardArmed = false; } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1957,11 +2071,12 @@ export class BridgeWorker { const fulfilledWorkspaceMutation = workspaceMutationApplied && settlement.status === 'fulfilled' && - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) && - (assignment.request.operation === 'write_file' || - assignment.request.operation === 'edit_file' || - assignment.request.operation === 'execute_command'); + (assignment.executionKind === 'workspace_programmatic' || + (assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + (assignment.request.operation === 'write_file' || + assignment.request.operation === 'edit_file' || + assignment.request.operation === 'execute_command'))); if (signal?.aborted === true) { if (assignment.runtimeSessionId != null || fulfilledWorkspaceMutation) { throw await this.quarantineWorkspace( diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 8f9fcd45..fa08a84c 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -950,6 +950,88 @@ test('worker executes a workspace tool assignment locally without acquiring a sa }); }); +test('worker executes programmatic Bash in the selected workspace and preserves its fence', async () => { + const programmaticRequests: object[] = []; + const quarantineEvents: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + async executeProgrammatic(workspaceId, request) { + programmaticRequests.push({ workspaceId, request }); + return { + session_id: 'session-1', + language: 'bash', + version: '5.2', + files: [], + run: { stdout: 'ready\n', stderr: '', code: 0, signal: null }, + }; + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + (reason) => quarantineEvents.push(`quarantine:${reason}`), + (reason) => quarantineEvents.push(`arm:${reason}`), + () => quarantineEvents.push('clear'), + ), + ], + ]), + fetchImpl: async () => Response.json({ protocolVersion: 1, accepted: true }), + }); + const request = { + body: { + language: 'bash' as const, + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }; + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-1', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request, + }); + + assert.deepEqual(programmaticRequests, [{ workspaceId: 'primary', request }]); + assert.deepEqual(quarantineEvents, [ + 'arm:Workspace programmatic execution is pending settlement', + 'clear', + ]); +}); + test('worker stops after Code API rejects a fulfilled workspace mutation', async () => { let quarantinedReason: string | undefined; let armed = 0; diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 5bfc07fe..dfda49fc 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -112,6 +112,8 @@ export interface SandboxWorkspaceToolsOptions { commandSandbox: WorkspaceCommandSandbox; /** Workspace IDs whose sandbox is configured and may run commands. */ commandWorkspaces: string[]; + /** Optional execution-scoped languages supplied by the same command sandbox. */ + programmaticLanguages?: BridgeWorkspaceToolCapabilities['programmaticLanguages']; } const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; @@ -1657,6 +1659,9 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { ...(base.listFileFeatures != null ? { listFileFeatures: base.listFileFeatures } : {}), + ...(options.programmaticLanguages?.length + ? { programmaticLanguages: [...options.programmaticLanguages] } + : {}), workspaces: base.workspaces.map((workspace) => ({ ...workspace, operations: [ diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 25b0bdaf..369b306c 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -414,6 +414,7 @@ router.post( supportedWorkspaceEditFileModes: ['single', 'batch'], supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], supportedWorkspaceListFileFeatures: ['after_path'], + supportedWorkspaceProgrammaticLanguages: ['bash'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/selection.ts b/service/src/bridge/selection.ts index 0959279f..f3926e8a 100644 --- a/service/src/bridge/selection.ts +++ b/service/src/bridge/selection.ts @@ -1,4 +1,5 @@ export const CODEAPI_BRIDGE_WORKER_HEADER = 'X-LibreChat-Code-Worker-ID'; +export const CODEAPI_BRIDGE_WORKSPACE_HEADER = 'X-LibreChat-Code-Workspace-ID'; export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; export class BridgeWorkerSelectionError extends Error { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index b09d3792..d6b91469 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -151,6 +151,26 @@ function supportsWorkspaceTool( return true; } +function supportsWorkspaceProgrammatic( + registration: RegisteredBridgeWorker, + workspaceId: string, + language: string, +): boolean { + const capabilities = registration.capabilities.workspaceTools; + const workspace = capabilities?.workspaces.find( + (candidate) => candidate.id === workspaceId, + ); + return ( + workspace != null && + capabilities?.operations.includes('execute_command') === true && + (workspace.operations == null || + workspace.operations.includes('execute_command')) && + capabilities.programmaticLanguages?.includes( + language as 'bash', + ) === true + ); +} + function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } @@ -700,6 +720,7 @@ export class RedisBridgeStore { body: t.PayloadBody; headers: Record; workspaceRequest?: WorkspaceToolRequest; + workspaceId?: string; runtimeSessionId?: string; deadlineAtMs: number; executionTimeoutMs?: number; @@ -709,6 +730,12 @@ export class RedisBridgeStore { registration: RegisteredBridgeWorker, ) => Promise; }): Promise { + if (args.workspaceRequest != null && args.workspaceId != null) { + throw new BridgeStoreError( + 'ASSIGNMENT_INVALID', + 'A bridge assignment cannot be both a workspace tool and programmatic execution', + ); + } if ( args.executionTimeoutMs !== undefined && (args.workspaceRequest == null || @@ -773,6 +800,19 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} does not advertise the requested workspace tool`, ); } + if ( + args.workspaceId != null && + !supportsWorkspaceProgrammatic( + registration, + args.workspaceId, + args.body.language, + ) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} does not advertise programmatic execution for the selected workspace`, + ); + } if ( args.runtimeSessionId !== undefined && (await this.dispatchCommand( @@ -799,14 +839,16 @@ export class RedisBridgeStore { const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; let workspaceLeaseSlot: number | undefined; + const selectedWorkspaceId = + args.workspaceRequest?.workspaceId ?? args.workspaceId; const workspaceSlots = - args.workspaceRequest != null && + selectedWorkspaceId != null && (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 ? new BridgeWorkspaceSlots(this.redis) : undefined; let resultCommitted = false; const admission = - args.workspaceRequest == null + selectedWorkspaceId == null ? undefined : new BridgeAdmissionQueue(this.redis); try { @@ -820,7 +862,7 @@ export class RedisBridgeStore { args.deadlineAtMs, workspaceSlots == null ? undefined - : args.workspaceRequest?.workspaceId, + : selectedWorkspaceId, ), args, 'Bridge admission enqueue', @@ -855,7 +897,7 @@ export class RedisBridgeStore { workerId: args.workerId, incarnationId: lockIncarnationId, assignmentId, - workspaceId: args.workspaceRequest!.workspaceId, + workspaceId: selectedWorkspaceId!, capacity: registration.capabilities.workspaceLeaseSlots!, expiresAtMs: Date.now() + ttlSeconds * 1000, }), @@ -910,7 +952,14 @@ export class RedisBridgeStore { ); } if ( - !supportsWorkspaceTool(current.registration, args.workspaceRequest!) + (args.workspaceRequest != null && + !supportsWorkspaceTool(current.registration, args.workspaceRequest)) || + (args.workspaceId != null && + !supportsWorkspaceProgrammatic( + current.registration, + args.workspaceId, + args.body.language, + )) ) { throw new BridgeStoreError( 'WORKER_MISMATCH', @@ -935,11 +984,14 @@ export class RedisBridgeStore { generation, leaseToken, leaseTokenHash: tokenHash(leaseToken), + ...(selectedWorkspaceId == null ? {} : { + workspaceFence: `native-workspace:${selectedWorkspaceId}`, + }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${args.workspaceRequest!.workspaceId}`, + workspaceFence: `native-workspace:${selectedWorkspaceId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -951,6 +1003,15 @@ export class RedisBridgeStore { executionKind: 'workspace_tool' as const, request: args.workspaceRequest, } + : args.workspaceId != null + ? { + executionKind: 'workspace_programmatic' as const, + workspaceId: args.workspaceId, + request: { + body: args.body, + headers: args.headers, + }, + } : { request: { body: args.body, @@ -1011,6 +1072,19 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} no longer advertises the requested workspace tool`, ); } + if ( + args.workspaceId != null && + !supportsWorkspaceProgrammatic( + replacement.registration, + args.workspaceId, + args.body.language, + ) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} no longer advertises programmatic execution for the selected workspace`, + ); + } registration = replacement.registration; readyToken = replacement.readyToken; } @@ -1034,11 +1108,24 @@ export class RedisBridgeStore { resultCommitted = true; return result; } catch (error) { - if (args.runtimeSessionId !== undefined) { + if (assignment.workspaceFence != null) { + // Native roots retain their own fence through result restoration. + // Do not quarantine unrelated roots or invalidate the worker lease. + await boundedCommand(this.redis.eval( + [ + "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", + "redis.call('SET', KEYS[1], 'quarantined:' .. ARGV[1])", + 'return 1', + ].join('\n'), + 1, + workspaceQuarantineKey(args.workerId, assignment.workspaceFence), + assignment.assignmentId, + ), this.redisCommandTimeoutMs, 'Bridge native workspace finalization quarantine'); + } else if (assignmentWorkspace(assignment) !== undefined) { await this.quarantine( args.workerId, assignment.incarnationId, - args.runtimeSessionId, + assignmentWorkspace(assignment)!, ); } throw error; @@ -1904,10 +1991,11 @@ export class RedisBridgeStore { : undefined; const cancelledMutation = signal.aborted && - workspaceRequest != null && - (workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file' || - workspaceRequest.operation === 'execute_command'); + (assignment.executionKind === 'workspace_programmatic' || + (workspaceRequest != null && + (workspaceRequest.operation === 'write_file' || + workspaceRequest.operation === 'edit_file' || + workspaceRequest.operation === 'execute_command'))); if (cancelledMutation) { try { // Keep the acknowledged assignment available long enough for the diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index 5b0186d0..f248aa40 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -77,6 +77,123 @@ test('dispatches a workspace tool only to a worker advertising its workspace and }); }); +for (const finalizationFails of [false, true]) test(`single-slot programmatic finalization retains the workspace fence (failure=${finalizationFails})`, async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const body = { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }; + const completion = store.dispatch({ + workerId: 'workspace-worker', + body, + headers: {}, + workspaceId: 'primary', + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + finalize: async settlement => { + if (finalizationFails) throw new Error('artifact restoration failed'); + return settlement; + }, + }); + + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + expect(assignment).toMatchObject({ + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { body }, + }); + await store.settle('workspace-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + session_id: 'session-1', + language: 'bash', + version: '5.2', + files: [], + run: { + stdout: 'ready\n', + stderr: '', + code: 0, + signal: null, + output: 'ready\n', + memory: null, + message: null, + status: null, + cpu_time: null, + wall_time: 0.01, + }, + }, + }); + + if (finalizationFails) { + await expect(completion).rejects.toThrow('artifact restoration failed'); + await expect(store.dispatch({ workerId: 'workspace-worker', body, headers: {}, + workspaceId: 'primary', deadlineAtMs: Date.now() + 1000, + signal: new AbortController().signal, + })).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + return; + } + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + result: { session_id: 'session-1' }, + }); +}); + +test('rejects programmatic execution without the workspace capability', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatch({ + workerId: 'workspace-worker', + body: { + language: 'bash', + version: '5.2', + session_id: 'session-2', + files: [{ name: 'main.sh', content: 'echo denied' }], + }, + headers: {}, + workspaceId: 'primary', + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + test('drains an acknowledged workspace mutation cancellation before releasing it', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index f86ce656..873db173 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -45,6 +45,7 @@ import { parseBoundedContentLength } from './http-limits'; import { validateEgressGatewayHardenedConfig } from './secure-startup'; import { isOpaqueObjectContentDisposition } from './file-metadata'; import { mapObjectDetails } from './file-object-resolver'; +import { isSupportedBridgeArtifactName } from '../../packages/code/src/protocol'; export const app: Express = express(); app.disable('x-powered-by'); @@ -52,29 +53,6 @@ validateEgressGatewayHardenedConfig(); app.use(traceHttpRequest('codeapi.egress_gateway.request')); app.use(httpMetricsMiddleware); -const SUPPORTED_OUTPUT_EXTENSIONS = new Set([ - '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', - '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', - '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', - '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', - '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', - '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', - '.xml', '.yaml', '.yml', - '.ics', '.ical', '.ifb', '.icalendar', - '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', - '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', - '.odt', '.ods', '.odp', '.rtf', - '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', - '.tif', '.tiff', '.webp', - '.eot', '.ttf', '.woff', '.woff2', - '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', - '.tf', '.tfvars', '.tfstate', '.hcl', - '.dockerfile', '.Dockerfile', '.dockerignore', - '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', - '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', - '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', -]); - type EgressAuditFields = { execHash?: string; requestExecHash?: string; @@ -248,18 +226,7 @@ function assertOutputFilenameAllowed(name: string): void { throw new EgressGrantError('malformed', 'Output filename must be canonical'); } if (!isDirkeepName(name)) { - const basename = path.posix.basename(name); - const ext = path.posix.extname(basename).toLowerCase(); - const dottedBasename = `.${basename}`; - const allowed = - (ext !== '' && SUPPORTED_OUTPUT_EXTENSIONS.has(ext)) || - SUPPORTED_OUTPUT_EXTENSIONS.has(basename) || - SUPPORTED_OUTPUT_EXTENSIONS.has(basename.toLowerCase()) || - (ext === '' && ( - SUPPORTED_OUTPUT_EXTENSIONS.has(dottedBasename) || - SUPPORTED_OUTPUT_EXTENSIONS.has(dottedBasename.toLowerCase()) - )); - if (!allowed) { + if (!isSupportedBridgeArtifactName(name)) { throw new EgressGrantError('scope_mismatch', 'Output filename extension is not supported'); } } diff --git a/service/src/egress-grant.test.ts b/service/src/egress-grant.test.ts index 32ef2988..cd26fcd7 100644 --- a/service/src/egress-grant.test.ts +++ b/service/src/egress-grant.test.ts @@ -5,6 +5,7 @@ import { env } from './config'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, + normalizeSelectedWorkspaceProgrammaticTimeoutMs, prepareSandboxJobSecurity, refreshEgressGrantClaims, timeoutMsToGrantSeconds, @@ -478,6 +479,13 @@ describe('egress encrypted grants and handles', () => { expect(() => normalizeProgrammaticTimeoutMs(0, 300000)).toThrow('timeout must be a positive number'); }); + test('budgets both selected-workspace replay passes inside the worker deadline', () => { + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(undefined, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(120_000, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(300_000, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(10_000, 20_000)).toBe(2_167); + }); + test('normalizes the gateway callback URL for sandbox-originated PTC', () => { expect(normalizeEgressGatewayUrl(' http://egress-gateway:3190/// ')).toBe('http://egress-gateway:3190'); expect(() => normalizeEgressGatewayUrl(' ')).toThrow('EGRESS_GATEWAY_URL is required'); diff --git a/service/src/preamble-bash.test.ts b/service/src/preamble-bash.test.ts index 815040d3..e35a77f5 100644 --- a/service/src/preamble-bash.test.ts +++ b/service/src/preamble-bash.test.ts @@ -1,10 +1,19 @@ import { execFileSync } from 'child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; import { describe, expect, test } from 'bun:test'; import { extractPendingFromStdout, type LCTool } from './preamble'; -import { generateBashReplayPostamble, generateBashReplayPreamble } from './preamble-bash'; +import { + generateBashReplayPostamble, + generateBashReplayPreamble, +} from './preamble-bash'; interface BashRunResult { stdout: string; @@ -60,9 +69,13 @@ function assemble(userCode: string, toolSet: LCTool[] = tools): string { ].join('\n'); } -function runBash(script: string, options: number | BashRunOptions = {}): BashRunResult { - const timeoutMs = typeof options === 'number' ? options : options.timeoutMs ?? 3000; - const history = typeof options === 'number' ? {} : options.history ?? {}; +function runBash( + script: string, + options: number | BashRunOptions = {}, +): BashRunResult { + const timeoutMs = + typeof options === 'number' ? options : (options.timeoutMs ?? 3000); + const history = typeof options === 'number' ? {} : (options.history ?? {}); const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-unit-')); const file = join(dir, 'main.sh'); const historyPath = join(dir, 'history.json'); @@ -99,46 +112,136 @@ function pendingNames(stdout: string): string[] { return (parsed.pending ?? []).map(call => call.tool_name).sort(); } +describe('generateBashReplayPreamble - private runtime directory', () => { + test('creates every replay tempfile beneath TMPDIR', () => { + const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-private-tmp-')); + const dataDir = join(dir, 'data'); + const runtimeDir = join(dir, 'runtime'); + const file = join(dir, 'main.sh'); + const historyPath = join(dataDir, 'history.json'); + mkdirSync(dataDir, { recursive: true }); + mkdirSync(runtimeDir, { recursive: true }); + writeFileSync(historyPath, '{}'); + writeFileSync( + file, + assemble(` +printf '%s\\n' "$_PTC_PENDING_FILE" "$_PTC_ERROR_FILE" "$_PTC_COUNTER_FILE" +`), + { mode: 0o755 }, + ); + + try { + const stdout = execFileSync('bash', [file], { + env: { + ...process.env, + PTC_HISTORY_PATH: historyPath, + TMPDIR: runtimeDir, + }, + encoding: 'utf8', + }); + const paths = stdout.trim().split('\n'); + expect(paths).toHaveLength(3); + expect( + paths.every(value => value.startsWith(`${runtimeDir}/`)), + ).toBe(true); + expect( + generateBashReplayPreamble({ executionId, tools }), + ).not.toContain('mktemp -t'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test('persists pending calls through the private native control path', () => { + const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-control-')); + const file = join(dir, 'main.sh'); + const historyPath = join(dir, 'history.json'); + const controlPath = join(dir, 'control.json'); + writeFileSync(file, assemble(`get_weather '{"city":"Paris"}'`), { + mode: 0o755, + }); + writeFileSync(historyPath, '{}'); + try { + execFileSync('bash', [file], { + env: { + ...process.env, + PTC_HISTORY_PATH: historyPath, + LIBRECHAT_CODE_CONTROL_PATH: controlPath, + TMPDIR: dir, + }, + encoding: 'utf8', + }); + expect(JSON.parse(readFileSync(controlPath, 'utf8'))).toMatchObject( + { + pending: [ + { + call_id: 'call_001', + tool_name: 'get_weather', + input: { city: 'Paris' }, + }, + ], + }, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe('generateBashReplayPreamble - command substitution pending emission', () => { test('emits ClickHouse-style object input with SQL quotes from double-quoted JSON', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` SVC="45886e06-932b-4cff-bb49-3f7281d80717" result=$(run_select_query_mcp_ClickHouse "{\\"serviceId\\":\\"$SVC\\",\\"query\\":\\"SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999\\"}") echo "AFTER: $result" -`, [clickHouseTool])); +`, + [clickHouseTool], + ), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); expect(parsed.pending).toHaveLength(1); - expect(parsed.pending?.[0]?.tool_name).toBe('run_select_query_mcp_ClickHouse'); + expect(parsed.pending?.[0]?.tool_name).toBe( + 'run_select_query_mcp_ClickHouse', + ); expect(parsed.pending?.[0]?.input).toEqual({ serviceId: '45886e06-932b-4cff-bb49-3f7281d80717', - query: - "SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999", + query: "SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999", }); expect(parsed.stdout).not.toContain('AFTER'); }); test('emits ClickHouse-style object input with shell-escaped SQL quotes', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` result=$(run_select_query_mcp_ClickHouse '{"serviceId":"45886e06-932b-4cff-bb49-3f7281d80717","query":"SELECT name, type FROM system.columns WHERE database='"'"'default'"'"' AND table='"'"'uk_prices_3'"'"' ORDER BY position"}') echo "AFTER: $result" -`, [clickHouseTool])); +`, + [clickHouseTool], + ), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); expect(parsed.pending).toHaveLength(1); - expect(parsed.pending?.[0]?.tool_name).toBe('run_select_query_mcp_ClickHouse'); + expect(parsed.pending?.[0]?.tool_name).toBe( + 'run_select_query_mcp_ClickHouse', + ); expect(parsed.pending?.[0]?.input).toEqual({ serviceId: '45886e06-932b-4cff-bb49-3f7281d80717', - query: - "SELECT name, type FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", + query: "SELECT name, type FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", }); expect(parsed.stdout).not.toContain('AFTER'); }); test('batches parallel ClickHouse-style command substitutions into one pending block', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` SVC="45886e06-932b-4cff-bb49-3f7281d80717" { @@ -158,7 +261,11 @@ SVC="45886e06-932b-4cff-bb49-3f7281d80717" wait echo "AFTER" -`, [clickHouseTool]), 3000); +`, + [clickHouseTool], + ), + 3000, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -168,7 +275,11 @@ echo "AFTER" 'run_select_query_mcp_ClickHouse', 'run_select_query_mcp_ClickHouse', ]); - expect(parsed.pending?.map(call => (call.input as { query: string }).query).sort()).toEqual([ + expect( + parsed.pending + ?.map(call => (call.input as { query: string }).query) + .sort(), + ).toEqual([ "SELECT name, engine, total_rows, formatReadableSize(total_bytes) AS size, sorting_key, partition_key FROM system.tables WHERE database='default' AND name IN ('uk_prices_3','weather_noaa_mt')", "SELECT name, type, comment FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", "SELECT name, type, comment FROM system.columns WHERE database='default' AND table='weather_noaa_mt' ORDER BY position", @@ -177,12 +288,14 @@ echo "AFTER" }); test('emits a command-substitution tool call before later user code while another job is running', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` sleep 0.2 & result=$(get_weather '{"city":"Madrid"}') echo "AFTER: $result" wait -`)); +`), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -194,12 +307,15 @@ wait }); test('batches background and command-substitution tool calls before command-substitution side effects', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Oslo"}' & result=$(calculate '{"expression":"2+3"}') echo "SIDE_EFFECT: $result" wait -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -210,10 +326,13 @@ wait }); test('waits for background compound commands that invoke tools later', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` (sleep 0.2; get_weather '{"city":"Paris"}') & echo "AFTER LAUNCH" -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -225,10 +344,13 @@ echo "AFTER LAUNCH" }); test('does not wait for unrelated background commands with tool names as arguments', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` bash -c 'sleep 2' get_weather & echo "DONE" -`), 700); +`), + 700, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -238,14 +360,17 @@ echo "DONE" }); test('does not treat arithmetic expansion as command substitution while batching background tools', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Oslo"}' & sleep 0.1 x=$((1+1)) calculate '{"expression":"2+3"}' & wait echo "DONE $x" -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -256,12 +381,15 @@ echo "DONE $x" }); test('handles backtick command substitution without waiting for unrelated background jobs', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` sleep 5 & result=\`get_weather '{"city":"Porto"}'\` echo "AFTER: $result" wait -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -281,7 +409,10 @@ wait echo "DONE" `; const firstRun = runBash(assemble(userCode)); - const firstParsed = extractPendingFromStdout(firstRun.stdout, executionId); + const firstParsed = extractPendingFromStdout( + firstRun.stdout, + executionId, + ); expect(firstRun.exitCode).toBe(0); expect(firstParsed.pending).toHaveLength(2); @@ -302,7 +433,10 @@ echo "DONE" }), ); const replayRun = runBash(assemble(userCode), { history }); - const replayParsed = extractPendingFromStdout(replayRun.stdout, executionId); + const replayParsed = extractPendingFromStdout( + replayRun.stdout, + executionId, + ); expect(replayRun.exitCode).toBe(0); expect(replayParsed.pending).toBeNull(); expect(replayParsed.stdout).toContain('"slot":"first"'); @@ -326,12 +460,15 @@ echo "DONE" }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' get_weather '{"city":"Paris"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -352,12 +489,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -385,12 +525,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -418,12 +561,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); diff --git a/service/src/preamble-bash.ts b/service/src/preamble-bash.ts index 4ec3fc05..c442b495 100644 --- a/service/src/preamble-bash.ts +++ b/service/src/preamble-bash.ts @@ -1,8 +1,5 @@ import type { LCTool } from './preamble'; -import { - buildScopedSentinel, - PTC_HISTORY_SANDBOX_PATH, -} from './ptc-constants'; +import { buildScopedSentinel, PTC_HISTORY_SANDBOX_PATH } from './ptc-constants'; export interface BashReplayPreambleConfig { executionId: string; @@ -42,12 +39,55 @@ export class BashToolNameCollisionError extends Error { } const BASH_RESERVED = new Set([ - 'if', 'then', 'else', 'elif', 'fi', 'case', 'esac', 'for', 'select', - 'while', 'until', 'do', 'done', 'in', 'function', 'time', 'coproc', - 'return', 'exit', 'break', 'continue', 'shift', 'export', 'readonly', - 'local', 'declare', 'typeset', 'unset', 'alias', 'unalias', 'source', - 'echo', 'printf', 'read', 'cd', 'pwd', 'kill', 'trap', 'wait', 'eval', - 'exec', 'jobs', 'bg', 'fg', 'set', 'let', 'test', 'true', 'false', + 'if', + 'then', + 'else', + 'elif', + 'fi', + 'case', + 'esac', + 'for', + 'select', + 'while', + 'until', + 'do', + 'done', + 'in', + 'function', + 'time', + 'coproc', + 'return', + 'exit', + 'break', + 'continue', + 'shift', + 'export', + 'readonly', + 'local', + 'declare', + 'typeset', + 'unset', + 'alias', + 'unalias', + 'source', + 'echo', + 'printf', + 'read', + 'cd', + 'pwd', + 'kill', + 'trap', + 'wait', + 'eval', + 'exec', + 'jobs', + 'bg', + 'fg', + 'set', + 'let', + 'test', + 'true', + 'false', ]); function normalizeBashFunctionName(name: string): string { @@ -60,10 +100,7 @@ function normalizeBashFunctionName(name: string): string { * the end-of-preamble `readonly -f` lockdown runs. Compared case- * insensitively because the `_PTC_` prefix is used for variables and * `_ptc_` for functions, and both live in the same identifier space. */ - if ( - BASH_RESERVED.has(normalized) || - /^_ptc_/i.test(normalized) - ) { + if (BASH_RESERVED.has(normalized) || /^_ptc_/i.test(normalized)) { normalized = normalized + '_tool'; } if (normalized === '') normalized = 'tool'; @@ -95,9 +132,12 @@ function escapeForBashEre(s: string): string { * Users capture results via command substitution; input is passed as a single * JSON object string argument (validated by jq). */ -export function generateBashReplayPreamble(config: BashReplayPreambleConfig): string { +export function generateBashReplayPreamble( + config: BashReplayPreambleConfig, +): string { const { executionId, tools } = config; - const { start: scopedStart, end: scopedEnd } = buildScopedSentinel(executionId); + const { start: scopedStart, end: scopedEnd } = + buildScopedSentinel(executionId); let preamble = `#!/bin/bash # ============================================================================ @@ -109,20 +149,25 @@ _PTC_EXECUTION_ID="${executionId}" _PTC_SENTINEL_START="${scopedStart}" _PTC_SENTINEL_END="${scopedEnd}" _PTC_HISTORY_PATH="\${PTC_HISTORY_PATH:-${PTC_HISTORY_SANDBOX_PATH}}" -_PTC_PENDING_FILE="$(mktemp -t _ptc_pending.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pending.XXXXXX)" -_PTC_ERROR_FILE="$(mktemp -t _ptc_error.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_error.XXXXXX)" -_PTC_CONSUMED_FILE="$(mktemp -t _ptc_consumed.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_consumed.XXXXXX)" -_PTC_SAW_BARE_TOOL_FILE="$(mktemp -t _ptc_saw_tool.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_saw_tool.XXXXXX)" -_PTC_PRE_TOOL_JOBS_FILE="$(mktemp -t _ptc_pre_tool_jobs.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pre_tool_jobs.XXXXXX)" -_PTC_PRE_TOOL_JOBS_READY_FILE="$(mktemp -t _ptc_pre_tool_jobs_ready.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pre_tool_jobs_ready.XXXXXX)" -_PTC_TOOL_JOBS_FILE="$(mktemp -t _ptc_tool_jobs.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_tool_jobs.XXXXXX)" -_PTC_WAIT_RAN_FILE="$(mktemp -t _ptc_wait_ran.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_wait_ran.XXXXXX)" -_PTC_SUPPRESS_SUBSHELL_TOOL_FILE="$(mktemp -t _ptc_suppress_subshell_tool.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_suppress_subshell_tool.XXXXXX)" -_PTC_SUPPRESS_SUBSHELL_TOOL_CLEAR_FILE="$(mktemp -t _ptc_suppress_subshell_tool_clear.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_suppress_subshell_tool_clear.XXXXXX)" +_PTC_CONTROL_PATH="\${LIBRECHAT_CODE_CONTROL_PATH:-}" +_PTC_RUNTIME_DIR="\${TMPDIR:-/tmp}" +_ptc_mktemp() { + mktemp "\${_PTC_RUNTIME_DIR%/}/$1.XXXXXX" +} +_PTC_PENDING_FILE="$(_ptc_mktemp _ptc_pending)" +_PTC_ERROR_FILE="$(_ptc_mktemp _ptc_error)" +_PTC_CONSUMED_FILE="$(_ptc_mktemp _ptc_consumed)" +_PTC_SAW_BARE_TOOL_FILE="$(_ptc_mktemp _ptc_saw_tool)" +_PTC_PRE_TOOL_JOBS_FILE="$(_ptc_mktemp _ptc_pre_tool_jobs)" +_PTC_PRE_TOOL_JOBS_READY_FILE="$(_ptc_mktemp _ptc_pre_tool_jobs_ready)" +_PTC_TOOL_JOBS_FILE="$(_ptc_mktemp _ptc_tool_jobs)" +_PTC_WAIT_RAN_FILE="$(_ptc_mktemp _ptc_wait_ran)" +_PTC_SUPPRESS_SUBSHELL_TOOL_FILE="$(_ptc_mktemp _ptc_suppress_subshell_tool)" +_PTC_SUPPRESS_SUBSHELL_TOOL_CLEAR_FILE="$(_ptc_mktemp _ptc_suppress_subshell_tool_clear)" # Counter must persist across subshells (command substitution) so call_ids # stay deterministic across cached/uncached calls. Bash variables set in a # subshell don't propagate back, so we use a file. -_PTC_COUNTER_FILE="$(mktemp -t _ptc_counter.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_counter.XXXXXX)" +_PTC_COUNTER_FILE="$(_ptc_mktemp _ptc_counter)" _PTC_LOCK_DIR="\${_PTC_PENDING_FILE}.lock" printf '0' > "$_PTC_COUNTER_FILE" : > "$_PTC_CONSUMED_FILE" @@ -243,7 +288,7 @@ _ptc_prune_finished_tool_jobs() { return 0 fi local _ptc_tmp_file - _ptc_tmp_file="$(mktemp -t _ptc_tool_jobs_live.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_tool_jobs_live.XXXXXX)" + _ptc_tmp_file="$(_ptc_mktemp _ptc_tool_jobs_live)" while IFS= read -r _ptc_pid; do [ -n "$_ptc_pid" ] || continue if kill -0 "$_ptc_pid" 2>/dev/null; then @@ -330,6 +375,17 @@ _ptc_maybe_emit_pending() { trap - DEBUG EXIT exit 1 fi + # Native BYOM workers use this private execution-scoped control file so a + # large stdout stream cannot truncate away the replay frame. Other + # backends continue to consume the stdout sentinel below. + if [ -n "$_PTC_CONTROL_PATH" ]; then + printf '%s' "$_ptc_payload" > "$_PTC_CONTROL_PATH" || { + printf 'failed to persist pending PTC tool calls\n' >&2 + _ptc_cleanup_tempfiles + trap - DEBUG EXIT + exit 1 + } + fi if [ "\${BASH_SUBSHELL:-0}" -eq 1 ]; then trap - DEBUG EXIT exit 0 @@ -512,7 +568,7 @@ _ptc_call_tool() { # Large input can exceed ARG_MAX via --argjson; write once, reuse path below. local _ptc_input_tmp - _ptc_input_tmp="$(mktemp -t _ptc_input.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_input.XXXXXX)" + _ptc_input_tmp="$(_ptc_mktemp _ptc_input)" printf '%s' "$_ptc_input" > "$_ptc_input_tmp" local _ptc_matches @@ -668,8 +724,12 @@ exit $_ptc_user_exit_code function generateBashToolStub(tool: LCTool): string { const fnName = normalizeBashFunctionName(tool.name); - const desc = (tool.description ?? '').split('\n').map(l => `# ${l}`).join('\n'); - const nameComment = fnName !== tool.name ? `# Original tool name: ${tool.name}\n` : ''; + const desc = (tool.description ?? '') + .split('\n') + .map(l => `# ${l}`) + .join('\n'); + const nameComment = + fnName !== tool.name ? `# Original tool name: ${tool.name}\n` : ''; const escapedToolName = escapeForBashDoubleQuote(tool.name); return `${nameComment}${desc ? desc + '\n' : ''}${fnName}() { local _default_input='{}' @@ -683,7 +743,8 @@ function generateBashToolStub(tool: LCTool): string { } function generateBashPendingDeferHelper(tools: readonly LCTool[]): string { - const toolNamesPattern = tools + const toolNamesPattern = + tools .map(tool => normalizeBashFunctionName(tool.name)) .map(escapeForBashEre) .join('|') || 'a^'; diff --git a/service/src/preamble.test.ts b/service/src/preamble.test.ts index bf50e3ec..803e1839 100644 --- a/service/src/preamble.test.ts +++ b/service/src/preamble.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import { buildScopedSentinel, createProgrammaticPayload, + extractPendingFromControlPayload, extractPendingFromStdout, generatePreamble, } from './preamble'; @@ -33,7 +34,9 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { expect(preamble).toMatch(/AF_UNIX/); expect(preamble).toMatch(/\.connect\(_TOOL_CALL_SOCKET\)/); /* Regression guard against reintroducing the user-spoofable check. */ - expect(preamble).not.toMatch(/if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/); + expect(preamble).not.toMatch( + /if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/, + ); }); test('caches the probe verdict at module load (before user code can plant a spoof)', () => { @@ -41,10 +44,14 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { /* The probe call must appear at top level of the preamble, NOT * inside _do_request. Otherwise a user could plant a regular file * at the path between calls and flip the gate per-request. */ - const probeCallIdx = preamble.indexOf('_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()'); + const probeCallIdx = preamble.indexOf( + '_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()', + ); expect(probeCallIdx).toBeGreaterThan(-1); /* _do_request must consult the cached verdict, not re-probe. */ - const doReqMatch = preamble.match(/def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/); + const doReqMatch = preamble.match( + /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, + ); expect(doReqMatch).not.toBeNull(); expect(doReqMatch![0]).toContain('_USE_TOOL_CALL_SOCKET'); expect(doReqMatch![0]).not.toContain('_probe_tool_call_socket('); @@ -55,7 +62,9 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { /* The fallback must still construct the URL from _CALLBACK_URL and * delegate to _tcp_request. Without this, runners without the * proxy bind-mount would have no way to reach the orchestrator. */ - const doReqMatch = preamble.match(/def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/); + const doReqMatch = preamble.match( + /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, + ); expect(doReqMatch).not.toBeNull(); expect(doReqMatch![0]).toContain('_CALLBACK_URL + path'); expect(doReqMatch![0]).toContain('_tcp_request('); @@ -67,24 +76,47 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { * exported. The path must remain hardcoded so the preamble does * not depend on env-var injection. */ expect(preamble).toContain('_TOOL_CALL_SOCKET = "/tmp/tcs.sock"'); - expect(preamble).not.toMatch(/os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/); + expect(preamble).not.toMatch( + /os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/, + ); expect(preamble).not.toMatch(/os\.environ\[['"]TOOL_CALL_SOCKET['"]\]/); }); }); describe('extractPendingFromStdout — input hash metadata', () => { + test('normalizes native control payload hashes instead of trusting the sandbox', () => { + const forgedHash = hashToolInput({ resource: 'B' }); + const expectedHash = hashToolInput({ resource: 'A' }); + const pending = extractPendingFromControlPayload( + JSON.stringify({ + pending: [ + { + call_id: 'call_001', + tool_name: 'authorize', + input: { resource: 'A' }, + input_hash: forgedHash, + }, + ], + }), + ); + expect(pending?.[0]?.input_hash).toBe(expectedHash); + expect(pending?.[0]?.input_hash).not.toBe(forgedHash); + }); + test('ignores sandbox-supplied input_hash and uses the parsed input hash', () => { const executionId = 'exec_hash_guard'; const { start, end } = buildScopedSentinel(executionId); const forgedHash = hashToolInput({ resource: 'B' }); const expectedHash = hashToolInput({ resource: 'A' }); const payload = { - pending: [{ + pending: [ + { call_id: 'call_001', tool_name: 'authorize', input: { resource: 'A' }, input_hash: forgedHash, - }], + }, + ], }; const parsed = extractPendingFromStdout( diff --git a/service/src/preamble.ts b/service/src/preamble.ts index 9ff75dc7..94685c6c 100644 --- a/service/src/preamble.ts +++ b/service/src/preamble.ts @@ -2,7 +2,10 @@ import fs from 'fs'; import path from 'path'; import type * as t from './types'; import { planLimits } from './config'; -import { generateBashReplayPreamble, generateBashReplayPostamble } from './preamble-bash'; +import { + generateBashReplayPreamble, + generateBashReplayPostamble, +} from './preamble-bash'; import { PTC_HISTORY_FILENAME, PTC_HISTORY_SANDBOX_PATH, @@ -11,10 +14,16 @@ import { buildScopedSentinel, isReservedPtcFilename, } from './ptc-constants'; -import { hashToolInput, pendingInputHashesFromRawPayload } from './tool-input-signature'; +import { + hashToolInput, + pendingInputHashesFromRawPayload, +} from './tool-input-signature'; // Load async matplotlib template for programmatic tool calling -const templateCodeAsync = fs.readFileSync(path.join(__dirname, 'matplotlib-async.py'), 'utf8'); +const templateCodeAsync = fs.readFileSync( + path.join(__dirname, 'matplotlib-async.py'), + 'utf8', +); // ============================================================================= // Programmatic Tool Calling Types & Preamble Generation @@ -88,11 +97,41 @@ function normalizePythonFunctionName(name: string): string { // Python keywords to avoid const pythonKeywords = new Set([ - 'False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', - 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', - 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', - 'lambda', 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', - 'try', 'while', 'with', 'yield' + 'False', + 'None', + 'True', + 'and', + 'as', + 'assert', + 'async', + 'await', + 'break', + 'class', + 'continue', + 'def', + 'del', + 'elif', + 'else', + 'except', + 'finally', + 'for', + 'from', + 'global', + 'if', + 'import', + 'in', + 'is', + 'lambda', + 'nonlocal', + 'not', + 'or', + 'pass', + 'raise', + 'return', + 'try', + 'while', + 'with', + 'yield', ]); if (pythonKeywords.has(normalized)) { @@ -137,11 +176,14 @@ function jsonSchemaToPythonType(schema: JsonSchemaProperty): string { * Sort property names so required parameters come before optional ones. * Uses a Set for O(1) lookups instead of repeated array includes() calls. */ -function getSortedPropertyNames(propertyNames: string[], required: string[]): string[] { +function getSortedPropertyNames( + propertyNames: string[], + required: string[], +): string[] { const requiredSet = new Set(required); return [ ...propertyNames.filter(name => requiredSet.has(name)), - ...propertyNames.filter(name => !requiredSet.has(name)) + ...propertyNames.filter(name => !requiredSet.has(name)), ]; } @@ -155,7 +197,10 @@ function schemaToParams(schema?: JsonSchema): string { const required = schema.required ?? []; const requiredSet = new Set(required); - const sortedNames = getSortedPropertyNames(Object.keys(schema.properties), required); + const sortedNames = getSortedPropertyNames( + Object.keys(schema.properties), + required, + ); const params: string[] = []; @@ -192,7 +237,11 @@ function inferReturnType(description?: string): string { const desc = description.toLowerCase(); - if (desc.includes('returns list') || desc.includes('returns array') || desc.includes('list of')) { + if ( + desc.includes('returns list') || + desc.includes('returns array') || + desc.includes('list of') + ) { return 'List[Dict[str, Any]]'; } if (desc.includes('returns dict') || desc.includes('returns object')) { @@ -225,7 +274,10 @@ function generateDocstring(tool: LCTool): string { doc += '\n\n Parameters:'; const required = tool.parameters.required ?? []; const requiredSet = new Set(required); - const sortedNames = getSortedPropertyNames(Object.keys(tool.parameters.properties), required); + const sortedNames = getSortedPropertyNames( + Object.keys(tool.parameters.properties), + required, + ); for (const name of sortedNames) { const propSchema = tool.parameters.properties[name]; @@ -253,7 +305,8 @@ function generateToolStub(tool: LCTool): string { const pythonFunctionName = normalizePythonFunctionName(tool.name); // If name was changed, add a comment - const nameComment = pythonFunctionName !== tool.name + const nameComment = + pythonFunctionName !== tool.name ? ` # Original tool name: ${tool.name}\n` : ''; @@ -451,7 +504,8 @@ async def _execute_tool_internal_async(tool_name: str, tool_input: Dict[str, Any */ export function generateReplayPreamble(config: ReplayPreambleConfig): string { const { executionId, tools } = config; - const { start: scopedStart, end: scopedEnd } = buildScopedSentinel(executionId); + const { start: scopedStart, end: scopedEnd } = + buildScopedSentinel(executionId); let preamble = ` # ============================================================================ @@ -564,6 +618,63 @@ export interface ExtractPendingResult { }> | null; } +export function extractPendingFromControlPayload( + rawPayload: string, +): ExtractPendingResult['pending'] { + let parsed: { pending?: unknown } | null = null; + try { + parsed = JSON.parse(rawPayload) as { pending?: unknown }; + } catch { + return null; + } + + const pendingField = parsed?.pending; + if (!Array.isArray(pendingField)) return null; + + const rawInputHashes = pendingInputHashesFromRawPayload(rawPayload); + type PendingWithIndex = { + c: { call_id: string; tool_name: string; input: unknown }; + index: number; + }; + const isPendingWithIndex = (entry: { + c: unknown; + index: number; + }): entry is PendingWithIndex => { + const { c } = entry; + return ( + c != null && + typeof c === 'object' && + typeof (c as { call_id?: unknown }).call_id === 'string' && + typeof (c as { tool_name?: unknown }).tool_name === 'string' + ); + }; + return pendingField + .map((c, index) => ({ c, index })) + .filter(isPendingWithIndex) + .map(({ c, index }) => { + const callSite = (c as { call_site?: unknown }).call_site; + const rawInputHash = rawInputHashes[index]; + const hasObjectInput = + c.input != null && typeof c.input === 'object'; + const input = (hasObjectInput ? c.input : {}) as Record< + string, + unknown + >; + return { + call_id: c.call_id, + tool_name: c.tool_name, + input, + input_hash: + hasObjectInput && typeof rawInputHash === 'string' + ? rawInputHash + : hashToolInput(input), + ...(typeof callSite === 'string' + ? { call_site: callSite } + : {}), + }; + }); +} + /** * Locate the last line whose trimmed content exactly equals `marker`. * Using full-line anchoring prevents user-provided tool payloads that happen @@ -581,7 +692,8 @@ function findSentinelLine( for (let i = lines.length - 1; i >= searchFromLine; i--) { if (lines[i].trim() === marker) { const startOffset = lineStartOffsets[i]; - const endOffset = i + 1 < lineStartOffsets.length + const endOffset = + i + 1 < lineStartOffsets.length ? lineStartOffsets[i + 1] - 1 : startOffset + lines[i].length; return { line: i, startOffset, endOffset }; @@ -619,48 +731,8 @@ export function extractPendingFromStdout( const payloadLines = lines.slice(startLine.line + 1, endLine.line); const rawPayload = payloadLines.join('\n').trim(); - let parsed: { pending?: unknown } | null = null; - try { - parsed = JSON.parse(rawPayload) as { pending?: unknown }; - } catch { - return { stdout, pending: null }; - } - - const pendingField = parsed?.pending; - if (!Array.isArray(pendingField)) return { stdout, pending: null }; - - const rawInputHashes = pendingInputHashesFromRawPayload(rawPayload); - type PendingWithIndex = { - c: { call_id: string; tool_name: string; input: unknown }; - index: number; - }; - const isPendingWithIndex = (entry: { c: unknown; index: number }): entry is PendingWithIndex => { - const { c } = entry; - return ( - c != null && - typeof c === 'object' && - typeof (c as { call_id?: unknown }).call_id === 'string' && - typeof (c as { tool_name?: unknown }).tool_name === 'string' - ); - }; - const pending = pendingField - .map((c, index) => ({ c, index })) - .filter(isPendingWithIndex) - .map(({ c, index }) => { - const callSite = (c as { call_site?: unknown }).call_site; - const rawInputHash = rawInputHashes[index]; - const hasObjectInput = c.input != null && typeof c.input === 'object'; - const input = (hasObjectInput ? c.input : {}) as Record; - return { - call_id: c.call_id, - tool_name: c.tool_name, - input, - input_hash: hasObjectInput && typeof rawInputHash === 'string' - ? rawInputHash - : hashToolInput(input), - ...(typeof callSite === 'string' ? { call_site: callSite } : {}), - }; - }); + const pending = extractPendingFromControlPayload(rawPayload); + if (pending == null) return { stdout, pending: null }; /** Strip only the sentinel block and leave every other byte of user * stdout untouched. Both the Python and bash preambles defensively @@ -674,7 +746,10 @@ export function extractPendingFromStdout( * emission, and anything else that depends on byte-accurate stdout. */ const rawHead = stdout.slice(0, startLine.startOffset); const head = rawHead.endsWith('\n') ? rawHead.slice(0, -1) : rawHead; - const tailStart = endLine.endOffset < stdout.length ? endLine.endOffset + 1 : stdout.length; + const tailStart = + endLine.endOffset < stdout.length + ? endLine.endOffset + 1 + : stdout.length; const tail = stdout.slice(tailStart); const cleaned = head + tail; @@ -688,11 +763,14 @@ export function extractPendingFromStdout( function wrapUserCodeInAsync(userCode: string): string { const lines = userCode.split('\n'); - let wrapped = '# ============================================================================\n'; + let wrapped = + '# ============================================================================\n'; wrapped += '# USER CODE BEGINS BELOW\n'; - wrapped += '# ============================================================================\n\n'; + wrapped += + '# ============================================================================\n\n'; wrapped += 'async def __user_main__():\n'; - wrapped += ' """Auto-generated wrapper for user code to support top-level await"""\n'; + wrapped += + ' """Auto-generated wrapper for user code to support top-level await"""\n'; // Indent all user code for (const line of lines) { @@ -743,10 +821,21 @@ const PROGRAMMATIC_RUN_TIMEOUT = 300000; // 5 minutes wall time * Create a payload for programmatic tool calling execution * Combines the tool preamble with user code */ -export function createProgrammaticPayload(options: CreateProgrammaticPayloadOptions): t.PayloadBody { +export function createProgrammaticPayload( + options: CreateProgrammaticPayloadOptions, +): t.PayloadBody { const { - req, session_id, execution_id, callbackUrl, callbackToken, tools, timeout, - mode = 'blocking', history, codeOverride, filesOverride, + req, + session_id, + execution_id, + callbackUrl, + callbackToken, + tools, + timeout, + mode = 'blocking', + history, + codeOverride, + filesOverride, language = 'python', } = options; const body = req.body as t.ProgrammaticRequestBody; @@ -762,7 +851,14 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti throw new Error('bash PTC is only supported in replay mode'); } return buildBashPayload({ - req, execution_id, session_id, tools, userCode, files, history, timeout, + req, + execution_id, + session_id, + tools, + userCode, + files, + history, + timeout, }); } @@ -771,7 +867,9 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti preamble = generateReplayPreamble({ executionId: execution_id, tools }); } else { if (!callbackUrl || !callbackToken) { - throw new Error('blocking PTC mode requires callbackUrl and callbackToken'); + throw new Error( + 'blocking PTC mode requires callbackUrl and callbackToken', + ); } preamble = generatePreamble({ callbackUrl, @@ -781,15 +879,21 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti }); } - const isPyPlot = userCode.includes('import matplotlib') || userCode.includes('import seaborn'); + const isPyPlot = + userCode.includes('import matplotlib') || + userCode.includes('import seaborn'); let finalCode: string; if (isPyPlot) { - const indentedUserCode = userCode.trim().split('\n').map(line => ` ${line}`).join('\n'); + const indentedUserCode = userCode + .trim() + .split('\n') + .map(line => ` ${line}`) + .join('\n'); const wrappedUserCode = templateCodeAsync.replace( /# BEGIN USER CODE\n[\s\S]*?# END USER CODE/, - `# BEGIN USER CODE\n${indentedUserCode}\n # END USER CODE` + `# BEGIN USER CODE\n${indentedUserCode}\n # END USER CODE`, ); finalCode = preamble + '\n' + wrappedUserCode; } else { @@ -797,7 +901,9 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti finalCode = preamble + wrappedUserCode; } - const run_memory_limit = planLimits[req.planId ?? '']?.run_memory_limit ?? planLimits.default.run_memory_limit; + const run_memory_limit = + planLimits[req.planId ?? '']?.run_memory_limit ?? + planLimits.default.run_memory_limit; const run_timeout = timeout ?? PROGRAMMATIC_RUN_TIMEOUT; const payload: t.PayloadBody = { @@ -809,8 +915,8 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti files: [ { name: 'main.py', - content: finalCode - } + content: finalCode, + }, ], session_id, }; @@ -851,13 +957,27 @@ function buildBashPayload(args: { history?: Record; timeout?: number; }): t.PayloadBody { - const { req, execution_id, session_id, tools, userCode, files, history, timeout } = args; - - const preamble = generateBashReplayPreamble({ executionId: execution_id, tools }); + const { + req, + execution_id, + session_id, + tools, + userCode, + files, + history, + timeout, + } = args; + + const preamble = generateBashReplayPreamble({ + executionId: execution_id, + tools, + }); const postamble = generateBashReplayPostamble(); const finalCode = preamble + userCode + '\n' + postamble; - const run_memory_limit = planLimits[req.planId ?? '']?.run_memory_limit ?? planLimits.default.run_memory_limit; + const run_memory_limit = + planLimits[req.planId ?? '']?.run_memory_limit ?? + planLimits.default.run_memory_limit; const run_timeout = timeout ?? PROGRAMMATIC_RUN_TIMEOUT; const payload: t.PayloadBody = { @@ -865,6 +985,8 @@ function buildBashPayload(args: { run_timeout, language: 'bash', version: '5.2.0', + execution_id, + replay_tool_count: tools.length, files: [ { name: 'main.sh', diff --git a/service/src/ptc-constants.test.ts b/service/src/ptc-constants.test.ts new file mode 100644 index 00000000..5028e385 --- /dev/null +++ b/service/src/ptc-constants.test.ts @@ -0,0 +1,9 @@ +import { expect, test } from 'bun:test'; +import { isReservedPtcFilename } from './ptc-constants'; + +test('reserves replay inputs and output control channels after normalization', () => { + for (const name of ['_ptc_history.json', '_ptc_pending_result.json', '_PTC_PENDING_RESULT.JSON', 'sub/../_ptc_pending_result.json', 'sub\\_ptc_pending_result.json']) { + expect(isReservedPtcFilename(name)).toBe(true); + } + expect(isReservedPtcFilename('_ptc_data.csv')).toBe(false); +}); diff --git a/service/src/ptc-constants.ts b/service/src/ptc-constants.ts index 5009c3c3..b7b3999b 100644 --- a/service/src/ptc-constants.ts +++ b/service/src/ptc-constants.ts @@ -15,8 +15,9 @@ export const PTC_HISTORY_SANDBOX_PATH = `/mnt/data/${PTC_HISTORY_FILENAME}`; * Returns `true` for any filename the submission layer must refuse. * * Two things make a name "reserved": - * 1. Its post-normalization basename is `_ptc_history.json` — the single - * runtime fixture the replay preamble injects into the submission dir. + * 1. Its post-normalization basename is `_ptc_history.json` or + * `_ptc_pending_result.json`, compared case-insensitively for macOS. + * These are the replay input and output control channels. * Any user-supplied file with that exact basename would shadow our * injected history and silently corrupt replay correctness, so we * reject it on the request path. The bash preamble's `_ptc_pending.*` @@ -61,7 +62,7 @@ export function isReservedPtcFilename(name: string): boolean { } if (escapes) return true; const basename = segments.length > 0 ? segments[segments.length - 1] : ''; - return basename === PTC_HISTORY_FILENAME; + return [PTC_HISTORY_FILENAME, '_ptc_pending_result.json'].includes(basename.toLowerCase()); } /** diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index a65bc89e..697271ee 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -62,6 +62,34 @@ describe('RemoteBridgeSandboxBackend', () => { }); }); + test('preserves an authenticated selected workspace on remote dispatch', async () => { + let dispatched: Parameters[0] | undefined; + const store = { + dispatch: async ( + args: Parameters[0], + ): ReturnType => { + dispatched = args; + return { + protocolVersion: 1 as const, + generation: 1, + leaseToken: 'a'.repeat(32), + incarnationId: 'incarnation-00000001', + status: 'fulfilled' as const, + result: { session_id: 'session-1', language: 'bash', version: '5.2', files: [] }, + }; + }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend(store, 'default-vm'); + + await backend.execute(request(), { ...context(), workspaceId: 'project-a' }); + + expect(dispatched).toMatchObject({ + workerId: 'user-vm', + workspaceId: 'project-a', + requireTenantBinding: true, + }); + }); + test('maps tenant authorization rejection to a bridge backend error', async () => { const store = { dispatch: async (): ReturnType => { diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index 0bee0038..6e06eda8 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -60,6 +60,7 @@ export class RemoteBridgeSandboxBackend implements SandboxBackend { (this.dynamicWorkers || ctx.bridgeWorkerId !== this.workerId), body: req.body, headers: req.headers, + ...(ctx.workspaceId != null ? { workspaceId: ctx.workspaceId } : {}), runtimeSessionId: ctx.runtimeSessionId, deadlineAtMs: ctx.deadlineAtMs ?? Date.now() + env.JOB_TIMEOUT, signal: ctx.signal, diff --git a/service/src/sandbox-backend/types.ts b/service/src/sandbox-backend/types.ts index fbaa2d20..e21982d9 100644 --- a/service/src/sandbox-backend/types.ts +++ b/service/src/sandbox-backend/types.ts @@ -39,6 +39,8 @@ export interface SandboxExecuteContext { canonicalUserId?: string; /** Trusted API-selected outbound worker. Presence requires a tenant-bound credential. */ bridgeWorkerId?: string; + /** Trusted selected workspace for native replay-mode PTC. */ + workspaceId?: string; /** Stable identifier for this queued iteration, used to derive an idempotent * stateless launch token. PTC replay reuses one executionId across every * iteration, so the executionId alone cannot separate them; the request body @@ -63,6 +65,7 @@ export type SandboxRawResponse = t.ExecuteResponse & { session_id: string; files?: t.FileRefs; run?: t.ExecuteResponse['run']; + pending_tool_calls_payload?: string; }; export interface SandboxBackend { diff --git a/service/src/sandbox-dispatch.test.ts b/service/src/sandbox-dispatch.test.ts index 435be9aa..5b1ef749 100644 --- a/service/src/sandbox-dispatch.test.ts +++ b/service/src/sandbox-dispatch.test.ts @@ -12,8 +12,10 @@ import { } from './execution-manifest'; const SECRET = 'test-secret'; -const PRIVATE_KEY = 'MC4CAQAwBQYDK2VwBCIEIBoxzSJjQ5jTVyuohHtlD+uDGqv/tZ6hQS2CmxuOg2Wn'; -const PUBLIC_KEY = 'MCowBQYDK2VwAyEAeY3PRoTS3adfU6E3gQUB5hSZdrdMSw6OrKkH4UhYh0U='; +const PRIVATE_KEY = + 'MC4CAQAwBQYDK2VwBCIEIBoxzSJjQ5jTVyuohHtlD+uDGqv/tZ6hQS2CmxuOg2Wn'; +const PUBLIC_KEY = + 'MCowBQYDK2VwAyEAeY3PRoTS3adfU6E3gQUB5hSZdrdMSw6OrKkH4UhYh0U='; function payload(overrides: Partial = {}): t.PayloadBody { return { @@ -25,14 +27,22 @@ function payload(overrides: Partial = {}): t.PayloadBody { }; } -function claims(overrides: Partial = {}): ExecutionManifestClaims { +function claims( + overrides: Partial = {}, +): ExecutionManifestClaims { return { v: EXECUTION_MANIFEST_VERSION, exec_id: 'exec_123', tenant_id: 'tenant_abc', user_id: 'user_123', session_key: 'tenant:tenant_abc:user:user_123', - input_files: [{ id: 'file_123', session_id: 'sess_input', name: 'inputs/data.csv' }], + input_files: [ + { + id: 'file_123', + session_id: 'sess_input', + name: 'inputs/data.csv', + }, + ], read_sessions: ['sess_input'], output_session_id: 'sess_output', max_upload_bytes: 1024, @@ -46,6 +56,20 @@ function claims(overrides: Partial = {}): ExecutionMani } describe('sandbox execute request dispatch', () => { + test('budgets every input and output batch before signing the request', () => { + const request = buildSandboxExecuteRequest({ + payload: payload({ files: Array.from({ length: 9 }, (_, index) => ({ name: `${index}.txt`, id: `file_${index}`, storage_session_id: 'input' })) }), + programmaticTransferReserveMs: 60_000, + executionManifestClaims: claims({ max_output_files: 10 }), + executionManifestSecret: SECRET, + executionManifestTtlSeconds: 300, + nowSeconds: 1_000, + }); + // Three download batches plus three upload batches share one reserve. + expect(request.body.transfer_timeout_ms).toBe(10_000); + const verified = verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_000 }); + expect(verified.execute_body_sha256).toBe(executionManifestBodySha256(request.body)); + }); test('keeps large egress grants out of HTTP headers', () => { const largeGrant = `ceg1.${'a'.repeat(24_000)}`; const request = buildSandboxExecuteRequest({ @@ -64,18 +88,27 @@ describe('sandbox execute request dispatch', () => { const request = buildSandboxExecuteRequest({ payload: payload(), executionManifestClaims: claims(), + maxOutputFileBytes: 1_000, executionManifestSecret: SECRET, executionManifestTtlSeconds: 300, nowSeconds: 1_000, }); expect(request.headers[EXECUTION_MANIFEST_HEADER]).toBeUndefined(); + expect(request.body.max_output_files).toBe(10); + expect(request.body.max_output_file_bytes).toBe(1_000); expect(request.body.execution_manifest).toEqual(expect.any(String)); - expect(verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifest(request.body.execution_manifest!, SECRET, { + nowSeconds: 1_100, + }), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); test('signs execution manifests with a private key when configured', () => { @@ -88,11 +121,19 @@ describe('sandbox execute request dispatch', () => { nowSeconds: 1_000, }); - expect(verifyExecutionManifestWithPublicKey(request.body.execution_manifest!, PUBLIC_KEY, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifestWithPublicKey( + request.body.execution_manifest!, + PUBLIC_KEY, + { nowSeconds: 1_100 }, + ), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); test('binds body-carried egress grants into signed execution manifests', () => { @@ -106,10 +147,16 @@ describe('sandbox execute request dispatch', () => { }); expect(request.body.egress_grant).toBe('ceg1.sealed-grant'); - expect(verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifest(request.body.execution_manifest!, SECRET, { + nowSeconds: 1_100, + }), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); }); diff --git a/service/src/sandbox-dispatch.ts b/service/src/sandbox-dispatch.ts index e3066905..340834b5 100644 --- a/service/src/sandbox-dispatch.ts +++ b/service/src/sandbox-dispatch.ts @@ -1,5 +1,14 @@ import type * as t from './types'; -import { executionManifestBodySha256, signExecutionManifestWithKey, type ExecutionManifestClaims } from './execution-manifest'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY, + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS, +} from '../../packages/code/src/protocol'; +import { + executionManifestBodySha256, + signExecutionManifestWithKey, + type ExecutionManifestClaims, +} from './execution-manifest'; interface BuildSandboxExecuteRequestArgs { payload: t.PayloadBody; @@ -9,6 +18,8 @@ interface BuildSandboxExecuteRequestArgs { executionManifestSecret: string; executionManifestTtlSeconds: number; nowSeconds?: number; + maxOutputFileBytes?: number; + programmaticTransferReserveMs?: number; } interface SandboxExecuteRequest { @@ -21,15 +32,31 @@ interface SandboxExecuteRequest { * ride in the JSON body instead of HTTP headers. Otherwise skill-heavy jobs can * fail with 431 before sandbox-runner reaches capability validation. */ -export function buildSandboxExecuteRequest(args: BuildSandboxExecuteRequestArgs): SandboxExecuteRequest { +export function buildSandboxExecuteRequest( + args: BuildSandboxExecuteRequestArgs, +): SandboxExecuteRequest { const body: t.PayloadBody = { ...args.payload }; - const headers: Record = { 'Content-Type': 'application/json' }; + const headers: Record = { + 'Content-Type': 'application/json', + }; if (args.egressGrantToken) { body.egress_grant = args.egressGrantToken; } + if (args.maxOutputFileBytes != null) { + body.max_output_file_bytes = args.maxOutputFileBytes; + } + if (args.programmaticTransferReserveMs != null) { + const batches = Math.max(1, + Math.ceil(body.files.filter(file => 'id' in file).length / BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY) + + Math.ceil((args.executionManifestClaims?.max_output_files ?? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY), + ); + body.transfer_timeout_ms = Math.min(BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS, + Math.max(1, Math.floor(args.programmaticTransferReserveMs / batches))); + } if (args.executionManifestClaims) { + body.max_output_files = args.executionManifestClaims.max_output_files; const nowSeconds = args.nowSeconds ?? Math.floor(Date.now() / 1000); body.execution_manifest = signExecutionManifestWithKey( { diff --git a/service/src/sandbox-egress.ts b/service/src/sandbox-egress.ts index f6549c09..e47806a2 100644 --- a/service/src/sandbox-egress.ts +++ b/service/src/sandbox-egress.ts @@ -3,6 +3,7 @@ import { env } from './config'; import { createGatewayPtcCallbackToken } from './egress-gateway-client'; import type { ExecutionManifestClaims } from './execution-manifest'; import type * as t from './types'; +import { programmaticTransferReserveMs } from '../../packages/code/src/protocol'; export type SandboxJobSecurity = { payload: t.PayloadBody; @@ -65,6 +66,9 @@ export function timeoutMsToGrantSeconds(timeoutMs: number): number { } const DEFAULT_PROGRAMMATIC_TIMEOUT_MS = 300000; +const SELECTED_WORKSPACE_REPLAY_PASSES = 2; +const SELECTED_WORKSPACE_SETTLEMENT_RESERVE_MS = 5_000; +const SELECTED_WORKSPACE_MAX_QUEUE_RESERVE_MS = 30_000; export function normalizeProgrammaticTimeoutMs( rawTimeout: unknown, @@ -80,6 +84,31 @@ export function normalizeProgrammaticTimeoutMs( return Math.min(Math.ceil(rawTimeout), maxTimeout); } +/** + * Selected-workspace Bash replay may run one read-only probe and one commit + * pass in its final iteration. Bound each pass so both plus settlement reserve + * fit inside the worker-owned JOB_TIMEOUT instead of advertising a duration + * the assignment cannot complete. + */ +export function normalizeSelectedWorkspaceProgrammaticTimeoutMs( + rawTimeout: unknown, + jobTimeoutMs = env.JOB_TIMEOUT, +): number { + const totalBudget = Math.max(1, Math.floor(jobTimeoutMs)); + const queueReserve = Math.min( + SELECTED_WORKSPACE_MAX_QUEUE_RESERVE_MS, + Math.floor(totalBudget / 5), + ); + const executionBudget = Math.max( + 1, + totalBudget - queueReserve - SELECTED_WORKSPACE_SETTLEMENT_RESERVE_MS - programmaticTransferReserveMs(totalBudget), + ); + return normalizeProgrammaticTimeoutMs( + rawTimeout, + Math.max(1, Math.floor(executionBudget / SELECTED_WORKSPACE_REPLAY_PASSES)), + ); +} + export async function sealPtcCallbackTokenForGateway(args: { executionId: string; sessionId: string; diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 9896518c..4de4a4d8 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -11,12 +11,19 @@ import { connection, getExecutionQueueBinding, } from '../queue'; -import { createProgrammaticPayload, extractPendingFromStdout } from '../preamble'; +import { + createProgrammaticPayload, + extractPendingFromControlPayload, + extractPendingFromStdout, +} from '../preamble'; import { findBashToolNameCollision } from '../preamble-bash'; import type { LCTool } from '../preamble'; import { isReservedPtcFilename } from '../ptc-constants'; import { internalServiceHeaders } from '../internal-service-auth'; -import { resolveOutputBucketSessionKey, SessionKeyResolutionError } from '../session-key'; +import { + resolveOutputBucketSessionKey, + SessionKeyResolutionError, +} from '../session-key'; import { getCredentialId, getPrincipalOrReject } from '../auth/principal'; import { getExecutionIdentity } from '../execution-identity'; import { PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION } from '../runtime-session/job-policy'; @@ -34,15 +41,25 @@ import { publicExecutionFailure } from '../utils'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, + normalizeSelectedWorkspaceProgrammaticTimeoutMs, prepareSandboxJobSecurity, sealPtcCallbackTokenForGateway, timeoutMsToGrantSeconds, } from '../sandbox-egress'; import { findUnregisteredToolCall } from '../tool-scope'; import { summarizeRequestedFiles } from '../execution-log'; -import { pollBlockingExecution, type BlockingPendingState } from './blocking-poll'; -import { clearSessionOwnership, recordSessionOwnership } from '../session-ownership'; -import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; +import { + pollBlockingExecution, + type BlockingPendingState, +} from './blocking-poll'; +import { + clearSessionOwnership, + recordSessionOwnership, +} from '../session-ownership'; +import { + FileRefAuthorizationError, + authorizeRequestedFiles, +} from './file-authorization'; import { buildReplayExecutionState, resolveReplayStateSandboxBackend, @@ -50,8 +67,10 @@ import { import { BridgeWorkerSelectionError, CODEAPI_BRIDGE_WORKER_HEADER, + CODEAPI_BRIDGE_WORKSPACE_HEADER, resolveBridgeWorkerSelection, } from '../bridge/selection'; +import { isValidBridgeWorkerId, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES } from '../../../packages/code/src/protocol'; import logger from '../logger'; import { type ExecutionState, @@ -145,26 +164,41 @@ async function retryToolCallServerRequest( ): Promise { let lastError: Error | undefined; - for (let attempt = 1; attempt <= TOOL_CALL_SERVER_RETRY_ATTEMPTS; attempt++) { + for ( + let attempt = 1; + attempt <= TOOL_CALL_SERVER_RETRY_ATTEMPTS; + attempt++ + ) { try { return await requestFn(); } catch (error) { lastError = error as Error; if (axios.isAxiosError(error)) { - if (error.response && error.response.status >= 400 && error.response.status < 500) { + if ( + error.response && + error.response.status >= 400 && + error.response.status < 500 + ) { throw error; } } if (attempt < TOOL_CALL_SERVER_RETRY_ATTEMPTS) { - logger.warn(`${context} failed (attempt ${attempt}/${TOOL_CALL_SERVER_RETRY_ATTEMPTS}), retrying...`, { + logger.warn( + `${context} failed (attempt ${attempt}/${TOOL_CALL_SERVER_RETRY_ATTEMPTS}), retrying...`, + { error: lastError.message, - }); - await new Promise(resolve => setTimeout(resolve, TOOL_CALL_SERVER_RETRY_DELAY * attempt)); + }, + ); + await new Promise(resolve => + setTimeout(resolve, TOOL_CALL_SERVER_RETRY_DELAY * attempt), + ); } } } - logger.error(`${context} failed after ${TOOL_CALL_SERVER_RETRY_ATTEMPTS} attempts`); + logger.error( + `${context} failed after ${TOOL_CALL_SERVER_RETRY_ATTEMPTS} attempts`, + ); throw lastError; } @@ -179,7 +213,9 @@ setInterval(() => { }, STALE_CLEANUP_INTERVAL_MS); function generateContinuationToken(execution_id: string): string { - return Buffer.from(JSON.stringify({ execution_id, ts: Date.now() })).toString('base64'); + return Buffer.from( + JSON.stringify({ execution_id, ts: Date.now() }), + ).toString('base64'); } /** Map a replay-continuation HTTP status to its operational outcome @@ -200,14 +236,21 @@ function classifyContinuationOutcome(statusCode: number): string { * timestamp is older than the execution-state TTL — without this, the * `ts` field was dead data and a client could replay an ancient token * against a freshly-reused-execution-id window. */ -function decodeContinuationToken(token: string): { execution_id: string } | null { +function decodeContinuationToken( + token: string, +): { execution_id: string } | null { try { - const parsed: unknown = JSON.parse(Buffer.from(token, 'base64').toString('utf-8')); + const parsed: unknown = JSON.parse( + Buffer.from(token, 'base64').toString('utf-8'), + ); if (parsed === null || typeof parsed !== 'object') { return null; } const candidate = parsed as { execution_id?: unknown; ts?: unknown }; - if (typeof candidate.execution_id !== 'string' || candidate.execution_id.length === 0) { + if ( + typeof candidate.execution_id !== 'string' || + candidate.execution_id.length === 0 + ) { return null; } if (typeof candidate.ts === 'number' && Number.isFinite(candidate.ts)) { @@ -226,13 +269,17 @@ function decodeContinuationToken(token: string): { execution_id: string } | null // Blocking mode (legacy path) // --------------------------------------------------------------------------- -function waitForExecutionState(execution_id: string, timeout: number): ReturnType { +function waitForExecutionState( + execution_id: string, + timeout: number, +): ReturnType { return pollBlockingExecution(execution_id, timeout, { getExecutionState, getBlockingResult, - getPending: async (id) => { + getPending: async id => { const response = await retryToolCallServerRequest( - () => axios.get( + () => + axios.get( `${env.TOOL_CALL_SERVER_URL}/sessions/${id}/pending`, { headers: internalServiceHeaders() }, ), @@ -240,7 +287,8 @@ function waitForExecutionState(execution_id: string, timeout: number): ReturnTyp ); return response.data; }, - isNotFound: (error) => axios.isAxiosError(error) && error.response?.status === 404, + isNotFound: error => + axios.isAxiosError(error) && error.response?.status === 404, sleep: () => new Promise(resolve => setTimeout(resolve, POLL_INTERVAL)), now: Date.now, }); @@ -295,7 +343,8 @@ async function runReplayIteration( }); if (DEBUG_MODE) { - const firstFile = rawPayload.files[0] as { content?: string } | undefined; + const firstFile = rawPayload.files[0] as + { content?: string } | undefined; logger.debug('Replay enqueue details', { execution_id: state.execution_id, historySize: Object.keys(history).length, @@ -320,7 +369,9 @@ async function runReplayIteration( state.executionProfile ?? env.EXECUTION_PROFILE, state.executionProfileSource ?? env.EXECUTION_PROFILE_SOURCE, ); - const job = await queue.add(Jobs.execute, { + const job = await queue.add( + Jobs.execute, + { code: state.userCode ?? '', userId, payload: sandboxSecurity.payload, @@ -332,17 +383,24 @@ async function runReplayIteration( canonicalUserId: state.canonicalUserId, executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, sandboxBackend: replayBackend, - ...(state.bridgeWorkerId != null ? { bridgeWorkerId: state.bridgeWorkerId } : {}), + ...(state.bridgeWorkerId != null + ? { bridgeWorkerId: state.bridgeWorkerId } + : {}), + ...(state.workspaceId != null + ? { workspaceId: state.workspaceId } + : {}), runtimeSessionMode: 'stateless', runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, executionManifestClaims: sandboxSecurity.executionManifestClaims, egressGrantClaims: sandboxSecurity.egressGrantClaims, egressGrantToken: sandboxSecurity.egressGrantToken, - }, { + }, + { removeOnComplete: { age: 60, count: 1 }, removeOnFail: { age: 180, count: 1 }, attempts: 1, - }); + }, + ); jobsSubmitted.inc({ language }); return job.waitUntilFinished(events, JOB_COMPLETION_WAIT_TIMEOUT_MS); @@ -365,18 +423,24 @@ async function handleReplayInitial( apiKeyId: string; userId: string; bridgeWorkerId?: string; + workspaceId?: string; }, ): Promise { - const { apiKeyId, userId, bridgeWorkerId } = params; - const { - code, - tools, - user_id, - files, - } = req.body as t.ProgrammaticRequestBody; + const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; + const { code, tools, user_id, files } = + req.body as t.ProgrammaticRequestBody; let timeout: number; try { - timeout = normalizeProgrammaticTimeoutMs((req.body as t.ProgrammaticRequestBody).timeout); + if (workspaceId != null && Array.isArray(files) && files.length > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES) { + throw new Error(`Selected-workspace execution allows at most ${BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES} input files; main and replay history occupy two reserved slots`); + } + timeout = workspaceId != null + ? normalizeSelectedWorkspaceProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ) + : normalizeProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ); } catch (error) { res.status(400).json({ error: (error as Error).message }); return; @@ -400,14 +464,23 @@ async function handleReplayInitial( }); return; } - const language: 'python' | 'bash' = requestedLanguage === 'bash' ? 'bash' : 'python'; + const language: 'python' | 'bash' = + requestedLanguage === 'bash' ? 'bash' : 'python'; + if (workspaceId != null && language !== 'bash') { + res.status(400).json({ + error: 'Selected-workspace programmatic execution supports bash only', + }); + return; + } if (!code) { res.status(400).json({ error: 'Missing required field: code' }); return; } - if (!tools || !Array.isArray(tools) || tools.length === 0) { - res.status(400).json({ error: 'Missing required field: tools (must be a non-empty array)' }); + if (!Array.isArray(tools) || (tools.length === 0 && workspaceId == null)) { + res.status(400).json({ + error: 'Missing required field: tools (must be non-empty unless a selected workspace executes bash)', + }); return; } if (tools.length > MAX_TOOLS_PER_REQUEST) { @@ -442,7 +515,8 @@ async function handleReplayInitial( files, store: connection, }); - (req.body as t.ProgrammaticRequestBody).files = authorizedFiles.length > 0 ? authorizedFiles : undefined; + (req.body as t.ProgrammaticRequestBody).files = + authorizedFiles.length > 0 ? authorizedFiles : undefined; } catch (error) { if (sendFileRefAuthorizationError(error, res, req)) return; logger.error('Error authorizing replay file refs:', error); @@ -456,7 +530,14 @@ async function handleReplayInitial( try { sessionKey = resolveOutputBucketSessionKey(req); } catch (error) { - if (sendSessionKeyResolutionError(error, res, req, 'programmatic /exec: resolveOutputBucketSessionKey')) { + if ( + sendSessionKeyResolutionError( + error, + res, + req, + 'programmatic /exec: resolveOutputBucketSessionKey', + ) + ) { return; } throw error; @@ -466,9 +547,9 @@ async function handleReplayInitial( const execution_id = nanoid(); const authContext = req.codeApiAuthContext; const identity = getExecutionIdentity(req, userId); - const isPyPlot = language === 'python' && ( - code.includes('import matplotlib') || code.includes('import seaborn') - ); + const isPyPlot = + language === 'python' && + (code.includes('import matplotlib') || code.includes('import seaborn')); await recordSessionOwnership(connection, session_id, sessionKey); @@ -487,6 +568,7 @@ async function handleReplayInitial( timeout, language, bridgeWorkerId, + workspaceId, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -506,13 +588,16 @@ async function handleReplayInitial( await setExecutionState(state); } catch (err) { if (err instanceof ExecutionStateTooLargeError) { - logger.warn('Rejecting replay request: ExecutionState exceeds Redis cap', { + logger.warn( + 'Rejecting replay request: ExecutionState exceeds Redis cap', + { execution_id, userId, apiKeyId, bytes: err.bytes, cap: err.cap, - }); + }, + ); await clearSessionOwnership(connection, session_id).catch(() => {}); ptcReplayStateOversize.inc(); res.status(413).json({ @@ -568,9 +653,15 @@ async function handleReplayContinuation( * adds outcome plumbing through `runAndRespond`. */ const startMs = performance.now(); res.once('finish', () => { - const labels = { mode: 'replay' as const, outcome: classifyContinuationOutcome(res.statusCode) }; + const labels = { + mode: 'replay' as const, + outcome: classifyContinuationOutcome(res.statusCode), + }; ptcReplayContinuations.inc(labels); - ptcReplayContinuationDuration.observe(labels, (performance.now() - startMs) / 1000); + ptcReplayContinuationDuration.observe( + labels, + (performance.now() - startMs) / 1000, + ); }); /** Reject oversized batches before we spend any CPU on per-entry @@ -621,9 +712,14 @@ async function handleReplayContinuation( call_site: emitted.call_site, }; }); - const deltaOrError = await computeToolHistoryDelta(state.execution_id, enrichedResults); + const deltaOrError = await computeToolHistoryDelta( + state.execution_id, + enrichedResults, + ); if ('error' in deltaOrError) { - res.status(deltaOrError.status ?? 400).json({ error: deltaOrError.error }); + res.status(deltaOrError.status ?? 400).json({ + error: deltaOrError.error, + }); return; } const delta = deltaOrError; @@ -642,7 +738,9 @@ async function handleReplayContinuation( }); if (!pre.ok) { if (pre.status === 403) { - logger.warn('Unauthorized replay continuation request rejected', { + logger.warn( + 'Unauthorized replay continuation request rejected', + { execution_id: state.execution_id, requestUserId: userId, requestApiKeyId: apiKeyId, @@ -650,7 +748,8 @@ async function handleReplayContinuation( executionUserId: state.userId, executionApiKeyId: state.apiKeyId, executionTenantId: state.tenantId, - }); + }, + ); } if (pre.cleanupOnReject === true) { await cleanupExecution(state.execution_id, 'replay'); @@ -672,7 +771,10 @@ async function handleReplayContinuation( * Redis MULTI/EXEC so counters and the hash can't drift out of sync * on a partial failure. */ state.callCount = (state.callCount ?? 0) + delta.newCallIds.length; - state.historyBytes = Math.max(0, (state.historyBytes ?? 0) + delta.bytesDelta); + state.historyBytes = Math.max( + 0, + (state.historyBytes ?? 0) + delta.bytesDelta, + ); state.lastActivity = Date.now(); try { await commitToolHistoryAndState(state, delta); @@ -687,14 +789,19 @@ async function handleReplayContinuation( * forward is a fresh execution with smaller inputs. Reap the * old execution to free the lock and Redis keys, then return * an actionable 413 instead of a generic 500. */ - logger.warn('Replay continuation rejected: ExecutionState exceeds Redis cap', { + logger.warn( + 'Replay continuation rejected: ExecutionState exceeds Redis cap', + { execution_id: state.execution_id, bytes: err.bytes, cap: err.cap, callCount: state.callCount, historyBytes: state.historyBytes, - }); - await cleanupExecution(state.execution_id, 'replay').catch(() => {}); + }, + ); + await cleanupExecution(state.execution_id, 'replay').catch( + () => {}, + ); ptcReplayStateOversize.inc(); res.status(413).json({ status: 'error', @@ -715,10 +822,13 @@ async function handleReplayContinuation( * the throw bubble to the top-level catch and become an opaque * 500 — clients (and load balancers) treat 5xx classes very * differently for retry policy. */ - logger.error('Failed to commit replay continuation; returning retryable 503', { + logger.error( + 'Failed to commit replay continuation; returning retryable 503', + { execution_id: state.execution_id, err: (err as Error).message, - }); + }, + ); res.status(503).json({ status: 'error', error: 'Failed to persist replay continuation; please retry the same request', @@ -764,11 +874,15 @@ async function runAndRespond( try { result = await runReplayIteration(req, state, apiKeyId, userId); } catch (err) { - logger.error('Replay iteration failed', { execution_id: state.execution_id, err }); + logger.error('Replay iteration failed', { + execution_id: state.execution_id, + err, + }); await cleanupExecution(state.execution_id, 'replay'); if (!isDisconnected()) { const publicFailure = publicExecutionFailure(err); - const message = publicFailure?.body.message ?? (err as Error).message; + const message = + publicFailure?.body.message ?? (err as Error).message; res.status(200).json({ status: 'error', error: message !== '' ? message : 'Sandbox execution failed', @@ -786,10 +900,19 @@ async function runAndRespond( return; } - const { stdout: cleanStdout, pending } = extractPendingFromStdout( + const extracted = extractPendingFromStdout( result.stdout, state.execution_id, ); + const cleanStdout = extracted.stdout; + const controlPayload = result.pending_tool_calls_payload; + const hasControlPayload = typeof controlPayload === 'string'; + const controlPending = hasControlPayload + ? extractPendingFromControlPayload(controlPayload) + : null; + const pending = hasControlPayload + ? (controlPending ?? []) + : extracted.pending; if (pending != null) { if (pending.length === 0) { @@ -806,7 +929,10 @@ async function runAndRespond( }); return; } - const unregisteredToolCall = findUnregisteredToolCall(pending, state.tools); + const unregisteredToolCall = findUnregisteredToolCall( + pending, + state.tools, + ); if (unregisteredToolCall != null) { logger.warn('Sandbox requested unregistered replay tool call', { execution_id: state.execution_id, @@ -866,11 +992,16 @@ async function runAndRespond( await setExecutionState(state); await refreshExecutionTtl(state.execution_id); } catch (err) { - logger.error('Failed to persist execution state before continuation; aborting', { + logger.error( + 'Failed to persist execution state before continuation; aborting', + { execution_id: state.execution_id, err: (err as Error).message, - }); - await cleanupExecution(state.execution_id, 'replay').catch(() => {}); + }, + ); + await cleanupExecution(state.execution_id, 'replay').catch( + () => {}, + ); if (!isDisconnected()) { if (err instanceof ExecutionStateTooLargeError) { /** A continuation that pushes `emittedCallIds` past the @@ -887,8 +1018,7 @@ async function runAndRespond( } else { res.status(503).json({ status: 'error', - error: - 'Failed to persist replay state; please retry the request from scratch', + error: 'Failed to persist replay state; please retry the request from scratch', session_id: state.session_id, }); } @@ -912,7 +1042,8 @@ async function runAndRespond( if (!isSandboxRunSuccess(result)) { await cleanupExecution(state.execution_id, 'replay'); - const errorMessage = result.message != null && result.message !== '' + const errorMessage = + result.message != null && result.message !== '' ? result.message : `Sandbox exited with code ${result.code ?? 'unknown'}`; res.status(200).json({ @@ -941,7 +1072,10 @@ async function runAndRespond( // Request entrypoint // --------------------------------------------------------------------------- -router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedRequest, res) => { +router.post( + '/exec/programmatic', + executionLimiter, + async (req: t.AuthenticatedRequest, res) => { const principal = getPrincipalOrReject(req, res); if (!principal) return; const apiKeyId = getCredentialId(req); @@ -954,13 +1088,12 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR return res.status(503).json({ error: 'Service is starting up' }); } - const { - continuation_token, - tool_results, - } = req.body as t.ProgrammaticRequestBody; + const { continuation_token, tool_results } = + req.body as t.ProgrammaticRequestBody; const rawBody = req.body as Record; const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; + let workspaceId: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -970,12 +1103,35 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), trustedWorkerId: principal.codeWorkerId, }); - bridgeWorkerId = bridgeSelection?.explicit === true - ? bridgeSelection.workerId - : undefined; + bridgeWorkerId = + bridgeSelection?.explicit === true || + (bridgeSelection != null && !env.BRIDGE_DYNAMIC_WORKERS) + ? bridgeSelection.workerId + : undefined; + const requestedWorkspaceId = req + .header(CODEAPI_BRIDGE_WORKSPACE_HEADER) + ?.trim(); + if ( + requestedWorkspaceId != null && + requestedWorkspaceId !== '' + ) { + if (bridgeWorkerId == null) { + return res.status(400).json({ + error: 'Workspace selection requires an authenticated bridge worker', + }); + } + if (!isValidBridgeWorkerId(requestedWorkspaceId)) { + return res + .status(400) + .json({ error: 'Invalid code workspace ID' }); + } + workspaceId = requestedWorkspaceId; + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { - return res.status(error.status).json({ error: error.message }); + return res + .status(error.status) + .json({ error: error.message }); } throw error; } @@ -1013,7 +1169,9 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR } const decoded = decodeContinuationToken(continuation_token); if (!decoded) { - return res.status(400).json({ error: 'Invalid continuation token' }); + return res + .status(400) + .json({ error: 'Invalid continuation token' }); } const existing = await getExecutionState(decoded.execution_id); if (existing?.mode === 'replay') { @@ -1036,9 +1194,23 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR }); } if (env.PTC_MODE === 'replay') { - return await handleReplayInitial(req, res, { apiKeyId, userId, bridgeWorkerId }); + return await handleReplayInitial(req, res, { + apiKeyId, + userId, + bridgeWorkerId, + workspaceId, + }); } - return await handleBlocking(req, res, { apiKeyId, userId, bridgeWorkerId }); + if (workspaceId != null) { + return res.status(400).json({ + error: 'Selected-workspace programmatic execution requires replay mode', + }); + } + return await handleBlocking(req, res, { + apiKeyId, + userId, + bridgeWorkerId, + }); } catch (err) { logger.error(`[${INSTANCE_ID}] Programmatic routing error:`, err); if (!res.headersSent) { @@ -1046,7 +1218,8 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR } return; } -}); + }, +); // --------------------------------------------------------------------------- // Blocking-mode handler (extracted from the original implementation). @@ -1059,46 +1232,47 @@ async function handleBlocking( params: { apiKeyId: string; userId: string; bridgeWorkerId?: string }, ): Promise> { const { apiKeyId, userId, bridgeWorkerId } = params; - const { - code, - tools, - user_id, - files, - continuation_token, - tool_results, - } = req.body as t.ProgrammaticRequestBody; + const { code, tools, user_id, files, continuation_token, tool_results } = + req.body as t.ProgrammaticRequestBody; let timeout: number; try { - timeout = normalizeProgrammaticTimeoutMs((req.body as t.ProgrammaticRequestBody).timeout); + timeout = normalizeProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ); } catch (error) { return res.status(400).json({ error: (error as Error).message }); } // CASE 1: Continuation - if (continuation_token != null && continuation_token !== '' && tool_results) { + if ( + continuation_token != null && + continuation_token !== '' && + tool_results + ) { const decoded = decodeContinuationToken(continuation_token); if (!decoded) { - return res.status(400).json({ error: 'Invalid continuation token' }); + return res + .status(400) + .json({ error: 'Invalid continuation token' }); } const { execution_id } = decoded; const execution = await getExecutionState(execution_id); if (!execution) { - return res.status(404).json({ error: 'Execution not found or expired' }); + return res + .status(404) + .json({ error: 'Execution not found or expired' }); } const identity = getExecutionIdentity(req, userId); if ( execution.userId !== userId || (execution.apiKeyId != null && execution.apiKeyId !== apiKeyId) || - ( - execution.tenantId != null && - execution.tenantId !== identity.storageNamespace - ) || - ( - execution.authContextHash != null && - execution.authContextHash !== req.codeApiAuthContext?.authContextHash - ) + (execution.tenantId != null && + execution.tenantId !== identity.storageNamespace) || + (execution.authContextHash != null && + execution.authContextHash !== + req.codeApiAuthContext?.authContextHash) ) { logger.warn('Unauthorized blocking continuation request rejected', { execution_id, @@ -1122,14 +1296,19 @@ async function handleBlocking( try { await retryToolCallServerRequest( - () => axios.post(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}/results`, { + () => + axios.post( + `${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}/results`, + { results: tool_results.map(r => ({ call_id: r.call_id, result: r.result, is_error: r.is_error ?? false, error_message: r.error_message, })), - }, { headers: internalServiceHeaders() }), + }, + { headers: internalServiceHeaders() }, + ), 'Submit tool results', ); @@ -1174,14 +1353,21 @@ async function handleBlocking( return res.status(400).json({ error: 'Missing required field: code' }); } if (!tools || !Array.isArray(tools) || tools.length === 0) { - return res.status(400).json({ error: 'Missing required field: tools (must be a non-empty array)' }); + return res + .status(400) + .json({ + error: 'Missing required field: tools (must be a non-empty array)', + }); } if (tools.length > MAX_TOOLS_PER_REQUEST) { - logger.warn(`Too many tools provided: ${tools.length}, limit is ${MAX_TOOLS_PER_REQUEST}`, { + logger.warn( + `Too many tools provided: ${tools.length}, limit is ${MAX_TOOLS_PER_REQUEST}`, + { execution_id: 'pre-creation', userId, toolCount: tools.length, - }); + }, + ); return res.status(400).json({ error: `Too many tools provided (${tools.length}). Maximum is ${MAX_TOOLS_PER_REQUEST}.`, }); @@ -1202,7 +1388,8 @@ async function handleBlocking( files, store: connection, }); - (req.body as t.ProgrammaticRequestBody).files = authorizedFiles.length > 0 ? authorizedFiles : undefined; + (req.body as t.ProgrammaticRequestBody).files = + authorizedFiles.length > 0 ? authorizedFiles : undefined; } catch (error) { if (sendFileRefAuthorizationError(error, res, req)) return; logger.error('Error authorizing programmatic file refs:', error); @@ -1215,7 +1402,14 @@ async function handleBlocking( try { sessionKey = resolveOutputBucketSessionKey(req); } catch (error) { - if (sendSessionKeyResolutionError(error, res, req, 'programmatic /exec-blocking: resolveOutputBucketSessionKey')) { + if ( + sendSessionKeyResolutionError( + error, + res, + req, + 'programmatic /exec-blocking: resolveOutputBucketSessionKey', + ) + ) { return; } throw error; @@ -1270,24 +1464,34 @@ async function handleBlocking( try { callbackUrl = normalizeEgressGatewayUrl(env.EGRESS_GATEWAY_URL); } catch (error) { - logger.error('Blocking PTC requires egress gateway callback URL:', error); + logger.error( + 'Blocking PTC requires egress gateway callback URL:', + error, + ); await cleanupExecution(execution_id, 'blocking'); - return res.status(503).json({ error: 'Egress gateway unavailable' }); + return res + .status(503) + .json({ error: 'Egress gateway unavailable' }); } let callbackToken: string; try { const toolCallResponse = await retryToolCallServerRequest( - () => axios.post<{ + () => + axios.post<{ success: boolean; callback_token: string; - }>(`${env.TOOL_CALL_SERVER_URL}/sessions`, { + }>( + `${env.TOOL_CALL_SERVER_URL}/sessions`, + { execution_id, session_id, timeout, tools, - }, { headers: internalServiceHeaders() }), + }, + { headers: internalServiceHeaders() }, + ), 'Create Tool Call Server session', ); @@ -1299,9 +1503,14 @@ async function handleBlocking( allowedToolNames: tools.map(tool => tool.name), }); } catch (error) { - logger.error('Failed to create Tool Call Server session or callback token:', error); + logger.error( + 'Failed to create Tool Call Server session or callback token:', + error, + ); await cleanupExecution(execution_id, 'blocking'); - return res.status(503).json({ error: 'Tool Call Server unavailable' }); + return res + .status(503) + .json({ error: 'Tool Call Server unavailable' }); } let rawPayload: t.PayloadBody; @@ -1316,10 +1525,15 @@ async function handleBlocking( timeout, }); } catch (error) { - logger.error('Failed to create payload', { execution_id, error: (error as Error).message }); + logger.error('Failed to create payload', { + execution_id, + error: (error as Error).message, + }); await cleanupExecution(execution_id, 'blocking'); return res.status(400).json({ - error: (error as Error).message || 'Failed to generate code payload', + error: + (error as Error).message || + 'Failed to generate code payload', }); } const sandboxSecurity = prepareSandboxJobSecurity({ @@ -1331,7 +1545,9 @@ async function handleBlocking( payload: rawPayload, }); - const job = await pyQueue.add(Jobs.execute, { + const job = await pyQueue.add( + Jobs.execute, + { code, userId, payload: sandboxSecurity.payload, @@ -1350,18 +1566,24 @@ async function handleBlocking( ...(bridgeWorkerId != null ? { bridgeWorkerId } : {}), runtimeSessionMode: 'stateless', runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, - executionManifestClaims: sandboxSecurity.executionManifestClaims, + executionManifestClaims: + sandboxSecurity.executionManifestClaims, egressGrantClaims: sandboxSecurity.egressGrantClaims, egressGrantToken: sandboxSecurity.egressGrantToken, - }, { + }, + { removeOnComplete: { age: 60, count: 1 }, removeOnFail: { age: 180, count: 1 }, attempts: 1, jobId: session_id, - }); + }, + ); jobsSubmitted.inc({ language: 'python' }); - logger.info('Job queued, polling for tool calls', { execution_id, session_id }); + logger.info('Job queued, polling for tool calls', { + execution_id, + session_id, + }); let clientDisconnected = false; req.on('close', async () => { @@ -1372,21 +1594,27 @@ async function handleBlocking( await job.remove(); await cleanupExecution(execution_id, 'blocking'); } catch (error) { - logger.error('Error cleaning up after client disconnect:', error); + logger.error( + 'Error cleaning up after client disconnect:', + error, + ); } }); job.waitUntilFinished(pyQueueEvents, JOB_COMPLETION_WAIT_TIMEOUT_MS) - .then(async (result) => { + .then(async result => { if (clientDisconnected) return; await setExecutionResult(execution_id, result); }) - .catch(async (error) => { + .catch(async error => { if (clientDisconnected) return; await setExecutionError(execution_id, error); }); - const state = await waitForExecutionState(execution_id, Math.min(timeout, MAX_POLL_TIME)); + const state = await waitForExecutionState( + execution_id, + Math.min(timeout, MAX_POLL_TIME), + ); if (state.status === 'waiting' && state.pending_calls) { return res.status(200).json({ @@ -1416,7 +1644,10 @@ async function handleBlocking( session_id, }); } catch (error) { - logger.error(`[${INSTANCE_ID}] Session ID: ${session_id} | Execution ID: ${execution_id} | Error:`, error); + logger.error( + `[${INSTANCE_ID}] Session ID: ${session_id} | Execution ID: ${execution_id} | Error:`, + error, + ); await cleanupExecution(execution_id, 'blocking'); return res.status(500).json({ error: 'Internal server error' }); } diff --git a/service/src/service/programmatic-state.test.ts b/service/src/service/programmatic-state.test.ts index fc84d8f8..81405021 100644 --- a/service/src/service/programmatic-state.test.ts +++ b/service/src/service/programmatic-state.test.ts @@ -83,6 +83,7 @@ describe('buildReplayExecutionState', () => { const state = build({ authContext, bridgeWorkerId: 'code-user_123', + workspaceId: 'project-a', sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -102,6 +103,7 @@ describe('buildReplayExecutionState', () => { authContextHash: 'hash_123', apiKeyId: 'key_legacy', bridgeWorkerId: 'code-user_123', + workspaceId: 'project-a', sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index 25571fed..e6bda59a 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -38,6 +38,7 @@ export interface BuildReplayExecutionStateParams { timeout: number; language: 'python' | 'bash'; bridgeWorkerId?: string; + workspaceId?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -66,6 +67,7 @@ export function buildReplayExecutionState( authContextHash: identity.authContextHash, apiKeyId: params.apiKeyId, bridgeWorkerId: params.bridgeWorkerId, + workspaceId: params.workspaceId, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 2254ee21..3b65cedf 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -116,6 +116,8 @@ export interface ExecutionState { apiKeyId?: string; /** Authenticated worker selection retained across every replay iteration. */ bridgeWorkerId?: string; + /** Selected workspace retained and bound across every replay iteration. */ + workspaceId?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 2a90eac7..f0a6da3b 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -3,7 +3,10 @@ import type { Request } from 'express'; import type { ExecutionManifestClaims } from '../execution-manifest'; import type { ExecutionIdentity } from '../execution-identity'; import type { CodeApiPrincipal } from '../auth/principal'; -import type { ExecutionProfile, SandboxBackendName } from '../execution-profile'; +import type { + ExecutionProfile, + SandboxBackendName, +} from '../execution-profile'; import { Jobs } from '@/enum/service'; /** @@ -149,7 +152,11 @@ export interface RequestBody { runtime_session_hint?: string; } -export type CreatePayload = { req: AuthenticatedRequest, session_id: string; isPyPlot?: boolean }; +export type CreatePayload = { + req: AuthenticatedRequest; + session_id: string; + isPyPlot?: boolean; +}; export interface FileObject { name: string; id: string; @@ -160,10 +167,12 @@ export interface FileObject { size?: number; lastModified?: string; etag?: string; - metadata?: { + metadata?: + | { 'content-type': string; 'original-filename': string; - } | undefined; + } + | undefined; versionId?: string | null; contentType?: string; } @@ -184,6 +193,14 @@ export type PayloadFileRef = { export interface PayloadBody { language: string; version: string; + /** Stable identity shared by all replay iterations of one execution. */ + execution_id?: string; + replay_tool_count?: number; + /** Manifest-bound upload ceiling exposed to remote workers. */ + max_output_files?: number; + /** Effective per-file ceiling after manifest and gateway policy intersect. */ + max_output_file_bytes?: number; + transfer_timeout_ms?: number; run_memory_limit?: number; run_timeout?: number; run_cpu_time?: number; @@ -238,6 +255,8 @@ export type ExecuteResult = { message?: string | null; status?: string | null; wall_time?: number | null; + /** Trusted worker control channel; avoids losing replay calls to stdout truncation. */ + pending_tool_calls_payload?: string; }; export interface LanguageConfig { @@ -265,6 +284,8 @@ export type JobData = { canonicalUserId?: string; /** Trusted dynamic outbound worker selection. */ bridgeWorkerId?: string; + /** Trusted selected workspace for native replay-mode PTC. */ + workspaceId?: string; /** Producer deployment identity. Optional only for pre-profile queued jobs. */ executionProfile?: ExecutionProfile; /** Required sandbox transport. Optional only for jobs queued before fencing. */ diff --git a/service/src/workers.ts b/service/src/workers.ts index ad20fd0f..dc2f491b 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -21,6 +21,7 @@ import { validateQueuedExecutionProfile, validateQueuedSandboxBackend, } from './execution-profile'; +import { BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, programmaticTransferReserveMs } from '../../packages/code/src/protocol'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; @@ -91,9 +92,15 @@ async function processJobInner(job: t.ExecuteJob): Promise { const delivery = prepareInputDelivery(payload, sandboxPayload); const sandboxRequest = buildSandboxExecuteRequest({ + ...(job.data.workspaceId == null ? {} : { programmaticTransferReserveMs: programmaticTransferReserveMs(env.JOB_TIMEOUT) }), payload: delivery.payload, egressGrantToken, executionManifestClaims, + maxOutputFileBytes: Math.min( + executionManifestClaims?.max_upload_bytes ?? env.EGRESS_GATEWAY_MAX_FILE_BYTES, + env.EGRESS_GATEWAY_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + ), executionManifestPrivateKey: env.EXECUTION_MANIFEST_PRIVATE_KEY, executionManifestSecret: env.EXECUTION_MANIFEST_SECRET, executionManifestTtlSeconds: env.EXECUTION_MANIFEST_TTL_SECONDS, @@ -149,6 +156,7 @@ async function processJobInner(job: t.ExecuteJob): Promise { tenantId: job.data.tenantId, canonicalUserId: job.data.canonicalUserId, bridgeWorkerId: job.data.bridgeWorkerId, + workspaceId: job.data.workspaceId, runtimeSessionId: runtimeSession.runtimeSessionId, runtimeSessionMode: runtimeSession.runtimeSessionMode, /* Stateful backends run this as a commit barrier after user code but @@ -184,6 +192,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { : {}), stdout, stderr, + ...(responseData.pending_tool_calls_payload != null + ? { pending_tool_calls_payload: responseData.pending_tool_calls_payload } + : {}), }; if (run) { From e4815fa6572a4380bd82678b5d500ce2d4c49fb2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 13:59:24 -0400 Subject: [PATCH 17/75] feat: Report Truncated Output Artifacts (#199) * feat: report truncated output artifacts * fix: classify omitted artifacts precisely * fix: preserve artifact scan invariants * fix: bound depth truncation probes * fix: bound capped directory enumeration * fix: constrain truncation probes across the job * fix: stop exhausted artifact probes --- api/README.md | 11 + api/src/job.ts | 279 ++++++++++++++- api/src/walker.test.ts | 375 +++++++++++++++++++++ service/src/execution-log.test.ts | 13 + service/src/execution-log.ts | 18 + service/src/service/blocking-poll.test.ts | 4 + service/src/service/blocking-poll.ts | 2 + service/src/service/programmatic-router.ts | 6 + service/src/types/service.ts | 12 + service/src/workers.ts | 3 + 10 files changed, 711 insertions(+), 12 deletions(-) diff --git a/api/README.md b/api/README.md index bbbe36ec..665a2931 100644 --- a/api/README.md +++ b/api/README.md @@ -94,6 +94,17 @@ Other package-format-compatible runtimes (Go, Rust, Java, GCC) can be installed Execute code in a sandboxed environment. +When supported output files are omitted because the response reaches its file +count limit, nesting or path limits, file-size limit, or a filesystem entry +cannot be read, the response includes `artifact_truncation`. Its `reasons` +object counts detected omissions by cause, `skipped_count` reports the total +detected omissions, and `skipped` contains up to 20 relative paths so callers +can match an expected output. Intentional filters such as unsupported file +extensions, hidden runtime directories, and unchanged session files do not +produce this marker when they can be classified within the bounded scan. A +depth-capped subtree that exceeds the metadata probe budget is reported +conservatively rather than allowing post-execution traversal to run unbounded. + ### `GET /api/v2/runtimes` List available language runtimes. diff --git a/api/src/job.ts b/api/src/job.ts index b7b82148..27604d29 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -36,9 +36,9 @@ import { SANDBOX_DIR_MODE, SANDBOX_FILE_MODE, ValidationError, + checkPathShape, hasRunnableSource, isDirkeep, - isValidPathShape, validateFilePath, isValidFilePath, } from './validation'; @@ -60,6 +60,15 @@ export { const AUTO_LOAD_DIRKEEP_TIMEOUT_MS = 10000; const AUTO_LOAD_DIRKEEP_RETRIES = 2; +const PTC_HISTORY_FILENAME = '_ptc_history.json'; +const TRUNCATION_PROBE_MAX_ENTRIES = 1000; +const TRUNCATION_PROBE_MAX_LEVELS = 10; +const TRUNCATION_PROBE_MAX_HASH_BYTES = 50_000_000; + +interface TruncationProbeState { + remainingEntries: number; + remainingHashBytes: number; +} /** Replaying the same sealed grant cannot repair an authorization denial. */ class InputAuthorizationError extends Error { @@ -647,8 +656,20 @@ interface ExecuteResult { session_id: string; files: FileRef[]; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; } +export type ArtifactTruncationReason = 'max_files' | 'depth' | 'size' | 'path' | 'unreadable'; + +export interface ArtifactTruncation { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; +} + +const MAX_REPORTED_TRUNCATED_PATHS = 20; + const jobQueue: Array<() => void> = []; async function acquireJobIdentity(log: Logger): Promise { @@ -693,6 +714,11 @@ export class Job { private pendingSurfaced = new Map(); private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; + private artifactTruncation: ArtifactTruncation | undefined; + private truncationProbeState: TruncationProbeState = { + remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, + remainingHashBytes: TRUNCATION_PROBE_MAX_HASH_BYTES, + }; private inputFileHashes = new Map(); private inputManifest = new Map(); private inputDestinations = new Map(); @@ -1673,6 +1699,7 @@ export class Job { version: this.runtime.version.raw, session_id: this.outputSessionId, files: this.sessionFiles, + ...(this.artifactTruncation ? { artifact_truncation: this.artifactTruncation } : {}), }; } @@ -1680,6 +1707,11 @@ export class Job { this.generatedFiles = []; this.sessionFiles = []; this.inheritedRefs = []; + this.artifactTruncation = undefined; + this.truncationProbeState = { + remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, + remainingHashBytes: TRUNCATION_PROBE_MAX_HASH_BYTES, + }; const inputByName = new Map(); for (const f of this.files) inputByName.set(f.name, f); @@ -1699,6 +1731,23 @@ export class Job { if (remaining > 0 && this.inheritedRefs.length > 0) { this.sessionFiles.push(...this.inheritedRefs.slice(0, remaining)); } + for (const ref of this.inheritedRefs.slice(remaining)) { + this.recordArtifactTruncation('max_files', ref.name); + } + } + + private recordArtifactTruncation(reason: ArtifactTruncationReason, relativePath: string): void { + this.artifactTruncation ??= { + code: 'artifact_truncated', + reasons: {}, + skipped: [], + skipped_count: 0, + }; + this.artifactTruncation.reasons[reason] = (this.artifactTruncation.reasons[reason] ?? 0) + 1; + this.artifactTruncation.skipped_count++; + if (this.artifactTruncation.skipped.length < MAX_REPORTED_TRUNCATED_PATHS) { + this.artifactTruncation.skipped.push(relativePath); + } } /** @@ -1722,6 +1771,7 @@ export class Job { isRegularFile = st.isFile(); } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: failed to lstat entry'); + this.recordArtifactTruncation('unreadable', relativePath); return 'skip'; } } @@ -1742,7 +1792,14 @@ export class Job { inputByName: Map, ): Promise<{ collected: boolean; truncated: boolean }> { const keepPath = path.join(relativePath, DIRKEEP); - if (!isValidPathShape(keepPath)) return { collected: false, truncated: false }; + const pathShapeError = checkPathShape(keepPath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + keepPath, + ); + return { collected: false, truncated: true }; + } const keepFullPath = path.join(fullPath, DIRKEEP); const inheritedKeep = inputByName.get(keepPath); @@ -1770,6 +1827,7 @@ export class Job { return this.createDirkeepMarker(keepPath, keepFullPath); } if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } const id = nanoid(); @@ -1819,6 +1877,7 @@ export class Job { if (!keepModified || keepInfo?.readOnly === true) return this.echoInheritedKeep(keepPath, inheritedKeep); if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } const refreshedId = nanoid(); @@ -1860,6 +1919,7 @@ export class Job { inheritedKeep: TFile, ): { collected: boolean; truncated: boolean } { if (this.inheritedRefs.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } this.inheritedRefs.push({ @@ -1886,6 +1946,7 @@ export class Job { keepFullPath: string, ): Promise<{ collected: boolean; truncated: boolean }> { if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } try { @@ -1944,6 +2005,7 @@ export class Job { if (existingFile.id && existingFile.storage_session_id) { if (this.inheritedRefs.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', relativePath); return { collected: false, truncated: true }; } this.inheritedRefs.push({ @@ -2003,9 +2065,29 @@ export class Job { size = st.size; } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: unable to stat file'); + this.recordArtifactTruncation('unreadable', relativePath); return { collected: false, truncated: false, stopLoop: false }; } + + const inputFileInfo = this.inputFileHashes.get(relativePath); + const existingFile = inputByName.get(relativePath); if (size > this.runtime.max_file_size) { + /* Only an inline entrypoint needs hashing to decide whether this is + * intentional request-input suppression. Every other oversized file + * is rejected immediately, preserving the scan's bounded I/O cost. */ + if (!inputFileInfo || existingFile?.id != null || relativePath !== this.entryPointName) { + this.recordArtifactTruncation('size', relativePath); + return { collected: false, truncated: false, stopLoop: false }; + } + try { + const currentHash = await this.computeFileHash(fullPath, true); + if (currentHash === inputFileInfo.hash) { + return { collected: true, truncated: false, stopLoop: false }; + } + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed to hash oversized entrypoint'); + } + this.recordArtifactTruncation('size', relativePath); return { collected: false, truncated: false, stopLoop: false }; } @@ -2015,8 +2097,6 @@ export class Job { * stat-only signature would wrongly suppress. Compute once per session/input * file and reuse for the suppression check, wasModified, and the surfaced * mark; non-session jobs still only hash their inputs. */ - const inputFileInfo = this.inputFileHashes.get(relativePath); - const existingFile = inputByName.get(relativePath); let contentHash: string | undefined; if (inputFileInfo != null || this.session != null) { try { @@ -2058,6 +2138,15 @@ export class Job { if (wasModified) this.log.info({ file: relativePath }, 'Input file was modified'); } + /* The unchanged inline entrypoint is executable request input, not an + * output artifact. Suppress it before applying output-size reporting; + * downloaded inputs still flow through the size limit below, preserving + * the existing response-cap behavior for inherited refs. */ + if (!wasModified && inputFileInfo && existingFile?.id == null + && relativePath === this.entryPointName) { + return { collected: true, truncated: false, stopLoop: false }; + } + const echoed = this.tryEchoUnchangedInput({ wasModified, inputFileInfo, @@ -2067,6 +2156,7 @@ export class Job { if (echoed) return { ...echoed, stopLoop: false }; if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', relativePath); return { collected: false, truncated: true, stopLoop: true }; } @@ -2105,10 +2195,130 @@ export class Job { const childStatus = await this.walkDir(fullPath, parentDepth + 1, inputByName); if (childStatus === 'collected') return { collected: true, truncated: false }; if (childStatus === 'skipped') return { collected: false, truncated: true }; - if (this.isOutputCapFull()) return { collected: false, truncated: true }; return this.handleEmptyDirectory(relativePath, fullPath, inputByName); } + /** Finds the first artifact that a scan cap would hide without reading file + * contents. Files below a depth boundary cannot be valid primed inputs, and + * symlinks/unsupported files/hidden runtime directories remain intentional + * exclusions. An empty directory represents a reportable `.dirkeep`. */ + private async findTruncatedArtifact( + dir: string, + inputByName: Map, + state = this.truncationProbeState, + probeDepth = 0, + rootPath = path.relative(this.submissionDir, dir) || '.', + respectSessionSuppression = false, + ): Promise { + /* The state is shared by every probe in this job. Once exhausted, return + * conservatively before opening yet another capped sibling directory. */ + if (state.remainingEntries <= 0) return rootPath; + let directory: fs.Dir; + try { + directory = await fsp.opendir(dir); + } catch (err) { + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + this.log.debug({ dir, err }, 'walkDir: unable to inspect depth-capped directory'); + this.recordArtifactTruncation('unreadable', relativeDir); + return undefined; + } + + let sawVisibleEntry = false; + let sawVisibleNonHiddenEntry = false; + try { + for await (const entry of directory) { + if (entry.name === PTC_HISTORY_FILENAME) continue; + sawVisibleEntry = true; + state.remainingEntries--; + if (state.remainingEntries < 0) return rootPath; + const fullPath = path.join(dir, entry.name); + const relativePath = path.relative(this.submissionDir, fullPath); + const kind = await this.classifyDirent(entry, fullPath, relativePath); + if (kind === 'skip') { + /* Ordinary walking counts symlinks/special entries as non-empty even + * though it does not surface them, so the probe must not invent a + * parent .dirkeep for that shape. */ + sawVisibleNonHiddenEntry = true; + continue; + } + if (kind === 'file') { + sawVisibleNonHiddenEntry = true; + if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; + const existingFile = inputByName.get(relativePath); + const inputFileInfo = this.inputFileHashes.get(relativePath); + if ( + respectSessionSuppression + && relativePath === this.entryPointName + && existingFile?.id == null + && inputFileInfo + ) { + try { + const st = await fsp.lstat(fullPath); + if (!st.isFile()) continue; + if (st.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= st.size; + if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during entrypoint cap probe'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + } + /* Once generated outputs fill the response cap, a persistent + * workspace may still contain unchanged artifacts from earlier + * turns. Ordinary walking suppresses those via their content hash, + * so the bounded cap probe must do the same or it reports a false + * max_files warning. Current-request inputs remain reportable: they + * would otherwise have been echoed into this response. */ + if (respectSessionSuppression && this.session && !existingFile) { + if (this.session.isPrimedReadOnly(relativePath)) continue; + try { + const st = await fsp.lstat(fullPath); + if (!st.isFile()) continue; + if (st.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= st.size; + const hash = await this.computeFileHash(fullPath, true); + if (this.session.isSurfaced(relativePath, hash)) continue; + if ( + this.session.isPrimedInput(relativePath) + && this.session.primedHash(relativePath) === hash + ) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during cap-probe hashing'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + } + return relativePath; + } + if (isHiddenDirectory(entry.name) && !inputsLiveUnder(inputByName, relativePath)) continue; + sawVisibleNonHiddenEntry = true; + /* The probe exists only to avoid false warnings for small, obviously + * unsupported-only subtrees. Once either budget is exhausted, report + * the capped root conservatively instead of defeating the scan bound. */ + if (probeDepth >= TRUNCATION_PROBE_MAX_LEVELS) return rootPath; + const nested = await this.findTruncatedArtifact( + fullPath, + inputByName, + state, + probeDepth + 1, + rootPath, + respectSessionSuppression, + ); + if (nested) return nested; + } + } catch (err) { + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + this.log.debug({ dir, err }, 'walkDir: failed during bounded directory inspection'); + this.recordArtifactTruncation('unreadable', relativeDir); + return undefined; + } + + return sawVisibleEntry && sawVisibleNonHiddenEntry + ? undefined + : path.join(path.relative(this.submissionDir, dir), DIRKEEP); + } + /** * Recursively scans the submission directory for output files. Returns a * status distinguishing truly empty directories from scans truncated by @@ -2120,14 +2330,30 @@ export class Job { depth: number, inputByName: Map, ): Promise<'collected' | 'empty' | 'skipped'> { - if (depth >= config.max_nesting_depth) return 'skipped'; - if (this.isOutputCapFull()) return 'skipped'; - + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + if (depth >= config.max_nesting_depth) { + const skippedPath = await this.findTruncatedArtifact(dir, inputByName); + if (skippedPath) this.recordArtifactTruncation('depth', skippedPath); + return 'skipped'; + } + if (this.isOutputCapFull()) { + const skippedPath = await this.findTruncatedArtifact( + dir, + inputByName, + this.truncationProbeState, + 0, + relativeDir, + true, + ); + if (skippedPath) this.recordArtifactTruncation('max_files', skippedPath); + return 'skipped'; + } let entries: fs.Dirent[]; try { entries = await fsp.readdir(dir, { withFileTypes: true }); } catch (err) { this.log.debug({ dir, err }, 'walkDir: unable to read directory'); + this.recordArtifactTruncation('unreadable', relativeDir); return 'skipped'; } @@ -2146,7 +2372,6 @@ export class Job { * separate npm packages so we can't import directly; the filename literal * is asserted-equal in `service/scripts/test-ptc-sentinel.ts` to catch * accidental drift in CI. */ - const PTC_HISTORY_FILENAME = '_ptc_history.json'; const isPtcReserved = (name: string): boolean => name === PTC_HISTORY_FILENAME; const nonDirkeepCount = entries.reduce( @@ -2166,13 +2391,10 @@ export class Job { let skippedHiddenDirs = 0; for (const entry of entries) { - if (this.isOutputCapFull()) { truncated = true; break; } if (isPtcReserved(entry.name)) continue; const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); - if (!isValidPathShape(relativePath)) continue; - const kind = await this.classifyDirent(entry, fullPath, relativePath); if (kind === 'skip') continue; @@ -2189,9 +2411,42 @@ export class Job { skippedHiddenDirs++; continue; } + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + const skippedPath = await this.findTruncatedArtifact( + fullPath, + inputByName, + this.truncationProbeState, + 0, + relativePath, + ); + if (skippedPath) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + skippedPath, + ); + truncated = true; + } + continue; + } const res = await this.walkSubdirectory(relativePath, fullPath, depth, inputByName); if (res.collected) hasCollectedChild = true; if (res.truncated) truncated = true; + if (this.isOutputCapFull() && this.artifactTruncation?.reasons.max_files) break; + continue; + } + + /* Check intentional filename filtering before path limits. Unsupported + * files never belong in files[], regardless of how long their path is. */ + if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; + + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + relativePath, + ); + truncated = true; continue; } diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index 91382468..543505a9 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -26,11 +26,19 @@ interface WalkerInternals { generatedFiles: Array<{ id: string; name: string; path: string }>; sessionFiles: Array<{ id: string; name: string; storage_session_id: string; modified_from?: { id: string; storage_session_id: string }; inherited?: true; entity_id?: string }>; inheritedRefs: Array<{ id: string; name: string; storage_session_id: string; inherited?: true; entity_id?: string }>; + artifactTruncation?: { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; + }; + truncationProbeState: { remainingEntries: number; remainingHashBytes: number }; pendingSurfaced: Map; inputFileHashes: Map; files: TFile[]; reusePrimedInput: (file: TFile) => Promise; writeFile: (file: TFile) => Promise; + computeFileHash: (filePath: string, noFollow?: boolean) => Promise; walkDir: (dir: string, depth: number, inputByName: Map) => Promise<'collected' | 'empty' | 'skipped'>; handleSessionFiles: () => Promise; } @@ -743,6 +751,10 @@ describe('walkDir / output caps', () => { await internals.walkDir(tmpDir, 0, new Map()); expect(internals.generatedFiles.length).toBeLessThanOrEqual(cap); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + }); }); it('respects max_output_files cap on inherited refs', async () => { @@ -771,6 +783,11 @@ describe('walkDir / output caps', () => { expect(internals.inheritedRefs.length).toBeLessThanOrEqual(cap); expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { max_files: 5 }, + skipped_count: 5, + }); }); }); @@ -792,6 +809,364 @@ describe('walkDir / depth cap', () => { const deepName = path.relative(tmpDir, path.join(cursor, 'deep.py')); expect(internals.generatedFiles.map(f => f.name)).not.toContain(deepName); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { depth: 1 }, + skipped_count: 1, + }); + expect(internals.artifactTruncation?.skipped[0]).toBe(deepName); + }); + + it('does not report a depth cap when the skipped subtree has only unsupported files', async () => { + let cursor = tmpDir; + for (let i = 0; i < config.max_nesting_depth; i++) { + cursor = path.join(cursor, `d${i}`); + await fsp.mkdir(cursor); + } + await fsp.writeFile(path.join(cursor, 'cache.bin'), 'ignored'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('bounds depth-cap eligibility probes and reports the capped subtree conservatively', async () => { + let cursor = tmpDir; + const totalDepth = config.max_nesting_depth + 12; + for (let i = 0; i < totalDepth; i++) { + cursor = path.join(cursor, `d${i}`); + await fsp.mkdir(cursor); + } + await fsp.writeFile(path.join(cursor, 'cache.bin'), 'ignored'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + const cappedRoot = Array.from( + { length: config.max_nesting_depth }, + (_, i) => `d${i}`, + ).join(path.sep); + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { depth: 1 }, + skipped: [cappedRoot], + skipped_count: 1, + }); + }); +}); + +describe('walkDir / artifact truncation details', () => { + it('reports oversized supported outputs while leaving them out of files', async () => { + await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); + const job = makeJob({ maxFileSize: 3 }); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { size: 1 }, + skipped: ['large.txt'], + skipped_count: 1, + }); + }); + + it('reports overlong output paths', async () => { + const directory = 'a'.repeat(200); + await fsp.mkdir(path.join(tmpDir, directory)); + const name = path.join(directory, `${'b'.repeat(60)}.txt`); + await fsp.writeFile(path.join(tmpDir, name), 'content'); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [name], + skipped_count: 1, + }); + }); + + it('does not report an overlong path for an unsupported output', async () => { + const directory = 'a'.repeat(200); + await fsp.mkdir(path.join(tmpDir, directory)); + await fsp.writeFile(path.join(tmpDir, directory, `${'b'.repeat(60)}.bin`), 'ignored'); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('reports an empty-directory marker whose appended path is too long', async () => { + const directory = 'a'.repeat(config.max_path_length - 6); + await fsp.mkdir(path.join(tmpDir, directory)); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [path.join(directory, DIRKEEP)], + skipped_count: 1, + }); + }); + + it('reports an explicit overlong .dirkeep exactly once', async () => { + const directory = 'a'.repeat(config.max_path_length - 6); + await fsp.mkdir(path.join(tmpDir, directory)); + const keepName = path.join(directory, DIRKEEP); + await fsp.writeFile(path.join(tmpDir, keepName), ''); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [keepName], + skipped_count: 1, + }); + }); + + it('uses a bounded probe instead of recursively walking an overlong directory', async () => { + const first = 'a'.repeat(200); + const second = 'b'.repeat(60); + const overlongDir = path.join(first, second); + await fsp.mkdir(path.join(tmpDir, overlongDir), { recursive: true }); + for (let i = 0; i < 1001; i++) { + await fsp.writeFile(path.join(tmpDir, overlongDir, `ignored-${i}.bin`), 'ignored'); + } + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [overlongDir], + skipped_count: 1, + }); + }); + + it('does not report an unchanged oversized inline entrypoint', async () => { + const name = 'main.py'; + const content = 'print(1)'; + const full = path.join(tmpDir, name); + await fsp.writeFile(full, content); + const inline: TFile = { name, content }; + const job = makeJob({ files: [inline], maxFileSize: 3 }); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.entryPointName = name; + internals.inputFileHashes.set(name, { hash: sha256(content), path: full }); + + await internals.walkDir(tmpDir, 0, buildInputByName([inline])); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('inspects a capped directory before deciding whether an artifact was omitted', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + await fsp.mkdir(path.join(tmpDir, 'ignored')); + await fsp.writeFile(path.join(tmpDir, 'ignored', 'cache.bin'), 'ignored'); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('reports a capped empty-directory marker', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + await fsp.mkdir(path.join(tmpDir, 'empty')); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: [path.join('empty', DIRKEEP)], + skipped_count: 1, + }); + }); + + it('does not hash ordinary oversized files in session mode', async () => { + await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); + const session = new SessionWorkspace({ runtimeSessionId: 'rt_large' }); + const internals = asInternals(makeJob({ maxFileSize: 3, session })); + internals.submissionDir = tmpDir; + let hashCalls = 0; + internals.computeFileHash = async () => { + hashCalls++; + return sha256('too large'); + }; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(hashCalls).toBe(0); + expect(internals.artifactTruncation?.reasons).toEqual({ size: 1 }); + }); + + it('keeps scanning for generated outputs when only inherited refs are capped', async () => { + await fsp.mkdir(path.join(tmpDir, 'a-ignored')); + await fsp.writeFile(path.join(tmpDir, 'a-ignored', 'cache.bin'), 'ignored'); + await fsp.writeFile(path.join(tmpDir, 'z-generated.txt'), 'new'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.inheritedRefs = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `inherited-${i}.txt`, + storage_session_id: 'previous', + inherited: true, + })); + internals.artifactTruncation = { + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['another-inherited.txt'], + skipped_count: 1, + }; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles.map(file => file.name)).toContain('z-generated.txt'); + }); + + it('bounds output-cap eligibility probes for wide unsupported-only directories', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + for (let i = 0; i < 1001; i++) { + await fsp.writeFile(path.join(tmpDir, `ignored-${i}.bin`), 'ignored'); + } + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['.'], + skipped_count: 1, + }); + }); + + it('shares the output-cap probe budget across sibling subtrees', async () => { + for (const dirname of ['b-ignored', 'c-ignored']) { + await fsp.mkdir(path.join(tmpDir, dirname)); + for (let i = 0; i < 600; i++) { + await fsp.writeFile(path.join(tmpDir, dirname, `ignored-${i}.bin`), 'ignored'); + } + } + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(path.join(tmpDir, 'b-ignored'), 1, new Map()); + await internals.walkDir(path.join(tmpDir, 'c-ignored'), 1, new Map()); + + expect(internals.artifactTruncation?.reasons).toEqual({ max_files: 1 }); + expect(internals.artifactTruncation?.skipped).toEqual(['c-ignored']); + }); + + it('does not report surfaced session artifacts during output-cap probing', async () => { + const name = 'old-output.txt'; + const content = 'already returned'; + await fsp.writeFile(path.join(tmpDir, name), content); + const session = new SessionWorkspace({ runtimeSessionId: 'rt_capped' }); + session.markSurfaced(name, sha256(content)); + const internals = asInternals(makeJob({ session })); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('does not reopen capped directories after the shared probe budget is exhausted', async () => { + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.truncationProbeState.remainingEntries = 0; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + const absentDir = path.join(tmpDir, 'not-opened'); + + await internals.walkDir(absentDir, 1, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['not-opened'], + skipped_count: 1, + }); + }); + + it('does not report an unchanged inline entrypoint during output-cap probing', async () => { + const directory = path.join(tmpDir, 'src'); + const name = path.join('src', 'main.py'); + const content = 'print(1)'; + await fsp.mkdir(directory); + await fsp.writeFile(path.join(tmpDir, name), content); + const inline: TFile = { name, content }; + const internals = asInternals(makeJob({ files: [inline] })); + internals.submissionDir = tmpDir; + internals.entryPointName = name; + internals.inputFileHashes.set(name, { hash: sha256(content), path: path.join(tmpDir, name) }); + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(directory, 1, buildInputByName([inline])); + + expect(internals.artifactTruncation).toBeUndefined(); }); }); diff --git a/service/src/execution-log.test.ts b/service/src/execution-log.test.ts index f04bad90..0f538125 100644 --- a/service/src/execution-log.test.ts +++ b/service/src/execution-log.test.ts @@ -28,6 +28,12 @@ describe('execution log summaries', () => { failed: 1, detail: 'private storage failure', }, + artifact_truncation: { + code: 'artifact_truncated', + reasons: { max_files: 2 }, + skipped: ['secret-one.txt', 'secret-two.txt'], + skipped_count: 2, + }, run: { code: 0, stdout: 'top secret stdout', @@ -42,6 +48,7 @@ describe('execution log summaries', () => { expect(JSON.stringify(summary)).not.toContain('sensitive stderr'); expect(JSON.stringify(summary)).not.toContain('combined output'); expect(JSON.stringify(summary)).not.toContain('private storage failure'); + expect(JSON.stringify(summary)).not.toContain('secret-one.txt'); expect(summary).toMatchObject({ session_id: 'sess_123', files: { count: 2, inheritedCount: 1, modifiedCount: 1 }, @@ -52,6 +59,12 @@ describe('execution log summaries', () => { delivered: 2, failed: 1, }, + artifact_truncation: { + code: 'artifact_truncated', + reasons: { max_files: 2 }, + skipped_count: 2, + reported_paths: 2, + }, run: { stdout: { length: 17, present: true }, stderr: { length: 16, present: true }, diff --git a/service/src/execution-log.ts b/service/src/execution-log.ts index 48a93cce..143e5541 100644 --- a/service/src/execution-log.ts +++ b/service/src/execution-log.ts @@ -19,6 +19,7 @@ type SandboxResponseLike = { version?: unknown; files?: unknown; artifact_delivery?: unknown; + artifact_truncation?: unknown; run?: RunLike; }; @@ -40,6 +41,22 @@ function summarizeArtifactDelivery(value: unknown): Record | un }; } +function summarizeArtifactTruncation(value: unknown): Record | undefined { + if (value == null || typeof value !== 'object' || Array.isArray(value)) return undefined; + const truncation = value as { + code?: unknown; + reasons?: unknown; + skipped?: unknown; + skipped_count?: unknown; + }; + return { + code: truncation.code, + reasons: truncation.reasons, + skipped_count: truncation.skipped_count, + reported_paths: Array.isArray(truncation.skipped) ? truncation.skipped.length : undefined, + }; +} + export function summarizeText(value: unknown): { length: number; present: boolean } { if (typeof value !== 'string') { return { length: 0, present: false }; @@ -87,6 +104,7 @@ export function summarizeSandboxResponse(data: SandboxResponseLike): Record { expect(await pollBlockingExecution('exec', 5, deps)).toEqual({ status: 'completed', stdout: result.stdout, stderr: '', files: [], artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, }); expect(deps.now()).toBe(2); }); diff --git a/service/src/service/blocking-poll.ts b/service/src/service/blocking-poll.ts index 8ce07fa8..03e386d4 100644 --- a/service/src/service/blocking-poll.ts +++ b/service/src/service/blocking-poll.ts @@ -34,6 +34,7 @@ export async function pollBlockingExecution( stderr?: string; files?: t.FileRefs; artifact_delivery?: t.ArtifactDeliveryFailure; + artifact_truncation?: t.ArtifactTruncation; }> { const start = deps.now(); while (deps.now() - start < timeout) { @@ -48,6 +49,7 @@ export async function pollBlockingExecution( stderr: result.stderr, files: result.files, artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, }; } } diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 4de4a4d8..93bbc0c5 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -1051,6 +1051,9 @@ async function runAndRespond( error: errorMessage, stdout: cleanStdout, stderr: result.stderr, + files: result.files, + artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, session_id: state.session_id, }); return; @@ -1064,6 +1067,7 @@ async function runAndRespond( stderr: result.stderr, files: result.files, artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, session_id: state.session_id, }); } @@ -1331,6 +1335,7 @@ async function handleBlocking( stderr: state.stderr ?? '', files: state.files ?? [], artifact_delivery: state.artifact_delivery, + artifact_truncation: state.artifact_truncation, session_id: execution.session_id, }); } @@ -1633,6 +1638,7 @@ async function handleBlocking( stderr: state.stderr ?? '', files: state.files ?? [], artifact_delivery: state.artifact_delivery, + artifact_truncation: state.artifact_truncation, session_id, }); } diff --git a/service/src/types/service.ts b/service/src/types/service.ts index f0a6da3b..d17b5a99 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -114,6 +114,15 @@ export interface ArtifactDeliveryFailure { failed: number; } +export type ArtifactTruncationReason = 'max_files' | 'depth' | 'size' | 'path' | 'unreadable'; + +export interface ArtifactTruncation { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; +} + export type ExecuteResponse = { run?: { stdout: string; @@ -133,6 +142,7 @@ export type ExecuteResponse = { session_id: string; files: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; }; export interface RequestBody { @@ -250,6 +260,7 @@ export type ExecuteResult = { stderr: string; files: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; code?: number | null; signal?: string | null; message?: string | null; @@ -392,6 +403,7 @@ export interface ProgrammaticResponse { stderr?: string; files?: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; /** Top-level execution session id (one sandbox PTC invocation). */ session_id?: string; tool_calls_made?: number; diff --git a/service/src/workers.ts b/service/src/workers.ts index dc2f491b..f11a5420 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -190,6 +190,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { ...(responseData.artifact_delivery != null ? { artifact_delivery: responseData.artifact_delivery } : {}), + ...(responseData.artifact_truncation != null + ? { artifact_truncation: responseData.artifact_truncation } + : {}), stdout, stderr, ...(responseData.pending_tool_calls_payload != null From f181b4deaa4ddf37a214875d9256932cb812a656 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 14:12:25 -0400 Subject: [PATCH 18/75] fix: cancel selected-workspace PTC across processes (#196) * fix: cancel replay jobs across API and worker processes * fix: drain worker cancellation watches promptly * fix: close programmatic cancellation races * fix: preserve cancellation response ordering * fix: close distributed cancellation races * fix: harden cancellation under concurrent load * fix: make cancellation ownership durable through completion * fix: recover durable replay outcomes across lost replies * fix: return atomic cancellation outcomes with aligned retention * fix: commit native results inside the workspace mutation fence * fix: claim programmatic execution before stalled-job redelivery --- docs/remote-bridge/README.md | 17 + packages/code/src/worker.test.ts | 643 ++++++++++++++--- packages/code/src/worker.ts | 203 ++++-- packages/code/src/workspace-worker.test.ts | 244 +++++++ service/src/bridge/concurrent-worker.test.ts | 20 +- service/src/config.spec.ts | 5 + service/src/config.ts | 9 +- service/src/job-cancellation-commit.test.ts | 431 ++++++++++++ service/src/job-cancellation.test.ts | 599 ++++++++++++++++ service/src/job-cancellation.ts | 664 ++++++++++++++++++ service/src/metrics.ts | 6 + service/src/middleware/limits.ts | 13 + service/src/programmatic-cancellation.test.ts | 234 ++++++ service/src/programmatic-cancellation.ts | 182 +++++ service/src/queue.ts | 39 +- service/src/redis-options.test.ts | 12 +- service/src/redis-options.ts | 6 + service/src/request-disconnect.test.ts | 60 ++ service/src/request-disconnect.ts | 49 ++ service/src/service/programmatic-router.ts | 351 +++++++-- service/src/types/service.ts | 6 + service/src/workers.ts | 399 ++++++++--- 22 files changed, 3872 insertions(+), 320 deletions(-) create mode 100644 service/src/job-cancellation-commit.test.ts create mode 100644 service/src/job-cancellation.test.ts create mode 100644 service/src/job-cancellation.ts create mode 100644 service/src/programmatic-cancellation.test.ts create mode 100644 service/src/programmatic-cancellation.ts create mode 100644 service/src/request-disconnect.test.ts create mode 100644 service/src/request-disconnect.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 99a0dece..b9de3ac2 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -256,6 +256,23 @@ execution. the currently registered incarnation. - Request cancellation is polled by the worker and aborts the local sandbox request. +- Replay PTC clients may attach a fresh `X-LibreChat-Code-Request-ID` to each + `/exec/programmatic` request and send that same opaque ID to + `POST /v1/exec/programmatic/cancel`. Code API binds the short-lived request + record to the authenticated principal, durably marks cancellation in Redis, + and publishes it to the worker process holding the BullMQ job. This explicit + path avoids relying on HTTP connection teardown, frees waiting jobs + immediately, and interrupts active remote-bridge assignments without polling + once per active job. + Cancellation and completed-result publication use an atomic Redis decision: + a late cancel returns `already_completed` instead of acknowledging Stop after + completion won. Ambiguous enqueue/cancellation errors retain replay ownership + until a durable fence or the original job deadline. Completed results are + retained temporarily (bounded to 16 MiB) so a lost BullMQ completion reply + does not cause sandbox effects to be repeated. Reconnect reconciliation reads + only small status markers, using one subscriber per process. + Roll out the matching Code API queue-worker processes before enabling this + endpoint on API replicas; pre-cancellation workers do not observe its markers. - A leased assignment remains in a Redis-backed delivery claim until the worker explicitly acknowledges it; reconnecting before acknowledgement redelivers the same fenced assignment instead of losing it after an HTTP disconnect. diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index 68d95b28..e26f212a 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -252,7 +252,10 @@ test('worker asks its supervisor to quarantine an ambiguous stateful runtime', a const quarantined: Array<{ sessionId: string; reason: string }> = []; const supervisor: RuntimeSupervisor = { async acquire() { - return { endpoint: 'http://127.0.0.1:3000/runtime', sessionId: 'rt-user-1' }; + return { + endpoint: 'http://127.0.0.1:3000/runtime', + sessionId: 'rt-user-1', + }; }, async reset() {}, async quarantine(sessionId, reason) { @@ -388,7 +391,10 @@ test('worker continues after an assignment-scoped settlement conflict', async () registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (init?.signal?.aborted === true) { @@ -397,15 +403,25 @@ test('worker continues after an assignment-scoped settlement conflict', async () if (url.endsWith('/lease')) { leases += 1; return new Response( - JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0, assignment }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/ack')) { leaseAcknowledged = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -511,7 +527,10 @@ test('worker refreshes its registration during a long assignment', async () => { registeredAt: new Date().toISOString(), leaseTtlMs: 100, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -576,7 +595,10 @@ test('worker schedules registration freshness from request start', async () => { registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -589,7 +611,10 @@ test('worker schedules registration freshness from request start', async () => { if (url.endsWith('/cancelled')) { return new Response( JSON.stringify({ protocolVersion: 1, cancelled: false }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -658,10 +683,13 @@ test('worker continues cancellation polling after a stalled response', async () }); } settlementAttempted = true; - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', @@ -694,6 +722,175 @@ test('worker continues cancellation polling after a stalled response', async () assert.equal(settlementAttempted, true); }); +test('worker stops an outstanding cancellation request before settling completed work', async () => { + let startCancellation!: () => void; + const cancellationStarted = new Promise((resolve) => { + startCancellation = resolve; + }); + let cancellationAborted = false; + let settlementAttempted = false; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + await cancellationStarted; + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + startCancellation(); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => { + cancellationAborted = true; + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 1, + cancellationTransportTimeoutMs: 10_000, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-while-cancellation-polling', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(cancellationAborted, true); + assert.equal(settlementAttempted, true); +}); + +test('worker aborts a retryable cancellation error body before settling completed work', async () => { + let cancellationBodyStarted!: () => void; + const cancellationStarted = new Promise((resolve) => { + cancellationBodyStarted = resolve; + }); + let cancellationBodyAborted = false; + let settlementAttempted = false; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + await cancellationStarted; + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + return new Response( + new ReadableStream({ + start(controller) { + cancellationBodyStarted(); + init?.signal?.addEventListener( + 'abort', + () => { + cancellationBodyAborted = true; + controller.error(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }, + }), + { status: 500 }, + ); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 1, + cancellationTransportTimeoutMs: 10_000, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-during-retryable-cancellation-response', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(cancellationBodyAborted, true); + assert.equal(settlementAttempted, true); +}); + +test('worker stops its cancellation delay before settling immediately completed work', async () => { + let settlementAttempted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 10_000, + fetchImpl: async (input) => { + const url = String(input); + if (url.endsWith('/execute')) { + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + throw new Error('cancellation transport should not start'); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-before-cancellation-polling', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(settlementAttempted, true); +}); + test('worker routes a hintless assignment to an ephemeral template session', async () => { let executeUrl = ''; let runtimeSessionHeader = ''; @@ -852,10 +1049,13 @@ test('worker preserves status for a non-JSON settlement rejection', async () => }, fetchImpl: async (input) => { if (String(input).endsWith('/execute')) { - return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ session_id: 'run-1', files: [] }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); } settlementAttempts += 1; return new Response('assignment fenced', { @@ -1032,7 +1232,10 @@ test('worker retries a known-clean rejection after shutdown until acknowledged', registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/execute')) { @@ -1046,18 +1249,20 @@ test('worker retries a known-clean rejection after shutdown until acknowledged', controller.abort(); throw new TypeError('connection reset'); } - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1097,8 +1302,7 @@ test('worker preserves a definite rejection when its heartbeat fails', async () token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1118,7 +1322,10 @@ test('worker preserves a definite rejection when its heartbeat fails', async () registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/execute')) { @@ -1139,7 +1346,10 @@ test('worker preserves a definite rejection when its heartbeat fails', async () } return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1170,8 +1380,7 @@ test('worker quarantines a stateful workspace after a sandbox 5xx response', asy token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1187,7 +1396,10 @@ test('worker quarantines a stateful workspace after a sandbox 5xx response', asy settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1217,8 +1429,7 @@ test('worker treats a non-JSON sandbox 4xx as a definite rejection', async () => token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1235,7 +1446,10 @@ test('worker treats a non-JSON sandbox 4xx as a definite rejection', async () => JSON.parse(String(init?.body) || '{}').status === 'rejected'; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1268,18 +1482,20 @@ test('worker quarantines a stateful workspace after the sandbox request aborts', }); } settlementAttempted = true; - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1331,13 +1547,23 @@ test('worker surfaces quarantine when shutdown aborts stateful execution', async registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/lease')) { return new Response( - JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0, assignment }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -1351,18 +1577,20 @@ test('worker surfaces quarantine when shutdown aborts stateful execution', async ); }); } - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1510,7 +1738,10 @@ test('worker subtracts lease response transit from the server budget', async () if (String(input).endsWith('/ack')) { return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } now += 50; @@ -1527,10 +1758,16 @@ test('worker subtracts lease response transit from the server budget', async () leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(0).toISOString(), remainingMs: 1_000, - request: { body: { language: 'bash' }, headers: {} }, + request: { + body: { language: 'bash' }, + headers: {}, + }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1572,21 +1809,28 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/ack')) { now += 10; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/settle')) { settlementAttempts += 1; - abandonedSettlement = JSON.parse( - String(init?.body), - ) as Record; + abandonedSettlement = JSON.parse(String(init?.body)) as Record< + string, + unknown + >; if (settlementAttempts === 1) { return new Response(JSON.stringify({ error: 'unavailable' }), { status: 503, @@ -1595,7 +1839,10 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( } return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -1611,15 +1858,24 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(0).toISOString(), remainingMs: 10, - request: { body: { language: 'bash' }, headers: {} }, + request: { + body: { language: 'bash' }, + headers: {}, + }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); - await assert.rejects(worker.lease(), /expired during lease acknowledgement/); + await assert.rejects( + worker.lease(), + /expired during lease acknowledgement/, + ); assert.equal(abandonedSettlement?.status, 'rejected'); assert.ok(registrations > 0); assert.equal(settlementAttempts, 2); @@ -1653,7 +1909,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as JSON.parse(String(init?.body) || '{}').status === 'rejected'; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/workers/register')) { @@ -1665,7 +1924,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -1685,7 +1947,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as request: { body: { language: 'bash' }, headers: {} }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1702,8 +1967,7 @@ test('worker clamps rejected settlement errors to the protocol limit', async () token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1716,11 +1980,16 @@ test('worker clamps rejected settlement errors to the protocol limit', async () headers: { 'Content-Type': 'application/json' }, }); } - const settlement = JSON.parse(String(init?.body)) as { error: string }; + const settlement = JSON.parse(String(init?.body)) as { + error: string; + }; rejection = settlement.error; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1747,8 +2016,7 @@ test('worker quarantines an explicitly dirty stateful sandbox response', async ( token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1761,13 +2029,19 @@ test('worker quarantines an explicitly dirty stateful sandbox response', async ( error: 'session_workspace_dirty', message: 'restore required', }), - { status: 409, headers: { 'Content-Type': 'application/json' } }, + { + status: 409, + headers: { 'Content-Type': 'application/json' }, + }, ); } settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1859,15 +2133,21 @@ test('worker uses the server-relative lease budget despite VM clock skew', async }, fetchImpl: async (input) => { if (String(input).endsWith('/execute')) { - return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ session_id: 'run-1', files: [] }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); } settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1886,7 +2166,6 @@ test('worker uses the server-relative lease budget despite VM clock skew', async assert.equal(settlementAttempted, true); }); - test('worker continues after an expired assignment settlement conflict', async () => { const controller = new AbortController(); let registrations = 0; @@ -1921,18 +2200,22 @@ test('worker continues after an expired assignment settlement conflict', async ( leases += 1; return Response.json({ protocolVersion: 1, - assignment: leases === 1 - ? { - protocolVersion: 1, - assignmentId: 'assignment-expired', - workerId: 'vm-1', - incarnationId, - generation: 1, - leaseToken: 'lease-token-that-is-long-enough-for-testing', - expiresAt: new Date(Date.now() + 10_000).toISOString(), - request: { body: { language: 'bash' }, headers: {} }, - } - : undefined, + assignment: + leases === 1 + ? { + protocolVersion: 1, + assignmentId: 'assignment-expired', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 10_000).toISOString(), + request: { + body: { language: 'bash' }, + headers: {}, + }, + } + : undefined, }); } if (url.endsWith('/execute')) { @@ -1940,7 +2223,10 @@ test('worker continues after an expired assignment settlement conflict', async ( } if (url.endsWith('/settle')) { return Response.json( - { error: 'Bridge assignment has expired', code: 'ASSIGNMENT_EXPIRED' }, + { + error: 'Bridge assignment has expired', + code: 'ASSIGNMENT_EXPIRED', + }, { status: 409 }, ); } @@ -2172,7 +2458,10 @@ test('paired worker rotates credentials throughout a long assignment', async () } if (url.endsWith('/execute')) { await new Promise((resolve) => setTimeout(resolve, 55)); - return Response.json({ session_id: 'run-long-rotation', files: [] }); + return Response.json({ + session_id: 'run-long-rotation', + files: [], + }); } return Response.json({ protocolVersion: 1, accepted: true }); }; @@ -2276,6 +2565,117 @@ test('paired worker cancels a stalled credential refresh after execution', async assert.equal(refreshAborted, true); }); +test('one concurrent caller cannot abort a credential refresh another caller still needs', async () => { + const key = createBridgeIdentity(); + const first = new AbortController(); + const second = new AbortController(); + let releaseRefresh!: () => void; + let refreshStarted!: () => void; + const started = new Promise((resolve) => { + refreshStarted = resolve; + }); + const released = new Promise((resolve) => { + releaseRefresh = resolve; + }); + let transportAborted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-shared-refresh', + expiresAt: new Date(Date.now() + 5).toISOString(), + }, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + assert.match(String(input), /credentials\/refresh$/); + refreshStarted(); + init?.signal?.addEventListener('abort', () => { + transportAborted = true; + }); + await released; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-shared-refresh-value', + expiresAt: new Date(Date.now() + 120_000).toISOString(), + }); + }, + }); + + const firstRefresh = worker.refreshCredential(first.signal); + await started; + const secondRefresh = worker.refreshCredential(second.signal); + first.abort(); + + await assert.rejects(firstRefresh, { name: 'AbortError' }); + assert.equal(transportAborted, false); + releaseRefresh(); + await secondRefresh; + assert.equal(transportAborted, false); +}); + +test('a new caller starts a fresh credential refresh after the last waiter aborts', async () => { + const key = createBridgeIdentity(); + const first = new AbortController(); + let refreshCount = 0; + let firstRefreshStarted!: () => void; + const started = new Promise((resolve) => { + firstRefreshStarted = resolve; + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-replacement-refresh', + expiresAt: new Date(Date.now() + 5).toISOString(), + }, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (_input, init) => { + refreshCount += 1; + if (refreshCount === 1) { + firstRefreshStarted(); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-replacement-refresh', + expiresAt: new Date(Date.now() + 120_000).toISOString(), + }); + }, + }); + + const abandoned = worker.refreshCredential(first.signal, Date.now() + 1_000); + await started; + first.abort(); + await assert.rejects(abandoned, { name: 'AbortError' }); + await worker.refreshCredential(undefined, Date.now() + 1_000); + + assert.equal(refreshCount, 2); +}); + test('paired worker refreshes conservatively before server clock calibration', async () => { const key = createBridgeIdentity(); let refreshCount = 0; @@ -2347,8 +2747,7 @@ test('paired worker charges initial credential refresh against the assignment de sandboxStarted = true; } if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2389,8 +2788,7 @@ test('paired worker rechecks the deadline after request serialization', async () codeApiUrl: 'https://code.example/v1', workerId: 'vm-1', incarnationId, - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', identity: { privateKey: key.privateKey, credential: 'credential-valid-during-serialization', @@ -2408,8 +2806,7 @@ test('paired worker rechecks the deadline after request serialization', async () sandboxStarted = true; } if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2471,8 +2868,7 @@ test('paired worker keeps endpoint validation failures known-clean', async () => const url = String(input); if (url.endsWith('/execute')) sandboxStarted = true; if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2758,11 +3154,13 @@ test('sandbox completion does not cancel an in-flight credential rotation', asyn } if (url.endsWith('/execute')) { await refreshStartedPromise; - return Response.json({ session_id: 'run-rotation-race', files: [] }); + return Response.json({ + session_id: 'run-rotation-race', + files: [], + }); } - settleAuthorization = ( - init?.headers as Record - ).Authorization; + settleAuthorization = (init?.headers as Record) + .Authorization; return Response.json({ protocolVersion: 1, accepted: true }); }; const worker = new BridgeWorker({ @@ -2799,8 +3197,41 @@ test('sandbox completion does not cancel an in-flight credential rotation', asyn ); }); +test('settlement does not drain another lane credential renewal', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', workerId: 'vm-1', token: 'fixture', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'] }, + fetchImpl: async (input) => String(input).endsWith('/execute') + ? Response.json({ session_id: 'independent-lane', files: [] }) + : Response.json({ protocolVersion: 1, accepted: true }), + }); + // A different lane owns this pending renewal. The settling lane has no + // maintenance waiter and must not consume its own lease on that promise. + Object.assign(worker, { credentialInFlight: { + promise: new Promise(() => {}), controller: new AbortController(), waiters: 1, + }, refreshCredential: async () => {} }); + const startedAt = Date.now(); + await worker.executeAndSettle({ + protocolVersion: 1, assignmentId: 'independent-lane', workerId: 'vm-1', + incarnationId, generation: 5, leaseToken: 'independent-lane-lease-token', + expiresAt: new Date(Date.now() + 600_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + assert.ok(Date.now() - startedAt < 500, 'unrelated renewal must not add a one-second drain'); +}); + test('reconnect delay uses bounded exponential jitter', () => { - assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 0), 500); - assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 1), 1_000); - assert.equal(reconnectDelayMs(10, 1_000, 30_000, () => 1), 30_000); + assert.equal( + reconnectDelayMs(0, 1_000, 30_000, () => 0), + 500, + ); + assert.equal( + reconnectDelayMs(0, 1_000, 30_000, () => 1), + 1_000, + ); + assert.equal( + reconnectDelayMs(10, 1_000, 30_000, () => 1), + 30_000, + ); }); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index a0517bd9..ffe1b350 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -38,6 +38,11 @@ export interface BridgeWorkerOptions { capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; workspaceProgrammatic?: { + /** + * True when a WorkspaceToolError without mutation uncertainty proves the + * selected workspace was not changed. + */ + mutationFailuresAreAtomic?: true; executeProgrammatic( workspaceId: string, request: BridgeWorkspaceProgrammaticRequest, @@ -99,6 +104,7 @@ const DEFAULT_REGISTRATION_TRANSPORT_TIMEOUT_MS = 10_000; const DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS = 10_000; const DEFAULT_CANCELLATION_POLL_INTERVAL_MS = 500; const DEFAULT_CANCELLATION_TRANSPORT_TIMEOUT_MS = 2_000; +const CREDENTIAL_REFRESH_SETTLEMENT_GRACE_MS = 1_000; const MIN_REGISTRATION_HEARTBEAT_MS = 25; const REGISTRATION_RETRY_DELAY_MS = 100; const CREDENTIAL_REFRESH_RETRY_DELAY_MS = 100; @@ -158,26 +164,29 @@ function workspaceCapabilitiesMatch( advertised.writeFileModes?.length === executor.writeFileModes?.length && (advertised.writeFileModes?.every( (mode, index) => mode === executor.writeFileModes?.[index], - ) ?? executor.writeFileModes == null) && + ) ?? + executor.writeFileModes == null) && advertised.editFileModes?.length === executor.editFileModes?.length && (advertised.editFileModes?.every( (mode, index) => mode === executor.editFileModes?.[index], - ) ?? executor.editFileModes == null) && - advertised.editFileFeatures?.length === - executor.editFileFeatures?.length && + ) ?? + executor.editFileModes == null) && + advertised.editFileFeatures?.length === executor.editFileFeatures?.length && (advertised.editFileFeatures?.every( (feature, index) => feature === executor.editFileFeatures?.[index], - ) ?? executor.editFileFeatures == null) && - advertised.listFileFeatures?.length === - executor.listFileFeatures?.length && + ) ?? + executor.editFileFeatures == null) && + advertised.listFileFeatures?.length === executor.listFileFeatures?.length && (advertised.listFileFeatures?.every( (feature, index) => feature === executor.listFileFeatures?.[index], - ) ?? executor.listFileFeatures == null) && + ) ?? + executor.listFileFeatures == null) && advertised.programmaticLanguages?.length === executor.programmaticLanguages?.length && (advertised.programmaticLanguages?.every( (language, index) => language === executor.programmaticLanguages?.[index], - ) ?? executor.programmaticLanguages == null) && + ) ?? + executor.programmaticLanguages == null) && advertised.workspaces.length === executor.workspaces.length && advertised.workspaces.every( (workspace, index) => @@ -189,7 +198,8 @@ function workspaceCapabilitiesMatch( (operation, operationIndex) => operation === executor.workspaces[index]?.operations?.[operationIndex], - ) ?? executor.workspaces[index]?.operations == null), + ) ?? + executor.workspaces[index]?.operations == null), ) ); } @@ -201,8 +211,7 @@ function registrationCompatibleCapabilities( if ( workspaceTools == null || (workspaceTools.operations.every( - (operation) => - operation === 'read_file' || operation === 'search_text', + (operation) => operation === 'read_file' || operation === 'search_text', ) && workspaceTools.workspaces.every( (workspace) => workspace.operations == null, @@ -211,8 +220,7 @@ function registrationCompatibleCapabilities( return capabilities; } const operations = workspaceTools.operations.filter( - (operation) => - operation === 'read_file' || operation === 'search_text', + (operation) => operation === 'read_file' || operation === 'search_text', ); if (operations.length === 0) { const { workspaceTools: _workspaceTools, ...compatible } = capabilities; @@ -221,7 +229,9 @@ function registrationCompatibleCapabilities( const workspaces = workspaceTools.workspaces.flatMap((workspace) => { if ( workspace.operations != null && - !operations.every((operation) => workspace.operations?.includes(operation)) + !operations.every((operation) => + workspace.operations?.includes(operation), + ) ) { return []; } @@ -386,7 +396,11 @@ export class BridgeWorker { private negotiatedWorkspaceSlots = 1; private concurrentRunning = false; private registrationInFlight?: Promise; - private credentialInFlight?: Promise; + private credentialInFlight?: { + promise: Promise; + controller: AbortController; + waiters: number; + }; private serverClockOffsetMs = MAX_PROOF_CLOCK_SKEW_MS; constructor(private readonly options: BridgeWorkerOptions) { @@ -434,7 +448,8 @@ export class BridgeWorker { (options.workspaceProgrammatic != null) !== (options.capabilities.workspaceTools?.programmaticLanguages?.includes( 'bash', - ) === true) + ) === + true) ) { throw new BridgeProtocolError( 'Workspace programmatic capability requires a matching executor', @@ -553,7 +568,9 @@ export class BridgeWorker { if (signal?.aborted) { abortRegistration(); } else { - signal?.addEventListener('abort', abortRegistration, { once: true }); + signal?.addEventListener('abort', abortRegistration, { + once: true, + }); } const timeoutMs = Math.min( Math.max(1, this.registrationTtlMs - 1), @@ -575,7 +592,10 @@ export class BridgeWorker { workerId: this.options.workerId, incarnationId: this.incarnationId, capabilities: this.maintenanceOnly - ? { ...capabilities, requiresReadyConfirmation: true } + ? { + ...capabilities, + requiresReadyConfirmation: true, + } : capabilities, }, registrationController.signal, @@ -709,7 +729,9 @@ export class BridgeWorker { } await this.runtimeSupervisor.reset(runtimeSessionId, signal); await this.timedRequest( - `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, + `${this.codeApiUrl}${bridgeWorkerPath( + this.options.workerId, + )}/workspaces/reset`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, @@ -745,7 +767,9 @@ export class BridgeWorker { // machine-local guard before the remote fence can be removed. await guard.assertAvailable(); await this.timedRequest( - `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, + `${this.codeApiUrl}${bridgeWorkerPath( + this.options.workerId, + )}/workspaces/reset`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, @@ -1051,20 +1075,58 @@ export class BridgeWorker { transportTimeoutMs = Number.POSITIVE_INFINITY, ): Promise { while (this.credentialInFlight) { - await this.credentialInFlight; + await this.waitForCredentialRefresh(this.credentialInFlight, signal); // A longer-lived caller may still need another refresh after this one. } + const controller = new AbortController(); const pending = this.refreshCredentialOwned( - signal, + controller.signal, validThroughMs, transportTimeoutMs, ); - this.credentialInFlight = pending; + const entry = { promise: pending, controller, waiters: 0 }; + this.credentialInFlight = entry; + void pending.then( + () => { + if (this.credentialInFlight === entry) + this.credentialInFlight = undefined; + }, + () => { + if (this.credentialInFlight === entry) + this.credentialInFlight = undefined; + }, + ); + await this.waitForCredentialRefresh(entry, signal); + } + + private async waitForCredentialRefresh( + entry: NonNullable, + signal?: AbortSignal, + ): Promise { + entry.waiters += 1; + let removeAbortListener = (): void => {}; + const aborted = new Promise((_, reject) => { + if (signal == null) return; + const abort = (): void => + reject( + signal.reason instanceof Error + ? signal.reason + : new DOMException('aborted', 'AbortError'), + ); + removeAbortListener = (): void => + signal.removeEventListener('abort', abort); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + }); try { - await pending; + await Promise.race([entry.promise, aborted]); } finally { - if (this.credentialInFlight === pending) + removeAbortListener(); + entry.waiters -= 1; + if (entry.waiters === 0 && this.credentialInFlight === entry) { this.credentialInFlight = undefined; + entry.controller.abort(); + } } } @@ -1118,7 +1180,7 @@ export class BridgeWorker { assignment: BridgeAssignment, stopSignal: AbortSignal, serverClockOffsetMs: number, - requestSignal?: AbortSignal, + maintenance: { refresh?: Promise }, ): Promise { const identity = this.options.identity; if (identity == null) return; @@ -1136,18 +1198,18 @@ export class BridgeWorker { await abortableDelay(waitMs, stopSignal); if (stopSignal.aborted || Date.now() >= assignmentDeadlineMs) return; try { - await this.refreshCredential( - requestSignal, + maintenance.refresh = this.refreshCredential( + stopSignal, Date.now() + serverClockOffsetMs + refreshWindowMs, ); + await maintenance.refresh; } catch (error) { if (stopSignal.aborted) return; const terminal = error instanceof BridgeProtocolError && (error.status === 401 || error.status === 403); const credentialRemainingMs = - Date.parse(identity.expiresAt) - - (Date.now() + serverClockOffsetMs); + Date.parse(identity.expiresAt) - (Date.now() + serverClockOffsetMs); if (terminal || credentialRemainingMs <= 0) throw error; await abortableDelay( Math.min( @@ -1156,6 +1218,8 @@ export class BridgeWorker { ), stopSignal, ); + } finally { + maintenance.refresh = undefined; } } } @@ -1352,6 +1416,7 @@ export class BridgeWorker { ); let credentialMaintenanceError: unknown; let credentialMaintenance: Promise | undefined; + const ownCredentialMaintenance: { refresh?: Promise } = {}; let settlement: BridgeSettlement; let ambiguousSandboxError: unknown; let ambiguousWorkspaceMutationError: unknown; @@ -1368,7 +1433,7 @@ export class BridgeWorker { assignment, credentialController.signal, serverClockOffsetMs, - signal, + ownCredentialMaintenance, ).catch((error) => { credentialMaintenanceError = error; executionController.abort(); @@ -1674,14 +1739,22 @@ export class BridgeWorker { ) { workspaceMutationGuardError = error; } + const knownAtomicWorkspaceToolFailure = + assignment.executionKind === 'workspace_tool' && + error instanceof WorkspaceToolError && + this.options.workspaceTools?.mutationFailuresAreAtomic === true && + !error.requiresQuarantine; + const knownAtomicProgrammaticFailure = + assignment.executionKind === 'workspace_programmatic' && + error instanceof WorkspaceToolError && + this.options.workspaceProgrammatic?.mutationFailuresAreAtomic === + true && + !error.requiresQuarantine; if ( workspaceMutationApplied || (workspaceMutationArmed && - !( - error instanceof WorkspaceToolError && - this.options.workspaceTools?.mutationFailuresAreAtomic === true && - !error.requiresQuarantine - )) + !knownAtomicWorkspaceToolFailure && + !knownAtomicProgrammaticFailure) ) { ambiguousWorkspaceMutationError = error; } @@ -1713,12 +1786,30 @@ export class BridgeWorker { clearTimeout(deadlineTimer); cancellationController.abort(); await cancellationWatcher; + // Only drain renewal joined by this assignment, never an unrelated lane's + // refresh. Leave settlement time inside the original assignment budget. + const credentialInFlight = ownCredentialMaintenance.refresh; + if (credentialInFlight != null && !credentialController.signal.aborted) { + let drainTimer: ReturnType | undefined; + await Promise.race([ + credentialInFlight.catch(() => undefined), + new Promise((resolve) => { + drainTimer = setTimeout( + resolve, + Math.min(CREDENTIAL_REFRESH_SETTLEMENT_GRACE_MS, + Math.max(0, Date.parse(assignment.expiresAt) - serverClockOffsetMs - Date.now() - 5_000)), + ); + }), + ]); + if (drainTimer != null) clearTimeout(drainTimer); + } credentialController.abort(); await credentialMaintenance; try { if (workspaceMutationGuardError != null) throw workspaceMutationGuardError; if (ambiguousWorkspaceMutationError != null) { + this.options.onError?.(ambiguousWorkspaceMutationError); throw await this.quarantineWorkspace( undefined, 'Worker stopped after a workspace mutation completed without a fulfilled settlement', @@ -1913,7 +2004,9 @@ export class BridgeWorker { return await lease.execute({ body, headers, signal }); } if (lease.endpoint == null) { - throw new BridgeProtocolError('Runtime lease does not provide an execution transport'); + throw new BridgeProtocolError( + 'Runtime lease does not provide an execution transport', + ); } const endpoint = lease.endpoint.replace(/\/+$/, ''); const response = await this.fetchImpl(`${endpoint}/execute`, { @@ -2170,17 +2263,23 @@ export class BridgeWorker { signal: AbortSignal, ): Promise { while (!signal.aborted && !executionController.signal.aborted) { - await this.delay( - Math.max( - 1, - this.options.cancellationPollIntervalMs ?? - DEFAULT_CANCELLATION_POLL_INTERVAL_MS, - ), - signal, - ); + try { + await this.delay( + Math.max( + 1, + this.options.cancellationPollIntervalMs ?? + DEFAULT_CANCELLATION_POLL_INTERVAL_MS, + ), + signal, + ); + } catch (error) { + if (signal.aborted || executionController.signal.aborted) return; + throw error; + } if (signal.aborted || executionController.signal.aborted) return; const pollController = new AbortController(); const abortPoll = (): void => pollController.abort(); + signal.addEventListener('abort', abortPoll, { once: true }); executionController.signal.addEventListener('abort', abortPoll, { once: true, }); @@ -2200,6 +2299,15 @@ export class BridgeWorker { incarnationId: this.incarnationId, }, pollController.signal, + (response) => { + // Once response headers arrive, drain the bounded body before a + // successful execution can settle. Otherwise a cancellation=true + // response racing command completion can be discarded. The + // transport timer and execution signal still cap the drain. + if (response.ok || response.status === 404) { + signal.removeEventListener('abort', abortPoll); + } + }, ); if (response.cancelled) { executionController.abort(); @@ -2213,6 +2321,7 @@ export class BridgeWorker { if (signal.aborted) return; } finally { clearTimeout(timeout); + signal.removeEventListener('abort', abortPoll); executionController.signal.removeEventListener('abort', abortPoll); } } @@ -2222,6 +2331,7 @@ export class BridgeWorker { url: string, body: object, signal?: AbortSignal, + onResponseHeaders?: (response: Response) => void, ): Promise { const requestBody = JSON.stringify(body); const response = await this.fetchImpl(url, { @@ -2233,6 +2343,7 @@ export class BridgeWorker { body: requestBody, signal, }); + onResponseHeaders?.(response); let payload: unknown; try { payload = await response.json(); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index fa08a84c..f6205cd4 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1032,6 +1032,250 @@ test('worker executes programmatic Bash in the selected workspace and preserves ]); }); +test('worker keeps a selected workspace usable after an atomic programmatic setup failure', async () => { + const lifecycle: string[] = []; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw new WorkspaceToolError( + 'Programmatic input download failed', + 'COMMAND_UNAVAILABLE', + ); + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + ], + ]), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-setup-failure', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }, + }); + + assert.deepEqual(lifecycle, ['arm', 'clear']); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'COMMAND_UNAVAILABLE'); +}); + +test('worker keeps a selected workspace usable after confirmed programmatic cancellation cleanup', async () => { + const lifecycle: string[] = []; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + ], + ]), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-cancelled-cleanly', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'sleep 30' }], + }, + headers: {}, + }, + }); + + assert.deepEqual(lifecycle, ['arm', 'clear']); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'EXECUTION_ABORTED'); +}); + +test('worker reports the underlying cause before quarantining an uncertain programmatic mutation', async () => { + const rootCause = new WorkspaceToolError( + 'Programmatic output upload failed', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + let reported: unknown; + let quarantined = false; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw rootCause; + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine(() => { + quarantined = true; + }), + ], + ]), + onError(error) { + reported = error; + }, + fetchImpl: async () => { + throw new Error('settlement must not run'); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-uncertain-failure', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }, + }), + BridgeWorkspaceQuarantinedError, + ); + + assert.equal(reported, rootCause); + assert.equal(quarantined, true); +}); + test('worker stops after Code API rejects a fulfilled workspace mutation', async () => { let quarantinedReason: string | undefined; let armed = 0; diff --git a/service/src/bridge/concurrent-worker.test.ts b/service/src/bridge/concurrent-worker.test.ts index 5aec49cf..ea5ac601 100644 --- a/service/src/bridge/concurrent-worker.test.ts +++ b/service/src/bridge/concurrent-worker.test.ts @@ -243,11 +243,25 @@ for (const failure of [ status: 'fulfilled', value: { status: 'fulfilled' }, }); - for (let i = 0; i < 300 && errors.length === 0; i++) + const diagnosticCount = cleanupFailure ? 1 : 2; + const quarantineAttemptCount = + failure === 'lost-response' + ? 2 + : failure === 'all-responses-lost' || failure === 'delivery-outage' + ? 3 + : 1; + for ( + let i = 0; + i < 300 && + (errors.length < diagnosticCount || + (!cleanupFailure && quarantineAttempts < quarantineAttemptCount)); + i++ + ) { await new Promise((resolve) => setTimeout(resolve, 5)); + } if (failure === 'delivery-outage') - expect(errors.length).toBeGreaterThanOrEqual(1); - else expect(errors.length).toBe(1); + expect(errors.length).toBeGreaterThanOrEqual(2); + else expect(errors.length).toBe(diagnosticCount); if (failure === 'lost-response') expect(quarantineAttempts).toBe(2); if (failure === 'delivery-outage') expect(quarantineAttempts).toBeGreaterThanOrEqual(3); diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 44b64b8b..69b416d8 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -99,6 +99,11 @@ describe('egress grant TTL configuration', () => { }); describe('job deadline accounting', () => { + it('never extends the producer deadline when worker configuration differs', () => { + expect(jobDeadlineAtMs(1_000, 300_000, 50_000, 91_000)).toBe(91_000); + expect(jobDeadlineAtMs(1_000, 30_000, 50_000, 91_000)).toBe(31_000); + expect(jobDeadlineAtMs(1_000, 300_000, 50_000, Number.NaN)).toBe(0); + }); it('counts time spent waiting in BullMQ against JOB_TIMEOUT', () => { expect(jobDeadlineAtMs(1_000, 300_000, 50_000)).toBe(301_000); }); diff --git a/service/src/config.ts b/service/src/config.ts index d025831e..55570dfe 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -95,10 +95,17 @@ export function jobDeadlineAtMs( enqueuedAtMs: number | undefined, timeoutMs: number, nowMs: number = Date.now(), + producerDeadlineAtMs?: number, ): number { - return Number.isFinite(enqueuedAtMs) && (enqueuedAtMs as number) > 0 + const localDeadline = Number.isFinite(enqueuedAtMs) && (enqueuedAtMs as number) > 0 ? (enqueuedAtMs as number) + timeoutMs : nowMs + timeoutMs; + if (producerDeadlineAtMs === undefined) return localDeadline; + // A worker with a larger JOB_TIMEOUT must not outlive the admission fence + // retained by its API producer. Malformed explicit deadlines fail closed. + return Number.isFinite(producerDeadlineAtMs) + ? Math.min(localDeadline, producerDeadlineAtMs) + : 0; } /** The worker stops user work at JOB_TIMEOUT, then may still need to terminate diff --git a/service/src/job-cancellation-commit.test.ts b/service/src/job-cancellation-commit.test.ts new file mode 100644 index 00000000..8ded29f2 --- /dev/null +++ b/service/src/job-cancellation-commit.test.ts @@ -0,0 +1,431 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test'; +import { startTestRedis } from './test/redis'; +import { RedisBridgeStore } from './bridge/store'; +import { + commitJobResult, + readCommittedJobResult, + requestJobCancellation, + JobCancellationRegistry, + jobCancellationInternals, + fenceJobCancellation, + waitForJobWithCancellation, + jobCancellationRetentionSeconds, + claimJobExecution, +} from './job-cancellation'; + +let redis: Awaited>; +beforeEach(async () => { + redis = await startTestRedis(); +}); +afterEach(async () => { + await redis.closeTestServer(); +}); +const target = { queueName: 'other', jobId: 'commit-race' }; + +for (const outcome of ['commit', 'stop', 'duplicate']) + test(`native mutation handoff commits or quarantines before root release (${outcome})`, async () => { + const store = new RedisBridgeStore(redis); + const workerId = 'handoff-worker'; + const incarnationId = 'incarnation-handoff-01'; + await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const controller = new AbortController(); + const dispatchArgs = { + workerId, + workspaceId: 'primary', + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'handoff-session', + files: [{ name: 'main.sh', content: 'echo mutation' }], + }, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }; + const completion = store.dispatch({ + ...dispatchArgs, + finalize: async settlement => { + if (outcome === 'stop') await requestJobCancellation(redis, target, 60); + if (outcome === 'duplicate') + await commitJobResult( + redis, + target, + { stdout: 'first mutation' }, + 60, + ); + if ( + (await commitJobResult( + redis, + target, + { stdout: 'mutation settled' }, + 60, + )) !== 'committed' + ) + throw new Error('handoff did not win'); + // This represents Stop during post-handoff egress cleanup. It must no + // longer turn the applied mutation into an acknowledged cancellation. + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + return settlement; + }, + }); + void completion.catch(() => undefined); + const assignment = await store.lease(workerId, incarnationId, 1_000); + if (assignment == null) throw new Error('Missing assignment'); + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'fulfilled', + result: { + session_id: 'handoff-session', + language: 'bash', + version: '5.2', + files: [], + }, + }); + if (outcome !== 'commit') { + await expect(completion).rejects.toThrow('handoff did not win'); + await expect(store.dispatch(dispatchArgs)).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + } else { + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + }); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'mutation settled' }, + }); + } + }); + +test('concurrent stalled-job redelivery claims at most one sandbox execution', async () => { + let executions = 0; + const attempt = async () => { + const claim = await claimJobExecution(redis, target, 60); + if (claim.status === 'claimed') executions += 1; + return claim; + }; + const results = await Promise.allSettled([attempt(), attempt()]); + expect(executions).toBe(1); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength( + 1, + ); + expect(results.filter(result => result.status === 'rejected')).toHaveLength( + 1, + ); + await expect(attempt()).rejects.toThrow('already claimed'); + expect(executions).toBe(1); + await commitJobResult(redis, target, { stdout: 'first result' }, 60); + expect(await attempt()).toEqual({ + status: 'completed', + result: { stdout: 'first result' }, + }); + expect(executions).toBe(1); + expect( + await commitJobResult(redis, target, { stdout: 'different result' }, 60), + ).toBe('already_completed'); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'first result' }, + }); +}); + +test('a lost execution-claim reply never authorizes a second attempt', async () => { + const lostReply = { + eval: async (...args: Parameters) => { + await redis.eval(...args); + throw new Error('claim reply lost'); + }, + } as unknown as typeof redis; + await expect(claimJobExecution(lostReply, target, 60)).rejects.toThrow( + 'claim reply lost', + ); + await expect(claimJobExecution(redis, target, 60)).rejects.toThrow( + 'already claimed', + ); +}); + +test('cancel-before-claim and missing completion payload fail closed', async () => { + await requestJobCancellation(redis, target, 60); + await expect(claimJobExecution(redis, target, 60)).rejects.toThrow( + 'cancelled', + ); + const completedTarget = { ...target, jobId: 'missing-payload-claim' }; + await commitJobResult(redis, completedTarget, { stdout: 'done' }, 60); + await redis.del( + `${jobCancellationInternals.cancellationKey(completedTarget)}:result`, + ); + await expect(claimJobExecution(redis, completedTarget, 60)).rejects.toThrow( + 'refusing re-execution', + ); +}); + +for (const corrupt of [false, true]) + test(`invalid committed result fails immediately without Redis retries (corrupt=${corrupt})`, async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = jobCancellationInternals.cancellationKey(target); + if (corrupt) await redis.set(`${key}:result`, '{invalid'); + else await redis.del(`${key}:result`); + let calls = 0; + const commands = { + eval: (...args: Parameters) => { + calls += 1; + return redis.eval(...args); + }, + } as unknown as typeof redis; + await expect( + fenceJobCancellation({ + commands, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 30_000, + }), + ).rejects.toThrow(); + expect(calls).toBe(1); + }); + +test('completion retention includes the API producer across timeout configuration drift', async () => { + const ttl = jobCancellationRetentionSeconds(30_000, 430); + expect(ttl).toBe(430); + expect(jobCancellationRetentionSeconds(300_000, 430)).toBe(780); + await commitJobResult(redis, target, { stdout: 'done' }, ttl); + const key = jobCancellationInternals.cancellationKey(target); + expect(await redis.ttl(key)).toBeGreaterThanOrEqual(429); + expect(await redis.ttl(`${key}:result`)).toBeGreaterThanOrEqual(429); +}); + +test('a late Stop renews completion evidence along with its request tombstone', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 1); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + const key = jobCancellationInternals.cancellationKey(target); + expect(await redis.ttl(key)).toBeGreaterThanOrEqual(59); + expect(await redis.ttl(`${key}:result`)).toBeGreaterThanOrEqual(59); +}); + +test('retention renewal does not lose subsecond time to rounded TTL readings', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = jobCancellationInternals.cancellationKey(target); + await redis.pexpire(key, 59_900); + const expiration = async () => + Number( + await redis.eval( + ` + local now = redis.call('TIME') + return tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + redis.call('PTTL', KEYS[1]) + `, + 1, + key, + ), + ); + const before = await expiration(); + await requestJobCancellation(redis, target, 60); + expect(await expiration()).toBeGreaterThan(before); +}); + +test('fencing returns the committed result without a vulnerable second Redis read', async () => { + const result = { stdout: 'one committed effect' }; + await commitJobResult(redis, target, result, 60); + let calls = 0; + const connectionDropsAfterDecision = { + eval: async (...args: Parameters) => { + calls += 1; + return redis.eval(...args); + }, + get: async () => { + throw new Error('connection lost after decision'); + }, + mget: async () => { + throw new Error('connection lost after decision'); + }, + } as unknown as typeof redis; + expect( + await fenceJobCancellation({ + commands: connectionDropsAfterDecision, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 1_000, + }), + ).toEqual({ status: 'completed', result }); + expect(calls).toBe(1); +}); + +test('disconnect returns a known completed result without waiting for a lost queue event', async () => { + const result = { stdout: 'done' }; + await commitJobResult(redis, target, result, 60); + const registry = new JobCancellationRegistry(redis); + const controller = new AbortController(); + controller.abort(); + const job = { + id: target.jobId, + queueName: target.queueName, + waitUntilFinished: () => new Promise(() => {}), + } as unknown as Parameters[0]['job']; + try { + expect( + await waitForJobWithCancellation({ + commands: redis, + registry, + job, + events: {} as Parameters< + typeof waitForJobWithCancellation + >[0]['events'], + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + signal: controller.signal, + }), + ).toEqual(result); + } finally { + await registry.close(); + } +}); + +test('durable cancellation wins even before its subscriber notification arrives', async () => { + expect(await requestJobCancellation(redis, target, 60)).toBe(true); + expect(await commitJobResult(redis, target, { stdout: 'late' }, 60)).toBe( + 'cancelled', + ); + expect(await readCommittedJobResult(redis, target)).toBeUndefined(); +}); + +test('committed results reject late Stop and survive a lost BullMQ completion reply', async () => { + const result = { stdout: 'one mutation', files: [] }; + expect(await commitJobResult(redis, target, result, 60)).toBe('committed'); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + expect(await readCommittedJobResult(redis, target)).toEqual({ result }); + expect( + await redis.get(jobCancellationInternals.cancellationKey(target)), + ).toBe('completed'); + const registry = new JobCancellationRegistry(redis); + const controller = new AbortController(); + try { + await registry.register(target, controller); + expect(controller.signal.aborted).toBe(false); + } finally { + await registry.close(); + } +}); + +test('concurrent cancellation and completion have exactly one winner', async () => { + const [cancelled, committed] = await Promise.all([ + requestJobCancellation(redis, target, 60), + commitJobResult(redis, target, { stdout: 'result' }, 60), + ]); + expect(Number(cancelled) + Number(committed === 'committed')).toBe(1); +}); + +test('a missing committed result fails closed instead of re-executing', async () => { + await commitJobResult(redis, target, { stdout: 'already applied' }, 60); + await redis.del(`${jobCancellationInternals.cancellationKey(target)}:result`); + await expect(readCommittedJobResult(redis, target)).rejects.toThrow( + 'refusing re-execution', + ); +}); + +test('an enqueue failure can recover a result that won cancellation fencing', async () => { + const result = { stdout: 'effect already applied' }; + await commitJobResult(redis, target, result, 60); + expect( + await fenceJobCancellation({ + commands: redis, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 5_000, + }), + ).toEqual({ status: 'completed', result }); + expect(await readCommittedJobResult(redis, target)).toEqual({ result }); +}); + +test('enqueue fencing still recovers completion after the original deadline', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + expect( + await fenceJobCancellation({ + commands: redis, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() - 1_000, + }), + ).toEqual({ status: 'completed', result: { stdout: 'done' } }); +}); + +test('Redis rejects commitment when recovery happens after the producer deadline', async () => { + const delayed = { + eval: async (...args: Parameters) => { + await new Promise(resolve => setTimeout(resolve, 150)); + return redis.eval(...args); + }, + } as unknown as typeof redis; + await expect( + commitJobResult(delayed, target, { stdout: 'late' }, 60, Date.now() + 100), + ).rejects.toThrow('exceeded its deadline'); + expect(await readCommittedJobResult(redis, target)).toBeUndefined(); +}); + +test('a timely durable commit remains successful when only its acknowledgement is late', async () => { + const delayedReply = { + eval: async (...args: Parameters) => { + const value = await redis.eval(...args); + await new Promise(resolve => setTimeout(resolve, 150)); + return value; + }, + } as unknown as typeof redis; + expect( + await commitJobResult( + delayedReply, + target, + { stdout: 'committed' }, + 60, + Date.now() + 100, + ), + ).toBe('committed'); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'committed' }, + }); +}); + +for (const failedStage of ['subscription', 'completion'] as const) { + test(`a lost ${failedStage} reply recovers the committed result instead of reporting failure`, async () => { + const result = { stdout: 'already applied once' }; + await commitJobResult(redis, target, result, 60); + const registry = new JobCancellationRegistry(redis); + if (failedStage === 'subscription') + registry.register = async () => { + throw new Error('lost reply'); + }; + const job = { + id: target.jobId, + queueName: target.queueName, + waitUntilFinished: () => Promise.reject(new Error('lost result event')), + } as unknown as Parameters[0]['job']; + try { + expect( + await waitForJobWithCancellation({ + commands: redis, + registry, + job, + events: {} as Parameters< + typeof waitForJobWithCancellation + >[0]['events'], + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + }), + ).toEqual(result); + } finally { + await registry.close(); + } + }); +} diff --git a/service/src/job-cancellation.test.ts b/service/src/job-cancellation.test.ts new file mode 100644 index 00000000..e23186e4 --- /dev/null +++ b/service/src/job-cancellation.test.ts @@ -0,0 +1,599 @@ +import { expect, test } from 'bun:test'; +import { EventEmitter } from 'node:events'; +import type IORedis from 'ioredis'; +import type { Job, QueueEvents } from 'bullmq'; +import { + CLIENT_DISCONNECT_REASON, + JobCancellationRegistry, + jobResultCommitFailure, + jobCancellationInternals, + removeJobIfWaiting, + requestJobCancellation, + throwIfJobAborted, + waitForJobWithCancellation, + fenceJobCancellation, +} from './job-cancellation'; + +class FakeSubscriber extends EventEmitter { + subscribed?: string; + closed = false; + subscribeFailures = 0; + + async subscribe(channel: string): Promise { + if (this.subscribeFailures > 0) { + this.subscribeFailures -= 1; + throw new Error('subscriber unavailable'); + } + this.subscribed = channel; + return 1; + } + + async quit(): Promise<'OK'> { + this.closed = true; + return 'OK'; + } + + disconnect(): void { + this.closed = true; + } +} + +class FakeTransaction { + readonly operations: unknown[][] = []; + + set(...args: unknown[]): this { + this.operations.push(['set', ...args]); + return this; + } + + publish(...args: unknown[]): this { + this.operations.push(['publish', ...args]); + return this; + } + + async exec(): Promise> { + return this.operations.map(() => [null, 'OK']); + } +} + +class FakeRedis { + readonly subscriber = new FakeSubscriber(); + duplicateCalls = 0; + readonly existing = new Set(); + readonly deleted: string[] = []; + readonly transactions: FakeTransaction[] = []; + mgetFailures = 0; + cancellationFailures = 0; + cancellationAttempts = 0; + + duplicate(): FakeSubscriber { + this.duplicateCalls += 1; + return this.subscriber; + } + + async get(key: string): Promise { + return this.existing.has(key) ? '1' : null; + } + + async eval( + _script: string, + _keys: number, + key: string, + _resultKey: string, + ttl: number, + channel: string, + payload: string, + ): Promise { + this.cancellationAttempts += 1; + if (this.cancellationFailures-- > 0) throw new Error('Redis unavailable'); + const transaction = this.multi(); + transaction.set(key, '1', 'EX', ttl); + transaction.publish(channel, payload); + await transaction.exec(); + return [1]; + } + + async mget(...keys: string[]): Promise> { + if (this.mgetFailures > 0) { + this.mgetFailures -= 1; + throw new Error('command connection unavailable'); + } + return keys.map(key => (this.existing.has(key) ? '1' : null)); + } + + async del(key: string): Promise { + this.deleted.push(key); + this.existing.delete(key); + return 1; + } + + multi(): FakeTransaction { + const transaction = new FakeTransaction(); + this.transactions.push(transaction); + return transaction; + } +} + +function redis(fake: FakeRedis): IORedis { + return fake as unknown as IORedis; +} + +test('idle registries allocate no subscriber connection', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + + await registry.close(); + + expect(fake.duplicateCalls).toBe(0); +}); + +test('shutdown disconnects a subscriber whose startup is still waiting for Redis', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribe = async () => new Promise(() => {}); + const registry = new JobCancellationRegistry(redis(fake)); + void registry + .register( + { queueName: 'other', jobId: 'shutdown-startup' }, + new AbortController(), + ) + .catch(() => undefined); + await registry.close(); + expect(fake.subscriber.closed).toBe(true); + expect(fake.subscriber.listenerCount('message')).toBe(0); +}, 1_000); + +test('failed subscription startup removes handlers before a bounded retry', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribeFailures = 1; + const registry = new JobCancellationRegistry(redis(fake)); + const first = new AbortController(); + + await expect( + registry.register({ queueName: 'other', jobId: 'job-failed-start' }, first), + ).rejects.toThrow('subscriber unavailable'); + expect(fake.subscriber.listenerCount('message')).toBe(0); + expect(fake.subscriber.listenerCount('ready')).toBe(0); + expect(fake.subscriber.listenerCount('error')).toBe(0); + + const second = new AbortController(); + await registry.register({ queueName: 'other', jobId: 'job-retry' }, second); + expect(fake.duplicateCalls).toBe(2); + expect(fake.subscriber.listenerCount('message')).toBe(1); + await registry.close(); +}); + +test('registry catches durable cancellation before subscriber registration', async () => { + const fake = new FakeRedis(); + const target = { queueName: 'other', jobId: 'job-1' }; + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + const registry = new JobCancellationRegistry(redis(fake)); + const controller = new AbortController(); + + await registry.register(target, controller); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + expect(fake.subscriber.subscribed).toBe(jobCancellationInternals.channel); + await registry.unregister(target); + await registry.close(); + expect(fake.subscriber.closed).toBe(true); +}); + +test('one pubsub listener cancels only the matching active job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const first = new AbortController(); + const second = new AbortController(); + await registry.register({ queueName: 'other', jobId: 'job-1' }, first); + await registry.register({ queueName: 'other', jobId: 'job-2' }, second); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-2' }), + ); + + expect(first.signal.aborted).toBe(false); + expect(second.signal.aborted).toBe(true); + await registry.close(); +}); + +test('one pubsub listener wakes every local waiter for the same job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-shared' }; + const first = new AbortController(); + const second = new AbortController(); + await registry.register(target, first); + await registry.register(target, second); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify(target), + ); + + expect(first.signal.aborted).toBe(true); + expect(second.signal.aborted).toBe(true); + expect(fake.duplicateCalls).toBe(1); + await registry.close(); +}); + +test('unregistering one local waiter preserves other waiters for the job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-shared-unregister' }; + const first = new AbortController(); + const second = new AbortController(); + await registry.register(target, first); + await registry.register(target, second); + await registry.unregister(target, first); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify(target), + ); + + expect(first.signal.aborted).toBe(false); + expect(second.signal.aborted).toBe(true); + await registry.close(); +}); + +test('subscriber reconnect reconciles active jobs against durable markers', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-reconnect' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + + fake.subscriber.emit('ready'); + await new Promise(resolve => setTimeout(resolve, 10)); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('subscriber reconnect retries durable-marker reconciliation', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-retry-reconcile' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + fake.mgetFailures = 1; + + fake.subscriber.emit('ready'); + await new Promise(resolve => setTimeout(resolve, 150)); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('terminal subscriber disconnect rebuilds the subscription and reconciles markers', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-terminal-reconnect' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + + fake.subscriber.emit('end'); + await new Promise(resolve => setTimeout(resolve, 10)); + + expect(fake.duplicateCalls).toBe(2); + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('cancellation writes a durable marker before publishing', async () => { + const fake = new FakeRedis(); + const target = { queueName: 'other', jobId: 'job-3' }; + + await requestJobCancellation(redis(fake), target, 42); + + expect(fake.transactions).toHaveLength(1); + expect(fake.transactions[0]?.operations).toEqual([ + ['set', jobCancellationInternals.cancellationKey(target), '1', 'EX', 42], + ['publish', jobCancellationInternals.channel, JSON.stringify(target)], + ]); +}); + +test('result commit barrier rejects cancellation observed after execution', () => { + const controller = new AbortController(); + expect(() => throwIfJobAborted(controller.signal)).not.toThrow(); + controller.abort(CLIENT_DISCONNECT_REASON); + expect(() => throwIfJobAborted(controller.signal)).toThrow( + CLIENT_DISCONNECT_REASON, + ); +}); + +test('result cleanup maps late cancellation to stable worker failures', () => { + const disconnected = new AbortController(); + disconnected.abort(CLIENT_DISCONNECT_REASON); + expect(jobResultCommitFailure(disconnected.signal, 30_000)?.message).toBe( + 'Job cancelled after client disconnected', + ); + + const deadline = new AbortController(); + deadline.abort('deadline'); + expect(jobResultCommitFailure(deadline.signal, 30_000)?.message).toBe( + 'Job timed out after 30000ms', + ); + expect( + jobResultCommitFailure(new AbortController().signal, 30_000), + ).toBeUndefined(); +}); + +test('disconnect frees a waiting job and rejects promptly', async () => { + const fake = new FakeRedis(); + const controller = new AbortController(); + let removed = false; + const never = new Promise(() => {}); + const job = { + id: 'job-4', + queueName: 'other', + waitUntilFinished: () => never, + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + + const registry = new JobCancellationRegistry(redis(fake)); + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + signal: controller.signal, + }); + controller.abort(CLIENT_DISCONNECT_REASON); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(removed).toBe(true); + expect(fake.cancellationAttempts).toBe(1); + expect(fake.transactions[0]?.operations[0]).toEqual([ + 'set', + jobCancellationInternals.cancellationKey({ + queueName: 'other', + jobId: 'job-4', + }), + '1', + 'EX', + 120, + ]); + await registry.close(); +}); + +test('registration failure fences and removes the already-enqueued job', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribeFailures = 1; + let removed = false; + const job = { + id: 'job-register-failure', + queueName: 'other', + waitUntilFinished: () => new Promise(() => {}), + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + const registry = new JobCancellationRegistry(redis(fake)); + + await expect( + waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }), + ).rejects.toThrow('subscriber unavailable'); + + expect(removed).toBe(true); + expect(fake.transactions[0]?.operations[0]).toEqual([ + 'set', + jobCancellationInternals.cancellationKey({ + queueName: 'other', + jobId: 'job-register-failure', + }), + '1', + 'EX', + 120, + ]); + await registry.close(); +}); + +test('a result rejection is owned while subscription registration is pending', async () => { + const fake = new FakeRedis(); + let release!: () => void; + fake.subscriber.subscribe = async () => { + await new Promise(resolve => { + release = resolve; + }); + return 1; + }; + const registry = new JobCancellationRegistry(redis(fake)); + const job = { + id: 'pending-registration', + queueName: 'other', + waitUntilFinished: () => Promise.reject(new Error('completion timeout')), + getState: async () => 'active', + remove: async () => {}, + } as unknown as Job; + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + }); + const rejection = waiting.catch((error: Error) => error); + // An unowned rejection fails the test runner on this event-loop turn. + await new Promise(resolve => setImmediate(resolve)); + release(); + expect(await rejection).toMatchObject({ message: 'completion timeout' }); + await registry.close(); +}); + +test('external cancellation frees a waiting job before rejecting its waiter', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + let removed = false; + const job = { + id: 'job-external-waiting', + queueName: 'other', + waitUntilFinished: () => new Promise(() => {}), + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }); + await new Promise(resolve => setImmediate(resolve)); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-external-waiting' }), + ); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(removed).toBe(true); + await registry.close(); +}); + +test('a separate cancellation request wakes the original job waiter', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const never = new Promise(() => {}); + const job = { + id: 'job-external-cancel', + queueName: 'other', + waitUntilFinished: () => never, + getState: async () => 'active', + remove: async () => undefined, + } as unknown as Job; + + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }); + await new Promise(resolve => setImmediate(resolve)); + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-external-cancel' }), + ); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(fake.deleted).toEqual([]); + await registry.close(); +}); + +test('queued removal never removes an active job', async () => { + let removed = false; + const job = { + getState: async () => 'active' as const, + remove: async () => { + removed = true; + }, + }; + + expect(await removeJobIfWaiting(job)).toBe(false); + expect(removed).toBe(false); +}); + +test('a failed cancellation write retains ownership until a durable retry succeeds', async () => { + const fake = new FakeRedis(); + fake.cancellationFailures = 2; + const target = { queueName: 'other', jobId: 'ambiguous-enqueue' }; + let released = false; + const fencing = fenceJobCancellation({ + commands: redis(fake), + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 500, + }).then(() => { + released = true; + }); + await new Promise(resolve => setTimeout(resolve, 10)); + expect(released).toBe(false); + await fencing; + expect(fake.cancellationAttempts).toBe(3); + expect(released).toBe(true); +}); + +test('an unavailable Redis cannot release ownership before the fixed job deadline', async () => { + const fake = new FakeRedis(); + fake.cancellationFailures = 1_000; + const startedAt = Date.now(); + await fenceJobCancellation({ + commands: redis(fake), + target: { queueName: 'other', jobId: 'offline' }, + ttlSeconds: 60, + deadlineAtMs: startedAt + 80, + }); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(80); + expect(fake.cancellationAttempts).toBeLessThanOrEqual(4); +}); + +test('a pending Redis write allocates no retry backlog and waits until the deadline', async () => { + const fake = new FakeRedis(); + let calls = 0; + fake.eval = async () => { + calls += 1; + return new Promise(() => {}); + }; + const startedAt = Date.now(); + await fenceJobCancellation({ + commands: redis(fake), + target: { queueName: 'other', jobId: 'pending-write' }, + ttlSeconds: 60, + deadlineAtMs: startedAt + 40, + }); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(39); + expect(calls).toBe(1); +}); + +test('queued removal frees waiting capacity and tolerates an activation race', async () => { + let removals = 0; + expect( + await removeJobIfWaiting({ + getState: async () => 'waiting', + remove: async () => { + removals += 1; + }, + }), + ).toBe(true); + expect( + await removeJobIfWaiting({ + getState: async () => 'waiting', + remove: async () => { + removals += 1; + throw new Error('job is active'); + }, + }), + ).toBe(false); + expect(removals).toBe(2); +}); diff --git a/service/src/job-cancellation.ts b/service/src/job-cancellation.ts new file mode 100644 index 00000000..4da3253d --- /dev/null +++ b/service/src/job-cancellation.ts @@ -0,0 +1,664 @@ +import type IORedis from 'ioredis'; +import type { Job, QueueEvents } from 'bullmq'; + +const JOB_CANCELLATION_PREFIX = 'codeapi:job-cancellation:v1'; +const JOB_CANCELLATION_CHANNEL = `${JOB_CANCELLATION_PREFIX}:events`; +export const CLIENT_DISCONNECT_REASON = 'client_disconnected'; +export const JOB_CANCELLED_MESSAGE = 'Job cancelled after client disconnected'; + +interface JobTarget { + queueName: string; + jobId: string; +} + +function targetKey(target: JobTarget): string { + return `${target.queueName}:${target.jobId}`; +} + +function cancellationKey(target: JobTarget): string { + return `${JOB_CANCELLATION_PREFIX}:${encodeURIComponent( + target.queueName, + )}:${encodeURIComponent(target.jobId)}`; +} + +function parseTarget(raw: string): JobTarget | undefined { + try { + const parsed = JSON.parse(raw) as Partial; + if ( + typeof parsed.queueName !== 'string' || + parsed.queueName.length === 0 || + parsed.queueName.length > 256 || + typeof parsed.jobId !== 'string' || + parsed.jobId.length === 0 || + parsed.jobId.length > 256 + ) { + return undefined; + } + return { queueName: parsed.queueName, jobId: parsed.jobId }; + } catch { + return undefined; + } +} + +/** + * Cross-process cancellation for BullMQ work. + * + * The durable marker closes the publish-before-subscribe race while one + * process-wide pub/sub connection makes active cancellation O(events), not + * O(active jobs) Redis polling. Only explicitly cancellable replay jobs use + * this path, so ordinary queue traffic pays no extra Redis round trips. + */ +export class JobCancellationRegistry { + private subscriber?: IORedis; + private readonly controllers = new Map< + string, + { target: JobTarget; controllers: Set } + >(); + private startPromise?: Promise; + private readonly subscriberEndHandlers = new WeakMap void>(); + private reconcileTimer?: ReturnType; + private subscriberRestartTimer?: ReturnType; + private reconcileRetryMs = 100; + private closed = false; + + constructor(private readonly commands: IORedis) {} + + private readonly onSubscriberError = (): void => { + // ioredis reconnects using the shared policy. The listener prevents a + // transient subscriber outage from becoming an uncaught process error. + }; + + private readonly onSubscriberReady = (): void => { + this.scheduleReconcile(0); + }; + + private readonly onSubscriberMessage = ( + channel: string, + raw: string, + ): void => { + if (channel !== JOB_CANCELLATION_CHANNEL) return; + const target = parseTarget(raw); + if (target == null) return; + for (const controller of this.controllers.get(targetKey(target)) + ?.controllers ?? []) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + }; + + private detachSubscriber(subscriber: IORedis): void { + subscriber.removeListener('error', this.onSubscriberError); + subscriber.removeListener('ready', this.onSubscriberReady); + subscriber.removeListener('message', this.onSubscriberMessage); + const onEnd = this.subscriberEndHandlers.get(subscriber); + if (onEnd != null) subscriber.removeListener('end', onEnd); + this.subscriberEndHandlers.delete(subscriber); + } + + private restartAfterTerminalDisconnect(subscriber: IORedis): void { + if (this.closed || this.subscriber !== subscriber) return; + this.detachSubscriber(subscriber); + this.subscriber = undefined; + this.startPromise = undefined; + if (this.controllers.size === 0) return; + void this.start().then( + () => this.scheduleReconcile(0), + () => this.scheduleSubscriberRestart(), + ); + } + + private scheduleSubscriberRestart(): void { + if ( + this.closed || + this.controllers.size === 0 || + this.startPromise != null || + this.subscriberRestartTimer != null + ) + return; + const retryMs = this.reconcileRetryMs; + this.reconcileRetryMs = Math.min(2_000, retryMs * 2); + this.subscriberRestartTimer = setTimeout(() => { + this.subscriberRestartTimer = undefined; + if ( + this.closed || + this.controllers.size === 0 || + this.startPromise != null + ) + return; + void this.start().then( + () => { + this.reconcileRetryMs = 100; + this.scheduleReconcile(0); + }, + () => this.scheduleSubscriberRestart(), + ); + }, retryMs); + } + + private async reconcile(): Promise { + const entries = [...this.controllers.values()]; + if (entries.length === 0) return; + const cancelled = await this.commands.mget( + ...entries.map(({ target }) => cancellationKey(target)), + ); + cancelled.forEach((value, index) => { + if (value === '1') { + for (const controller of entries[index]?.controllers ?? []) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + } + }); + } + + private scheduleReconcile(delayMs: number): void { + if ( + this.closed || + this.controllers.size === 0 || + this.reconcileTimer != null + ) { + return; + } + this.reconcileTimer = setTimeout(() => { + this.reconcileTimer = undefined; + void this.reconcile().then( + () => { + this.reconcileRetryMs = 100; + }, + () => { + const retryMs = this.reconcileRetryMs; + this.reconcileRetryMs = Math.min(2_000, retryMs * 2); + this.scheduleReconcile(retryMs); + }, + ); + }, delayMs); + } + + private start(): Promise { + if (this.closed) { + return Promise.reject(new Error('Job cancellation registry is closed')); + } + if (this.startPromise != null) return this.startPromise; + const starting = (async (): Promise => { + const subscriber = this.commands.duplicate(); + this.subscriber = subscriber; + subscriber.on('error', this.onSubscriberError); + subscriber.on('ready', this.onSubscriberReady); + subscriber.on('message', this.onSubscriberMessage); + const onEnd = (): void => this.restartAfterTerminalDisconnect(subscriber); + this.subscriberEndHandlers.set(subscriber, onEnd); + subscriber.on('end', onEnd); + try { + await subscriber.subscribe(JOB_CANCELLATION_CHANNEL); + } catch (error) { + this.detachSubscriber(subscriber); + if (this.subscriber === subscriber) this.subscriber = undefined; + subscriber.disconnect(false); + throw error; + } + })(); + this.startPromise = starting; + void starting.catch(() => { + if (this.startPromise === starting) this.startPromise = undefined; + }); + return starting; + } + + async register( + target: JobTarget, + controller: AbortController, + ): Promise { + const key = targetKey(target); + const entry = this.controllers.get(key) ?? { + target, + controllers: new Set(), + }; + entry.controllers.add(controller); + this.controllers.set(key, entry); + try { + await this.start(); + if ((await this.commands.get(cancellationKey(target))) === '1') { + controller.abort(CLIENT_DISCONNECT_REASON); + } + } catch (error) { + entry.controllers.delete(controller); + if (entry.controllers.size === 0) this.controllers.delete(key); + throw error; + } + } + + async unregister( + target: JobTarget, + controller?: AbortController, + ): Promise { + const key = targetKey(target); + const entry = this.controllers.get(key); + if (controller == null) { + this.controllers.delete(key); + } else if (entry != null) { + entry.controllers.delete(controller); + if (entry.controllers.size === 0) this.controllers.delete(key); + } + // Markers expire by TTL. Deleting one here can erase the only evidence + // needed by another replica whose subscriber was reconnecting. + } + + async close(): Promise { + this.closed = true; + this.controllers.clear(); + if (this.reconcileTimer != null) clearTimeout(this.reconcileTimer); + this.reconcileTimer = undefined; + if (this.subscriberRestartTimer != null) + clearTimeout(this.subscriberRestartTimer); + this.subscriberRestartTimer = undefined; + const subscriber = this.subscriber; + this.subscriber = undefined; + this.startPromise = undefined; + if (subscriber == null) return; + this.detachSubscriber(subscriber); + // This socket only carries notifications. Disconnect it before awaiting + // anything: subscribe() may be queued through an indefinite Redis outage. + subscriber.disconnect(false); + } +} + +async function cancelJobInRedis( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, + includeResult: boolean, +): Promise { + // Cancellation and result publication have ONE durable winner. Pub/sub is + // only a notification; it must not decide whether Stop was accepted. + return commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == 'completed' then + -- Keep completion evidence at least as long as the requesting process + -- requires, even across API/worker config differences. Attached request + -- tombstones never renew independently of this decision. + local requestedTtlMs = tonumber(ARGV[1]) * 1000 + for i = 1, 2 do + if redis.call('PTTL', KEYS[i]) < requestedTtlMs then + redis.call('PEXPIRE', KEYS[i], requestedTtlMs) + end + end + if ARGV[4] == '1' then return {0, redis.call('GET', KEYS[2])} end + return {0} + end + if state and state ~= '1' then return {-1} end + redis.call('SET', KEYS[1], '1', 'EX', ARGV[1]) + redis.call('PUBLISH', ARGV[2], ARGV[3]) + return {1} + `, + 2, + cancellationKey(target), + `${cancellationKey(target)}:result`, + Math.max(1, ttlSeconds), + JOB_CANCELLATION_CHANNEL, + JSON.stringify(target), + includeResult ? '1' : '0', + ); +} + +export async function requestJobCancellation( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, +): Promise { + const decision = await cancelJobInRedis(commands, target, ttlSeconds, false); + if (!Array.isArray(decision) || ![0, 1].includes(decision[0])) { + throw new Error('Invalid durable cancellation decision'); + } + return decision[0] === 1; +} + +export type JobFenceOutcome = + | { status: 'cancelled' | 'expired' } + | { status: 'completed'; result: T }; + +function decodeCommittedResult(value: unknown): { result: T } { + if (typeof value !== 'string') { + throw new Error( + 'Committed programmatic result expired; refusing re-execution', + ); + } + const decoded: unknown = JSON.parse(value); + if ( + decoded == null || + typeof decoded !== 'object' || + !Object.prototype.hasOwnProperty.call(decoded, 'result') + ) { + throw new Error( + 'Invalid committed programmatic result; refusing re-execution', + ); + } + return decoded as { result: T }; +} + +export function jobCancellationRetentionSeconds( + localTimeoutMs: number, + producerTtlSeconds = 0, +): number { + return Math.max( + Math.ceil(localTimeoutMs / 1_000) * 2 + 180, + Number.isFinite(producerTtlSeconds) ? producerTtlSeconds : 0, + ); +} + +/** Retain the actual result so a BullMQ retry after a lost completion reply + * cannot repeat sandbox mutations. Keep the status small: reconnect MGETs must + * never load every active job's output into each API/worker replica. */ +export async function commitJobResult( + commands: IORedis, + target: JobTarget, + result: T, + ttlSeconds: number, + deadlineAtMs = Number.MAX_SAFE_INTEGER, +): Promise<'committed' | 'cancelled' | 'already_completed'> { + const serialized = JSON.stringify({ result }); + if (Buffer.byteLength(serialized) > 16 * 1024 * 1024) { + throw new Error('Programmatic completion exceeds the 16 MiB result limit'); + } + const decision = await commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == '1' then return 0 end + if state == 'completed' then return 2 end + if state then return -2 end + if not state then + local now = redis.call('TIME') + if tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) >= tonumber(ARGV[3]) then + return -1 + end + -- One write command, so an OOM cannot publish just half the decision. + redis.call('MSET', KEYS[1], 'completed', KEYS[2], ARGV[1]) + redis.call('EXPIRE', KEYS[1], ARGV[2]) + redis.call('EXPIRE', KEYS[2], ARGV[2]) + end + return 1 + `, + 2, + cancellationKey(target), + `${cancellationKey(target)}:result`, + serialized, + Math.max(1, ttlSeconds), + deadlineAtMs, + ); + if (decision === -1) + throw new Error('Job result commitment exceeded its deadline'); + if (decision === 1) return 'committed'; + if (decision === 0) return 'cancelled'; + if (decision === 2) return 'already_completed'; + throw new Error('Invalid durable result commitment'); +} + +/** BullMQ lock loss can redeliver a job while its first processor still runs. + * Claim once before any sandbox work, retaining the claim through the job's + * recovery horizon. An ambiguous/stalled attempt is never permission to rerun. + * Completion lookup and claim are atomic, so there is no read-then-start gap. */ +export async function claimJobExecution( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, +): Promise<{ status: 'claimed' } | { status: 'completed'; result: T }> { + const key = cancellationKey(target); + const decision = await commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == 'completed' then return {0, redis.call('GET', KEYS[2])} end + if state == '1' then return {-1} end + if state then return {-3} end + if redis.call('SET', KEYS[3], '1', 'NX', 'EX', ARGV[1]) then return {1} end + return {-2} + `, + 3, + key, + `${key}:result`, + `${key}:execution`, + Math.max(1, ttlSeconds), + ); + if (!Array.isArray(decision)) throw new Error('Invalid execution claim'); + if (decision[0] === 1) return { status: 'claimed' }; + if (decision[0] === 0) + return { + status: 'completed', + result: decodeCommittedResult(decision[1]).result, + }; + if (decision[0] === -1) throw new Error(JOB_CANCELLED_MESSAGE); + if (decision[0] === -2) + throw new Error( + 'Programmatic job already claimed; refusing duplicate execution', + ); + throw new Error('Invalid durable execution claim'); +} + +export async function readCommittedJobResult( + commands: IORedis, + target: JobTarget, +): Promise<{ result: T } | undefined> { + const [state, value] = await commands.mget( + cancellationKey(target), + `${cancellationKey(target)}:result`, + ); + if (state !== 'completed') return undefined; + return decodeCommittedResult(value); +} + +/** Do not release replay ownership on an ambiguous Redis failure. Keep one + * outstanding marker write, retry rejected writes with bounded backoff, and + * retain ownership until it succeeds or the job's ORIGINAL deadline expires. + * A delayed queue.add must carry that same timestamp into the worker. */ +export async function fenceJobCancellation(args: { + commands: IORedis; + target: JobTarget; + ttlSeconds: number; + deadlineAtMs: number; +}): Promise> { + let retryMs = 25; + let firstAttempt = true; + while (firstAttempt || Date.now() < args.deadlineAtMs) { + firstAttempt = false; + // If a lost enqueue reply arrives after the execution deadline, still + // give a healthy Redis one bounded opportunity to return completion's + // winning decision. Never translate a known committed effect to failure. + const remainingMs = args.deadlineAtMs - Date.now(); + let timer: ReturnType | undefined; + let decision: unknown; + try { + decision = await Promise.race([ + cancelJobInRedis(args.commands, args.target, args.ttlSeconds, true), + new Promise(resolve => { + timer = setTimeout( + () => resolve(undefined), + remainingMs > 0 ? remainingMs : 1_000, + ); + }), + ]); + } catch { + await new Promise(resolve => + setTimeout( + resolve, + Math.min(retryMs, Math.max(0, args.deadlineAtMs - Date.now())), + ), + ); + retryMs = Math.min(1_000, retryMs * 2); + continue; + } finally { + if (timer != null) clearTimeout(timer); + } + // Only transport failures retry. Corrupt/missing durable results are + // deterministic invariant failures, not an invitation to extend their TTL. + if (decision === undefined) return { status: 'expired' }; + if (!Array.isArray(decision)) + throw new Error('Invalid durable cancellation decision'); + if (decision[0] === 1) return { status: 'cancelled' }; + if (decision[0] === 0) + return { + status: 'completed', + result: decodeCommittedResult(decision[1]).result, + }; + throw new Error('Invalid durable cancellation decision'); + } + return { status: 'expired' }; +} + +const REMOVABLE_JOB_STATES = new Set([ + 'waiting', + 'delayed', + 'prioritized', + 'waiting-children', +]); + +/** Frees queued capacity without ever removing an active or settled job. */ +export async function removeJobIfWaiting( + job: Pick, +): Promise { + if (!REMOVABLE_JOB_STATES.has(await job.getState())) return false; + try { + await job.remove(); + return true; + } catch { + // A worker may have activated the job between getState() and remove(). + // The durable marker remains authoritative for that race. + return false; + } +} + +export function programmaticCancellationError(): Error { + return new DOMException( + 'Programmatic execution request disconnected', + 'AbortError', + ); +} + +/** Commit barrier for result-processing stages that may yield after execution. */ +export function throwIfJobAborted(signal: AbortSignal): void { + if (!signal.aborted) return; + if (signal.reason instanceof Error) throw signal.reason; + throw new DOMException( + typeof signal.reason === 'string' ? signal.reason : 'Job aborted', + 'AbortError', + ); +} + +/** Maps cancellation observed during asynchronous result cleanup to the same + * stable worker failure used by the main execution catch path. */ +export function jobResultCommitFailure( + signal: AbortSignal, + jobTimeoutMs: number, +): Error | undefined { + if (!signal.aborted) return undefined; + return new Error( + signal.reason === CLIENT_DISCONNECT_REASON + ? JOB_CANCELLED_MESSAGE + : `Job timed out after ${jobTimeoutMs}ms`, + ); +} + +export async function waitForJobWithCancellation(args: { + commands: IORedis; + registry: JobCancellationRegistry; + job: Job; + events: QueueEvents; + timeoutMs: number; + cancellationTtlSeconds: number; + deadlineAtMs?: number; + signal?: AbortSignal; +}): Promise { + const { + commands, + registry, + job, + events, + timeoutMs, + cancellationTtlSeconds, + signal, + } = args; + const completion = job.waitUntilFinished(events, timeoutMs); + // Subscription startup can itself wait for Redis recovery. Own the losing + // promise immediately, before any await, rather than after registration. + void completion.catch(() => undefined); + const target = { queueName: job.queueName, jobId: String(job.id) }; + const deadlineAtMs = args.deadlineAtMs ?? Date.now() + timeoutMs; + let fencing: Promise> | undefined; + const fence = (): Promise> => + (fencing ??= fenceJobCancellation({ + commands, + target, + ttlSeconds: cancellationTtlSeconds, + deadlineAtMs, + })); + const externalController = new AbortController(); + try { + await registry.register(target, externalController); + } catch (error) { + void completion.catch(() => undefined); + const outcome = await fence(); + if (outcome.status === 'completed') return outcome.result; + await removeJobIfWaiting(job).catch(() => false); + throw error; + } + + let removeAbortListener = (): void => {}; + const disconnected = new Promise((resolve, reject) => { + let cancelling = false; + const cancel = (): void => { + if (cancelling) return; + cancelling = true; + void fence() + .then(async outcome => { + if (outcome.status === 'completed') { + resolve(outcome.result); + return; + } + // Removing a waiting job immediately frees queue capacity. An active + // job cannot be removed; its worker observes the durable marker or + // pub/sub event and aborts the sandbox transport instead. + await removeJobIfWaiting(job).catch(() => false); + reject(programmaticCancellationError()); + }) + .catch(reject); + }; + if (signal != null) { + removeAbortListener = (): void => + signal.removeEventListener('abort', cancel); + signal.addEventListener('abort', cancel, { once: true }); + if (signal.aborted) cancel(); + } + }); + const cancelled = new Promise((_, reject) => { + const cancel = (): void => { + void removeJobIfWaiting(job).then( + () => reject(programmaticCancellationError()), + () => reject(programmaticCancellationError()), + ); + }; + externalController.signal.addEventListener('abort', cancel, { + once: true, + }); + if (externalController.signal.aborted) cancel(); + }); + + // A cancelled request stops awaiting the BullMQ result, so attach a sink to + // the losing promise before racing it to avoid an unhandled late rejection. + void completion.catch(() => undefined); + try { + return await Promise.race([completion, disconnected, cancelled]); + } catch (error) { + // Includes waitUntilFinished timeouts and registration/transport errors, + // not only explicit Stop. Replay cleanup is unsafe until this barrier. + const outcome = await fence(); + if (outcome.status === 'completed') return outcome.result; + throw error; + } finally { + removeAbortListener(); + await registry + .unregister(target, externalController) + .catch(() => undefined); + } +} + +export const jobCancellationInternals = { + channel: JOB_CANCELLATION_CHANNEL, + cancellationKey, + parseTarget, +}; diff --git a/service/src/metrics.ts b/service/src/metrics.ts index adfd9872..42c5536f 100644 --- a/service/src/metrics.ts +++ b/service/src/metrics.ts @@ -117,6 +117,12 @@ export const jobsFailed = new Counter({ labelNames: ['language'] as const, }); +export const jobsCancelled = new Counter({ + name: 'codeapi_jobs_cancelled_total', + help: 'Total number of jobs cancelled after the calling client disconnected', + labelNames: ['language'] as const, +}); + export const activeJobs = new Gauge({ name: 'codeapi_active_jobs', help: 'Number of jobs currently being processed', diff --git a/service/src/middleware/limits.ts b/service/src/middleware/limits.ts index 099261a1..9ffaeb9d 100644 --- a/service/src/middleware/limits.ts +++ b/service/src/middleware/limits.ts @@ -182,6 +182,19 @@ export const executionLimiter = createRateLimiter( } ); +/** Keep Stop available when execution admission is full, while independently + * bounding request-id churn in the cancellation registry. */ +export const cancellationLimiter = createRateLimiter( + 'exec-cancel', + env.EXEC_LIMIT_WINDOW, + Math.max(80, env.EXEC_MAX_REQUESTS * 4), + { + message: 'Too many CodeAPI cancellation requests.', + structuredBody: true, + logRejections: true, + } +); + export const uploadLimiter = createRateLimiter( 'upload', env.UPLOAD_LIMIT_WINDOW, diff --git a/service/src/programmatic-cancellation.test.ts b/service/src/programmatic-cancellation.test.ts new file mode 100644 index 00000000..53a4340e --- /dev/null +++ b/service/src/programmatic-cancellation.test.ts @@ -0,0 +1,234 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test'; +import type IORedis from 'ioredis'; +import { startTestRedis } from './test/redis'; +import { commitJobResult, requestJobCancellation } from './job-cancellation'; +import { + attachProgrammaticCancellationTarget, + cancelProgrammaticRequest, + normalizeProgrammaticRequestId, + programmaticCancellationInternals, + releaseProgrammaticCancellation, + reserveProgrammaticCancellation, +} from './programmatic-cancellation'; + +let redis: IORedis & { closeTestServer(): Promise }; + +beforeEach(async () => { + redis = await startTestRedis(); +}); + +afterEach(async () => { + await redis.closeTestServer(); +}); + +test('normalizes only bounded opaque request IDs', () => { + expect(normalizeProgrammaticRequestId('request_123456789')).toBe( + 'request_123456789', + ); + expect(normalizeProgrammaticRequestId(' short ')).toBeUndefined(); + expect( + normalizeProgrammaticRequestId('../request_123456789'), + ).toBeUndefined(); + expect(normalizeProgrammaticRequestId('a'.repeat(129))).toBeUndefined(); +}); + +test('Stop cannot extend an attached tombstone before the outcome command succeeds', async () => { + const requestId = 'request_no_split_renewal'; + const owner = 'owner-a'; + const target = { queueName: 'other', jobId: 'split-renewal' }; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }); + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target, + ttlSeconds: 60, + }); + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = programmaticCancellationInternals.requestKey(requestId); + await redis.pexpire(key, 5_000); + const before = await redis.pttl(key); + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 600, + }), + ).toEqual({ status: 'accepted', target }); + // Simulate losing Redis before requestJobCancellation: no second command. + expect(await redis.pttl(key)).toBeLessThanOrEqual(before); + expect(await requestJobCancellation(redis, target, 600)).toBe(false); +}); + +test('cancellation before queue attachment is retained atomically', async () => { + const requestId = 'request_early_cancel_123'; + const owner = 'owner-a'; + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toEqual({ status: 'accepted' }); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('cancelled'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target: { queueName: 'other', jobId: '42' }, + ttlSeconds: 60, + }), + ).toBe('cancelled'); +}); + +test('cancellation after attachment returns the exact queue target', async () => { + const requestId = 'request_attached_cancel_1'; + const owner = 'owner-a'; + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('active'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target: { queueName: 'other', jobId: '43' }, + ttlSeconds: 60, + }), + ).toBe('active'); + + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toEqual({ + status: 'accepted', + target: { queueName: 'other', jobId: '43' }, + }); +}); + +test('overlapping requests from the same owner cannot share cancellation state', async () => { + const requestId = 'request_duplicate_owner_1'; + const owner = 'owner-a'; + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('active'); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('duplicate'); +}); + +test('a different principal cannot reserve, attach, cancel, or release a request', async () => { + const requestId = 'request_owned_cancel_123'; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-b', + ttlSeconds: 60, + }), + ).toBe('forbidden'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner: 'owner-b', + target: { queueName: 'other', jobId: '44' }, + ttlSeconds: 60, + }), + ).toBe('forbidden'); + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner: 'owner-b', + ttlSeconds: 60, + }), + ).toEqual({ status: 'forbidden' }); + await releaseProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-b', + }); + expect( + await redis.exists(programmaticCancellationInternals.requestKey(requestId)), + ).toBe(1); +}); + +test('settlement retains a bounded target tombstone for late Stop classification', async () => { + const requestId = 'request_release_cancel_1'; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + const target = { queueName: 'other', jobId: 'settled-job' }; + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner: 'owner-a', + target, + ttlSeconds: 60, + }); + await commitJobResult(redis, target, { stdout: 'done' }, 60); + await releaseProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + }); + const key = programmaticCancellationInternals.requestKey(requestId); + expect(await redis.exists(key)).toBe(1); + expect(await redis.ttl(key)).toBeGreaterThan(0); + expect(await redis.ttl(key)).toBeLessThanOrEqual(60); + const cancelled = await cancelProgrammaticRequest({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + expect(cancelled).toEqual({ status: 'accepted', target }); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); +}); diff --git a/service/src/programmatic-cancellation.ts b/service/src/programmatic-cancellation.ts new file mode 100644 index 00000000..12a6694d --- /dev/null +++ b/service/src/programmatic-cancellation.ts @@ -0,0 +1,182 @@ +import { createHash } from 'node:crypto'; +import type IORedis from 'ioredis'; +import type { AuthenticatedRequest } from './types'; +import { getCredentialId } from './auth/principal'; +import { getExecutionIdentity } from './execution-identity'; + +export const CODEAPI_PROGRAMMATIC_REQUEST_HEADER = + 'X-LibreChat-Code-Request-ID'; +const REQUEST_PREFIX = 'codeapi:programmatic-cancellation:v1'; +const REQUEST_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; + +interface CancellationTarget { + queueName: string; + jobId: string; +} + +export type CancellationRequestResult = + | { status: 'accepted'; target?: CancellationTarget } + | { status: 'forbidden' }; + +function requestKey(requestId: string): string { + return `${REQUEST_PREFIX}:${requestId}`; +} + +export function normalizeProgrammaticRequestId( + value: unknown, +): string | undefined { + if (typeof value !== 'string') return undefined; + const trimmed = value.trim(); + return REQUEST_ID_PATTERN.test(trimmed) ? trimmed : undefined; +} + +export function programmaticCancellationOwner( + req: AuthenticatedRequest, + userId: string, +): string { + const identity = getExecutionIdentity(req, userId); + return createHash('sha256') + .update( + JSON.stringify([ + identity.storageNamespace, + identity.canonicalUserId, + getCredentialId(req), + identity.authContextHash ?? '', + ]), + ) + .digest('hex'); +} + +const RESERVE_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local ttl = tonumber(ARGV[2]) +local existing = redis.call('HGET', key, 'owner') +if existing and existing ~= owner then return -1 end +if redis.call('HGET', key, 'reserved') == '1' then return -2 end +if not existing then + redis.call('HSET', key, 'owner', owner, 'cancelled', '0') +end +redis.call('HSET', key, 'reserved', '1') +redis.call('EXPIRE', key, ttl) +return tonumber(redis.call('HGET', key, 'cancelled') or '0') +`; + +const ATTACH_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local queueName = ARGV[2] +local jobId = ARGV[3] +local ttl = tonumber(ARGV[4]) +if redis.call('HGET', key, 'owner') ~= owner then return -1 end +redis.call('HSET', key, 'queueName', queueName, 'jobId', jobId) +redis.call('EXPIRE', key, ttl) +return tonumber(redis.call('HGET', key, 'cancelled') or '0') +`; + +const CANCEL_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local ttl = tonumber(ARGV[2]) +local existing = redis.call('HGET', key, 'owner') +if existing and existing ~= owner then return {-1} end +if not existing then redis.call('HSET', key, 'owner', owner) end +redis.call('HSET', key, 'cancelled', '1') +local queueName = redis.call('HGET', key, 'queueName') +local jobId = redis.call('HGET', key, 'jobId') +-- Once attached, never extend this mapping independently of the job decision. +-- Its original admission TTL already covers execution and late cancellation. +if queueName and jobId then return {1, queueName, jobId} end +redis.call('EXPIRE', key, ttl) +return {1} +`; + +const RELEASE_SCRIPT = ` +if redis.call('HGET', KEYS[1], 'owner') == ARGV[1] then + -- Keep the owner/target tombstone through its existing bounded TTL. A Stop + -- racing response delivery must still reach the job's completion decision. + return redis.call('HSET', KEYS[1], 'finished', '1') +end +return 0 +`; + +export async function reserveProgrammaticCancellation(args: { + redis: IORedis; + requestId: string; + owner: string; + ttlSeconds: number; +}): Promise<'active' | 'cancelled' | 'duplicate' | 'forbidden'> { + const result = Number( + await args.redis.eval( + RESERVE_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + Math.max(1, args.ttlSeconds), + ), + ); + if (result === -1) return 'forbidden'; + if (result === -2) return 'duplicate'; + return result === 1 ? 'cancelled' : 'active'; +} + +export async function attachProgrammaticCancellationTarget(args: { + redis: IORedis; + requestId: string; + owner: string; + target: CancellationTarget; + ttlSeconds: number; +}): Promise<'active' | 'cancelled' | 'forbidden'> { + const result = Number( + await args.redis.eval( + ATTACH_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + args.target.queueName, + args.target.jobId, + Math.max(1, args.ttlSeconds), + ), + ); + if (result < 0) return 'forbidden'; + return result === 1 ? 'cancelled' : 'active'; +} + +export async function cancelProgrammaticRequest(args: { + redis: IORedis; + requestId: string; + owner: string; + ttlSeconds: number; +}): Promise { + const raw = await args.redis.eval( + CANCEL_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + Math.max(1, args.ttlSeconds), + ); + const result = Array.isArray(raw) ? raw.map(String) : []; + if (result[0] === '-1') return { status: 'forbidden' }; + if (result.length >= 3) { + return { + status: 'accepted', + target: { queueName: result[1]!, jobId: result[2]! }, + }; + } + return { status: 'accepted' }; +} + +export async function releaseProgrammaticCancellation(args: { + redis: IORedis; + requestId: string; + owner: string; +}): Promise { + await args.redis.eval( + RELEASE_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + ); +} + +export const programmaticCancellationInternals = { requestKey }; diff --git a/service/src/queue.ts b/service/src/queue.ts index 54fea308..72c2fbb1 100644 --- a/service/src/queue.ts +++ b/service/src/queue.ts @@ -1,6 +1,7 @@ // src/queue.ts import IORedis from 'ioredis'; import { Queue, QueueEvents } from 'bullmq'; +import type { Job } from 'bullmq'; import { setMaxListeners } from 'events'; import type { CommonRedisOptions } from 'ioredis'; import type * as tls from 'tls'; @@ -17,19 +18,13 @@ import type { SandboxBackendName, } from './execution-profile'; import logger from './logger'; -import { redisKeepAliveOptions } from './redis-options'; +import { redisKeepAliveOptions, redisReconnectDelay } from './redis-options'; import { bullmqQueueJobs, registerBullmqQueueMetricsCollector } from './metrics'; - -const MAX_RECONNECT_ATTEMPTS = 5; -const RECONNECT_DELAY = 2000; +import { JobCancellationRegistry } from './job-cancellation'; const retryStrategy: CommonRedisOptions['retryStrategy'] = (times) => { - if (times > MAX_RECONNECT_ATTEMPTS) { - logger.error(`Failed to connect to Redis after ${times} attempts`); - return null; - } logger.warn(`Retrying Redis connection attempt ${times}`); - return RECONNECT_DELAY; + return redisReconnectDelay(times); }; const reconnectOnError: CommonRedisOptions['reconnectOnError'] = (err) => { @@ -60,6 +55,7 @@ const connection = new IORedis({ ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } : {}) }); +const jobCancellationRegistry = new JobCancellationRegistry(connection); // Global queues - no INSTANCE_ID prefix // This enables horizontal scaling where any worker can process any job @@ -110,6 +106,19 @@ export function getExecutionQueueBinding( return { ...getQueueResources(name), language }; } +/** + * Resolve a job only from this deployment's already-open queue set. Every + * homogeneous API replica opens both execution queues at startup, so this + * supports cross-replica cancellation without allocating attacker-shaped + * QueueEvents connections for arbitrary names recovered from Redis. + */ +export async function getExistingExecutionJob( + queueName: string, + jobId: string, +): Promise | undefined> { + return queueResources.get(queueName)?.queue.getJob(jobId); +} + const { queue: pyQueue, events: pyQueueEvents } = getQueueResources(queueNames.python); const { queue: otherQueue, events: otherQueueEvents } = getQueueResources(queueNames.other); @@ -163,8 +172,16 @@ export async function closeQueueConnections(): Promise { [...queueResources.values()].flatMap(({ queue, events }) => [ queue.close(), events.close(), - ]), + ]).concat(jobCancellationRegistry.close()), ); } -export { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, queueNames, connection }; +export { + pyQueue, + otherQueue, + pyQueueEvents, + otherQueueEvents, + queueNames, + connection, + jobCancellationRegistry, +}; diff --git a/service/src/redis-options.test.ts b/service/src/redis-options.test.ts index 1bb942e6..9795e279 100644 --- a/service/src/redis-options.test.ts +++ b/service/src/redis-options.test.ts @@ -1,5 +1,15 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { redisKeepAliveMs, redisKeepAliveOptions } from './redis-options'; +import { + redisKeepAliveMs, + redisKeepAliveOptions, + redisReconnectDelay, +} from './redis-options'; + +test('long-lived command connections keep recovering with a bounded retry delay', () => { + expect(redisReconnectDelay(1)).toBe(100); + expect(redisReconnectDelay(6)).toBe(600); + expect(redisReconnectDelay(1_000)).toBe(2_000); +}); describe('Redis keepalive options', () => { afterEach(() => { diff --git a/service/src/redis-options.ts b/service/src/redis-options.ts index 98d3f5b1..1f2c8e42 100644 --- a/service/src/redis-options.ts +++ b/service/src/redis-options.ts @@ -1,5 +1,11 @@ import type { CommonRedisOptions } from 'ioredis'; +/** Long-lived queue/cancellation command and subscriber connections must both + * recover after an outage. Never leave a live process with a terminal client. */ +export function redisReconnectDelay(attempt: number): number { + return Math.min(2_000, 100 * Math.max(1, attempt)); +} + export function redisKeepAliveMs(): number { const raw = process.env.REDIS_KEEP_ALIVE_MS; const trimmed = raw?.trim(); diff --git a/service/src/request-disconnect.test.ts b/service/src/request-disconnect.test.ts new file mode 100644 index 00000000..15660aca --- /dev/null +++ b/service/src/request-disconnect.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from 'bun:test'; +import { EventEmitter } from 'node:events'; +import type { Response } from 'express'; +import type { AuthenticatedRequest } from './types'; +import { CLIENT_DISCONNECT_REASON } from './job-cancellation'; +import { observeRequestDisconnect } from './request-disconnect'; + +function requestAndResponse(options: { + requestAborted?: boolean; + requestDestroyed?: boolean; + responseDestroyed?: boolean; +} = {}): { + req: AuthenticatedRequest & EventEmitter; + res: Response & EventEmitter; +} { + const req = Object.assign(new EventEmitter(), { + aborted: options.requestAborted ?? false, + destroyed: options.requestDestroyed ?? false, + }) as AuthenticatedRequest & EventEmitter; + const res = Object.assign(new EventEmitter(), { + destroyed: options.responseDestroyed ?? false, + writableFinished: false, + }) as Response & EventEmitter; + return { req, res }; +} + +test('a consumed Bun request stream is not mistaken for a disconnect', () => { + const { req, res } = requestAndResponse({ requestDestroyed: true }); + const observer = observeRequestDisconnect(req, res); + + expect(observer.isDisconnected()).toBe(false); + expect(observer.signal.aborted).toBe(false); + observer.dispose(); +}); + +test('current and future transport abandonment abort exactly once', () => { + const current = requestAndResponse({ requestAborted: true }); + const currentObserver = observeRequestDisconnect(current.req, current.res); + expect(currentObserver.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + + const future = requestAndResponse(); + const futureObserver = observeRequestDisconnect(future.req, future.res); + future.res.emit('close'); + future.req.emit('aborted'); + expect(futureObserver.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + expect(future.req.listenerCount('aborted')).toBe(0); + expect(future.res.listenerCount('close')).toBe(0); +}); + +test('a completed response disposes listeners without aborting', () => { + const { req, res } = requestAndResponse(); + const observer = observeRequestDisconnect(req, res); + (res as unknown as { writableFinished: boolean }).writableFinished = true; + res.emit('finish'); + res.emit('close'); + + expect(observer.isDisconnected()).toBe(false); + expect(req.listenerCount('aborted')).toBe(0); + expect(res.listenerCount('close')).toBe(0); +}); diff --git a/service/src/request-disconnect.ts b/service/src/request-disconnect.ts new file mode 100644 index 00000000..2a4d7a9d --- /dev/null +++ b/service/src/request-disconnect.ts @@ -0,0 +1,49 @@ +import type { Response } from 'express'; +import type { AuthenticatedRequest } from './types'; +import { CLIENT_DISCONNECT_REASON } from './job-cancellation'; + +export interface RequestDisconnectObserver { + signal: AbortSignal; + isDisconnected(): boolean; + dispose(): void; +} + +/** + * Observe a genuinely abandoned HTTP response across Node and Bun. + * + * Bun may mark the consumed IncomingMessage stream as `destroyed` while the + * response remains healthy, so request stream destruction is deliberately not + * treated as a disconnect. Express' `aborted` event and ServerResponse's + * pre-finish `close` event are the portable abandonment signals. + */ +export function observeRequestDisconnect( + req: AuthenticatedRequest, + res: Response, +): RequestDisconnectObserver { + const controller = new AbortController(); + let disposed = false; + const dispose = (): void => { + if (disposed) return; + disposed = true; + req.removeListener('aborted', disconnect); + res.removeListener('close', disconnect); + res.removeListener('finish', dispose); + }; + const disconnect = (): void => { + if (!res.writableFinished && !controller.signal.aborted) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + dispose(); + }; + + req.once('aborted', disconnect); + res.once('close', disconnect); + res.once('finish', dispose); + if (req.aborted || res.destroyed) disconnect(); + + return { + signal: controller.signal, + isDisconnected: () => controller.signal.aborted, + dispose, + }; +} diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 93bbc0c5..bb140013 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -4,13 +4,32 @@ import { Router } from 'express'; import type { Response } from 'express'; import type * as t from '../types'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; -import { executionLimiter } from '../middleware/limits'; +import { cancellationLimiter, executionLimiter } from '../middleware/limits'; import { pyQueue, pyQueueEvents, connection, + jobCancellationRegistry, getExecutionQueueBinding, + getExistingExecutionJob, } from '../queue'; +import { + JOB_CANCELLED_MESSAGE, + programmaticCancellationError, + removeJobIfWaiting, + requestJobCancellation, + fenceJobCancellation, + waitForJobWithCancellation, +} from '../job-cancellation'; +import { + CODEAPI_PROGRAMMATIC_REQUEST_HEADER, + attachProgrammaticCancellationTarget, + cancelProgrammaticRequest, + normalizeProgrammaticRequestId, + programmaticCancellationOwner, + releaseProgrammaticCancellation, + reserveProgrammaticCancellation, +} from '../programmatic-cancellation'; import { createProgrammaticPayload, extractPendingFromControlPayload, @@ -38,6 +57,7 @@ import { Jobs } from '../enum'; import { env, jobCompletionWaitTimeoutMs } from '../config'; import { resolveQueuedSandboxBackend } from '../execution-profile'; import { publicExecutionFailure } from '../utils'; +import { observeRequestDisconnect } from '../request-disconnect'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, @@ -107,6 +127,18 @@ const JOB_COMPLETION_WAIT_TIMEOUT_MS = jobCompletionWaitTimeoutMs( env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS, env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, ); +const PROGRAMMATIC_CANCELLATION_TTL_SECONDS = + Math.ceil(JOB_COMPLETION_WAIT_TIMEOUT_MS / 1000) + 60; + +interface ReplayRequestCancellation { + signal: AbortSignal; + isDisconnected(): boolean; + request?: { + requestId: string; + owner: string; + cancelledBeforeStart: boolean; + }; +} const router = Router(); @@ -322,7 +354,10 @@ async function runReplayIteration( state: ExecutionState, apiKeyId: string, userId: string, + signal?: AbortSignal, + cancellation?: { requestId: string; owner: string }, ): Promise { + if (signal?.aborted) throw programmaticCancellationError(); const history = await loadToolHistory(state.execution_id); const rawPayload = buildReplayPayload(req, state, history); const sessionKey = state.sessionKey ?? state.userId; @@ -369,41 +404,83 @@ async function runReplayIteration( state.executionProfile ?? env.EXECUTION_PROFILE, state.executionProfileSource ?? env.EXECUTION_PROFILE_SOURCE, ); - const job = await queue.add( - Jobs.execute, - { - code: state.userCode ?? '', - userId, - payload: sandboxSecurity.payload, - apiKeyId, - isPyPlot: state.isPyPlot ?? false, - principalSource: state.principalSource, - executionId: state.execution_id, - tenantId: state.tenantId, - canonicalUserId: state.canonicalUserId, - executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, - sandboxBackend: replayBackend, - ...(state.bridgeWorkerId != null - ? { bridgeWorkerId: state.bridgeWorkerId } - : {}), - ...(state.workspaceId != null - ? { workspaceId: state.workspaceId } - : {}), - runtimeSessionMode: 'stateless', - runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, - executionManifestClaims: sandboxSecurity.executionManifestClaims, - egressGrantClaims: sandboxSecurity.egressGrantClaims, - egressGrantToken: sandboxSecurity.egressGrantToken, - }, - { - removeOnComplete: { age: 60, count: 1 }, - removeOnFail: { age: 180, count: 1 }, - attempts: 1, - }, + if (signal?.aborted) throw programmaticCancellationError(); + const cancellationTarget = { queueName: queue.name, jobId: nanoid() }; + if (cancellation != null) { + const attachment = await attachProgrammaticCancellationTarget({ + redis: connection, + requestId: cancellation.requestId, + owner: cancellation.owner, + target: cancellationTarget, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + if (attachment === 'forbidden') { + throw new Error('Programmatic cancellation request ownership changed'); + } + if (attachment === 'cancelled') { + throw programmaticCancellationError(); + } + } + const submittedAtMs = Date.now(); + const deadlineAtMs = submittedAtMs + env.JOB_TIMEOUT; + let job: Awaited>; + try { + job = await queue.add( + Jobs.execute, + { + code: state.userCode ?? '', + userId, + payload: sandboxSecurity.payload, + apiKeyId, + isPyPlot: state.isPyPlot ?? false, + principalSource: state.principalSource, + executionId: state.execution_id, + tenantId: state.tenantId, + canonicalUserId: state.canonicalUserId, + executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, + sandboxBackend: replayBackend, + ...(state.bridgeWorkerId != null ? { bridgeWorkerId: state.bridgeWorkerId } : {}), + ...(state.workspaceId != null ? { workspaceId: state.workspaceId } : {}), + cancellable: true, + deadlineAtMs, + cancellationTtlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + runtimeSessionMode: 'stateless', + runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, + executionManifestClaims: sandboxSecurity.executionManifestClaims, + egressGrantClaims: sandboxSecurity.egressGrantClaims, + egressGrantToken: sandboxSecurity.egressGrantToken, + }, + { + removeOnComplete: { age: 60, count: 1 }, + removeOnFail: { age: 180, count: 1 }, + attempts: 1, + jobId: cancellationTarget.jobId, + timestamp: submittedAtMs, + }, ); + } catch (error) { + // Redis may have enqueued the job even though its reply was lost. + // Preserve replay ownership until cancellation is durable or the job's + // fixed worker deadline prevents a late admission from executing. + const outcome = await fenceJobCancellation({ + commands: connection, target: cancellationTarget, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, deadlineAtMs, + }); + if (outcome.status === 'completed') return outcome.result; + throw error; + } jobsSubmitted.inc({ language }); - return job.waitUntilFinished(events, JOB_COMPLETION_WAIT_TIMEOUT_MS); + return waitForJobWithCancellation({ + commands: connection, + registry: jobCancellationRegistry, + job, + events, + timeoutMs: JOB_COMPLETION_WAIT_TIMEOUT_MS, + cancellationTtlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + deadlineAtMs, + signal, + }); } function isSandboxRunSuccess(result: t.ExecuteResult): boolean { @@ -425,6 +502,7 @@ async function handleReplayInitial( bridgeWorkerId?: string; workspaceId?: string; }, + cancellation: ReplayRequestCancellation, ): Promise { const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; const { code, tools, user_id, files } = @@ -543,6 +621,19 @@ async function handleReplayInitial( throw error; } + if ( + cancellation.signal.aborted || + cancellation.request?.cancelledBeforeStart === true + ) { + if (!cancellation.isDisconnected()) { + res.status(200).json({ + status: 'error', + error: 'Programmatic execution request cancelled', + }); + } + return; + } + const session_id = nanoid(); const execution_id = nanoid(); const authContext = req.codeApiAuthContext; @@ -622,7 +713,7 @@ async function handleReplayInitial( timeout, }); - await runAndRespond(req, res, state, apiKeyId, userId); + await runAndRespond(req, res, state, apiKeyId, userId, cancellation); } async function handleReplayContinuation( @@ -634,6 +725,7 @@ async function handleReplayContinuation( decoded: { execution_id: string }; tool_results: NonNullable; }, + cancellation: ReplayRequestCancellation, ): Promise { const { apiKeyId, userId, decoded, tool_results } = params; @@ -692,6 +784,20 @@ async function handleReplayContinuation( res.status(404).json({ error: 'Execution not found or expired' }); return; } + if ( + cancellation.signal.aborted || + cancellation.request?.cancelledBeforeStart === true + ) { + await cleanupExecution(state.execution_id, 'replay'); + if (!cancellation.isDisconnected()) { + res.status(200).json({ + status: 'error', + error: 'Programmatic execution request cancelled', + session_id: state.session_id, + }); + } + return; + } /** Compute the delta against already-persisted history first so the * cap checks see the real impact of this batch (new call_ids only * advance `callCount`; overwrites may shrink or grow `historyBytes` @@ -845,7 +951,14 @@ async function handleReplayContinuation( }); } - await runAndRespond(req, res, state, apiKeyId, userId); + await runAndRespond( + req, + res, + state, + apiKeyId, + userId, + cancellation, + ); } finally { await releaseExecutionLock(decoded.execution_id, lockToken); } @@ -857,29 +970,29 @@ async function runAndRespond( state: ExecutionState, apiKeyId: string, userId: string, + cancellation: ReplayRequestCancellation, ): Promise { - /** Read disconnect state through `isDisconnected()` rather than a - * direct boolean. The `req.on('close', ...)` handler flips the flag - * during awaits, but `@typescript-eslint/no-unnecessary-condition` - * (correctly per TS semantics) narrows a directly-mutated `let`/object - * member to its literal value after an early-return `if (...) return`, - * even across awaits. A function call is opaque to that narrowing. */ - let disconnected = false; - const isDisconnected = (): boolean => disconnected; - req.on('close', () => { - if (!res.writableEnded) disconnected = true; - }); - let result: t.ExecuteResult; try { - result = await runReplayIteration(req, state, apiKeyId, userId); + result = await runReplayIteration( + req, + state, + apiKeyId, + userId, + cancellation.signal, + cancellation.request, + ); } catch (err) { - logger.error('Replay iteration failed', { - execution_id: state.execution_id, - err, - }); + const cancelled = + (err as Error).name === 'AbortError' || + (err as Error).message === JOB_CANCELLED_MESSAGE; + logger.log(cancelled ? 'info' : 'error', 'Replay iteration failed', { + execution_id: state.execution_id, + cancelled, + err, + }); await cleanupExecution(state.execution_id, 'replay'); - if (!isDisconnected()) { + if (!cancellation.isDisconnected()) { const publicFailure = publicExecutionFailure(err); const message = publicFailure?.body.message ?? (err as Error).message; @@ -892,7 +1005,7 @@ async function runAndRespond( return; } - if (isDisconnected()) { + if (cancellation.isDisconnected()) { logger.info('Client disconnected during replay; cleaning up', { execution_id: state.execution_id, }); @@ -1002,7 +1115,7 @@ async function runAndRespond( await cleanupExecution(state.execution_id, 'replay').catch( () => {}, ); - if (!isDisconnected()) { + if (!cancellation.isDisconnected()) { if (err instanceof ExecutionStateTooLargeError) { /** A continuation that pushes `emittedCallIds` past the * `MAX_EXECUTION_STATE_BYTES` cap is a client-input sizing @@ -1076,6 +1189,67 @@ async function runAndRespond( // Request entrypoint // --------------------------------------------------------------------------- +router.post( + '/exec/programmatic/cancel', + cancellationLimiter, + async (req: t.AuthenticatedRequest, res) => { + const principal = getPrincipalOrReject(req, res); + if (!principal) return; + const requestId = normalizeProgrammaticRequestId( + (req.body as Record)?.request_id, + ); + if (requestId == null) { + res.status(400).json({ error: 'Invalid or missing request_id' }); + return; + } + try { + const owner = programmaticCancellationOwner(req, principal.userId); + const cancellation = await cancelProgrammaticRequest({ + redis: connection, + requestId, + owner, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + if (cancellation.status === 'forbidden') { + res.status(403).json({ error: 'Programmatic request belongs to another principal' }); + return; + } + if (cancellation.target != null) { + const accepted = await requestJobCancellation( + connection, + cancellation.target, + PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + ); + if (!accepted) { + res.status(200).json({ status: 'already_completed' }); + return; + } + try { + const queuedJob = await getExistingExecutionJob( + cancellation.target.queueName, + cancellation.target.jobId, + ); + if (queuedJob != null) await removeJobIfWaiting(queuedJob); + } catch (error) { + logger.warn('Failed to remove cancelled waiting execution', { + requestId, + queueName: cancellation.target?.queueName, + jobId: cancellation.target?.jobId, + error: (error as Error).message, + }); + } + } + res.status(202).json({ status: 'cancellation_requested' }); + } catch (error) { + logger.error('Failed to request programmatic execution cancellation', { + requestId, + error: (error as Error).message, + }); + res.status(503).json({ error: 'Cancellation service unavailable' }); + } + }, +); + router.post( '/exec/programmatic', executionLimiter, @@ -1095,6 +1269,11 @@ router.post( const { continuation_token, tool_results } = req.body as t.ProgrammaticRequestBody; const rawBody = req.body as Record; + const rawRequestId = req.header(CODEAPI_PROGRAMMATIC_REQUEST_HEADER); + const requestId = normalizeProgrammaticRequestId(rawRequestId); + if (rawRequestId != null && requestId == null) { + return res.status(400).json({ error: 'Invalid programmatic request ID' }); + } const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; @@ -1151,7 +1330,51 @@ router.post( }); } + const disconnectObserver = observeRequestDisconnect(req, res); + + const cancellation: ReplayRequestCancellation = { + signal: disconnectObserver.signal, + isDisconnected: disconnectObserver.isDisconnected, + }; + let reservedCancellation: { requestId: string; owner: string } | undefined; + try { + if (requestId != null) { + const owner = programmaticCancellationOwner(req, userId); + let reservation: Awaited>; + try { + reservation = await reserveProgrammaticCancellation({ + redis: connection, + requestId, + owner, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + } catch (error) { + logger.error('Failed to reserve programmatic cancellation request', { + requestId, + error: (error as Error).message, + }); + if (!cancellation.isDisconnected()) { + return res.status(503).json({ error: 'Cancellation service unavailable' }); + } + return; + } + if (reservation === 'forbidden' || reservation === 'duplicate') { + if (!cancellation.isDisconnected()) { + return res.status(409).json({ + error: 'Programmatic request ID is already in use', + }); + } + return; + } + reservedCancellation = { requestId, owner }; + cancellation.request = { + requestId, + owner, + cancelledBeforeStart: reservation === 'cancelled', + }; + } + /** For continuations, peek at the stored execution to route by the * mode it was started in rather than the current process default. * Without this, a replay-mode execution resumed via an instance @@ -1184,7 +1407,7 @@ router.post( userId, decoded, tool_results, - }); + }, cancellation); } return await handleBlocking(req, res, { apiKeyId, userId }); } @@ -1203,7 +1426,7 @@ router.post( userId, bridgeWorkerId, workspaceId, - }); + }, cancellation); } if (workspaceId != null) { return res.status(400).json({ @@ -1221,6 +1444,20 @@ router.post( return res.status(500).json({ error: 'Internal server error' }); } return; + } finally { + disconnectObserver.dispose(); + if (reservedCancellation != null) { + await releaseProgrammaticCancellation({ + redis: connection, + requestId: reservedCancellation.requestId, + owner: reservedCancellation.owner, + }).catch(error => { + logger.warn('Failed to release programmatic cancellation request', { + requestId: reservedCancellation?.requestId, + error: (error as Error).message, + }); + }); + } } }, ); diff --git a/service/src/types/service.ts b/service/src/types/service.ts index d17b5a99..a0c78486 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -297,6 +297,12 @@ export type JobData = { bridgeWorkerId?: string; /** Trusted selected workspace for native replay-mode PTC. */ workspaceId?: string; + /** Opts replay jobs into durable client-disconnect cancellation. */ + cancellable?: boolean; + /** Absolute producer budget; queue-worker configuration may only tighten it. */ + deadlineAtMs?: number; + /** Producer request tombstones must never outlive the completion decision. */ + cancellationTtlSeconds?: number; /** Producer deployment identity. Optional only for pre-profile queued jobs. */ executionProfile?: ExecutionProfile; /** Required sandbox transport. Optional only for jobs queued before fencing. */ diff --git a/service/src/workers.ts b/service/src/workers.ts index f11a5420..a7153b53 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -1,69 +1,148 @@ import axios from 'axios'; import { Worker } from 'bullmq'; import type * as t from './types'; -import { filterSystemLogs, applySystemReplacements, getAxiosErrorDetails, sandboxErrorMessageFromAxios } from './utils'; -import { jobProcessingDuration, jobsCompleted, jobsFailed, activeJobs, workerRunning } from './metrics'; -import { connection, queueNames } from './queue'; +import { + filterSystemLogs, + applySystemReplacements, + getAxiosErrorDetails, + sandboxErrorMessageFromAxios, +} from './utils'; +import { + jobProcessingDuration, + jobsCancelled, + jobsCompleted, + jobsFailed, + activeJobs, + workerRunning, +} from './metrics'; +import { connection, jobCancellationRegistry, queueNames } from './queue'; import { env, jobDeadlineAtMs } from './config'; import { summarizeSandboxResponse, summarizeText } from './execution-log'; -import { createGatewayEgressGrant, restoreGatewaySandboxResult, revokeGatewayEgressGrant } from './egress-gateway-client'; +import { + createGatewayEgressGrant, + restoreGatewaySandboxResult, + revokeGatewayEgressGrant, +} from './egress-gateway-client'; import { refreshEgressGrantClaims } from './sandbox-egress'; import { buildSandboxExecuteRequest } from './sandbox-dispatch'; import { prepareInputDelivery } from './runtime-session/input-delivery'; import { SessionFilesError } from './runtime-session/files'; import { resolveRuntimeSessionForJob } from './runtime-session/job-policy'; -import { getSandboxBackend, SandboxBackendError, type SandboxRawResponse } from './sandbox-backend'; +import { + getSandboxBackend, + SandboxBackendError, + type SandboxRawResponse, +} from './sandbox-backend'; import { isSyntheticPrincipalSource } from './auth/synthetic'; import { withSpan, withTraceContext } from './telemetry'; import { workerDeadlineFailure } from './worker-error'; +import { + CLIENT_DISCONNECT_REASON, + JOB_CANCELLED_MESSAGE, + jobResultCommitFailure, + commitJobResult, + claimJobExecution, + jobCancellationRetentionSeconds, + throwIfJobAborted, +} from './job-cancellation'; import logger from './logger'; import { validateQueuedExecutionProfile, validateQueuedSandboxBackend, } from './execution-profile'; -import { BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, programmaticTransferReserveMs } from '../../packages/code/src/protocol'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + programmaticTransferReserveMs, +} from '../../packages/code/src/protocol'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; function isAbortError(error: unknown): boolean { - return axios.isAxiosError(error) && (error.name === 'AbortError' || error.code === 'ERR_CANCELED'); + return ( + axios.isAxiosError(error) && + (error.name === 'AbortError' || error.code === 'ERR_CANCELED') + ); } async function processJob(job: t.ExecuteJob): Promise { - return withTraceContext(job.data._otel, () => withSpan('codeapi.job.process', { - 'messaging.system': 'bullmq', - 'messaging.operation.name': 'process', - 'messaging.message.id': typeof job.id === 'string' ? job.id : String(job.id ?? ''), - 'codeapi.language': job.data.payload?.language ?? 'unknown', - 'codeapi.execution_profile': job.data.executionProfile ?? 'legacy', - 'codeapi.worker_execution_profile': env.EXECUTION_PROFILE, - }, () => processJobInner(job), 'CONSUMER')); + return withTraceContext(job.data._otel, () => + withSpan( + 'codeapi.job.process', + { + 'messaging.system': 'bullmq', + 'messaging.operation.name': 'process', + 'messaging.message.id': + typeof job.id === 'string' ? job.id : String(job.id ?? ''), + 'codeapi.language': job.data.payload?.language ?? 'unknown', + 'codeapi.execution_profile': job.data.executionProfile ?? 'legacy', + 'codeapi.worker_execution_profile': env.EXECUTION_PROFILE, + }, + () => processJobInner(job), + 'CONSUMER', + ), + ); } async function processJobInner(job: t.ExecuteJob): Promise { const { payload, isPyPlot } = job.data; - const isSyntheticJob = job.data.isSynthetic === true || isSyntheticPrincipalSource(job.data.principalSource); + const isSyntheticJob = + job.data.isSynthetic === true || + isSyntheticPrincipalSource(job.data.principalSource); const language = payload?.language ?? 'unknown'; const endTimer = jobProcessingDuration.startTimer({ language }); activeJobs.inc({ language }); const controller = new AbortController(); - const deadlineAtMs = jobDeadlineAtMs(job.timestamp, env.JOB_TIMEOUT); + const cancellationTarget = + job.data.cancellable === true && job.id != null + ? { queueName: job.queueName, jobId: String(job.id) } + : undefined; + let cancellationRegistered = false; + const deadlineAtMs = jobDeadlineAtMs( + job.timestamp, + env.JOB_TIMEOUT, + Date.now(), + job.data.deadlineAtMs, + ); const remainingBudgetMs = Math.max(0, deadlineAtMs - Date.now()); - const timer = remainingBudgetMs > 0 - ? setTimeout(() => controller.abort(), remainingBudgetMs) - : undefined; - if (remainingBudgetMs === 0) controller.abort(); + const timer = + remainingBudgetMs > 0 + ? setTimeout(() => controller.abort('deadline'), remainingBudgetMs) + : undefined; + if (remainingBudgetMs === 0) controller.abort('deadline'); let egressGrantId: string | undefined; let egressGrantTokenForRestore: string | undefined; let revokeReason = 'completed'; + let completedResult = false; + let resultToCommit: t.ExecuteResult | undefined; + let resultCommittedAtHandoff = false; + const commitAtHandoff = + cancellationTarget != null && + job.data.workspaceId != null && + env.SANDBOX_BACKEND === 'remote-bridge'; try { + if (cancellationTarget != null) { + await jobCancellationRegistry.register(cancellationTarget, controller); + cancellationRegistered = true; + const claim = await claimJobExecution( + connection, + cancellationTarget, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + ); + if (claim.status === 'completed') return claim.result; + } if (controller.signal.aborted) { throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); } - validateQueuedExecutionProfile(job.data.executionProfile, env.EXECUTION_PROFILE); + validateQueuedExecutionProfile( + job.data.executionProfile, + env.EXECUTION_PROFILE, + ); validateQueuedSandboxBackend( job.data.sandboxBackend, env.SANDBOX_BACKEND, @@ -77,7 +156,10 @@ async function processJobInner(job: t.ExecuteJob): Promise { const nowSeconds = Math.floor(Date.now() / 1000); const prepared = await createGatewayEgressGrant({ payload, - claims: refreshEgressGrantClaims(job.data.egressGrantClaims, nowSeconds), + claims: refreshEgressGrantClaims( + job.data.egressGrantClaims, + nowSeconds, + ), isSynthetic: isSyntheticJob, signal: controller.signal, }); @@ -85,19 +167,27 @@ async function processJobInner(job: t.ExecuteJob): Promise { sandboxPayload = prepared.payload; egressGrantToken = prepared.egressGrantToken; egressGrantTokenForRestore = prepared.egressGrantToken; - executionManifestClaims = (env.EXECUTION_MANIFEST_PRIVATE_KEY || env.EXECUTION_MANIFEST_SECRET) - ? prepared.executionManifestClaims - : undefined; + executionManifestClaims = + env.EXECUTION_MANIFEST_PRIVATE_KEY || env.EXECUTION_MANIFEST_SECRET + ? prepared.executionManifestClaims + : undefined; } const delivery = prepareInputDelivery(payload, sandboxPayload); const sandboxRequest = buildSandboxExecuteRequest({ - ...(job.data.workspaceId == null ? {} : { programmaticTransferReserveMs: programmaticTransferReserveMs(env.JOB_TIMEOUT) }), + ...(job.data.workspaceId == null + ? {} + : { + programmaticTransferReserveMs: programmaticTransferReserveMs( + env.JOB_TIMEOUT, + ), + }), payload: delivery.payload, egressGrantToken, executionManifestClaims, maxOutputFileBytes: Math.min( - executionManifestClaims?.max_upload_bytes ?? env.EGRESS_GATEWAY_MAX_FILE_BYTES, + executionManifestClaims?.max_upload_bytes ?? + env.EGRESS_GATEWAY_MAX_FILE_BYTES, env.EGRESS_GATEWAY_MAX_FILE_BYTES, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, ), @@ -121,21 +211,41 @@ async function processJobInner(job: t.ExecuteJob): Promise { * the transformed object makes that second call an idempotent no-op. */ const resultRestoreToken = egressGrantTokenForRestore; const finalizedSandboxResults = new WeakSet(); - const finalizeSandboxResult = async (result: SandboxRawResponse): Promise => { - if ( - resultRestoreToken === undefined || - resultRestoreToken.length === 0 || - finalizedSandboxResults.has(result) - ) { - return result; + const finalizeSandboxResult = async ( + result: SandboxRawResponse, + ): Promise => { + if (finalizedSandboxResults.has(result)) return result; + const restored = + resultRestoreToken == null || resultRestoreToken.length === 0 + ? result + : await restoreGatewaySandboxResult({ + grantId: egressGrantId, + egressGrantToken: resultRestoreToken, + result, + isSynthetic: isSyntheticJob, + signal: controller.signal, + }); + if (commitAtHandoff && cancellationTarget != null) { + // The bridge still owns its mutation fence here. A failed/ambiguous + // commit quarantines that root before it can serve a caller retry. + throwIfJobAborted(controller.signal); + const mapped = mapSandboxResult(restored); + const committed = await commitJobResult( + connection, + cancellationTarget, + mapped, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + deadlineAtMs, + ); + if (committed === 'cancelled') throw new Error(JOB_CANCELLED_MESSAGE); + if (committed === 'already_completed') + throw new Error('Duplicate mutation handoff; quarantining workspace'); + resultToCommit = mapped; + resultCommittedAtHandoff = true; } - const restored = await restoreGatewaySandboxResult({ - grantId: egressGrantId, - egressGrantToken: resultRestoreToken, - result, - isSynthetic: isSyntheticJob, - signal: controller.signal, - }); finalizedSandboxResults.add(restored); return restored; }; @@ -162,76 +272,115 @@ async function processJobInner(job: t.ExecuteJob): Promise { /* Stateful backends run this as a commit barrier after user code but * before checkpointing/reusing the mutated workspace. Stateless/HTTP * paths retain the worker-owned fallback immediately below. */ - sessionResultFinalizer: resultRestoreToken !== undefined && resultRestoreToken.length > 0 - ? finalizeSandboxResult - : undefined, + sessionResultFinalizer: + commitAtHandoff || + (resultRestoreToken !== undefined && resultRestoreToken.length > 0) + ? finalizeSandboxResult + : undefined, }, ); const responseData = await finalizeSandboxResult(responseRaw); + // Cancellation can arrive after sandbox exit while artifact restoration + // yields. Do not let BullMQ commit a success after Stop was acknowledged. + if (!resultCommittedAtHandoff) throwIfJobAborted(controller.signal); - if (!isSyntheticJob) { - logger.info('Sandbox response', summarizeSandboxResponse(responseData)); - } + function mapSandboxResult( + responseData: SandboxRawResponse, + ): t.ExecuteResult { + if (!isSyntheticJob) { + logger.info('Sandbox response', summarizeSandboxResponse(responseData)); + } - const { files } = responseData; - const run = responseData.run; - const stdout = applySystemReplacements(run?.stdout ?? ''); - const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); + const { files } = responseData; + const run = responseData.run; + const stdout = applySystemReplacements(run?.stdout ?? ''); + const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); - const result: t.ExecuteResult = { - session_id: responseData.session_id, - /* `files` is optional on the sandbox response (e.g. dry-run - * execute with no outputs); the public `ExecuteResult.files` is - * required and downstream callers always iterate it. Default to - * `[]` so the strictened response type from Phase B doesn't - * surface a regression that wasn't there before. */ - files: files ?? [], - ...(responseData.artifact_delivery != null - ? { artifact_delivery: responseData.artifact_delivery } - : {}), - ...(responseData.artifact_truncation != null - ? { artifact_truncation: responseData.artifact_truncation } - : {}), - stdout, - stderr, - ...(responseData.pending_tool_calls_payload != null - ? { pending_tool_calls_payload: responseData.pending_tool_calls_payload } - : {}), - }; + const result: t.ExecuteResult = { + session_id: responseData.session_id, + /* `files` is optional on the sandbox response (e.g. dry-run + * execute with no outputs); the public `ExecuteResult.files` is + * required and downstream callers always iterate it. Default to + * `[]` so the strictened response type from Phase B doesn't + * surface a regression that wasn't there before. */ + files: files ?? [], + ...(responseData.artifact_delivery != null + ? { artifact_delivery: responseData.artifact_delivery } + : {}), + ...(responseData.artifact_truncation != null + ? { artifact_truncation: responseData.artifact_truncation } + : {}), + stdout, + stderr, + ...(responseData.pending_tool_calls_payload != null + ? { + pending_tool_calls_payload: + responseData.pending_tool_calls_payload, + } + : {}), + }; - if (run) { - result.code = run.code ?? null; - result.signal = run.signal != null ? String(run.signal) : null; - result.message = run.message ?? null; - result.status = run.status ?? null; - result.wall_time = (run as Record).wall_time as number | null ?? null; - } + if (run) { + result.code = run.code ?? null; + result.signal = run.signal != null ? String(run.signal) : null; + result.message = run.message ?? null; + result.status = run.status ?? null; + result.wall_time = + ((run as Record).wall_time as number | null) ?? null; + } - if (result.message || result.signal) { - logger.warn('Sandbox execution error metadata', { - session_id: responseData.session_id, - code: result.code, - signal: result.signal, - message: summarizeText(result.message), - status: result.status, - wall_time: result.wall_time, - }); + if (result.message || result.signal) { + logger.warn('Sandbox execution error metadata', { + session_id: responseData.session_id, + code: result.code, + signal: result.signal, + message: summarizeText(result.message), + status: result.status, + wall_time: result.wall_time, + }); + } + + return result; } + const result = resultToCommit ?? mapSandboxResult(responseData); + completedResult = true; + resultToCommit = result; return result; } catch (error) { - revokeReason = controller.signal.aborted || isAbortError(error) ? 'timeout' : 'failed'; + // Bridge fence cleanup can fail after the outcome was durably committed. + // Preserve the winning result; the bridge retains/quarantines its fence. + if (resultCommittedAtHandoff && resultToCommit != null) + return resultToCommit; + const clientDisconnected = + controller.signal.aborted && + controller.signal.reason === CLIENT_DISCONNECT_REASON; + revokeReason = clientDisconnected + ? 'cancelled' + : controller.signal.aborted || isAbortError(error) + ? 'timeout' + : 'failed'; const errorDetails = getAxiosErrorDetails(error); - logger.error('Error processing job', errorDetails); + if (clientDisconnected) { + logger.info('Job cancelled after client disconnected', { + queueName: job.queueName, + jobId: job.id, + executionId: job.data.executionId, + }); + } else { + logger.error('Error processing job', errorDetails); + } const deadlineFailure = workerDeadlineFailure( error, - controller.signal.aborted, + controller.signal.aborted && !clientDisconnected, env.JOB_TIMEOUT, ); if (deadlineFailure) { throw deadlineFailure; + } else if (clientDisconnected) { + throw new Error(JOB_CANCELLED_MESSAGE); } else if (error instanceof SandboxBackendError) { throw new Error(`${error.code}: ${error.message}`); } else if (error instanceof SessionFilesError) { @@ -251,17 +400,67 @@ async function processJobInner(job: t.ExecuteJob): Promise { if (egressGrantId || egressGrantTokenForRestore) { await revokeGatewayEgressGrant({ grantId: egressGrantId, - egressGrantToken: egressGrantId ? undefined : egressGrantTokenForRestore, + egressGrantToken: egressGrantId + ? undefined + : egressGrantTokenForRestore, isSynthetic: isSyntheticJob, reason: revokeReason, timeoutMs: env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, }).catch(error => { - logger.error('Failed to revoke egress grant', { grantId: egressGrantId, error: getAxiosErrorDetails(error) }); + logger.error('Failed to revoke egress grant', { + grantId: egressGrantId, + error: getAxiosErrorDetails(error), + }); }); } + let lateCommitFailure = + completedResult && !resultCommittedAtHandoff + ? jobResultCommitFailure(controller.signal, env.JOB_TIMEOUT) + : undefined; + if ( + completedResult && + !resultCommittedAtHandoff && + cancellationTarget != null && + lateCommitFailure == null + ) { + try { + const committed = await commitJobResult( + connection, + cancellationTarget, + resultToCommit, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + deadlineAtMs, + ); + if (committed === 'cancelled') { + lateCommitFailure = new Error(JOB_CANCELLED_MESSAGE); + } else if (committed === 'already_completed') { + lateCommitFailure = new Error( + 'Duplicate result handoff; refusing replacement', + ); + } + } catch (error) { + lateCommitFailure = + error instanceof Error ? error : new Error('Result commit failed'); + } + } if (timer) clearTimeout(timer); + if (cancellationTarget != null && cancellationRegistered) { + await jobCancellationRegistry + .unregister(cancellationTarget, controller) + .catch(error => { + logger.warn('Failed to clear queued execution cancellation state', { + queueName: cancellationTarget.queueName, + jobId: cancellationTarget.jobId, + error: getAxiosErrorDetails(error), + }); + }); + } endTimer(); activeJobs.dec({ language }); + if (lateCommitFailure != null) throw lateCommitFailure; } } @@ -304,21 +503,31 @@ otherWorker.on('completed', job => { }); pyWorker.on('failed', (job, err) => { + if (err.message === JOB_CANCELLED_MESSAGE) { + logger.info(`[${WORKER_ID}] Python job ${job?.id} cancelled`); + jobsCancelled.inc({ language: 'python' }); + return; + } logger.error(`[${WORKER_ID}] Python job ${job?.id} failed`, err); jobsFailed.inc({ language: 'python' }); }); otherWorker.on('failed', (job, err) => { + if (err.message === JOB_CANCELLED_MESSAGE) { + logger.info(`[${WORKER_ID}] Other job ${job?.id} cancelled`); + jobsCancelled.inc({ language: 'other' }); + return; + } logger.error(`[${WORKER_ID}] Other job ${job?.id} failed`, err); jobsFailed.inc({ language: 'other' }); }); -pyWorker.on('error', (err) => { +pyWorker.on('error', err => { logger.error(`[${WORKER_ID}] Python worker error`, err); workerRunning.set({ worker_type: 'python' }, 0); }); -otherWorker.on('error', (err) => { +otherWorker.on('error', err => { logger.error(`[${WORKER_ID}] Other worker error`, err); workerRunning.set({ worker_type: 'other' }, 0); }); From 3e7b107c74960ab42197fe01d2b05a1709e4b762 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:43:49 -0400 Subject: [PATCH 19/75] fix: classify capped artifact probe candidates (#206) --- api/src/job.ts | 56 +++++++++++++++++++++++++++++++++--------- api/src/walker.test.ts | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 12 deletions(-) diff --git a/api/src/job.ts b/api/src/job.ts index 27604d29..a5b7ded2 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -2208,7 +2208,7 @@ export class Job { state = this.truncationProbeState, probeDepth = 0, rootPath = path.relative(this.submissionDir, dir) || '.', - respectSessionSuppression = false, + isOutputCapProbe = false, ): Promise { /* The state is shared by every probe in this job. Once exhausted, return * conservatively before opening yet another capped sibling directory. */ @@ -2246,17 +2246,51 @@ export class Job { if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; const existingFile = inputByName.get(relativePath); const inputFileInfo = this.inputFileHashes.get(relativePath); + let capProbeStat: fs.Stats | undefined; + if (isOutputCapProbe) { + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + relativePath, + ); + continue; + } + try { + capProbeStat = await fsp.lstat(fullPath); + if (!capProbeStat.isFile()) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during cap-probe stat'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + if (capProbeStat.size > this.runtime.max_file_size) { + /* Match handleRegularFile's one exception: an unchanged inline + * entrypoint is request input rather than an oversized output. */ + if (!inputFileInfo || existingFile?.id != null || relativePath !== this.entryPointName) { + this.recordArtifactTruncation('size', relativePath); + continue; + } + if (capProbeStat.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat.size; + try { + if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during oversized entrypoint cap probe'); + } + this.recordArtifactTruncation('size', relativePath); + continue; + } + } if ( - respectSessionSuppression + isOutputCapProbe && relativePath === this.entryPointName && existingFile?.id == null && inputFileInfo ) { try { - const st = await fsp.lstat(fullPath); - if (!st.isFile()) continue; - if (st.size > state.remainingHashBytes) return rootPath; - state.remainingHashBytes -= st.size; + if (capProbeStat!.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat!.size; if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: failed during entrypoint cap probe'); @@ -2270,13 +2304,11 @@ export class Job { * so the bounded cap probe must do the same or it reports a false * max_files warning. Current-request inputs remain reportable: they * would otherwise have been echoed into this response. */ - if (respectSessionSuppression && this.session && !existingFile) { + if (isOutputCapProbe && this.session && !existingFile) { if (this.session.isPrimedReadOnly(relativePath)) continue; try { - const st = await fsp.lstat(fullPath); - if (!st.isFile()) continue; - if (st.size > state.remainingHashBytes) return rootPath; - state.remainingHashBytes -= st.size; + if (capProbeStat!.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat!.size; const hash = await this.computeFileHash(fullPath, true); if (this.session.isSurfaced(relativePath, hash)) continue; if ( @@ -2303,7 +2335,7 @@ export class Job { state, probeDepth + 1, rootPath, - respectSessionSuppression, + isOutputCapProbe, ); if (nested) return nested; } diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index 543505a9..ba6f4af3 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -1021,6 +1021,49 @@ describe('walkDir / artifact truncation details', () => { }); }); + it('classifies an oversized supported file by size when the output cap is full', async () => { + await fsp.writeFile(path.join(tmpDir, 'oversized.txt'), 'too large'); + const internals = asInternals(makeJob({ maxFileSize: 3 })); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { size: 1 }, + skipped: ['oversized.txt'], + skipped_count: 1, + }); + }); + + it('classifies an overlong supported path by path when the output cap is full', async () => { + const directory = 'a'.repeat(200); + const filename = path.join(directory, `${'b'.repeat(60)}.txt`); + await fsp.mkdir(path.join(tmpDir, directory)); + await fsp.writeFile(path.join(tmpDir, filename), 'output'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [filename], + skipped_count: 1, + }); + }); + it('does not hash ordinary oversized files in session mode', async () => { await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); const session = new SessionWorkspace({ runtimeSessionId: 'rt_large' }); From 6ca38b23fd0b40eab66d82b0d8783e9168a092f9 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:44:01 -0400 Subject: [PATCH 20/75] feat: Report Deleted Code Session Files (#200) * fix: report deleted persisted files * fix: reconcile deletions across code runtimes * fix: preserve protected session inputs * fix: classify reserved runtime paths --- api/README.md | 7 + api/src/job.ts | 57 ++++- api/src/session-workspace.test.ts | 5 + api/src/session-workspace.ts | 6 + api/src/walker.test.ts | 196 ++++++++++++++++++ packages/code/src/native-programmatic.test.ts | 112 ++++++++++ packages/code/src/native-programmatic.ts | 44 +++- service/src/service/blocking-poll.test.ts | 2 + service/src/service/blocking-poll.ts | 2 + service/src/service/programmatic-router.ts | 4 + service/src/types/service.ts | 3 + service/src/workers.ts | 3 + 12 files changed, 424 insertions(+), 17 deletions(-) diff --git a/api/README.md b/api/README.md index 665a2931..38e4cd98 100644 --- a/api/README.md +++ b/api/README.md @@ -94,6 +94,13 @@ Other package-format-compatible runtimes (Go, Rust, Java, GCC) can be installed Execute code in a sandboxed environment. +When a persisted input file is removed during execution, a complete artifact +scan reports its relative path in `deleted_files`. Callers can use this +explicit list to remove stale file references from their next session request. +The field is omitted when no persisted inputs were removed or when artifact +scanning is incomplete, so truncation or unreadable paths cannot be mistaken +for deletions. + When supported output files are omitted because the response reaches its file count limit, nesting or path limits, file-size limit, or a filesystem entry cannot be read, the response includes `artifact_truncation`. Its `reasons` diff --git a/api/src/job.ts b/api/src/job.ts index a5b7ded2..0939de08 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -655,6 +655,8 @@ interface ExecuteResult { /** Top-level execution session id (one sandbox `/exec` invocation). */ session_id: string; files: FileRef[]; + /** Persisted input paths that no longer exist after this execution. */ + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; } @@ -714,6 +716,8 @@ export class Job { private pendingSurfaced = new Map(); private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; + private presentInputFiles = new Set(); + private deletedFiles: string[] = []; private artifactTruncation: ArtifactTruncation | undefined; private truncationProbeState: TruncationProbeState = { remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, @@ -1699,6 +1703,9 @@ export class Job { version: this.runtime.version.raw, session_id: this.outputSessionId, files: this.sessionFiles, + ...(this.deletedFiles.length > 0 + ? { deleted_files: this.deletedFiles } + : {}), ...(this.artifactTruncation ? { artifact_truncation: this.artifactTruncation } : {}), }; } @@ -1707,6 +1714,8 @@ export class Job { this.generatedFiles = []; this.sessionFiles = []; this.inheritedRefs = []; + this.presentInputFiles.clear(); + this.deletedFiles = []; this.artifactTruncation = undefined; this.truncationProbeState = { remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, @@ -1720,6 +1729,26 @@ export class Job { await this.walkDir(this.submissionDir, 0, inputByName); } catch (error) { this.log.error({ err: error }, 'Error scanning submission directory'); + this.recordArtifactTruncation('unreadable', '.'); + } + + if (this.artifactTruncation == null) { + const returnedNames = new Set([ + ...this.sessionFiles.map(file => file.name), + ...this.inheritedRefs.map(file => file.name), + ]); + for (const file of this.files) { + if ( + file.id != null && + file.storage_session_id != null && + this.inputFileHashes.get(file.name)?.readOnly !== true && + !this.presentInputFiles.has(file.name) && + !returnedNames.has(file.name) + ) { + this.deletedFiles.push(file.name); + this.session?.forgetPrimed(file.name); + } + } } /* Generated files get priority in sessionFiles; fill remaining slots up @@ -2227,13 +2256,18 @@ export class Job { let sawVisibleNonHiddenEntry = false; try { for await (const entry of directory) { - if (entry.name === PTC_HISTORY_FILENAME) continue; - sawVisibleEntry = true; - state.remainingEntries--; - if (state.remainingEntries < 0) return rootPath; const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); const kind = await this.classifyDirent(entry, fullPath, relativePath); + if (kind === 'file' && entry.name === PTC_HISTORY_FILENAME) { + if (inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } + continue; + } + sawVisibleEntry = true; + state.remainingEntries--; + if (state.remainingEntries < 0) return rootPath; if (kind === 'skip') { /* Ordinary walking counts symlinks/special entries as non-empty even * though it does not surface them, so the probe must not invent a @@ -2243,6 +2277,9 @@ export class Job { } if (kind === 'file') { sawVisibleNonHiddenEntry = true; + if (inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; const existingFile = inputByName.get(relativePath); const inputFileInfo = this.inputFileHashes.get(relativePath); @@ -2423,13 +2460,21 @@ export class Job { let skippedHiddenDirs = 0; for (const entry of entries) { - if (isPtcReserved(entry.name)) continue; - const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); const kind = await this.classifyDirent(entry, fullPath, relativePath); if (kind === 'skip') continue; + if (kind === 'file' && inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } + + /* A by-reference input may legitimately use the reserved replay-history + * basename on the ordinary execution endpoint. It remains hidden from + * output collection, but must be observed before the runtime fixture is + * skipped so an untouched input is not reported as deleted. */ + if (kind === 'file' && isPtcReserved(entry.name)) continue; + if (kind === 'dir') { /* Skip hidden directories (basename starts with `.`) unless the user * explicitly primed something under them. Matplotlib, pip, and other diff --git a/api/src/session-workspace.test.ts b/api/src/session-workspace.test.ts index 87ca71df..1a5320c6 100644 --- a/api/src/session-workspace.test.ts +++ b/api/src/session-workspace.test.ts @@ -113,6 +113,11 @@ describe('SessionWorkspace state', () => { expect(ws.isPrimedInput('in.csv')).toBe(false); ws.markPrimed('in.csv', 'file_abc'); expect(ws.primedInputId('in.csv')).toBe('file_abc'); + ws.markSurfaced('in.csv', 'old-output'); + ws.forgetPrimed('in.csv'); + expect(ws.primedInputId('in.csv')).toBeUndefined(); + expect(ws.isSurfaced('in.csv', 'old-output')).toBe(false); + ws.markPrimed('in.csv', 'file_abc'); /* read-only primes report as not-primed so the caller re-downloads them * (a reused on-disk copy could have been tampered via the writable dir). */ diff --git a/api/src/session-workspace.ts b/api/src/session-workspace.ts index 3f2105bc..82ed6c89 100644 --- a/api/src/session-workspace.ts +++ b/api/src/session-workspace.ts @@ -202,6 +202,12 @@ export class SessionWorkspace { this.primed.set(relPath, { id: storageFileId, readOnly, hash }); } + /** Clears input lineage after execution proves that the path was deleted. */ + forgetPrimed(relPath: string): void { + this.primed.delete(relPath); + this.forget(relPath); + } + markDirty(reason: string): void { this.dirty = reason; logger.error( diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index ba6f4af3..44e252a6 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -26,6 +26,8 @@ interface WalkerInternals { generatedFiles: Array<{ id: string; name: string; path: string }>; sessionFiles: Array<{ id: string; name: string; storage_session_id: string; modified_from?: { id: string; storage_session_id: string }; inherited?: true; entity_id?: string }>; inheritedRefs: Array<{ id: string; name: string; storage_session_id: string; inherited?: true; entity_id?: string }>; + presentInputFiles: Set; + deletedFiles: string[]; artifactTruncation?: { code: 'artifact_truncated'; reasons: Partial>; @@ -39,6 +41,14 @@ interface WalkerInternals { reusePrimedInput: (file: TFile) => Promise; writeFile: (file: TFile) => Promise; computeFileHash: (filePath: string, noFollow?: boolean) => Promise; + findTruncatedArtifact: ( + dir: string, + inputByName: Map, + state?: { remainingEntries: number; remainingHashBytes: number }, + probeDepth?: number, + rootPath?: string, + respectSessionSuppression?: boolean, + ) => Promise; walkDir: (dir: string, depth: number, inputByName: Map) => Promise<'collected' | 'empty' | 'skipped'>; handleSessionFiles: () => Promise; } @@ -1300,6 +1310,192 @@ describe('handleSessionFiles / priority-fill composition', () => { }); }); +describe('handleSessionFiles / persisted input deletion', () => { + it('reports a persisted input that no longer exists', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual(['removed.txt']); + }); + + it('retains a read-only persisted input when sandbox code removes its local copy', async () => { + const inherited: TFile = { + id: 'skill-id', + storage_session_id: 'skill-session', + name: path.join('skills', 'review', 'SKILL.md'), + }; + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + internals.inputFileHashes.set(inherited.name, { + hash: sha256('trusted-skill'), + path: path.join(tmpDir, inherited.name), + originalId: inherited.id, + originalSessionId: inherited.storage_session_id, + readOnly: true, + }); + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + }); + + it('tracks a persisted input using the reserved PTC history basename', async () => { + const inherited: TFile = { + id: 'history-id', + storage_session_id: 'prior-session', + name: path.join('fixtures', '_ptc_history.json'), + }; + await fsp.mkdir(path.join(tmpDir, 'fixtures')); + await fsp.writeFile(path.join(tmpDir, inherited.name), '{}'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect(internals.generatedFiles.map(file => file.name)).not.toContain(inherited.name); + }); + + it('traverses a directory that uses the reserved PTC history basename', async () => { + const inherited: TFile = { + id: 'nested-id', + storage_session_id: 'prior-session', + name: path.join('_ptc_history.json', 'data.csv'), + }; + await fsp.mkdir(path.join(tmpDir, '_ptc_history.json')); + await fsp.writeFile(path.join(tmpDir, inherited.name), 'persisted'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); + + it('tracks a reserved persisted input during a capped subtree probe', async () => { + const inherited: TFile = { + id: 'history-id', + storage_session_id: 'prior-session', + name: path.join('fixtures', '_ptc_history.json'), + }; + const fixtures = path.join(tmpDir, 'fixtures'); + await fsp.mkdir(fixtures); + await fsp.writeFile(path.join(tmpDir, inherited.name), '{}'); + await fsp.writeFile(path.join(fixtures, 'unsupported.bin'), 'binary'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + const skipped = await internals.findTruncatedArtifact( + fixtures, + new Map([[inherited.name, inherited]]) + ); + + expect(skipped).toBeUndefined(); + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); + + it('does not report a surviving input that is unsupported as an output artifact', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'archive.bin', + }; + await fsp.writeFile(path.join(tmpDir, inherited.name), 'binary-placeholder'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.deletedFiles).toEqual([]); + }); + + it('suppresses deletion reporting when the artifact scan is incomplete', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + await fsp.writeFile(path.join(tmpDir, 'too-large.txt'), 'too large'); + const internals = asInternals(makeJob({ files: [inherited], maxFileSize: 3 })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.artifactTruncation?.reasons).toEqual({ size: 1 }); + expect(internals.deletedFiles).toEqual([]); + }); + + it('does not report an inherited marker that is returned for an empty directory', async () => { + const name = path.join('empty', DIRKEEP); + const inherited: TFile = { + id: 'marker-id', + storage_session_id: 'prior-session', + name, + }; + await fsp.mkdir(path.join(tmpDir, 'empty')); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect([ + ...internals.sessionFiles, + ...internals.inheritedRefs, + ].map(file => file.name)).toContain(name); + }); + + it('clears stateful priming lineage when a persisted input is deleted', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + const session = new SessionWorkspace({ runtimeSessionId: 'rt_deleted' }); + session.markPrimed(inherited.name, inherited.id!, true, 'old-hash'); + session.markSurfaced(inherited.name, 'old-output-hash'); + const internals = asInternals(makeJob({ files: [inherited], session })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([inherited.name]); + expect(session.isPrimedInput(inherited.name)).toBe(false); + expect(session.isSurfaced(inherited.name, 'old-output-hash')).toBe(false); + }); + + it('tracks surviving persisted inputs during capped subtree probes', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: path.join('assets', 'model.bin'), + }; + await fsp.mkdir(path.join(tmpDir, 'assets')); + await fsp.writeFile( + path.join(tmpDir, inherited.name), + 'unsupported-but-persisted', + ); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.findTruncatedArtifact( + tmpDir, + new Map([[inherited.name, inherited]]), + ); + + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); +}); + describe('walkDir / dirent classification', () => { it('ignores symlinks (never classifies them as file or dir)', async () => { await fsp.writeFile(path.join(tmpDir, 'real.py'), 'print(1)'); diff --git a/packages/code/src/native-programmatic.test.ts b/packages/code/src/native-programmatic.test.ts index 82650f95..902c707a 100644 --- a/packages/code/src/native-programmatic.test.ts +++ b/packages/code/src/native-programmatic.test.ts @@ -121,6 +121,118 @@ test('stages skill files privately and returns generated artifacts', async () => } }); +test('reports persisted inputs deleted by selected-workspace execution', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-delete-test-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const server = createServer((_req, res) => res.end('persisted input')); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(() => resolve()))); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await rm(join(dataDirectory, 'input.txt')); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + + const result = await executor.execute({ + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'rm input.txt' }, + { + name: 'input.txt', + id: 'input-id', + storage_session_id: 'input-session', + }, + ], + }, + }, 'primary'); + + assert.deepEqual(result.files, []); + assert.deepEqual(result.deleted_files, ['input.txt']); +}); + +test('retains read-only persisted inputs removed by selected-workspace execution', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-readonly-delete-test-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + let downloads = 0; + const server = createServer((_req, res) => { + downloads++; + res.setHeader('X-Read-Only', 'true'); + res.end('trusted skill'); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(() => resolve()))); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await rm(join(dataDirectory, 'skills', 'review', 'SKILL.md')); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + const request = { + headers: {}, + body: { + language: 'bash' as const, + version: '5.2.0', + execution_id: 'readonly-execution', + session_id: 'execution-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'rm skills/review/SKILL.md' }, + { + name: 'skills/review/SKILL.md', + id: 'skill-id', + storage_session_id: 'skill-session', + input_cache_key: 'a'.repeat(64), + }, + ], + }, + }; + + const result = await executor.execute(request, 'primary'); + const replay = await executor.execute(request, 'primary'); + + assert.equal(downloads, 1); + assert.equal(result.deleted_files, undefined); + assert.equal(replay.deleted_files, undefined); +}); + test('reports unsupported and rejected artifacts without invalidating a completed command', async () => { const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-test-')); const uploads = new Map(); diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts index 42974b79..bd2700f2 100644 --- a/packages/code/src/native-programmatic.ts +++ b/packages/code/src/native-programmatic.ts @@ -50,6 +50,7 @@ type ProgrammaticResult = { version: string; session_id: string; files: ProgrammaticFileResult[]; + deleted_files?: string[]; artifact_delivery?: { code: 'artifact_delivery_failed'; status: 'partial' | 'failed'; @@ -75,8 +76,11 @@ type ProgrammaticResult = { type InputBaseline = { sha256: string; source?: { id: string; storage_session_id: string }; + readOnly?: boolean; }; +type CachedInput = { bytes: Buffer; readOnly: boolean }; + function sha256(value: Uint8Array): string { return createHash('sha256').update(value).digest('hex'); } @@ -227,7 +231,7 @@ export class NativeWorkspaceProgrammaticExecutor { /** Parent-process cache: sandboxed children cannot inspect this memory. */ private readonly inputCache = new Map< string, - { bytes: Buffer; lastUsed: number } + CachedInput & { lastUsed: number } >(); private inputCacheBytes = 0; @@ -272,14 +276,15 @@ export class NativeWorkspaceProgrammaticExecutor { : undefined; } - private cachedInput(key: string): Buffer | undefined { + private cachedInput(key: string): CachedInput | undefined { const cached = this.inputCache.get(key); if (!cached) return undefined; cached.lastUsed = Date.now(); - return cached.bytes; + return { bytes: cached.bytes, readOnly: cached.readOnly }; } - private cacheInput(key: string, bytes: Buffer): void { + private cacheInput(key: string, input: CachedInput): void { + const { bytes } = input; if (bytes.byteLength > INPUT_CACHE_MAX_BYTES) return; const existing = this.inputCache.get(key); if (existing) this.inputCacheBytes -= existing.bytes.byteLength; @@ -300,7 +305,7 @@ export class NativeWorkspaceProgrammaticExecutor { this.inputCache.get(oldestKey)!.bytes.byteLength; this.inputCache.delete(oldestKey); } - this.inputCache.set(key, { bytes, lastUsed: Date.now() }); + this.inputCache.set(key, { ...input, lastUsed: Date.now() }); this.inputCacheBytes += bytes.byteLength; } @@ -310,7 +315,7 @@ export class NativeWorkspaceProgrammaticExecutor { executionId: string | undefined, signal?: AbortSignal, transferTimeoutMs = TRANSFER_TIMEOUT_MS, - ): Promise { + ): Promise { const key = this.cacheKey(executionId, file); const cached = key ? this.cachedInput(key) : undefined; if (cached) return cached; @@ -341,8 +346,12 @@ export class NativeWorkspaceProgrammaticExecutor { response, controller.signal, ); - if (key) this.cacheInput(key, bytes); - return bytes; + const input = { + bytes, + readOnly: response.headers.get('x-read-only')?.toLowerCase() === 'true', + }; + if (key) this.cacheInput(key, input); + return input; } finally { clearTimeout(timer); signal?.removeEventListener('abort', abort); @@ -393,9 +402,9 @@ export class NativeWorkspaceProgrammaticExecutor { request.body.files, TRANSFER_CONCURRENCY, async (file): Promise => { - const bytes = + const input = 'content' in file - ? Buffer.from(file.content) + ? { bytes: Buffer.from(file.content), readOnly: false } : await this.downloadInput( file, grant!, @@ -403,6 +412,7 @@ export class NativeWorkspaceProgrammaticExecutor { signal, request.body.transfer_timeout_ms, ); + const { bytes } = input; totalInputBytes += bytes.byteLength; if ( totalInputBytes > @@ -421,6 +431,7 @@ export class NativeWorkspaceProgrammaticExecutor { await writeFile(path, bytes, { flag: 'wx', mode: 0o600 }); baselines.set(file.name, { sha256: sha256(bytes), + ...(input.readOnly ? { readOnly: true } : {}), ...('id' in file ? { source: { @@ -578,7 +589,15 @@ export class NativeWorkspaceProgrammaticExecutor { } const outputSessionId = request.body.output_session_id; - const outputNames = (await listRegularFiles(dataDirectory)).filter( + const survivingNames = new Set(await listRegularFiles(dataDirectory)); + const deletedFiles = refFiles + .filter( + file => + baselines.get(file.name)?.readOnly !== true && + !survivingNames.has(file.name), + ) + .map(file => file.name); + const outputNames = [...survivingNames].filter( name => name !== EXECUTION_MAIN_FILE && name !== EXECUTION_HISTORY_FILE && @@ -731,6 +750,7 @@ export class NativeWorkspaceProgrammaticExecutor { performance.now() - startedAt, undefined, artifactDelivery, + deletedFiles, ); } catch (error) { if (!commandDispatched) { @@ -785,6 +805,7 @@ export class NativeWorkspaceProgrammaticExecutor { elapsedMs: number, pendingToolCallsPayload?: string, artifactDelivery?: ProgrammaticResult['artifact_delivery'], + deletedFiles: string[] = [], ): ProgrammaticResult { return { language: 'bash', @@ -795,6 +816,7 @@ export class NativeWorkspaceProgrammaticExecutor { session_id: request.body.output_session_id ?? request.body.session_id, files, + ...(deletedFiles.length > 0 ? { deleted_files: deletedFiles } : {}), ...(artifactDelivery ? { artifact_delivery: artifactDelivery } : {}), ...(pendingToolCallsPayload ? { pending_tool_calls_payload: pendingToolCallsPayload } diff --git a/service/src/service/blocking-poll.test.ts b/service/src/service/blocking-poll.test.ts index 35190157..96903d1f 100644 --- a/service/src/service/blocking-poll.test.ts +++ b/service/src/service/blocking-poll.test.ts @@ -4,6 +4,7 @@ import type * as t from '../types'; const result: t.ExecuteResult = { session_id: 'session', stdout: 'successful code', stderr: '', files: [], + deleted_files: ['removed.txt'], artifact_delivery: { code: 'artifact_delivery_failed', status: 'failed', attempted: 1, delivered: 0, failed: 1, }, @@ -29,6 +30,7 @@ describe('blocking worker settlement', () => { const deps = fixture(); expect(await pollBlockingExecution('exec', 5, deps)).toEqual({ status: 'completed', stdout: result.stdout, stderr: '', files: [], + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, }); diff --git a/service/src/service/blocking-poll.ts b/service/src/service/blocking-poll.ts index 03e386d4..1505f818 100644 --- a/service/src/service/blocking-poll.ts +++ b/service/src/service/blocking-poll.ts @@ -33,6 +33,7 @@ export async function pollBlockingExecution( stdout?: string; stderr?: string; files?: t.FileRefs; + deleted_files?: string[]; artifact_delivery?: t.ArtifactDeliveryFailure; artifact_truncation?: t.ArtifactTruncation; }> { @@ -48,6 +49,7 @@ export async function pollBlockingExecution( stdout: result.stdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, }; diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index bb140013..1063fbfe 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -1165,6 +1165,7 @@ async function runAndRespond( stdout: cleanStdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, session_id: state.session_id, @@ -1179,6 +1180,7 @@ async function runAndRespond( stdout: cleanStdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, session_id: state.session_id, @@ -1571,6 +1573,7 @@ async function handleBlocking( stdout: state.stdout ?? '', stderr: state.stderr ?? '', files: state.files ?? [], + deleted_files: state.deleted_files, artifact_delivery: state.artifact_delivery, artifact_truncation: state.artifact_truncation, session_id: execution.session_id, @@ -1874,6 +1877,7 @@ async function handleBlocking( stdout: state.stdout ?? '', stderr: state.stderr ?? '', files: state.files ?? [], + deleted_files: state.deleted_files, artifact_delivery: state.artifact_delivery, artifact_truncation: state.artifact_truncation, session_id, diff --git a/service/src/types/service.ts b/service/src/types/service.ts index a0c78486..0404ad16 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -141,6 +141,7 @@ export type ExecuteResponse = { /** Top-level execution session id (one sandbox `/exec` invocation). */ session_id: string; files: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; }; @@ -259,6 +260,7 @@ export type ExecuteResult = { stdout: string; stderr: string; files: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; code?: number | null; @@ -408,6 +410,7 @@ export interface ProgrammaticResponse { stdout?: string; stderr?: string; files?: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; /** Top-level execution session id (one sandbox PTC invocation). */ diff --git a/service/src/workers.ts b/service/src/workers.ts index a7153b53..dbfd544b 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -305,6 +305,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { * `[]` so the strictened response type from Phase B doesn't * surface a regression that wasn't there before. */ files: files ?? [], + ...(responseData.deleted_files != null + ? { deleted_files: responseData.deleted_files } + : {}), ...(responseData.artifact_delivery != null ? { artifact_delivery: responseData.artifact_delivery } : {}), From 926569e38975acc39ff202ef5800c83e8e0bc464 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:44:10 -0400 Subject: [PATCH 21/75] fix: preserve trusted jq path for PTC (#207) --- packages/code/src/native-process.test.ts | 14 ++++++++++-- packages/code/src/native-process.ts | 11 +++++----- packages/code/src/native-sandbox.ts | 5 +++++ service/src/preamble-bash.test.ts | 8 +++++++ service/src/preamble-bash.ts | 27 ++++++++++++------------ 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index e96f1e65..2f38d642 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -3,7 +3,7 @@ import { EventEmitter } from 'node:events'; import test from 'node:test'; import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import { NativeProcessWorkspaceCommandSandbox, @@ -199,6 +199,7 @@ test('programmatic executor resolves and scopes credentials to its command', asy { workspaceRoot: tmpdir(), programmaticFileUpstream: 'http://127.0.0.1:3190', + environment: { PATH: '/sandbox-only' }, maskedEnvironment: { variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], async resolve() { @@ -225,9 +226,18 @@ test('programmatic executor resolves and scopes credentials to its command', asy JSON.stringify(fake.options).includes('per-programmatic-secret'), false, ); - const message = fake.messages.find( + const message = fake.messages.find( candidate => candidate.type === 'programmatic', )!; + const prepareMessage = fake.messages.find( + candidate => candidate.type === 'prepare', + )!; + assert.equal(typeof prepareMessage.options.jqPath, 'string'); + assert.equal(prepareMessage.options.jqPath.startsWith('/'), true); + assert.equal( + '/sandbox-only'.split(':').includes(dirname(prepareMessage.options.jqPath)), + false, + ); assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); assert.equal( message.wrappedCommand, diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index d2ea0d72..4ed3ffec 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -61,7 +61,7 @@ export async function trustedProgrammaticExecutable(candidate: string, workspace async function resolveProgrammaticShell( options: NativeProcessSandboxOptions, -): Promise { +): Promise<{ shellPath: string; jqPath: string }> { const environment = options.environment ?? process.env; const shellPath = options.shellPath != null @@ -99,7 +99,7 @@ async function resolveProgrammaticShell( 'COMMAND_UNAVAILABLE', ); } - return shellPath; + return { shellPath, jqPath }; } /** Only OS discovery and conventional proxy settings cross into the executor. @@ -211,9 +211,9 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan } private async start(): Promise { - const programmaticShellPath = this.options.programmaticFileUpstream + const programmaticExecutables = this.options.programmaticFileUpstream ? await resolveProgrammaticShell(this.options) - : this.options.shellPath; + : undefined; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], @@ -299,7 +299,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan protectedPaths, allowedDomains, homeDirectory, - shellPath: programmaticShellPath ?? shellPath, + shellPath: programmaticExecutables?.shellPath ?? shellPath, + jqPath: programmaticExecutables?.jqPath, programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 8d150d9f..91e9832d 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -169,6 +169,8 @@ export interface NativeSrtWorkspaceCommandSandboxOptions { platform?: NodeJS.Platform; /** Trusted shell path used by SRT on POSIX hosts. */ shellPath?: string; + /** Trusted jq path used by generated programmatic scripts. */ + jqPath?: string; /** Host-owned credentials exposed only as SRT sentinels inside the sandbox. */ maskedEnvironment?: { variables: Array<{ @@ -702,6 +704,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox '_ptc_pending_result.json', ), LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', + ...(this.options.jqPath + ? { LIBRECHAT_CODE_JQ_PATH: this.options.jqPath } + : {}), PTC_HISTORY_PATH: join( canonicalDataDirectory, '_ptc_history.json', diff --git a/service/src/preamble-bash.test.ts b/service/src/preamble-bash.test.ts index e35a77f5..e110abdc 100644 --- a/service/src/preamble-bash.test.ts +++ b/service/src/preamble-bash.test.ts @@ -186,6 +186,14 @@ printf '%s\\n' "$_PTC_PENDING_FILE" "$_PTC_ERROR_FILE" "$_PTC_COUNTER_FILE" rmSync(dir, { recursive: true, force: true }); } }); + + test('uses the trusted jq path instead of resolving jq through PATH', () => { + const preamble = generateBashReplayPreamble({ executionId, tools }); + expect(preamble).toContain( + '_PTC_JQ_PATH="${LIBRECHAT_CODE_JQ_PATH:-jq}"', + ); + expect(preamble).not.toMatch(/(^|[|;(]\s*)jq\s/m); + }); }); describe('generateBashReplayPreamble - command substitution pending emission', () => { diff --git a/service/src/preamble-bash.ts b/service/src/preamble-bash.ts index c442b495..31ed4c6b 100644 --- a/service/src/preamble-bash.ts +++ b/service/src/preamble-bash.ts @@ -150,6 +150,7 @@ _PTC_SENTINEL_START="${scopedStart}" _PTC_SENTINEL_END="${scopedEnd}" _PTC_HISTORY_PATH="\${PTC_HISTORY_PATH:-${PTC_HISTORY_SANDBOX_PATH}}" _PTC_CONTROL_PATH="\${LIBRECHAT_CODE_CONTROL_PATH:-}" +_PTC_JQ_PATH="\${LIBRECHAT_CODE_JQ_PATH:-jq}" _PTC_RUNTIME_DIR="\${TMPDIR:-/tmp}" _ptc_mktemp() { mktemp "\${_PTC_RUNTIME_DIR%/}/$1.XXXXXX" @@ -222,7 +223,7 @@ _ptc_sha256() { _ptc_hash_input() { local _ptc_canonical - _ptc_canonical=$(printf '%s' "$1" | jq -cS . 2>/dev/null) || return 1 + _ptc_canonical=$(printf '%s' "$1" | "$_PTC_JQ_PATH" -cS . 2>/dev/null) || return 1 printf '%s' "$_ptc_canonical" | _ptc_sha256 } @@ -369,7 +370,7 @@ _ptc_maybe_emit_pending() { return 0 fi local _ptc_payload - if ! _ptc_payload=$(jq -c -s '{pending:.}' "$_PTC_PENDING_FILE" 2>/dev/null); then + if ! _ptc_payload=$("$_PTC_JQ_PATH" -c -s '{pending:.}' "$_PTC_PENDING_FILE" 2>/dev/null); then printf 'failed to serialize pending PTC tool calls\\n' >&2 _ptc_cleanup_tempfiles trap - DEBUG EXIT @@ -480,7 +481,7 @@ _ptc_history_matches_by_signature() { return 0 fi # Path, not inline: large input can exceed ARG_MAX via --argjson. - jq -c \\ + "$_PTC_JQ_PATH" -c \\ --arg nm "$_ptc_name" \\ --arg site "$_ptc_call_site" \\ --arg hash "$_ptc_input_hash" \\ @@ -504,7 +505,7 @@ _ptc_first_unconsumed_history_match() { local _ptc_key while IFS= read -r _ptc_match; do [ -n "$_ptc_match" ] || continue - _ptc_key=$(printf '%s' "$_ptc_match" | jq -r '.key // empty' 2>/dev/null) + _ptc_key=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -r '.key // empty' 2>/dev/null) if [ -n "$_ptc_key" ] && ! grep -Fxq "$_ptc_key" "$_PTC_CONSUMED_FILE" 2>/dev/null; then printf '%s' "$_ptc_match" return 0 @@ -516,15 +517,15 @@ _ptc_first_unconsumed_history_match() { _ptc_print_history_entry() { local _ptc_entry="$1" local _ptc_is_err - _ptc_is_err=$(printf '%s' "$_ptc_entry" | jq -r 'if type == "object" then (.is_error // false) else false end' 2>/dev/null) + _ptc_is_err=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -r 'if type == "object" then (.is_error // false) else false end' 2>/dev/null) if [ "$_ptc_is_err" = "true" ]; then local _ptc_msg - _ptc_msg=$(printf '%s' "$_ptc_entry" | jq -r '.error_message // "tool execution failed"' 2>/dev/null) + _ptc_msg=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -r '.error_message // "tool execution failed"' 2>/dev/null) _ptc_write_error "$_ptc_msg" exit 1 fi local _ptc_result - _ptc_result=$(printf '%s' "$_ptc_entry" | jq -c 'if type == "object" and has("result") then .result else . end' 2>/dev/null || printf 'null') + _ptc_result=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -c 'if type == "object" and has("result") then .result else . end' 2>/dev/null || printf 'null') printf '%s' "$_ptc_result" return 0 } @@ -535,7 +536,7 @@ _ptc_history_entry_matches_current_call() { local _ptc_input_file="$3" local _ptc_input_hash="$4" # Path, same ARG_MAX reason as above. - printf '%s' "$_ptc_entry" | jq -e \\ + printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -e \\ --arg nm "$_ptc_name" \\ --arg hash "$_ptc_input_hash" \\ --slurpfile inp_arr "$_ptc_input_file" \\ @@ -555,7 +556,7 @@ _ptc_call_tool() { local _ptc_call_site="\${BASH_LINENO[1]:-\${BASH_LINENO[0]:-0}}" # Reject extra trailing JSON values instead of silently dropping them. - if ! printf '%s' "$_ptc_input" | jq -e -n '[inputs] as $docs | ($docs | length) == 1 and ($docs[0] | type) == "object"' >/dev/null 2>&1; then + if ! printf '%s' "$_ptc_input" | "$_PTC_JQ_PATH" -e -n '[inputs] as $docs | ($docs | length) == 1 and ($docs[0] | type) == "object"' >/dev/null 2>&1; then _ptc_write_error "tool input for $_ptc_name must be a single JSON object, got: $_ptc_input" exit 1 fi @@ -583,8 +584,8 @@ _ptc_call_tool() { if [ -n "$_ptc_match" ] && [ "$_ptc_match" != "null" ]; then local _ptc_matched_call_id local _ptc_matched_entry - _ptc_matched_call_id=$(printf '%s' "$_ptc_match" | jq -r '.key' 2>/dev/null) - _ptc_matched_entry=$(printf '%s' "$_ptc_match" | jq -c '.value' 2>/dev/null) + _ptc_matched_call_id=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -r '.key' 2>/dev/null) + _ptc_matched_entry=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -c '.value' 2>/dev/null) printf '%s\\n' "$_ptc_matched_call_id" >> "$_PTC_CONSUMED_FILE" _ptc_mark_counter_at_least "$_ptc_matched_call_id" _ptc_release_lock @@ -598,7 +599,7 @@ _ptc_call_tool() { while :; do _ptc_call_id=$(_ptc_next_call_id) if [ -r "$_PTC_HISTORY_PATH" ]; then - _ptc_entry=$(jq -c --arg id "$_ptc_call_id" '.[$id] // empty' "$_PTC_HISTORY_PATH" 2>/dev/null || printf '') + _ptc_entry=$("$_PTC_JQ_PATH" -c --arg id "$_ptc_call_id" '.[$id] // empty' "$_PTC_HISTORY_PATH" 2>/dev/null || printf '') else _ptc_entry="" fi @@ -614,7 +615,7 @@ _ptc_call_tool() { fi done - if ! printf '%s' "$_ptc_input" | jq -c -n \\ + if ! printf '%s' "$_ptc_input" | "$_PTC_JQ_PATH" -c -n \\ --arg cid "$_ptc_call_id" \\ --arg nm "$_ptc_name" \\ --arg hash "$_ptc_input_hash" \\ From 10ac19b74f87ec2ff4c9ff7e81198cc1f6b5e830 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:51:48 -0400 Subject: [PATCH 22/75] fix: isolate native PTC readiness and watchdog phases (#208) --- packages/code/src/native-process-child.ts | 89 ++++++-- packages/code/src/native-process.test.ts | 234 +++++++++++++++++++++- packages/code/src/native-process.ts | 140 ++++++++++--- packages/code/src/native-programmatic.ts | 15 +- packages/code/src/native-sandbox.ts | 17 +- 5 files changed, 444 insertions(+), 51 deletions(-) diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index 2e8beb38..c3b9614b 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -11,7 +11,12 @@ import type { // argv credentials, bridge token, or persisted pairing material is required. let sandbox: NativeSrtWorkspaceCommandSandbox | undefined; let programmaticExecutor: NativeWorkspaceProgrammaticExecutor | undefined; +let programmaticReady: Promise | undefined; +let programmaticFileUpstream: string | undefined; let active: { id: string; controller: AbortController } | undefined; +let commitAcknowledgement: + | { id: string; acknowledge(): void } + | undefined; let busy = false; let credentials: Record = {}; let wrappedCommand: string | undefined; @@ -25,6 +30,33 @@ function reply(message: object): void { /* Parent was lost. */ } } +async function awaitCommitAcknowledgement( + id: string, + signal: AbortSignal, +): Promise { + await new Promise((resolve, reject) => { + const abort = () => { + commitAcknowledgement = undefined; + reject( + new WorkspaceToolError( + 'Programmatic execution aborted before commit', + 'EXECUTION_ABORTED', + ), + ); + }; + commitAcknowledgement = { + id, + acknowledge() { + signal.removeEventListener('abort', abort); + commitAcknowledgement = undefined; + resolve(); + }, + }; + signal.addEventListener('abort', abort, { once: true }); + reply({ id, phase: 'commit' }); + if (signal.aborted) abort(); + }); +} let shuttingDown = false; const shutdown = () => { if (shuttingDown) return; @@ -59,19 +91,29 @@ process.on('message', async (raw: unknown) => { workspaceId?: string; credentials?: Record; wrappedCommand?: string; + programmaticShellPath?: string; + programmaticJqPath?: string; }; if (!message || typeof message.id !== 'string') return; if (message.type === 'cancel') { if (active?.id === message.id) active.controller.abort(); return; } + if (message.type === 'commit-ack') { + if (commitAcknowledgement?.id === message.id) { + commitAcknowledgement.acknowledge(); + } + return; + } if (busy) return; busy = true; + let mutationStarted = false; try { let result: unknown; if (message.type === 'prepare' && !sandbox) { - const { variables, programmaticFileUpstream, ...options } = + const { variables, programmaticFileUpstream: upstream, ...options } = message.options; + programmaticFileUpstream = upstream; sandbox = new NativeSrtWorkspaceCommandSandbox({ ...options, ...(variables @@ -89,32 +131,55 @@ process.on('message', async (raw: unknown) => { : {}), }); await sandbox.prepare(); - programmaticExecutor = programmaticFileUpstream - ? new NativeWorkspaceProgrammaticExecutor({ - sandbox, - upstreamUrl: programmaticFileUpstream, - }) - : undefined; - await programmaticExecutor?.prepare(); } else if (message.type === 'execute' && sandbox) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; + mutationStarted = true; result = await sandbox.execute(message.request, active.controller.signal); } else if ( message.type === 'programmatic' && sandbox && - programmaticExecutor && + programmaticFileUpstream && message.programmaticRequest && - typeof message.workspaceId === 'string' + typeof message.workspaceId === 'string' && + typeof message.programmaticShellPath === 'string' && + typeof message.programmaticJqPath === 'string' ) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; + if (!programmaticExecutor) { + programmaticExecutor = new NativeWorkspaceProgrammaticExecutor({ + sandbox, + upstreamUrl: programmaticFileUpstream, + shellPath: message.programmaticShellPath, + jqPath: message.programmaticJqPath, + }); + programmaticReady = programmaticExecutor.prepare( + active.controller.signal, + ); + } + try { + await programmaticReady; + } catch (error) { + programmaticExecutor = undefined; + programmaticReady = undefined; + throw error; + } result = await programmaticExecutor.execute( message.programmaticRequest, message.workspaceId, active.controller.signal, + { + async beforeCommit() { + await awaitCommitAcknowledgement( + message.id, + active!.controller.signal, + ); + mutationStarted = true; + }, + }, ); } else if (message.type === 'close' && sandbox) { await sandbox.close(); @@ -134,11 +199,11 @@ process.on('message', async (raw: unknown) => { mutation: error instanceof WorkspaceToolError ? error.mutationMayHaveCommitted - : true, + : mutationStarted, requiresQuarantine: error instanceof WorkspaceToolError ? error.requiresQuarantine - : true, + : mutationStarted, }); } finally { active = undefined; diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 2f38d642..89651845 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -11,6 +11,7 @@ import { trustedProgrammaticExecutable, } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; +import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS } from './protocol.js'; test('preflight rejects relative and workspace-controlled executables including symlinks', async t => { const root = await mkdtemp(join(tmpdir(), 'native-ptc-path-')); @@ -99,6 +100,24 @@ function fixture( }; } +class ObservedWatchdogSandbox extends NativeProcessWorkspaceCommandSandbox { + readonly watchdogTimeouts: number[] = []; + readonly watchdogCallbacks: Array<() => void> = []; + + protected override scheduleRpcTimeout( + callback: () => void, + timeoutMs: number, + ): ReturnType { + this.watchdogTimeouts.push(timeoutMs); + this.watchdogCallbacks.push(callback); + return super.scheduleRpcTimeout(callback, timeoutMs); + } + + fireLatestWatchdog(): void { + this.watchdogCallbacks.at(-1)?.(); + } +} + test('executor bootstrap excludes bridge credentials and Node injection variables', async () => { assert.deepEqual( nativeExecutorEnvironment({ @@ -229,13 +248,12 @@ test('programmatic executor resolves and scopes credentials to its command', asy const message = fake.messages.find( candidate => candidate.type === 'programmatic', )!; - const prepareMessage = fake.messages.find( - candidate => candidate.type === 'prepare', - )!; - assert.equal(typeof prepareMessage.options.jqPath, 'string'); - assert.equal(prepareMessage.options.jqPath.startsWith('/'), true); + assert.equal(typeof message.programmaticShellPath, 'string'); + assert.equal(message.programmaticShellPath.startsWith('/'), true); + assert.equal(typeof message.programmaticJqPath, 'string'); + assert.equal(message.programmaticJqPath.startsWith('/'), true); assert.equal( - '/sandbox-only'.split(':').includes(dirname(prepareMessage.options.jqPath)), + '/sandbox-only'.split(':').includes(dirname(message.programmaticJqPath)), false, ); assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); @@ -246,8 +264,109 @@ test('programmatic executor resolves and scopes credentials to its command', asy await sandbox.close(); }); +test('omitted PTC timeout gives the commit watchdog the protocol execution default', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('message', { id: message.id, ok: true, result: {} }); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + replay_tool_count: 0, + max_output_files: 0, + files: [{ name: 'main.sh', content: 'sleep 45' }], + }, + }); + assert.ok( + sandbox.watchdogTimeouts.at(-1)! > + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ); + await sandbox.close(); +}); + +test('PTC watchdog budgets staging separately and resets when commit begins', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('message', { id: message.id, ok: true, result: {} }); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + run_timeout: 1_000, + replay_tool_count: 0, + max_output_files: 0, + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }); + assert.deepEqual(sandbox.watchdogTimeouts.slice(-2), [65_000, 6_000]); + assert.ok( + fake.messages.some(message => message.type === 'commit-ack'), + 'the child must not enter the mutating phase before the parent arms it', + ); + await sandbox.close(); +}); + +test('PTC-only preflight failures do not disable ordinary native commands', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + shellPath: '/definitely/missing/bash', + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + assert.deepEqual(await sandbox.execute(request), result); + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + !error.mutationMayHaveCommitted, + ); + assert.deepEqual(await sandbox.execute(request), result); + await sandbox.close(); +}); + test('programmatic executor preserves a child-reported pre-dispatch failure', async () => { - const fake = fixture((child, message) => + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') { + child.emit('message', { id: message.id, ok: true, result }); + return; + } child.emit('message', { id: message.id, ok: false, @@ -255,8 +374,39 @@ test('programmatic executor preserves a child-reported pre-dispatch failure', as errorMessage: 'Programmatic input download failed', mutation: false, requiresQuarantine: false, + }); + }); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, }), + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, ); + assert.deepEqual(await sandbox.execute(request), result); + await sandbox.close(); +}); + +test('executor loss during programmatic staging is not an uncertain workspace mutation', async () => { + const fake = fixture((child, message) => { + if (message.type === 'programmatic') child.emit('exit', 1); + }); const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: tmpdir(), @@ -283,6 +433,76 @@ test('programmatic executor preserves a child-reported pre-dispatch failure', as await sandbox.close(); }); +test('programmatic staging watchdog expires without claiming a workspace mutation', async () => { + let staged!: () => void; + const staging = new Promise(resolve => { + staged = resolve; + }); + const fake = fixture((_child, message) => { + if (message.type === 'programmatic') staged(); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + const execution = sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }); + await staging; + sandbox.fireLatestWatchdog(); + + await assert.rejects( + execution, + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, + ); + assert.equal(fake.killCalls, 1); + await sandbox.close(); +}); + +test('executor loss after programmatic commit starts remains an uncertain mutation', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('exit', 1); + }); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted && + error.requiresQuarantine, + ); + await sandbox.close(); +}); + test('executor loss after dispatch is an uncertain mutation and is never replayed', async () => { const fake = fixture(child => child.emit('exit', 1)); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 4ed3ffec..81bbd9c4 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -7,6 +7,7 @@ import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, isWorkspaceToolRequest, isWorkspaceToolResult, @@ -29,6 +30,13 @@ export type NativeProcessSandboxOptions = Omit< }; const execFileAsync = promisify(execFile); +const PROGRAMMATIC_STAGING_TIMEOUT_MS = 60_000; +const PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; +const RPC_SETTLEMENT_SLACK_MS = 5_000; + +type RpcTimeoutBudget = + | number + | { stagingMs: number; commitMs: number }; async function systemProgrammaticExecutable( name: string, @@ -183,11 +191,16 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan private closing?: Promise; private failed = false; private terminationTimer?: ReturnType; + private programmaticExecutables?: Promise<{ + shellPath: string; + jqPath: string; + }>; private pending?: { id: string; resolve(value: unknown): void; reject(error: Error): void; mutation: boolean; + commit?(): void; }; constructor( @@ -199,6 +212,14 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan ) => ChildProcess = fork, ) {} + /** Overridable only for deterministic watchdog tests. */ + protected scheduleRpcTimeout( + callback: () => void, + timeoutMs: number, + ): ReturnType { + return setTimeout(callback, timeoutMs); + } + async prepare(): Promise { if (this.failed || this.closing) throw this.unavailable(false); if (this.ready) return this.ready; @@ -210,10 +231,22 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return new NativeExecutorUnavailableError(mutation); } + private async resolveProgrammaticExecutables(): Promise<{ + shellPath: string; + jqPath: string; + }> { + this.programmaticExecutables ??= resolveProgrammaticShell(this.options); + try { + return await this.programmaticExecutables; + } catch (error) { + // An operator may install or repair this optional dependency while the + // worker stays online. Keep ordinary execution live and let PTC retry. + this.programmaticExecutables = undefined; + throw error; + } + } + private async start(): Promise { - const programmaticExecutables = this.options.programmaticFileUpstream - ? await resolveProgrammaticShell(this.options) - : undefined; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], @@ -237,6 +270,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan code?: unknown; errorMessage?: unknown; fatal?: unknown; + phase?: unknown; }; if ( !message || @@ -246,6 +280,25 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return; const pending = this.pending; if (!pending) return; + if (message.phase === 'commit') { + pending.mutation = true; + const commit = pending.commit; + pending.commit = undefined; + commit?.(); + try { + child.send({ type: 'commit-ack', id: pending.id }, error => { + if (!error) return; + this.failed = true; + this.terminate(); + pending.reject(this.unavailable(true)); + }); + } catch { + this.failed = true; + this.terminate(); + pending.reject(this.unavailable(true)); + } + return; + } if (message.fatal === true) this.failed = true; if (message.ok === true) pending.resolve(message.result); else { @@ -299,8 +352,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan protectedPaths, allowedDomains, homeDirectory, - shellPath: programmaticExecutables?.shellPath ?? shellPath, - jqPath: programmaticExecutables?.jqPath, + shellPath, programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, @@ -376,9 +428,12 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan ); let credentials: Record | undefined; let wrappedCommand: string | undefined; + let programmaticExecutables: { shellPath: string; jqPath: string }; try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); + programmaticExecutables = await this.resolveProgrammaticExecutables(); + if (signal?.aborted) throw new Error('aborted'); credentials = await this.options.maskedEnvironment?.resolve(signal); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( @@ -407,19 +462,11 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan workspaceId, credentials, wrappedCommand, + programmaticShellPath: programmaticExecutables.shellPath, + programmaticJqPath: programmaticExecutables.jqPath, }, - (request.body.run_timeout ?? 30_000) * - ((request.body.replay_tool_count ?? 0) > 0 ? 2 : 1) + - (Math.ceil( - request.body.files.filter(file => 'id' in file).length / 4, - ) + - Math.ceil( - (request.body.max_output_files ?? - BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, - )) * - (request.body.transfer_timeout_ms ?? 30_000) + - 5_000, - true, + this.programmaticWatchdogBudget(request), + false, signal, ); if (signal?.aborted) { @@ -437,6 +484,35 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return result; } + private programmaticWatchdogBudget( + request: BridgeWorkspaceProgrammaticRequest, + ): Exclude { + const runTimeoutMs = Math.min( + request.body.run_timeout ?? BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ); + const transferTimeoutMs = + request.body.transfer_timeout_ms ?? PROGRAMMATIC_TRANSFER_TIMEOUT_MS; + const inputBatches = Math.ceil( + request.body.files.filter(file => 'id' in file).length / 4, + ); + const outputBatches = Math.ceil( + (request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, + ); + return { + stagingMs: + PROGRAMMATIC_STAGING_TIMEOUT_MS + + inputBatches * transferTimeoutMs + + ((request.body.replay_tool_count ?? 0) > 0 ? runTimeoutMs : 0) + + RPC_SETTLEMENT_SLACK_MS, + commitMs: + runTimeoutMs + + outputBatches * transferTimeoutMs + + RPC_SETTLEMENT_SLACK_MS, + }; + } + private async executeOnce( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, @@ -498,7 +574,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan private async rpc( type: string, payload: object, - timeoutMs: number, + timeout: RpcTimeoutBudget, mutation: boolean, signal?: AbortSignal, ): Promise { @@ -506,7 +582,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan throw this.unavailable(false); const id = randomUUID(); const child = this.child; - let timer: ReturnType; + let timer: ReturnType | undefined; const abort = () => { try { if (child.connected) @@ -518,12 +594,24 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan }; try { return await new Promise((resolve, reject) => { - this.pending = { id, resolve, reject, mutation }; - timer = setTimeout(() => { - this.failed = true; - this.terminate(); - reject(this.unavailable(mutation)); - }, timeoutMs); + const schedule = (timeoutMs: number): void => { + if (timer) clearTimeout(timer); + timer = this.scheduleRpcTimeout(() => { + this.failed = true; + this.terminate(); + reject(this.unavailable(this.pending?.mutation ?? mutation)); + }, timeoutMs); + }; + this.pending = { + id, + resolve, + reject, + mutation, + ...(typeof timeout === 'number' + ? {} + : { commit: () => schedule(timeout.commitMs) }), + }; + schedule(typeof timeout === 'number' ? timeout : timeout.stagingMs); signal?.addEventListener('abort', abort, { once: true }); const sendFailed = () => { this.failed = true; @@ -540,7 +628,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan if (signal?.aborted) abort(); }); } finally { - clearTimeout(timer!); + if (timer) clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending = undefined; } diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts index bd2700f2..9b6eea9e 100644 --- a/packages/code/src/native-programmatic.ts +++ b/packages/code/src/native-programmatic.ts @@ -216,6 +216,8 @@ export interface NativeWorkspaceProgrammaticOptions { Pick >; upstreamUrl: string; + shellPath?: string; + jqPath?: string; fetchImpl?: typeof fetch; } @@ -362,6 +364,7 @@ export class NativeWorkspaceProgrammaticExecutor { request: BridgeWorkspaceProgrammaticRequest, workspaceId: string, signal?: AbortSignal, + lifecycle?: { beforeCommit?(): Promise | void }, ): Promise { if (!isBridgeWorkspaceProgrammaticRequest(request)) { throw new WorkspaceToolError( @@ -456,7 +459,10 @@ export class NativeWorkspaceProgrammaticExecutor { errorOnExist: true, mode: constants.COPYFILE_FICLONE, }); - if (!probe) commandDispatched = true; + if (!probe) { + await lifecycle?.beforeCommit?.(); + commandDispatched = true; + } return await this.options.sandbox.executeProgrammatic( { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -473,7 +479,12 @@ export class NativeWorkspaceProgrammaticExecutor { }, directory, signal, - { probe, workspaceRoot }, + { + probe, + workspaceRoot, + shellPath: this.options.shellPath, + jqPath: this.options.jqPath, + }, ); }; diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 91e9832d..550d0d52 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -661,7 +661,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox request: WorkspaceExecuteCommandRequest, dataDirectory: string, signal?: AbortSignal, - options?: { probe?: boolean; workspaceRoot?: string }, + options?: { + probe?: boolean; + workspaceRoot?: string; + shellPath?: string; + jqPath?: string; + }, ): Promise { if (this.execution || this.closing) { throw new WorkspaceToolError( @@ -703,9 +708,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox canonicalDataDirectory, '_ptc_pending_result.json', ), - LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', - ...(this.options.jqPath - ? { LIBRECHAT_CODE_JQ_PATH: this.options.jqPath } + LIBRECHAT_CODE_BASH_PATH: + options?.shellPath ?? this.options.shellPath ?? '/bin/bash', + ...((options?.jqPath ?? this.options.jqPath) + ? { + LIBRECHAT_CODE_JQ_PATH: + options?.jqPath ?? this.options.jqPath, + } : {}), PTC_HISTORY_PATH: join( canonicalDataDirectory, From 3a2c2a0a974b3b01c1c509e40cc414f75b23faab Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 18:30:33 -0400 Subject: [PATCH 23/75] feat: Declare Named Worker Project Environments (#209) * feat: declare named worker project environments * fix: preserve environment trust and negotiated action boundaries * Harden environment loading and executor identity * Protect environment root traversal and exact config bytes * Reject self-controlled environment root aliases * Check filesystem identities at environment trust boundaries * Validate environment containment across Linux mount aliases * Handle stacked mounts conservatively without blocking unrelated paths --- packages/code/README.md | 52 +++ packages/code/package-lock.json | 18 +- packages/code/package.json | 3 +- packages/code/src/cli.ts | 249 +++++++++--- packages/code/src/environment-live.test.ts | 117 ++++++ packages/code/src/environment-mount.test.ts | 73 ++++ packages/code/src/environment-mount.ts | 123 ++++++ packages/code/src/environment.test.ts | 348 +++++++++++++++++ packages/code/src/environment.ts | 399 ++++++++++++++++++++ packages/code/src/private-storage.ts | 10 +- packages/code/src/protocol.ts | 239 ++++++++++-- packages/code/src/worker.ts | 21 +- packages/code/src/workspace-worker.test.ts | 50 +++ packages/code/src/workspace.ts | 3 + 14 files changed, 1617 insertions(+), 88 deletions(-) create mode 100644 packages/code/src/environment-live.test.ts create mode 100644 packages/code/src/environment-mount.test.ts create mode 100644 packages/code/src/environment-mount.ts create mode 100644 packages/code/src/environment.test.ts create mode 100644 packages/code/src/environment.ts diff --git a/packages/code/README.md b/packages/code/README.md index 87d9dded..c3bb16d3 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -652,3 +652,55 @@ To recover a quarantined native root: The workspace selector in LibreChat must preserve these registered IDs. Adding roots here does not grant a principal access or change an agent's selected root. +# Named project environments + +An operator can keep a project definition outside the coding workspace and start +the worker with `librechat-code run --environment /operator/app.yaml +--allow-workspace-commands --allow-workspace-writes`. Existing pairing settings +still identify the machine and its principal. Repeat `--environment` for independent, +non-overlapping roots (up to 32). Do not combine definitions with workspace directory, +ID, or name flags or environment variables. + +```yaml +name: app-dev +root: /projects/app +repo: example/app +ref: main +setup: + command: npm ci + timeoutMs: 300000 +actions: + - name: typecheck + command: npm run typecheck + timeoutMs: 120000 +``` + +The root must already exist; relative roots resolve from the YAML file's directory. +Repository and ref are descriptive metadata, not a clone or checkout instruction. +No Git repository is required. Definitions are loaded once at startup, hashed into +the worker's policy identity, and protected from sandbox writes. All definition +files must be outside every registered root. Unknown fields are rejected. +On Linux, startup also verifies the mount namespace so bind mounts cannot expose +definitions or their controlling paths through a workspace. The mount table is +bounded to 4 MiB, with at most 256 exposed mount boundaries; stacked and hidden +mount mappings are considered conservatively. Operators must keep mount topology stable while the +worker runs. This inspection happens at startup, not on the command hot path. + +Setup is an operator-authorized startup command under the configured native sandbox +policy. It requires commands to be enabled, runs once per worker startup before +registration, and must be idempotent for restarts. Its timeout is bounded to five +minutes and captured output to 8 KiB. Setup failure prevents registration. A crash +or uncertain termination retains the existing workspace quarantine marker; inspect +the workspace before clearing quarantine. No setup output is sent to the model. + +Named actions are fixed commands without model-supplied substitution. The bridge +advertises only their names and the definition fingerprint, never their shell source +or host root. A command request can select `environmentAction: { name, fingerprint }`; +the worker resolves the command from its loaded definition and rejects stale revisions, +unknown names, other roots, or a changed working directory. Actions use ordinary +command authorization, queueing, cancellation and quarantine. They never override +deployment approval rules or expand the pairing's principal scope. + +Rollout: update Code API and the LibreChat environment-descriptor consumer before +enabling this opt-in flag on a worker. Older validators reject the additional metadata. +Existing workers without `--environment` continue to use their existing registration. diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json index ac9affc8..426b950f 100644 --- a/packages/code/package-lock.json +++ b/packages/code/package-lock.json @@ -10,7 +10,8 @@ "license": "Apache-2.0", "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", - "koffi": "3.2.1" + "koffi": "3.2.1", + "yaml": "2.9.1" }, "bin": { "librechat-code": "dist/cli.js" @@ -414,6 +415,21 @@ "dev": true, "license": "MIT" }, + "node_modules/yaml": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/packages/code/package.json b/packages/code/package.json index 452a8be7..b00ca807 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -65,6 +65,7 @@ }, "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", - "koffi": "3.2.1" + "koffi": "3.2.1", + "yaml": "2.9.1" } } diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 289b0dc4..5498af63 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -5,6 +5,11 @@ import { realpath, stat } from 'node:fs/promises'; import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, + EnvironmentWorkspaceTools, +} from './environment.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -61,7 +66,8 @@ function workspaceSecurityIdentity( configuredToken: string | undefined, ): string { return ( - pairedPublicKey ?? required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken) + pairedPublicKey ?? + required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken) ); } @@ -70,7 +76,8 @@ function workspaceQuarantinePath(options: { workerId: string; workspaceRoot?: string; }): string { - const override = process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim(); + const override = + process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim(); if (override) return override; return defaultWorkspaceQuarantinePath({ ...options, @@ -88,7 +95,7 @@ function list(value: string | undefined): string[] { return ( value ?.split(',') - .map((item) => item.trim()) + .map(item => item.trim()) .filter(Boolean) ?? [] ); } @@ -127,7 +134,7 @@ function option(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index >= 0) return args[index + 1]; return args - .find((value) => value.startsWith(`${name}=`)) + .find(value => value.startsWith(`${name}=`)) ?.slice(name.length + 1); } @@ -175,7 +182,9 @@ function githubCredentials(): { try { parsedApiUrl = new URL(apiUrl); } catch { - throw new Error('LIBRECHAT_CODE_GITHUB_API_URL must be a valid URL'); + throw new Error( + 'LIBRECHAT_CODE_GITHUB_API_URL must be a valid URL', + ); } apiHost = parsedApiUrl.hostname.toLowerCase() === 'api.github.com' @@ -288,7 +297,7 @@ async function relay(): Promise { process.stdout.write( `librechat-code: file relay listening at ${handle.url}\n`, ); - await new Promise((resolve) => { + await new Promise(resolve => { process.once('SIGINT', resolve); process.once('SIGTERM', resolve); }); @@ -299,6 +308,47 @@ async function run( runtimeSessionId?: string, args: string[] = [], ): Promise { + const environmentPaths: string[] = []; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--environment') { + const path = args[++i]; + if (!path || path.startsWith('--')) + throw new Error('--environment requires a YAML file'); + environmentPaths.push(path); + } else if (args[i].startsWith('--environment=')) { + const path = args[i].slice('--environment='.length); + if (!path) throw new Error('--environment requires a YAML file'); + environmentPaths.push(path); + } + } + if (environmentPaths.length > 32) + throw new Error('At most 32 environments may be registered'); + const environments = await Promise.all( + environmentPaths.map(loadCodeEnvironment), + ); + if ( + environments.length && + (runtimeSessionId != null || + args.some(arg => + [ + '--worker-dir', + '--default-workspace', + '--workspace', + '--workspace-id', + '--workspace-name', + ].some(flag => arg === flag || arg.startsWith(`${flag}=`)), + ) || + [ + process.env.LIBRECHAT_CODE_WORKER_DIR, + process.env.LIBRECHAT_CODE_WORKSPACE_ID, + process.env.LIBRECHAT_CODE_WORKSPACE_NAME, + ].some(value => value?.trim()) || + process.env.LIBRECHAT_CODE_DEFAULT_WORKSPACE?.trim().toLowerCase() === 'true') + ) { + throw new Error( + '--environment cannot be combined with workspace directory, ID, or name settings', + ); + } const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); @@ -347,7 +397,8 @@ async function run( ); } const nsjailDockerMode = - runtimeMode === 'docker-nsjail' || runtimeMode === 'docker-macos-nsjail'; + runtimeMode === 'docker-nsjail' || + runtimeMode === 'docker-macos-nsjail'; const sandboxEndpoint = process.env.LIBRECHAT_CODE_SANDBOX_ENDPOINT ?? 'http://127.0.0.1:2000/api/v2'; @@ -374,16 +425,18 @@ async function run( runtimeSessionId == null && (fileRelayUpstream?.length ?? 0) > 0; const workspaceId = + environments[0]?.definition.name ?? option(args, '--workspace-id') ?? process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? 'primary'; const explicitWorkerDirectory = - runtimeSessionId == null + environments[0]?.definition.root ?? + (runtimeSessionId == null ? nonEmpty( option(args, '--worker-dir') ?? process.env.LIBRECHAT_CODE_WORKER_DIR?.trim(), ) - : undefined; + : undefined); const useDefaultWorkspace = runtimeSessionId == null && (args.includes('--default-workspace') || @@ -403,11 +456,25 @@ async function run( option(args, '--command-sandbox') ?? process.env.LIBRECHAT_CODE_COMMAND_SANDBOX?.trim().toLowerCase() ?? (nsjailDockerMode ? 'runtime' : 'native-srt'); - if (commandSandboxMode !== 'native-srt' && commandSandboxMode !== 'runtime') { + if ( + commandSandboxMode !== 'native-srt' && + commandSandboxMode !== 'runtime' + ) { throw new Error( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + if (environments.length && commandSandboxMode !== 'native-srt') { + throw new Error('Environment definitions require native-srt'); + } + if ( + environments.some(environment => environment.definition.setup) && + !allowWorkspaceCommands + ) { + throw new Error( + 'Environment setup requires --allow-workspace-commands', + ); + } const nativeProgrammaticEnabled = allowWorkspaceCommands && commandSandboxMode === 'native-srt' && @@ -486,7 +553,8 @@ async function run( } } const mutationQuarantinePath = - (allowWorkspaceWrites || allowWorkspaceCommands) && canonicalWorkerDirectory + (allowWorkspaceWrites || allowWorkspaceCommands) && + canonicalWorkerDirectory ? workspaceQuarantinePath({ codeApiUrl, workerId, @@ -508,14 +576,27 @@ async function run( root: canonicalWorkerDirectory, writable: allowWorkspaceWrites, name: + environments[0]?.definition.name ?? option(args, '--workspace-name') ?? process.env.LIBRECHAT_CODE_WORKSPACE_NAME?.trim() ?? (useDefaultWorkspace ? workspaceId - : defaultWorkspaceName(workerDirectory!, workspaceId)), + : defaultWorkspaceName( + workerDirectory!, + workspaceId, + )), }, ] : []; + for (const environment of environments.slice(1)) { + roots.push({ + id: environment.definition.name, + name: environment.definition.name, + root: environment.definition.root, + writable: allowWorkspaceWrites, + }); + } + await assertEnvironmentDefinitionsOutsideRoots(environments, roots); for (let i = 0; i < args.length; i++) { if ( args[i] === '--workspace' && @@ -551,16 +632,18 @@ async function run( if (roots.length > 32) throw new Error('At most 32 workspace roots may be registered'); const rootIdentities = await Promise.all( - roots.map((root) => stat(root.root)), + roots.map(root => stat(root.root)), ); - const normalized = roots.map((root) => root.root); + const normalized = roots.map(root => root.root); for (let i = 0; i < roots.length; i++) for (let j = 0; j < i; j++) { const inside = (a: string, b: string): boolean => { const path = relative(a, b); return ( path === '' || - (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + (path !== '..' && + !path.startsWith(`..${sep}`) && + !isAbsolute(path)) ); }; if ( @@ -578,7 +661,9 @@ async function run( workspaceLeaseSlots > 1 && (!allowWorkspaceCommands || commandSandboxMode !== 'native-srt') ) { - throw new Error('Concurrent workspace leases require native-srt commands'); + throw new Error( + 'Concurrent workspace leases require native-srt commands', + ); } if ( roots.length > 1 && @@ -589,7 +674,7 @@ async function run( ); } const rootQuarantinePaths = new Map( - roots.map((root) => [ + roots.map(root => [ root.id, workspaceQuarantinePath({ codeApiUrl, @@ -676,7 +761,10 @@ async function run( token: createHmac( 'sha256', pairedIdentity?.privateKey ?? - required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken), + required( + 'LIBRECHAT_CODE_WORKER_TOKEN', + configuredToken, + ), ) .update('librechat-code-file-relay-v1') .digest('hex'), @@ -712,8 +800,11 @@ async function run( image: runtimeImage, ...(nsjailDockerMode && runtimeSessionId == null ? (() => { - const { seccompProfile, packagesPath, profileRevision } = - nsjailLaunchProfile!; + const { + seccompProfile, + packagesPath, + profileRevision, + } = nsjailLaunchProfile!; return { capabilities: MACOS_NSJAIL_CAPABILITIES, securityOptions: [`seccomp=${seccompProfile}`], @@ -733,10 +824,12 @@ async function run( httpClient: 'bun' as const, environment: { SANDBOX_USE_CGROUPV2: 'false', - SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: + 'false', ...(workspaceMount ? { - SANDBOX_EXTERNAL_WORKSPACE_ENABLED: 'true', + SANDBOX_EXTERNAL_WORKSPACE_ENABLED: + 'true', SANDBOX_EXTERNAL_WORKSPACE_ROOT: workspaceMount.target, SANDBOX_EXTERNAL_WORKSPACE_TOKEN: @@ -745,15 +838,21 @@ async function run( : {}), ...(fileRelayProfile ? { - EGRESS_GATEWAY_URL: fileRelayProfile.url, + EGRESS_GATEWAY_URL: + fileRelayProfile.url, SANDBOX_PRIME_CONCURRENCY: String( - fileRelayLimits!.maxConcurrentRequests, + fileRelayLimits! + .maxConcurrentRequests, ), - SANDBOX_UPLOAD_CONCURRENCY: String( - fileRelayLimits!.maxConcurrentRequests, + SANDBOX_UPLOAD_CONCURRENCY: + String( + fileRelayLimits! + .maxConcurrentRequests, ), - SANDBOX_FILE_RELAY_TOKEN: fileRelayProfile.token, - SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_FILE_RELAY_TOKEN: + fileRelayProfile.token, + SANDBOX_REQUIRE_EGRESS_MANIFEST: + 'true', SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: executionManifestPublicKey!, } @@ -766,8 +865,10 @@ async function run( bindMounts: [workspaceMount], environment: { SANDBOX_EXTERNAL_WORKSPACE_ENABLED: 'true', - SANDBOX_EXTERNAL_WORKSPACE_ROOT: workspaceMount.target, - SANDBOX_EXTERNAL_WORKSPACE_TOKEN: workspaceCommandToken!, + SANDBOX_EXTERNAL_WORKSPACE_ROOT: + workspaceMount.target, + SANDBOX_EXTERNAL_WORKSPACE_TOKEN: + workspaceCommandToken!, }, } : {}), @@ -781,6 +882,7 @@ async function run( commandPolicy, protectedPaths: [ identityPath, + ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), github.privateKeyPath, ].filter((path): path is string => path != null), @@ -814,7 +916,7 @@ async function run( ? roots.length > 1 || workspaceLeaseSlots > 1 ? new NativeWorkspaceCommandPool( new Map( - roots.map((root) => [ + roots.map(root => [ root.id, { ...nativeOptions, workspaceRoot: root.root }, ]), @@ -826,7 +928,7 @@ async function run( if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ workspaceTools, - commandWorkspaces: roots.map((root) => root.id), + commandWorkspaces: roots.map(root => root.id), ...(nativeProgrammaticEnabled ? { programmaticLanguages: ['bash'] } : {}), @@ -839,6 +941,12 @@ async function run( }), }); } + if (workspaceTools && environments.length) { + workspaceTools = new EnvironmentWorkspaceTools( + workspaceTools, + environments, + ); + } const capabilities = { statefulWorkspace, sandboxProfile: @@ -854,6 +962,11 @@ async function run( policyDigest: createHash('sha256') .update(policy) .update( + environments.length + ? `\0environments\0${environments.map(environment => environment.fingerprint).join('\0')}` + : '', + ) + .update( allowWorkspaceCommands && commandSandboxMode === 'native-srt' ? `\0native-srt\0${serializeNativeSrtCommandPolicy(commandPolicy)}\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` : '', @@ -863,7 +976,9 @@ async function run( ...(workspaceLeaseSlots > 1 ? { workspaceLeaseSlots, requiresReadyConfirmation: true } : {}), - ...(workspaceTools ? { workspaceTools: workspaceTools.capabilities } : {}), + ...(workspaceTools + ? { workspaceTools: workspaceTools.capabilities } + : {}), }; if (!isValidBridgeWorkerCapabilities(capabilities)) { await fileRelaySupervisor?.stop().catch(() => undefined); @@ -874,6 +989,39 @@ async function run( try { await github.provider?.getCredential(controller.signal); await nativeCommandSandbox?.prepare(); + for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { + const setup = environment.definition.setup; + if (!setup || !nativeCommandSandbox) continue; + const id = environment.definition.name; + const guard = workspaceMutationGuard( + rootQuarantinePaths.get(id)!, + workerId, + id, + incarnationId, + ); + await guard.assertAvailable(); + await guard.arm('Environment setup did not settle', 'setup'); + const result = await nativeCommandSandbox.execute( + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: id, + command: setup.command, + timeoutMs: setup.timeoutMs, + maxOutputBytes: 8192, + }, + controller.signal, + ); + await guard.clear('setup'); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error( + `Environment ${id} setup failed; inspect the setup command before restarting`, + ); + } + process.stdout.write( + `librechat-code: environment ${id} prepared\n`, + ); + } } catch (error) { await nativeCommandSandbox?.close().catch(() => undefined); await fileRelaySupervisor?.stop().catch(() => undefined); @@ -895,7 +1043,7 @@ async function run( ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( - roots.map((root) => [ + roots.map(root => [ root.id, workspaceMutationGuard( rootQuarantinePaths.get(root.id)!, @@ -913,7 +1061,8 @@ async function run( roots.length === 1 ? { async assertAvailable() { - const record = await loadWorkspaceMutationQuarantine( + const record = + await loadWorkspaceMutationQuarantine( mutationQuarantinePath, ); if (record != null) { @@ -925,15 +1074,18 @@ async function run( } }, async arm(reason) { - await saveWorkspaceMutationQuarantine(mutationQuarantinePath, { + await saveWorkspaceMutationQuarantine( + mutationQuarantinePath, + { version: 1, workerId, workspaceId, ownerId: incarnationId, quarantinedAt: new Date().toISOString(), reason, - }); }, + ); + }, async clear() { await clearWorkspaceMutationQuarantine( mutationQuarantinePath, @@ -950,7 +1102,7 @@ async function run( : undefined, onIdentityChange: pairedIdentity && identityPath - ? async (identity) => { + ? async identity => { await saveBridgeIdentity(identityPath, { ...pairedIdentity, credential: identity.credential, @@ -959,10 +1111,12 @@ async function run( } : undefined, onRegistered: fileRelaySupervisor - ? async (registration) => { + ? async registration => { if ( registration.registrationGeneration == null || - !Number.isSafeInteger(registration.registrationGeneration) || + !Number.isSafeInteger( + registration.registrationGeneration, + ) || registration.registrationGeneration < 1 ) { throw new Error( @@ -975,10 +1129,14 @@ async function run( ); } : undefined, - onError: (error) => { + onError: error => { const message = - error instanceof Error ? error.message : 'unknown bridge error'; - process.stderr.write(`librechat-code: reconnecting after ${message}\n`); + error instanceof Error + ? error.message + : 'unknown bridge error'; + process.stderr.write( + `librechat-code: reconnecting after ${message}\n`, + ); }, }); if (runtimeSessionId !== undefined) { @@ -994,7 +1152,10 @@ async function run( if (resetNativeRoot != null) { await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); - await worker.resetNativeWorkspace(resetNativeRoot, controller.signal); + await worker.resetNativeWorkspace( + resetNativeRoot, + controller.signal, + ); process.stdout.write( `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}\n`, ); diff --git a/packages/code/src/environment-live.test.ts b/packages/code/src/environment-live.test.ts new file mode 100644 index 00000000..bb84c8b5 --- /dev/null +++ b/packages/code/src/environment-live.test.ts @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, mkdir, readFile, writeFile, rm } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +for (const { succeeds, reset } of [ + { succeeds: true, reset: false }, + { succeeds: false, reset: false }, + { succeeds: true, reset: true }, +]) { + test( + `real CLI environment setup gates registration (success=${succeeds}, reset=${reset})`, + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 20_000, + }, + async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-live-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + await mkdir(root); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + `name: project\nroot: project\nsetup:\n command: 'printf prepared > prepared.txt; exit ${succeeds ? 0 : 2}'\n timeoutMs: 5000\n`, + ); + let registrations = 0; + let receive: (() => void) | undefined; + const registered = new Promise(resolve => { + receive = resolve; + }); + const server = createServer(async (request, response) => { + request.resume(); + if (request.url?.endsWith('/register')) { + registrations++; + if (reset) + await assert.rejects( + readFile(join(root, 'prepared.txt')), + { code: 'ENOENT' }, + ); + else + assert.equal( + await readFile(join(root, 'prepared.txt'), 'utf8'), + 'prepared', + ); + receive?.(); + } + response.writeHead(503).end(); + }); + await new Promise(resolve => + server.listen(0, '127.0.0.1', resolve), + ); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const child = spawn( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--environment', + path, + '--allow-workspace-commands', + ...(reset + ? ['--reset-workspace-quarantine', 'project'] + : []), + ], + { + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + LIBRECHAT_CODE_URL: `http://127.0.0.1:${address.port}/v1`, + LIBRECHAT_CODE_WORKER_ID: 'environment-test', + LIBRECHAT_CODE_WORKER_TOKEN: 'test-only-token', + LIBRECHAT_CODE_DEFAULT_WORKSPACE: 'false', + LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE: join( + directory, + 'quarantine.json', + ), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + const exited = once(child, 'exit'); + t.after(() => child.kill('SIGKILL')); + let stderr = ''; + child.stderr.on('data', chunk => { + stderr += chunk.toString(); + }); + if (succeeds) { + await Promise.race([ + registered, + exited.then(() => { + throw new Error(stderr); + }), + ]); + child.kill('SIGTERM'); + await exited; + assert.ok(registrations > 0); + } else { + const [code] = await exited; + assert.notEqual(code, 0); + assert.match(stderr, /Environment project setup failed/); + assert.equal(registrations, 0); + } + }, + ); +} diff --git a/packages/code/src/environment-mount.test.ts b/packages/code/src/environment-mount.test.ts new file mode 100644 index 00000000..fdd5b581 --- /dev/null +++ b/packages/code/src/environment-mount.test.ts @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { assertEnvironmentMountIsolation } from './environment-mount.js'; + +const base = '1 0 8:1 / / rw - ext4 /dev/root rw\n'; +test('mount coordinates reject definition aliases in both directions and mounted files', () => { + for (const entry of [ + '2 1 8:1 /workspace/config /operator rw - ext4 /dev/root rw', + '2 1 8:1 /operator /workspace/config rw - ext4 /dev/root rw', + '2 1 8:1 /operator/app.yaml /workspace/app.yaml rw - ext4 /dev/root rw', + '2 1 8:1 /workspace/app.yaml /operator/app.yaml rw - ext4 /dev/root rw', + ]) + assert.throws( + () => + assertEnvironmentMountIsolation( + base + entry, + ['/operator/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); +}); +test('mount coordinates retain safe separate filesystems and escaped paths', () => { + assertEnvironmentMountIsolation( + base + '2 1 9:1 / /workspace rw - ext4 /dev/other rw', + ['/operator/app.yaml'], + ['/workspace'], + ); + assertEnvironmentMountIsolation( + base, + ['/operator/app.yaml'], + ['/workspace'], + ); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + + '2 1 8:1 /workspace/my\\040config /operator rw - ext4 /dev/root rw', + ['/operator/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); + assert.throws(() => assertEnvironmentMountIsolation('invalid', [], [])); + assertEnvironmentMountIsolation( + base + '2 1 9:1 / / rw - ext4 /dev/other rw', + ['/operator/app.yaml'], + ['/workspace'], + ); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + '2 1 9:1 / / rw - ext4 /dev/other rw', + ['/workspace/config/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); + const many = Array.from( + { length: 257 }, + (_, index) => + `${index + 2} 1 9:1 / /workspace/m${index} rw - ext4 /dev/other rw`, + ).join('\n'); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + many, + ['/operator/app.yaml'], + ['/workspace'], + ), + /Too many/, + ); +}); diff --git a/packages/code/src/environment-mount.ts b/packages/code/src/environment-mount.ts new file mode 100644 index 00000000..bbc8483e --- /dev/null +++ b/packages/code/src/environment-mount.ts @@ -0,0 +1,123 @@ +import { open } from 'node:fs/promises'; +import { posix } from 'node:path'; + +interface Mount { + device: string; + root: string; + point: string; +} +const inside = (root: string, path: string): boolean => + path === root || path.startsWith(root === '/' ? '/' : `${root}/`); +const decode = (path: string): string => { + if (!path.startsWith('/') || /\\(?!040|011|012|134)/.test(path)) + throw new Error('Invalid environment mount table'); + return path.replace(/\\(040|011|012|134)/g, (_, octal: string) => + String.fromCharCode(parseInt(octal, 8)), + ); +}; + +/** Compare filesystem coordinates, not mount aliases. Include mounted descendants of each grant. */ +export function createEnvironmentMountIsolation( + table: string, +): (controls: readonly string[], roots: readonly string[]) => void { + if (Buffer.byteLength(table) > 4 * 1024 * 1024) + throw new Error('Environment mount table exceeds limit'); + const mounts: Mount[] = table + .trimEnd() + .split('\n') + .map(line => { + const fields = line.split(' '); + const separator = fields.indexOf('-', 6); + if ( + separator < 6 || + fields.length !== separator + 4 || + !/^\d+:\d+$/.test(fields[2] ?? '') + ) + throw new Error('Invalid environment mount table'); + return { + device: fields[2], + root: decode(fields[3] ?? ''), + point: decode(fields[4] ?? ''), + }; + }); + const cache = new Map(); + const coordinate = (path: string): { device: string; path: string }[] => { + const cached = cache.get(path); + if (cached) return cached; + // Include every possible backing mapping. Hidden/stacked mounts may cause + // conservative rejection but must never hide an accessible control path. + const result = mounts + .filter(mount => inside(mount.point, path)) + .map(mount => ({ + device: mount.device, + path: posix.join(mount.root, posix.relative(mount.point, path)), + })); + if (!result.length || result.length > 256) + throw new Error( + 'Environment path has an unsupported mount mapping', + ); + cache.set(path, result); + return result; + }; + return (controls, roots) => { + const points = new Set(roots); + for (const mount of mounts) + if (roots.some(root => inside(root, mount.point))) + points.add(mount.point); + if (points.size > 256) + throw new Error('Too many workspace mount boundaries'); + const exposed = [...points].flatMap(coordinate); + if (exposed.length > 1024) + throw new Error('Too many workspace mount mappings'); + for (const control of controls) { + const target = coordinate(control); + if ( + target.some(target => + exposed.some( + root => + root.device === target.device && + inside(root.path, target.path), + ), + ) + ) { + throw new Error( + 'Environment control path is writable through a workspace mount alias', + ); + } + } + }; +} + +export function assertEnvironmentMountIsolation( + table: string, + controls: readonly string[], + roots: readonly string[], +): void { + createEnvironmentMountIsolation(table)(controls, roots); +} + +export async function readEnvironmentMountTable(): Promise { + if (process.platform !== 'linux') return undefined; + const handle = await open('/proc/self/mountinfo', 'r'); + try { + const buffer = Buffer.alloc(4 * 1024 * 1024 + 1); + let length = 0; + while (length < buffer.length) { + const result = await handle.read( + buffer, + length, + buffer.length - length, + null, + ); + if (!result.bytesRead) break; + length += result.bytesRead; + } + if (length === buffer.length) + throw new Error('Environment mount table exceeds limit'); + return new TextDecoder('utf-8', { fatal: true }).decode( + buffer.subarray(0, length), + ); + } finally { + await handle.close(); + } +} diff --git a/packages/code/src/environment.test.ts b/packages/code/src/environment.test.ts new file mode 100644 index 00000000..9080f6a2 --- /dev/null +++ b/packages/code/src/environment.test.ts @@ -0,0 +1,348 @@ +import assert from 'node:assert/strict'; +import { + mkdtemp, + mkdir, + writeFile, + rm, + symlink, + link, + open, + realpath, +} from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + parseCodeEnvironment, + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, + EnvironmentWorkspaceTools, +} from './environment.js'; +import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; +import { isValidBridgeWorkspaceToolCapabilities } from './protocol.js'; +import type { WorkspaceExecuteCommandRequest } from './protocol.js'; + +test('environment YAML validates setup and rejects unsupported policy or action fields', () => { + const definition = parseCodeEnvironment( + 'name: app\nroot: ./project\nsetup:\n command: npm ci\n', + ); + assert.equal(definition.setup?.timeoutMs, 300_000); + for (const suffix of [ + 'scope: { users: [anyone] }', + 'actions: [{}]', + 'unknown: true', + 'setup: { command: npm ci, timeoutMs: 600000 }', + 'setup: { command: npm ci, timeoutMs: -1 }', + 'setup: { command: npm ci, env: { SECRET: x } }', + 'name: duplicate', + 'repo: https://token@github.com/a/b', + ]) + assert.throws(() => + parseCodeEnvironment(`name: app\nroot: ./project\n${suffix}\n`), + ); + assert.throws(() => parseCodeEnvironment('name: &id app\nroot: *id')); + assert.throws(() => parseCodeEnvironment('x'.repeat(65_537))); + for (const field of ['setup', 'actions']) { + const command = '漢'.repeat(12_000); + const suffix = + field === 'setup' + ? `setup: { command: '${command}' }` + : `actions: [{ name: test, command: '${command}' }]`; + assert.throws(() => + parseCodeEnvironment(`name: app\nroot: project\n${suffix}`), + ); + } +}); + +test('named actions use the loaded definition, reject stale revisions and preserve command restrictions', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-action-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const local = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'app', root: directory }], + }); + const executed: WorkspaceExecuteCommandRequest[] = []; + const commands = new SandboxWorkspaceTools({ + workspaceTools: local, + commandWorkspaces: ['app'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute(request) { + executed.push(request); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'app', + stdout: '', + stderr: '', + exitCode: 0, + timedOut: false, + truncated: false, + }; + }, + }, + }); + const environments = [ + { + path: '/operator/environment.yaml', + fingerprint: 'a'.repeat(64), + definition: { + name: 'app', + root: directory, + actions: [ + { name: 'test', command: 'npm test', timeoutMs: 2000 }, + ], + }, + }, + ]; + const tools = new EnvironmentWorkspaceTools(commands, environments); + assert.ok(isValidBridgeWorkspaceToolCapabilities(tools.capabilities)); + assert.deepEqual(tools.capabilities.workspaces[0].environment?.actions, [ + 'test', + ]); + assert.equal( + JSON.stringify(tools.capabilities).includes('npm test'), + false, + ); + const request: WorkspaceExecuteCommandRequest = { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'app', + command: 'untrusted placeholder', + timeoutMs: 5000, + environmentAction: { name: 'test', fingerprint: 'a'.repeat(64) }, + }; + await tools.execute(request); + assert.equal(executed[0].command, 'npm test'); + assert.equal(executed[0].timeoutMs, 2000); + assert.equal(executed[0].environmentAction, undefined); + for (const altered of [ + { ...request, workspaceId: 'other' }, + { ...request, cwd: 'nested' }, + { + ...request, + environmentAction: { name: 'test', fingerprint: 'b'.repeat(64) }, + }, + { + ...request, + environmentAction: { name: 'other', fingerprint: 'a'.repeat(64) }, + }, + ]) + await assert.rejects( + tools.execute(altered), + /unavailable or its definition changed/, + ); + await assert.rejects(commands.execute(request), /not resolved/); + const readOnly = new EnvironmentWorkspaceTools(local, environments); + assert.deepEqual( + readOnly.capabilities.workspaces[0].environment?.actions, + [], + ); + await assert.rejects(readOnly.execute(request)); + assert.equal(executed.length, 1); +}); + +test('environment roots resolve relative to the definition and fingerprints cover setup', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-definition-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: "printf first" }\n', + ); + const first = await loadCodeEnvironment(path); + assert.ok(first.definition.root.endsWith('/project')); + await assertEnvironmentDefinitionsOutsideRoots( + [first], + [{ id: 'app', root: first.definition.root }], + ); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: "printf second" }\n', + ); + assert.notEqual( + (await loadCodeEnvironment(path)).fingerprint, + first.fingerprint, + ); + await assert.rejects(() => + assertEnvironmentDefinitionsOutsideRoots( + [first], + [ + { + id: 'parent', + root: first.definition.root.slice(0, -'/project'.length), + }, + ], + ), + ); + await symlink(path, join(directory, 'project', 'alias.yaml')); + const alias = await loadCodeEnvironment( + join(directory, 'project', 'alias.yaml'), + ); + await assert.rejects(() => + assertEnvironmentDefinitionsOutsideRoots( + [alias], + [{ id: 'app', root: first.definition.root }], + ), + ); + assert.equal( + (await loadCodeEnvironment(join(directory, 'project', 'alias.yaml'))) + .path, + first.path, + ); +}); + +test('rejects a trusted definition with an in-workspace hard link', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-hardlink-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile(path, 'name: app\nroot: project\n'); + await link(path, join(directory, 'project', 'alias.yaml')); + await assert.rejects(loadCodeEnvironment(path), /one link/); +}); + +test('rejects nested aliases passing through a workspace-controlled link', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-nested-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + const trusted = join(directory, 'trusted'); + await mkdir(root); + await mkdir(trusted); + await writeFile( + join(trusted, 'environment.yaml'), + `name: app\nroot: ${root}\n`, + ); + await symlink(trusted, join(root, 'pivot')); + await symlink(join(root, 'pivot'), join(directory, 'alias')); + const loaded = await loadCodeEnvironment( + join(directory, 'alias', 'environment.yaml'), + ); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [{ id: 'app', root }], + ), + /outside|mount alias/, + ); +}); + +test('reads complete definitions despite short filesystem reads', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-short-read-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: echo prepared }\n', + ); + const sample = await open(path); + const prototype = Object.getPrototypeOf(sample); + const read = prototype.read; + await sample.close(); + t.mock.method( + prototype, + 'read', + function ( + this: unknown, + buffer: Buffer, + offset: number, + length: number, + position: number, + ) { + return read.call( + this, + buffer, + offset, + Math.min(length, 7), + position, + ); + }, + ); + assert.equal( + (await loadCodeEnvironment(path)).definition.setup?.command, + 'echo prepared', + ); +}); + +test('rejects a root routed through another workspace and malformed UTF-8', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-root-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const rootA = join(directory, 'a'); + const rootB = join(directory, 'b'); + await mkdir(rootA); + await mkdir(rootB); + await symlink(rootA, join(rootB, 'pivot')); + const path = join(directory, 'environment.yaml'); + await writeFile(path, `name: a\nroot: ${join(rootB, 'pivot')}\n`); + const loaded = await loadCodeEnvironment(path); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [ + { id: 'a', root: rootA }, + { id: 'b', root: rootB }, + ], + ), + /root traversal|mount alias/, + ); + await symlink(rootA, join(rootA, 'self-pivot')); + await writeFile(path, `name: a\nroot: ${join(rootA, 'self-pivot')}\n`); + const selfControlled = await loadCodeEnvironment(path); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [selfControlled], + [{ id: 'a', root: rootA }], + ), + /root traversal|mount alias/, + ); + await writeFile( + path, + Buffer.concat([ + Buffer.from(`name: a\nroot: ${rootA}\nsetup: { command: echo `), + Buffer.from([0xff]), + Buffer.from(' }'), + ]), + ); + await assert.rejects(loadCodeEnvironment(path), /encoded data/); +}); + +test('rejects a FIFO definition without waiting for a writer', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-fifo-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'environment.yaml'); + execFileSync('mkfifo', ['-m', '600', path], { timeout: 2000 }); + await assert.rejects(loadCodeEnvironment(path), /Invalid environment file/); +}); + +test('rejects a filesystem-identical control directory despite a different root path', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-identity-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const trusted = join(directory, 'trusted'); + const alias = join(directory, 'alias'); + const project = join(directory, 'project'); + await mkdir(trusted); + await mkdir(project); + await symlink(trusted, alias); + const path = join(trusted, 'environment.yaml'); + await writeFile(path, `name: app\nroot: ${project}\n`); + const loaded = await loadCodeEnvironment(path); + // Unlike realpath-based containment, inode comparison also covers bind-mount aliases. + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [{ id: 'alias', root: alias }], + ), + /outside|mount alias/, + ); +}); diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts new file mode 100644 index 00000000..700fc0dc --- /dev/null +++ b/packages/code/src/environment.ts @@ -0,0 +1,399 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { open, realpath, stat } from 'node:fs/promises'; +import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; +import { parseDocument } from 'yaml'; +import { + assertPrivateStorageAcl, + assertPrivateStorageAncestors, +} from './private-storage.js'; +import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, +} from './protocol.js'; +import type { LocalWorkspaceConfig } from './workspace.js'; +import { WorkspaceToolError } from './workspace.js'; +import { + createEnvironmentMountIsolation, + readEnvironmentMountTable, +} from './environment-mount.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; +import type { WorkspaceToolRequest, WorkspaceToolResult } from './protocol.js'; + +export interface CodeEnvironmentDefinition { + name: string; + root: string; + repo?: string; + ref?: string; + setup?: { command: string; timeoutMs: number }; + actions?: { name: string; command: string; timeoutMs: number }[]; +} + +export interface LoadedCodeEnvironment { + path: string; + sourceParents?: string[]; + rootPaths?: string[]; + definition: CodeEnvironmentDefinition; + fingerprint: string; +} + +function record(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function text(value: unknown, max: number): value is string { + return ( + typeof value === 'string' && + value.trim().length > 0 && + value.length <= max && + !value.includes('\0') + ); +} + +export function parseCodeEnvironment( + source: string, +): CodeEnvironmentDefinition { + if (Buffer.byteLength(source) > 65_536) + throw new Error('Environment file exceeds 64 KiB'); + const document = parseDocument(source, { + schema: 'core', + uniqueKeys: true, + }); + if (document.errors.length || document.warnings.length) { + throw new Error('Invalid environment YAML'); + } + const value: unknown = document.toJS({ maxAliasCount: 0 }); + if ( + !record(value) || + Object.keys(value).some( + key => + !['name', 'root', 'repo', 'ref', 'setup', 'actions'].includes( + key, + ), + ) || + !text(value.name, 64) || + !/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(value.name) || + !text(value.root, 4096) || + (value.repo !== undefined && + (!text(value.repo, 256) || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(value.repo))) || + (value.ref !== undefined && + (!text(value.ref, 256) || /[\r\n]/.test(value.ref))) + ) { + throw new Error( + 'Invalid environment definition: expected name, root, optional repo, ref and setup', + ); + } + let setup: CodeEnvironmentDefinition['setup']; + if (value.setup !== undefined) { + if ( + !record(value.setup) || + Object.keys(value.setup).some( + key => !['command', 'timeoutMs'].includes(key), + ) || + !text(value.setup.command, 16_384) || + Buffer.byteLength(value.setup.command) > + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) { + throw new Error('Invalid environment setup'); + } + const timeoutMs = + value.setup.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS; + if ( + typeof timeoutMs !== 'number' || + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS + ) { + throw new Error( + `Environment setup timeout must be between 1 and ${BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS} ms`, + ); + } + setup = { command: value.setup.command, timeoutMs }; + } + let actions: CodeEnvironmentDefinition['actions']; + if (value.actions !== undefined) { + if (!Array.isArray(value.actions) || value.actions.length > 32) + throw new Error('Invalid environment actions'); + const names = new Set(); + actions = value.actions.map((action: unknown) => { + if ( + !record(action) || + Object.keys(action).some( + key => !['name', 'command', 'timeoutMs'].includes(key), + ) || + !text(action.name, 64) || + !/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(action.name) || + names.has(action.name) || + !text(action.command, 16_384) || + Buffer.byteLength(action.command) > + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) + throw new Error('Invalid environment action'); + const timeoutMs = action.timeoutMs ?? 30_000; + if ( + typeof timeoutMs !== 'number' || + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS + ) + throw new Error('Invalid environment action timeout'); + names.add(action.name); + return { name: action.name, command: action.command, timeoutMs }; + }); + } + return { + name: value.name, + root: value.root, + ...(typeof value.repo === 'string' ? { repo: value.repo } : {}), + ...(typeof value.ref === 'string' ? { ref: value.ref } : {}), + ...(setup ? { setup } : {}), + ...(actions ? { actions } : {}), + }; +} + +/** Resolve actions only against the worker-owned snapshot, after normal command admission. */ +export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly environments: Map; + + constructor( + private readonly delegate: WorkspaceToolExecutor, + environments: LoadedCodeEnvironment[], + ) { + this.mutationFailuresAreAtomic = delegate.mutationFailuresAreAtomic; + this.environments = new Map( + environments.map(environment => [ + environment.definition.name, + environment, + ]), + ); + this.capabilities = { + ...delegate.capabilities, + workspaces: delegate.capabilities.workspaces.map(workspace => { + const environment = this.environments.get(workspace.id); + if (!environment) return workspace; + const operations = + workspace.operations ?? delegate.capabilities.operations; + return { + ...workspace, + environment: { + fingerprint: environment.fingerprint, + ...(environment.definition.repo + ? { repo: environment.definition.repo } + : {}), + ...(environment.definition.ref + ? { ref: environment.definition.ref } + : {}), + actions: operations.includes('execute_command') + ? (environment.definition.actions ?? []).map( + action => action.name, + ) + : [], + }, + }; + }), + }; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise { + if ( + request.operation !== 'execute_command' || + !request.environmentAction + ) { + return this.delegate.execute(request, signal); + } + const environment = this.environments.get(request.workspaceId); + const action = environment?.definition.actions?.find( + action => action.name === request.environmentAction?.name, + ); + if ( + !environment || + environment.fingerprint !== request.environmentAction.fingerprint || + !action || + (request.cwd !== undefined && request.cwd !== '.') + ) { + throw new WorkspaceToolError( + 'Environment action is unavailable or its definition changed', + 'INVALID_REQUEST', + ); + } + const { environmentAction: _action, ...commandRequest } = request; + return this.delegate.execute( + { + ...commandRequest, + command: action.command, + timeoutMs: Math.min( + request.timeoutMs ?? action.timeoutMs, + action.timeoutMs, + ), + cwd: '.', + }, + signal, + ); + } +} + +export async function loadCodeEnvironment( + path: string, +): Promise { + const sourcePath = resolve(path); + const sourceParents = await assertPrivateStorageAncestors(sourcePath); + const canonicalPath = await realpath(sourcePath); + const handle = await open( + canonicalPath, + constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW, + ); + let definition: CodeEnvironmentDefinition; + try { + const metadata = await handle.stat(); + const self = process.getuid?.(); + if ( + metadata.nlink !== 1 || + (metadata.mode & 0o022) !== 0 || + (self !== undefined && metadata.uid !== self && metadata.uid !== 0) + ) { + throw new Error( + 'Environment definitions must have one link, a trusted owner and no group or other write permissions', + ); + } + await assertPrivateStorageAcl(handle, canonicalPath); + if (!metadata.isFile() || metadata.size > 65_536) + throw new Error('Invalid environment file'); + const buffer = Buffer.alloc(65_537); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const result = await handle.read( + buffer, + bytesRead, + buffer.length - bytesRead, + bytesRead, + ); + if (result.bytesRead === 0) break; + bytesRead += result.bytesRead; + } + const after = await handle.stat(); + if ( + bytesRead !== metadata.size || + after.size !== metadata.size || + after.mtimeMs !== metadata.mtimeMs || + after.ctimeMs !== metadata.ctimeMs + ) { + throw new Error('Environment definition changed while reading'); + } + definition = parseCodeEnvironment( + new TextDecoder('utf-8', { fatal: true }).decode( + buffer.subarray(0, bytesRead), + ), + ); + } finally { + await handle.close(); + } + const rootPath = resolve(dirname(canonicalPath), definition.root); + const rootPaths = await assertPrivateStorageAncestors(rootPath); + const root = await realpath(rootPath); + if (!(await stat(root)).isDirectory()) + throw new Error('Environment root must be a directory'); + definition = { ...definition, root }; + return { + path: canonicalPath, + sourceParents, + rootPaths, + definition, + fingerprint: createHash('sha256') + .update(JSON.stringify(definition)) + .digest('hex'), + }; +} + +/** A workspace must never be able to rewrite a definition used on the next startup. */ +export async function assertEnvironmentDefinitionsOutsideRoots( + environments: readonly LoadedCodeEnvironment[], + roots: readonly LocalWorkspaceConfig[], +): Promise { + if (!environments.length) return; + const mountTable = await readEnvironmentMountTable(); + if (mountTable !== undefined) { + const assertMountIsolation = + createEnvironmentMountIsolation(mountTable); + assertMountIsolation( + environments.flatMap(environment => [ + environment.path, + ...(environment.sourceParents ?? []), + ]), + roots.map(root => root.root), + ); + for (const environment of environments) { + assertMountIsolation( + environment.rootPaths ?? [], + roots + .filter(root => root.id !== environment.definition.name) + .map(root => root.root), + ); + assertMountIsolation( + (environment.rootPaths ?? []).filter( + path => path !== environment.definition.root, + ), + roots + .filter(root => root.id === environment.definition.name) + .map(root => root.root), + ); + } + } + const identities = new Map>(); + const identity = (path: string): Promise => { + let result = identities.get(path); + if (!result) { + result = stat(path).then( + metadata => `${metadata.dev}:${metadata.ino}`, + ); + identities.set(path, result); + } + return result; + }; + for (const environment of environments) { + for (const root of roots) { + const rootIdentity = await identity(root.root); + // No granted workspace may control how this root resolves on restart. + { + for (const component of environment.rootPaths ?? []) { + const path = relative(root.root, component); + if (path === '' && root.id === environment.definition.name) + continue; + if ( + (await identity(component)) === rootIdentity || + path === '' || + (!isAbsolute(path) && + path !== '..' && + !path.startsWith(`..${sep}`)) + ) { + throw new Error( + 'Environment root traversal crosses a workspace-controlled component', + ); + } + } + } + for (const controlPath of [ + environment.path, + ...(environment.sourceParents ?? []), + ]) { + const path = relative(root.root, controlPath); + if ( + (await identity(controlPath)) === rootIdentity || + path === '' || + (!isAbsolute(path) && + path !== '..' && + !path.startsWith(`..${sep}`)) + ) { + throw new Error( + 'Environment definitions must be outside every registered workspace root', + ); + } + } + } + } +} diff --git a/packages/code/src/private-storage.ts b/packages/code/src/private-storage.ts index 25f23a2a..d896de33 100644 --- a/packages/code/src/private-storage.ts +++ b/packages/code/src/private-storage.ts @@ -48,12 +48,15 @@ export async function removePrivateStorageAcl( * links one component at a time so even intermediate link targets are checked. * Other local accounts cannot replace a checked entry: its parent is either * non-writable or sticky and the entry belongs to this account or root. + * Returns every traversed entry, including intermediate symlinks, so callers + * can also enforce containment restrictions without resolving those entries away. */ export async function assertPrivateStorageAncestors( path: string, allowMissing = false, -): Promise { +): Promise { assertPrivateStorageSupported(); + const visited: string[] = []; const uid = process.getuid!(); let current = '/'; const pending = (isAbsolute(path) ? path : `${process.cwd()}/${path}`).split('/'); @@ -63,7 +66,8 @@ export async function assertPrivateStorageAncestors( if (allowMissing && error.code === 'ENOENT') return undefined; throw error; }); - if (metadata === undefined) return; + if (metadata === undefined) return visited; + visited.push(current); if (metadata.uid !== uid && metadata.uid !== 0) { throw new BridgeProtocolError( `${current} is owned by another account (uid ${metadata.uid}), ` + @@ -102,7 +106,7 @@ export async function assertPrivateStorageAncestors( } let next = pending.shift(); while (next === '' || next === '.') next = pending.shift(); - if (next === undefined) return; + if (next === undefined) return visited; current = next === '..' ? dirname(current) : `${current === '/' ? '' : current}/${next}`; } } diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index b92d21ac..9199fcf0 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -21,7 +21,8 @@ export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES = 100; -export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY = 4; export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; @@ -41,26 +42,119 @@ export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; * locally instead of discovering the mismatch only after mutating a workspace. */ const BRIDGE_ARTIFACT_EXTENSIONS = new Set([ - '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', - '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', - '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', - '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', - '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', - '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', - '.xml', '.yaml', '.yml', - '.ics', '.ical', '.ifb', '.icalendar', - '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', - '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', - '.odt', '.ods', '.odp', '.rtf', - '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', - '.tif', '.tiff', '.webp', - '.eot', '.ttf', '.woff', '.woff2', - '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', - '.tf', '.tfvars', '.tfstate', '.hcl', - '.dockerfile', '.Dockerfile', '.dockerignore', - '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', - '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', - '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', + '.c', + '.cs', + '.cpp', + '.go', + '.java', + '.js', + '.kt', + '.kts', + '.lua', + '.php', + '.pl', + '.ps1', + '.py', + '.r', + '.rb', + '.rs', + '.scala', + '.sh', + '.sql', + '.swift', + '.ts', + '.jsx', + '.tsx', + '.groovy', + '.css', + '.htm', + '.html', + '.less', + '.sass', + '.scss', + '.svg', + '.svelte', + '.vue', + '.adoc', + '.asciidoc', + '.md', + '.rst', + '.tex', + '.txt', + '.wiki', + '.csv', + '.json', + '.bson', + '.json5', + '.jsonl', + '.parquet', + '.tsv', + '.xml', + '.yaml', + '.yml', + '.ics', + '.ical', + '.ifb', + '.icalendar', + '.conf', + '.env', + '.gitignore', + '.ini', + '.properties', + '.toml', + '.doc', + '.docx', + '.pdf', + '.ppt', + '.pptx', + '.xls', + '.xlsx', + '.odt', + '.ods', + '.odp', + '.rtf', + '.avif', + '.bmp', + '.gif', + '.ico', + '.jpeg', + '.jpg', + '.png', + '.tif', + '.tiff', + '.webp', + '.eot', + '.ttf', + '.woff', + '.woff2', + '.7z', + '.bz2', + '.gz', + '.gzip', + '.rar', + '.tar', + '.zip', + '.tf', + '.tfvars', + '.tfstate', + '.hcl', + '.dockerfile', + '.Dockerfile', + '.dockerignore', + '.helmignore', + '.helmfile', + '.jenkinsfile', + '.vagrantfile', + '.eslintrc', + '.prettierrc', + '.editorconfig', + '.nomad', + '.bat', + '.cmd', + '.deb', + '.log', + '.rpm', + '.vbs', ]); function portableBasename(name: string): string { @@ -94,7 +188,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.css': 'text/css', '.csv': 'text/csv', '.doc': 'application/msword', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.docx': + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', '.gif': 'image/gif', '.gz': 'application/gzip', '.gzip': 'application/gzip', @@ -123,7 +218,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.pdf': 'application/pdf', '.png': 'image/png', '.ppt': 'application/vnd.ms-powerpoint', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.pptx': + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', '.py': 'text/x-python', '.rst': 'text/x-rst', '.rtf': 'application/rtf', @@ -143,7 +239,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.woff': 'font/woff', '.woff2': 'font/woff2', '.xls': 'application/vnd.ms-excel', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.xlsx': + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.xml': 'application/xml', '.yaml': 'application/yaml', '.yml': 'application/yaml', @@ -180,6 +277,12 @@ export interface BridgeWorkspaceDescriptor { name?: string; /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ operations?: BridgeWorkspaceToolOperation[]; + environment?: { + fingerprint: string; + repo?: string; + ref?: string; + actions: string[]; + }; } export interface BridgeWorkspaceToolCapabilities { @@ -291,7 +394,8 @@ interface WorkspaceEditFileRequestBase { expectedBaseSha256?: string; } -export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequestBase { +export interface WorkspaceSingleEditFileRequest + extends WorkspaceEditFileRequestBase { /** Legacy single-edit form. */ oldText: string; /** Legacy single-edit form. */ @@ -299,7 +403,8 @@ export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequest edits?: never; } -export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestBase { +export interface WorkspaceBatchEditFileRequest + extends WorkspaceEditFileRequestBase { /** Ordered exact replacements applied atomically as one file mutation. */ edits: WorkspaceTextEdit[]; oldText?: never; @@ -307,7 +412,8 @@ export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestB } export type WorkspaceEditFileRequest = - WorkspaceSingleEditFileRequest | WorkspaceBatchEditFileRequest; + | WorkspaceSingleEditFileRequest + | WorkspaceBatchEditFileRequest; export interface WorkspaceTextEdit { oldText: string; @@ -330,20 +436,23 @@ interface WorkspacePreviewEditRequestBase { path: string; } -export interface WorkspaceSinglePreviewEditRequest extends WorkspacePreviewEditRequestBase { +export interface WorkspaceSinglePreviewEditRequest + extends WorkspacePreviewEditRequestBase { oldText: string; newText: string; edits?: never; } -export interface WorkspaceBatchPreviewEditRequest extends WorkspacePreviewEditRequestBase { +export interface WorkspaceBatchPreviewEditRequest + extends WorkspacePreviewEditRequestBase { edits: WorkspaceTextEdit[]; oldText?: never; newText?: never; } export type WorkspacePreviewEditRequest = - WorkspaceSinglePreviewEditRequest | WorkspaceBatchPreviewEditRequest; + | WorkspaceSinglePreviewEditRequest + | WorkspaceBatchPreviewEditRequest; export interface WorkspacePreviewEditResult { protocolVersion: BridgeProtocolVersion; @@ -368,6 +477,7 @@ export interface WorkspaceExecuteCommandRequest { timeoutMs?: number; /** Aggregate UTF-8 stdout and stderr budget. */ maxOutputBytes?: number; + environmentAction?: { name: string; fingerprint: string }; } export interface WorkspaceExecuteCommandResult { @@ -452,6 +562,7 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ ]); const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText']); const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ + 'environmentAction', 'protocolVersion', 'operation', 'workspaceId', @@ -720,7 +831,8 @@ export function isWorkspaceToolErrorCode( } export type BridgeSettlement = - BridgeFulfilledSettlement | BridgeRejectedSettlement; + | BridgeFulfilledSettlement + | BridgeRejectedSettlement; export interface BridgeSettlementResponse { protocolVersion: BridgeProtocolVersion; @@ -806,7 +918,8 @@ export function isBridgeWorkspaceProgrammaticRequest( (body.transfer_timeout_ms !== undefined && (!Number.isSafeInteger(body.transfer_timeout_ms) || Number(body.transfer_timeout_ms) < 1 || - Number(body.transfer_timeout_ms) > BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || + Number(body.transfer_timeout_ms) > + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || (body.run_timeout !== undefined && (!Number.isSafeInteger(body.run_timeout) || Number(body.run_timeout) < 1 || @@ -826,7 +939,8 @@ export function isBridgeWorkspaceProgrammaticRequest( if ( !isSafePortableRelativePath(file.name) || file.name === '.' || - portableBasename(file.name).toLowerCase() === '_ptc_pending_result.json' || + portableBasename(file.name).toLowerCase() === + '_ptc_pending_result.json' || normalizePortableRelativePath(file.name) !== file.name || names.has(file.name) ) { @@ -875,7 +989,9 @@ export function isBridgeWorkspaceProgrammaticRequest( const segments = name.split('/'); let ancestor = ''; for (let index = 0; index < segments.length - 1; index += 1) { - ancestor = ancestor ? `${ancestor}/${segments[index]}` : segments[index]!; + ancestor = ancestor + ? `${ancestor}/${segments[index]}` + : segments[index]!; if (names.has(ancestor)) return false; } } @@ -1105,6 +1221,22 @@ export function isWorkspaceToolRequest( } if (request.operation === 'execute_command') { return ( + (request.environmentAction === undefined || + (typeof request.environmentAction === 'object' && + request.environmentAction !== null && + Object.keys(request.environmentAction).length === 2 && + typeof (request.environmentAction as { name?: unknown }) + .name === 'string' && + /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test( + (request.environmentAction as { name: string }).name, + ) && + typeof ( + request.environmentAction as { fingerprint?: unknown } + ).fingerprint === 'string' && + /^[a-f0-9]{64}$/.test( + (request.environmentAction as { fingerprint: string }) + .fingerprint, + ))) && hasOnlyKeys(request, WORKSPACE_COMMAND_REQUEST_KEYS) && typeof request.command === 'string' && request.command.trim().length > 0 && @@ -1438,11 +1570,17 @@ export function isValidBridgeWorkspaceToolCapabilities( const descriptor = workspace as Record; if ( Object.keys(descriptor).some( - key => key !== 'id' && key !== 'name' && key !== 'operations', + key => + key !== 'id' && + key !== 'name' && + key !== 'operations' && + key !== 'environment', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || + (descriptor.environment !== undefined && + !isValidCodeEnvironmentDescriptor(descriptor.environment)) || (descriptor.name !== undefined && (typeof descriptor.name !== 'string' || descriptor.name.trim().length === 0 || @@ -1469,6 +1607,37 @@ export function isValidBridgeWorkspaceToolCapabilities( }); } +export function isValidCodeEnvironmentDescriptor( + value: unknown, +): value is NonNullable { + if (typeof value !== 'object' || value === null) return false; + const environment = value as Record; + return ( + Object.keys(environment).every(key => + ['fingerprint', 'repo', 'ref', 'actions'].includes(key), + ) && + typeof environment.fingerprint === 'string' && + /^[a-f0-9]{64}$/.test(environment.fingerprint) && + (environment.repo === undefined || + (typeof environment.repo === 'string' && + environment.repo.length <= 256 && + /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(environment.repo))) && + (environment.ref === undefined || + (typeof environment.ref === 'string' && + environment.ref.trim().length > 0 && + environment.ref.length <= 256 && + !/[\0\r\n]/.test(environment.ref))) && + Array.isArray(environment.actions) && + environment.actions.length <= 32 && + environment.actions.every( + name => + typeof name === 'string' && + /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(name), + ) && + new Set(environment.actions).size === environment.actions.length + ); +} + export function isValidBridgeWorkerCapabilities( value: unknown, ): value is BridgeWorkerCapabilities { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index ffe1b350..86e85872 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -192,6 +192,13 @@ function workspaceCapabilitiesMatch( (workspace, index) => workspace.id === executor.workspaces[index]?.id && workspace.name === executor.workspaces[index]?.name && + workspace.environment?.fingerprint === executor.workspaces[index]?.environment?.fingerprint && + workspace.environment?.repo === executor.workspaces[index]?.environment?.repo && + workspace.environment?.ref === executor.workspaces[index]?.environment?.ref && + workspace.environment?.actions.length === executor.workspaces[index]?.environment?.actions.length && + (workspace.environment?.actions.every( + (action, actionIndex) => action === executor.workspaces[index]?.environment?.actions[actionIndex], + ) ?? executor.workspaces[index]?.environment == null) && workspace.operations?.length === executor.workspaces[index]?.operations?.length && (workspace.operations?.every( @@ -236,7 +243,9 @@ function registrationCompatibleCapabilities( return []; } const { operations: _operations, ...compatibleWorkspace } = workspace; - return [compatibleWorkspace]; + return [{ ...compatibleWorkspace, ...(workspace.environment ? { + environment: { ...workspace.environment, actions: [] }, + } : {}) }]; }); if (workspaces.length === 0) { const { workspaceTools: _workspaceTools, ...compatible } = capabilities; @@ -312,13 +321,17 @@ function supportedWorkspaceCapabilities( editOperations.has(operation), ); const workspaces = desired.workspaces.flatMap((workspace) => { - if (workspace.operations == null) return [workspace]; - const workspaceOperations = workspace.operations.filter((operation) => + const workspaceOperations = (workspace.operations ?? operations).filter((operation) => operations.includes(operation), ); return workspaceOperations.length === 0 ? [] - : [{ ...workspace, operations: workspaceOperations }]; + : [{ ...workspace, + ...(workspace.operations ? { operations: workspaceOperations } : {}), + ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { + environment: { ...workspace.environment, actions: [] }, + } : {}), + }]; }); if (workspaces.length === 0) return undefined; const editFileFeatures = desired.editFileFeatures?.filter((feature) => diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index f6205cd4..d97735dc 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -7,6 +7,30 @@ import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('worker clears named actions when command execution is not negotiated', async () => { + const workspaceTools = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary', environment: { fingerprint: 'a'.repeat(64), actions: ['test'] } }], + }; + const registrations: Array = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], workspaceTools }, + workspaceMutationQuarantine: mutationQuarantine(), + workspaceTools: { capabilities: workspaceTools, async execute() { throw new Error('not executed'); } }, + fetchImpl: async (_input, init) => { + registrations.push(JSON.parse(String(init?.body)).capabilities.workspaceTools); + return Response.json({ protocolVersion: 1, workerId: 'vm-1', incarnationId, + registeredAt: new Date().toISOString(), leaseTtlMs: 60000, supportedWorkspaceToolOperations: ['read_file'] }); + }, + }); + await worker.register(); + assert.ok(registrations.length > 0); + for (const registration of registrations) assert.deepEqual(registration.workspaces[0].environment.actions, []); +}); + const listWorkspaceCapabilities = { protocolVersion: 1 as const, operations: [ @@ -2422,6 +2446,32 @@ test('worker refuses to advertise workspace tools without a matching executor', ); }); +test('worker refuses environment metadata that differs from its executor', () => { + const environment = { fingerprint: 'a'.repeat(64), repo: 'owner/repo', ref: 'main', actions: [] as string[] }; + const workspaceTools = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', environment }], + }; + for (const changed of [ + undefined, + { ...environment, fingerprint: 'b'.repeat(64) }, + { ...environment, repo: 'other/repo' }, + { ...environment, ref: 'other' }, + { ...environment, actions: ['test'] }, + ]) { + assert.throws(() => new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], workspaceTools }, + workspaceTools: { + capabilities: { ...workspaceTools, workspaces: [{ id: 'primary', ...(changed ? { environment: changed } : {}) }] }, + async execute() { throw new Error('not executed'); }, + }, + }), /workspace tool capabilities require a matching executor/i); + } +}); + test('worker requires durable quarantine before advertising command execution', () => { const workspaceCapabilities = { protocolVersion: 1 as const, diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index dfda49fc..91e89f54 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1693,6 +1693,9 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { if (request.operation !== 'execute_command') { return this.options.workspaceTools.execute(request, signal); } + if (request.environmentAction) { + throw new WorkspaceToolError('Environment action was not resolved by this worker', 'INVALID_REQUEST'); + } if (!this.commandWorkspaces.has(request.workspaceId)) { throw new WorkspaceToolError( 'Command execution is disabled for this workspace', From f2dcb93b78578fe0fe5eeb7cf1b6a965a5d036f1 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 20:01:54 -0400 Subject: [PATCH 24/75] fix: Allow Trusted Own-Root Environment Symlinks (#212) * fix: Allow Trusted Own-Root Environment Symlinks * fix: Check Alias Parent Ownership by Filesystem Identity * fix: Enforce Parent Ownership Across Every Environment Path --- .github/workflows/ci.yml | 2 + packages/code/src/environment.test.ts | 62 +++++++++++++++++++++++++++ packages/code/src/environment.ts | 28 +++++++++++- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f041e2d6..bbf9e302 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,6 +194,8 @@ jobs: node-version: 24.16.0 - run: npm ci - run: npm run build + - name: Native environment containment tests + run: node --test dist/environment.test.js - name: Native ACL and credential lifecycle tests run: node --test dist/macos-storage.test.js dist/private-storage.test.js dist/storage.test.js dist/github.test.js diff --git a/packages/code/src/environment.test.ts b/packages/code/src/environment.test.ts index 9080f6a2..a8b5b036 100644 --- a/packages/code/src/environment.test.ts +++ b/packages/code/src/environment.test.ts @@ -193,6 +193,68 @@ test('environment roots resolve relative to the definition and fingerprints cove ); }); +test('accepts an own root through trusted external symlinks without allowing other root identities', async t => { + const directory = await realpath(await mkdtemp(join(tmpdir(), 'code-env-own-alias-'))); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + await mkdir(root); + const alias = join(directory, 'alias'); + await symlink(root, alias); + await symlink(alias, join(directory, 'nested-alias')); + const path = join(directory, 'environment.yaml'); + for (const selected of [alias, join(directory, 'nested-alias')]) { + await writeFile(path, `name: app\nroot: ${selected}\n`); + const loaded = await loadCodeEnvironment(path); + assert.equal(loaded.definition.root, root); + await assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'other', root }]), + /root traversal|mount alias/, + ); + } +}); + +test('rejects own-root links hidden by parent aliases or filesystem casing', async t => { + const directory = await realpath(await mkdtemp(join(tmpdir(), 'code-env-parent-alias-'))); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'Project'); + await mkdir(root); + await symlink(root, join(root, 'self')); + const outside = join(directory, 'outside'); + await mkdir(outside); + await symlink(root, join(outside, 'back')); + await symlink(outside, join(root, 'pivot')); + const alias = join(directory, 'parent-alias'); + await symlink(root, alias); + const path = join(directory, 'environment.yaml'); + const selectedRoots = [join(alias, 'self'), join(alias, 'pivot', 'back')]; + try { + if (await realpath(join(directory, 'project')) === root) { + selectedRoots.push(join(directory, 'project', 'self')); + selectedRoots.push(join(directory, 'project', 'pivot', 'back')); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + for (const selected of selectedRoots) { + await writeFile(path, `name: app\nroot: ${selected}\n`); + const loaded = await loadCodeEnvironment(path); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]), + /root traversal|mount alias/, + ); + } + const definition = join(outside, 'environment.yaml'); + await writeFile(definition, `name: app\nroot: ${root}\n`); + for (const selected of selectedRoots.filter(path => path.endsWith('/back'))) { + const loaded = await loadCodeEnvironment(selected.replace(/back$/, 'environment.yaml')); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]), + /outside|mount alias/, + ); + } +}); + test('rejects a trusted definition with an in-workspace hard link', async t => { const directory = await mkdtemp(join(tmpdir(), 'code-env-hardlink-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 700fc0dc..c2b2cf1f 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -355,6 +355,25 @@ export async function assertEnvironmentDefinitionsOutsideRoots( } return result; }; + // The entry's parent, not its symlink target, determines who can replace it. + // Compare ancestor identities so casing and directory aliases cannot make a + // workspace-controlled entry look external on case-insensitive filesystems. + const canonicalParents = new Map>(); + const controlsEntry = async (component: string, rootIdentity: string): Promise => { + const directory = dirname(component); + let canonical = canonicalParents.get(directory); + if (!canonical) { + canonical = realpath(directory); + canonicalParents.set(directory, canonical); + } + let parent = await canonical; + while (true) { + if ((await identity(parent)) === rootIdentity) return true; + const next = dirname(parent); + if (next === parent) return false; + parent = next; + } + }; for (const environment of environments) { for (const root of roots) { const rootIdentity = await identity(root.root); @@ -362,10 +381,14 @@ export async function assertEnvironmentDefinitionsOutsideRoots( { for (const component of environment.rootPaths ?? []) { const path = relative(root.root, component); - if (path === '' && root.id === environment.definition.name) + const sameRoot = (await identity(component)) === rootIdentity; + const controlled = await controlsEntry(component, rootIdentity); + // A trusted external alias may select its own root, but a + // link beneath that root is still writable by the workspace. + if (sameRoot && !controlled && root.id === environment.definition.name) continue; if ( - (await identity(component)) === rootIdentity || + controlled || sameRoot || path === '' || (!isAbsolute(path) && path !== '..' && @@ -383,6 +406,7 @@ export async function assertEnvironmentDefinitionsOutsideRoots( ]) { const path = relative(root.root, controlPath); if ( + (await controlsEntry(controlPath, rootIdentity)) || (await identity(controlPath)) === rootIdentity || path === '' || (!isAbsolute(path) && From 840537b9d95a3ad6c761729a198c4e9a0d020d90 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 20:02:05 -0400 Subject: [PATCH 25/75] fix: Retain Quarantine After Failed Environment Setup (#213) * fix: Retain Quarantine After Failed Environment Setup * test: Run Native Environment Setup Lifecycle in CI * fix: Document and Verify Local Setup Quarantine Recovery --- .github/workflows/ci.yml | 4 ++ packages/code/README.md | 11 ++++-- packages/code/src/cli.ts | 4 +- packages/code/src/environment-live.test.ts | 43 ++++++++++++++++++---- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbf9e302..2bdeda21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,10 @@ jobs: run: node --test dist/environment.test.js - name: Native ACL and credential lifecycle tests run: node --test dist/macos-storage.test.js dist/private-storage.test.js dist/storage.test.js dist/github.test.js + - name: Native environment setup lifecycle tests + env: + LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' + run: node --test dist/environment-live.test.js lambda-microvm-provisioning: name: Lambda MicroVM Provisioning diff --git a/packages/code/README.md b/packages/code/README.md index c3bb16d3..8b98618f 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -689,9 +689,14 @@ worker runs. This inspection happens at startup, not on the command hot path. Setup is an operator-authorized startup command under the configured native sandbox policy. It requires commands to be enabled, runs once per worker startup before registration, and must be idempotent for restarts. Its timeout is bounded to five -minutes and captured output to 8 KiB. Setup failure prevents registration. A crash -or uncertain termination retains the existing workspace quarantine marker; inspect -the workspace before clearing quarantine. No setup output is sent to the model. +minutes and captured output to 8 KiB. Setup failure prevents registration. A nonzero +exit, timeout, crash or uncertain termination retains the workspace quarantine marker; +inspect the workspace before running `librechat-code clear-workspace-quarantine +--worker-dir --workspace-id ` with the same +deployment and identity configuration. Only use the separate +`--reset-workspace-quarantine ` run option afterward if a server +fence also needs clearing. Only successful setup automatically clears its marker. +No setup output is sent to the model. Named actions are fixed commands without model-supplied substitution. The bridge advertises only their names and the definition fingerprint, never their shell source diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 5498af63..00eca576 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1012,12 +1012,12 @@ async function run( }, controller.signal, ); - await guard.clear('setup'); if (result.exitCode !== 0 || result.timedOut) { throw new Error( - `Environment ${id} setup failed; inspect the setup command before restarting`, + `Environment ${id} setup failed; inspect the workspace and use clear-workspace-quarantine with its root and workspace ID before restarting`, ); } + await guard.clear('setup'); process.stdout.write( `librechat-code: environment ${id} prepared\n`, ); diff --git a/packages/code/src/environment-live.test.ts b/packages/code/src/environment-live.test.ts index bb84c8b5..9d442982 100644 --- a/packages/code/src/environment-live.test.ts +++ b/packages/code/src/environment-live.test.ts @@ -8,13 +8,14 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; -for (const { succeeds, reset } of [ - { succeeds: true, reset: false }, - { succeeds: false, reset: false }, - { succeeds: true, reset: true }, +for (const { succeeds, reset, timesOut } of [ + { succeeds: true, reset: false, timesOut: false }, + { succeeds: false, reset: false, timesOut: false }, + { succeeds: false, reset: false, timesOut: true }, + { succeeds: true, reset: true, timesOut: false }, ]) { test( - `real CLI environment setup gates registration (success=${succeeds}, reset=${reset})`, + `real CLI environment setup gates registration (success=${succeeds}, reset=${reset}, timeout=${timesOut})`, { skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', timeout: 20_000, @@ -27,7 +28,7 @@ for (const { succeeds, reset } of [ const path = join(directory, 'environment.yaml'); await writeFile( path, - `name: project\nroot: project\nsetup:\n command: 'printf prepared > prepared.txt; exit ${succeeds ? 0 : 2}'\n timeoutMs: 5000\n`, + `name: project\nroot: project\nsetup:\n command: 'printf prepared >> prepared.txt; ${timesOut ? 'sleep 10' : `exit ${succeeds ? 0 : 2}`}'\n timeoutMs: ${timesOut ? 1000 : 5000}\n`, ); let registrations = 0; let receive: (() => void) | undefined; @@ -38,6 +39,7 @@ for (const { succeeds, reset } of [ request.resume(); if (request.url?.endsWith('/register')) { registrations++; + await assert.rejects(readFile(join(directory, 'quarantine.json')), { code: 'ENOENT' }); if (reset) await assert.rejects( readFile(join(root, 'prepared.txt')), @@ -61,10 +63,15 @@ for (const { succeeds, reset } of [ }); const address = server.address(); assert.ok(address && typeof address !== 'string'); - const child = spawn( + const start = (clear = false) => spawn( process.execPath, [ fileURLToPath(new URL('./cli.js', import.meta.url)), + ...(clear ? [ + 'clear-workspace-quarantine', + '--worker-dir', root, + '--workspace-id', 'project', + ] : [ 'run', '--environment', path, @@ -72,6 +79,7 @@ for (const { succeeds, reset } of [ ...(reset ? ['--reset-workspace-quarantine', 'project'] : []), + ]), ], { env: { @@ -90,6 +98,7 @@ for (const { succeeds, reset } of [ stdio: ['ignore', 'pipe', 'pipe'], }, ); + const child = start(); const exited = once(child, 'exit'); t.after(() => child.kill('SIGKILL')); let stderr = ''; @@ -111,6 +120,26 @@ for (const { succeeds, reset } of [ assert.notEqual(code, 0); assert.match(stderr, /Environment project setup failed/); assert.equal(registrations, 0); + const marker = await readFile(join(directory, 'quarantine.json'), 'utf8'); + assert.equal(JSON.parse(marker).workspaceId, 'project'); + const before = await readFile(join(root, 'prepared.txt'), 'utf8'); + const retry = start(); + t.after(() => retry.kill('SIGKILL')); + let retryStderr = ''; + retry.stderr.on('data', chunk => { retryStderr += chunk.toString(); }); + const [retryCode] = await once(retry, 'exit'); + assert.notEqual(retryCode, 0); + assert.match(retryStderr, /quarantined/); + assert.equal(await readFile(join(root, 'prepared.txt'), 'utf8'), before); + assert.equal(await readFile(join(directory, 'quarantine.json'), 'utf8'), marker); + assert.equal(registrations, 0); + const recovery = start(true); + t.after(() => recovery.kill('SIGKILL')); + const [recoveryCode] = await once(recovery, 'exit'); + assert.equal(recoveryCode, 0); + await assert.rejects(readFile(join(directory, 'quarantine.json')), { code: 'ENOENT' }); + assert.equal(await readFile(join(root, 'prepared.txt'), 'utf8'), before); + assert.equal(registrations, 0); } }, ); From 9a3f5dbd6fedf42462239d135c2c25ec8e737628 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 15 Sep 2026 14:25:50 -0400 Subject: [PATCH 26/75] fix: Allow Lambda MicroVM metadata in hardened mode (#215) --- api/src/secure-startup.test.ts | 12 ++++++++++++ api/src/secure-startup.ts | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/api/src/secure-startup.test.ts b/api/src/secure-startup.test.ts index c1558e85..e1291712 100644 --- a/api/src/secure-startup.test.ts +++ b/api/src/secure-startup.test.ts @@ -113,6 +113,18 @@ describe('hardened sandbox-runner startup config', () => { expect(() => validateHardenedSandboxStartup()).toThrow('REDIS_HOST'); }); + test('allows Lambda MicroVM image metadata while rejecting AWS credentials', () => { + setValidHardenedConfig(); + process.env.AWS_LAMBDA_MICROVM_IMAGE_ARN = 'arn:aws:lambda:us-east-1:123456789012:microvm-image:codeapi'; + process.env.AWS_LAMBDA_MICROVM_IMAGE_NAME = 'codeapi'; + process.env.AWS_LAMBDA_MICROVM_IMAGE_VERSION = '3'; + process.env.AWS_REGION = 'us-east-1'; + expect(() => validateHardenedSandboxStartup()).not.toThrow(); + + process.env.AWS_ACCESS_KEY_ID = 'access-key'; + expect(() => validateHardenedSandboxStartup()).toThrow('AWS_ACCESS_KEY_ID'); + }); + test('rejects missing manifest verifier and wrong forwarding target', () => { setValidHardenedConfig(); config.egress_gateway_url = ''; diff --git a/api/src/secure-startup.ts b/api/src/secure-startup.ts index 04f5638a..326bf810 100644 --- a/api/src/secure-startup.ts +++ b/api/src/secure-startup.ts @@ -1,6 +1,13 @@ import { config } from './config'; import { workspaceIsolationConfigErrors } from './workspace-isolation'; +const ALLOWED_LAMBDA_MICROVM_AWS_ENV = new Set([ + 'AWS_LAMBDA_MICROVM_IMAGE_ARN', + 'AWS_LAMBDA_MICROVM_IMAGE_NAME', + 'AWS_LAMBDA_MICROVM_IMAGE_VERSION', + 'AWS_REGION', +]); + export class SandboxSecureStartupError extends Error { constructor(message: string) { super(message); @@ -57,7 +64,7 @@ function forbiddenEnvNames(): string[] { if (name === 'CODEAPI_HARDENED_SANDBOX_MODE') continue; if (name.startsWith('CODEAPI_')) forbidden.push(name); if (name.startsWith('REDIS_')) forbidden.push(name); - if (name.startsWith('AWS_')) forbidden.push(name); + if (name.startsWith('AWS_') && !ALLOWED_LAMBDA_MICROVM_AWS_ENV.has(name)) forbidden.push(name); if (name.startsWith('S3_')) forbidden.push(name); if (name.startsWith('MINIO_')) forbidden.push(name); if (/(SECRET|TOKEN|PASSWORD|PRIVATE_KEY)/.test(name)) forbidden.push(name); From c03d309429f152ae11a6d9503d3804dbaab9d228 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 00:38:02 -0400 Subject: [PATCH 27/75] docs: add self-hosted worker setup runbook (#219) --- docs/remote-bridge/README.md | 5 + docs/remote-bridge/worker-runbook.md | 527 +++++++++++++++++++++++++++ packages/code/README.md | 3 + 3 files changed, 535 insertions(+) create mode 100644 docs/remote-bridge/worker-runbook.md diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index b9de3ac2..f66499b2 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -3,6 +3,10 @@ Remote Code Bridge makes an operator-owned VM a stateful Code API execution environment without exposing that VM to inbound internet traffic. +For an end-to-end host setup, including pairing, named environments, systemd, +launchd, GitHub App credentials, upgrades, verification, and recovery, see the +[self-hosted worker runbook](./worker-runbook.md). + ```text LibreChat -> Code API -> Redis assignment ^ | @@ -200,6 +204,7 @@ Expose the Code API deployment as an environment under the Agents endpoint: endpoints: agents: statefulCodeSessions: + allowedEnvironments: [user, agent-user, conversation] environments: - id: my-vm name: My VM diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md new file mode 100644 index 00000000..7511a265 --- /dev/null +++ b/docs/remote-bridge/worker-runbook.md @@ -0,0 +1,527 @@ +# Self-hosted worker runbook + +This runbook attaches an operator-controlled laptop or VM to a LibreChat Code +API deployment. The worker makes an outbound HTTPS connection; it does not +open an inbound port. The same procedure works for one machine or many +principal-bound machines. + +The guide uses a named environment and the native Sandbox Runtime (SRT). It +covers a restricted personal-machine deployment and the `trusted-vm` preset, +where a separate VM boundary is responsible for most host isolation. + +## 1. Understand the boundaries + +Four independently managed components participate: + +1. LibreChat stores the environment record, resolves its principal, applies + administrator/user policy, and selects the worker for a conversation. +2. Code API authenticates the selection, queues and fences assignments, and + exposes the outbound bridge. +3. `@librechat/code` runs on the attached machine, owns local workspace + admission, rotates its bridge credential, and executes tools through SRT. +4. The machine owner controls the OS, workspace, network, credentials, and + service lifecycle. + +Pairing authenticates a worker. It does not make the host trustworthy, attest +the host policy, or replace tool approval. A `trusted-vm` worker is appropriate +only when the VM boundary is already operated as the security boundary. + +## 2. Configure LibreChat and Code API + +Deploy Code API's remote-bridge profile before pairing a machine. At minimum, +use paired authentication and dynamic routing so one bridge can serve many +principal-bound workers: + +```dotenv +CODEAPI_SANDBOX_BACKEND=remote-bridge +CODEAPI_EXECUTION_PROFILE=stateful +CODEAPI_RUNTIME_SESSION_MODE=affinity +CODEAPI_BRIDGE_AUTH_MODE=paired +CODEAPI_BRIDGE_DYNAMIC_WORKERS=true +CODEAPI_BRIDGE_TOKEN= +``` + +The administrator token belongs only on the Code API/control-plane host. Never +put it on an attached machine. Configure Redis and the remaining Code API +settings as described in the [Remote Code Bridge guide](./README.md). + +Expose that Code API endpoint to LibreChat and explicitly choose which +state-sharing scopes the deployment permits: + +```yaml +endpoints: + agents: + capabilities: + [ + deferred_tools, + execute_code, + file_search, + web_search, + artifacts, + subagents, + actions, + context, + skills, + memory, + ask_user_question, + tools, + chain, + ocr, + stateful_code_sessions, + ] + statefulCodeSessions: + allowedEnvironments: [user, agent-user, conversation] + environments: + - id: attached-workers + name: Attached machines + type: attached + baseURL: https://code.example.com/v1 + default: true +``` + +The example preserves LibreChat's default capabilities and adds the opt-in +`stateful_code_sessions` capability. Adjust the list to the deployment's +policy. Exactly one configured Code environment must be the default. The three +sharing scopes mean: + +- `user`: reuse an environment for the signed-in user; +- `agent-user`: reuse it for one agent and user; and +- `conversation`: isolate reuse to one conversation. + +These scopes determine session reuse. They do not weaken a worker's filesystem +root or share one user's principal-bound machine with another user. + +Enable stateful code sessions on the intended agent and select the attached +environment. Start with file writes and command execution set to `ask`; expose +`allow` or `deny` only when the deployment and machine policy permit them. +Worker capabilities are a ceiling: a conversation setting cannot enable a +command or write that the worker did not advertise. + +## 3. Roll out compatible consumers first + +Before enabling `--environment` on a worker: + +1. Deploy a LibreChat version that accepts named environment descriptors. +2. Deploy the matching Code API API and queue-worker processes. +3. Update `@librechat/code` on the attached machine. +4. Only then restart the worker with `--environment`. + +An old worker remains compatible with new consumers until the opt-in flag is +used. An old strict consumer can reject a new worker's environment metadata. +During a rolling deployment, update every API/queue replica before changing +workers. + +Record the exact source commit or package version at every tier. Do not infer a +worker's version from the Code API server: the worker is a separate process on +a separate machine. + +## 4. Prepare the worker host + +Install: + +- Node.js 20.11 or newer (Node.js 24 is supported); +- Git; +- `bubblewrap`, `socat`, and `ripgrep` on Linux; +- Bash 5.2 or newer and `jq` when Bash Programmatic Tool Calling is enabled; + and +- GitHub CLI and Git LFS only when the workflows need them. + +For example, install the system dependencies on Ubuntu with: + +```bash +sudo apt-get update +sudo apt-get install -y bash bubblewrap git jq ripgrep socat +``` + +On macOS, install the optional PTC and GitHub tools with: + +```bash +brew install bash gh git-lfs jq ripgrep +``` + +Install Node.js through the host's managed package source or version manager. +Bun is not required by `@librechat/code`. + +Keep source, application state, environment definitions, and credentials in +separate paths. For example: + +```text +/opt/librechat-code/releases// pinned worker source/build +/srv/code-workspaces/ coding roots +/etc/librechat-code/environments/ operator-owned YAML definitions +~/.config/librechat/code/ paired identity +~/.config/librechat-code/github-app.pem optional GitHub App key +``` + +Every ancestor of a definition, identity, key, quarantine file, or workspace +root must be owned by the worker account or root. It must not be writable by +group or other users. Sticky shared directories such as `/tmp` are handled +separately, but should not hold durable configuration. + +The workspace remains writable by its owner. For a dedicated service account, +a typical root is: + +```bash +sudo install -d -o librechat-code -g librechat-code -m 0750 /srv/code-workspaces +``` + +Do not register a home directory or another root containing credentials, +shell history, SSH keys, or unrelated projects. + +## 5. Install a pinned worker + +Use a published version when available. To install from source, keep a pinned +checkout and build only the worker package: + +```bash +git clone https://github.com/LibreChat-AI/code-interpreter.git /opt/librechat-code/source +cd /opt/librechat-code/source +git fetch origin main +git checkout --detach +npm ci --prefix packages/code +npm run build --prefix packages/code +cd packages/code +sudo npm link +``` + +Confirm that `/usr/local/bin/librechat-code` resolves to the intended build. +Do not replace a running release until the new build and its native imports +have succeeded. Keeping releases in commit-named directories makes rollback a +service-path change instead of a rebuild. + +## 6. Pair the machine + +Create a pairing in LibreChat's Code environments UI when available. The +pairing must be bound to the intended deployment, tenant, user, role, or group. +The code is single-use and expires after ten minutes. + +Redeem it on the worker machine: + +```bash +librechat-code pair https://code.example.com/v1 '' \ + --worker-id code-example123 +``` + +Run pairing as the same operating-system account that will run the worker. If +the systemd service uses `User=librechat-code`, run the command as that account +or supply an explicit identity path the account can read and replace. + +The CLI generates the Ed25519 private key locally and saves the identity under +`~/.config/librechat/code/` with owner-only permissions. Do not transmit or +copy that file through chat. The bridge credential expires after fifteen +minutes, but a running worker rotates it automatically. A normal restart does +not require re-pairing. + +For a custom location, use `--identity` during pairing and set +`LIBRECHAT_CODE_IDENTITY_FILE` in the service. Keep the worker ID stable: agent +defaults and conversations refer to the environment record associated with +that identity. + +## 7. Define named environments + +Store definitions outside every workspace root. A broad, multi-project VM can +preserve an existing `primary` binding without pretending the root is one Git +repository: + +```yaml +# /etc/librechat-code/environments/primary.yaml +name: primary +root: /srv/code-workspaces +``` + +For a single project, descriptive repository metadata and fixed actions may be +useful: + +```yaml +name: app-dev +root: /srv/code-workspaces/app +repo: example/app +ref: main +setup: + command: npm ci + timeoutMs: 300000 +actions: + - name: typecheck + command: npm run typecheck + timeoutMs: 120000 + - name: test + command: npm test + timeoutMs: 300000 +``` + +Important semantics: + +- `name` is both the workspace ID and its current display name. Preserve an + existing ID such as `primary` to preserve agent/conversation bindings. +- `repo` and `ref` are labels. They do not clone, fetch, or check out anything. +- `root` must already exist. Relative roots resolve from the definition file. +- Setup runs before registration on every worker start. It must be idempotent. +- A setup failure or timeout prevents registration and leaves a durable + quarantine marker for operator inspection. +- Actions are fixed operator commands. The model selects only the action name + and fingerprint; it cannot inject arguments, a command, or a working + directory. +- Actions still pass through LibreChat approval and worker command policy. +- Up to 32 roots may be declared, and they must not overlap. A broad parent + environment cannot coexist with child project environments. + +Definitions contain policy rather than secrets. A root-owned file may be +readable by the service account, but must not be group/other writable. For +example: + +```bash +sudo install -d -o root -g librechat-code -m 0750 /etc/librechat-code/environments +sudo install -o root -g librechat-code -m 0640 primary.yaml \ + /etc/librechat-code/environments/primary.yaml +``` + +Do not combine `--environment` with `--worker-dir`, `--default-workspace`, +`--workspace`, `--workspace-id`, or `--workspace-name`. Remove the equivalent +`LIBRECHAT_CODE_WORKER_DIR`, `LIBRECHAT_CODE_WORKSPACE_ID`, and +`LIBRECHAT_CODE_WORKSPACE_NAME` settings too. + +## 8. Choose a command policy + +For a personal machine, use the default `restricted` policy and explicitly +allow only required network destinations. + +For a separately secured VM whose outer boundary is managed by the operator, +the worker may use: + +```text +--allow-workspace-writes +--allow-workspace-commands +--command-policy-preset trusted-vm +``` + +`trusted-vm` is a policy preset, not an unsandboxed execution mode. SRT still +protects the bridge identity, GitHub credentials, worker configuration, and +control sockets. The preset deliberately permits broader workspace and network +behavior because the VM owner accepts responsibility for the host boundary. + +LibreChat's tool approval remains independent. Enabling commands on a worker +does not authorize a user or agent to bypass `ask` or `deny` policy. + +## 9. Optionally configure a GitHub App + +Prefer a GitHub App over a personal token. Install it only on repositories the +agent may use and grant the minimum permissions its workflows require. Git +clone/fetch/push generally needs repository Contents access; API-based pull +request workflows also need Pull requests access. + +Store the downloaded private key outside every workspace. Unlike an +environment definition, the key must have no group or other access and must be +readable by the service account: + +```bash +install -d -m 0700 ~/.config/librechat-code +install -m 0600 app.private-key.pem ~/.config/librechat-code/github-app.pem +``` + +Configure the worker, preferably in a separate service drop-in: + +```ini +[Service] +Environment=LIBRECHAT_CODE_GITHUB_APP_ID=12345 +Environment=LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 +Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/librechat-code/github-app.pem +``` + +The trusted worker mints short-lived installation tokens. Sandboxed commands +receive masked Git/`gh` credentials only for the configured GitHub hosts; the +token is not written to the repository, remote URL, or Git configuration. + +## 10. Run under systemd + +Use a dedicated service account in a multi-user deployment. This example keeps +the paired identity in its default location: + +```ini +# /etc/systemd/system/librechat-code.service +[Unit] +Description=LibreChat attached code worker +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=librechat-code +Group=librechat-code +WorkingDirectory=/srv/code-workspaces +Environment=HOME=/home/librechat-code +Environment=NODE_ENV=production +Environment=LIBRECHAT_CODE_WORKER_ID=code-example123 +ExecStart=/usr/local/bin/librechat-code run \ + --environment /etc/librechat-code/environments/primary.yaml \ + --allow-workspace-writes \ + --allow-workspace-commands +Restart=always +RestartSec=5s +TimeoutStopSec=35s +KillMode=control-group +UMask=0077 +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` + +For a trusted VM, append `--command-policy-preset trusted-vm` to `ExecStart`. +After installing or changing a unit or drop-in, reload it before restart: + +```bash +sudo systemd-analyze verify librechat-code.service +sudo systemctl daemon-reload +sudo systemctl enable --now librechat-code.service +``` + +`systemctl restart` alone does not load a changed unit definition. + +## 11. Run under launchd on macOS + +Use absolute executable and release paths in the property list. Keep the paired +identity in the logged-in user's private configuration directory: + +```xml +ProgramArguments + + /absolute/path/to/node + /opt/librechat-code/releases/COMMIT/packages/code/dist/cli.js + run + --environment + /Users/worker/.config/librechat/code/environments/primary.yaml + --allow-workspace-writes + --allow-workspace-commands + +EnvironmentVariables + + LIBRECHAT_CODE_WORKER_ID + code-example123 + LIBRECHAT_CODE_IDENTITY_FILE + /Users/worker/.config/librechat/code/code-example123.json + +``` + +Editing the plist does not update launchd's cached job. Reload it: + +```bash +launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/ai.librechat.code.plist +launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/ai.librechat.code.plist +``` + +`launchctl kickstart -k` restarts the already-loaded definition and therefore +continues using stale paths after a plist edit. + +## 12. Verify the complete path + +Do not stop at “the process is running.” Check: + +1. The service command points to the intended version and environment file. +2. The native executor child started. +3. The worker has an established outbound HTTPS connection to Code API. +4. Code API reports the worker `online: true` and `ready: true` with the expected + workspace IDs and operations. +5. LibreChat lists the environment for the expected principal. +6. A disposable chat can select the workspace, read a file, perform an approved + write, execute a command, and retain state on the next turn. +7. Stop/cancellation prevents a delayed mutation. +8. A denied action remains denied even when the worker uses `trusted-vm`. + +Useful host checks: + +```bash +systemctl show librechat-code.service -p ExecStart -p MainPID -p NRestarts +journalctl -u librechat-code.service --since '10 minutes ago' +ss -tpn | grep librechat-code +``` + +The Code API status endpoint requires its administrator credential. Filter the +response before sharing it; do not expose tokens, pairings, bindings, or host +paths in logs or chat. + +## 13. Upgrade and roll back + +For each upgrade: + +1. Read the release notes and confirm whether LibreChat/Code API consumers must + land first. +2. Stage and build the new worker beside the current release. +3. Run focused package/native checks. +4. Update the service path or pinned checkout. +5. Reload the service manager definition when it changed. +6. Restart once and verify the complete path above. +7. Retain the previous release until the worker has completed real work. + +For source-linked installations, verify both `git rev-parse HEAD` and the +actual executable target. Updating a Code API checkout on another host does not +update this worker. + +Rollback by restoring the previous executable/service path and restarting. Do +not roll a new-metadata worker back behind the minimum consumer version while +it still advertises named environments. + +## 14. Recover safely + +### Expired bridge credential + +A running worker refreshes its short-lived credential automatically. If a +machine is offline long enough that refresh can no longer authenticate, issue +a fresh one-time pairing for the same worker ID and redeem it with a newly +generated keypair. Reusing the worker ID preserves the LibreChat environment +record and its agent assignments; creating a new ID creates a new environment. + +### Failed environment setup or uncertain mutation + +Inspect or restore the affected workspace first. Then, with the normal worker +stopped, clear the local quarantine using the same identity/deployment context: + +```bash +librechat-code clear-workspace-quarantine \ + --worker-dir /srv/code-workspaces/app \ + --workspace-id app-dev +``` + +If Code API also retains a server-side workspace fence, run the normal worker +configuration once with `--reset-workspace-quarantine app-dev`, wait for it to +exit successfully, and then start the normal service. The reset flag does not +replace the local clear command. + +Never clear quarantine merely to make the worker start. It represents a setup, +command, cancellation, or settlement whose effects may be incomplete. + +## 15. Common failures + +- **`--environment cannot be combined...`:** remove old workspace flags and + equivalent environment variables. +- **Definition or root rejected as replaceable:** remove group/other write + permission from every path ancestor; keep owner write. +- **Worker starts but old command/path remains:** run + `systemctl daemon-reload`, or fully boot out/bootstrap a changed launchd + plist. +- **Worker online but not ready:** check native sandbox preparation, definition + validation, setup, quarantine, and readiness logs. +- **Setup repeats on restart:** setup is intentionally per-start; make it + idempotent or remove it. +- **Git works on the host but not in tools:** verify the App installation, + permissions, private-key mode/owner, and allowed GitHub domains. +- **Repository label is present but files are absent:** `repo`/`ref` are + metadata; clone or mount the repository yourself. +- **Existing chats lose their workspace:** preserve the original workspace ID + in `name`, commonly `primary`. +- **Multiple project roots are rejected:** roots cannot overlap; remove the + broad parent or keep it as the only environment. + +## Final checklist + +- [ ] LibreChat and every Code API replica support the worker protocol. +- [ ] Worker version/source commit is recorded. +- [ ] Pairing is principal-bound and the identity file is private. +- [ ] Definitions are outside roots and immutable to sandboxed tools. +- [ ] Workspace ancestors are not group/other writable. +- [ ] GitHub App is optional, least-privilege, and installed only where needed. +- [ ] Approval policy remains enforced independently of worker capability. +- [ ] Service manager uses the intended executable and configuration. +- [ ] Worker is online, ready, and advertises the expected workspace. +- [ ] Read, approved mutation, command, persistence, denial, and cancellation + are tested. +- [ ] Upgrade and quarantine-recovery procedures are recorded for the operator. diff --git a/packages/code/README.md b/packages/code/README.md index 8b98618f..3cc61677 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -3,6 +3,9 @@ Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed code environment to LibreChat Code API. +For a complete machine setup and operations guide, see the +[self-hosted worker runbook](../../docs/remote-bridge/worker-runbook.md). + The CLI owns the runtime-supervisor seam. Native workspace commands use Anthropic's open-source Sandbox Runtime (SRT) on the worker machine. The bundled endpoint adapter can also connect to an already-running loopback Code From 6dbf03ba13db9e0c894d06b73412a43cfd39cb0f Mon Sep 17 00:00:00 2001 From: Jackson Riding <99007683+jacksonriding@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:09:43 +1000 Subject: [PATCH 28/75] fix: bound memory buffering for streamed file uploads (#218) --- service/src/file-server.ts | 67 +------------------- service/src/minio-client.test.ts | 101 +++++++++++++++++++++++++++++++ service/src/minio-client.ts | 69 +++++++++++++++++++++ 3 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 service/src/minio-client.test.ts create mode 100644 service/src/minio-client.ts diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 22302042..00b91e4a 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -11,11 +11,11 @@ import { sendFileDownload } from './file-download'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; -import { Client } from 'minio'; +import { createMinioClient } from './minio-client'; import { nanoid } from 'nanoid'; import { PassThrough } from 'stream'; import { pipeline } from 'stream/promises'; -import type { BucketItem, BucketItemStat, ClientOptions } from 'minio'; +import type { BucketItem, BucketItemStat, Client } from 'minio'; import type { Readable } from 'stream'; import type * as tls from 'tls'; import type * as t from './types'; @@ -40,69 +40,6 @@ app.use(httpMetricsMiddleware); const bucketName = process.env.MINIO_BUCKET ?? 'test-bucket'; -type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; - -async function createMinioClient(): Promise { - const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; - const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); - const useIrsa = irsaExplicit || irsaEnvVars; - - const baseConfig: ClientOptions = { - endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', - port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), - useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', - region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', - }; - - if (useIrsa) { - logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { - tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, - roleArn: process.env.AWS_ROLE_ARN, - region: baseConfig.region, - }); - - /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module - * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) - */ - let IamAwsProviderClass: new (opts: object) => unknown; - try { - const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (primaryError) { - try { - // Fallback for bun: resolve path using require if available (CJS context) - let resolvePath = 'node_modules/minio/'; - try { - // eslint-disable-next-line @typescript-eslint/no-require-imports - resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); - } catch { - // require.resolve not available (ESM context), use default path - } - const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (fallbackError) { - logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); - throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); - } - } - - const credentialsProvider = new IamAwsProviderClass({}); - - return new Client({ - ...baseConfig, - credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], - }); - } - - logger.info('Using explicit credentials for MinIO/S3 authentication'); - return new Client({ - ...baseConfig, - accessKey: process.env.MINIO_ACCESS_KEY ?? '', - secretKey: process.env.MINIO_SECRET_KEY ?? '', - sessionToken: process.env.MINIO_SESSION_TOKEN, - }); -} - let minioClient: Client; let storageInitialized = false; diff --git a/service/src/minio-client.test.ts b/service/src/minio-client.test.ts new file mode 100644 index 00000000..13259d03 --- /dev/null +++ b/service/src/minio-client.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from 'bun:test'; +import { createHash } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { createMinioClient } from './minio-client'; + +const MiB = 1024 * 1024; + +// Exercise the real SDK against a local S3 HTTP fixture. No storage account, +// Redis, or file-server listener is needed to test the production client. +test.each([1024, 8 * MiB, 20 * MiB + 17])( + 'unknown-length upload of %i bytes uses bounded parts without losing bytes', + async size => { + const parts = new Map(); + const lengths: number[] = []; + const uploaded: { body?: Buffer; contentType?: string | null; originalFilename?: string | null } = {}; + const xml = (body: string) => new Response(body, { + headers: { 'Content-Type': 'application/xml' }, + }); + const server = Bun.serve({ + hostname: '127.0.0.1', + port: 0, + async fetch(req) { + const url = new URL(req.url); + if (req.method === 'GET' && url.searchParams.has('uploads')) { + return xml('false'); + } + if (req.method === 'POST' && url.searchParams.has('uploads')) { + uploaded.contentType = req.headers.get('content-type'); + uploaded.originalFilename = req.headers.get('x-amz-meta-original-filename'); + return xml('test-upload'); + } + if (req.method === 'PUT' && url.searchParams.has('partNumber')) { + const body = Buffer.from(await req.arrayBuffer()); + lengths.push(body.length); + if (Number(req.headers.get('content-length')) !== body.length || + req.headers.get('content-md5') !== createHash('md5').update(body).digest('base64')) { + return new Response('Invalid part length or checksum', { status: 400 }); + } + parts.set(Number(url.searchParams.get('partNumber')), body); + return new Response(null, { + headers: { ETag: `"${createHash('md5').update(body).digest('hex')}"` }, + }); + } + if (req.method === 'POST' && url.searchParams.has('uploadId')) { + const manifest = await req.text(); + const ordered = [...manifest.matchAll(/(\d+)<\/PartNumber>/g)] + .map(match => parts.get(Number(match[1]))); + if (ordered.length !== parts.size || ordered.some(part => !part)) { + return new Response('Invalid multipart completion', { status: 400 }); + } + uploaded.body = Buffer.concat(ordered as Buffer[]); + return xml('http://localhost/test-bucket/input.bintest-bucketinput.bin"complete"'); + } + return new Response('Unexpected S3 request', { status: 400 }); + }, + }); + const settings: Record = { + MINIO_ENDPOINT: '127.0.0.1', + MINIO_PORT: String(server.port), + MINIO_NO_PORT: 'false', + MINIO_USE_SSL: 'false', + MINIO_REGION: 'us-east-1', + MINIO_USE_IRSA: 'false', + AWS_WEB_IDENTITY_TOKEN_FILE: undefined, + AWS_ROLE_ARN: undefined, + MINIO_ACCESS_KEY: 'test-access', + MINIO_SECRET_KEY: 'test-secret', + MINIO_SESSION_TOKEN: undefined, + }; + const saved = Object.fromEntries(Object.keys(settings).map(key => [key, process.env[key]])); + try { + for (const [key, value] of Object.entries(settings)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + const client = await createMinioClient(); + const expected = Buffer.alloc(size); + for (let i = 0; i < expected.length; i++) expected[i] = i % 251; + function* chunks() { + for (let offset = 0; offset < size; offset += 64 * 1024) { + yield expected.subarray(offset, Math.min(size, offset + 64 * 1024)); + } + } + await client.putObject('test-bucket', 'input.bin', Readable.from(chunks()), undefined, { + 'Content-Type': 'application/octet-stream', + 'X-Amz-Meta-Original-Filename': 'input.bin', + }); + expect(lengths.length).toBe(Math.ceil(size / (8 * MiB))); + expect(lengths.every(length => length <= 8 * MiB)).toBe(true); + expect(uploaded.body?.equals(expected)).toBe(true); + expect(uploaded.contentType).toBe('application/octet-stream'); + expect(uploaded.originalFilename).toBe('input.bin'); + } finally { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + await server.stop(true); + } + }, +); diff --git a/service/src/minio-client.ts b/service/src/minio-client.ts new file mode 100644 index 00000000..07af2891 --- /dev/null +++ b/service/src/minio-client.ts @@ -0,0 +1,69 @@ +import { Client, type ClientOptions } from 'minio'; +import logger from './fileServerLogger'; + +type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; + +export async function createMinioClient(): Promise { + const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; + const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); + const useIrsa = irsaExplicit || irsaEnvVars; + + const baseConfig: ClientOptions = { + // Unknown-length streams otherwise grow SDK parts to 528 MiB (the 5 TiB + // object limit / 10,000 parts). Bound each multipart buffer instead. + partSize: 8 * 1024 * 1024, + endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', + port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), + useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', + region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', + }; + + if (useIrsa) { + logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { + tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, + roleArn: process.env.AWS_ROLE_ARN, + region: baseConfig.region, + }); + + /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module + * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) + */ + let IamAwsProviderClass: new (opts: object) => unknown; + try { + const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (primaryError) { + try { + // Fallback for bun: resolve path using require if available (CJS context) + let resolvePath = 'node_modules/minio/'; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); + } catch { + // require.resolve not available (ESM context), use default path + } + const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (fallbackError) { + logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); + throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); + } + } + + const credentialsProvider = new IamAwsProviderClass({}); + + return new Client({ + ...baseConfig, + credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], + }); + } + + logger.info('Using explicit credentials for MinIO/S3 authentication'); + return new Client({ + ...baseConfig, + accessKey: process.env.MINIO_ACCESS_KEY ?? '', + secretKey: process.env.MINIO_SECRET_KEY ?? '', + sessionToken: process.env.MINIO_SESSION_TOKEN, + }); +} + From 3a3003a4ca4e31b9c120f3bd55238a4a74a8aabd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 01:26:03 -0400 Subject: [PATCH 29/75] ci: Automate Auditable Main Releases (#216) * fix: decouple repository release versions * ci: automate releases after successful main builds --- .github/scripts/next-release-version.sh | 65 +++++++++++++++ .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 100 +++++++++++++++++++----- CONTRIBUTING.md | 10 ++- README.md | 4 +- docs/RELEASING.md | 66 ++++++++++------ tests/release-versioning.sh | 56 +++++++++++++ 7 files changed, 255 insertions(+), 49 deletions(-) create mode 100755 .github/scripts/next-release-version.sh create mode 100755 tests/release-versioning.sh diff --git a/.github/scripts/next-release-version.sh b/.github/scripts/next-release-version.sh new file mode 100755 index 00000000..a5766479 --- /dev/null +++ b/.github/scripts/next-release-version.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +CURRENT_TAG="$1" +REVISION_RANGE="$2" + +if [[ ! "$CURRENT_TAG" =~ ^v([0-9]+)[.]([0-9]+)[.]([0-9]+)$ ]]; then + echo "current release must be a stable vMAJOR.MINOR.PATCH tag (got '$CURRENT_TAG')" >&2 + exit 2 +fi + +# Documentation, workflow, and test-only changes remain auditable in git but do +# not produce a deployable release. Any unrecognised path is treated as +# deployable so a newly added runtime component cannot silently miss a release. +DEPLOYABLE=false +while IFS= read -r path; do + case "$path" in + .github/*|docs/*|tests/*|*.md|*/README|*/README.*|*.test.*|*.spec.*) + ;; + *) + DEPLOYABLE=true + break + ;; + esac +done < <(git diff --name-only "$REVISION_RANGE") + +if [ "$DEPLOYABLE" = "false" ]; then + exit 0 +fi + +COMMIT_MESSAGES="$(git log --format='%s%n%b' "$REVISION_RANGE")" +BUMP=patch + +if grep -Eq '^[[:alnum:]_-]+(\([^)]*\))?!:' <<<"$COMMIT_MESSAGES" \ + || grep -Eq '^BREAKING([ -])CHANGE:' <<<"$COMMIT_MESSAGES"; then + BUMP=major +elif grep -Eq '^feat(\([^)]*\))?:' <<<"$COMMIT_MESSAGES"; then + BUMP=minor +fi + +VERSION="${CURRENT_TAG#v}" +IFS=. read -r MAJOR MINOR PATCH <<<"$VERSION" + +case "$BUMP" in + major) + MAJOR=$((MAJOR + 1)) + MINOR=0 + PATCH=0 + ;; + minor) + MINOR=$((MINOR + 1)) + PATCH=0 + ;; + patch) + PATCH=$((PATCH + 1)) + ;; +esac + +printf 'v%s.%s.%s\n' "$MAJOR" "$MINOR" "$PATCH" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bdeda21..07622ee7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,9 @@ jobs: - name: Compose bridge configuration run: node tests/compose-bridge-config.cjs + - name: Release versioning + run: tests/release-versioning.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 55c7a116..c47b1a0d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,11 @@ # file is inert inside the monorepo — GitHub only runs workflows from the repo # root — and becomes a root workflow in the published repo. # -# Two entry points feed one job: +# Three entry points feed one job: +# +# * successful CI on main — automatically releases deployable changes. The +# next repository version follows Conventional Commit intent; changes that +# only touch docs, workflows, or tests do not cut a release. # # * workflow_dispatch — pick a version in the Actions UI. The chart is # packaged before the tag is created, so a packaging failure aborts while @@ -18,10 +22,13 @@ name: Release on: + workflow_run: + workflows: ['CI'] + types: [completed] workflow_dispatch: inputs: version: - description: 'Version to release, e.g. v2.0.0 or v2.1.0-rc1. Must match helm/codeapi/Chart.yaml appVersion.' + description: 'Repository version to release, e.g. v1.0.0 or v1.1.0-rc1.' required: true type: string draft: @@ -36,12 +43,20 @@ permissions: contents: write concurrency: - group: release-${{ github.event.inputs.version || github.ref_name }} + # Automatic runs serialize against one another. If main advances before an + # older run starts, version resolution skips the stale SHA and the newest + # successful run releases the full range instead. + group: release-${{ github.event_name == 'workflow_run' && 'main' || github.event.inputs.version || github.ref_name }} cancel-in-progress: false jobs: release: name: Tag and publish + if: >- + github.event_name != 'workflow_run' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main') runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -50,19 +65,60 @@ jobs: # Full history and tags: resolving whether this release is the newest # stable one compares it against every other tag in the repository. fetch-depth: 0 + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.ref }} - name: Resolve and validate version id: version env: EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} INPUT_VERSION: ${{ github.event.inputs.version }} INPUT_DRAFT: ${{ github.event.inputs.draft }} REF_NAME: ${{ github.ref_name }} REF_TYPE: ${{ github.ref_type }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + SKIP=false + if [ "$EVENT_NAME" = "workflow_run" ]; then + if [ "$(git rev-parse HEAD)" != "$HEAD_SHA" ]; then + echo "::error::Checked out SHA does not match the successful CI run" + exit 1 + fi + + git fetch --no-tags origin main:refs/remotes/origin/main + if [ "$(git rev-parse refs/remotes/origin/main)" != "$HEAD_SHA" ]; then + echo "main advanced after this CI run; the newer successful run will release the combined changes" + SKIP=true + fi + + # A rerun after tag creation but before release publication resumes + # the missing release rather than incrementing the version again. + EXACT_TAG="$({ git tag --points-at HEAD || true; } | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' | sort -V | tail -n 1)" + if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then + if gh release view "$EXACT_TAG" >/dev/null 2>&1; then + echo "$EXACT_TAG already publishes this commit; nothing to do" + SKIP=true + else + VERSION="$EXACT_TAG" + fi + elif [ "$SKIP" = "false" ]; then + PREVIOUS_TAG="$(git tag --merged HEAD \ + | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ + | sort -V \ + | tail -n 1)" + if [ -z "$PREVIOUS_TAG" ]; then + echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" + exit 1 + fi + VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" + if [ -z "$VERSION" ]; then + echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" + SKIP=true + fi + fi + elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then # Releases describe what shipped to main. Dispatching from a topic # branch would tag a commit that is not on the release line. if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then @@ -74,6 +130,11 @@ jobs: VERSION="$REF_NAME" fi + if [ "$SKIP" = "true" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # A bare "2.0.0" typed into the dispatch box is accepted; everything # downstream works with the v-prefixed form the tag actually uses. case "$VERSION" in @@ -82,14 +143,15 @@ jobs: esac if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then - echo "::error::Release tags must be v.. or v..-rcN, for example v2.0.0 or v2.1.0-rc1 (got '$VERSION')" + echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" exit 1 fi - # v2.1.0-rc1 -> 2.1.0. Release candidates carry the version they are - # candidates for, so they compare against the same appVersion. - BASE_VERSION="${VERSION%%-rc*}" - BASE_VERSION="${BASE_VERSION#v}" + if [ "$EVENT_NAME" = "workflow_run" ] \ + && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + echo "::error::Calculated tag $VERSION already exists on a different commit" + exit 1 + fi read_chart_field() { grep -m1 "^$1:" helm/codeapi/Chart.yaml \ @@ -98,14 +160,6 @@ jobs: APP_VERSION="$(read_chart_field appVersion)" CHART_VERSION="$(read_chart_field version)" - # The tag is the app version. Requiring the bump to have landed on - # main first keeps a deployed chart from reporting a version that no - # release ever carried. - if [ "$APP_VERSION" != "$BASE_VERSION" ]; then - echo "::error::Tag $VERSION does not match helm/codeapi/Chart.yaml appVersion ($APP_VERSION). Land the appVersion bump on main before releasing." - exit 1 - fi - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." @@ -142,8 +196,8 @@ jobs: fi { + echo "skip=false" echo "version=$VERSION" - echo "base_version=$BASE_VERSION" echo "app_version=$APP_VERSION" echo "chart_version=$CHART_VERSION" echo "prerelease=$PRERELEASE" @@ -157,6 +211,7 @@ jobs: # ci.yml depend on it. - name: Package Helm chart id: chart + if: steps.version.outputs.skip != 'true' run: | set -euo pipefail @@ -183,17 +238,20 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Create tag - if: github.event_name == 'workflow_dispatch' + if: steps.version.outputs.skip != 'true' && github.event_name != 'push' env: VERSION: ${{ steps.version.outputs.version }} run: | set -euo pipefail git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git tag -a "$VERSION" -m "$VERSION" - git push origin "refs/tags/$VERSION" + if ! git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + git tag -a "$VERSION" -m "$VERSION" + git push origin "refs/tags/$VERSION" + fi - name: Publish release + if: steps.version.outputs.skip != 'true' env: GH_TOKEN: ${{ github.token }} VERSION: ${{ steps.version.outputs.version }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37ecc984..e7b95797 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,10 +25,12 @@ Practical consequences: ## Releases Tagged releases are cut from `main` as `vMAJOR.MINOR.PATCH` (with `-rcN` for -release candidates), and each one carries the packaged Helm chart. The version -comes from `helm/codeapi/Chart.yaml`'s `appVersion`, so a version bump lands on -`main` through the pull request flow above before it can be released. See -[docs/RELEASING.md](docs/RELEASING.md) for the full process. +release candidates), and each one carries the packaged Helm chart. Repository, +API, service, and chart versions advance independently; component version bumps +land on `main` through the pull request flow above before they are included in a +release. Successful `main` CI automatically releases deployable changes while +documentation, workflow, and test-only changes are skipped. See +[docs/RELEASING.md](docs/RELEASING.md) for the full process and manual path. ## Development diff --git a/README.md b/README.md index bad5b66e..3856b254 100644 --- a/README.md +++ b/README.md @@ -108,14 +108,14 @@ Deployments should pin a [tagged release](https://github.com/LibreChat-AI/code-i rather than track `main`, which moves whenever an internal snapshot is merged: ```bash -git clone --branch v2.0.0 --depth 1 https://github.com/LibreChat-AI/code-interpreter.git +git clone --branch v1.0.0 --depth 1 https://github.com/LibreChat-AI/code-interpreter.git ``` Every release attaches `codeapi-.tgz`, the packaged Helm chart with its Redis and MinIO subcharts vendored: ```bash -helm install codeapi ./codeapi-0.3.0.tgz -f my-values.yaml +helm install codeapi ./codeapi-0.3.1.tgz -f my-values.yaml ``` Versions are `vMAJOR.MINOR.PATCH`, with `-rcN` release candidates published as diff --git a/docs/RELEASING.md b/docs/RELEASING.md index dd00c96d..3b2ff3ef 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -6,44 +6,66 @@ tags are cut. ## Versioning A release is named `vMAJOR.MINOR.PATCH`, optionally with a `-rcN` suffix for a -release candidate — `v2.0.0`, `v2.1.0-rc1`. That version is the **app -version**: `helm/codeapi/Chart.yaml`'s `appVersion` is its source of truth, and -the release workflow refuses any tag that disagrees with it. A release -candidate carries the version it is a candidate for, so `v2.1.0-rc1` also -requires `appVersion: "2.1.0"`. +release candidate — `v1.0.0`, `v1.1.0-rc1`. This is the public repository's +release sequence and is independent from the versions of the components it +contains. The first public release is therefore `v1.0.0` even though the API, +service, and Helm chart already have their own version histories. -Two other version numbers are deliberately independent: +Component versions are deliberately independent: +- `helm/codeapi/Chart.yaml`'s `appVersion` identifies the API version deployed + by the chart. - `helm/codeapi/Chart.yaml`'s `version` is the **chart** version. Bump it when the chart's templates or values change, not when the app changes. It names the packaged chart attached to the release (`codeapi-.tgz`). - `service/package.json`'s `version` tracks the Lambda service package alone. -By convention `api/package.json`'s `version` is kept in step with `appVersion`, -so the API package and the tag agree. Nothing enforces it. +By convention `api/package.json`'s `version` is kept in step with `appVersion`. +Nothing enforces it. -## Cutting a release +## Automatic releases -1. Land the `appVersion` bump on `main` first. `main` takes no direct pushes - (see [CONTRIBUTING.md](../CONTRIBUTING.md)), so it arrives through a sync - pull request from the internal monorepo or a community pull request. Bump - the chart `version` too if the chart changed. +After the full **CI** workflow succeeds for the current tip of `main`, the +release workflow examines everything since the last stable repository tag. It +cuts a release when that range changes deployable files and skips ranges that +only change documentation, GitHub workflows, or tests. If several merges land +while CI is running, the newest successful run releases them together. + +The next version follows Conventional Commit intent across the unreleased +range: + +- a `BREAKING CHANGE:` footer or `type!:` subject bumps the major version; +- a `feat:` subject bumps the minor version; +- every other deployable change bumps the patch version. + +This makes the safe fallback a patch release even when a merge title does not +follow the convention. The workflow packages the Helm chart before creating +the tag, so a packaging failure leaves the version available for a retry. A +rerun also resumes publication if the tag was created before a later step +failed. + +## Manual releases + +1. Land every intended component version bump on `main` first. `main` takes no + direct pushes (see [CONTRIBUTING.md](../CONTRIBUTING.md)), so changes arrive + through a sync pull request from the internal monorepo or a community pull + request. Bump the chart `version` only if the chart changed. 2. Run the **Release** workflow from the Actions tab against `main`, entering - the version (`v2.1.0`). Tick *draft* to review the generated notes before - they go public. + the next repository version (`v1.1.0`). Tick *draft* to review the generated + notes before they go public. -The workflow validates the version, packages the Helm chart, then creates the -annotated tag and publishes the release. Packaging runs before tagging so a -failure — a rate-limited subchart pull, most likely — leaves the version -unused and the run safe to retry. +Use this path when intentionally overriding the automatically selected version, +cutting a release candidate, or recovering while automatic releases are +disabled. The workflow validates the version, packages the Helm chart, then +creates the annotated tag and publishes the release. A tag pushed by hand works as well, and takes the same path from validation onward: ```bash git checkout main && git pull -git tag -a v2.1.0 -m v2.1.0 -git push origin v2.1.0 +git tag -a v1.1.0 -m v1.1.0 +git push origin v1.1.0 ``` ## What the release contains @@ -64,7 +86,7 @@ repository, so re-cutting an older patch cannot drag it backwards. Delete the release and its tag, then re-run the workflow: ```bash -gh release delete v2.1.0 --cleanup-tag --yes +gh release delete v1.1.0 --cleanup-tag --yes ``` Republishing the same version is only safe while nobody has deployed it. Once diff --git a/tests/release-versioning.sh b/tests/release-versioning.sh new file mode 100755 index 00000000..3ab80bef --- /dev/null +++ b/tests/release-versioning.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RESOLVER="$ROOT/.github/scripts/next-release-version.sh" +TEST_REPO="$(mktemp -d)" +trap 'rm -rf "$TEST_REPO"' EXIT + +git -C "$TEST_REPO" init -q +git -C "$TEST_REPO" config user.name test +git -C "$TEST_REPO" config user.email test@example.com + +commit_file() { + local path="$1" + local content="$2" + local message="$3" + + mkdir -p "$TEST_REPO/$(dirname "$path")" + printf '%s\n' "$content" > "$TEST_REPO/$path" + git -C "$TEST_REPO" add "$path" + git -C "$TEST_REPO" commit -q -m "$message" +} + +assert_version() { + local expected="$1" + local actual + actual="$(cd "$TEST_REPO" && bash "$RESOLVER" v1.2.3 v1.2.3..HEAD)" + if [ "$actual" != "$expected" ]; then + echo "expected '$expected', got '$actual'" >&2 + exit 1 + fi +} + +commit_file api/runtime.ts initial 'chore: initial release' +git -C "$TEST_REPO" tag v1.2.3 + +commit_file docs/guide.md docs 'docs: clarify deployment' +assert_version '' + +commit_file api/runtime.ts fix 'fix: repair execution' +assert_version v1.2.4 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file api/runtime.ts feature 'feat: add execution mode' +assert_version v1.3.0 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file api/runtime.ts breaking 'feat!: replace execution protocol' +assert_version v2.0.0 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file service/config.ts config 'chore: tune runtime defaults' +assert_version v1.2.4 + +echo 'release versioning tests passed' From f6cdfb3658a0f61c81cb6c63b73eaacf03417463 Mon Sep 17 00:00:00 2001 From: Jackson Riding <99007683+jacksonriding@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:26:27 +1000 Subject: [PATCH 30/75] fix: forward input-file limit into sandbox guests (#217) --- launcher/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 7771b9b9..27f08f92 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -433,6 +433,7 @@ fn is_allowed_guest_env_key(key: &str, egress_gateway_enabled: bool) -> bool { "SANDBOX_LOG_LEVEL", "SANDBOX_MAX_CONCURRENT_JOBS", "SANDBOX_MAX_FILE_SIZE", + "SANDBOX_MAX_INPUT_FILES", "SANDBOX_MAX_NESTING_DEPTH", "SANDBOX_MAX_OPEN_FILES", "SANDBOX_MAX_OUTPUT_FILES", @@ -850,6 +851,13 @@ mod tests { } } + #[test] + fn guest_env_allowlist_forwards_input_file_limit_in_both_egress_modes() { + for egress_gateway_enabled in [false, true] { + assert!(is_allowed_guest_env_key("SANDBOX_MAX_INPUT_FILES", egress_gateway_enabled)); + } + } + #[test] fn guest_env_allowlist_preserves_legacy_file_server_url_only_without_egress_gateway() { assert!(is_allowed_guest_env_key("FILE_SERVER_URL", false)); From b35c503fd2fe7be412d95c0eef6db50a09aad280 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 01:51:42 -0400 Subject: [PATCH 31/75] feat: Inspect Local Coding Projects (#221) * feat: add bounded local project inventory * fix: report incomplete Git metadata reads * fix: preserve incomplete discovery and remote identities * fix: finalize discovery budgets and nested remote identities * fix(code): stop project traversal at filesystem budget boundaries --- docs/remote-bridge/projects.md | 69 ++++++ packages/code/README.md | 37 +++ packages/code/src/cli.ts | 11 + packages/code/src/projects.test.ts | 368 +++++++++++++++++++++++++++++ packages/code/src/projects.ts | 295 +++++++++++++++++++++++ 5 files changed, 780 insertions(+) create mode 100644 docs/remote-bridge/projects.md create mode 100644 packages/code/src/projects.test.ts create mode 100644 packages/code/src/projects.ts diff --git a/docs/remote-bridge/projects.md b/docs/remote-bridge/projects.md new file mode 100644 index 00000000..987ce112 --- /dev/null +++ b/docs/remote-bridge/projects.md @@ -0,0 +1,69 @@ +# Projects and worktrees + +The intended experience is to select a project on an attached machine, choose +the current checkout or a new worktree, and have subsequent tool calls and +approval resumes use that selection without repeating a working directory. + +## Delivery sequence + +1. Local project inventory (`librechat-code projects --root `). + Discover bounded Git metadata without changing registration or authority. +2. Negotiated project selection across the worker, Code API, and LibreChat. + Persist the selection and enforce the same project boundary in file tools, + commands, programmatic execution, environment actions, and approval resumes. +3. Worktree creation and setup with durable operation receipts. Publish a new + selection only after Git creation, setup, and registration have completed. +4. Worktree selection in the composer and an approved agent operation. Allow + an authenticated owner to narrow a selection to a newly created worktree + with a compare-and-set against the prior conversation decision. +5. Explicit listing and removal, with binding checks and recovery for uncertain + outcomes. Retention policy determines eligibility, not automatic permission + to destroy uncommitted work or unpublished commits. + +Only the first step is implemented by the inventory command. Existing explicit +workspace registration remains available for independent project directories. + +## Boundaries that must remain consistent + +- Discovery metadata is advisory. A remote is not an authorization grant, a + trusted repository identity, or automatically a codegraph repository ID. + Keep the host in normalized remotes to distinguish identically named repos. +- A discovery root may authorize enumeration while an execution root narrows + writes to one selected project. A broad parent must not remain an independent + concurrent execution lane alongside its descendants. +- Path-derived IDs must be scoped by the registered root. Admission must + validate the current directory identity; inventory cannot reserve a path + against replacement after discovery. +- Discovery must not run on every status request. A future worker catalog + needs bounded caching, coalesced refreshes, and explicit generation changes. +- A linked worktree shares Git metadata with its parent. Project IDs alone + cannot make those metadata mutations independent. Admission needs both a + filesystem boundary and coordination for the common Git directory. +- A worktree beneath its parent checkout overlaps that checkout. Either use + disjoint execution roots under a discovery grant or explicitly exclude and + coordinate descendant worktrees before relaxing root exclusion. +- Setup and dependency links must remain within the execution policy. Sharing + writable dependency directories between supposedly isolated worktrees + reintroduces overlap and requires an explicit operator decision. +- Dynamic registration requires versioned capabilities and fenced catalog + generations. Old consumers must not silently drop a project selection and + execute against its broader parent. Deploy consumers before producers. +- Create, setup, registration, and conversation binding form a recoverable + lifecycle. A network retry must find the same worktree, not create a second + one. Failed setup leaves it unavailable; uncertain mutation quarantines it. +- Approval decisions must include the exact target and operation. Creating a + branch changes repository state and follows mutation policy. An additive + operation is not automatically exempt from required approval. +- Cleanup must coordinate live bindings and active execution. A missing remote + branch alone does not prove a worktree is disposable. + +## Acceptance cases for selection and lifecycle + +Verify separate repositories under one discovery root, two chats sharing one +project, two worktrees sharing Git metadata, directory replacement, stale +catalog generations, old/new consumer combinations, and cross-principal access. +Exercise file tools, commands, programmatic execution, and environment actions +through the same persisted selection. Include pause/resume, cancellation during +creation and setup, process death before registration, retry after binding, and +removal racing an active conversation. Use disposable local fixtures before +testing the hosted deployment. diff --git a/packages/code/README.md b/packages/code/README.md index 3cc61677..48bf769d 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -14,6 +14,43 @@ container/NsJail profile. The worker connects outbound to Code API, long-polls for assignments, sends them to the local runtime, and returns fenced results. The VM does not need an inbound public port. +## Inspect local projects + +Before registering a directory containing several checkouts, inspect its Git +projects on the worker machine: + +```bash +librechat-code projects --root /srv/projects +``` + +The command prints JSON with `projects`, `truncated`, and `incomplete`. Each +project contains a path relative to the requested directory, a path-derived ID, +and the current origin, branch, and HEAD. Origins are normalized to +`host[:port]/namespace/repository`, including nested namespaces; URL credentials, +query strings, and fragments are omitted. An unsupported configured origin is +redacted to null and marks the inventory incomplete. +A detached HEAD has a null branch; an unborn branch has a null HEAD. IDs stay +stable when branches change, but moving or renaming a directory changes its ID. +IDs are local to the supplied discovery root. + +Discovery runs only when requested. Its default limits are three directory +levels, 10,000 entries, 256 projects, and a ten-second processing budget with +bounded Git subprocess output and timeouts. +The time budget starts before resolving the root and is checked between native +filesystem operations; it cannot interrupt a kernel call stalled on a filesystem. +Use a responsive local filesystem. Discovery skips hidden directories, +dependencies, symlinks, and children of an identified repository. Linked +worktrees and submodules using a `.git` file are skipped and set `incomplete`: +their shared Git metadata needs separate admission before independent execution. +An empty project list does not prevent registering a non-Git directory. + +This is a local inventory command. It does not clone, register roots, pair a +worker, change the sandbox, or automatically select a conversation workspace. +For the existing picker and independent lease slots, explicitly register the +chosen non-overlapping project directories with `--workspace` or `--environment`. +Do not also register their parent directory. Treat the inventory as a snapshot; +normal workspace admission must validate any directory selected from it. + ## Pair Hardened deployments use a one-time code instead of copying a long-lived diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 00eca576..7406e267 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -5,6 +5,7 @@ import { realpath, stat } from 'node:fs/promises'; import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { discoverProjects } from './projects.js'; import { loadCodeEnvironment, assertEnvironmentDefinitionsOutsideRoots, @@ -1237,6 +1238,16 @@ async function clearMutationQuarantine(args: string[]): Promise { async function main(): Promise { const args = process.argv.slice(2); + if (args[0] === 'projects') { + const root = option(args, '--root'); + if (!root || args.slice(1).some((arg, index, rest) => + arg !== '--root' && rest[index - 1] !== '--root' && !arg.startsWith('--root='))) { + throw new Error('Usage: librechat-code projects --root '); + } + const inventory = await discoverProjects({ root }); + process.stdout.write(`${JSON.stringify(inventory, null, 2)}\n`); + return; + } if (args[0] === 'relay') { await relay(); return; diff --git a/packages/code/src/projects.test.ts b/packages/code/src/projects.test.ts new file mode 100644 index 00000000..98ce0949 --- /dev/null +++ b/packages/code/src/projects.test.ts @@ -0,0 +1,368 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + chmod, + mkdtemp, + mkdir, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import fs from 'node:fs/promises'; +import { syncBuiltinESMExports } from 'node:module'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; +import { discoverProjects, projectRemote } from './projects.js'; + +const exec = promisify(execFile); +async function fixture(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), 'code-projects-')); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} +async function repo(root: string, path: string) { + const directory = join(root, path); + await mkdir(directory, { recursive: true }); + await exec('git', ['init', '--initial-branch=dev', directory]); + return directory; +} + +test('discovers real sibling repositories with stable IDs and bounded metadata', async t => { + const root = await fixture(t); + const a = await repo(root, 'a'); + await repo(root, 'nested/b'); + await exec('git', [ + '-C', + a, + 'remote', + 'add', + 'origin', + 'https://user:secret@github.com/example/app.git?token=secret', + ]); + await exec('git', [ + '-C', + a, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '--allow-empty', + '-m', + 'initial', + ]); + const before = await discoverProjects({ root }); + assert.equal(before.incomplete, false); + assert.equal(before.truncated, false); + assert.deepEqual( + before.projects.map(p => p.path), + ['a', 'nested/b'] + ); + assert.equal(before.projects[0].remote, 'github.com/example/app'); + assert.equal(before.projects[0].branch, 'dev'); + assert.match(before.projects[0].head!, /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/); + assert.equal(before.projects[1].head, null); + assert.ok(!JSON.stringify(before).includes('secret')); + await exec('git', ['-C', a, 'checkout', '-b', 'next']); + const after = await discoverProjects({ root }); + assert.deepEqual( + after.projects.map(p => p.id), + before.projects.map(p => p.id) + ); + assert.equal(after.projects[0].branch, 'next'); +}); + +test('does not walk dependencies, hidden directories, symlinks or repository children', async t => { + const root = await fixture(t); + await repo(root, 'node_modules/ignored'); + await repo(root, '.hidden/ignored'); + await repo(root, 'parent'); + await repo(root, 'parent/nested'); + const outside = await fixture(t); + await repo(outside, 'external'); + await symlink(outside, join(root, 'alias'), 'dir'); + const inventory = await discoverProjects({ root }); + assert.deepEqual( + inventory.projects.map(p => p.path), + ['parent'] + ); +}); + +test('linked worktrees are reported incomplete until shared git metadata is admitted', async t => { + const root = await fixture(t); + await mkdir(join(root, 'linked')); + await writeFile( + join(root, 'linked', '.git'), + 'gitdir: /outside/metadata\n' + ); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.deepEqual(inventory.projects, []); +}); + +test('oversized Git metadata is incomplete rather than silently reported absent', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + await exec('git', [ + '-C', + directory, + 'config', + 'remote.origin.url', + 'x'.repeat(10_000), + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].remote, null); +}); + +test('a valid branch beyond the metadata bound reports incomplete', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + const branch = ['a'.repeat(100), 'b'.repeat(100), 'c'.repeat(100)].join( + '/' + ); + await exec('git', [ + '-C', + directory, + 'symbolic-ref', + 'HEAD', + `refs/heads/${branch}`, + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].branch, null); +}); + +test('unreadable Git markers report incomplete', async t => { + if (process.platform === 'win32' || process.getuid?.() === 0) { + t.skip('requires POSIX permissions under an unprivileged account'); + return; + } + const root = await fixture(t); + const directory = await repo(root, 'app'); + await chmod(directory, 0o400); + try { + assert.equal( + (await discoverProjects({ root: directory })).incomplete, + true + ); + } finally { + await chmod(directory, 0o700); + } +}); + +test('root resolution consumes the processing budget and pre-abort wins', async t => { + const root = await fixture(t); + let ticks = 0; + t.mock.method(Date, 'now', () => (ticks++ === 0 ? 0 : 20_000)); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.truncated, true); + assert.deepEqual(inventory.projects, []); + const reason = new Error('cancelled before filesystem access'); + await assert.rejects( + discoverProjects({ + root: join(root, 'missing'), + signal: AbortSignal.abort(reason), + }), + error => error === reason + ); +}); + +test('empty directory completion checks a late deadline and cancellation', async t => { + const root = await fixture(t); + const original = fs.opendir; + let clock = 0; + let cancel: AbortController | undefined; + t.mock.method(Date, 'now', () => clock); + const openMock = t.mock.method( + fs, + 'opendir', + async (path: Parameters[0]) => { + const directory = await original(path); + clock = 20_000; + cancel?.abort(); + return directory; + } + ); + syncBuiltinESMExports(); + try { + assert.equal((await discoverProjects({ root })).truncated, true); + clock = 0; + cancel = new AbortController(); + await assert.rejects( + discoverProjects({ root, signal: cancel.signal }), + { name: 'AbortError' } + ); + } finally { + openMock.mock.restore(); + syncBuiltinESMExports(); + } +}); + +test('late filesystem boundaries stop before starting the next operation', async t => { + const root = await fixture(t); + for (const boundary of [3, 4, 5]) { + for (const abort of [false, true]) { + let calls = 0; + let clock = 0; + const controller = new AbortController(); + const now = t.mock.method(Date, 'now', () => clock); + const mocks = ['lstat', 'realpath', 'opendir'].map(name => { + const original = fs[name as 'lstat']; + return t.mock.method( + fs, + name as 'lstat', + async (...args: Parameters) => { + calls++; + try { + return await original(...args); + } finally { + if (calls === boundary) { + clock = 20_000; + if (abort) controller.abort(); + } + } + } + ); + }); + syncBuiltinESMExports(); + try { + const discovery = discoverProjects({ + root, + signal: controller.signal, + }); + if (abort) + await assert.rejects(discovery, { name: 'AbortError' }); + else assert.equal((await discovery).truncated, true); + assert.equal(calls, boundary); + } finally { + for (const mock of mocks) mock.mock.restore(); + now.mock.restore(); + syncBuiltinESMExports(); + } + } + } +}); + +test('unsupported configured origins are distinguishable from missing origins', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + await exec('git', [ + '-C', + directory, + 'config', + 'remote.origin.url', + '/private/local/repo', + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].remote, null); +}); + +test('project, entry and depth ceilings report partial discovery', async t => { + const root = await fixture(t); + await repo(root, 'a'); + await repo(root, 'b'); + await repo(root, 'nested/deeper/c'); + assert.equal( + (await discoverProjects({ root, maxProjects: 1 })).truncated, + true + ); + assert.equal( + (await discoverProjects({ root, maxEntries: 1 })).truncated, + true + ); + assert.equal( + (await discoverProjects({ root, maxDepth: 1 })).truncated, + true + ); + await assert.rejects(discoverProjects({ root, maxDepth: 100 }), /limit/); + await assert.rejects( + discoverProjects({ root, signal: AbortSignal.abort() }) + ); +}); + +test('root checkout uses dot and detached HEAD has no branch', async t => { + const root = await fixture(t); + await repo(root, '.'); + await exec('git', [ + '-C', + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '--allow-empty', + '-m', + 'initial', + ]); + await exec('git', ['-C', root, 'checkout', '--detach']); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.projects[0].path, '.'); + assert.equal(inventory.projects[0].branch, null); +}); + +test('repository identity retains host and drops credentials, query and fragments', () => { + assert.equal( + projectRemote('ssh://git@example.com:2222/org/repo.git'), + 'example.com:2222/org/repo' + ); + assert.equal( + projectRemote('https://example.com:8443/org/repo.git'), + 'example.com:8443/org/repo' + ); + assert.equal( + projectRemote('git@github.com:org/repo.git'), + 'github.com/org/repo' + ); + assert.equal( + projectRemote('ssh://git@example.com/org/repo.git'), + 'example.com/org/repo' + ); + assert.equal( + projectRemote('https://token@example.com/org/repo.git?secret#fragment'), + 'example.com/org/repo' + ); + assert.equal(projectRemote('/home/user/private'), null); + assert.equal(projectRemote('file:///home/user/private'), null); + assert.equal( + projectRemote('https://example.com/group/subgroup/repo.git'), + 'example.com/group/subgroup/repo' + ); + assert.equal( + projectRemote('git@example.com:group/subgroup/repo.git'), + 'example.com/group/subgroup/repo' + ); +}); + +test('CLI inventories a real checkout without pairing or starting a worker', async t => { + const root = await fixture(t); + await repo(root, 'app'); + const { stdout, stderr } = await exec( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'projects', + '--root', + root, + ], + { env: { PATH: process.env.PATH }, timeout: 15_000 } + ); + const result = JSON.parse(stdout); + assert.equal(stderr, ''); + assert.equal(result.projects[0].path, 'app'); + assert.equal(result.projects[0].branch, 'dev'); + assert.equal(result.incomplete, false); + await assert.rejects( + exec(process.execPath, [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'projects', + ]), + /Usage: librechat-code projects/ + ); +}); diff --git a/packages/code/src/projects.ts b/packages/code/src/projects.ts new file mode 100644 index 00000000..1e2fcbd3 --- /dev/null +++ b/packages/code/src/projects.ts @@ -0,0 +1,295 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { lstat, opendir, realpath } from 'node:fs/promises'; +import { isAbsolute, relative, resolve, sep } from 'node:path'; +import { promisify } from 'node:util'; + +const exec = promisify(execFile); +const skipped = new Set(['node_modules', 'vendor']); + +export interface LocalProject { + id: string; + path: string; + remote: string | null; + branch: string | null; + head: string | null; +} + +export interface ProjectInventory { + projects: LocalProject[]; + truncated: boolean; + incomplete: boolean; +} + +export interface ProjectDiscoveryOptions { + root: string; + maxDepth?: number; + maxProjects?: number; + maxEntries?: number; + timeoutMs?: number; + signal?: AbortSignal; +} + +/** Public repository identity only: never propagate credentials or URL query data. */ +export function projectRemote(value: string): string | null { + let host: string; + let path: string; + try { + const scp = /^(?:[^/@:\s]+@)?([^/:\s]+):([^\s]+)$/.exec(value); + if (scp && !value.includes('://')) { + host = scp[1]; + path = scp[2]; + } else { + const url = new URL(value); + if (!['https:', 'http:', 'ssh:', 'git:'].includes(url.protocol)) + return null; + host = url.host; + path = url.pathname.replace(/^\//, ''); + } + path = path.replace(/\.git$/, ''); + if ( + !/^[A-Za-z0-9.-]+(?::[0-9]+)?$/.test(host) || + !/^[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)+$/.test(path) + ) + return null; + if (path.split('/').some(part => part === '.' || part === '..')) + return null; + return `${host.toLowerCase()}/${path}`; + } catch { + return null; + } +} + +function limit( + value: number | undefined, + fallback: number, + maximum: number +): number { + const resolved = value ?? fallback; + if (!Number.isSafeInteger(resolved) || resolved < 1 || resolved > maximum) + throw new Error('Invalid project discovery limit'); + return resolved; +} + +/** Bounded local inventory; it does not grant roots or mutate a checkout. */ +export async function discoverProjects( + options: ProjectDiscoveryOptions +): Promise { + const maxDepth = limit(options.maxDepth, 3, 16); + const maxProjects = limit(options.maxProjects, 256, 256); + const maxEntries = limit(options.maxEntries, 10_000, 100_000); + const timeoutMs = limit(options.timeoutMs, 10_000, 60_000); + const deadline = Date.now() + timeoutMs; + const result: ProjectInventory = { + projects: [], + truncated: false, + incomplete: false, + }; + let entries = 0; + const expired = (): boolean => { + options.signal?.throwIfAborted(); + if (Date.now() < deadline) return false; + result.truncated = true; + return true; + }; + options.signal?.throwIfAborted(); + const root = await realpath(options.root); + if (expired()) return result; + const rootStat = await lstat(root); + if (expired()) return result; + if (!rootStat.isDirectory()) + throw new Error('Project root must be a directory'); + const queue = [{ path: root, depth: 0 }]; + const git = async ( + path: string, + args: string[], + expectedExitCodes: number[] = [] + ): Promise => { + if (expired()) return null; + try { + const { stdout } = await exec( + 'git', + [ + '--no-optional-locks', + '-C', + path, + '-c', + 'core.fsmonitor=false', + ...args, + ], + { + env: { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }, + encoding: 'utf8', + maxBuffer: 4096, + timeout: Math.max(1, Math.min(1500, deadline - Date.now())), + signal: options.signal, + } + ); + return stdout.trim(); + } catch (error) { + options.signal?.throwIfAborted(); + const expected = + error instanceof Error && + 'code' in error && + typeof error.code === 'number' && + expectedExitCodes.includes(error.code); + if (!expected) result.incomplete = true; + return null; + } + }; + for (let index = 0; index < queue.length; index++) { + if (expired()) break; + const current = queue[index]; + try { + // Revalidate queued directories; never traverse a replaced symlink. + const currentStat = await lstat(current.path); + if (expired()) break; + if (currentStat.isSymbolicLink()) { + result.incomplete = true; + continue; + } + const canonical = await realpath(current.path); + if (expired()) break; + const rel = relative(root, canonical); + if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + result.incomplete = true; + continue; + } + const marker = await lstat(resolve(current.path, '.git')).catch( + error => { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + throw error; + return undefined; + } + ); + if (expired()) break; + if (marker) { + // Linked worktrees and submodules need separate shared-gitdir admission. + if (!marker.isDirectory() || marker.isSymbolicLink()) { + result.incomplete = true; + continue; + } + if (result.projects.length === maxProjects) { + result.truncated = true; + break; + } + const top = await git(current.path, [ + 'rev-parse', + '--show-toplevel', + ]); + if (expired()) break; + const canonicalTop = top + ? await realpath(top).catch(() => null) + : null; + if (expired()) break; + if (!top || canonicalTop !== canonical) { + result.incomplete = true; + continue; + } + const remote = await git( + current.path, + [ + 'config', + '--local', + '--no-includes', + '--get', + 'remote.origin.url', + ], + [1] + ); + const branch = await git( + current.path, + ['symbolic-ref', '--quiet', '--short', 'HEAD'], + [1] + ); + const head = await git( + current.path, + ['rev-parse', '--verify', 'HEAD'], + branch ? [128] : [] + ); + const path = rel.split(sep).join('/') || '.'; + const normalizedRemote = remote ? projectRemote(remote) : null; + const validBranch = + branch && + branch.length <= 256 && + !/[\x00-\x1f\x7f]/.test(branch) + ? branch + : null; + const validHead = + head && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(head) + ? head + : null; + if ( + (remote !== null && normalizedRemote === null) || + (branch !== null && validBranch === null) || + (head !== null && validHead === null) + ) + result.incomplete = true; + result.projects.push({ + id: `project-${createHash('sha256') + .update(path) + .digest('hex') + .slice(0, 32)}`, + path, + remote: normalizedRemote, + branch: validBranch, + head: validHead, + }); + continue; + } + if (current.depth === maxDepth) { + result.truncated = true; + continue; + } + const directory = await opendir(current.path); + // Close a newly opened handle even when cancellation won during open. + try { + if (expired()) { + await directory.close(); + break; + } + } catch (error) { + await directory.close(); + throw error; + } + for await (const entry of directory) { + if (expired() || ++entries > maxEntries) { + result.truncated = true; + result.projects.sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0 + ); + return result; + } + if ( + entry.isDirectory() && + !entry.name.startsWith('.') && + !skipped.has(entry.name) + ) + queue.push({ + path: resolve(current.path, entry.name), + depth: current.depth + 1, + }); + } + } catch { + options.signal?.throwIfAborted(); + result.incomplete = true; + } + } + result.projects.sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0 + ); + expired(); + return result; +} From c688b30d2ebb0af0fc9d603766b9ebcb4e2a8c9d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 07:38:19 -0400 Subject: [PATCH 32/75] fix: make automatic release tip check read-only (#225) --- .github/workflows/release.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c47b1a0d..ea8a5364 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,7 @@ name: Release on: workflow_run: workflows: ['CI'] + branches: [main] types: [completed] workflow_dispatch: inputs: @@ -87,8 +88,12 @@ jobs: exit 1 fi - git fetch --no-tags origin main:refs/remotes/origin/main - if [ "$(git rev-parse refs/remotes/origin/main)" != "$HEAD_SHA" ]; then + REMOTE_MAIN_SHA="$(git ls-remote origin refs/heads/main | awk '{print $1}')" + if [ -z "$REMOTE_MAIN_SHA" ]; then + echo "::error::Could not resolve the current main branch tip" + exit 1 + fi + if [ "$REMOTE_MAIN_SHA" != "$HEAD_SHA" ]; then echo "main advanced after this CI run; the newer successful run will release the combined changes" SKIP=true fi From 4c7b224358ac1647416f070935931abe088008ef Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 09:01:32 -0400 Subject: [PATCH 33/75] feat: Discover Bounded Repository Instructions for Attached Workspaces (#226) * Discover bounded repository instructions for opted-in workspaces * Verify snapshot digests and cross-platform confinement --- packages/code/REPOSITORY-INSTRUCTIONS.md | 29 ++++ packages/code/src/cli.ts | 5 +- packages/code/src/instructions.test.ts | 206 +++++++++++++++++++++++ packages/code/src/instructions.ts | 89 ++++++++++ packages/code/src/protocol.ts | 40 +++++ packages/code/src/worker.ts | 26 ++- packages/code/src/workspace.ts | 25 ++- service/src/bridge/instructions.test.ts | 123 ++++++++++++++ 8 files changed, 539 insertions(+), 4 deletions(-) create mode 100644 packages/code/REPOSITORY-INSTRUCTIONS.md create mode 100644 packages/code/src/instructions.test.ts create mode 100644 packages/code/src/instructions.ts create mode 100644 service/src/bridge/instructions.test.ts diff --git a/packages/code/REPOSITORY-INSTRUCTIONS.md b/packages/code/REPOSITORY-INSTRUCTIONS.md new file mode 100644 index 00000000..5f8ce529 --- /dev/null +++ b/packages/code/REPOSITORY-INSTRUCTIONS.md @@ -0,0 +1,29 @@ +# Repository instructions + +After updating Code API and LibreChat to versions supporting repository instruction metadata, +add `--repository-instructions` to the existing `librechat-code run` command. Keep all existing +pairing, workspace, sandbox and environment arguments. This is an explicit machine-owner opt-in; +without it, no instructions are advertised or automatically read. + +Only each registered workspace root is examined. `AGENTS.md` takes precedence; `CLAUDE.md` +is considered only when `AGENTS.md` does not exist. Symlinks, directories, unreadable files, +invalid UTF-8 and binary files are omitted. There is no parent-directory walk or globbing. + +Metadata refreshes with the existing worker registration heartbeat. It does not change worker +identity, lease slots or workspace permissions. The next run after that refresh sees changed +metadata; this is not a filesystem watch or a guarantee of immediate edit visibility. + +The descriptor contains only the relative filename, bounded snapshot byte count, SHA-256 and +truncation flag. SHA-256 identifies the delivered UTF-8 snapshot, not unseen bytes beyond the +32 KiB cap. An incomplete trailing UTF-8 character is omitted. Content travels only through the +authorized `read_file` operation, using `instructionSha256` to reject reads if the snapshot changed. +This mode preserves newlines and is independent of the ordinary line-oriented reader. + +LibreChat caches verified content with bounded capacity and principal/machine/workspace scoping. +Its per-agent setting selects `prefer`, `defer` or `off`; it cannot enable discovery on a worker +that did not opt in or widen filesystem, network, or approval permissions. + +Rollout: update LibreChat and Code API first, then update workers and enable the flag. Older +workers continue unchanged. An older Code API rejecting the metadata causes the worker to retry +without it and omit metadata until restart. Do not enable this flag against an older LibreChat +instance: older clients validate workspace descriptors strictly. diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 7406e267..946b08d0 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -684,11 +684,13 @@ async function run( }), ]), ); - let workspaceTools: WorkspaceToolExecutor | undefined = workerDirectory + const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, + repositoryInstructions: args.includes('--repository-instructions'), }) : undefined; + let workspaceTools: WorkspaceToolExecutor | undefined = localWorkspaceTools; if (allowWorkspaceCommands && !canonicalWorkerDirectory) { throw new Error('Workspace commands require a registered directory'); } @@ -1030,6 +1032,7 @@ async function run( } try { const worker = new BridgeWorker({ + instructionDescriptors: () => localWorkspaceTools?.instructionDescriptors() ?? Promise.resolve(undefined), codeApiUrl, token: configuredToken, identity: workerIdentity, diff --git a/packages/code/src/instructions.test.ts b/packages/code/src/instructions.test.ts new file mode 100644 index 00000000..42baafe9 --- /dev/null +++ b/packages/code/src/instructions.test.ts @@ -0,0 +1,206 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, writeFile, rm, symlink, realpath } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { readRepositoryInstructions } from './instructions.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import { isWorkspaceToolResult } from './protocol.js'; +import { BridgeWorker } from './worker.js'; + +test('instruction discovery selects one fixed name and refuses symlink fallback', async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'repository-instructions-')), + ); + try { + await writeFile(join(root, 'CLAUDE.md'), 'fallback\n'); + assert.equal( + (await readRepositoryInstructions(root))?.content, + 'fallback\n', + ); + await writeFile(join(root, 'AGENTS.md'), 'preferred\n'); + assert.equal( + (await readRepositoryInstructions(root))?.content, + 'preferred\n', + ); + await rm(join(root, 'AGENTS.md')); + await symlink(join(root, 'CLAUDE.md'), join(root, 'AGENTS.md')); + assert.equal(await readRepositoryInstructions(root), undefined); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('registration refreshes instruction metadata without changing the worker incarnation', async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'repository-instructions-')), + ); + try { + await writeFile(join(root, 'AGENTS.md'), 'first'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + repositoryInstructions: true, + }); + const registrations: Array<{ + incarnationId: string; + capabilities: { + workspaceTools: { + workspaces: Array<{ + instructions?: Array<{ sha256: string }>; + }>; + }; + }; + }> = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'test', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'test', + runtimes: [], + workspaceTools: tools.capabilities, + }, + workspaceTools: tools, + instructionDescriptors: () => tools.instructionDescriptors(), + fetchImpl: async (_url, init) => { + registrations.push(JSON.parse(String(init?.body))); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + supportedWorkspaceToolOperations: [ + 'read_file', + 'search_text', + 'list_files', + ], + supportedWorkspaceListFileFeatures: ['after_path'], + }); + }, + }); + await worker.register(); + const first = + registrations.at(-1)!.capabilities.workspaceTools.workspaces[0] + .instructions![0].sha256; + await writeFile(join(root, 'AGENTS.md'), 'second'); + await worker.register(); + assert.notEqual( + registrations.at(-1)!.capabilities.workspaceTools.workspaces[0] + .instructions![0].sha256, + first, + ); + assert.ok( + registrations.every( + item => item.incarnationId === 'incarnation-00000001', + ), + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('bounded UTF-8 snapshots refresh and hash-fence authorized reads', async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'repository-instructions-')), + ); + try { + await writeFile(join(root, 'AGENTS.md'), 'a'.repeat(32767) + '🙂tail'); + const snapshot = await readRepositoryInstructions(root); + assert.equal(snapshot?.descriptor.truncated, true); + assert.equal(snapshot?.descriptor.bytes, 32767); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + repositoryInstructions: true, + }); + const request = { + protocolVersion: 1 as const, + operation: 'read_file' as const, + workspaceId: 'primary', + path: 'AGENTS.md', + instructionSha256: snapshot!.descriptor.sha256, + }; + const result = await tools.execute(request); + assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal(isWorkspaceToolResult(request, { ...result, content: 'forged', endLine: 1 }), false); + assert.equal( + (await tools.instructionDescriptors())?.get('primary')?.[0]?.sha256, + snapshot?.descriptor.sha256, + ); + await writeFile(join(root, 'AGENTS.md'), 'new instructions\n'); + await assert.rejects( + tools.execute(request), + /changed or are unavailable/, + ); + assert.notEqual( + (await tools.instructionDescriptors())?.get('primary')?.[0]?.sha256, + snapshot?.descriptor.sha256, + ); + const disabled = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + assert.equal(await disabled.instructionDescriptors(), undefined); + await assert.rejects(disabled.execute(request)); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('an older bridge can reject metadata without breaking later registrations', async () => { + let metadataRequests = 0; + let accepted = 0; + const capabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', name: 'Primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'test', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'test', + runtimes: [], + workspaceTools: capabilities, + }, + workspaceTools: { + capabilities, + execute: async () => { + throw new Error('not expected'); + }, + }, + instructionDescriptors: async () => new Map([['primary', []]]), + fetchImpl: async (_url, init) => { + const body = JSON.parse(String(init?.body)); + if ( + body.capabilities.workspaceTools?.workspaces[0].instructions !== + undefined + ) { + metadataRequests++; + return Response.json( + { error: 'Unknown workspace field' }, + { status: 400 }, + ); + } + accepted++; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + supportedWorkspaceToolOperations: ['read_file'], + }); + }, + }); + await worker.register(); + await worker.register(); + assert.equal(metadataRequests, 1); + assert.ok(accepted >= 2); +}); diff --git a/packages/code/src/instructions.ts b/packages/code/src/instructions.ts new file mode 100644 index 00000000..ee3e8127 --- /dev/null +++ b/packages/code/src/instructions.ts @@ -0,0 +1,89 @@ +import { constants } from 'node:fs'; +import { open, lstat, realpath, stat } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { resolve, relative, isAbsolute, sep } from 'node:path'; + +import { REPOSITORY_INSTRUCTION_MAX_BYTES } from './protocol.js'; +import type { RepositoryInstructionDescriptor } from './protocol.js'; +export interface RepositoryInstructionSnapshot { + descriptor: RepositoryInstructionDescriptor; + content: string; +} + +/** Fixed-name, root-confined discovery. An unreadable AGENTS.md never selects a fallback. */ +export async function readRepositoryInstructions( + root: string, +): Promise { + let path: RepositoryInstructionDescriptor['path'] = 'AGENTS.md'; + try { + await lstat(resolve(root, path)); + } catch (error) { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + return; + path = 'CLAUDE.md'; + } + let handle: Awaited> | undefined; + try { + const candidate = resolve(root, path); + if (!(await lstat(candidate)).isFile()) return; + handle = await open( + candidate, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const opened = await handle.stat(); + const canonical = await realpath(candidate); + const offset = relative(root, canonical); + const current = await stat(canonical); + if ( + !opened.isFile() || + isAbsolute(offset) || + offset === '..' || + offset.startsWith(`..${sep}`) || + opened.dev !== current.dev || + opened.ino !== current.ino + ) + return; + const buffer = Buffer.alloc(REPOSITORY_INSTRUCTION_MAX_BYTES + 1); + let length = 0; + while (length < buffer.length) { + const read = await handle.read( + buffer, + length, + buffer.length - length, + length, + ); + if (read.bytesRead === 0) break; + length += read.bytesRead; + } + const truncated = length > REPOSITORY_INSTRUCTION_MAX_BYTES; + const decoder = new TextDecoder('utf-8', { + fatal: true, + ignoreBOM: true, + }); + const content = decoder.decode( + buffer.subarray( + 0, + Math.min(length, REPOSITORY_INSTRUCTION_MAX_BYTES), + ), + { stream: truncated }, + ); + if (content.includes('\0')) return; + return { + descriptor: { + path, + bytes: Buffer.byteLength(content), + sha256: createHash('sha256').update(content).digest('hex'), + truncated, + }, + content, + }; + } catch { + return; + } finally { + await handle?.close().catch(() => undefined); + } +} diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 9199fcf0..c5353ae2 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -1,3 +1,5 @@ +import { createHash } from 'node:crypto'; + export const BRIDGE_PROTOCOL_VERSION = 1 as const; export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; export const BRIDGE_SANDBOX_PROFILE_MAX_LENGTH = 128; @@ -275,6 +277,7 @@ export type WorkspaceProgrammaticLanguage = 'bash'; export interface BridgeWorkspaceDescriptor { id: string; name?: string; + instructions?: RepositoryInstructionDescriptor[]; /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ operations?: BridgeWorkspaceToolOperation[]; environment?: { @@ -308,6 +311,27 @@ export interface WorkspaceReadFileRequest { path: string; startLine?: number; maxLines?: number; + /** Requests an exact bounded instruction snapshot, not a line-oriented file read. */ + instructionSha256?: string; +} + +export const REPOSITORY_INSTRUCTION_MAX_BYTES = 32 * 1024; +export interface RepositoryInstructionDescriptor { + path: 'AGENTS.md' | 'CLAUDE.md'; + /** Bytes in the bounded UTF-8 snapshot, whose digest is sha256. */ + bytes: number; + sha256: string; + truncated: boolean; +} + +export function isRepositoryInstructionDescriptor(value: unknown): value is RepositoryInstructionDescriptor { + if (value == null || typeof value !== 'object') return false; + const descriptor = value as Record; + return Object.keys(descriptor).every(key => ['path', 'bytes', 'sha256', 'truncated'].includes(key)) && + (descriptor.path === 'AGENTS.md' || descriptor.path === 'CLAUDE.md') && + Number.isSafeInteger(descriptor.bytes) && Number(descriptor.bytes) >= 0 && Number(descriptor.bytes) <= REPOSITORY_INSTRUCTION_MAX_BYTES && + typeof descriptor.sha256 === 'string' && /^[a-f0-9]{64}$/.test(descriptor.sha256) && + typeof descriptor.truncated === 'boolean'; } export interface WorkspaceReadFileResult { @@ -510,6 +534,7 @@ export type WorkspaceToolResult = | WorkspaceExecuteCommandResult; const WORKSPACE_READ_REQUEST_KEYS = new Set([ + 'instructionSha256', 'protocolVersion', 'operation', 'workspaceId', @@ -1139,6 +1164,12 @@ export function isWorkspaceToolRequest( return false; } if (request.operation === 'read_file') { + if (request.instructionSha256 !== undefined) { + return hasOnlyKeys(request, WORKSPACE_READ_REQUEST_KEYS) && + typeof request.instructionSha256 === 'string' && /^[a-f0-9]{64}$/.test(request.instructionSha256) && + (request.path === 'AGENTS.md' || request.path === 'CLAUDE.md') && + request.startLine === undefined && request.maxLines === undefined; + } return ( hasOnlyKeys(request, WORKSPACE_READ_REQUEST_KEYS) && isSafePortableRelativePath(request.path) && @@ -1282,6 +1313,13 @@ export function isWorkspaceToolResult( } if (request.operation === 'read_file') { + if (request.instructionSha256 !== undefined) { + return hasOnlyKeys(result, WORKSPACE_READ_RESULT_KEYS) && result.path === request.path && + typeof result.content === 'string' && new TextEncoder().encode(result.content).byteLength <= REPOSITORY_INSTRUCTION_MAX_BYTES && + createHash('sha256').update(result.content).digest('hex') === request.instructionSha256 && + result.startLine === 1 && result.endLine === result.content.split('\n').length && + result.nextStartLine === undefined; + } const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; const content = @@ -1574,11 +1612,13 @@ export function isValidBridgeWorkspaceToolCapabilities( key !== 'id' && key !== 'name' && key !== 'operations' && + key !== 'instructions' && key !== 'environment', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || + (descriptor.instructions !== undefined && (!Array.isArray(descriptor.instructions) || descriptor.instructions.length > 1 || !descriptor.instructions.every(isRepositoryInstructionDescriptor))) || (descriptor.environment !== undefined && !isValidCodeEnvironmentDescriptor(descriptor.environment)) || (descriptor.name !== undefined && diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 86e85872..697a3c55 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -23,6 +23,7 @@ import type { BridgeWorkerRegistrationResponse, BridgeWorkspaceToolOperation, BridgeWorkspaceProgrammaticRequest, + RepositoryInstructionDescriptor, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -37,6 +38,7 @@ export interface BridgeWorkerOptions { runtimeSupervisor?: RuntimeSupervisor; capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; + instructionDescriptors?: () => Promise | undefined>; workspaceProgrammatic?: { /** * True when a WorkspaceToolError without mutation uncertainty proves the @@ -396,6 +398,7 @@ export class BridgeWorker { private readonly compatibleCapabilities: BridgeWorkerCapabilities; private registrationCapabilities: BridgeWorkerCapabilities; private activeCapabilities: BridgeWorkerCapabilities; + private instructionMetadataSupported = true; private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; private maintenanceOnly = false; @@ -597,6 +600,8 @@ export class BridgeWorker { const registrationStartedAtMs = Date.now(); let registration: BridgeWorkerRegistrationResponse; try { + const instructions = this.instructionMetadataSupported ? await this.options.instructionDescriptors?.() : undefined; + let includeInstructions = this.instructionMetadataSupported; const register = (capabilities: BridgeWorkerCapabilities) => this.request( `${this.codeApiUrl}/bridge/workers/register`, @@ -609,12 +614,29 @@ export class BridgeWorker { ...capabilities, requiresReadyConfirmation: true, } - : capabilities, + : includeInstructions && instructions && capabilities.workspaceTools ? { + ...capabilities, + workspaceTools: { ...capabilities.workspaceTools, + workspaces: capabilities.workspaceTools.workspaces.map(workspace => ({ ...workspace, + ...((workspace.operations ?? capabilities.workspaceTools!.operations).includes('read_file') + ? { instructions: [...(instructions.get(workspace.id) ?? [])] } : {}), + })), + }, + } : capabilities, }, registrationController.signal, ); try { - registration = await register(this.registrationCapabilities); + try { + registration = await register(this.registrationCapabilities); + } catch (error) { + if (!(instructions && error instanceof BridgeProtocolError && error.status === 400)) { + throw error; + } + includeInstructions = false; + this.instructionMetadataSupported = false; + registration = await register(this.registrationCapabilities); + } } catch (error) { if ( !(error instanceof BridgeProtocolError) || diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 91e89f54..87b444e9 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -45,6 +45,7 @@ import type { } from './protocol.js'; export { isWorkspaceToolRequest, isWorkspaceToolResult }; +import { readRepositoryInstructions } from './instructions.js'; export type { WorkspaceReadFileRequest, WorkspaceReadFileResult, @@ -75,6 +76,7 @@ export interface LocalWorkspaceConfig { export interface LocalWorkspaceToolsOptions { workspaces: LocalWorkspaceConfig[]; + repositoryInstructions?: boolean; } export interface WorkspaceToolExecutor { @@ -1429,6 +1431,16 @@ async function withinListDeadline( export class LocalWorkspaceTools implements WorkspaceToolExecutor { readonly capabilities: BridgeWorkspaceToolCapabilities; readonly mutationFailuresAreAtomic = true as const; + private repositoryInstructions = false; + + async instructionDescriptors() { + if (!this.repositoryInstructions) return undefined; + const entries = await Promise.all([...this.roots].map(async ([id, { root }]) => { + const snapshot = await readRepositoryInstructions(root); + return [id, snapshot ? [snapshot.descriptor] : []] as const; + })); + return new Map(entries); + } private constructor( private readonly roots: ReadonlyMap, @@ -1508,7 +1520,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { writable: workspace.writable === true, }); } - return new LocalWorkspaceTools( + const tools = new LocalWorkspaceTools( roots, operations, workspaces, @@ -1517,6 +1529,8 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { capabilities.editFileFeatures, capabilities.listFileFeatures, ); + tools.repositoryInstructions = options.repositoryInstructions === true; + return tools; } async execute( @@ -1579,6 +1593,15 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ); } + if (request.instructionSha256 !== undefined) { + const snapshot = this.repositoryInstructions ? await readRepositoryInstructions(root) : undefined; + if (!snapshot || snapshot.descriptor.path !== request.path || snapshot.descriptor.sha256 !== request.instructionSha256) { + throw new WorkspaceToolError('Repository instructions changed or are unavailable', 'INVALID_PATH'); + } + return { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'read_file', workspaceId: request.workspaceId, + path: request.path, content: snapshot.content, startLine: 1, endLine: snapshot.content.split('\n').length, + truncated: snapshot.descriptor.truncated }; + } const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; if ( diff --git a/service/src/bridge/instructions.test.ts b/service/src/bridge/instructions.test.ts new file mode 100644 index 00000000..63edf614 --- /dev/null +++ b/service/src/bridge/instructions.test.ts @@ -0,0 +1,123 @@ +import { createServer } from 'node:http'; +import { mkdtemp, realpath, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { expect, test } from 'bun:test'; +import express from 'express'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { BridgeWorker } from '../../../packages/code/src/worker'; +import { LocalWorkspaceTools } from '../../../packages/code/src/workspace'; +import { applyPrincipal } from '../auth/principal'; +import { createWorkspaceToolsRouter } from '../workspace-tools/router'; +import { createBridgeRouter } from './router'; +import { RedisBridgeStore } from './store'; +import { RedisBridgePairingStore } from './pairing'; + +test('repository snapshots traverse the HTTP bridge and real workspace executor', async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'instruction-http-')), + ); + const redis = new RedisMock() as unknown as Redis; + const store = new RedisBridgeStore(redis); + const controller = new AbortController(); + const app = express(); + app.use(express.json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store, + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'local-test-only', + configuredWorkerId: 'instructions-worker', + }), + ); + app.use( + '/v1', + (req, _res, next) => { + applyPrincipal(req, { + userId: 'test-user', + tenantId: 'test-tenant', + principalSource: 'librechat_jwt', + }); + next(); + }, + createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'instructions-worker', + dynamicWorkers: false, + store, + }), + ); + const server = createServer(app); + let running: Promise | undefined; + try { + await new Promise(resolve => + server.listen(0, '127.0.0.1', resolve), + ); + const address = server.address(); + if (!address || typeof address === 'string') + throw new Error('Missing address'); + const base = `http://127.0.0.1:${address.port}/v1`; + await writeFile( + join(root, 'AGENTS.md'), + 'Exact\r\nrepository guidance\n', + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + repositoryInstructions: true, + }); + const worker = new BridgeWorker({ + codeApiUrl: base, + token: 'local-test-only', + workerId: 'instructions-worker', + sandboxEndpoint: 'http://127.0.0.1:1/api/v2', + leaseWaitMs: 50, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'test', + runtimes: [], + workspaceTools: tools.capabilities, + }, + workspaceTools: tools, + instructionDescriptors: () => tools.instructionDescriptors(), + }); + await worker.register(); + const status = await fetch( + `${base}/bridge/workers/instructions-worker/status`, + { headers: { Authorization: 'Bearer local-test-only' } }, + ).then(r => r.json()); + const descriptor = + status.capabilities.workspaceTools.workspaces[0].instructions[0]; + expect(descriptor.path).toBe('AGENTS.md'); + running = worker.run(controller.signal); + const request = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'AGENTS.md', + instructionSha256: descriptor.sha256, + }; + const read = () => + fetch(`${base}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + }); + const response = await read(); + expect(response.status).toBe(200); + expect((await response.json()).content).toBe( + 'Exact\r\nrepository guidance\n', + ); + await writeFile(join(root, 'AGENTS.md'), 'changed'); + expect((await read()).status).toBe(422); + } finally { + controller.abort(); + await running; + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + redis.disconnect(); + await rm(root, { recursive: true, force: true }); + } +}, 10000); From 95bfcbd1dc03cf21e98d70e168891bb4e18980a3 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 10:37:24 -0400 Subject: [PATCH 34/75] feat: Register Selected Coding Projects (#222) * feat(code): register explicitly selected project roots * fix(code): reject shared Git metadata for selected projects * fix(code): pin selected project identity through executor admission * fix(code): preserve full filesystem identity precision * Check selected project identity before replay staging * Anchor replay copies to the verified working directory * fix: Bind Selected Project Operations to Held Directory Descriptors * test: Cover Selected Project PTC and Load Native Fixtures Before Platform Simulation * fix: Keep Native Root Bindings Worker-Local and Verify Directory Ancestry * fix: Anchor Project Admission and Preserve Search Permissions --- .github/workflows/ci.yml | 5 + packages/code/README.md | 44 ++ packages/code/src/cli.ts | 34 +- packages/code/src/instructions.ts | 2 +- packages/code/src/native-process.ts | 2 + .../code/src/native-programmatic-live.test.ts | 10 +- packages/code/src/native-sandbox.test.ts | 114 +++++ packages/code/src/native-sandbox.ts | 51 +- packages/code/src/project-roots.test.ts | 383 +++++++++++++++ packages/code/src/project-roots.ts | 167 +++++++ packages/code/src/root-access.test.ts | 359 ++++++++++++++ packages/code/src/root-access.ts | 460 ++++++++++++++++++ packages/code/src/root-exec.ts | 18 + packages/code/src/root-identity.ts | 27 + packages/code/src/workspace.ts | 28 +- 15 files changed, 1691 insertions(+), 13 deletions(-) create mode 100644 packages/code/src/project-roots.test.ts create mode 100644 packages/code/src/project-roots.ts create mode 100644 packages/code/src/root-access.test.ts create mode 100644 packages/code/src/root-access.ts create mode 100644 packages/code/src/root-exec.ts create mode 100644 packages/code/src/root-identity.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07622ee7..72f32178 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,6 +197,11 @@ jobs: node-version: 24.16.0 - run: npm ci - run: npm run build + - name: Selected project root containment tests + run: | + command -v rg || brew install ripgrep + node --test dist/root-access.test.js + node --test --test-name-pattern='selected command|selected replay copy|programmatic probes reject' dist/native-sandbox.test.js - name: Native environment containment tests run: node --test dist/environment.test.js - name: Native ACL and credential lifecycle tests diff --git a/packages/code/README.md b/packages/code/README.md index 48bf769d..720a0b35 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -51,6 +51,50 @@ chosen non-overlapping project directories with `--workspace` or `--environment` Do not also register their parent directory. Treat the inventory as a snapshot; normal workspace admission must validate any directory selected from it. +## Register selected projects + +After pairing, use paths from `projects --root` to register individual checkouts: + +```bash +librechat-code run --project-root /srv/projects \ + --project web --project services/api \ + --allow-workspace-writes --allow-workspace-commands +``` + +Only the explicitly listed checkouts become execution roots. The discovery +directory is not registered, and adding a new sibling repository does not grant +access to it. In LibreChat, select the project in the existing workspace picker; +the conversation stores that selection for subsequent tools and approval resumes. +An agent's default workspace and the user's recent selection work as before. + +Project IDs are derived from the canonical discovery directory and relative +project path, not the branch or selection order. Keep both paths unchanged across +restarts to retain chat bindings. Moving a checkout changes its ID. These are +registration IDs, not the root-local IDs printed by the inventory command. + +Up to 32 selected projects are supported. Each must be a standalone Git checkout; +linked worktrees, symlink traversal, overlapping roots, and duplicate selections +are rejected. Existing native sandbox, command/write permissions, lease-slot and +quarantine rules still apply. This mode cannot be combined with `--environment`, +`--worker-dir`, `--workspace`, default-workspace, or workspace ID/name settings. +Existing registrations are not migrated automatically; use a new conversation +when switching registration mode. Non-Git directories still use the existing +workspace flags. Named environment setup/actions still use `--environment`. + +Selected projects require macOS or Linux (including WSL2). Each request opens +and verifies the admitted directory, then retains that descriptor through file +access, repository-instruction loading, command startup, and replay copying. +Renaming a project cannot redirect an in-flight request to a replacement checkout; +subsequent requests reject the changed identity. Restart with an explicitly +selected replacement to admit it. Descriptors close when requests settle, and +independent workspaces do not share a current directory or global execution lock. + +This reuses the existing workspace protocol. Programmatic tool calling requires +a LibreChat version that preserves the selected workspace across initial +execution and replay, plus the worker's normal programmatic prerequisites. +Installation alone does not restart workers or change registration; update your +worker service arguments explicitly. + ## Pair Hardened deployments use a one-time code instead of copying a long-lived diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 946b08d0..524c5853 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -6,6 +6,7 @@ import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { discoverProjects } from './projects.js'; +import { loadProjectRoots, projectRootArguments } from './project-roots.js'; import { loadCodeEnvironment, assertEnvironmentDefinitionsOutsideRoots, @@ -309,6 +310,26 @@ async function run( runtimeSessionId?: string, args: string[] = [], ): Promise { + const projectArgs = projectRootArguments(args); + if ( + projectArgs && + (runtimeSessionId != null || + args.some(arg => + ['--environment', '--worker-dir', '--default-workspace', + '--workspace', '--workspace-id', '--workspace-name'].some( + flag => arg === flag || arg.startsWith(`${flag}=`), + ), + ) || + [process.env.LIBRECHAT_CODE_WORKER_DIR, + process.env.LIBRECHAT_CODE_WORKSPACE_ID, + process.env.LIBRECHAT_CODE_WORKSPACE_NAME].some(value => value?.trim()) || + process.env.LIBRECHAT_CODE_DEFAULT_WORKSPACE?.trim().toLowerCase() === 'true') + ) { + throw new Error('Project selection cannot be combined with other workspace registration settings'); + } + const projectRoots = projectArgs + ? await loadProjectRoots(projectArgs.root, projectArgs.projects) + : []; const environmentPaths: string[] = []; for (let i = 0; i < args.length; i++) { if (args[i] === '--environment') { @@ -426,11 +447,13 @@ async function run( runtimeSessionId == null && (fileRelayUpstream?.length ?? 0) > 0; const workspaceId = + projectRoots[0]?.id ?? environments[0]?.definition.name ?? option(args, '--workspace-id') ?? process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? 'primary'; const explicitWorkerDirectory = + projectRoots[0]?.root ?? environments[0]?.definition.root ?? (runtimeSessionId == null ? nonEmpty( @@ -468,6 +491,9 @@ async function run( if (environments.length && commandSandboxMode !== 'native-srt') { throw new Error('Environment definitions require native-srt'); } + if (projectRoots.length && commandSandboxMode !== 'native-srt') { + throw new Error('Project selections require native-srt'); + } if ( environments.some(environment => environment.definition.setup) && !allowWorkspaceCommands @@ -575,8 +601,10 @@ async function run( { id: workspaceId, root: canonicalWorkerDirectory, + identity: projectRoots[0]?.identity, writable: allowWorkspaceWrites, name: + projectRoots[0]?.name ?? environments[0]?.definition.name ?? option(args, '--workspace-name') ?? process.env.LIBRECHAT_CODE_WORKSPACE_NAME?.trim() ?? @@ -597,6 +625,9 @@ async function run( writable: allowWorkspaceWrites, }); } + for (const project of projectRoots.slice(1)) { + roots.push({ ...project, writable: allowWorkspaceWrites }); + } await assertEnvironmentDefinitionsOutsideRoots(environments, roots); for (let i = 0; i < args.length; i++) { if ( @@ -882,6 +913,7 @@ async function run( }); const nativeOptions: NativeProcessSandboxOptions = { workspaceRoot: canonicalWorkerDirectory!, + workspaceIdentity: roots[0]?.identity, commandPolicy, protectedPaths: [ identityPath, @@ -921,7 +953,7 @@ async function run( new Map( roots.map(root => [ root.id, - { ...nativeOptions, workspaceRoot: root.root }, + { ...nativeOptions, workspaceRoot: root.root, workspaceIdentity: root.identity }, ]), ), workspaceLeaseSlots, diff --git a/packages/code/src/instructions.ts b/packages/code/src/instructions.ts index ee3e8127..1a1e8cad 100644 --- a/packages/code/src/instructions.ts +++ b/packages/code/src/instructions.ts @@ -1,5 +1,5 @@ import { constants } from 'node:fs'; -import { open, lstat, realpath, stat } from 'node:fs/promises'; +import { open, lstat, realpath, stat } from './root-access.js'; import { createHash } from 'node:crypto'; import { resolve, relative, isAbsolute, sep } from 'node:path'; diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 81bbd9c4..70cb2bd1 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -336,6 +336,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan child.on('disconnect', lost); const { workspaceRoot, + workspaceIdentity, commandPolicy, protectedPaths, allowedDomains, @@ -348,6 +349,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan { options: { workspaceRoot, + workspaceIdentity, commandPolicy, protectedPaths, allowedDomains, diff --git a/packages/code/src/native-programmatic-live.test.ts b/packages/code/src/native-programmatic-live.test.ts index 2bc22356..92cab6ea 100644 --- a/packages/code/src/native-programmatic-live.test.ts +++ b/packages/code/src/native-programmatic-live.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { createServer } from 'node:http'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { mkdtemp, readFile, realpath, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; @@ -8,17 +8,20 @@ import type { AddressInfo } from 'node:net'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { resolveNativeSrtCommandPolicy } from './native-policy.js'; -test('real SRT prevents speculative network effects under trusted-vm', { +for (const selected of [false, true]) { +test(`real SRT prevents speculative network effects under trusted-vm${selected ? ' for a selected project' : ''}`, { skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', timeout: 30_000, }, async () => { - const root = await mkdtemp(join(tmpdir(), 'native-ptc-effects-')); + const root = await realpath(await mkdtemp(join(tmpdir(), 'native-ptc-effects-'))); + const identity = await stat(root, { bigint: true }); let effects = 0; const server = createServer((_req, res) => { effects += 1; res.end('ok'); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); const port = (server.address() as AddressInfo).port; const executor = new NativeProcessWorkspaceCommandSandbox({ workspaceRoot: root, + ...(selected ? { workspaceIdentity: { path: root, dev: String(identity.dev), ino: String(identity.ino) } } : {}), commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), programmaticFileUpstream: `http://127.0.0.1:${port}`, }); @@ -39,3 +42,4 @@ test('real SRT prevents speculative network effects under trusted-vm', { } } }); +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 3252ee50..e610a0fc 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { EventEmitter } from 'node:events'; +import { mkdirSync, renameSync, writeFileSync } from 'node:fs'; import { access, chmod, @@ -236,6 +237,119 @@ test('programmatic probes use a copy-on-write workspace without mutating the pro assert.equal(await readFile(join(snapshot, 'state.txt'), 'utf8'), 'probe-only'); }); +test('selected command cwd stays bound when replacement happens while wrapping', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-project-command-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(await realpath(parent), 'project'); + await mkdir(root); + await writeFile(join(root, 'identity.txt'), 'original'); + const identity = await stat(root, { bigint: true }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + workspaceIdentity: { path: root, dev: String(identity.dev), ino: String(identity.ino) }, + manager: fakeManager({ beforeWrap: async () => { + await rename(root, `${root}.old`); + await mkdir(root); + await writeFile(join(root, 'identity.txt'), 'replacement'); + } }).manager, + }); + t.after(() => sandbox.close()); + const result = await sandbox.execute({ ...request, command: 'cat identity.txt; printf written > result.txt' }); + assert.equal(result.exitCode, 0, result.stderr); + assert.equal(result.stdout, 'original'); + assert.equal(await readFile(join(`${root}.old`, 'result.txt'), 'utf8'), 'written'); + await assert.rejects(access(join(root, 'result.txt'))); +}); + +test('selected command cancellation kills the exec trampoline process group', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-project-cancel-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = await realpath(parent); + const identity = await stat(root, { bigint: true }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + workspaceIdentity: { path: root, dev: String(identity.dev), ino: String(identity.ino) }, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const controller = new AbortController(); + const running = sandbox.execute({ ...request, timeoutMs: 5000, + command: 'printf started > started; sleep 3; printf late > late' }, controller.signal); + const rejected = assert.rejects(running, error => error instanceof WorkspaceToolError && error.code === 'EXECUTION_ABORTED'); + const deadline = Date.now() + 3000; + while (true) { + try { await access(join(root, 'started')); break; } catch { /* Wait for the actual child. */ } + if (Date.now() > deadline) throw new Error('Selected command did not start'); + await new Promise(resolve => setTimeout(resolve, 20)); + } + controller.abort(); + await rejected; + await new Promise(resolve => setTimeout(resolve, 3100)); + await assert.rejects(access(join(root, 'late'))); +}); + +test('selected replay copy stays on the verified directory after pathname replacement', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-project-copy-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(await realpath(parent), 'project'); + await mkdir(root); + await writeFile(join(root, 'identity.txt'), 'original'); + const identity = await stat(root, { bigint: true }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + workspaceIdentity: { path: root, dev: identity.dev.toString(), ino: identity.ino.toString() }, + manager: fakeManager().manager, + spawnCommand(command, args, options) { + assert.equal(command, process.execPath); + renameSync(root, `${root}.old`); + mkdirSync(root); + writeFileSync(join(root, 'identity.txt'), 'replacement'); + return spawn(command, args, options); + }, + }); + t.after(() => sandbox.close()); + const directory = await sandbox.createExecutionDirectory(); + let snapshot: string; + try { + snapshot = await sandbox.createProgrammaticProbeWorkspace(directory); + } catch (error) { + if (error instanceof CopyOnWriteCloneUnavailableError) { + t.skip('host filesystem does not support copy-on-write cloning'); + return; + } + throw error; + } + assert.equal(await readFile(join(root, 'identity.txt'), 'utf8'), 'replacement'); + assert.equal(await readFile(join(snapshot, 'identity.txt'), 'utf8'), 'original'); +}); + +test('programmatic probes reject a replaced selected project before copying', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-project-probe-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(await realpath(parent), 'project'); + await mkdir(root); + const identity = await stat(root, { bigint: true }); + let copies = 0; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + workspaceIdentity: { path: root, dev: identity.dev.toString(), ino: identity.ino.toString() }, + manager: fakeManager().manager, + spawnCommand() { + copies++; + throw new Error('must not copy a replaced project'); + }, + }); + t.after(() => sandbox.close()); + const executionDirectory = await sandbox.createExecutionDirectory(); + await rename(root, join(parent, 'original')); + await mkdir(root); + await assert.rejects( + sandbox.createProgrammaticProbeWorkspace(executionDirectory), + /Selected project changed before probe staging/, + ); + assert.equal(copies, 0); +}); + test('programmatic probes do not hide clone implementation failures as unsupported filesystems', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 550d0d52..84602a35 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -13,6 +13,9 @@ import { import { constants as fsConstants } from 'node:fs'; import { access, mkdtemp, open, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { withWorkspaceRoot, WorkspaceRootAccessError, spawnWithinWorkspace, realpath as rootedRealpath, stat as rootedStat } from './root-access.js'; import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; @@ -157,6 +160,7 @@ type SpawnCommand = ( ) => ChildProcessWithoutNullStreams; export interface NativeSrtWorkspaceCommandSandboxOptions { + workspaceIdentity?: WorkspaceRootIdentity; workspaceRoot: string; commandPolicy?: NativeSrtCommandPolicy; /** Trusted worker files that must never become workspace-readable or writable. */ @@ -343,6 +347,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const root = await realpath(this.options.workspaceRoot); + if (this.options.workspaceIdentity && !await matchesWorkspaceRoot(root, this.options.workspaceIdentity)) { + throw new WorkspaceToolError('Selected project changed before sandbox admission', 'REGISTRATION_INVALID'); + } if (!(await stat(root)).isDirectory()) { throw new WorkspaceToolError( 'Native sandbox workspace is unavailable', @@ -573,7 +580,20 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox executionDirectory: string, signal?: AbortSignal, ): Promise { + try { + return await withWorkspaceRoot(this.options.workspaceRoot, this.options.workspaceIdentity, + () => this.createBoundProgrammaticProbeWorkspace(executionDirectory, signal)); + } catch (error) { + if (error instanceof WorkspaceRootAccessError) throw new WorkspaceToolError('Selected project changed before probe staging', 'REGISTRATION_INVALID'); + throw error; + } + } + + private async createBoundProgrammaticProbeWorkspace(executionDirectory: string, signal?: AbortSignal): Promise { await this.initialize(); + if (this.options.workspaceIdentity && !await matchesWorkspaceRoot(this.options.workspaceRoot, this.options.workspaceIdentity)) { + throw new WorkspaceToolError('Selected project changed before probe staging', 'REGISTRATION_INVALID'); + } const scratchDirectory = this.scratchDirectory; const root = this.canonicalRoot; let parent: string; @@ -608,8 +628,11 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.platform === 'darwin' ? ['-cR', root, destination] : ['--archive', '--reflink=always', root, destination]; + const identity = this.options.workspaceIdentity; + const copyArgs = identity ? [...args.slice(0, -2), '.', destination] : args; await new Promise((resolveCopy, rejectCopy) => { - const child = this.spawnCommand('/bin/cp', args, { + const child = spawnWithinWorkspace(this.spawnCommand, '/bin/cp', copyArgs, { + ...(identity ? { cwd: root } : {}), env: { PATH: this.environment.PATH, LANG: this.environment.LANG, @@ -800,6 +823,26 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox sandboxScratchDirectory?: string, workspaceRoot?: string, ): Promise { + try { + return await withWorkspaceRoot(this.options.workspaceRoot, workspaceRoot ? undefined : this.options.workspaceIdentity, + () => this.executeBound(request, signal, trustedEnvironment, customConfig, sandboxScratchDirectory, workspaceRoot)); + } catch (error) { + if (error instanceof WorkspaceRootAccessError) throw new WorkspaceToolError(error.message, 'REGISTRATION_INVALID'); + throw error; + } + } + + private async executeBound( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, + trustedEnvironment?: NodeJS.ProcessEnv, + customConfig?: Partial, + sandboxScratchDirectory?: string, + workspaceRoot?: string, + ): Promise { + if (this.options.workspaceIdentity && !await matchesWorkspaceRoot(this.options.workspaceRoot, this.options.workspaceIdentity)) { + throw new WorkspaceToolError('Selected project changed after sandbox admission', 'REGISTRATION_INVALID'); + } if ( !isWorkspaceToolRequest(request) || request.operation !== 'execute_command' @@ -819,8 +862,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const root = workspaceRoot ?? this.canonicalRoot!; let cwd: string; try { - cwd = await realpath(resolve(root, request.cwd ?? '.')); - if (!isWithin(root, cwd) || !(await stat(cwd)).isDirectory()) + cwd = await rootedRealpath(resolve(root, request.cwd ?? '.')); + if (!isWithin(root, cwd) || !(await rootedStat(cwd)).isDirectory()) throw new Error('invalid cwd'); } catch { throw new WorkspaceToolError( @@ -939,7 +982,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox (resolvePromise, reject) => { let child: ChildProcessWithoutNullStreams; try { - child = this.spawnCommand( + child = spawnWithinWorkspace(this.spawnCommand, wrapped.argv[0], wrapped.argv.slice(1), { diff --git a/packages/code/src/project-roots.test.ts b/packages/code/src/project-roots.test.ts new file mode 100644 index 00000000..f88e523f --- /dev/null +++ b/packages/code/src/project-roots.test.ts @@ -0,0 +1,383 @@ +import assert from 'node:assert/strict'; +import { execFile, spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; +import { + mkdtemp, + mkdir, + readFile, + rename, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; +import { loadProjectRoots, projectRootArguments } from './project-roots.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import type { BridgeWorkerCapabilities } from './protocol.js'; +import { WorkspaceRootAccess } from './root-access.js'; + +const exec = promisify(execFile); +test('admission validates the captured root even while a replacement checkout occupies its path', async t => { + const root = await fixture(t); + const selected = join(root, 'app'); + await writeFile( + join(selected, '.git/commondir'), + '../../nested/api/.git\n', + ); + const originalOpen = WorkspaceRootAccess.open; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const held = await originalOpen(...args); + await rename(selected, `${selected}-original`); + await exec('git', ['init', '--initial-branch=dev', selected]); + const originalClose = held.close.bind(held); + held.close = async () => { + await rm(selected, { recursive: true, force: true }); + await rename(`${selected}-original`, selected); + await originalClose(); + }; + return held; + }, + ); + await assert.rejects( + loadProjectRoots(root, ['app']), + /Git common directory/, + ); + assert.equal( + await readFile(join(selected, '.git/commondir'), 'utf8'), + '../../nested/api/.git\n', + ); +}); + +test('admission cannot borrow a replacement checkout Git validity', async t => { + const root = await fixture(t); + const selected = join(root, 'app'); + await rm(join(selected, '.git/HEAD')); + const originalOpen = WorkspaceRootAccess.open; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const held = await originalOpen(...args); + await rename(selected, `${selected}-original`); + await exec('git', ['init', '--initial-branch=dev', selected]); + const originalClose = held.close.bind(held); + held.close = async () => { + await rm(selected, { recursive: true, force: true }); + await rename(`${selected}-original`, selected); + await originalClose(); + }; + return held; + }, + ); + await assert.rejects( + loadProjectRoots(root, ['app']), + /standalone Git checkout/, + ); +}); +test( + 'real worker CLI registers only explicitly selected project roots', + { timeout: 10000 }, + async t => { + const root = await fixture(t); + let accept: (capabilities: BridgeWorkerCapabilities) => void = () => {}; + const registered = new Promise(resolve => { + accept = resolve; + }); + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on('data', (chunk: Buffer) => chunks.push(chunk)); + request.on('end', () => { + response.setHeader('Content-Type', 'application/json'); + if (request.url?.endsWith('/bridge/workers/register')) { + const body = JSON.parse( + Buffer.concat(chunks).toString(), + ) as { + workerId: string; + incarnationId: string; + capabilities: BridgeWorkerCapabilities; + }; + accept(body.capabilities); + response.end( + JSON.stringify({ + protocolVersion: 1, + workerId: body.workerId, + incarnationId: body.incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + }), + ); + } else + response.end( + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + }), + ); + }); + }); + await new Promise(resolve => + server.listen(0, '127.0.0.1', resolve), + ); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const child = spawn( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--project-root', + root, + '--project', + 'app', + '--project', + 'nested/api', + ], + { + env: { + PATH: process.env.PATH, + LIBRECHAT_CODE_URL: `http://127.0.0.1:${address.port}/v1`, + LIBRECHAT_CODE_WORKER_ID: 'test-worker', + LIBRECHAT_CODE_WORKER_TOKEN: 'test-token', + }, + stdio: 'ignore', + }, + ); + t.after(() => { + if (child.exitCode === null) child.kill('SIGKILL'); + }); + const capabilities = await registered; + child.kill(); + await once(child, 'exit'); + assert.deepEqual( + capabilities.workspaceTools?.workspaces?.map( + workspace => workspace.name, + ), + ['app', 'nested/api'], + ); + assert.ok( + capabilities.workspaceTools?.workspaces?.every(workspace => + workspace.id.startsWith('project-'), + ), + ); + assert.equal(JSON.stringify(capabilities).includes(root), false); + }, +); + +async function fixture(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), 'selected-projects-')); + t.after(() => rm(root, { recursive: true, force: true })); + for (const name of ['app', 'nested/api']) { + await mkdir(join(root, name), { recursive: true }); + await exec('git', ['init', '--initial-branch=dev', join(root, name)]); + } + return root; +} + +test('selected projects retain identity across order and branch changes without granting their parent', async t => { + const root = await fixture(t); + const selected = await loadProjectRoots(root, ['app', 'nested/api']); + assert.deepEqual( + selected.map(project => project.name), + ['app', 'nested/api'], + ); + assert.ok( + selected.every(project => project.root !== root && !project.writable), + ); + await exec('git', [ + '-C', + join(root, 'app'), + 'symbolic-ref', + 'HEAD', + 'refs/heads/next', + ]); + const restarted = await loadProjectRoots(root, ['nested/api', 'app']); + assert.equal(restarted[1].id, selected[0].id); + assert.equal(restarted[0].id, selected[1].id); + const otherRoot = await fixture(t); + assert.notEqual( + (await loadProjectRoots(otherRoot, ['app']))[0].id, + selected[0].id, + ); +}); + +test('real file operations use the selected project boundary', async t => { + const root = await fixture(t); + const selected = await loadProjectRoots(root, ['app', 'nested/api']); + const tools = await LocalWorkspaceTools.create({ + workspaces: selected.map(project => ({ ...project, writable: true })), + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: selected[1].id, + path: 'created.txt', + content: 'second project', + }); + assert.equal( + await readFile(join(root, 'nested/api/created.txt'), 'utf8'), + 'second project', + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: selected[0].id, + path: '../nested/api/created.txt', + }), + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'nested/api/created.txt', + }), + ); + await assert.rejects(readFile(join(root, 'app/created.txt'))); +}); + +test('rejects missing, escaped, aliased, duplicate and linked-worktree selections', async t => { + const root = await fixture(t); + await mkdir(join(root, 'linked')); + await writeFile( + join(root, 'linked/.git'), + 'gitdir: ../app/.git/worktrees/linked\n', + ); + await symlink(join(root, 'app'), join(root, 'alias'), 'dir'); + for (const projects of [ + [], + ['..'], + ['/tmp'], + ['missing'], + ['alias'], + ['linked'], + ['app', './app'], + Array(33).fill('app'), + ]) { + await assert.rejects(loadProjectRoots(root, projects)); + } + await assert.rejects( + loadProjectRoots(root, ['.']), + /standalone Git checkout/, + ); +}); + +test('project CLI arguments require explicit bounded selections', () => { + assert.equal(projectRootArguments(['run']), undefined); + assert.deepEqual( + projectRootArguments([ + 'run', + '--project-root=/srv/projects', + '--project', + 'app', + '--project=nested/api', + ]), + { root: '/srv/projects', projects: ['app', 'nested/api'] }, + ); + for (const args of [ + ['--project-root'], + ['--project-root=/srv'], + ['--project=app'], + ['--project-root=/srv', '--project-root=/other', '--project=app'], + ['--project-root=/srv', '--project', '--allow-workspace-writes'], + ]) { + assert.throws(() => projectRootArguments(args)); + } +}); + +test('rejects a directory-form git marker redirecting to shared metadata', async t => { + const root = await fixture(t); + await writeFile( + join(root, 'app/.git/commondir'), + '../../nested/api/.git\n', + ); + await assert.rejects( + loadProjectRoots(root, ['app']), + /Git common directory/, + ); +}); + +test('replacement after selection cannot become a file or native execution root', async t => { + const root = await fixture(t); + const outside = await fixture(t); + const [selected] = await loadProjectRoots(root, ['app']); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ ...selected, writable: true }], + }); + await rename(selected.root, `${selected.root}-previous`); + await symlink(join(outside, 'app'), selected.root, 'dir'); + await assert.rejects( + LocalWorkspaceTools.create({ workspaces: [selected] }), + /Invalid workspace registration/, + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: selected.id, + path: 'escaped.txt', + content: 'blocked', + }), + /changed after admission/, + ); + const executor = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: selected.root, + workspaceIdentity: selected.identity, + }); + try { + await assert.rejects(executor.prepare()); + } finally { + await executor.close().catch(() => undefined); + } + await assert.rejects(readFile(join(outside, 'app/escaped.txt')), { + code: 'ENOENT', + }); +}); + +test('CLI rejects mixed registration and overlapping selected projects before connecting', async t => { + const root = await fixture(t); + await exec('git', ['init', '--initial-branch=dev', root]); + for (const extra of [ + ['--worker-dir', root], + ['--project', '.'], + ]) { + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--project-root', + root, + '--project', + 'app', + ...extra, + ], + { + encoding: 'utf8', + timeout: 5000, + env: { + PATH: process.env.PATH, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1', + LIBRECHAT_CODE_WORKER_ID: 'test-worker', + LIBRECHAT_CODE_WORKER_TOKEN: 'test-token', + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /cannot be combined|must not overlap/); + } +}); diff --git a/packages/code/src/project-roots.ts b/packages/code/src/project-roots.ts new file mode 100644 index 00000000..06e910ec --- /dev/null +++ b/packages/code/src/project-roots.ts @@ -0,0 +1,167 @@ +import { createHash } from 'node:crypto'; +import { lstat, realpath } from 'node:fs/promises'; +import { basename, isAbsolute, relative, resolve, sep } from 'node:path'; +import { + lstat as rootedLstat, + spawn, + withWorkspaceRoot, +} from './root-access.js'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { LocalWorkspaceConfig } from './workspace.js'; + +/** Validate the selected checkout itself, never rediscover it via its pathname. */ +async function validateCheckout(root: string): Promise { + const marker = await rootedLstat(resolve(root, '.git')).catch( + () => undefined, + ); + if (!marker?.isDirectory() || marker.isSymbolicLink()) + throw new Error( + 'Select a standalone Git checkout, not a parent directory or linked worktree', + ); + const common = await rootedLstat(resolve(root, '.git', 'commondir')).catch( + error => { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + throw error; + return undefined; + }, + ); + if (common) + throw new Error( + 'Selected projects must not share a Git common directory', + ); + await new Promise((accept, reject) => { + const child = spawn( + 'git', + [ + '--no-optional-locks', + '--git-dir=.git', + '--work-tree=.', + '-c', + 'core.fsmonitor=false', + 'rev-parse', + '--is-inside-work-tree', + ], + { + cwd: root, + env: { + PATH: process.env.PATH, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + LC_ALL: 'C', + }, + }, + ); + let output = ''; + let exceeded = false; + const timer = setTimeout(() => { + exceeded = true; + child.kill('SIGKILL'); + }, 1500); + child.stdout.on('data', (chunk: Buffer) => { + if (output.length + chunk.length > 4096) { + exceeded = true; + child.kill('SIGKILL'); + } else output += chunk.toString(); + }); + child.stderr.resume(); + child.stdin.end(); + child.once('error', reject); + child.once('close', code => { + clearTimeout(timer); + if (!exceeded && code === 0 && output.trim() === 'true') accept(); + else + reject( + new Error( + 'Select a standalone Git checkout, not a parent directory or linked worktree', + ), + ); + }); + }); +} + +/** Explicit operator selections, not an automatically expanding execution grant. */ +export async function loadProjectRoots( + directory: string, + selections: string[], +): Promise { + if (!selections.length || selections.length > 32) + throw new Error('Choose between 1 and 32 projects'); + const root = await realpath(directory); + const paths = new Set(); + const projects: LocalWorkspaceConfig[] = []; + for (const selection of selections) { + if (!selection || isAbsolute(selection) || selection.includes('\0')) + throw new Error('Project paths must be relative to --project-root'); + const path = resolve(root, selection); + const rel = relative(root, path); + if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) + throw new Error('Project paths must stay inside --project-root'); + const canonical = await realpath(path); + const directoryIdentity = await lstat(path, { bigint: true }); + if (canonical !== path || !directoryIdentity.isDirectory()) + throw new Error( + 'Selected projects must be directories without symlink traversal', + ); + if (paths.has(canonical)) + throw new Error('Duplicate project selection'); + paths.add(canonical); + const portablePath = rel.split(sep).join('/') || '.'; + const identity = { + path: canonical, + dev: directoryIdentity.dev.toString(), + ino: directoryIdentity.ino.toString(), + }; + await withWorkspaceRoot(canonical, identity, () => + validateCheckout(canonical), + ); + if (!(await matchesWorkspaceRoot(canonical, identity))) + throw new Error('Selected project changed during admission'); + projects.push({ + identity, + id: `project-${createHash('sha256') + .update(`${root}\0${portablePath}`) + .digest('hex') + .slice(0, 32)}`, + name: (portablePath === '.' ? basename(root) : portablePath).slice( + 0, + 64, + ), + root: canonical, + }); + } + return projects; +} + +export function projectRootArguments( + args: string[], +): { root: string; projects: string[] } | undefined { + let root: string | undefined; + const projects: string[] = []; + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + const flag = arg.split('=')[0]; + if (flag !== '--project-root' && flag !== '--project') continue; + const value = arg.includes('=') + ? arg.slice(flag.length + 1) + : args[++index]; + if (!value || value.startsWith('--')) + throw new Error(`${flag} requires a value`); + if (flag === '--project') projects.push(value); + else { + if (root !== undefined) + throw new Error('Only one --project-root may be supplied'); + root = value; + } + } + if (root === undefined && !projects.length) return undefined; + if (root === undefined || !projects.length) + throw new Error( + '--project-root requires at least one --project relative/path', + ); + return { root, projects }; +} diff --git a/packages/code/src/root-access.test.ts b/packages/code/src/root-access.test.ts new file mode 100644 index 00000000..bb757464 --- /dev/null +++ b/packages/code/src/root-access.test.ts @@ -0,0 +1,359 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { constants } from 'node:fs'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + WorkspaceRootAccess, + withWorkspaceRoot, + open, + realpath, + stat, + lstat, + rename, + link, + unlink, + spawn, +} from './root-access.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import type { WorkspaceToolRequest } from './protocol.js'; + +test('search-only directories support known files and command cwd without enumeration', async t => { + if (process.getuid?.() === 0) + return t.skip( + 'requires an unprivileged user to verify search permissions', + ); + const root = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-search-')), + ); + const nested = join(root, 'nested'); + await fs.mkdir(nested); + await fs.writeFile(join(nested, 'known'), 'known-value'); + const identity = await fs.stat(root, { bigint: true }); + t.after(async () => { + await fs.chmod(root, 0o700); + await fs.chmod(nested, 0o700); + await fs.rm(root, { recursive: true, force: true }); + }); + await fs.chmod(root, 0o111); + await fs.chmod(nested, 0o111); + await assert.rejects(fs.readdir(nested), { code: 'EACCES' }); + await withWorkspaceRoot( + root, + { path: root, dev: String(identity.dev), ino: String(identity.ino) }, + async () => { + const reader = await open(join(nested, 'known'), 'r'); + try { + assert.equal(await reader.readFile('utf8'), 'known-value'); + } finally { + await reader.close(); + } + assert.equal((await stat(nested)).isDirectory(), true); + assert.equal(await realpath(nested), nested); + await new Promise((accept, reject) => { + const child = spawn('/bin/cat', ['known'], { cwd: nested }); + let output = ''; + child.stdout!.on('data', chunk => { + output += chunk.toString(); + }); + child.once('error', reject); + child.once('close', code => { + try { + assert.equal(code, 0); + assert.equal(output, 'known-value'); + accept(); + } catch (error) { + reject(error); + } + }); + }); + }, + ); +}); + +test('held roots allow internal directory links and reject external ancestors', async t => { + const directory = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-links-')), + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const root = join(directory, 'root'); + await fs.mkdir(join(root, 'nested'), { recursive: true }); + await fs.mkdir(join(directory, 'outside')); + await fs.writeFile(join(root, 'nested', 'value'), 'inside'); + await fs.symlink('nested', join(root, 'inside')); + await fs.symlink('../outside', join(root, 'outside')); + const identity = await fs.stat(root, { bigint: true }); + const originalOpen = WorkspaceRootAccess.open; + let held: WorkspaceRootAccess | undefined; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + held = await originalOpen(...args); + return held; + }, + ); + await withWorkspaceRoot( + root, + { path: root, dev: String(identity.dev), ino: String(identity.ino) }, + async () => { + assert.equal( + (await lstat(join(root, 'inside'))).isSymbolicLink(), + true, + ); + const reader = await open(join(root, 'inside', 'value'), 'r'); + try { + assert.equal(await reader.readFile('utf8'), 'inside'); + } finally { + await reader.close(); + } + await assert.rejects( + open( + join(root, 'outside', 'escape'), + constants.O_CREAT | constants.O_WRONLY, + 0o600, + ), + { code: 'EACCES' }, + ); + }, + ); + assert.equal(held?.handle.fd, -1); + assert.deepEqual(await fs.readdir(join(directory, 'outside')), []); +}); + +test('held root file operations cannot be redirected by replacing its pathname', async () => { + const directory = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-access-')), + ); + const root = join(directory, 'project'); + await fs.mkdir(root); + await fs.writeFile(join(root, 'original'), 'original'); + const identity = await fs.stat(root, { bigint: true }); + try { + await withWorkspaceRoot( + root, + { + path: root, + dev: String(identity.dev), + ino: String(identity.ino), + }, + async () => { + await fs.rename(root, `${root}.old`); + await fs.mkdir(root); + await fs.writeFile(join(root, 'original'), 'replacement'); + assert.equal( + await realpath(join(root, 'original')), + join(root, 'original'), + ); + assert.equal((await stat(root)).isDirectory(), true); + assert.equal( + (await lstat(join(root, 'original'))).isFile(), + true, + ); + const reader = await open(join(root, 'original'), 'r'); + try { + assert.equal(await reader.readFile('utf8'), 'original'); + } finally { + await reader.close(); + } + const writer = await open( + join(root, 'new'), + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, + 0o600, + ); + try { + await writer.writeFile('new'); + await writer.sync(); + } finally { + await writer.close(); + } + await link(join(root, 'new'), join(root, 'linked')); + await rename(join(root, 'new'), join(root, 'renamed')); + await unlink(join(root, 'linked')); + const child = spawn('/bin/sh', ['-c', 'cat original'], { + cwd: root, + }); + let output = ''; + let error = ''; + child.stdout.on('data', chunk => { + output += chunk; + }); + child.stderr.on('data', chunk => { + error += chunk; + }); + child.stdin.end(); + const code = await new Promise(resolve => + child.on('close', resolve), + ); + assert.equal(code, 0, error); + assert.equal(output, 'original'); + }, + ); + assert.equal( + await fs.readFile(join(root, 'original'), 'utf8'), + 'replacement', + ); + assert.equal( + await fs.readFile(join(`${root}.old`, 'renamed'), 'utf8'), + 'new', + ); + assert.deepEqual(await fs.readdir(root), ['original']); + await assert.rejects( + WorkspaceRootAccess.open(root, { + path: root, + dev: String(identity.dev), + ino: String(identity.ino), + }), + ); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } +}); + +for (const operation of [ + 'read_file', + 'write_file', + 'edit_file', + 'list_files', + 'search_text', + 'instructions', +] as const) { + test(`selected ${operation} stays bound when the root is replaced after acquisition`, async t => { + const directory = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-caller-')), + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + await fs.mkdir(root); + await fs.writeFile(join(root, 'original.txt'), 'original'); + await fs.writeFile(join(root, 'AGENTS.md'), 'Original instructions'); + const identity = await fs.stat(root, { bigint: true }); + const tools = await LocalWorkspaceTools.create({ + repositoryInstructions: true, + workspaces: [ + { + id: 'selected', + root, + writable: true, + identity: { + path: root, + dev: String(identity.dev), + ino: String(identity.ino), + }, + }, + ], + }); + const originalOpen = WorkspaceRootAccess.open; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const access = await originalOpen(...args); + await fs.rename(root, `${root}.old`); + await fs.mkdir(root); + await fs.writeFile( + join(root, 'replacement.txt'), + 'replacement', + ); + await fs.writeFile( + join(root, 'AGENTS.md'), + 'Replacement instructions', + ); + return access; + }, + ); + if (operation === 'instructions') { + const descriptors = await tools.instructionDescriptors(); + const { createHash } = await import('node:crypto'); + assert.equal( + descriptors?.get('selected')?.[0].sha256, + createHash('sha256') + .update('Original instructions') + .digest('hex'), + ); + } else { + const request = { + protocolVersion: 1, + workspaceId: 'selected', + operation, + ...(operation === 'write_file' + ? { path: 'new.txt', content: 'created', overwrite: false } + : {}), + ...(operation === 'read_file' ? { path: 'original.txt' } : {}), + ...(operation === 'edit_file' + ? { + path: 'original.txt', + oldText: 'original', + newText: 'edited', + } + : {}), + ...(operation === 'search_text' ? { query: 'original' } : {}), + } as WorkspaceToolRequest; + const result = await tools.execute(request); + assert.equal( + JSON.stringify(result).includes('replacement.txt'), + false, + ); + if (operation === 'read_file') + assert.equal( + (result as { content: string }).content, + 'original', + ); + if (operation === 'write_file') + assert.equal( + await fs.readFile(join(`${root}.old`, 'new.txt'), 'utf8'), + 'created', + ); + if (operation === 'edit_file') + assert.equal( + await fs.readFile( + join(`${root}.old`, 'original.txt'), + 'utf8', + ), + 'edited', + ); + } + assert.deepEqual((await fs.readdir(root)).sort(), [ + 'AGENTS.md', + 'replacement.txt', + ]); + }); +} + +test('simultaneous roots retain independent descriptor contexts and release on failure', async t => { + const directory = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-concurrency-')), + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + await Promise.all( + ['a', 'b'].map(async name => { + const root = join(directory, name); + await fs.mkdir(root); + await fs.writeFile(join(root, 'value'), name); + const identity = await fs.stat(root, { bigint: true }); + await assert.rejects( + withWorkspaceRoot( + root, + { + path: root, + dev: String(identity.dev), + ino: String(identity.ino), + }, + async () => { + await fs.rename(root, `${root}.old`); + await fs.mkdir(root); + const file = await open(join(root, 'value'), 'r'); + try { + assert.equal(await file.readFile('utf8'), name); + } finally { + await file.close(); + } + throw new Error('cancelled request'); + }, + ), + /cancelled request/, + ); + }), + ); +}); diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts new file mode 100644 index 00000000..4aeb6100 --- /dev/null +++ b/packages/code/src/root-access.ts @@ -0,0 +1,460 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { spawn as spawnProcess } from 'node:child_process'; +import { constants, closeSync, fstatSync, readlinkSync } from 'node:fs'; +import * as fs from 'node:fs/promises'; +import { + basename, + dirname, + isAbsolute, + relative, + resolve, + sep, +} from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import type { + SpawnOptionsWithoutStdio, + ChildProcessWithoutNullStreams, +} from 'node:child_process'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +type NativeCall = (...args: (string | number | Buffer)[]) => number; +interface NativeLibrary { + func(signature: string): NativeCall; +} +interface NativeRuntime { + load(path: null): NativeLibrary; + errno(): number; + os: { errno: Record }; +} +let nativeRuntime: NativeRuntime | undefined; +let library: NativeLibrary | undefined; +function runtime(): NativeRuntime { + // Code API imports workspace contracts without installing native worker + // dependencies. Load the POSIX implementation only for selected roots. + return (nativeRuntime ??= createRequire(import.meta.url)( + 'koffi', + ) as NativeRuntime); +} +function bind(signature: string): NativeCall | undefined { + if (!['darwin', 'linux'].includes(process.platform)) return undefined; + let call: NativeCall | undefined; + return (...args) => { + library ??= runtime().load(null); + call ??= library.func(signature); + return call(...args); + }; +} +const nativeOpenAt = bind( + 'int openat(int dirfd, const char *path, int flags, ...)', +); +const O_CLOEXEC = process.platform === 'darwin' ? 0x1000000 : 0x80000; +// Anchors need search, not directory enumeration permission. +const DIRECTORY_ACCESS = + constants.O_DIRECTORY | + (process.platform === 'darwin' + ? 0x40000000 /* O_SEARCH */ + : 0x200000) /* O_PATH */; +const openAt = nativeOpenAt + ? (fd: number, path: string, flags: number, mode: number): number => + nativeOpenAt(fd, path, flags | O_CLOEXEC, 'unsigned int', mode) + : undefined; +const renameAt = bind( + 'int renameat(int fromfd, const char *from, int tofd, const char *to)', +); +const linkAt = bind( + 'int linkat(int fromfd, const char *from, int tofd, const char *to, int flags)', +); +const unlinkAt = bind('int unlinkat(int dirfd, const char *path, int flags)'); +const getPath = + process.platform === 'darwin' + ? bind('int fcntl(int fd, int command, ...)') + : undefined; + +function nativeError(): NodeJS.ErrnoException { + const errno = runtime().errno(); + const code = + Object.entries(runtime().os.errno).find( + ([, value]) => value === errno, + )?.[0] ?? 'EIO'; + return Object.assign( + new Error(`Workspace descriptor access failed: ${code}`), + { code }, + ); +} + +function checked(fd: number): number { + if (fd < 0) throw nativeError(); + return fd; +} + +function descriptorPath(fd: number): string { + return process.platform === 'linux' + ? `/proc/self/fd/${fd}` + : `/dev/fd/${fd}`; +} + +function physicalPath(fd: number): string { + if (process.platform === 'linux') return readlinkSync(descriptorPath(fd)); + const buffer = Buffer.alloc(1024); + if (!getPath || getPath(fd, 50 /* F_GETPATH */, 'void *', buffer) !== 0) + throw nativeError(); + return buffer.subarray(0, buffer.indexOf(0)).toString(); +} + +function offset(root: string, path: string): string { + const value = relative(root, path); + if (isAbsolute(value) || value === '..' || value.startsWith(`..${sep}`)) { + throw Object.assign(new Error('Path is outside the held workspace'), { + code: 'EACCES', + }); + } + return value || '.'; +} + +/** A request owns one directory descriptor, not a replaceable pathname grant. */ +export class WorkspaceRootAccessError extends Error {} +export class WorkspaceRootAccess { + private constructor( + readonly path: string, + readonly handle: fs.FileHandle, + ) {} + + static async open( + path: string, + identity: WorkspaceRootIdentity, + ): Promise { + if (!openAt || path !== identity.path) + throw new WorkspaceRootAccessError( + 'Selected project root access is unavailable', + ); + const handle = await fs + .open(path, DIRECTORY_ACCESS | constants.O_NOFOLLOW) + .catch(() => { + throw new WorkspaceRootAccessError( + 'Selected project changed after admission', + ); + }); + try { + const current = await handle.stat({ bigint: true }); + if ( + !current.isDirectory() || + current.dev.toString() !== identity.dev || + current.ino.toString() !== identity.ino + ) { + throw new WorkspaceRootAccessError( + 'Selected project changed after admission', + ); + } + return new WorkspaceRootAccess(path, handle); + } catch (error) { + await handle.close(); + throw error; + } + } + + close(): Promise { + return this.handle.close(); + } + + directory(path: string): number { + const fd = checked( + openAt!( + this.handle.fd, + offset(this.path, path), + DIRECTORY_ACCESS, + 0, + ), + ); + try { + this.assertDirectoryAncestor(fd); + this.canonical(fd); + return fd; + } catch (error) { + closeSync(fd); + throw error; + } + } + + private canonical(fd: number): string { + return resolve( + this.path, + offset(physicalPath(this.handle.fd), physicalPath(fd)), + ); + } + + /** Paths are presentation, not proof of ancestry: a renamed root can make + * an outside symlink target temporarily occupy its old textual prefix. */ + private assertDirectoryAncestor(fd: number): void { + const root = fstatSync(this.handle.fd, { bigint: true }); + let current = fd; + try { + for (let depth = 0; depth <= 128; depth++) { + const identity = fstatSync(current, { bigint: true }); + if (identity.dev === root.dev && identity.ino === root.ino) + return; + const parent = checked( + openAt!( + current, + '..', + DIRECTORY_ACCESS | constants.O_NOFOLLOW, + 0, + ), + ); + if (current !== fd) closeSync(current); + current = parent; + const ancestor = fstatSync(parent, { bigint: true }); + if ( + ancestor.dev === identity.dev && + ancestor.ino === identity.ino + ) + break; + } + throw Object.assign( + new Error( + 'Directory is outside the held workspace or exceeds its ancestry limit', + ), + { code: 'EACCES' }, + ); + } finally { + if (current !== fd) closeSync(current); + } + } + + private parent(path: string): { fd: number; name: string } { + const local = offset(this.path, path); + const fd = checked( + openAt!(this.handle.fd, dirname(local), DIRECTORY_ACCESS, 0), + ); + try { + this.assertDirectoryAncestor(fd); + this.canonical(fd); + return { fd, name: basename(local) }; + } catch (error) { + closeSync(fd); + throw error; + } + } + + async openFile( + path: string, + flags: number, + mode = 0o666, + ): Promise { + const parent = this.parent(path); + let fd: number | undefined; + try { + fd = checked(openAt!(parent.fd, parent.name, flags, mode)); + // Node owns the duplicate, so callers retain native FileHandle semantics + // for asynchronous I/O, fsync, ownership and deterministic close. + const accessMode = flags & (constants.O_WRONLY | constants.O_RDWR); + const duplicate = await fs.open( + descriptorPath(fd), + accessMode | constants.O_NONBLOCK, + ); + try { + const expected = fstatSync(fd, { bigint: true }); + const actual = await duplicate.stat({ bigint: true }); + if (expected.dev !== actual.dev || expected.ino !== actual.ino) + throw new Error('Descriptor duplication changed identity'); + return duplicate; + } catch (error) { + await duplicate.close(); + throw error; + } + } finally { + if (fd !== undefined) closeSync(fd); + closeSync(parent.fd); + } + } + + stat(path: string, follow = true): ReturnType { + const parent = this.parent(path); + let fd: number | undefined; + try { + const flags = + process.platform === 'linux' + ? 0x200000 /* O_PATH */ | + (follow ? 0 : constants.O_NOFOLLOW) + : 0x8000 /* O_EVTONLY */ | + (follow ? 0 : 0x200000); /* O_SYMLINK */ + fd = this.metadataDescriptor(parent.fd, parent.name, flags); + return fstatSync(fd); + } finally { + if (fd !== undefined) closeSync(fd); + closeSync(parent.fd); + } + } + + private metadataDescriptor( + parent: number, + name: string, + flags: number, + ): number { + if (process.platform === 'darwin') { + // O_EVTONLY still requests read permission on a directory. Search-only + // descriptors preserve known-path metadata/cwd access without enumeration. + const directory = openAt!( + parent, + name, + DIRECTORY_ACCESS | + (flags & 0x200000 /* O_SYMLINK */ + ? constants.O_NOFOLLOW + : 0), + 0, + ); + if (directory >= 0) return directory; + const error = nativeError(); + if (error.code !== 'ENOTDIR' && error.code !== 'ELOOP') throw error; + } + return checked(openAt!(parent, name, flags, 0)); + } + + realpath(path: string): string { + const parent = this.parent(path); + let fd: number | undefined; + try { + fd = this.metadataDescriptor( + parent.fd, + parent.name, + process.platform === 'linux' + ? 0x200000 /* O_PATH */ + : 0x8000 /* O_EVTONLY */, + ); + return this.canonical(fd); + } finally { + if (fd !== undefined) closeSync(fd); + closeSync(parent.fd); + } + } + + install(from: string, to: string, link: boolean): void { + const source = this.parent(from); + let target: ReturnType | undefined; + try { + target = this.parent(to); + const result = link + ? linkAt!(source.fd, source.name, target.fd, target.name, 0) + : renameAt!(source.fd, source.name, target.fd, target.name); + if (result !== 0) throw nativeError(); + } finally { + closeSync(source.fd); + if (target) closeSync(target.fd); + } + } + + unlink(path: string): void { + const parent = this.parent(path); + try { + if (unlinkAt!(parent.fd, parent.name, 0) !== 0) throw nativeError(); + } finally { + closeSync(parent.fd); + } + } +} + +const context = new AsyncLocalStorage(); +export async function withWorkspaceRoot( + root: string, + identity: WorkspaceRootIdentity | undefined, + action: () => Promise, +): Promise { + if (!identity) return action(); + const access = await WorkspaceRootAccess.open(root, identity); + try { + return await context.run(access, action); + } finally { + await access.close(); + } +} + +type SpawnCommand = ( + command: string, + args: string[], + options: SpawnOptionsWithoutStdio, +) => ChildProcessWithoutNullStreams; +export function spawnWithinWorkspace( + spawner: SpawnCommand, + command: string, + args: string[], + options: SpawnOptionsWithoutStdio, +): ChildProcessWithoutNullStreams { + const access = context.getStore(); + if (!access) return spawner(command, args, options); + if (typeof options.cwd !== 'string') + throw new Error('Workspace process requires a working directory'); + const fd = access.directory(options.cwd); + try { + const env = { ...(options.env ?? process.env) }; + delete env.NODE_OPTIONS; + delete env.NODE_PATH; + return spawner( + process.execPath, + [ + fileURLToPath(new URL('./root-exec.js', import.meta.url)), + command, + ...args, + ], + { + ...options, + cwd: '/', + env, + // The trusted bootstrap consumes fd 3 before exec. Commands receive no + // root descriptor, bridge socket or new long-lived supervising process. + stdio: [ + ...(Array.isArray(options.stdio) + ? options.stdio.slice(0, 3) + : ['pipe', 'pipe', 'pipe']), + fd, + ], + } as SpawnOptionsWithoutStdio, + ); + } finally { + closeSync(fd); + } +} + +export const spawn = (( + command: string, + args: string[], + options: SpawnOptionsWithoutStdio, +) => + spawnWithinWorkspace( + spawnProcess, + command, + args, + options, + )) as typeof spawnProcess; + +// Only workspace filesystem consumers import these adapters. Unselected legacy +// roots retain their existing behavior; concurrent selected roots never share fd state. +export const open = async ( + path: string, + flags: number | 'r', + mode?: number, +): Promise => + context + .getStore() + ?.openFile(path, flags === 'r' ? constants.O_RDONLY : flags, mode) ?? + fs.open(path, flags, mode); +export const stat = async (path: string) => + context.getStore()?.stat(path) ?? fs.stat(path); +export const lstat = async (path: string) => + context.getStore()?.stat(path, false) ?? fs.lstat(path); +export const realpath = async (path: string): Promise => + context.getStore()?.realpath(path) ?? fs.realpath(path); +export const rename = async (from: string, to: string): Promise => { + const access = context.getStore(); + if (access) access.install(from, to, false); + else await fs.rename(from, to); +}; +export const link = async (from: string, to: string): Promise => { + const access = context.getStore(); + if (access) access.install(from, to, true); + else await fs.link(from, to); +}; +export const unlink = async (path: string): Promise => { + const access = context.getStore(); + if (access) access.unlink(path); + else await fs.unlink(path); +}; diff --git a/packages/code/src/root-exec.ts b/packages/code/src/root-exec.ts new file mode 100644 index 00000000..4cd7132c --- /dev/null +++ b/packages/code/src/root-exec.ts @@ -0,0 +1,18 @@ +import koffi from 'koffi'; + +// Private exec trampoline. The parent supplies an already validated directory +// on fd 3. fchdir is process-local here and never changes the bridge's cwd. +const lib = koffi.load(null); +const fchdir = lib.func('int fchdir(int fd)'); +const close = lib.func('int close(int fd)'); +const execvp = lib.func('int execvp(const char *file, const char **argv)'); +const fcntl = lib.func('int fcntl(int fd, int command, ...)'); +const args = process.argv.slice(2); +if (args.length === 0 || fchdir(3) !== 0 || close(3) !== 0) process.exit(125); +// Node marks its standard streams close-on-exec during startup. Preserve only +// the three conventional streams; every internal descriptor stays closed. +for (const fd of [0, 1, 2]) { + if (fcntl(fd, 2 /* F_SETFD */, 'int', 0) !== 0) process.exit(125); +} +execvp(args[0], [...args, null]); +process.exit(126); diff --git a/packages/code/src/root-identity.ts b/packages/code/src/root-identity.ts new file mode 100644 index 00000000..3ddfe453 --- /dev/null +++ b/packages/code/src/root-identity.ts @@ -0,0 +1,27 @@ +import { lstat, realpath } from 'node:fs/promises'; + +export interface WorkspaceRootIdentity { + path: string; + dev: string; + ino: string; +} + +/** Revalidation of a trusted snapshot, never a fresh grant to a replacement. */ +export async function matchesWorkspaceRoot( + root: string, + identity: WorkspaceRootIdentity +): Promise { + if (root !== identity.path) return false; + try { + const current = await lstat(root, { bigint: true }); + return ( + current.isDirectory() && + !current.isSymbolicLink() && + current.dev.toString() === identity.dev && + current.ino.toString() === identity.ino && + (await realpath(root)) === identity.path + ); + } catch { + return false; + } +} diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 87b444e9..65757d39 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,10 +1,11 @@ -import { spawn } from 'node:child_process'; import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { link, lstat, open, realpath, rename, stat, unlink } from 'node:fs/promises'; +import { link, lstat, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; import { BRIDGE_PROTOCOL_VERSION, @@ -67,6 +68,7 @@ export type { }; export interface LocalWorkspaceConfig { + identity?: WorkspaceRootIdentity; id: string; name?: string; root: string; @@ -336,6 +338,7 @@ async function readConfinedFile( } interface WorkspaceRoot { + identity?: WorkspaceRootIdentity; root: string; writable: boolean; } @@ -1435,8 +1438,8 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { async instructionDescriptors() { if (!this.repositoryInstructions) return undefined; - const entries = await Promise.all([...this.roots].map(async ([id, { root }]) => { - const snapshot = await readRepositoryInstructions(root); + const entries = await Promise.all([...this.roots].map(async ([id, { root, identity }]) => { + const snapshot = await withWorkspaceRoot(root, identity, () => readRepositoryInstructions(root)); return [id, snapshot ? [snapshot.descriptor] : []] as const; })); return new Map(entries); @@ -1508,6 +1511,8 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { let canonicalRoot: string; try { canonicalRoot = await realpath(workspace.root); + if (workspace.identity && !await matchesWorkspaceRoot(canonicalRoot, workspace.identity)) throw new Error(); + await withWorkspaceRoot(canonicalRoot, workspace.identity, async () => undefined); if (!(await stat(canonicalRoot)).isDirectory()) throw new Error(); } catch { throw new WorkspaceToolError( @@ -1516,6 +1521,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ); } roots.set(workspace.id, { + identity: workspace.identity, root: canonicalRoot, writable: workspace.writable === true, }); @@ -1536,6 +1542,20 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { async execute( request: WorkspaceToolRequest, signal?: AbortSignal, + ): Promise { + const workspace = this.roots.get(request?.workspaceId); + if (!workspace?.identity) return this.executeBound(request, signal); + try { + return await withWorkspaceRoot(workspace.root, workspace.identity, () => this.executeBound(request, signal)); + } catch (error) { + if (error instanceof WorkspaceRootAccessError) throw new WorkspaceToolError(error.message, 'REGISTRATION_INVALID'); + throw error; + } + } + + private async executeBound( + request: WorkspaceToolRequest, + signal?: AbortSignal, ): Promise { if (signal?.aborted) { throw new WorkspaceToolError( From 672e19522900d1b78b4b32bd2e60587d8d3b1080 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 11:51:18 -0400 Subject: [PATCH 35/75] fix: Report workspace admission capacity without ambiguous timeout errors (#227) * fix: Distinguish workspace admission capacity from execution expiry * test: Preserve execution uncertainty while classifying blocked follow-ups * fix: Classify admission expiry at the enqueue boundary --- .github/workflows/ci.yml | 7 + packages/code/README.md | 13 ++ service/src/bridge/concurrent-worker.test.ts | 2 +- service/src/bridge/fleet.test.ts | 132 ++++++++++++++++++ service/src/bridge/store.ts | 22 ++- service/src/bridge/worker-admission.test.ts | 24 +++- .../src/sandbox-backend/remote-bridge.test.ts | 1 + service/src/sandbox-backend/remote-bridge.ts | 1 + service/src/workspace-tools/router.test.ts | 2 + service/src/workspace-tools/router.ts | 2 + 10 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 service/src/bridge/fleet.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72f32178..7259ec87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,13 @@ jobs: - name: Install Redis for ledger integration tests run: sudo apt-get update && sudo apt-get install -y redis-server + - name: Verify fleet admission with isolated Redis + run: | + redis_socket="$RUNNER_TEMP/byom-admission.sock" + redis-server --port 0 --unixsocket "$redis_socket" --save '' --appendonly no --daemonize yes + trap 'redis-cli -s "$redis_socket" shutdown nosave' EXIT + BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts + - name: Build service run: bun run build diff --git a/packages/code/README.md b/packages/code/README.md index 720a0b35..08dc54a7 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -692,6 +692,19 @@ librechat-code run \ --allow-workspace-commands ``` +Slots are per machine, not a fleet-wide execution limit. A busy machine does not +consume another machine's slots. Requests for the same root remain serialized, +including commands started through background tools. Independent checkouts can +use different slots; selecting subdirectories beneath one registered parent root +does not create separate scheduling boundaries. Linked Git worktrees share Git +metadata and are not supported by selected-project registration. + +Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP +503, `Retry-After: 1`) means the operation was not assigned or started; wait for +capacity before submitting it again. This is distinct from `ASSIGNMENT_EXPIRED` +or a transport timeout after dispatch, where execution may have occurred and +mutations must not be blindly retried. No automatic retry is added by this policy. + Keep the existing URL, pairing/identity, and network policy configuration. The primary root keeps its configured workspace ID (default `primary`). Repeat `--workspace id=path` to add named roots, up to the protocol's 32-root limit. diff --git a/service/src/bridge/concurrent-worker.test.ts b/service/src/bridge/concurrent-worker.test.ts index ea5ac601..c107576a 100644 --- a/service/src/bridge/concurrent-worker.test.ts +++ b/service/src/bridge/concurrent-worker.test.ts @@ -281,7 +281,7 @@ for (const failure of [ ).rejects.toMatchObject({ code: failure === 'delivery-outage' - ? 'ASSIGNMENT_EXPIRED' + ? 'WORKSPACE_QUEUE_TIMEOUT' : 'WORKSPACE_QUARANTINED', }); await expect( diff --git a/service/src/bridge/fleet.test.ts b/service/src/bridge/fleet.test.ts new file mode 100644 index 00000000..3eac5088 --- /dev/null +++ b/service/src/bridge/fleet.test.ts @@ -0,0 +1,132 @@ +import { expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import { randomUUID } from 'node:crypto'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment } from './store'; + +/** Opt-in integration check against a disposable Redis, never a deployment database. */ +test.skipIf(!process.env.BRIDGE_TEST_REDIS_URL)( + 'admission saturation is isolated across machines and independent roots', + async () => { + const redis = new Redis(process.env.BRIDGE_TEST_REDIS_URL!); + const store = new RedisBridgeStore(redis, 60, 1000, 2); + const prefix = `fleet-${randomUUID()}`; + const machines = [`${prefix}-a`, `${prefix}-b`]; + const incarnationId = 'fleet-incarnation'; + const pending: Promise[] = []; + const controller = new AbortController(); + const dispatch = ( + workerId: string, + workspaceId: string, + budgetMs = 3000, + ) => { + const result = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + budgetMs, + executionTimeoutMs: 5000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId, + path: 'probe', + }, + }); + void result.catch(() => undefined); + pending.push(result); + return result; + }; + const settle = async (assignment: CodeBridgeAssignment) => { + await store.acknowledgeLease( + assignment.workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + await store.settle(assignment.workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'probe complete', + }); + }; + try { + for (const workerId of machines) { + const generation = await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 2, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'a' }, { id: 'b' }], + }, + }, + }); + await store.confirmReady(workerId, incarnationId, generation); + } + const busy = dispatch(machines[0], 'a'); + const held = await store.lease( + machines[0], + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + expect(held).toBeDefined(); + const blocked = dispatch(machines[0], 'a', 300); + const independent = dispatch(machines[0], 'b'); + const otherMachine = dispatch(machines[1], 'a'); + const root = await store.lease( + machines[0], + incarnationId, + 1000, + undefined, + undefined, + 1, + ); + const remote = await store.lease( + machines[1], + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + expect(root?.request).toMatchObject({ workspaceId: 'b' }); + expect(remote?.workerId).toBe(machines[1]); + await settle(root!); + await settle(remote!); + await Promise.all([independent, otherMachine]); + await expect(blocked).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + await settle(held!); + await busy; + expect( + await store.lease( + machines[0], + incarnationId, + 20, + undefined, + undefined, + 0, + ), + ).toBeUndefined(); + } finally { + controller.abort(); + await Promise.allSettled(pending); + await redis.quit(); + } + }, +); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index d6b91469..4eaabd7f 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -47,6 +47,7 @@ export class BridgeStoreError extends Error { | 'WORKER_UNAUTHORIZED' | 'WORKER_BUSY' | 'WORKER_QUEUE_FULL' + | 'WORKSPACE_QUEUE_TIMEOUT' | 'ASSIGNMENT_EXPIRED' | 'ASSIGNMENT_FENCED' | 'ASSIGNMENT_NOT_FOUND' @@ -838,6 +839,7 @@ export class RedisBridgeStore { ); const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; + let enqueueAttempted = false; let workspaceLeaseSlot: number | undefined; const selectedWorkspaceId = args.workspaceRequest?.workspaceId ?? args.workspaceId; @@ -1024,12 +1026,14 @@ export class RedisBridgeStore { this.assertDispatchActive(args.signal, args.deadlineAtMs); assignment.incarnationId = registration.incarnationId; queued = await this.dispatchCommand( - () => - this.enqueueForActiveIncarnation( + () => { + enqueueAttempted = true; + return this.enqueueForActiveIncarnation( assignment!, ttlSeconds, readyToken, - ), + ); + }, args, 'Bridge assignment enqueue', ); @@ -1130,6 +1134,18 @@ export class RedisBridgeStore { } throw error; } + } catch (error) { + // Once enqueue starts, even a lost Redis response may hide execution. + if ( + admission != null && !enqueueAttempted && !args.signal.aborted && + error instanceof BridgeStoreError && error.code === 'ASSIGNMENT_EXPIRED' + ) { + throw new BridgeStoreError( + 'WORKSPACE_QUEUE_TIMEOUT', + 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', + ); + } + throw error; } finally { if (admission != null) { // Expiry remains the fallback if Redis is unavailable during cancellation. diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index db2e5956..655748b1 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -1,4 +1,4 @@ -import { afterEach, expect, test } from 'bun:test'; +import { afterEach, expect, spyOn, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; @@ -114,7 +114,7 @@ test('an expired queued call never reaches the worker and does not strand later const assignment = await store.lease(workerId, incarnationId, 1000); await expect( dispatch('expired', new AbortController(), 25, 1000), - ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + ).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); const third = dispatch('third'); await settle(assignment); await first; @@ -170,3 +170,23 @@ test('execution expires independently of an unused queue allowance', async () => expect(Date.parse(assignment!.expiresAt) - Date.now()).toBeLessThanOrEqual(150); await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); }); + +test('expiry after generation allocation but before enqueue is definitely not started', async () => { + await register(); + const now = Date.now; + const incr = redis.incr.bind(redis); + let expired = false; + const clock = spyOn(Date, 'now').mockImplementation(() => now() + (expired ? 10_000 : 0)); + const generation = spyOn(redis, 'incr').mockImplementation(async (key) => { + const value = await incr(key); + expired = true; + return value; + }); + try { + await expect(dispatch('not-enqueued')).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); + } finally { + clock.mockRestore(); + generation.mockRestore(); + } + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +}); diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index 697271ee..4f50898e 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -108,6 +108,7 @@ describe('RemoteBridgeSandboxBackend', () => { WORKER_UNAUTHORIZED: ['BRIDGE_WORKER_UNAUTHORIZED', false, 403, 'Code environment is not authorized for this tenant'], WORKER_BUSY: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], WORKER_QUEUE_FULL: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], + WORKSPACE_QUEUE_TIMEOUT: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], ASSIGNMENT_EXPIRED: ['BRIDGE_DEADLINE_EXCEEDED', false, 504, 'Code environment execution timed out'], ASSIGNMENT_FENCED: ['BRIDGE_ASSIGNMENT_FENCED', false, 409, 'Code environment assignment is fenced; inspect the execution before retrying'], ASSIGNMENT_NOT_FOUND: ['BRIDGE_ASSIGNMENT_NOT_FOUND', false, 409, 'Code environment assignment is no longer available; inspect the execution before retrying'], diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index 6e06eda8..2cdbd767 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -19,6 +19,7 @@ const bridgeErrorCodes = { WORKER_UNAUTHORIZED: 'BRIDGE_WORKER_UNAUTHORIZED', WORKER_BUSY: 'BRIDGE_WORKER_BUSY', WORKER_QUEUE_FULL: 'BRIDGE_WORKER_BUSY', + WORKSPACE_QUEUE_TIMEOUT: 'BRIDGE_WORKER_BUSY', ASSIGNMENT_EXPIRED: 'BRIDGE_DEADLINE_EXCEEDED', ASSIGNMENT_FENCED: 'BRIDGE_ASSIGNMENT_FENCED', ASSIGNMENT_NOT_FOUND: 'BRIDGE_ASSIGNMENT_NOT_FOUND', diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index ae291a4b..04738b8a 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -320,6 +320,7 @@ test.each([ ['ASSIGNMENT_EXPIRED', 504], ['WORKER_OFFLINE', 503], ['WORKER_BUSY', 503], + ['WORKSPACE_QUEUE_TIMEOUT', 503], ['WORKER_MISMATCH', 409], ] as const)('logs store rejection %s with actual HTTP %i', async (errorCode, expectedStatus) => { const app = express(); @@ -360,6 +361,7 @@ test.each([ }), }); expect(response.status).toBe(expectedStatus); + expect(response.headers.get('retry-after')).toBe(errorCode === 'WORKSPACE_QUEUE_TIMEOUT' ? '1' : null); await response.text(); expect(logSpy).toHaveBeenCalledTimes(1); expect(logSpy).toHaveBeenCalledWith( diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 17085963..eb89370e 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -36,6 +36,7 @@ function asyncRoute(handler: (req: AuthenticatedRequest, res: Response) => Promi } export function bridgeStoreStatus(error: BridgeStoreError): number { + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') return 503; if (error.code === 'WORKER_QUEUE_FULL') return 429; if (error.code === 'WORKER_UNAUTHORIZED') return 403; if (error.code === 'ASSIGNMENT_INVALID') return 400; @@ -172,6 +173,7 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) } catch (error) { if (error instanceof BridgeStoreError) { outcome.errorCode = error.code; + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); res.status(bridgeStoreStatus(error)).json({ error: error.message, code: error.code, From fd9a4fa65e0a5189957032c0046eb286311fda62 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:33:43 -0400 Subject: [PATCH 36/75] ci: Fix Release Version Resolution for Untagged and Resumed Runs (#233) * ci: Fix Release Version Resolution for Untagged and Resumed Runs The release workflow resolved its version in one inline shell block under `set -euo pipefail`, where two paths could not succeed. Filtering tags through `grep` made a no-match fatal. On the ordinary untagged tip of `main`, `git tag --points-at HEAD | grep -E '^v[0-9]+...'` exits 1, and the step died before reaching its skip handling or `next-release-version.sh`, so a deployable commit could not obtain a release version (#228). Selecting stable tags now reads exit 1 as an empty answer while exit 2 and above still fail the release, which also lets the missing-previous-tag case report its own error. The rerun-resume path then rejected the tag it had itself chosen. With a stable tag already pointing at `HEAD` and no release published, the version comes from that tag, and the following existence check failed merely because the ref existed (#229). It now compares the tag's commit against the release commit, so only a tag on some other commit is a collision; `Create tag` already tolerates a tag that exists. The block moved into `.github/scripts/resolve-release-version.sh`, beside the `next-release-version.sh` it calls, so `tests/release-version-resolution.sh` can cover every path: automatic, resumed, skipped, dispatched, pushed-tag, and the runs that must be refused, each against a throwaway repository with a stubbed `gh`. * fix: harden release resolver execution --------- Co-authored-by: Lia Co-authored-by: Danny Avila --- .github/scripts/resolve-release-version.sh | 195 +++++++++++++++ .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 153 ++---------- tests/release-version-resolution.sh | 265 +++++++++++++++++++++ 4 files changed, 483 insertions(+), 133 deletions(-) create mode 100755 .github/scripts/resolve-release-version.sh create mode 100755 tests/release-version-resolution.sh diff --git a/.github/scripts/resolve-release-version.sh b/.github/scripts/resolve-release-version.sh new file mode 100755 index 00000000..ef5f8667 --- /dev/null +++ b/.github/scripts/resolve-release-version.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +# Resolves which version a release run publishes, or decides that it publishes +# nothing, and records the decision in $GITHUB_OUTPUT. Extracted from +# .github/workflows/release.yml so the four entry paths — an automatic release +# after CI, a rerun resuming a release whose tag was already cut, a dispatch, +# and a pushed tag — are covered by tests/release-version-resolution.sh. +# +# Inputs arrive as environment variables, mirroring the workflow's env block: +# +# EVENT_NAME github.event_name +# HEAD_SHA github.event.workflow_run.head_sha +# INPUT_VERSION github.event.inputs.version +# INPUT_DRAFT github.event.inputs.draft +# REF_NAME github.ref_name +# REF_TYPE github.ref_type +# GH_TOKEN a token `gh release view` can read releases with +# +# Outputs: skip, and for a real release version, app_version, chart_version, +# prerelease, latest, draft. Run from the repository root; the chart fields are +# read from helm/codeapi/Chart.yaml relative to it. + +set -euo pipefail + +EVENT_NAME="${EVENT_NAME:-}" +HEAD_SHA="${HEAD_SHA:-}" +INPUT_VERSION="${INPUT_VERSION:-}" +INPUT_DRAFT="${INPUT_DRAFT:-}" +REF_NAME="${REF_NAME:-}" +REF_TYPE="${REF_TYPE:-}" +GITHUB_OUTPUT="${GITHUB_OUTPUT:?GITHUB_OUTPUT must name the step output file}" + +STABLE_TAG_PATTERN='^v[0-9]+[.][0-9]+[.][0-9]+$' + +# `grep` exits 1 when nothing matches, and under `pipefail` that would abort the +# step. A commit with no stable tag is the ordinary state of `main`, so a +# no-match reads as an empty answer while a genuine grep failure — exit 2 and +# above — still fails the release. +select_stable_tags() { + local status=0 + grep -E "$STABLE_TAG_PATTERN" || status=$? + [ "$status" -le 1 ] +} + +# Highest stable tag among those `git tag` selects, empty when there are none. +newest_stable_tag() { + git tag "$@" | select_stable_tags | sort -V | tail -n 1 +} + +# The commit a tag resolves to. The caller first verifies that the ref exists, +# so a failure here means the tag ultimately names a non-commit object. +tag_commit() { + git rev-parse -q --verify "refs/tags/$1^{commit}" +} + +SKIP=false +VERSION="" +HEAD_COMMIT="" + +if [ "$EVENT_NAME" = "workflow_run" ]; then + HEAD_COMMIT="$(git rev-parse HEAD)" + if [ "$HEAD_COMMIT" != "$HEAD_SHA" ]; then + echo "::error::Checked out SHA does not match the successful CI run" + exit 1 + fi + + REMOTE_MAIN_SHA="$(git ls-remote origin refs/heads/main | awk '{print $1}')" + if [ -z "$REMOTE_MAIN_SHA" ]; then + echo "::error::Could not resolve the current main branch tip" + exit 1 + fi + if [ "$REMOTE_MAIN_SHA" != "$HEAD_SHA" ]; then + echo "main advanced after this CI run; the newer successful run will release the combined changes" + SKIP=true + fi + + # A rerun after tag creation but before release publication resumes the + # missing release rather than incrementing the version again. + EXACT_TAG="$(newest_stable_tag --points-at HEAD)" + if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then + if gh release view "$EXACT_TAG" >/dev/null 2>&1; then + echo "$EXACT_TAG already publishes this commit; nothing to do" + SKIP=true + else + VERSION="$EXACT_TAG" + fi + elif [ "$SKIP" = "false" ]; then + PREVIOUS_TAG="$(newest_stable_tag --merged HEAD)" + if [ -z "$PREVIOUS_TAG" ]; then + echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" + exit 1 + fi + VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" + if [ -z "$VERSION" ]; then + echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" + SKIP=true + fi + fi +elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then + # Releases describe what shipped to main. Dispatching from a topic branch + # would tag a commit that is not on the release line. + if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then + echo "::error::Releases must be cut from main; this run is on '$REF_NAME'" + exit 1 + fi + VERSION="$INPUT_VERSION" +else + VERSION="$REF_NAME" +fi + +if [ "$SKIP" = "true" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 +fi + +# A bare "2.0.0" typed into the dispatch box is accepted; everything downstream +# works with the v-prefixed form the tag actually uses. +case "$VERSION" in + v*) ;; + *) VERSION="v$VERSION" ;; +esac + +if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then + echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" + exit 1 +fi + +if [ "$EVENT_NAME" = "workflow_run" ]; then + # A tag already pointing at this commit is the resumed release above, and the + # publish steps tolerate it. Only a tag on some other commit is a collision. + if git show-ref --verify --quiet "refs/tags/$VERSION"; then + if ! EXISTING_TAG_COMMIT="$(tag_commit "$VERSION")"; then + echo "::error::Calculated tag $VERSION already exists but does not point to a commit" + exit 1 + fi + if [ "$EXISTING_TAG_COMMIT" != "$HEAD_COMMIT" ]; then + echo "::error::Calculated tag $VERSION already exists on a different commit" + exit 1 + fi + fi +fi + +read_chart_field() { + grep -m1 "^$1:" helm/codeapi/Chart.yaml \ + | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]*#.*//; s/^[\"']//; s/[\"']\$//" +} +APP_VERSION="$(read_chart_field appVersion)" +CHART_VERSION="$(read_chart_field version)" + +if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." + exit 1 +fi + +case "$VERSION" in + *-rc*) PRERELEASE=true ;; + *) PRERELEASE=false ;; +esac + +# `latest` moves only when this is the highest stable version, so re-cutting an +# older patch cannot drag it backwards. The tag under dispatch does not exist +# yet, hence adding it to the comparison. +LATEST=false +if [ "$PRERELEASE" = "false" ]; then + HIGHEST_STABLE="$( + { + git tag --list 'v[0-9]*' + printf '%s\n' "$VERSION" + } \ + | select_stable_tags \ + | sort -V \ + | tail -n 1 + )" + if [ "$HIGHEST_STABLE" = "$VERSION" ]; then + LATEST=true + fi +fi + +DRAFT=false +if [ "$INPUT_DRAFT" = "true" ]; then + DRAFT=true +fi + +{ + echo "skip=false" + echo "version=$VERSION" + echo "app_version=$APP_VERSION" + echo "chart_version=$CHART_VERSION" + echo "prerelease=$PRERELEASE" + echo "latest=$LATEST" + echo "draft=$DRAFT" +} >> "$GITHUB_OUTPUT" + +echo "Releasing $VERSION (chart $CHART_VERSION, appVersion $APP_VERSION, prerelease=$PRERELEASE, latest=$LATEST, draft=$DRAFT)" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7259ec87..b645f4b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,9 @@ jobs: - name: Release versioning run: tests/release-versioning.sh + - name: Release version resolution + run: tests/release-version-resolution.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea8a5364..684f1654 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,6 +61,20 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: + # `workflow_run` can be rerun for a commit older than this resolver. Save + # the helper from the revision that supplied this workflow before the + # release checkout replaces the working tree with that historical SHA. + - name: Checkout release workflow + if: github.event_name == 'workflow_run' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 1 + ref: ${{ github.workflow_sha }} + + - name: Preserve release resolver + if: github.event_name == 'workflow_run' + run: install -m 755 .github/scripts/resolve-release-version.sh "$RUNNER_TEMP/resolve-release-version.sh" + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: # Full history and tags: resolving whether this release is the newest @@ -78,139 +92,12 @@ jobs: REF_NAME: ${{ github.ref_name }} REF_TYPE: ${{ github.ref_type }} GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - SKIP=false - if [ "$EVENT_NAME" = "workflow_run" ]; then - if [ "$(git rev-parse HEAD)" != "$HEAD_SHA" ]; then - echo "::error::Checked out SHA does not match the successful CI run" - exit 1 - fi - - REMOTE_MAIN_SHA="$(git ls-remote origin refs/heads/main | awk '{print $1}')" - if [ -z "$REMOTE_MAIN_SHA" ]; then - echo "::error::Could not resolve the current main branch tip" - exit 1 - fi - if [ "$REMOTE_MAIN_SHA" != "$HEAD_SHA" ]; then - echo "main advanced after this CI run; the newer successful run will release the combined changes" - SKIP=true - fi - - # A rerun after tag creation but before release publication resumes - # the missing release rather than incrementing the version again. - EXACT_TAG="$({ git tag --points-at HEAD || true; } | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' | sort -V | tail -n 1)" - if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then - if gh release view "$EXACT_TAG" >/dev/null 2>&1; then - echo "$EXACT_TAG already publishes this commit; nothing to do" - SKIP=true - else - VERSION="$EXACT_TAG" - fi - elif [ "$SKIP" = "false" ]; then - PREVIOUS_TAG="$(git tag --merged HEAD \ - | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ - | sort -V \ - | tail -n 1)" - if [ -z "$PREVIOUS_TAG" ]; then - echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" - exit 1 - fi - VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" - if [ -z "$VERSION" ]; then - echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" - SKIP=true - fi - fi - elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then - # Releases describe what shipped to main. Dispatching from a topic - # branch would tag a commit that is not on the release line. - if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then - echo "::error::Releases must be cut from main; this run is on '$REF_NAME'" - exit 1 - fi - VERSION="$INPUT_VERSION" - else - VERSION="$REF_NAME" - fi - - if [ "$SKIP" = "true" ]; then - echo "skip=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # A bare "2.0.0" typed into the dispatch box is accepted; everything - # downstream works with the v-prefixed form the tag actually uses. - case "$VERSION" in - v*) ;; - *) VERSION="v$VERSION" ;; - esac - - if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then - echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" - exit 1 - fi - - if [ "$EVENT_NAME" = "workflow_run" ] \ - && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then - echo "::error::Calculated tag $VERSION already exists on a different commit" - exit 1 - fi - - read_chart_field() { - grep -m1 "^$1:" helm/codeapi/Chart.yaml \ - | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]*#.*//; s/^[\"']//; s/[\"']\$//" - } - APP_VERSION="$(read_chart_field appVersion)" - CHART_VERSION="$(read_chart_field version)" - - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ - && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then - echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." - exit 1 - fi - - case "$VERSION" in - *-rc*) PRERELEASE=true ;; - *) PRERELEASE=false ;; - esac - - # `latest` moves only when this is the highest stable version, so - # re-cutting an older patch cannot drag it backwards. The tag under - # dispatch does not exist yet, hence adding it to the comparison. - LATEST=false - if [ "$PRERELEASE" = "false" ]; then - HIGHEST_STABLE="$( - { - git tag --list 'v[0-9]*' - printf '%s\n' "$VERSION" - } \ - | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ - | sort -V \ - | tail -n 1 - )" - if [ "$HIGHEST_STABLE" = "$VERSION" ]; then - LATEST=true - fi - fi - - DRAFT=false - if [ "$INPUT_DRAFT" = "true" ]; then - DRAFT=true - fi - - { - echo "skip=false" - echo "version=$VERSION" - echo "app_version=$APP_VERSION" - echo "chart_version=$CHART_VERSION" - echo "prerelease=$PRERELEASE" - echo "latest=$LATEST" - echo "draft=$DRAFT" - } >> "$GITHUB_OUTPUT" - - echo "Releasing $VERSION (chart $CHART_VERSION, appVersion $APP_VERSION, prerelease=$PRERELEASE, latest=$LATEST, draft=$DRAFT)" + RESOLVER_PATH: ${{ github.event_name == 'workflow_run' && format('{0}/resolve-release-version.sh', runner.temp) || '.github/scripts/resolve-release-version.sh' }} + # The resolution itself lives in a script so that every path through it + # — automatic release, resumed release, dispatch, pushed tag, and the + # runs that must skip or fail — is covered by + # tests/release-version-resolution.sh in CI. + run: "$RESOLVER_PATH" # helm is preinstalled on ubuntu-latest, the same way the chart tests in # ci.yml depend on it. diff --git a/tests/release-version-resolution.sh b/tests/release-version-resolution.sh new file mode 100755 index 00000000..c70d50d3 --- /dev/null +++ b/tests/release-version-resolution.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash + +# Covers .github/scripts/resolve-release-version.sh: the version a release run +# publishes, and the runs that have to skip or fail instead. Every case builds a +# throwaway repository with an `origin` the resolver can query and a stubbed +# `gh`, so nothing here reaches the network or the real repository. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# The resolver is deliberately kept outside each throwaway checkout. That +# mirrors release.yml preserving the workflow revision in RUNNER_TEMP before a +# workflow_run checks out the possibly historical release commit. +RESOLVER="$ROOT/.github/scripts/resolve-release-version.sh" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +REPO="$WORK/repo" +ORIGIN="$WORK/origin.git" +OUTPUT="$WORK/github_output" +LOG="$WORK/log" +CASE='' +STATUS=0 +FAILURES=0 + +# `gh release view` is the only gh call the resolver makes. PUBLISHED lists the +# releases that already exist; anything else must not be invoked at all. +mkdir -p "$WORK/bin" +cat > "$WORK/bin/gh" <<'STUB' +#!/usr/bin/env bash +if [ "$1" = 'release' ] && [ "$2" = 'view' ]; then + for published in ${PUBLISHED:-}; do + if [ "$published" = "$3" ]; then + exit 0 + fi + done + exit 1 +fi +echo "unexpected gh invocation: $*" >&2 +exit 2 +STUB +chmod +x "$WORK/bin/gh" +PATH="$WORK/bin:$PATH" + +git_repo() { + git -C "$REPO" "$@" +} + +# A fresh repository whose layout matches what the resolver reads from the +# checkout: the version bump script it shells out to, and the chart it takes the +# app and chart versions from, trailing comments and quotes included. +new_case() { + CASE="$1" + rm -rf "$REPO" "$ORIGIN" + git init -q --bare "$ORIGIN" + git init -q -b main "$REPO" + git_repo config user.name test + git_repo config user.email test@example.com + git_repo remote add origin "$ORIGIN" + mkdir -p "$REPO/.github/scripts" "$REPO/helm/codeapi" + cp "$ROOT/.github/scripts/next-release-version.sh" "$REPO/.github/scripts/" + cat > "$REPO/helm/codeapi/Chart.yaml" <<'CHART' +apiVersion: v2 +name: codeapi +version: 0.3.1 # Chart version +appVersion: "2.0.0" # App version +CHART + commit api/runtime.ts initial 'chore: initial import' + publish_main +} + +commit() { + local path="$1" content="$2" message="$3" + mkdir -p "$REPO/$(dirname "$path")" + printf '%s\n' "$content" > "$REPO/$path" + git_repo add "$path" + git_repo commit -q -m "$message" +} + +publish_main() { + git_repo push -q origin main +} + +head_sha() { + git_repo rev-parse HEAD +} + +# Runs the resolver in the throwaway repository. Arguments are KEY=VALUE pairs +# standing in for the workflow's env block. +resolve() { + : > "$OUTPUT" + set +e + (cd "$REPO" && env GITHUB_OUTPUT="$OUTPUT" "$@" bash "$RESOLVER") > "$LOG" 2>&1 + STATUS=$? + set -e +} + +fail() { + echo "FAIL [$CASE] $1" >&2 + sed 's/^/ | /' "$LOG" >&2 + FAILURES=$((FAILURES + 1)) +} + +expect_status() { + if [ "$STATUS" != "$1" ]; then + fail "exit status: expected $1, got $STATUS" + fi +} + +expect_output() { + local actual + actual="$(sed -n "s/^$1=//p" "$OUTPUT" | tail -n 1)" + if [ "$actual" != "$2" ]; then + fail "output $1: expected '$2', got '$actual'" + fi +} + +expect_log() { + if ! grep -qF "$1" "$LOG"; then + fail "expected log to mention: $1" + fi +} + +# An untagged tip of main is the ordinary automatic-release path: the version +# comes from Conventional Commit intent since the last stable tag. Filtering +# tags with `grep` used to abort the step here, because no match under +# `pipefail` looks like a command failure. +new_case 'automatic release from an untagged commit' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip false +expect_output version v1.2.4 +expect_output app_version 2.0.0 +expect_output chart_version 0.3.1 +expect_output prerelease false +expect_output latest true +expect_output draft false + +new_case 'documentation-only range releases nothing' +git_repo tag v1.2.3 +commit docs/guide.md docs 'docs: clarify deployment' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip true +expect_output version '' +expect_log 'no release needed' + +new_case 'a repository without a stable tag reports why' +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'Automatic releases require an existing stable' + +# The rerun-to-publish recovery path: a previous run created the tag and then +# failed before the release existed. +new_case 'a rerun resumes the tag that already points at HEAD' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +git_repo tag v1.2.4 +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" PUBLISHED='' +expect_status 0 +expect_output skip false +expect_output version v1.2.4 +expect_output latest true + +new_case 'a published tag at HEAD releases nothing twice' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +git_repo tag v1.2.4 +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" PUBLISHED='v1.2.4' +expect_status 0 +expect_output skip true +expect_log 'already publishes this commit' + +new_case 'a calculated tag held by another commit is a collision' +git_repo tag v1.2.3 +git_repo checkout -q -b elsewhere +commit api/runtime.ts diverged 'fix: unrelated work' +git_repo tag v1.2.4 +git_repo checkout -q main +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'already exists on a different commit' + +new_case 'a calculated tag held by a non-commit object is a collision' +git_repo tag v1.2.3 +blob="$(printf 'not a commit\n' | git_repo hash-object -w --stdin)" +git_repo update-ref refs/tags/v1.2.4 "$blob" +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'already exists but does not point to a commit' + +new_case 'a stale CI run defers to the newer tip' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +commit api/runtime.ts advanced 'fix: land more work' +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip true +expect_log 'main advanced after this CI run' + +new_case 'a dispatched version may omit the v prefix' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=2.0.0 INPUT_DRAFT=true +expect_status 0 +expect_output version v2.0.0 +expect_output prerelease false +expect_output latest true +expect_output draft true + +new_case 'a release candidate is a prerelease and never latest' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=v1.3.0-rc1 +expect_status 0 +expect_output version v1.3.0-rc1 +expect_output prerelease true +expect_output latest false + +new_case 'dispatching from a topic branch is refused' +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=feature/x \ + INPUT_VERSION=v1.3.0 +expect_status 1 +expect_log 'Releases must be cut from main' + +new_case 'dispatching an existing version is refused' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=v1.2.3 +expect_status 1 +expect_log 'already exists' + +new_case 'a malformed version is refused' +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=1.2 +expect_status 1 +expect_log 'Release tags must be' + +new_case 'a pushed older patch tag does not become latest' +git_repo tag v9.9.9 +resolve EVENT_NAME=push REF_TYPE=tag REF_NAME=v1.0.1 +expect_status 0 +expect_output version v1.0.1 +expect_output prerelease false +expect_output latest false + +if [ "$FAILURES" -ne 0 ]; then + echo "$FAILURES release version resolution assertion(s) failed" >&2 + exit 1 +fi + +echo 'release version resolution tests passed' From 54aca9262ecc1eca6ad3746e6f5b72169897afcf Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 09:50:44 -0400 Subject: [PATCH 37/75] feat: Route GitHub App credentials per repository (#236) * feat: Route GitHub App credentials per repository * test: Make repository routing assertion deterministic * fix: Harden repository credential routing * fix: Bound shared GitHub credential refreshes * fix: Bind GitHub credentials to admitted workspaces --- docs/remote-bridge/worker-runbook.md | 19 +- packages/code/README.md | 14 +- packages/code/src/cli.test.ts | 59 ++- packages/code/src/cli.ts | 52 ++- packages/code/src/github.test.ts | 442 +++++++++++++++++++++- packages/code/src/github.ts | 445 ++++++++++++++++++++--- packages/code/src/native-process.test.ts | 5 +- packages/code/src/native-process.ts | 12 +- packages/code/src/native-sandbox.test.ts | 45 +++ packages/code/src/native-sandbox.ts | 26 +- 10 files changed, 1037 insertions(+), 82 deletions(-) diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 7511a265..16c24742 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -323,13 +323,21 @@ Configure the worker, preferably in a separate service drop-in: ```ini [Service] Environment=LIBRECHAT_CODE_GITHUB_APP_ID=12345 -Environment=LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/librechat-code/github-app.pem ``` -The trusted worker mints short-lived installation tokens. Sandboxed commands -receive masked Git/`gh` credentials only for the configured GitHub hosts; the -token is not written to the repository, remote URL, or Git configuration. +Install the same App separately on every personal account or organization the +worker is allowed to use. The trusted worker resolves the correct installation +from the repository containing each command's working directory, then mints and +caches a repository-scoped token. Cross-repository work therefore does not +require changing an installation ID or restarting the worker. Set +`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation +fallback. + +Sandboxed commands receive masked Git/`gh` credentials only for the configured +GitHub hosts; the token is not written to the repository, remote URL, or Git +configuration. Git commits receive the App bot's canonical no-reply identity so +GitHub renders the bot profile and avatar. ## 10. Run under systemd @@ -518,7 +526,8 @@ command, cancellation, or settlement whose effects may be incomplete. - [ ] Pairing is principal-bound and the identity file is private. - [ ] Definitions are outside roots and immutable to sandboxed tools. - [ ] Workspace ancestors are not group/other writable. -- [ ] GitHub App is optional, least-privilege, and installed only where needed. +- [ ] GitHub App is optional, least-privilege, and installed on every account + the worker is expected to use. - [ ] Approval policy remains enforced independently of worker capability. - [ ] Service manager uses the intended executable and configuration. - [ ] Worker is online, ready, and advertises the expected workspace. diff --git a/packages/code/README.md b/packages/code/README.md index 08dc54a7..69d85b16 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -283,14 +283,24 @@ repositories the agent may access: ```bash LIBRECHAT_CODE_GITHUB_APP_ID=12345 \ -LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 \ LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/secure/librechat-agent.pem \ librechat-code run --worker-dir /path/to/project --allow-workspace-commands ``` The private key must be an owner-only regular file outside the workspace. It is read only by the trusted worker, which mints and refreshes short-lived -installation tokens. A personal access token is supported as a fallback with +installation tokens. At startup, the worker binds each explicitly admitted +workspace root to its Git repository. Commands in those independent roots can +use simultaneous installations on personal accounts and organizations without +being restarted or reconfigured, while a command cannot gain access by changing +its workspace's remote URL. Tokens are scoped and cached per repository. For +compatibility with deployments +that intentionally bind a worker to one installation, set the optional legacy +`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. + +App-authenticated commits use the GitHub App bot's canonical no-reply identity, +so GitHub links them to the bot profile and avatar. A personal access token is +supported as a fallback with `LIBRECHAT_CODE_GITHUB_TOKEN`, but the GitHub App is the safer default because its repository access and permissions can be narrowly installed and revoked. Native Windows credential storage is unavailable until native DACL removal and diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 18456072..f39b28d9 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -1,5 +1,9 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; +import { generateKeyPairSync } from 'node:crypto'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; @@ -233,6 +237,59 @@ test('CLI validates GitHub App credentials before worker registration', () => { assert.doesNotMatch(result.stderr, /fetch failed/); }); +test('CLI accepts repository-routed GitHub App authentication without a fixed installation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'cli-github-routing-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const preload = join(directory, 'fetch.mjs'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + await writeFile( + preload, + ` + globalThis.fetch = async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia-by-librechat' }); + if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { + return Response.json({ id: 328778573, login: 'lia-by-librechat[bot]', type: 'Bot' }); + } + throw new Error('test stopped after GitHub App validation'); + }; + `, + ); + const result = spawnSync( + process.execPath, + [ + '--import', + preload, + fileURLToPath(new URL('./cli.js', import.meta.url)), + ], + { + encoding: 'utf8', + timeout: 10_000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: directory, + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: privateKeyPath, + }, + }, + ); + assert.notEqual(result.status, 0); + assert.doesNotMatch(result.stderr, /GitHub App authentication requires/); + assert.doesNotMatch(result.stderr, /installation ID/i); +}); + test('CLI requires a runtime image for Docker supervision', () => { const result = spawnSync( process.execPath, @@ -449,6 +506,6 @@ test('CLI host-only enterprise configuration sends App JWTs to GHES, never GitHu }, }); assert.equal(result.status, 1, result.stderr); - assert.match(result.stderr, /GITHUB_REQUEST:https:\/\/github\.example\.test\/api\/v3\/app\/installations\/456\/access_tokens/); + assert.match(result.stderr, /GITHUB_REQUEST:https:\/\/github\.example\.test\/api\/v3\/app/); assert.doesNotMatch(result.stderr, /GITHUB_REQUEST:https:\/\/api\.github\.com/); }); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 524c5853..96757d57 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -46,6 +46,8 @@ import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, GitHubAppCredentialProvider, + gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForDirectory, gitHubCommandCredentialEnvironment, gitHubMaskedCredentialVariables, StaticGitHubCredentialProvider, @@ -149,6 +151,7 @@ function githubCredentials(): { host: string; privateKeyPath?: string; mode?: 'app' | 'token'; + repositoryRouting?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -161,9 +164,9 @@ function githubCredentials(): { ); const appValues = [appId, installationId, privateKeyPath]; const hasApp = appValues.some(Boolean); - if (hasApp && !appValues.every(Boolean)) { + if (hasApp && (!appId || !privateKeyPath)) { throw new Error( - 'GitHub App authentication requires LIBRECHAT_CODE_GITHUB_APP_ID, LIBRECHAT_CODE_GITHUB_INSTALLATION_ID, and LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE', + 'GitHub App authentication requires LIBRECHAT_CODE_GITHUB_APP_ID and LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE; LIBRECHAT_CODE_GITHUB_INSTALLATION_ID is an optional legacy fallback', ); } if (hasApp && token) { @@ -203,6 +206,7 @@ function githubCredentials(): { return { host, mode: 'app', + repositoryRouting: !installationId, policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, @@ -212,7 +216,7 @@ function githubCredentials(): { privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, - installationId: installationId!, + installationId, privateKeyPath: privateKeyPath!, host, apiUrl, @@ -715,6 +719,19 @@ async function run( }), ]), ); + // Bind credentials to immutable, explicitly admitted roots. The repository + // remote is operator input at startup, never an authorization input that a + // sandboxed command may change for its next invocation. + const admittedGitHubRepositories = github.provider && github.repositoryRouting + ? new Map( + await Promise.all( + roots.map(async root => [ + root.root, + await gitHubRepositoryForDirectory(root.root, github.host), + ] as const), + ), + ) + : undefined; const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, @@ -928,18 +945,34 @@ async function run( ...(github.provider ? { maskedEnvironment: { - variables: gitHubMaskedCredentialVariables(github.host), - async resolve(signal?: AbortSignal) { + variables: gitHubMaskedCredentialVariables( + github.host, + ), + async resolve(signal?: AbortSignal, cwd?: string) { + const repository = cwd && admittedGitHubRepositories + ? gitHubRepositoryForAdmittedDirectory( + cwd, + admittedGitHubRepositories, + ) + : undefined; + if (!repository && github.repositoryRouting) { + return {}; + } return gitHubCommandCredentialEnvironment( - await github.provider!.getCredential(signal), + await github.provider!.getCredential(signal, repository), github.host, ); }, - wrapCommand(command: string, platform: NodeJS.Platform) { + wrapCommand( + command: string, + platform: NodeJS.Platform, + environment: Readonly>, + ) { return wrapGitHubCredentialCommand( command, github.host, platform, + environment, ); }, }, @@ -1022,7 +1055,10 @@ async function run( ); } try { - await github.provider?.getCredential(controller.signal); + await github.provider?.validate?.(controller.signal); + if (github.provider && !github.provider.validate) { + await github.provider.getCredential(controller.signal); + } await nativeCommandSandbox?.prepare(); for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { const setup = environment.definition.setup; diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 4b028239..73e02927 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -1,4 +1,5 @@ import { generateKeyPairSync } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; import { chmod, mkdtemp, @@ -8,12 +9,14 @@ import { writeFile, } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import test from 'node:test'; import assert from 'node:assert/strict'; import { GITHUB_ALLOWED_DOMAINS, + GITHUB_AUTHOR_EMAIL_ENV_NAME, + GITHUB_AUTHOR_NAME_ENV_NAME, GitHubAppCredentialProvider, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, @@ -22,6 +25,8 @@ import { gitHubMaskedCredentialVariables, GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, + gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForDirectory, normalizeGitHubHost, wrapGitHubCredentialCommand, } from './github.js'; @@ -136,6 +141,358 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = assert.equal(calls, 1); }); +test('routes and scopes GitHub App tokens per repository installation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-routing-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + const calls: Array<{ url: string; body?: string }> = []; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input, init) => { + const url = String(input); + calls.push({ url, body: typeof init?.body === 'string' ? init.body : undefined }); + if (url.endsWith('/app')) { + return Response.json({ slug: 'lia-by-librechat' }); + } + if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { + return Response.json({ + id: 328778573, + login: 'lia-by-librechat[bot]', + type: 'Bot', + }); + } + if (url.endsWith('/repos/danny-avila/LibreChat/installation')) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/repos/LibreChat-AI/code-interpreter/installation')) { + return Response.json({ id: 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + return Response.json( + { + token: `ghs_${installation}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }, + { status: 201 }, + ); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + + await provider.validate(); + const [personal, organization] = await Promise.all([ + provider.getCredential(undefined, 'danny-avila/LibreChat'), + provider.getCredential(undefined, 'LibreChat-AI/code-interpreter'), + ]); + assert.equal( + (await provider.getCredential(undefined, 'danny-avila/LibreChat')).value, + personal.value, + ); + assert.equal(personal.value, 'ghs_111_abcdefghijklmnopqrstuvwxyz'); + assert.equal(organization.value, 'ghs_222_abcdefghijklmnopqrstuvwxyz'); + assert.deepEqual(personal.actor, { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }); + assert.equal( + calls.filter(call => call.url.includes('/repos/danny-avila/')).length, + 1, + ); + assert.deepEqual( + calls + .filter(call => call.url.endsWith('/access_tokens')) + .map(call => JSON.parse(call.body ?? '{}')) + .map(body => body.repositories[0]) + .sort(), + [ + 'LibreChat', + 'code-interpreter', + ], + ); +}); + +test('keeps a shared token refresh alive when one waiter is cancelled', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-cancel-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let releaseToken!: (response: Response) => void; + const tokenResponse = new Promise(resolve => { + releaseToken = resolve; + }); + let mintCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + mintCount += 1; + return tokenResponse; + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + const firstController = new AbortController(); + const first = provider.getCredential( + firstController.signal, + 'acme/project', + ); + const second = provider.getCredential(undefined, 'acme/project'); + firstController.abort(new Error('first command cancelled')); + await assert.rejects(first, /first command cancelled/); + const third = provider.getCredential(undefined, 'acme/project'); + releaseToken( + Response.json({ + token: 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }), + ); + assert.equal( + (await second).value, + 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + ); + assert.equal((await third).value, 'ghs_shared_abcdefghijklmnopqrstuvwxyz'); + assert.equal(mintCount, 1); +}); + +test('refreshes a cached repository installation after App reinstallation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-reinstall-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let now = new Date('2030-01-01T00:00:00Z'); + let lookupCount = 0; + let oldMintCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => now, + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + lookupCount += 1; + return Response.json({ id: lookupCount === 1 ? 111 : 222 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + oldMintCount += 1; + return oldMintCount === 1 + ? Response.json({ + token: 'ghs_old_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }) + : Response.json({}, { status: 404 }); + } + if (url.endsWith('/app/installations/222/access_tokens')) { + return Response.json({ + token: 'ghs_new_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T02:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + assert.equal( + (await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_old_abcdefghijklmnopqrstuvwxyz', + ); + now = new Date('2030-01-01T00:56:00Z'); + assert.equal( + (await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_new_abcdefghijklmnopqrstuvwxyz', + ); + assert.equal(lookupCount, 2); +}); + +test('validates a configured fixed installation by minting its token', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-fixed-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let minted = false; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + minted = true; + return Response.json({ + token: 'ghs_fixed_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + assert.equal(minted, true); +}); + +test('discovers the GitHub repository from a command working directory', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-repo-')); + t.after(() => rm(directory, { recursive: true, force: true })); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'add', + 'origin', + 'git@github.com:LibreChat-AI/code-interpreter.git', + ]); + assert.equal( + await gitHubRepositoryForDirectory(directory), + 'LibreChat-AI/code-interpreter', + ); + assert.equal( + await gitHubRepositoryForDirectory(directory, 'github.example.test'), + undefined, + ); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'set-url', + 'origin', + 'https://github.example.test:8443/acme/project.git', + ]); + assert.equal( + await gitHubRepositoryForDirectory(directory, 'github.example.test'), + 'acme/project', + ); +}); + +test('keeps repository authorization bound to the admitted workspace root', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-binding-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const nested = join(directory, 'packages', 'app'); + await mkdir(nested, { recursive: true }); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'add', + 'origin', + 'git@github.com:acme/allowed.git', + ]); + const admitted = new Map([ + [directory, await gitHubRepositoryForDirectory(directory)], + ]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'set-url', + 'origin', + 'git@github.com:acme/not-authorized.git', + ]); + assert.equal( + gitHubRepositoryForAdmittedDirectory(nested, admitted), + 'acme/allowed', + ); + assert.equal( + gitHubRepositoryForAdmittedDirectory(dirname(directory), admitted), + undefined, + ); +}); + +test('uses the configured GHES host for the App bot no-reply identity', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-ghes-identity-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath, + host: 'github.example.test', + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + return Response.json({ + token: 'ghs_enterprise_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + const credential = await provider.getCredential(); + assert.deepEqual(credential.actor, { + name: 'lia[bot]', + email: '1234+lia[bot]@users.noreply.github.example.test', + }); + const wrapped = wrapGitHubCredentialCommand( + 'git commit -m test', + 'github.example.test', + 'linux', + gitHubCommandCredentialEnvironment(credential, 'github.example.test'), + ); + assert.match( + wrapped, + /user\.email=1234\+lia\[bot\]@users\.noreply\.github\.example\.test/, + ); +}); + test('builds process-scoped Git HTTPS authorization without embedding credentials in URLs', async () => { const provider = new StaticGitHubCredentialProvider( 'github_pat_abcdefghijklmnopqrstuvwxyz', @@ -177,6 +534,76 @@ test('adds a GitHub CLI token only to the command-sandbox credential bundle', as ); }); +test('binds Git commits to the GitHub App bot identity', () => { + const environment = gitHubCommandCredentialEnvironment({ + value: 'ghs_abcdefghijklmnopqrstuvwxyz', + actor: { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }, + }); + assert.equal(environment[GITHUB_AUTHOR_NAME_ENV_NAME], 'lia-by-librechat[bot]'); + assert.equal( + environment[GITHUB_AUTHOR_EMAIL_ENV_NAME], + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + ); + const variables = gitHubMaskedCredentialVariables('github.com'); + assert.ok(!variables.some(variable => variable.name === GITHUB_AUTHOR_NAME_ENV_NAME)); + assert.ok(!variables.some(variable => variable.name === GITHUB_AUTHOR_EMAIL_ENV_NAME)); + const wrapped = wrapGitHubCredentialCommand( + 'git commit -m test', + 'github.com', + 'linux', + environment, + ); + assert.match(wrapped, /user\.name=/); + assert.match(wrapped, /user\.email=/); +}); + +test('records the canonical App bot as Git author and committer', async (t) => { + if (process.platform === 'win32') { + t.skip('POSIX command wrapper integration is unavailable on Windows'); + return; + } + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-author-')); + t.after(() => rm(directory, { recursive: true, force: true })); + execFileSync('git', ['init', directory]); + const environment = gitHubCommandCredentialEnvironment({ + value: 'ghs_abcdefghijklmnopqrstuvwxyz', + actor: { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }, + }); + const wrapped = wrapGitHubCredentialCommand( + 'git commit --allow-empty -m test', + 'github.com', + process.platform, + environment, + ); + execFileSync('/bin/bash', ['-lc', wrapped], { + cwd: directory, + env: { PATH: process.env.PATH, ...environment }, + }); + assert.equal( + execFileSync( + 'git', + [ + '-C', + directory, + 'show', + '-s', + '--format=%an|%ae|%cn|%ce', + 'HEAD', + ], + { encoding: 'utf8' }, + ).trim(), + 'lia-by-librechat[bot]|328778573+lia-by-librechat[bot]@users.noreply.github.com|lia-by-librechat[bot]|328778573+lia-by-librechat[bot]@users.noreply.github.com', + ); +}); + test('selects the GitHub CLI token variable for public and enterprise hosts', () => { assert.equal(gitHubCliTokenEnvironmentName('github.com'), 'GH_TOKEN'); assert.equal( @@ -217,6 +644,9 @@ test('composes the masked credential with SRT Git configuration inside the sandb 'git push', 'github.com', 'darwin', + { + [GITHUB_CREDENTIAL_ENV_NAME]: 'masked-authorization', + }, ); assert.match(wrapped, /http\.proxyAuthMethod=basic/); assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); @@ -227,6 +657,16 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.ok(!wrapped.includes('github_pat_')); }); +test('omits Git authorization when repository routing resolves no credential', () => { + const wrapped = wrapGitHubCredentialCommand( + 'git clone https://github.com/LibreChat-AI/LibreChat.git', + 'github.com', + 'linux', + {}, + ); + assert.doesNotMatch(wrapped, /Authorization: Basic/); +}); + test('targets GitHub CLI at an enterprise host without exposing its token', () => { const wrapped = wrapGitHubCredentialCommand( 'gh pr create', diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index c727e70d..4c79fa37 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -1,10 +1,15 @@ import { constants } from 'node:fs'; +import { execFile } from 'node:child_process'; import { createHash, createPrivateKey, sign } from 'node:crypto'; import { open } from 'node:fs/promises'; -import { dirname } from 'node:path'; +import { dirname, isAbsolute, relative, sep } from 'node:path'; +import { promisify } from 'node:util'; +import { projectRemote } from './projects.js'; import { assertPrivateStorageAcl, assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; export const GITHUB_CREDENTIAL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHORIZATION'; +export const GITHUB_AUTHOR_NAME_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHOR_NAME'; +export const GITHUB_AUTHOR_EMAIL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHOR_EMAIL'; export const GITHUB_ALLOWED_DOMAINS = [ 'github.com', '*.github.com', @@ -18,15 +23,24 @@ export const GITHUB_ALLOWED_DOMAINS = [ export interface GitHubCredential { value: string; expiresAt?: Date; + actor?: { + name: string; + email: string; + }; } export interface GitHubCredentialProvider { - getCredential(signal?: AbortSignal): Promise; + getCredential( + signal?: AbortSignal, + repository?: string, + ): Promise; + validate?(signal?: AbortSignal): Promise; } export interface GitHubAppCredentialProviderOptions { appId: string; - installationId: string; + /** Legacy fixed installation. Omit to resolve the installation per repository. */ + installationId?: string; privateKeyPath: string; apiUrl?: string; /** Git HTTPS hostname; non-public hosts default to the GHES /api/v3 base. */ @@ -36,6 +50,110 @@ export interface GitHubAppCredentialProviderOptions { platform?: NodeJS.Platform; } +const execFileAsync = promisify(execFile); +const GITHUB_SHARED_REQUEST_TIMEOUT_MS = 30_000; + +async function waitForShared( + promise: Promise, + signal?: AbortSignal, +): Promise { + if (!signal) return promise; + signal.throwIfAborted(); + return new Promise((resolve, reject) => { + const aborted = () => { + try { + signal.throwIfAborted(); + } catch (error) { + reject(error); + } + }; + signal.addEventListener('abort', aborted, { once: true }); + void promise.then(resolve, reject).finally(() => { + signal.removeEventListener('abort', aborted); + }); + }); +} + +function repositoryName(value: string): { owner: string; name: string } { + const match = /^([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)$/.exec(value); + if (!match) throw new Error('GitHub repository must be owner/name'); + return { owner: match[1], name: match[2] }; +} + +/** Resolve only the repository containing the admitted command cwd. */ +export async function gitHubRepositoryForDirectory( + cwd: string, + host = 'github.com', + signal?: AbortSignal, +): Promise { + let remote: string; + try { + const result = await execFileAsync( + 'git', + [ + '--no-optional-locks', + '-C', + cwd, + '-c', + 'core.fsmonitor=false', + 'config', + '--local', + '--no-includes', + '--get', + 'remote.origin.url', + ], + { + env: { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }, + encoding: 'utf8', + maxBuffer: 4096, + timeout: 1500, + signal, + }, + ); + remote = result.stdout.trim(); + } catch { + signal?.throwIfAborted(); + return undefined; + } + const normalized = projectRemote(remote); + if (!normalized) return undefined; + const separator = normalized.indexOf('/'); + const remoteHost = normalized + .slice(0, separator) + .replace(/:[1-9][0-9]*$/, ''); + if (remoteHost !== normalizeGitHubHost(host)) { + return undefined; + } + const repository = normalized.slice(separator + 1); + repositoryName(repository); + return repository; +} + +/** Return the startup-bound repository for the admitted root containing cwd. */ +export function gitHubRepositoryForAdmittedDirectory( + cwd: string, + repositories: ReadonlyMap, +): string | undefined { + for (const [root, repository] of repositories) { + const path = relative(root, cwd); + if ( + path === '' || + (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + ) { + return repository; + } + } + return undefined; +} + function base64UrlJson(value: unknown): string { return Buffer.from(JSON.stringify(value)).toString('base64url'); } @@ -96,8 +214,13 @@ function createAppJwt(appId: string, privateKey: string, now: Date): string { } export class GitHubAppCredentialProvider implements GitHubCredentialProvider { - private cached?: GitHubCredential; + private readonly cached = new Map(); + private readonly inFlight = new Map>(); + private readonly installationIds = new Map(); + private actor?: GitHubCredential['actor']; + private actorInFlight?: Promise>; private readonly apiUrl: string; + private readonly host: string; constructor(private readonly options: GitHubAppCredentialProviderOptions) { if ((options.platform ?? process.platform) === 'win32') { @@ -106,10 +229,12 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ); } assertPositiveIdentifier('GitHub App ID', options.appId); - assertPositiveIdentifier( - 'GitHub App installation ID', - options.installationId, - ); + if (options.installationId != null) { + assertPositiveIdentifier( + 'GitHub App installation ID', + options.installationId, + ); + } const host = options.host == null ? undefined : normalizeGitHubHost(options.host); const apiUrl = new URL(options.apiUrl ?? ( host != null && host !== 'github.com' @@ -126,58 +251,246 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { if (host != null && host !== apiHost) { throw new Error('LIBRECHAT_CODE_GITHUB_HOST must match the GitHub App API hostname'); } + this.host = host ?? apiHost; this.apiUrl = apiUrl.href.replace(/\/+$/, ''); } - async getCredential(signal?: AbortSignal): Promise { - const now = (this.options.now ?? (() => new Date()))(); - if ( - this.cached?.expiresAt != null && - this.cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 - ) { - return this.cached; - } + private async appJwt(now: Date): Promise { const privateKey = await readPrivateKey(this.options.privateKeyPath); - const jwt = createAppJwt(this.options.appId, privateKey, now); + return createAppJwt(this.options.appId, privateKey, now); + } + + private async request( + path: string, + jwt: string, + signal?: AbortSignal, + init?: RequestInit, + ): Promise { const request = this.options.fetch ?? globalThis.fetch; - const response = await request( - `${this.apiUrl}/app/installations/${this.options.installationId}/access_tokens`, - { - method: 'POST', - redirect: 'error', - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${jwt}`, - 'X-GitHub-Api-Version': '2022-11-28', - }, - signal, + return request(`${this.apiUrl}${path}`, { + redirect: 'error', + ...init, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${jwt}`, + 'X-GitHub-Api-Version': '2022-11-28', + ...init?.headers, }, + signal, + }); + } + + private async resolveActor( + jwt: string, + signal?: AbortSignal, + ): Promise> { + if (this.actor) return this.actor; + if (!this.actorInFlight) { + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const appResponse = await this.request('/app', jwt, sharedSignal); + if (!appResponse.ok) { + throw new Error( + `GitHub App identity request failed with status ${appResponse.status}`, + ); + } + const app = (await appResponse.json()) as { slug?: unknown }; + if ( + typeof app.slug !== 'string' || + !/^[A-Za-z0-9-]+$/.test(app.slug) + ) { + throw new Error('GitHub App identity response is invalid'); + } + const login = `${app.slug}[bot]`; + const userResponse = await this.request( + `/users/${encodeURIComponent(login)}`, + jwt, + sharedSignal, + ); + if (!userResponse.ok) { + throw new Error( + `GitHub App bot identity request failed with status ${userResponse.status}`, + ); + } + const user = (await userResponse.json()) as { + id?: unknown; + login?: unknown; + type?: unknown; + }; + if ( + !Number.isSafeInteger(user.id) || + Number(user.id) <= 0 || + user.login !== login || + user.type !== 'Bot' + ) { + throw new Error('GitHub App bot identity response is invalid'); + } + return { + name: login, + email: `${user.id}+${login}@users.noreply.${this.host}`, + }; + })(); + this.actorInFlight = pending; + void pending.then( + actor => { + this.actor = actor; + if (this.actorInFlight === pending) this.actorInFlight = undefined; + }, + () => { + if (this.actorInFlight === pending) this.actorInFlight = undefined; + }, + ); + } + return waitForShared(this.actorInFlight, signal); + } + + async validate(signal?: AbortSignal): Promise { + const now = (this.options.now ?? (() => new Date()))(); + const jwt = await this.appJwt(now); + await this.resolveActor(jwt, signal); + if (this.options.installationId) { + await this.getCredential(signal); + } + } + + private async resolveInstallationId( + repository: string, + jwt: string, + signal?: AbortSignal, + ): Promise { + if (this.options.installationId) return this.options.installationId; + const cached = this.installationIds.get(repository); + if (cached) return cached; + const { owner, name } = repositoryName(repository); + const response = await this.request( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, + jwt, + signal, ); if (!response.ok) { throw new Error( - `GitHub App token request failed with status ${response.status}`, + response.status === 404 + ? `GitHub App is not installed for ${repository}` + : `GitHub App installation lookup failed with status ${response.status}`, ); } - const body = (await response.json()) as { - token?: unknown; - expires_at?: unknown; - }; - if ( - typeof body.token !== 'string' || - body.token.length < 20 || - typeof body.expires_at !== 'string' - ) { - throw new Error('GitHub App token response is invalid'); + const body = (await response.json()) as { id?: unknown }; + if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { + throw new Error('GitHub App installation response is invalid'); + } + const installationId = String(body.id); + this.installationIds.set(repository, installationId); + return installationId; + } + + async getCredential( + signal?: AbortSignal, + repository?: string, + ): Promise { + signal?.throwIfAborted(); + if (!this.options.installationId && !repository) { + throw new Error( + 'GitHub App authentication requires a GitHub repository for this command', + ); } - const expiresAt = new Date(body.expires_at); + if (repository) repositoryName(repository); + const now = (this.options.now ?? (() => new Date()))(); + const key = this.options.installationId ?? repository!; + const cached = this.cached.get(key); if ( - !Number.isFinite(expiresAt.getTime()) || - expiresAt.getTime() <= now.getTime() + cached?.expiresAt != null && + cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 ) { - throw new Error('GitHub App token expiry is invalid'); + return cached; } - this.cached = { value: body.token, expiresAt }; - return this.cached; + const existing = this.inFlight.get(key); + if (existing) return waitForShared(existing, signal); + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const jwt = await this.appJwt(now); + const scopedRepository = repository + ? repositoryName(repository).name + : undefined; + const installationId = await this.resolveInstallationId( + repository ?? '', + jwt, + sharedSignal, + ); + let response = await this.request( + `/app/installations/${installationId}/access_tokens`, + jwt, + sharedSignal, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + ...(this.options.installationId + ? {} + : { body: JSON.stringify({ repositories: [scopedRepository] }) }), + }, + ); + if (!this.options.installationId && response.status === 404) { + this.installationIds.delete(repository!); + const refreshedInstallationId = await this.resolveInstallationId( + repository!, + jwt, + sharedSignal, + ); + response = await this.request( + `/app/installations/${refreshedInstallationId}/access_tokens`, + jwt, + sharedSignal, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ repositories: [scopedRepository] }), + }, + ); + } + if (!response.ok) { + throw new Error( + `GitHub App token request failed with status ${response.status}`, + ); + } + const body = (await response.json()) as { + token?: unknown; + expires_at?: unknown; + }; + if ( + typeof body.token !== 'string' || + body.token.length < 20 || + typeof body.expires_at !== 'string' + ) { + throw new Error('GitHub App token response is invalid'); + } + const expiresAt = new Date(body.expires_at); + if ( + !Number.isFinite(expiresAt.getTime()) || + expiresAt.getTime() <= now.getTime() + ) { + throw new Error('GitHub App token expiry is invalid'); + } + const credential = { + value: body.token, + expiresAt, + ...(this.actor ? { actor: this.actor } : {}), + }; + this.cached.set(key, credential); + return credential; + })(); + this.inFlight.set(key, pending); + const clearPending = () => { + if (this.inFlight.get(key) === pending) this.inFlight.delete(key); + }; + void pending.then(clearPending, clearPending); + return waitForShared(pending, signal); } } @@ -211,6 +524,12 @@ export function gitHubCommandCredentialEnvironment( return { ...gitHubCredentialEnvironment(credential), [gitHubCliTokenEnvironmentName(host)]: credential.value, + ...(credential.actor + ? { + [GITHUB_AUTHOR_NAME_ENV_NAME]: credential.actor.name, + [GITHUB_AUTHOR_EMAIL_ENV_NAME]: credential.actor.email, + } + : {}), }; } @@ -260,12 +579,10 @@ export function gitHubAuthenticationPolicyIdentity(options: { return `${identity}:fingerprint:${fingerprint}`; } if (options.mode !== 'app') return identity; - if (!options.appId || !options.installationId) { - throw new Error( - 'GitHub App policy identity requires an App and installation ID', - ); + if (!options.appId) { + throw new Error('GitHub App policy identity requires an App ID'); } - return `${identity}:app:${options.appId}:installation:${options.installationId}`; + return `${identity}:app:${options.appId}:installation:${options.installationId ?? 'repository'}`; } export function normalizeGitHubHost(value: string): string { @@ -284,24 +601,48 @@ export function wrapGitHubCredentialCommand( command: string, host = 'github.com', platform: NodeJS.Platform = process.platform, + environment: Readonly> = {}, ): string { const key = `http.https://${host}/.extraheader`; const cliHost = host === 'github.com' ? undefined : host; + const hasCredential = Boolean(environment[GITHUB_CREDENTIAL_ENV_NAME]); + const actorName = environment[GITHUB_AUTHOR_NAME_ENV_NAME]; + const actorEmail = environment[GITHUB_AUTHOR_EMAIL_ENV_NAME]; + const noReplyHost = `users.noreply.${normalizeGitHubHost(host)}` + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const hasActor = + /^[A-Za-z0-9_.-]+\[bot\]$/.test(actorName ?? '') && + new RegExp( + `^[1-9][0-9]+\\+[A-Za-z0-9_.-]+\\[bot\\]@${noReplyHost}$`, + ).test(actorEmail ?? ''); if (platform === 'win32') { return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', ...(cliHost ? [`set "GH_HOST=${cliHost}"`] : []), - `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, + ...(hasCredential + ? [`set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`] + : ['set "GIT_CONFIG_PARAMETERS="']), + ...(hasActor + ? [`set "GIT_CONFIG_PARAMETERS=%GIT_CONFIG_PARAMETERS% 'user.name=${actorName}' 'user.email=${actorEmail}'"`] + : []), `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, + `set "${GITHUB_AUTHOR_NAME_ENV_NAME}="`, + `set "${GITHUB_AUTHOR_EMAIL_ENV_NAME}="`, command, ].join(' && '); } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', ...(cliHost ? [`export GH_HOST=${cliHost}`] : []), - `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, + ...(hasCredential + ? [`export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`] + : ['export GIT_CONFIG_PARAMETERS=']), + ...(hasActor + ? [`export GIT_CONFIG_PARAMETERS="\${GIT_CONFIG_PARAMETERS} 'user.name=${actorName}' 'user.email=${actorEmail}'"`] + : []), `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, + `unset ${GITHUB_AUTHOR_NAME_ENV_NAME} ${GITHUB_AUTHOR_EMAIL_ENV_NAME}`, command, ].join(';\n'); } diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 89651845..b3016937 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -181,12 +181,14 @@ test('executor forwards the resolved command policy without worker credentials', test('executor hands credentials over IPC only for the current command', async () => { const fake = fixture(); + let credentialCwd: string | undefined; const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: '/workspace', maskedEnvironment: { variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], - async resolve() { + async resolve(_signal, cwd) { + credentialCwd = cwd; return { TOKEN: 'per-command-secret' }; }, wrapCommand(command) { @@ -208,6 +210,7 @@ test('executor hands credentials over IPC only for the current command', async ( assert.deepEqual(fake.messages[1].credentials, { TOKEN: 'per-command-secret', }); + assert.equal(credentialCwd, '/workspace'); assert.equal(fake.messages[1].wrappedCommand, 'wrapped printf ok'); await sandbox.close(); }); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 70cb2bd1..ba411042 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -2,7 +2,7 @@ import { execFile, fork } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { constants as fsConstants } from 'node:fs'; import { access, realpath } from 'node:fs/promises'; -import { isAbsolute, join, relative, sep } from 'node:path'; +import { isAbsolute, join, relative, resolve, sep } from 'node:path'; import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; @@ -436,11 +436,15 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan if (signal?.aborted) throw new Error('aborted'); programmaticExecutables = await this.resolveProgrammaticExecutables(); if (signal?.aborted) throw new Error('aborted'); - credentials = await this.options.maskedEnvironment?.resolve(signal); + credentials = await this.options.maskedEnvironment?.resolve( + signal, + this.options.workspaceRoot, + ); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( NATIVE_PROGRAMMATIC_COMMAND, process.platform, + credentials ?? {}, ); if (signal?.aborted) throw new Error('aborted'); } catch (error) { @@ -529,11 +533,13 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); - credentials = await this.options.maskedEnvironment?.resolve(signal); + const cwd = resolve(this.options.workspaceRoot, request.cwd ?? '.'); + credentials = await this.options.maskedEnvironment?.resolve(signal, cwd); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( request.command, process.platform, + credentials ?? {}, ); if (signal?.aborted) throw new Error('aborted'); } catch (error) { diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index e610a0fc..b1e4c5df 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1031,6 +1031,51 @@ test('masks a host credential for only its injection host and restores the paren }); }); +test('keeps trusted public command context out of credential masking', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + allowedDomains: ['github.com'], + maskedEnvironment: { + variables: [ + { + name: 'LIBRECHAT_CODE_TEST_CREDENTIAL', + injectHosts: ['github.com'], + }, + ], + async resolve() { + return { + LIBRECHAT_CODE_TEST_CREDENTIAL: 'real-secret', + LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY: 'lia[bot]', + }; + }, + wrapCommand(command, _platform, environment) { + assert.equal( + environment.LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY, + 'lia[bot]', + ); + return `export LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY="${environment.LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY}"; ${command}`; + }, + }, + manager: fake.manager, + }); + + const result = await sandbox.execute({ + ...request, + command: + 'printf "%s|%s" "$LIBRECHAT_CODE_TEST_CREDENTIAL" "$LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY"', + }); + + assert.equal(result.stdout, 'Authorization: Bearer srt-sentinel|lia[bot]'); + assert.ok( + !fake.config?.credentials?.envVars?.some( + variable => variable.name === 'LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY', + ), + ); +}); + test('serializes credential handoff across concurrent sandbox instances', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 84602a35..5bf61c50 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -182,8 +182,15 @@ export interface NativeSrtWorkspaceCommandSandboxOptions { injectHosts: string[]; extract?: string; }>; - resolve(signal?: AbortSignal): Promise>; - wrapCommand?(command: string, platform: NodeJS.Platform): string; + resolve( + signal?: AbortSignal, + cwd?: string, + ): Promise>; + wrapCommand?( + command: string, + platform: NodeJS.Platform, + environment: Readonly>, + ): string; }; } @@ -872,18 +879,19 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const commandId = `librechat-code-${randomUUID()}`; - const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand - ? this.options.maskedEnvironment.wrapCommand( - request.command, - this.platform, - ) - : request.command; let wrapped: Awaited< ReturnType >; try { const credentialEnvironment = - await this.options.maskedEnvironment?.resolve(signal); + await this.options.maskedEnvironment?.resolve(signal, cwd); + const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand + ? this.options.maskedEnvironment.wrapCommand( + request.command, + this.platform, + credentialEnvironment ?? {}, + ) + : request.command; wrapped = await this.withTemporaryHostEnvironment( { ...TRUSTED_GIT_ENVIRONMENT, From dc48249741e9d9bd010d19a3d5fcf56e2c159b79 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 11:12:23 -0400 Subject: [PATCH 38/75] fix: Authenticate GitHub App bot identity lookup (#237) --- packages/code/src/github.test.ts | 61 ++++++++++++++++++++++---------- packages/code/src/github.ts | 54 +++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 26 deletions(-) diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 73e02927..352bf1c3 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -104,23 +104,30 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = { mode: 0o600 }, ); await chmod(privateKeyPath, 0o600); - let calls = 0; + const calls: Array<{ url: string; authorization: string | null }> = []; const request = async ( - _input: string | URL | Request, + input: string | URL | Request, init?: RequestInit, ) => { - calls += 1; - assert.match( - String(new Headers(init?.headers).get('authorization')), - /^Bearer eyJ/, - ); - return new Response( - JSON.stringify({ + const url = String(input); + const authorization = new Headers(init?.headers).get('authorization'); + calls.push({ url, authorization }); + if (url.endsWith('/app')) { + assert.match(String(authorization), /^Bearer eyJ/); + return Response.json({ slug: 'lia' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + assert.match(String(authorization), /^Bearer eyJ/); + return Response.json({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z', - }), - { status: 201 }, - ); + }, { status: 201 }); + } + if (url.endsWith('/users/lia%5Bbot%5D')) { + assert.equal(authorization, 'Bearer ghs_abcdefghijklmnopqrstuvwxyz'); + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); }; const provider = new GitHubAppCredentialProvider({ appId: '123', @@ -138,7 +145,15 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = (await provider.getCredential()).value, 'ghs_abcdefghijklmnopqrstuvwxyz', ); - assert.equal(calls, 1); + assert.equal(calls.filter(call => call.url.endsWith('/app')).length, 1); + assert.equal( + calls.filter(call => call.url.endsWith('/access_tokens')).length, + 1, + ); + assert.equal( + calls.filter(call => call.url.endsWith('/users/lia%5Bbot%5D')).length, + 1, + ); }); test('routes and scopes GitHub App tokens per repository installation', async (t) => { @@ -767,16 +782,26 @@ test('App JWT requests use the resolved public or enterprise endpoint', async (t now: () => new Date('2030-01-01T00:00:00Z'), fetch: async (input, init) => { calls++; - assert.equal(String(input), `${expected}/app/installations/456/access_tokens`); - assert.equal(init?.method, 'POST'); + const url = String(input); assert.equal(init?.redirect, 'error'); - assert.match(new Headers(init?.headers).get('authorization')!, /^Bearer eyJ/); - return new Response(JSON.stringify({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z' }), { status: 201 }); + const authorization = new Headers(init?.headers).get('authorization'); + if (url === `${expected}/app`) { + assert.match(authorization!, /^Bearer eyJ/); + return Response.json({ slug: 'lia' }); + } + if (url === `${expected}/app/installations/456/access_tokens`) { + assert.equal(init?.method, 'POST'); + assert.match(authorization!, /^Bearer eyJ/); + return new Response(JSON.stringify({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z' }), { status: 201 }); + } + assert.equal(url, `${expected}/users/lia%5Bbot%5D`); + assert.equal(authorization, 'Bearer ghs_abcdefghijklmnopqrstuvwxyz'); + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); }, }); await provider.getCredential(); await provider.getCredential(); - assert.equal(calls, 1); + assert.equal(calls, 3); } }); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 4c79fa37..fb6b6fdc 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -217,6 +217,8 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private readonly cached = new Map(); private readonly inFlight = new Map>(); private readonly installationIds = new Map(); + private appLogin?: string; + private appLoginInFlight?: Promise; private actor?: GitHubCredential['actor']; private actorInFlight?: Promise>; private readonly apiUrl: string; @@ -280,12 +282,12 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { }); } - private async resolveActor( + private async resolveAppLogin( jwt: string, signal?: AbortSignal, - ): Promise> { - if (this.actor) return this.actor; - if (!this.actorInFlight) { + ): Promise { + if (this.appLogin) return this.appLogin; + if (!this.appLoginInFlight) { const pending = (async () => { const sharedSignal = AbortSignal.timeout( GITHUB_SHARED_REQUEST_TIMEOUT_MS, @@ -303,10 +305,41 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ) { throw new Error('GitHub App identity response is invalid'); } - const login = `${app.slug}[bot]`; + return `${app.slug}[bot]`; + })(); + this.appLoginInFlight = pending; + void pending.then( + login => { + this.appLogin = login; + if (this.appLoginInFlight === pending) { + this.appLoginInFlight = undefined; + } + }, + () => { + if (this.appLoginInFlight === pending) { + this.appLoginInFlight = undefined; + } + }, + ); + } + return waitForShared(this.appLoginInFlight, signal); + } + + private async resolveActor( + jwt: string, + installationToken: string, + signal?: AbortSignal, + ): Promise> { + if (this.actor) return this.actor; + if (!this.actorInFlight) { + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const login = await this.resolveAppLogin(jwt, sharedSignal); const userResponse = await this.request( `/users/${encodeURIComponent(login)}`, - jwt, + installationToken, sharedSignal, ); if (!userResponse.ok) { @@ -349,7 +382,7 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { async validate(signal?: AbortSignal): Promise { const now = (this.options.now ?? (() => new Date()))(); const jwt = await this.appJwt(now); - await this.resolveActor(jwt, signal); + await this.resolveAppLogin(jwt, signal); if (this.options.installationId) { await this.getCredential(signal); } @@ -477,10 +510,15 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ) { throw new Error('GitHub App token expiry is invalid'); } + const actor = await this.resolveActor( + jwt, + body.token, + sharedSignal, + ); const credential = { value: body.token, expiresAt, - ...(this.actor ? { actor: this.actor } : {}), + actor, }; this.cached.set(key, credential); return credential; From 277fa7742d383eb1b6606ec228cdafe36af043a4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 14:43:11 -0400 Subject: [PATCH 39/75] feat: Advertise Workspace Command Timeout Ceiling (#238) --- packages/code/src/protocol.ts | 2 ++ service/src/bridge/index.ts | 1 + service/src/bridge/router.test.ts | 54 ++++++++++++++++++++++++++++++- service/src/bridge/router.ts | 15 +++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c5353ae2..27220735 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -712,6 +712,8 @@ export interface BridgeWorkerStatusResponse { online: boolean; ready: boolean; leaseExpiresInMs?: number; + /** Server-owned execution ceiling for workspace commands. Omitted by legacy servers. */ + maxCommandTimeoutMs?: number; capabilities?: BridgeWorkerCapabilities; } diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index a1ec5d78..16857fad 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -21,4 +21,5 @@ export default createBridgeRouter({ adminToken: env.BRIDGE_TOKEN, configuredWorkerId: env.BRIDGE_WORKER_ID, allowDynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + maxCommandTimeoutMs: env.JOB_TIMEOUT, }); diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index af0e765c..63640111 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -10,7 +10,10 @@ import { createBridgeIdentity, signBridgeRequest, } from '../../../packages/code/src/identity'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { + BRIDGE_PROTOCOL_VERSION, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, +} from '../../../packages/code/src/protocol'; import { RedisBridgePairingStore } from './pairing'; import { createBridgeRouter } from './router'; import { RedisBridgeStore } from './store'; @@ -115,6 +118,55 @@ describe('paired bridge HTTP API', () => { expect(unauthorized.status).toBe(401); }); + test('advertises the effective server command timeout for command-capable workers', async () => { + const store = new RedisBridgeStore(redis); + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store, + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'command-worker', + maxCommandTimeoutMs: 900_000, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'command-worker', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + const response = await fetch( + `http://127.0.0.1:${address.port}/v1/bridge/workers/command-worker/status`, + { headers: { Authorization: 'Bearer strong-administrator-bootstrap-token' } }, + ); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + workerId: 'command-worker', + maxCommandTimeoutMs: BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + }); + }); + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 369b306c..73870772 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -7,6 +7,7 @@ import type { BridgePrincipalType, BridgeWorkerBinding } from './pairing'; import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, @@ -36,6 +37,7 @@ export interface BridgeRouterOptions { adminToken: string; configuredWorkerId?: string; allowDynamicWorkers?: boolean; + maxCommandTimeoutMs?: number; } function sameToken(left: string, right: string): boolean { @@ -132,6 +134,16 @@ function isSettlement(value: unknown): value is CodeBridgeSettlement { export function createBridgeRouter(options: BridgeRouterOptions): Router { const router = Router(); if (options.enabled === false) return router; + if ( + options.maxCommandTimeoutMs !== undefined && + (!Number.isSafeInteger(options.maxCommandTimeoutMs) || options.maxCommandTimeoutMs < 1) + ) { + throw new RangeError('Workspace command timeout must be a positive safe integer'); + } + const maxCommandTimeoutMs = + options.maxCommandTimeoutMs == null + ? undefined + : Math.min(options.maxCommandTimeoutMs, BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS); const configuredWorker = (workerId: string): boolean => options.allowDynamicWorkers === true || @@ -324,10 +336,13 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { return; } const status = await options.store.workerStatus(workerId); + const supportsCommands = + status.capabilities?.workspaceTools?.operations.includes('execute_command') === true; res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, workerId, ...status, + ...(supportsCommands && maxCommandTimeoutMs != null ? { maxCommandTimeoutMs } : {}), }); }), ); From 95ebbd3cca39948c42e92d6c8228ace6a2f44298 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 21:42:37 -0400 Subject: [PATCH 40/75] =?UTF-8?q?=F0=9F=8C=B3=20feat:=20Provision=20Conver?= =?UTF-8?q?sation-Scoped=20Code=20Worktrees=20(#239)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: provision conversation-scoped code worktrees * fix: isolate conversation checkout metadata * docs: clarify isolated conversation checkouts * fix: revalidate conversation checkout sources * fix: preserve synchronous legacy execution startup * fix: harden conversation worktree lifecycle * test: use canonical workspace isolation keys * fix: secure conversation worktree provisioning * fix: preserve isolated workspace lifecycle * fix: harden conversation worktree provisioning * fix: fence worktree setup and credential routing * fix: use kernel-backed provisioning locks * fix: load worktree locking only when provisioned * fix: retain conversation provisioning ownership through recovery * fix: reserve provisioning before launching checkout writers * fix: pin Git provisioning inputs and close instance admission gaps --- packages/code/README.md | 59 +- packages/code/src/cli.ts | 190 +++- packages/code/src/git-snapshot.ts | 343 ++++++ packages/code/src/native-pool.test.ts | 120 ++- packages/code/src/native-pool.ts | 106 +- packages/code/src/native-process.test.ts | 1 + packages/code/src/native-sandbox.test.ts | 31 + packages/code/src/process-lock.test.ts | 61 ++ packages/code/src/process-lock.ts | 59 ++ packages/code/src/protocol.test.ts | 42 +- packages/code/src/protocol.ts | 42 +- packages/code/src/root-identity.ts | 16 + packages/code/src/worker-slots.test.ts | 35 +- packages/code/src/worker.ts | 163 ++- packages/code/src/workspace-cli.test.ts | 32 + packages/code/src/workspace-instances.test.ts | 234 +++++ packages/code/src/workspace-instances.ts | 232 ++++ packages/code/src/workspace-worker.test.ts | 100 ++ packages/code/src/worktrees.test.ts | 988 ++++++++++++++++++ packages/code/src/worktrees.ts | 718 +++++++++++++ service/src/bridge/concurrent-store.test.ts | 69 +- service/src/bridge/router.ts | 1 + service/src/bridge/store.test.ts | 12 +- service/src/bridge/store.ts | 55 +- service/src/bridge/workspace-instance.test.ts | 40 + service/src/bridge/workspace-instance.ts | 17 + service/src/service/programmatic-router.ts | 34 +- .../src/service/programmatic-state.test.ts | 16 + service/src/service/programmatic-state.ts | 12 + service/src/service/replay-state.ts | 2 + service/src/types/service.ts | 4 + service/src/workspace-tools/router.test.ts | 31 + service/src/workspace-tools/router.ts | 19 +- 33 files changed, 3803 insertions(+), 81 deletions(-) create mode 100644 packages/code/src/git-snapshot.ts create mode 100644 packages/code/src/process-lock.test.ts create mode 100644 packages/code/src/process-lock.ts create mode 100644 packages/code/src/workspace-instances.test.ts create mode 100644 packages/code/src/workspace-instances.ts create mode 100644 packages/code/src/worktrees.test.ts create mode 100644 packages/code/src/worktrees.ts create mode 100644 service/src/bridge/workspace-instance.test.ts create mode 100644 service/src/bridge/workspace-instance.ts diff --git a/packages/code/README.md b/packages/code/README.md index 69d85b16..e30b80cb 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -706,8 +706,63 @@ Slots are per machine, not a fleet-wide execution limit. A busy machine does not consume another machine's slots. Requests for the same root remain serialized, including commands started through background tools. Independent checkouts can use different slots; selecting subdirectories beneath one registered parent root -does not create separate scheduling boundaries. Linked Git worktrees share Git -metadata and are not supported by selected-project registration. +does not create separate scheduling boundaries. + +To bind each conversation to an isolated checkout of the selected Git +repository, configure worker-owned conversation worktrees: + +```sh +librechat-code run \ + --worker-dir /projects/LibreChat \ + --workspace-lease-slots 4 \ + --conversation-worktree-root /var/lib/librechat-code/worktrees \ + --conversation-worktree-max 64 \ + --conversation-worktree-clone-timeout-ms 300000 \ + --allow-workspace-writes \ + --allow-workspace-commands +``` + +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_ROOT` and +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX` are the environment equivalents; +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS` controls the bounded +clone budget (five minutes by default, from 30 seconds through 30 minutes). +The storage root must be owner-controlled, must not overlap a registered +workspace, and every registered source must be a Git repository. The worker +creates a deterministic branch in an isolated local checkout for the opaque +conversation identity supplied by LibreChat. Each checkout owns its writable +Git metadata and object storage, without alternates or hardlinks to the source. +Provisioning pins the source Git-directory and object-store identities. It copies +Git data through no-follow, descriptor-relative reads into private staging before +running Git; source hooks and config includes are not used. The clone budget +also bounds this snapshot. Local hardlinks only connect private staging to its +new checkout, never to the source; staging is removed before setup. Source +alternates admitted at worker startup are materialized into independent objects. +Git metadata replacement requires operator recovery, not automatic re-admission. +Host paths remain private. The configured count +is a hard per-machine quota, provisioning is serialized, and operations for one +conversation remain serialized while different conversations may occupy +different lease slots. Recognizable abandoned checkouts without a lifecycle +record are discarded before admission. New provisioning reserves its record +before starting Git or setup; a worker crash leaves that checkout reserved for +operator recovery because child processes might still be running. +Reservations count even when a crash happens before a checkout directory exists. + +Cancellation also covers waiting for the provisioning lock, cloning, and setup. +The worker waits for setup cleanup before releasing the assignment. If cleanup +cannot be confirmed, the checkout stays reserved and fails closed on restart. +A completed checkout with a changed source identity or invalid completion record +is preserved for operator recovery, including any uncommitted work. After stopping +the worker and confirming no executor still uses the checkout, an operator can +archive the affected checkout and its adjacent `.complete` record before retrying. +Also archive any adjacent `.source` staging directory. Pre-release version-1 +completion records are deliberately preserved but not admitted by this version; +they do not contain the required source Git identity binding. + +GitHub App routing is inherited from the operator-admitted source repository; +commands cannot select a different installation by rewriting a worktree remote. +Legacy requests without a conversation identity continue to use the selected +source root. Older Code API deployments do not negotiate the capability, so the +worker omits it until every request path understands the isolation boundary. Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, `Retry-After: 1`) means the operation was not assigned or started; wait for diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 96757d57..26c60e9a 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -36,6 +36,9 @@ import { import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; import { resolveNativeSrtCommandPolicy, serializeNativeSrtCommandPolicy, @@ -61,8 +64,9 @@ import type { WorkspaceToolExecutor } from './workspace.js'; import { BRIDGE_WORKSPACE_NAME_MAX_LENGTH, BridgeProtocolError, - isValidBridgeWorkerCapabilities, - isValidBridgeWorkerId, + isValidBridgeWorkerCapabilities, + isValidBridgeWorkerId, + workspaceIsolationKey, } from './protocol.js'; function workspaceSecurityIdentity( @@ -600,6 +604,32 @@ async function run( ); if (workspaceLeaseSlots > 8) throw new Error('Workspace lease slots cannot exceed 8'); + const conversationWorktreeRoot = + option(args, '--conversation-worktree-root') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_ROOT?.trim(); + const conversationWorktreeMax = positiveInteger( + 'LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX', + option(args, '--conversation-worktree-max') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX, + 64, + ); + if (conversationWorktreeMax > 1024) { + throw new Error('Conversation worktree capacity cannot exceed 1024'); + } + const conversationWorktreeCloneTimeoutMs = positiveInteger( + 'LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS', + option(args, '--conversation-worktree-clone-timeout-ms') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS, + 5 * 60_000, + ); + if ( + conversationWorktreeCloneTimeoutMs < 30_000 || + conversationWorktreeCloneTimeoutMs > 30 * 60_000 + ) { + throw new Error( + 'Conversation worktree clone timeout must be between 30000 and 1800000 milliseconds', + ); + } const roots: LocalWorkspaceConfig[] = canonicalWorkerDirectory ? [ { @@ -701,6 +731,16 @@ async function run( 'Concurrent workspace leases require native-srt commands', ); } + if ( + conversationWorktreeRoot && + (!allowWorkspaceCommands || + commandSandboxMode !== 'native-srt' || + workspaceLeaseSlots < 2) + ) { + throw new Error( + 'Conversation worktrees require native-srt commands and at least two workspace lease slots', + ); + } if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() @@ -732,6 +772,14 @@ async function run( ), ) : undefined; + const repositoriesByWorkspace = admittedGitHubRepositories + ? new Map( + roots.map((root) => [ + root.id, + admittedGitHubRepositories.get(root.root), + ]), + ) + : undefined; const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, @@ -981,7 +1029,7 @@ async function run( }; const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? roots.length > 1 || workspaceLeaseSlots > 1 + ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot ? new NativeWorkspaceCommandPool( new Map( roots.map(root => [ @@ -1008,6 +1056,105 @@ async function run( incarnationId, }), }); + } + const conversationWorktrees = conversationWorktreeRoot + ? new GitWorktreeManager({ + cloneTimeoutMs: conversationWorktreeCloneTimeoutMs, + maxCount: conversationWorktreeMax, + root: conversationWorktreeRoot, + ...(nativeCommandSandbox instanceof NativeWorkspaceCommandPool + ? { + prepareInstance: async (instance, signal) => { + const setup = environments.find( + (environment) => + environment.definition.name === instance.sourceWorkspaceId, + )?.definition.setup; + if (!setup) return; + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + instance.root, + repositoriesByWorkspace?.get(instance.sourceWorkspaceId), + ); + } + const id = internalWorkspaceId(instance.sourceWorkspaceId, instance.id); + await nativeCommandSandbox.registerRoot(id, { + ...nativeOptions, + workspaceIdentity: instance.identity, + workspaceRoot: instance.root, + }); + const result = await nativeCommandSandbox.execute( + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: id, + command: setup.command, + timeoutMs: setup.timeoutMs, + maxOutputBytes: 8192, + }, + signal, + ); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error( + `Environment ${instance.sourceWorkspaceId} setup failed for its conversation worktree`, + ); + } + }, + discardInstance: async (instance) => { + await nativeCommandSandbox.unregisterRoot( + internalWorkspaceId(instance.sourceWorkspaceId, instance.id), + ); + admittedGitHubRepositories?.delete(instance.root); + }, + } + : {}), + sources: new Map( + await Promise.all( + roots.map(async (root) => [ + root.id, + { + root: root.root, + identity: + root.identity ?? + (await captureWorkspaceRootIdentity(root.root)), + }, + ] as const), + ), + ), + }) + : undefined; + let conversationWorkspaceTools: GitWorktreeWorkspaceTools | undefined; + if (conversationWorktrees && workspaceTools) { + if (!(nativeCommandSandbox instanceof NativeWorkspaceCommandPool)) { + throw new Error('Conversation worktrees require a native command pool'); + } + conversationWorkspaceTools = new GitWorktreeWorkspaceTools({ + commandPool: nativeCommandSandbox, + delegate: workspaceTools, + manager: conversationWorktrees, + onResolve(workspaceId, root) { + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + root, + repositoriesByWorkspace?.get(workspaceId), + ); + } + }, + sources: new Map( + roots.map((root) => [ + root.id, + { + command: { + ...nativeOptions, + workspaceIdentity: root.identity, + workspaceRoot: root.root, + }, + repositoryInstructions: args.includes('--repository-instructions'), + writable: root.writable ?? false, + }, + ]), + ), + }); + workspaceTools = conversationWorkspaceTools; } if (workspaceTools && environments.length) { workspaceTools = new EnvironmentWorkspaceTools( @@ -1059,6 +1206,7 @@ async function run( if (github.provider && !github.provider.validate) { await github.provider.getCredential(controller.signal); } + await conversationWorktrees?.prepare(); await nativeCommandSandbox?.prepare(); for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { const setup = environment.definition.setup; @@ -1110,7 +1258,34 @@ async function run( capabilities, workspaceTools, ...(nativeProgrammaticEnabled && nativeCommandSandbox - ? { workspaceProgrammatic: nativeCommandSandbox } + ? { + workspaceProgrammatic: + conversationWorkspaceTools ?? nativeCommandSandbox, + } + : {}), + ...(conversationWorktrees + ? { + workspaceQuarantineResolver: async ( + selectedWorkspaceId: string, + workspaceInstanceId: string, + ) => + workspaceMutationGuard( + defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: await conversationWorktrees.plannedRoot( + selectedWorkspaceId, + workspaceInstanceId, + ), + }), + workerId, + workspaceIsolationKey( + selectedWorkspaceId, + workspaceInstanceId, + ), + incarnationId, + ), + } : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { @@ -1222,14 +1397,19 @@ async function run( } const resetNativeRoot = option(args, '--reset-workspace-quarantine'); if (resetNativeRoot != null) { + const resetWorkspaceInstance = option( + args, + '--reset-workspace-instance', + ); await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); await worker.resetNativeWorkspace( resetNativeRoot, controller.signal, + resetWorkspaceInstance, ); process.stdout.write( - `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}\n`, + `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}\n`, ); return; } diff --git a/packages/code/src/git-snapshot.ts b/packages/code/src/git-snapshot.ts new file mode 100644 index 00000000..7c57beda --- /dev/null +++ b/packages/code/src/git-snapshot.ts @@ -0,0 +1,343 @@ +import { constants, close, fstat, read } from 'node:fs'; +import { mkdir, open, opendir, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +const closeFd = promisify(close); +const statFd = promisify(fstat); +const readFd = promisify(read); +let binding: + | Promise<{ + openat: (fd: number, name: string, flags: number) => number; + errno: () => number; + }> + | undefined; + +async function childFd( + parent: number, + name: string +): Promise { + if (!name || name === '.' || name === '..' || name.includes('/')) + throw new Error('Invalid Git metadata entry'); + binding ??= import('koffi').then(({ default: koffi }) => ({ + openat: koffi + .load(null) + .func('int openat(int dirfd, const char *path, int flags)'), + errno: () => koffi.errno(), + })); + const native = await binding; + // Node does not expose O_CLOEXEC. Set it atomically with openat so unrelated + // concurrent executor spawns cannot inherit privileged source descriptors. + if (process.platform !== 'darwin' && process.platform !== 'linux') + throw new Error('Git snapshots require a POSIX host'); + const closeOnExec = process.platform === 'darwin' ? 0x1000000 : 0x80000; + const fd = native.openat( + parent, + name, + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK | + closeOnExec + ); + if (fd >= 0) return fd; + if (native.errno() === 2) return undefined; // ENOENT on supported POSIX hosts + throw new Error('Git metadata entry is unavailable or is a symbolic link'); +} + +async function withDirectory( + identity: WorkspaceRootIdentity, + operation: (fd: number) => Promise +): Promise { + const handle = await open( + identity.path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW + ); + try { + const current = await handle.stat({ bigint: true }); + if ( + current.dev.toString() !== identity.dev || + current.ino.toString() !== identity.ino + ) { + throw new Error('Source Git metadata changed after admission'); + } + return await operation(handle.fd); + } finally { + await handle.close(); + } +} + +async function textAt( + parent: number, + name: string, + limit = 16 * 1024 +): Promise { + const fd = await childFd(parent, name); + if (fd == null) return undefined; + try { + const metadata = await statFd(fd); + if (!metadata.isFile() || metadata.size > limit) + throw new Error('Invalid Git metadata file'); + const buffer = Buffer.alloc(metadata.size); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await readFd( + fd, + buffer, + offset, + buffer.length - offset, + offset + ); + if (!bytesRead) throw new Error('Git metadata changed during snapshot'); + offset += bytesRead; + } + return buffer.toString('utf8'); + } finally { + await closeFd(fd); + } +} + +async function alternatesAt(parent: number): Promise { + const fd = await childFd(parent, 'info'); + if (fd == null) return undefined; + try { + if (!(await statFd(fd)).isDirectory()) + throw new Error('Invalid Git objects info directory'); + return await textAt(fd, 'alternates'); + } finally { + await closeFd(fd); + } +} + +/** Copies only regular files/directories through descriptor-relative, no-follow opens. + * Renaming a parent or replacing a child with a symlink never expands the read grant. + * No source config, hooks, object alternates, or executable helpers reach Git. + */ +async function copyEntry( + parent: number, + name: string, + destination: string, + signal: AbortSignal | undefined, + depth = 0 +): Promise { + signal?.throwIfAborted(); + if (depth > 64) + throw new Error('Git metadata nesting exceeds snapshot limit'); + const fd = await childFd(parent, name); + if (fd == null) return; + try { + const metadata = await statFd(fd); + if (metadata.isDirectory()) { + await mkdir(destination, { recursive: true, mode: 0o700 }); + const directory = await opendir(`/dev/fd/${fd}`); + for await (const entry of directory) { + await copyEntry( + fd, + entry.name, + join(destination, entry.name), + signal, + depth + 1 + ); + } + } else if (metadata.isFile()) { + const target = await open(destination, 'w', 0o600); + try { + const buffer = Buffer.alloc(128 * 1024); + let offset = 0; + while (offset < metadata.size) { + signal?.throwIfAborted(); + const { bytesRead } = await readFd( + fd, + buffer, + 0, + Math.min(buffer.length, metadata.size - offset), + offset + ); + if (!bytesRead) + throw new Error('Git metadata changed during snapshot'); + await target.writeFile(buffer.subarray(0, bytesRead)); + offset += bytesRead; + } + } finally { + await target.close(); + } + } else { + throw new Error('Git snapshot requires regular files and directories'); + } + } finally { + await closeFd(fd); + } +} + +export class GitSourceSnapshot { + private constructor( + private readonly source: WorkspaceRootIdentity, + private readonly gitDirectory: WorkspaceRootIdentity, + private readonly common: WorkspaceRootIdentity, + private readonly gitfile: string | undefined, + private readonly commondir: string | undefined, + private readonly objects: Array<{ + identity: WorkspaceRootIdentity; + alternates: string | undefined; + }> + ) {} + + get fingerprint(): string { + return createHash('sha256') + .update( + JSON.stringify([ + this.gitDirectory, + this.common, + this.gitfile, + this.commondir, + this.objects, + ]) + ) + .digest('hex'); + } + + static async admit( + source: WorkspaceRootIdentity + ): Promise { + let gitfile: string | undefined; + await withDirectory(source, async (fd) => { + const git = await childFd(fd, '.git'); + if (git == null) + throw new Error('Source workspace is not a Git repository'); + try { + if (!(await statFd(git)).isDirectory()) + gitfile = await textAt(fd, '.git'); + } finally { + await closeFd(git); + } + }); + if (gitfile != null && !/^gitdir: .+\n?$/.test(gitfile)) + throw new Error('Invalid source Git directory pointer'); + const gitDirectory = await captureWorkspaceRootIdentity( + gitfile == null + ? join(source.path, '.git') + : resolve(source.path, gitfile.slice(8).trim()) + ); + const commondir = await withDirectory(gitDirectory, (fd) => + textAt(fd, 'commondir') + ); + const common = + commondir == null + ? gitDirectory + : await captureWorkspaceRootIdentity( + resolve(gitDirectory.path, commondir.trim()) + ); + const objects: Array<{ + identity: WorkspaceRootIdentity; + alternates: string | undefined; + }> = []; + const visit = async (path: string): Promise => { + const identity = await captureWorkspaceRootIdentity(path); + if (objects.some((entry) => entry.identity.path === identity.path)) + return; + if (objects.length >= 32) + throw new Error('Too many source Git object stores'); + const alternates = await withDirectory(identity, alternatesAt); + objects.push({ identity, alternates }); + for (const alternate of alternates?.split('\n').filter(Boolean) ?? []) { + if (alternate.startsWith('"')) + throw new Error('Quoted Git alternate paths are unsupported'); + await visit(resolve(identity.path, alternate)); + } + }; + await visit(join(common.path, 'objects')); + const admitted = new GitSourceSnapshot( + source, + gitDirectory, + common, + gitfile, + commondir, + objects + ); + await admitted.validate(); + return admitted; + } + + async validate(): Promise { + await withDirectory(this.source, async (fd) => { + if (this.gitfile != null) { + if ((await textAt(fd, '.git')) !== this.gitfile) + throw new Error('Source Git metadata changed after admission'); + } else { + // Compare the directory reached from the admitted root, not just its name. + const child = await childFd(fd, '.git'); + if (child == null) + throw new Error('Source Git metadata changed after admission'); + try { + const current = await promisify(fstat)(child, { + bigint: true, + }); + if ( + current.dev.toString() !== this.gitDirectory.dev || + current.ino.toString() !== this.gitDirectory.ino + ) { + throw new Error('Source Git metadata changed after admission'); + } + } finally { + await closeFd(child); + } + } + }); + await withDirectory(this.gitDirectory, async (fd) => { + if ((await textAt(fd, 'commondir')) !== this.commondir) + throw new Error('Source Git metadata changed after admission'); + }); + await withDirectory(this.common, async () => {}); + for (const entry of this.objects) { + await withDirectory(entry.identity, async (fd) => { + if ((await alternatesAt(fd)) !== entry.alternates) + throw new Error('Source Git alternates changed after admission'); + }); + } + } + + async copyTo(destination: string, signal?: AbortSignal): Promise { + await this.validate(); + await mkdir(destination, { mode: 0o700 }); + await mkdir(join(destination, 'objects'), { mode: 0o700 }); + await mkdir(join(destination, 'refs'), { mode: 0o700 }); + await withDirectory(this.gitDirectory, (fd) => + copyEntry(fd, 'HEAD', join(destination, 'HEAD'), signal) + ); + await withDirectory(this.common, async (fd) => { + for (const name of ['refs', 'packed-refs', 'shallow']) + await copyEntry(fd, name, join(destination, name), signal); + // Kept outside Git's config name; caller may query origin with --no-includes. + const config = await textAt(fd, 'config', 1024 * 1024); + if (config != null) + await writeFile(join(destination, 'source-config'), config, { + mode: 0o600, + }); + }); + for (const entry of [...this.objects].reverse()) { + await withDirectory(entry.identity, async (fd) => { + const directory = await opendir(`/dev/fd/${fd}`); + for await (const child of directory) { + // Object info (notably alternates) never crosses into private staging. + if (child.name === 'pack' || /^[a-f0-9]{2}$/.test(child.name)) { + await copyEntry( + fd, + child.name, + join(destination, 'objects', child.name), + signal + ); + } + } + }); + } + await writeFile( + join(destination, 'config'), + '[core]\nrepositoryformatversion = 0\nbare = true\n', + { mode: 0o600 } + ); + await this.validate(); + signal?.throwIfAborted(); + } +} diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index cad30a52..ce25caa4 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -5,7 +5,7 @@ import { WorkspaceToolError } from './workspace.js'; import type { WorkspaceExecuteCommandRequest } from './protocol.js'; const roots = new Map( - ['a', 'b', 'c'].map((id) => [id, { workspaceRoot: `/fixture/${id}` }]), + ['a', 'b', 'c'].map((id) => [id, { workspaceRoot: `/fixture/${id}` }]) ); const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ protocolVersion: 1, @@ -14,6 +14,36 @@ const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ command: 'fixture', }); +test('failed provisioning roots are removed only after confirmed executor cleanup', async () => { + let failClose = true; + const pool = new NativeWorkspaceCommandPool(roots, 2, () => ({ + async prepare() {}, + async execute() { + throw new Error('not used'); + }, + async close() { + if (failClose) throw new Error('cleanup unconfirmed'); + }, + })); + await pool.registerRoot('instance', { workspaceRoot: '/fixture/instance' }); + // Allocate this executor without dispatching a command. + const controller = new AbortController(); + controller.abort(); + await assert.rejects( + pool.execute(request('instance'), controller.signal), + /cancelled/ + ); + await assert.rejects(pool.unregisterRoot('instance'), /cleanup unconfirmed/); + failClose = false; + await pool.unregisterRoot('instance'); + await assert.rejects(pool.execute(request('instance')), /unavailable/); + await pool.close(); + await assert.rejects( + pool.registerRoot('new', { workspaceRoot: '/fixture/new' }), + /unavailable/ + ); +}); + test('native pool preflights every registered root with bounded concurrency', async () => { const prepared: string[] = []; let active = 0; @@ -22,7 +52,7 @@ test('native pool preflights every registered root with bounded concurrency', as async prepare() { active += 1; peak = Math.max(peak, active); - await new Promise(resolve => setTimeout(resolve, 5)); + await new Promise((resolve) => setTimeout(resolve, 5)); prepared.push(options.workspaceRoot); active -= 1; }, @@ -38,6 +68,90 @@ test('native pool preflights every registered root with bounded concurrency', as await pool.close(); }); +test('native pool admits worker-owned roots after startup', async () => { + const created: string[] = []; + const pool = new NativeWorkspaceCommandPool( + new Map([['primary', { workspaceRoot: '/fixture/primary' }]]), + 2, + (options) => ({ + async prepare() {}, + async close() {}, + async execute(req) { + created.push(options.workspaceRoot); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }) + ); + await pool.registerRoot('conversation', { + workspaceRoot: '/fixture/conversation', + }); + await pool.execute(request('conversation')); + assert.deepEqual(created, ['/fixture/conversation']); + await assert.rejects( + async () => + pool.registerRoot('conversation', { + workspaceRoot: '/fixture/replaced', + }), + { code: 'REGISTRATION_INVALID' } + ); + await pool.close(); +}); + +test('native pool retires a cached executor when a root inode changes', async () => { + let created = 0; + let closed = 0; + const pool = new NativeWorkspaceCommandPool( + new Map([['primary', { workspaceRoot: '/fixture/primary' }]]), + 2, + () => { + created++; + return { + async prepare() {}, + async close() { + closed++; + }, + async execute(req) { + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }; + } + ); + const options = (ino: string) => ({ + workspaceRoot: '/fixture/conversation', + workspaceIdentity: { + path: '/fixture/conversation', + dev: '1', + ino, + }, + }); + await pool.registerRoot('conversation', options('1')); + await pool.execute(request('conversation')); + await pool.registerRoot('conversation', options('2')); + await pool.execute(request('conversation')); + + assert.equal(created, 2); + assert.equal(closed, 1); + await pool.close(); +}); + test('a known-clean executor failure is retired without replaying the command', async () => { let created = 0; let executed = 0; @@ -55,7 +169,7 @@ test('a known-clean executor failure is retired without replaying the command', throw new WorkspaceToolError( 'prepare failed', 'COMMAND_UNAVAILABLE', - false, + false ); return { protocolVersion: 1, diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 766409b1..59db3e0a 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -12,9 +12,7 @@ interface Entry { NativeProcessWorkspaceCommandSandbox, 'prepare' | 'execute' | 'close' > & - Partial< - Pick - >; + Partial>; busy: boolean; } @@ -27,12 +25,12 @@ export class NativeWorkspaceCommandPool { private allocation: Promise = Promise.resolve(); private closing = false; constructor( - private readonly roots: ReadonlyMap, + roots: ReadonlyMap, private readonly capacity: number, private readonly createSandbox: ( - options: NativeProcessSandboxOptions, + options: NativeProcessSandboxOptions ) => Entry['sandbox'] = (options) => - new NativeProcessWorkspaceCommandSandbox(options), + new NativeProcessWorkspaceCommandSandbox(options) ) { if ( !Number.isSafeInteger(capacity) || @@ -42,6 +40,74 @@ export class NativeWorkspaceCommandPool { ) { throw new Error('Native executor capacity must be between 1 and 8'); } + this.roots = new Map(roots); + } + + private readonly roots: Map; + + /** Confirm executor cleanup before a failed provisioning attempt removes its root. */ + async unregisterRoot(id: string): Promise { + const pending = this.allocation.then(async () => { + const entry = this.entries.get(id); + if (entry?.busy) { + throw new WorkspaceToolError( + 'Native workspace still executing', + 'COMMAND_UNAVAILABLE' + ); + } + if (entry) await entry.sandbox.close(); + this.entries.delete(id); + this.roots.delete(id); + }); + this.allocation = pending.catch(() => undefined); + await pending; + } + + /** Add or safely replace a worker-owned isolated root. */ + async registerRoot( + id: string, + options: NativeProcessSandboxOptions + ): Promise { + const pending = this.allocation.then(async () => { + if (this.closing) { + throw new WorkspaceToolError( + 'Native workspace unavailable', + 'REGISTRATION_INVALID' + ); + } + const existing = this.roots.get(id); + if (!existing) { + this.roots.set(id, options); + return; + } + if (existing.workspaceRoot !== options.workspaceRoot) { + throw new WorkspaceToolError( + 'Native workspace identity changed', + 'REGISTRATION_INVALID' + ); + } + if ( + existing.workspaceIdentity?.dev === options.workspaceIdentity?.dev && + existing.workspaceIdentity?.ino === options.workspaceIdentity?.ino && + existing.workspaceIdentity?.path === options.workspaceIdentity?.path + ) { + return; + } + const entry = this.entries.get(id); + if (entry?.busy) { + throw new WorkspaceToolError( + 'Native workspace changed during execution', + 'REGISTRATION_INVALID' + ); + } + if (entry) { + await entry.sandbox.close(); + this.entries.delete(id); + } + this.roots.set(id, options); + }); + this.allocation = pending.catch(() => undefined); + await pending; } private allocate(root: string): Promise { @@ -50,23 +116,23 @@ export class NativeWorkspaceCommandPool { if (this.closing || !options) throw new WorkspaceToolError( 'Native workspace unavailable', - 'REGISTRATION_INVALID', + 'REGISTRATION_INVALID' ); let entry = this.entries.get(root); if (entry?.busy) throw new WorkspaceToolError( 'Native workspace already executing', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); if (!entry) { if (this.entries.size >= this.capacity) { const idle = [...this.entries].find( - ([, candidate]) => !candidate.busy, + ([, candidate]) => !candidate.busy ); if (!idle) throw new WorkspaceToolError( 'Native executor capacity reached', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); await idle[1].sandbox.close(); this.entries.delete(idle[0]); @@ -88,7 +154,7 @@ export class NativeWorkspaceCommandPool { if (error instanceof WorkspaceToolError) throw error; throw new WorkspaceToolError( 'Native executor allocation failed', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); }); this.allocation = checked.catch(() => undefined); @@ -112,14 +178,14 @@ export class NativeWorkspaceCommandPool { entry.busy = false; } } - }, - ), + } + ) ); } async execute( request: WorkspaceExecuteCommandRequest, - signal?: AbortSignal, + signal?: AbortSignal ): Promise { const entry = await this.allocate(request.workspaceId); let enteredExecutor = false; @@ -127,7 +193,7 @@ export class NativeWorkspaceCommandPool { if (signal?.aborted) throw new WorkspaceToolError( 'Command cancelled before dispatch', - 'EXECUTION_ABORTED', + 'EXECUTION_ABORTED' ); enteredExecutor = true; return await entry.sandbox.execute(request, signal); @@ -156,7 +222,7 @@ export class NativeWorkspaceCommandPool { async executeProgrammatic( workspaceId: string, request: BridgeWorkspaceProgrammaticRequest, - signal?: AbortSignal, + signal?: AbortSignal ): Promise { const entry = await this.allocate(workspaceId); let enteredExecutor = false; @@ -164,19 +230,19 @@ export class NativeWorkspaceCommandPool { if (signal?.aborted) throw new WorkspaceToolError( 'Programmatic execution cancelled before dispatch', - 'EXECUTION_ABORTED', + 'EXECUTION_ABORTED' ); enteredExecutor = true; if (!entry.sandbox.executeProgrammatic) { throw new WorkspaceToolError( 'Native programmatic executor is unavailable', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); } return await entry.sandbox.executeProgrammatic( workspaceId, request, - signal, + signal ); } catch (error) { if ( @@ -202,7 +268,7 @@ export class NativeWorkspaceCommandPool { this.closing = true; await this.allocation; const results = await Promise.allSettled( - [...this.entries.values()].map((entry) => entry.sandbox.close()), + [...this.entries.values()].map((entry) => entry.sandbox.close()) ); this.entries.clear(); const errors = results diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index b3016937..5f7eedbb 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -146,6 +146,7 @@ test('executor bootstrap excludes bridge credentials and Node injection variable assert.deepEqual(fake.options?.execArgv, []); assert.deepEqual(fake.options?.env, { PATH: '/bin' }); assert.equal(JSON.stringify(fake.messages).includes('secret'), false); + assert.equal('gitSharedObjectDirectory' in fake.messages[0].options, false); await sandbox.close(); }); diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index b1e4c5df..eab72966 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -590,6 +590,37 @@ test('trusted-vm permits unmatched egress and local development sockets', async ]); }); +test('recreated executors never grant reads through a replaced Git object directory', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-worktree-')); + const root = join(parent, 'worktree'); + const gitSharedObjectDirectory = join(parent, 'source.git', 'objects'); + await mkdir(join(root, '.git'), { recursive: true }); + await mkdir(gitSharedObjectDirectory, { recursive: true }); + await symlink(gitSharedObjectDirectory, join(root, '.git', 'objects')); + t.after(() => rm(parent, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + + await sandbox.prepare(); + + assert.equal( + fake.config?.filesystem.allowRead?.includes( + await realpath(gitSharedObjectDirectory), + ), + false, + ); + assert.equal( + fake.config?.filesystem.allowWrite?.includes( + await realpath(gitSharedObjectDirectory), + ), + false, + ); +}); + test('provides an isolated scratch directory to commands and restores the host environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/process-lock.test.ts b/packages/code/src/process-lock.test.ts new file mode 100644 index 00000000..3594a1a9 --- /dev/null +++ b/packages/code/src/process-lock.test.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { withProcessLock } from './process-lock.js'; + +test( + 'kernel lock survives contention and is released when the owning process crashes', + { timeout: 10_000 }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-lock-')); + const path = join(directory, '.provision.lock'); + const moduleUrl = new URL('./process-lock.js', import.meta.url).href; + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { withProcessLock } from ${JSON.stringify(moduleUrl)}; + await withProcessLock(${JSON.stringify(path)}, async () => { + process.stdout.write('locked'); + await new Promise(() => { setInterval(() => {}, 1000); }); + }); + `, + ], + { stdio: ['ignore', 'pipe', 'pipe'] } + ); + const closed = once(child, 'close'); + t.after(async () => { + child.kill('SIGKILL'); + await closed; + await rm(directory, { recursive: true, force: true }); + }); + await once(child.stdout!, 'data'); + let entered = false; + await assert.rejects( + withProcessLock( + path, + async () => { + entered = true; + }, + AbortSignal.timeout(100) + ) + ); + assert.equal(entered, false); + child.kill('SIGKILL'); + await closed; + await withProcessLock( + path, + async () => { + entered = true; + }, + AbortSignal.timeout(1000) + ); + assert.equal(entered, true); + } +); diff --git a/packages/code/src/process-lock.ts b/packages/code/src/process-lock.ts new file mode 100644 index 00000000..aeca561a --- /dev/null +++ b/packages/code/src/process-lock.ts @@ -0,0 +1,59 @@ +import { constants } from 'node:fs'; +import { open } from 'node:fs/promises'; +import { setTimeout as delay } from 'node:timers/promises'; + +const LOCK_EX = 2; +const LOCK_NB = 4; +const LOCK_UN = 8; +let binding: + | Promise<{ + flock: (fd: number, operation: number) => number; + eagain: number; + errno: () => number; + }> + | undefined; + +async function lockBinding() { + binding ??= import('koffi').then(({ default: koffi }) => ({ + flock: koffi.load(null).func('int flock(int fd, int operation)'), + eagain: koffi.os.errno.EAGAIN, + errno: () => koffi.errno(), + })); + return await binding; +} + +/** Process-lifetime advisory lock; the kernel releases it on crash or restart. */ +export async function withProcessLock( + path: string, + operation: () => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted(); + if (process.platform !== 'darwin' && process.platform !== 'linux') { + throw new Error('Conversation worktree locking requires a POSIX host'); + } + const native = await lockBinding(); + const handle = await open( + path, + constants.O_CREAT | constants.O_RDWR | constants.O_NOFOLLOW, + 0o600 + ); + try { + for (;;) { + signal?.throwIfAborted(); + if (native.flock(handle.fd, LOCK_EX | LOCK_NB) === 0) break; + const errno = native.errno(); + if (errno !== native.eagain) { + throw new Error( + `Conversation worktree lock failed with errno ${errno}` + ); + } + await delay(50, undefined, { signal }); + } + signal?.throwIfAborted(); + return await operation(); + } finally { + native.flock(handle.fd, LOCK_UN); + await handle.close(); + } +} diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 08cba1c7..5c26c462 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -10,6 +10,7 @@ import { isValidBridgeWorkerId, isWorkspaceToolRequest, isWorkspaceToolResult, + workspaceIsolationKey, } from './protocol.js'; import type { WorkspaceEditFileRequest, @@ -90,6 +91,19 @@ test('bridgeWorkerPath encodes worker-controlled path segments', () => { ); }); +test('workspace isolation keys keep roots and instances in disjoint namespaces', () => { + const instanceId = 'a'.repeat(64); + assert.notEqual( + workspaceIsolationKey(`foo:git-worktree:${instanceId}`), + workspaceIsolationKey('foo', instanceId), + ); + assert.equal(workspaceIsolationKey('foo'), 'foo'); + assert.notEqual( + workspaceIsolationKey('foo'), + workspaceIsolationKey('workspace:foo'), + ); +}); + test('bridge worker IDs reject path, whitespace, and oversized values', () => { assert.equal(isValidBridgeWorkerId('engineering-vm:1'), true); assert.equal(isValidBridgeWorkerId('engineering/vm'), false); @@ -262,6 +276,20 @@ test('workspace file listing accepts only bounded portable requests and results' afterPath: 'src/app.ts', }; assert.equal(isWorkspaceToolRequest(request), true); + assert.equal( + isWorkspaceToolRequest({ + ...request, + workspaceInstanceId: 'a'.repeat(64), + }), + true, + ); + assert.equal( + isWorkspaceToolRequest({ + ...request, + workspaceInstanceId: 'conversation-1', + }), + false, + ); assert.equal( isWorkspaceToolRequest({ ...request, path: '../outside' }), false, @@ -596,7 +624,11 @@ test('workspace capabilities allow per-workspace operation restrictions', () => operations: ['read_file', 'write_file'], workspaces: [ { id: 'readonly', operations: ['read_file'] }, - { id: 'writable', operations: ['read_file', 'write_file'] }, + { + id: 'writable', + operations: ['read_file', 'write_file'], + workspaceInstances: ['git_worktree'], + }, ], }, }; @@ -660,6 +692,7 @@ test('workspace programmatic requests accept only stable input cache identities' max_output_files: 50, max_output_file_bytes: 10_000_000, session_id: 'session-1', + workspace_instance_id: 'a'.repeat(64), files: [ { name: 'main.sh', content: 'echo ready' }, { @@ -672,6 +705,13 @@ test('workspace programmatic requests accept only stable input cache identities' }, }; assert.equal(isBridgeWorkspaceProgrammaticRequest(request), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + ...request, + body: { ...request.body, workspace_instance_id: '../escape' }, + }), + false, + ); assert.equal( isBridgeWorkspaceProgrammaticRequest({ ...request, diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 27220735..c678a1af 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -259,6 +259,16 @@ export function bridgeArtifactMediaType(name: string): string { export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; +/** Collision-free identity shared by scheduling and worker quarantine state. */ +export function workspaceIsolationKey( + workspaceId: string, + instanceId?: string, +): string { + return instanceId === undefined + ? workspaceId + : `\0git-worktree\0${workspaceId}\0${instanceId}`; +} + export type BridgeWorkspaceToolOperation = | 'read_file' | 'search_text' @@ -280,6 +290,8 @@ export interface BridgeWorkspaceDescriptor { instructions?: RepositoryInstructionDescriptor[]; /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ operations?: BridgeWorkspaceToolOperation[]; + /** Worker-owned isolation schemes available beneath this selected root. */ + workspaceInstances?: ['git_worktree']; environment?: { fingerprint: string; repo?: string; @@ -308,6 +320,7 @@ export interface WorkspaceReadFileRequest { protocolVersion: BridgeProtocolVersion; operation: 'read_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; startLine?: number; maxLines?: number; @@ -350,6 +363,7 @@ export interface WorkspaceSearchTextRequest { protocolVersion: BridgeProtocolVersion; operation: 'search_text'; workspaceId: string; + workspaceInstanceId?: string; query: string; path?: string; maxResults?: number; @@ -374,6 +388,7 @@ export interface WorkspaceListFilesRequest { protocolVersion: BridgeProtocolVersion; operation: 'list_files'; workspaceId: string; + workspaceInstanceId?: string; path?: string; maxResults?: number; /** Continue strictly after this canonical path from a previous page. */ @@ -394,6 +409,7 @@ export interface WorkspaceWriteFileRequest { protocolVersion: BridgeProtocolVersion; operation: 'write_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; content: string; /** False requires an atomic create and refuses to replace an existing file. */ @@ -413,6 +429,7 @@ interface WorkspaceEditFileRequestBase { protocolVersion: BridgeProtocolVersion; operation: 'edit_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; /** Refuses the mutation unless current file bytes match this preview revision. */ expectedBaseSha256?: string; @@ -457,6 +474,7 @@ interface WorkspacePreviewEditRequestBase { protocolVersion: BridgeProtocolVersion; operation: 'preview_edit'; workspaceId: string; + workspaceInstanceId?: string; path: string; } @@ -494,6 +512,7 @@ export interface WorkspaceExecuteCommandRequest { protocolVersion: BridgeProtocolVersion; operation: 'execute_command'; workspaceId: string; + workspaceInstanceId?: string; /** Shell source evaluated only inside the selected sandbox runtime. */ command: string; /** Portable path relative to the workspace root; defaults to '.'. */ @@ -538,6 +557,7 @@ const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'startLine', 'maxLines', @@ -546,6 +566,7 @@ const WORKSPACE_SEARCH_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'query', 'path', 'maxResults', @@ -554,6 +575,7 @@ const WORKSPACE_LIST_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'maxResults', 'afterPath', @@ -562,6 +584,7 @@ const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'content', 'overwrite', @@ -570,6 +593,7 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'oldText', 'newText', @@ -580,6 +604,7 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'oldText', 'newText', @@ -591,6 +616,7 @@ const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'command', 'cwd', 'timeoutMs', @@ -702,6 +728,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceListFileFeatures?: WorkspaceListFileFeature[]; /** PTC languages this Code API can safely route into a selected workspace. */ supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; + /** Workspace isolation schemes this Code API understands and can route. */ + supportedWorkspaceInstanceTypes?: ['git_worktree']; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -748,6 +776,7 @@ export type BridgeProgrammaticPayloadFile = export interface BridgeWorkspaceProgrammaticBody { language: 'bash'; version: string; + workspace_instance_id?: string; /** Stable identity shared by every replay iteration of one execution. */ execution_id?: string; /** Declared replay tools; zero allows the worker to skip the probe pass. */ @@ -912,6 +941,9 @@ export function isBridgeWorkspaceProgrammaticRequest( typeof body.version !== 'string' || body.version.length === 0 || body.version.length > BRIDGE_RUNTIME_MAX_LENGTH || + (body.workspace_instance_id !== undefined && + (typeof body.workspace_instance_id !== 'string' || + !/^[a-f0-9]{64}$/.test(body.workspace_instance_id))) || (body.execution_id !== undefined && (typeof body.execution_id !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || @@ -1161,7 +1193,10 @@ export function isWorkspaceToolRequest( if ( request.protocolVersion !== BRIDGE_PROTOCOL_VERSION || typeof request.workspaceId !== 'string' || - !isValidBridgeWorkerId(request.workspaceId) + !isValidBridgeWorkerId(request.workspaceId) || + (request.workspaceInstanceId !== undefined && + (typeof request.workspaceInstanceId !== 'string' || + !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) ) { return false; } @@ -1614,12 +1649,17 @@ export function isValidBridgeWorkspaceToolCapabilities( key !== 'id' && key !== 'name' && key !== 'operations' && + key !== 'workspaceInstances' && key !== 'instructions' && key !== 'environment', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || + (descriptor.workspaceInstances !== undefined && + (!Array.isArray(descriptor.workspaceInstances) || + descriptor.workspaceInstances.length !== 1 || + descriptor.workspaceInstances[0] !== 'git_worktree')) || (descriptor.instructions !== undefined && (!Array.isArray(descriptor.instructions) || descriptor.instructions.length > 1 || !descriptor.instructions.every(isRepositoryInstructionDescriptor))) || (descriptor.environment !== undefined && !isValidCodeEnvironmentDescriptor(descriptor.environment)) || diff --git a/packages/code/src/root-identity.ts b/packages/code/src/root-identity.ts index 3ddfe453..dd3bb2f5 100644 --- a/packages/code/src/root-identity.ts +++ b/packages/code/src/root-identity.ts @@ -6,6 +6,22 @@ export interface WorkspaceRootIdentity { ino: string; } +/** Capture the inode-bound identity of a canonical workspace grant. */ +export async function captureWorkspaceRootIdentity( + root: string, +): Promise { + const canonical = await realpath(root); + const current = await lstat(canonical, { bigint: true }); + if (!current.isDirectory() || current.isSymbolicLink()) { + throw new Error('Workspace root must be a real directory'); + } + return { + path: canonical, + dev: current.dev.toString(), + ino: current.ino.toString(), + }; +} + /** Revalidation of a trusted snapshot, never a fresh grant to a replacement. */ export async function matchesWorkspaceRoot( root: string, diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index 6e8cc8b1..fda6c659 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -5,6 +5,7 @@ import type { BridgeAssignment, BridgeWorkspaceToolCapabilities, } from './protocol.js'; +import { workspaceIsolationKey } from './protocol.js'; const capabilities: BridgeWorkspaceToolCapabilities = { protocolVersion: 1, @@ -229,7 +230,8 @@ for (const cancelled of [false, true]) { rejectUnexecutedAssignment: () => Promise; executeOwned: () => Promise; }; - internals.activeWorkspaceAssignments.set('a', { + const workspaceKey = workspaceIsolationKey('a'); + internals.activeWorkspaceAssignments.set(workspaceKey, { id: 'previous', done: new Promise(() => {}), }); @@ -257,7 +259,10 @@ for (const cancelled of [false, true]) { controller.signal, ); assert.equal(rejected, true); - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceKey)?.id, + 'previous', + ); }); } @@ -281,7 +286,8 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b executeOwned: (assignment: BridgeAssignment) => Promise; }; let release!: () => void; - internals.activeWorkspaceAssignments.set('a', { + const workspaceKey = workspaceIsolationKey('a'); + internals.activeWorkspaceAssignments.set(workspaceKey, { id: 'previous', done: new Promise((resolve) => { release = resolve; @@ -290,7 +296,10 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b let executed = false; internals.executeOwned = async (assignment) => { executed = true; - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'next'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceKey)?.id, + 'next', + ); assert.ok(assignment.remainingMs! < 1000 && assignment.remainingMs! > 0); }; const pending = worker.executeAndSettle({ @@ -306,7 +315,7 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b } as BridgeAssignment); await new Promise((resolve) => setTimeout(resolve, 5)); assert.equal(executed, false); - internals.activeWorkspaceAssignments.delete('a'); + internals.activeWorkspaceAssignments.delete(workspaceKey); release(); await pending; assert.equal(executed, true); @@ -332,14 +341,19 @@ test('programmatic work on an independent workspace bypasses another root cleanu >; executeOwned: (assignment: BridgeAssignment) => Promise; }; - internals.activeWorkspaceAssignments.set('a', { + const workspaceAKey = workspaceIsolationKey('a'); + const workspaceBKey = workspaceIsolationKey('b'); + internals.activeWorkspaceAssignments.set(workspaceAKey, { id: 'previous', done: new Promise(() => {}), }); let executed = false; internals.executeOwned = async () => { executed = true; - assert.equal(internals.activeWorkspaceAssignments.get('b')?.id, 'next'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceBKey)?.id, + 'next', + ); }; await worker.executeAndSettle({ assignmentId: 'next', @@ -357,6 +371,9 @@ test('programmatic work on an independent workspace bypasses another root cleanu }, } as BridgeAssignment); assert.equal(executed, true); - assert.equal(internals.activeWorkspaceAssignments.has('b'), false); - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); + assert.equal(internals.activeWorkspaceAssignments.has(workspaceBKey), false); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceAKey)?.id, + 'previous', + ); }); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 697a3c55..2a5896fd 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -7,6 +7,7 @@ import { bridgeWorkerPath, isBridgeWorkspaceProgrammaticRequest, isWorkspaceToolResult, + workspaceIsolationKey, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; @@ -54,6 +55,13 @@ export interface BridgeWorkerOptions { workspaceMutationQuarantine?: WorkspaceMutationQuarantine; /** Required per-root durable guards when opting into concurrent workspace leases. */ workspaceQuarantines?: ReadonlyMap; + /** Resolve a durable guard for a worker-owned dynamic workspace instance. */ + workspaceQuarantineResolver?: ( + workspaceId: string, + workspaceInstanceId: string, + ) => + | WorkspaceMutationQuarantine + | Promise; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; @@ -208,7 +216,14 @@ function workspaceCapabilitiesMatch( operation === executor.workspaces[index]?.operations?.[operationIndex], ) ?? - executor.workspaces[index]?.operations == null), + executor.workspaces[index]?.operations == null) && + workspace.workspaceInstances?.length === + executor.workspaces[index]?.workspaceInstances?.length && + (workspace.workspaceInstances?.every( + (instanceType, instanceIndex) => + instanceType === + executor.workspaces[index]?.workspaceInstances?.[instanceIndex], + ) ?? executor.workspaces[index]?.workspaceInstances == null), ) ); } @@ -223,7 +238,8 @@ function registrationCompatibleCapabilities( (operation) => operation === 'read_file' || operation === 'search_text', ) && workspaceTools.workspaces.every( - (workspace) => workspace.operations == null, + (workspace) => + workspace.operations == null && workspace.workspaceInstances == null, )) ) { return capabilities; @@ -244,7 +260,11 @@ function registrationCompatibleCapabilities( ) { return []; } - const { operations: _operations, ...compatibleWorkspace } = workspace; + const { + operations: _operations, + workspaceInstances: _workspaceInstances, + ...compatibleWorkspace + } = workspace; return [{ ...compatibleWorkspace, ...(workspace.environment ? { environment: { ...workspace.environment, actions: [] }, } : {}) }]; @@ -329,6 +349,12 @@ function supportedWorkspaceCapabilities( return workspaceOperations.length === 0 ? [] : [{ ...workspace, + ...(workspace.workspaceInstances != null && + registration.supportedWorkspaceInstanceTypes?.includes( + 'git_worktree', + ) + ? { workspaceInstances: workspace.workspaceInstances } + : { workspaceInstances: undefined }), ...(workspace.operations ? { operations: workspaceOperations } : {}), ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { environment: { ...workspace.environment, actions: [] }, @@ -479,12 +505,23 @@ export class BridgeWorker { operation === 'execute_command', ) === true && options.workspaceMutationQuarantine == null && - options.workspaceQuarantines == null + options.workspaceQuarantines == null && + options.workspaceQuarantineResolver == null ) { throw new BridgeProtocolError( 'Workspace mutation capabilities require durable quarantine storage', ); } + if ( + options.capabilities.workspaceTools?.workspaces.some( + (root) => (root.workspaceInstances?.length ?? 0) > 0, + ) && + options.workspaceQuarantineResolver == null + ) { + throw new BridgeProtocolError( + 'Workspace instance capabilities require a durable quarantine resolver', + ); + } if ((options.capabilities.workspaceLeaseSlots ?? 1) > 1) { if ( options.capabilities.requiresReadyConfirmation !== true || @@ -785,14 +822,25 @@ export class BridgeWorker { async resetNativeWorkspace( workspaceId: string, signal?: AbortSignal, + workspaceInstanceId?: string, ): Promise { - const guard = this.options.workspaceQuarantines?.get(workspaceId); + const workspace = this.options.capabilities.workspaceTools?.workspaces.find( + (root) => root.id === workspaceId, + ); + const key = workspaceIsolationKey(workspaceId, workspaceInstanceId); + const guard = + workspaceInstanceId == null + ? this.options.workspaceQuarantines?.get(workspaceId) + : await this.options.workspaceQuarantineResolver?.( + workspaceId, + workspaceInstanceId, + ); if ( !guard || this.activeWorkspaceAssignments.size > 0 || - !this.options.capabilities.workspaceTools?.workspaces.some( - (root) => root.id === workspaceId, - ) + workspace == null || + (workspaceInstanceId != null && + workspace.workspaceInstances?.includes('git_worktree') !== true) ) { throw new BridgeProtocolError( 'Native workspace reset requires an idle registered root', @@ -808,14 +856,14 @@ export class BridgeWorker { { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, - runtimeSessionId: `native-workspace:${workspaceId}`, + runtimeSessionId: `native-workspace:${key}`, confirmDiscarded: true, }, this.options.resetTransportTimeoutMs ?? DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, signal, ); - this.quarantinedWorkspaces.delete(workspaceId); + this.quarantinedWorkspaces.delete(key); } async lease( @@ -1340,8 +1388,18 @@ export class BridgeWorker { private workspaceGuard( assignment: BridgeAssignment, - ): WorkspaceMutationQuarantine | undefined { - const workspaceId = this.assignmentWorkspaceId(assignment); + ): + | WorkspaceMutationQuarantine + | Promise + | undefined { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + const instanceId = this.assignmentWorkspaceInstanceId(assignment); + if (workspaceId != null && instanceId != null) { + return this.options.workspaceQuarantineResolver?.( + workspaceId, + instanceId, + ); + } return workspaceId != null ? (this.options.workspaceQuarantines?.get(workspaceId) ?? this.options.workspaceMutationQuarantine) @@ -1350,6 +1408,15 @@ export class BridgeWorker { private assignmentWorkspaceId( assignment: BridgeAssignment, + ): string | undefined { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + if (workspaceId == null) return undefined; + const instanceId = this.assignmentWorkspaceInstanceId(assignment); + return workspaceIsolationKey(workspaceId, instanceId); + } + + private assignmentBaseWorkspaceId( + assignment: BridgeAssignment, ): string | undefined { if ( assignment.executionKind === 'workspace_tool' && @@ -1366,11 +1433,33 @@ export class BridgeWorker { return undefined; } + private assignmentWorkspaceInstanceId( + assignment: BridgeAssignment, + ): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.workspaceInstanceId; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + return assignment.request.body.workspace_instance_id; + } + return undefined; + } + private async executeOwned( assignment: BridgeAssignment, signal?: AbortSignal, ): Promise { - const guard = this.workspaceGuard(assignment); + const unresolvedGuard = this.workspaceGuard(assignment); + const guard = + unresolvedGuard instanceof Promise + ? await unresolvedGuard + : unresolvedGuard; if (signal?.aborted === true) { throw signal.reason instanceof Error ? signal.reason @@ -1484,12 +1573,17 @@ export class BridgeWorker { throw new BridgeProtocolError('Invalid workspace tool request'); } const workspaceRequest = assignment.request; + const workspaceKey = this.assignmentWorkspaceId(assignment)!; try { - if (this.quarantinedWorkspaces.has(workspaceRequest.workspaceId)) { + if (this.quarantinedWorkspaces.has(workspaceKey)) { throw new Error('Workspace requires an explicit quarantine reset'); } - if (this.options.workspaceQuarantines != null) + if ( + this.options.workspaceQuarantines != null || + this.options.workspaceQuarantineResolver != null + ) { await guard?.assertAvailable(); + } } catch (error) { throw new BridgeWorkspaceQuarantinedError( 'Workspace is quarantined', @@ -1513,6 +1607,14 @@ export class BridgeWorker { if (workspace == null) { throw new BridgeProtocolError('Workspace is not advertised'); } + if ( + workspaceRequest.workspaceInstanceId != null && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Workspace instance type is not advertised', + ); + } if ( workspace.operations != null && !workspace.operations.includes(workspaceRequest.operation) @@ -1575,7 +1677,7 @@ export class BridgeWorker { if (isMutation) { this.mutationGuardArmed = true; try { - this.armedWorkspaces.add(workspaceRequest.workspaceId); + this.armedWorkspaces.add(workspaceKey); await guard!.arm( `Workspace mutation ${workspaceRequest.operation} is pending settlement`, assignment.assignmentId, @@ -1630,12 +1732,17 @@ export class BridgeWorker { 'Worker does not provide valid selected-workspace programmatic execution', ); } + const workspaceKey = this.assignmentWorkspaceId(assignment)!; try { - if (this.quarantinedWorkspaces.has(workspaceId)) { + if (this.quarantinedWorkspaces.has(workspaceKey)) { throw new Error('Workspace requires an explicit quarantine reset'); } - if (this.options.workspaceQuarantines != null) + if ( + this.options.workspaceQuarantines != null || + this.options.workspaceQuarantineResolver != null + ) { await guard?.assertAvailable(); + } } catch (error) { throw new BridgeWorkspaceQuarantinedError( 'Workspace is quarantined', @@ -1657,9 +1764,17 @@ export class BridgeWorker { 'Selected-workspace programmatic execution is not advertised', ); } + if ( + assignment.request.body.workspace_instance_id != null && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Workspace instance type is not advertised', + ); + } this.mutationGuardArmed = true; try { - this.armedWorkspaces.add(workspaceId); + this.armedWorkspaces.add(workspaceKey); await guard!.arm( 'Workspace programmatic execution is pending settlement', assignment.assignmentId, @@ -2079,11 +2194,15 @@ export class BridgeWorker { ): Promise { if (runtimeSessionId == null) { try { - await ( + const unresolvedGuard = assignment == null ? this.options.workspaceMutationQuarantine - : this.workspaceGuard(assignment) - )?.quarantine(message, cause, assignment?.assignmentId); + : this.workspaceGuard(assignment); + const guard = + unresolvedGuard instanceof Promise + ? await unresolvedGuard + : unresolvedGuard; + await guard?.quarantine(message, cause, assignment?.assignmentId); return new BridgeWorkspaceQuarantinedError(message, cause); } catch (error) { return new BridgeWorkspaceQuarantinedError( diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index c1bc7fce..1de68818 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -115,6 +115,38 @@ test('CLI supports native SRT by default and validates explicit runtime mode', a assert.match(noWorkspace.stderr, /require.*registered directory/i); }); +test('CLI requires concurrent native slots for conversation worktrees', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-conversation-')); + const workspaceRoot = join(root, 'workspace'); + const worktreeRoot = join(root, 'worktrees'); + await mkdir(workspaceRoot); + t.after(() => rm(root, { recursive: true, force: true })); + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspaceRoot, + '--allow-workspace-writes', + '--allow-workspace-commands', + '--conversation-worktree-root', + worktreeRoot, + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /at least two workspace lease slots/i); +}); + test('CLI advertises explicitly enabled writes without exposing the workspace root', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); diff --git a/packages/code/src/workspace-instances.test.ts b/packages/code/src/workspace-instances.test.ts new file mode 100644 index 00000000..a1cd6859 --- /dev/null +++ b/packages/code/src/workspace-instances.test.ts @@ -0,0 +1,234 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { GitWorktreeWorkspaceTools } from './workspace-instances.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { readRepositoryInstructions } from './instructions.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; + +const execFileAsync = promisify(execFile); + +async function repository(): Promise<{ parent: string; root: string }> { + const parent = await mkdtemp(join(tmpdir(), 'librechat-instance-tools-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', root]); + await writeFile(join(root, 'README.md'), 'source\n'); + await writeFile(join(root, 'AGENTS.md'), 'follow repository rules\n'); + await execFileAsync('git', ['-C', root, 'add', 'README.md', 'AGENTS.md']); + await execFileAsync('git', [ + '-C', + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'initial', + ]); + return { parent, root: await realpath(root) }; +} + +async function source(root: string) { + return { root, identity: await captureWorkspaceRootIdentity(root) }; +} + +test('routes each conversation to its own writable Git worktree', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: true, + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager, + sources: new Map([ + ['primary', { repositoryInstructions: true, writable: true }], + ]), + }); + const firstId = 'a'.repeat(64); + const secondId = 'b'.repeat(64); + + assert.deepEqual(tools.capabilities.workspaces[0]?.workspaceInstances, [ + 'git_worktree', + ]); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: 'conversation.txt', + content: 'first', + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: secondId, + path: 'conversation.txt', + content: 'second', + }); + + const first = await manager.resolve('primary', firstId); + const second = await manager.resolve('primary', secondId); + assert.equal( + await readFile(join(first.root, 'conversation.txt'), 'utf8'), + 'first', + ); + assert.equal( + await readFile(join(second.root, 'conversation.txt'), 'utf8'), + 'second', + ); + await assert.rejects(readFile(join(fixture.root, 'conversation.txt')), { + code: 'ENOENT', + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: 'conversation.txt', + }); + assert.equal(result.workspaceId, 'primary'); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, 'first'); + + await writeFile(join(fixture.root, 'AGENTS.md'), 'local repository rules\n'); + const instructions = await readRepositoryInstructions(fixture.root); + assert.ok(instructions); + const instructionResult = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: instructions.descriptor.path, + instructionSha256: instructions.descriptor.sha256, + }); + assert.equal(instructionResult.operation, 'read_file'); + assert.equal(instructionResult.content, 'local repository rules\n'); +}); + +test('reports provisioning rejection as an atomic workspace error', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager: new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }), + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: true }], + ]), + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: 'a'.repeat(64), + path: 'first.txt', + content: 'first', + }); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: 'b'.repeat(64), + path: 'second.txt', + content: 'second', + }), + { + code: 'WRITE_UNAVAILABLE', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }, + ); +}); + +test('rebuilds file executors only after operator recovery releases the reservation', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager, + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: true }], + ]), + }); + const instanceId = 'c'.repeat(64); + const request = { + protocolVersion: 1 as const, + operation: 'read_file' as const, + workspaceId: 'primary', + workspaceInstanceId: instanceId, + path: 'README.md', + }; + await tools.execute(request); + const initial = await manager.resolve('primary', instanceId); + await rm(initial.root, { recursive: true, force: true }); + + await assert.rejects(tools.execute(request), { + code: 'WRITE_UNAVAILABLE', + }); + await assert.rejects(tools.execute(request), /capacity is exhausted/); + // Missing checkout directories do not prove an interrupted writer is gone. + // Simulate operator recovery after confirming there is no active executor. + await rm(`${initial.root}.complete`); + const recovered = await tools.execute(request); + assert.equal(recovered.operation, 'read_file'); + assert.equal(recovered.content, 'source'); +}); + +test('leaves legacy requests on the selected source workspace', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: false }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager: new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }), + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: false }], + ]), + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, 'source'); +}); diff --git a/packages/code/src/workspace-instances.ts b/packages/code/src/workspace-instances.ts new file mode 100644 index 00000000..68e76a10 --- /dev/null +++ b/packages/code/src/workspace-instances.ts @@ -0,0 +1,232 @@ +import { createHash } from 'node:crypto'; + +import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, + WorkspaceToolRequest, + WorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +interface WorkspaceInstanceSource { + command?: NativeProcessSandboxOptions; + repositoryInstructions: boolean; + writable: boolean; +} + +export interface GitWorktreeWorkspaceToolsOptions { + commandPool?: NativeWorkspaceCommandPool; + delegate: WorkspaceToolExecutor; + manager: GitWorktreeManager; + onResolve?: (workspaceId: string, root: string) => void; + sources: ReadonlyMap; +} + +export function internalWorkspaceId(workspaceId: string, instanceId: string): string { + return `instance-${createHash('sha256') + .update(`${workspaceId}\0${instanceId}`) + .digest('hex')}`; +} + +function publicResult( + result: WorkspaceToolResult, + workspaceId: string, +): WorkspaceToolResult { + return { ...result, workspaceId }; +} + +/** Resolve an opaque conversation binding into an isolated Git worktree. */ +export class GitWorktreeWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly executors = new Map< + string, + { identity: WorkspaceRootIdentity; value: Promise } + >(); + + constructor(private readonly options: GitWorktreeWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + this.capabilities = { + ...options.delegate.capabilities, + workspaces: options.delegate.capabilities.workspaces.map((workspace) => ({ + ...workspace, + ...(options.sources.has(workspace.id) + ? { workspaceInstances: ['git_worktree' as const] } + : {}), + })), + }; + } + + private async executor( + workspaceId: string, + instanceId: string, + signal?: AbortSignal, + ): Promise<{ + executor: LocalWorkspaceTools; + identity: WorkspaceRootIdentity; + internalId: string; + root: string; + }> { + const source = this.options.sources.get(workspaceId); + if (!source) { + throw new WorkspaceToolError( + 'Workspace does not allow conversation worktrees', + 'INVALID_REQUEST', + ); + } + let instance; + try { + instance = await this.options.manager.resolve( + workspaceId, + instanceId, + signal, + ); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + if ( + signal?.aborted || + (error instanceof Error && error.name === 'AbortError') + ) { + throw new WorkspaceToolError( + 'Conversation worktree provisioning aborted', + 'EXECUTION_ABORTED', + ); + } + throw new WorkspaceToolError( + error instanceof Error + ? error.message + : 'Conversation worktree provisioning failed', + 'WRITE_UNAVAILABLE', + ); + } + this.options.onResolve?.(workspaceId, instance.root); + const internalId = internalWorkspaceId(workspaceId, instanceId); + const key = `${workspaceId}\0${instanceId}`; + let cached = this.executors.get(key); + if ( + cached == null || + cached.identity.dev !== instance.identity.dev || + cached.identity.ino !== instance.identity.ino || + cached.identity.path !== instance.identity.path + ) { + cached = { + identity: instance.identity, + value: LocalWorkspaceTools.create({ + repositoryInstructions: source.repositoryInstructions, + workspaces: [ + { + id: internalId, + identity: instance.identity, + root: instance.root, + writable: source.writable, + }, + ], + }), + }; + this.executors.set(key, cached); + } + return { + executor: await cached.value, + identity: instance.identity, + internalId, + root: instance.root, + }; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise { + if (!request.workspaceInstanceId) { + return await this.options.delegate.execute(request, signal); + } + if ( + request.operation === 'read_file' && + request.instructionSha256 !== undefined + ) { + const { workspaceInstanceId: _workspaceInstanceId, ...sourceRequest } = + request; + return await this.options.delegate.execute(sourceRequest, signal); + } + const { workspaceInstanceId, ...baseRequest } = request; + const source = this.options.sources.get(request.workspaceId); + const resolved = await this.executor( + request.workspaceId, + workspaceInstanceId, + signal, + ); + const isolatedRequest = { + ...baseRequest, + workspaceId: resolved.internalId, + } as WorkspaceToolRequest; + if (request.operation === 'execute_command') { + if (!source?.command || !this.options.commandPool) { + throw new WorkspaceToolError( + 'Conversation worktree commands are unavailable', + 'COMMAND_DISABLED', + ); + } + await this.options.commandPool.registerRoot(resolved.internalId, { + ...source.command, + workspaceIdentity: resolved.identity, + workspaceRoot: resolved.root, + }); + return publicResult( + await this.options.commandPool.execute( + isolatedRequest as WorkspaceExecuteCommandRequest, + signal, + ), + request.workspaceId, + ); + } + return publicResult( + await resolved.executor.execute(isolatedRequest, signal), + request.workspaceId, + ); + } + + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const instanceId = request.body.workspace_instance_id; + if (!instanceId) { + if (!this.options.commandPool) { + throw new WorkspaceToolError( + 'Workspace programmatic execution is unavailable', + 'COMMAND_DISABLED', + ); + } + return await this.options.commandPool.executeProgrammatic( + workspaceId, + request, + signal, + ); + } + const source = this.options.sources.get(workspaceId); + if (!source?.command || !this.options.commandPool) { + throw new WorkspaceToolError( + 'Conversation worktree programmatic execution is unavailable', + 'COMMAND_DISABLED', + ); + } + const resolved = await this.executor(workspaceId, instanceId, signal); + await this.options.commandPool.registerRoot(resolved.internalId, { + ...source.command, + workspaceIdentity: resolved.identity, + workspaceRoot: resolved.root, + }); + return await this.options.commandPool.executeProgrammatic( + resolved.internalId, + request, + signal, + ); + } +} diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index d97735dc..96a5d1f0 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -7,6 +7,19 @@ import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('instance advertisement requires a guard resolver even for reads and with base guards', () => { + for (const operation of ['read_file', 'write_file'] as const) { + const workspaceTools = { protocolVersion: 1 as const, operations: [operation], workspaces: [{ id: 'primary', workspaceInstances: ['git_worktree'] as ['git_worktree'] }] }; + assert.throws(() => new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools }, + workspaceQuarantines: new Map([['primary', mutationQuarantine()]]), + workspaceTools: { capabilities: workspaceTools, async execute() { throw new Error('must not execute'); } }, + }), /instance capabilities require a durable quarantine resolver/); + } +}); + test('worker clears named actions when command execution is not negotiated', async () => { const workspaceTools = { protocolVersion: 1 as const, @@ -974,6 +987,93 @@ test('worker executes a workspace tool assignment locally without acquiring a sa }); }); +test('worker isolates dynamic worktree guards from collision-shaped root IDs', async () => { + const instanceId = 'a'.repeat(64); + const collisionRoot = `foo:git-worktree:${instanceId}`; + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [ + { id: 'foo', workspaceInstances: ['git_worktree'] as ['git_worktree'] }, + { id: collisionRoot }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute(request) { + return { + protocolVersion: 1, + operation: 'write_file', + workspaceId: request.workspaceId, + path: 'result.txt', + created: true, + bytesWritten: 2, + }; + }, + }, + workspaceQuarantines: new Map([ + [ + collisionRoot, + mutationQuarantine( + undefined, + () => lifecycle.push('root:arm'), + () => lifecycle.push('root:clear'), + ), + ], + ]), + workspaceQuarantineResolver: async () => + mutationQuarantine( + undefined, + () => lifecycle.push('instance:arm'), + () => lifecycle.push('instance:clear'), + ), + fetchImpl: async () => + Response.json({ protocolVersion: 1, accepted: true }), + }); + const assignment = (workspaceId: string, suffix: string) => ({ + protocolVersion: 1 as const, + assignmentId: `assignment-${suffix}`, + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: `lease-token-that-is-long-enough-${suffix}`, + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool' as const, + request: { + protocolVersion: 1 as const, + operation: 'write_file' as const, + workspaceId, + path: 'result.txt', + content: 'ok', + ...(workspaceId === 'foo' ? { workspaceInstanceId: instanceId } : {}), + }, + }); + + await worker.executeAndSettle(assignment('foo', 'instance')); + await worker.executeAndSettle(assignment(collisionRoot, 'root')); + + assert.deepEqual(lifecycle, [ + 'instance:arm', + 'instance:clear', + 'root:arm', + 'root:clear', + ]); +}); + test('worker executes programmatic Bash in the selected workspace and preserves its fence', async () => { const programmaticRequests: object[] = []; const quarantineEvents: string[] = []; diff --git a/packages/code/src/worktrees.test.ts b/packages/code/src/worktrees.test.ts new file mode 100644 index 00000000..c353c24e --- /dev/null +++ b/packages/code/src/worktrees.test.ts @@ -0,0 +1,988 @@ +import assert from 'node:assert/strict'; +import { execFile, spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rename, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { GitWorktreeManager } from './worktrees.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { GitSourceSnapshot } from './git-snapshot.js'; + +const execFileAsync = promisify(execFile); + +test('aborting private staging releases its reservation only after cleanup', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const controller = new AbortController(); + const copy = GitSourceSnapshot.prototype.copyTo; + const mocked = t.mock.method( + GitSourceSnapshot.prototype, + 'copyTo', + async function ( + this: GitSourceSnapshot, + destination: string, + signal?: AbortSignal + ) { + await copy.call(this, destination, signal); + controller.abort(); + } + ); + const id = 'b'.repeat(64); + const path = await manager.plannedRoot('primary', id); + await assert.rejects(manager.resolve('primary', id, controller.signal), { + name: 'AbortError', + }); + for (const name of [path, `${path}.source`, `${path}.complete`]) + await assert.rejects(stat(name), { code: 'ENOENT' }); + mocked.mock.restore(); + assert.ok(await manager.resolve('primary', 'c'.repeat(64))); +}); + +test('setup cannot publish a checkout that redirects its Git metadata', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance) => { + await writeFile( + join(instance.root, '.git', 'commondir'), + join(fixture.root, '.git') + ); + }, + }); + await assert.rejects( + manager.resolve('primary', 'd'.repeat(64)), + /must not redirect/ + ); +}); + +test('private snapshots preserve SHA-256 repositories and the configured origin', async (t) => { + const fixture = await repository('sha256'); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await git( + fixture.root, + 'remote', + 'add', + 'origin', + 'https://github.com/example/repo.git' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const instance = await manager.resolve('primary', 'a'.repeat(64)); + assert.equal( + await git(instance.root, 'rev-parse', '--show-object-format'), + 'sha256' + ); + assert.equal( + await git(instance.root, 'remote', 'get-url', 'origin'), + 'https://github.com/example/repo.git' + ); + assert.equal( + await readFile(join(instance.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('sidecar-only crash reservations consume quota after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const admitted = await source(fixture.root); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', admitted]]), + }; + const manager = new GitWorktreeManager(options); + const path = await manager.plannedRoot('primary', 'a'.repeat(64)); + await mkdir(join(path, '..'), { recursive: true }); + await writeFile( + `${path}.complete`, + JSON.stringify({ + version: 2, + source: admitted.identity, + sourceGit: (await GitSourceSnapshot.admit(admitted.identity)).fingerprint, + provisioningFailed: true, + }) + ); + const restarted = new GitWorktreeManager(options); + await assert.rejects( + restarted.resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); + await assert.rejects( + restarted.resolve('primary', 'a'.repeat(64)), + /operator recovery required/ + ); + await assert.rejects(stat(path), { code: 'ENOENT' }); +}); + +test('linked-worktree sources retain their own HEAD and admitted common objects', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const linked = join(fixture.parent, 'linked'); + await git(fixture.root, 'worktree', 'add', '-b', 'linked', linked); + await writeFile(join(linked, 'linked.txt'), 'linked\n'); + await git(linked, 'add', 'linked.txt'); + await git( + linked, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'linked' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(linked)]]), + }); + const instance = await manager.resolve('primary', 'e'.repeat(64)); + assert.equal( + await readFile(join(instance.root, 'linked.txt'), 'utf8'), + 'linked\n' + ); +}); + +test('restart cannot rebind a completed checkout to replacement source Git metadata', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const id = 'f'.repeat(64); + const instance = await new GitWorktreeManager(options).resolve('primary', id); + await writeFile(join(instance.root, 'uncommitted.txt'), 'keep'); + await rename(join(fixture.root, '.git'), join(fixture.root, '.git-original')); + await git(fixture.root, 'init'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /source identity changed/ + ); + assert.equal( + await readFile(join(instance.root, 'uncommitted.txt'), 'utf8'), + 'keep' + ); +}); + +test('rejects source Git redirection without changing the admitted working-directory inode', async (t) => { + const fixture = await repository(); + const other = await repository(); + t.after(() => + Promise.all( + [fixture, other].map(({ parent }) => + rm(parent, { recursive: true, force: true }) + ) + ) + ); + const manager = new GitWorktreeManager({ + maxCount: 2, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await manager.prepare(); + await rename(join(fixture.root, '.git'), join(fixture.root, '.git-original')); + await writeFile( + join(fixture.root, '.git'), + `gitdir: ${join(other.root, '.git')}\n` + ); + await assert.rejects( + manager.resolve('primary', 'c'.repeat(64)), + /Git metadata changed/ + ); +}); + +test('source replacement after validation cannot redirect the private snapshot', async (t) => { + const fixture = await repository(); + const other = await repository(); + t.after(() => + Promise.all( + [fixture, other].map(({ parent }) => + rm(parent, { recursive: true, force: true }) + ) + ) + ); + const snapshot = await GitSourceSnapshot.admit( + ( + await source(fixture.root) + ).identity + ); + const validate = snapshot.validate.bind(snapshot); + t.mock.method(snapshot, 'validate', async () => { + await validate(); + await rename( + join(fixture.root, '.git'), + join(fixture.root, '.git-original') + ); + await symlink(join(other.root, '.git'), join(fixture.root, '.git')); + }); + await assert.rejects(snapshot.copyTo(join(fixture.parent, 'snapshot'))); + await assert.rejects( + stat(join(fixture.parent, 'snapshot', 'objects', 'pack')), + { code: 'ENOENT' } + ); +}); + +test('cached instances reject object-directory redirection before executor recreation', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'd'.repeat(64); + const instance = await manager.resolve('primary', id); + await rename( + join(instance.root, '.git', 'objects'), + join(instance.root, '.git', 'objects-original') + ); + await symlink( + join(fixture.root, '.git', 'objects'), + join(instance.root, '.git', 'objects') + ); + await assert.rejects( + manager.resolve('primary', id), + /does not own its Git objects/ + ); +}); + +test('snapshot rejects symlinked refs and never copies source hooks or config includes', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await git(fixture.root, 'config', 'include.path', '/not-readable/config'); + const snapshot = await GitSourceSnapshot.admit( + ( + await source(fixture.root) + ).identity + ); + await snapshot.copyTo(join(fixture.parent, 'snapshot')); + assert.equal( + await readFile(join(fixture.parent, 'snapshot', 'config'), 'utf8'), + '[core]\nrepositoryformatversion = 0\nbare = true\n' + ); + await assert.rejects(stat(join(fixture.parent, 'snapshot', 'hooks')), { + code: 'ENOENT', + }); + await symlink( + join(fixture.root, 'README.md'), + join(fixture.root, '.git', 'refs', 'bad') + ); + await assert.rejects( + snapshot.copyTo(join(fixture.parent, 'snapshot-bad')), + /symbolic link/ + ); +}); + +test( + 'restart preserves a checkout reserved by a crashed provisioning process', + { timeout: 10_000 }, + async (t) => { + const fixture = await repository(); + const admitted = await source(fixture.root); + const storage = join(fixture.parent, 'instances'); + const id = '9'.repeat(64); + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { GitWorktreeManager } from ${JSON.stringify( + new URL('./worktrees.js', import.meta.url).href + )}; + const manager = new GitWorktreeManager({ + maxCount: 1, root: ${JSON.stringify(storage)}, + sources: new Map([['primary', ${JSON.stringify(admitted)}]]), + prepareInstance: async () => { + process.stdout.write('setup-started'); + await new Promise(() => { setInterval(() => {}, 1000); }); + }, + }); + await manager.resolve('primary', ${JSON.stringify(id)}); + `, + ], + { stdio: ['ignore', 'pipe', 'pipe'] } + ); + const closed = once(child, 'close'); + t.after(async () => { + child.kill('SIGKILL'); + await closed; + await rm(fixture.parent, { recursive: true, force: true }); + }); + await once(child.stdout!, 'data'); + child.kill('SIGKILL'); + await closed; + const restarted = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', admitted]]), + }); + await assert.rejects( + restarted.resolve('primary', id), + /operator recovery required/ + ); + await assert.rejects( + restarted.resolve('primary', '8'.repeat(64)), + /capacity is exhausted/ + ); + assert.equal( + await readFile( + join(await restarted.plannedRoot('primary', id), 'README.md'), + 'utf8' + ), + 'source\n' + ); + } +); + +test('cached checkouts revalidate their admitted source without deleting user work', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'f'.repeat(64); + const instance = await manager.resolve('primary', id); + await writeFile(join(instance.root, 'pending.txt'), 'user work'); + await rename(fixture.root, `${fixture.root}.original`); + await mkdir(fixture.root); + await assert.rejects( + manager.resolve('primary', id), + /source changed after admission/ + ); + assert.equal( + await readFile(join(instance.root, 'pending.txt'), 'utf8'), + 'user work' + ); +}); + +test('preserves a failed setup checkout until executor cleanup is confirmed', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async () => { + throw new Error('setup failed'); + }, + discardInstance: async () => { + throw new Error('child cleanup unconfirmed'); + }, + }; + const manager = new GitWorktreeManager(options); + const id = 'a'.repeat(64); + await assert.rejects(manager.resolve('primary', id), /cleanup unconfirmed/); + const root = await manager.plannedRoot('primary', id); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'source\n'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /operator recovery required/ + ); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'source\n'); +}); + +test('cancellation waits for setup cleanup before releasing provisioning ownership', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let started!: () => void; + const setupStarted = new Promise((resolve) => { + started = resolve; + }); + let cleanupFinished = false; + let setupRoot = ''; + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance, signal) => { + const reservation = JSON.parse( + await readFile(`${instance.root}.complete`, 'utf8') + ); + assert.equal(reservation.provisioningFailed, true); + setupRoot = instance.root; + started(); + try { + await delay(60_000, undefined, { signal }); + await writeFile(join(instance.root, 'LATE'), 'should never happen'); + } finally { + await delay(20); + cleanupFinished = true; + } + }, + }); + const controller = new AbortController(); + const pending = manager.resolve('primary', 'a'.repeat(64), controller.signal); + const rejected = assert.rejects(pending, { name: 'AbortError' }); + await setupStarted; + controller.abort(); + await rejected; + assert.equal(cleanupFinished, true); + await assert.rejects(stat(setupRoot), { code: 'ENOENT' }); + await assert.rejects(stat(`${setupRoot}.complete`), { code: 'ENOENT' }); +}); + +test('cancels lock wait without provisioning while another caller continues', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let started!: () => void; + const setupStarted = new Promise((resolve) => { + started = resolve; + }); + let release!: () => void; + const released = new Promise((resolve) => { + release = resolve; + }); + t.after(release); + let setups = 0; + const options = { + maxCount: 2, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async () => { + setups++; + started(); + await released; + }, + }; + const active = new GitWorktreeManager(options).resolve( + 'primary', + 'a'.repeat(64) + ); + await setupStarted; + const controller = new AbortController(); + const manager = new GitWorktreeManager(options); + const waiting = manager.resolve('primary', 'b'.repeat(64), controller.signal); + const rejected = assert.rejects(waiting, { name: 'AbortError' }); + await delay(75); + controller.abort(); + await rejected; + assert.equal(setups, 1); + release(); + await active; + await assert.rejects( + stat(await manager.plannedRoot('primary', 'b'.repeat(64))), + { code: 'ENOENT' } + ); +}); + +test('recovery preserves unknown directories and malformed completion records', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const manager = new GitWorktreeManager(options); + const first = await manager.resolve('primary', 'a'.repeat(64)); + const unrelated = join(options.root, 'operator-backups', 'important'); + await mkdir(unrelated, { recursive: true }); + await writeFile(join(unrelated, 'notes'), 'keep'); + await manager.resolve('primary', 'b'.repeat(64)); + assert.equal(await readFile(join(unrelated, 'notes'), 'utf8'), 'keep'); + await writeFile(`${first.root}.complete`, '{"version":0}'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'a'.repeat(64)), + /completion record is invalid/ + ); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'c'.repeat(64)), + /completion record is invalid/ + ); + assert.equal( + await readFile(join(first.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +async function git(root: string, ...args: string[]): Promise { + const result = await execFileAsync('git', ['-C', root, ...args], { + encoding: 'utf8', + env: { + PATH: process.env.PATH, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + LC_ALL: 'C', + }, + }); + return result.stdout.trim(); +} + +async function repository( + objectFormat = 'sha1' +): Promise<{ parent: string; root: string }> { + const parent = await mkdtemp(join(tmpdir(), 'librechat-worktrees-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', `--object-format=${objectFormat}`, root]); + await writeFile(join(root, 'README.md'), 'source\n'); + await git(root, 'add', 'README.md'); + await git( + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'initial' + ); + return { parent, root: await realpath(root) }; +} + +async function source(root: string) { + return { root, identity: await captureWorkspaceRootIdentity(root) }; +} + +test('creates and reuses an isolated worktree for one conversation identity', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'a'.repeat(64); + + const [first, concurrent] = await Promise.all([ + manager.resolve('primary', id), + manager.resolve('primary', id), + ]); + assert.deepEqual(concurrent, first); + assert.notEqual(first.root, fixture.root); + assert.equal( + (await realpath(join(first.root, '.git', 'objects'))).startsWith( + first.root + ), + true + ); + const instanceCommon = await realpath( + await git( + first.root, + 'rev-parse', + '--path-format=absolute', + '--git-common-dir' + ) + ); + assert.equal(instanceCommon.startsWith(first.root), true); + assert.equal( + await readFile(join(first.root, 'README.md'), 'utf8'), + 'source\n' + ); + + await writeFile(join(first.root, 'README.md'), 'conversation\n'); + assert.equal( + await readFile(join(fixture.root, 'README.md'), 'utf8'), + 'source\n' + ); + + const restarted = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + assert.equal((await restarted.resolve('primary', id)).root, first.root); +}); + +test('replaces an incomplete checkout before admitting it after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const id = 'c'.repeat(64); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const first = await manager.resolve('primary', id); + await writeFile(join(first.root, 'README.md'), 'partial mutation\n'); + await rm(`${first.root}.complete`); + + const restarted = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const recovered = await restarted.resolve('primary', id); + assert.equal( + await readFile(join(recovered.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('does not count an incomplete checkout against capacity after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const abandoned = await manager.resolve('primary', 'c'.repeat(64)); + await rm(`${abandoned.root}.complete`); + const staleMarker = `${abandoned.root}.complete.1.tmp`; + await writeFile(staleMarker, '1\n'); + + const restarted = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const replacement = await restarted.resolve('primary', 'd'.repeat(64)); + assert.equal((await stat(replacement.root)).isDirectory(), true); + await assert.rejects(stat(abandoned.root), { code: 'ENOENT' }); + await assert.rejects(stat(staleMarker), { code: 'ENOENT' }); +}); + +test('keeps a conversation checkout independent of source object pruning', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await writeFile(join(fixture.root, 'SECOND.md'), 'second\n'); + await git(fixture.root, 'add', 'SECOND.md'); + await git( + fixture.root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'second' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const instance = await manager.resolve('primary', 'e'.repeat(64)); + const retainedHead = await git(instance.root, 'rev-parse', 'HEAD'); + + await git(fixture.root, 'reset', '--hard', 'HEAD~1'); + await git(fixture.root, 'reflog', 'expire', '--expire=now', '--all'); + await git(fixture.root, 'gc', '--prune=now'); + + assert.equal(await git(instance.root, 'rev-parse', 'HEAD'), retainedHead); + assert.equal( + await readFile(join(instance.root, 'SECOND.md'), 'utf8'), + 'second\n' + ); + await assert.rejects( + readFile(join(instance.root, '.git', 'objects', 'info', 'alternates')), + { code: 'ENOENT' } + ); +}); + +test('dissociates a checkout from inherited source alternates', async (t) => { + const upstream = await repository(); + const sharedParent = await mkdtemp( + join(tmpdir(), 'librechat-shared-source-') + ); + const sharedRoot = join(sharedParent, 'source'); + t.after(() => + Promise.all([ + rm(upstream.parent, { recursive: true, force: true }), + rm(sharedParent, { recursive: true, force: true }), + ]) + ); + await execFileAsync('git', ['clone', '--shared', upstream.root, sharedRoot]); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(sharedParent, 'instances'), + sources: new Map([['primary', await source(await realpath(sharedRoot))]]), + }); + const instance = await manager.resolve('primary', 'f'.repeat(64)); + await rm(upstream.root, { recursive: true, force: true }); + + assert.equal( + await git(instance.root, 'rev-parse', 'HEAD^{commit}'), + await git(instance.root, 'rev-parse', 'HEAD') + ); + await assert.rejects( + readFile(join(instance.root, '.git', 'objects', 'info', 'alternates')), + { code: 'ENOENT' } + ); +}); + +test('provisions an orphan branch for a repository with an unborn HEAD', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-empty-source-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', root]); + t.after(() => rm(parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(parent, 'instances'), + sources: new Map([['primary', await source(await realpath(root))]]), + }); + + const instance = await manager.resolve('primary', '0'.repeat(64)); + assert.match( + await git(instance.root, 'branch', '--show-current'), + /^librechat\/conversation-/ + ); + await assert.rejects(git(instance.root, 'rev-parse', '--verify', 'HEAD')); +}); + +test('rejects replacement of the admitted worktree storage root', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + await manager.resolve('primary', '1'.repeat(64)); + await rename(storage, `${storage}.original`); + await mkdir(storage, { mode: 0o700 }); + + await assert.rejects( + manager.resolve('primary', '1'.repeat(64)), + /storage changed after admission/ + ); +}); + +test('keeps conversations and source repositories isolated', async (t) => { + const first = await repository(); + const second = await repository(); + t.after(() => + Promise.all([ + rm(first.parent, { recursive: true, force: true }), + rm(second.parent, { recursive: true, force: true }), + ]) + ); + const storage = await mkdtemp(join(tmpdir(), 'librechat-worktree-storage-')); + t.after(() => rm(storage, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([ + ['first', await source(first.root)], + ['second', await source(second.root)], + ]), + }); + + const firstConversation = await manager.resolve('first', '1'.repeat(64)); + const secondConversation = await manager.resolve('first', '2'.repeat(64)); + const otherRepository = await manager.resolve('second', '1'.repeat(64)); + assert.equal( + new Set([ + firstConversation.root, + secondConversation.root, + otherRepository.root, + ]).size, + 3 + ); +}); + +test('rejects invalid identities, overlapping storage and exhausted capacity', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + assert.throws( + () => + new GitWorktreeManager({ + cloneTimeoutMs: 29_999, + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([ + [ + 'primary', + { + root: fixture.root, + identity: { + path: fixture.root, + dev: '1', + ino: '1', + }, + }, + ], + ]), + }), + /clone timeout/ + ); + const overlapping = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.root, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await assert.rejects( + overlapping.resolve('primary', 'a'.repeat(64)), + /must not overlap/ + ); + + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await assert.rejects(manager.resolve('primary', '../escape'), /SHA-256/); + const first = await manager.resolve('primary', 'a'.repeat(64)); + assert.equal((await stat(first.root)).isDirectory(), true); + await assert.rejects( + manager.resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); +}); + +test('serializes provisioning across manager instances sharing storage', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const results = await Promise.allSettled([ + new GitWorktreeManager(options).resolve('primary', 'a'.repeat(64)), + new GitWorktreeManager(options).resolve('primary', 'b'.repeat(64)), + ]); + assert.equal( + results.filter((result) => result.status === 'fulfilled').length, + 1 + ); + assert.equal( + results.filter((result) => result.status === 'rejected').length, + 1 + ); + assert.match( + ( + results.find( + (result) => result.status === 'rejected' + ) as PromiseRejectedResult + ).reason.message, + /capacity is exhausted/ + ); +}); + +test('prepares a new checkout before publishing its completion marker', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let attempts = 0; + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance: { root: string }) => { + attempts += 1; + if (attempts === 1) throw new Error('setup failed'); + await writeFile(join(instance.root, 'prepared'), 'yes\n'); + }, + }; + const id = 'c'.repeat(64); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /setup failed/ + ); + const instance = await new GitWorktreeManager(options).resolve('primary', id); + assert.equal( + await readFile(join(instance.root, 'prepared'), 'utf8'), + 'yes\n' + ); + assert.equal(attempts, 2); +}); + +test('preserves a completed checkout when its admitted source changes', async (t) => { + const first = await repository(); + const second = await repository(); + t.after(() => rm(first.parent, { recursive: true, force: true })); + t.after(() => rm(second.parent, { recursive: true, force: true })); + await writeFile(join(second.root, 'README.md'), 'replacement\n'); + await git(second.root, 'add', 'README.md'); + await git( + second.root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'replacement' + ); + const storage = join(first.parent, 'instances'); + const id = 'e'.repeat(64); + const original = await new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(first.root)]]), + }).resolve('primary', id); + await writeFile(join(original.root, 'UNCOMMITTED.md'), 'user work\n'); + await assert.rejects( + new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(second.root)]]), + }).resolve('primary', id), + /source identity changed/ + ); + assert.equal( + await readFile(join(original.root, 'UNCOMMITTED.md'), 'utf8'), + 'user work\n' + ); + assert.equal( + await readFile(join(original.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('rejects a source whose admitted filesystem identity changed', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const metadata = await stat(fixture.root, { bigint: true }); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([ + [ + 'primary', + { + root: fixture.root, + identity: { + path: fixture.root, + dev: metadata.dev.toString(), + ino: (metadata.ino + 1n).toString(), + }, + }, + ], + ]), + }); + + await assert.rejects( + manager.resolve('primary', 'd'.repeat(64)), + /source changed after admission/ + ); +}); diff --git a/packages/code/src/worktrees.ts b/packages/code/src/worktrees.ts new file mode 100644 index 00000000..ab19aa38 --- /dev/null +++ b/packages/code/src/worktrees.ts @@ -0,0 +1,718 @@ +import { execFile } from 'node:child_process'; +import { createHash, randomUUID } from 'node:crypto'; +import { + lstat, + mkdir, + readFile, + readdir, + realpath, + rename, + rm, + stat, + writeFile, +} from 'node:fs/promises'; +import { isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { promisify } from 'node:util'; + +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { assertPrivateStorageAncestors } from './private-storage.js'; +import { withProcessLock } from './process-lock.js'; +import { GitSourceSnapshot } from './git-snapshot.js'; + +const execFileAsync = promisify(execFile); +const WORKTREE_INSTANCE_PATTERN = /^[a-f0-9]{64}$/; +const COMPLETION_TEMP_PATTERN = /^[a-f0-9]{64}\.complete\.[a-f0-9-]+\.tmp$/; +const GIT_TIMEOUT_MS = 30_000; +const DEFAULT_CLONE_TIMEOUT_MS = 5 * 60_000; + +export interface GitWorktreeSource { + identity: WorkspaceRootIdentity; + root: string; +} + +export interface GitWorktreeInstance { + id: string; + identity: WorkspaceRootIdentity; + root: string; + sourceWorkspaceId: string; +} + +export interface GitWorktreeManagerOptions { + cloneTimeoutMs?: number; + maxCount: number; + root: string; + sources: ReadonlyMap; + prepareInstance?: ( + instance: GitWorktreeInstance, + signal?: AbortSignal + ) => Promise; + discardInstance?: (instance: GitWorktreeInstance) => Promise | void; +} + +const PROVISIONING_LOCK = '.provision.lock'; + +function isInside(parent: string, candidate: string): boolean { + const path = relative(parent, candidate); + return ( + path === '' || + (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + ); +} + +function gitEnvironment(): NodeJS.ProcessEnv { + return { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }; +} + +async function git( + root: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS +): Promise { + const execution = execFileAsync( + 'git', + ['--no-optional-locks', '-C', root, ...args], + { + encoding: 'utf8', + env: gitEnvironment(), + maxBuffer: 16 * 1024, + signal, + timeout, + } + ); + const closed = new Promise((resolve) => + execution.child.once('close', () => resolve()) + ); + try { + return (await execution).stdout.trim(); + } finally { + // execFile's AbortError callback can run before its child exits. Retain the + // provisioning lock and directory until the writer is actually gone. + const killTimer = setTimeout(() => execution.child.kill('SIGKILL'), 1000); + killTimer.unref(); + try { + await closed; + } finally { + clearTimeout(killTimer); + } + } +} + +async function sourceConfig( + root: string, + key: string, + signal?: AbortSignal +): Promise { + try { + const remote = await git( + root, + [ + 'config', + '--no-includes', + '--file', + join(root, 'source-config'), + '--get', + key, + ], + signal + ); + return remote || undefined; + } catch { + signal?.throwIfAborted(); + return undefined; + } +} + +async function hasCommittedHead( + root: string, + signal?: AbortSignal +): Promise { + try { + await git(root, ['rev-parse', '--verify', 'HEAD'], signal); + return true; + } catch (error) { + signal?.throwIfAborted(); + if (error instanceof Error && 'code' in error && error.code === 128) { + return false; + } + throw error; + } +} + +async function directoryIdentity(path: string): Promise { + const metadata = await lstat(path, { bigint: true }); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new Error('Conversation worktree must be a real directory'); + } + return { + path, + dev: metadata.dev.toString(), + ino: metadata.ino.toString(), + }; +} + +export class GitWorktreeManager { + private readonly instances = new Map(); + private readonly sourceSnapshots = new Map< + string, + Promise + >(); + private canonicalRoot?: Promise<{ + identity: WorkspaceRootIdentity; + path: string; + }>; + + constructor(private readonly options: GitWorktreeManagerOptions) { + if ( + !Number.isSafeInteger(options.maxCount) || + options.maxCount < 1 || + options.maxCount > 1024 || + options.sources.size === 0 + ) { + throw new Error( + 'Conversation worktree capacity must be between 1 and 1024' + ); + } + if ( + options.cloneTimeoutMs !== undefined && + (!Number.isSafeInteger(options.cloneTimeoutMs) || + options.cloneTimeoutMs < GIT_TIMEOUT_MS || + options.cloneTimeoutMs > 30 * 60_000) + ) { + throw new Error( + 'Conversation worktree clone timeout must be between 30000 and 1800000 milliseconds' + ); + } + } + + private async root(): Promise { + this.canonicalRoot ??= (async () => { + const configuredRoot = resolve(this.options.root); + await assertPrivateStorageAncestors(configuredRoot, true); + await mkdir(configuredRoot, { + mode: 0o700, + recursive: true, + }); + await assertPrivateStorageAncestors(configuredRoot); + const root = await realpath(this.options.root); + await assertPrivateStorageAncestors(root); + const metadata = await stat(root); + if ( + !metadata.isDirectory() || + (process.platform !== 'win32' && (metadata.mode & 0o022) !== 0) + ) { + throw new Error( + 'Conversation worktree root must not be group or world writable' + ); + } + for (const source of this.options.sources.values()) { + const sourceRoot = await realpath(source.root); + if (isInside(sourceRoot, root) || isInside(root, sourceRoot)) { + throw new Error( + 'Conversation worktree storage must not overlap a source workspace' + ); + } + } + return { + identity: await directoryIdentity(root), + path: root, + }; + })(); + const root = await this.canonicalRoot; + if (!(await matchesWorkspaceRoot(root.path, root.identity))) { + throw new Error('Conversation worktree storage changed after admission'); + } + return root.path; + } + + private key(sourceWorkspaceId: string, instanceId: string): string { + return `${sourceWorkspaceId}\0${instanceId}`; + } + + private branch(sourceWorkspaceId: string, instanceId: string): string { + const source = createHash('sha256') + .update(sourceWorkspaceId) + .digest('hex') + .slice(0, 8); + return `librechat/conversation-${source}-${instanceId.slice(0, 31)}`; + } + + private async instancePath( + sourceWorkspaceId: string, + instanceId: string + ): Promise { + const sourceDirectory = createHash('sha256') + .update(sourceWorkspaceId) + .digest('hex') + .slice(0, 24); + return join(await this.root(), sourceDirectory, instanceId); + } + + async plannedRoot( + sourceWorkspaceId: string, + instanceId: string + ): Promise { + if (!WORKTREE_INSTANCE_PATTERN.test(instanceId)) { + throw new Error( + 'Conversation worktree identity must be a SHA-256 digest' + ); + } + if (!this.options.sources.has(sourceWorkspaceId)) { + throw new Error('Conversation worktree source is unavailable'); + } + return await this.instancePath(sourceWorkspaceId, instanceId); + } + + async prepare(): Promise { + await this.root(); + await Promise.all( + [...this.options.sources].map(async ([_workspaceId, source]) => { + const sourceRoot = await this.admittedSourceRoot(source); + await this.sourceSnapshot(sourceRoot, source); + }) + ); + } + + private async admittedSourceRoot(source: GitWorktreeSource): Promise { + const sourceRoot = await realpath(source.root); + if (!(await matchesWorkspaceRoot(sourceRoot, source.identity))) { + throw new Error('Conversation worktree source changed after admission'); + } + return sourceRoot; + } + + private async sourceSnapshot( + root: string, + source: GitWorktreeSource + ): Promise { + let snapshot = this.sourceSnapshots.get(root); + if (!snapshot) { + snapshot = GitSourceSnapshot.admit(source.identity); + this.sourceSnapshots.set(root, snapshot); + } + const admitted = await snapshot; + await admitted.validate(); + return admitted; + } + + private async countInstances(): Promise { + const root = await this.root(); + const sourceDirectories = await readdir(root, { withFileTypes: true }); + let count = 0; + for (const sourceDirectory of sourceDirectories) { + if (sourceDirectory.name.startsWith(PROVISIONING_LOCK)) continue; + if ( + !/^[a-f0-9]{24}$/.test(sourceDirectory.name) || + !sourceDirectory.isDirectory() || + sourceDirectory.isSymbolicLink() + ) + continue; + const entries = await readdir(join(root, sourceDirectory.name), { + withFileTypes: true, + }); + const reserved = new Set(); + for (const entry of entries) { + const id = entry.name.endsWith('.complete') + ? entry.name.slice(0, -9) + : ''; + if (!WORKTREE_INSTANCE_PATTERN.test(id)) continue; + if (!entry.isFile() || entry.isSymbolicLink()) + throw new Error('Invalid worktree reservation'); + if ( + await this.hasCompletionMarker(join(root, sourceDirectory.name, id)) + ) { + reserved.add(id); + count += 1; + } + } + for (const entry of entries) { + if (entry.isFile() && COMPLETION_TEMP_PATTERN.test(entry.name)) { + await rm(join(root, sourceDirectory.name, entry.name), { + force: true, + }); + continue; + } + if ( + !WORKTREE_INSTANCE_PATTERN.test(entry.name) || + !entry.isDirectory() || + entry.isSymbolicLink() + ) + continue; + const path = join(root, sourceDirectory.name, entry.name); + if (!reserved.has(entry.name)) { + await rm(path, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + } + } + } + return count; + } + + private async withProvisioningLock( + operation: () => Promise, + signal?: AbortSignal + ): Promise { + return await withProcessLock( + join(await this.root(), PROVISIONING_LOCK), + operation, + signal + ); + } + + private completionMarker(path: string): string { + return `${path}.complete`; + } + + private async hasCompletionMarker( + path: string, + source?: WorkspaceRootIdentity, + sourceGit?: string + ): Promise { + try { + const record = JSON.parse( + await readFile(this.completionMarker(path), 'utf8') + ) as { + version?: unknown; + source?: Partial; + provisioningFailed?: boolean; + sourceGit?: string; + }; + const valid = + record.version === 2 && + typeof record.sourceGit === 'string' && + WORKTREE_INSTANCE_PATTERN.test(record.sourceGit) && + typeof record.source?.path === 'string' && + typeof record.source.dev === 'string' && + typeof record.source.ino === 'string'; + if (!valid) + throw new Error( + 'Conversation worktree completion record is invalid; existing checkout preserved' + ); + if ( + source != null && + (record.source!.path !== source.path || + record.source!.dev !== source.dev || + record.source!.ino !== source.ino || + record.sourceGit !== sourceGit) + ) { + throw new Error( + 'Conversation worktree source identity changed; existing checkout preserved' + ); + } + if (source != null && record.provisioningFailed) { + throw new Error( + 'Conversation worktree setup cleanup is unconfirmed; operator recovery required' + ); + } + return true; + } catch (error) { + if ( + error instanceof Error && + 'code' in error && + error.code === 'ENOENT' + ) { + return false; + } + throw error; + } + } + + private async writeCompletionMarker( + path: string, + source: WorkspaceRootIdentity, + sourceGit: string, + provisioningFailed = false + ): Promise { + const marker = this.completionMarker(path); + const temporary = `${marker}.${randomUUID()}.tmp`; + try { + await writeFile( + temporary, + `${JSON.stringify({ + version: 2, + source, + sourceGit, + ...(provisioningFailed ? { provisioningFailed: true } : {}), + })}\n`, + { mode: 0o600, flag: 'wx' } + ); + await rename(temporary, marker); + } finally { + await rm(temporary, { force: true }); + } + } + + private async validateRepository( + sourceWorkspaceId: string, + instanceId: string, + path: string, + signal?: AbortSignal + ): Promise { + const canonicalPath = await realpath(path); + if (canonicalPath !== path || !isInside(await this.root(), canonicalPath)) { + throw new Error( + 'Conversation worktree escaped its configured storage root' + ); + } + signal?.throwIfAborted(); + const instanceCommon = join(canonicalPath, '.git'); + const gitMetadata = await lstat(instanceCommon); + if ( + !gitMetadata.isDirectory() || + gitMetadata.isSymbolicLink() || + (await realpath(instanceCommon)) !== instanceCommon + ) { + throw new Error('Conversation worktree does not own its Git metadata'); + } + try { + await lstat(join(instanceCommon, 'commondir')); + throw new Error( + 'Conversation worktree must not redirect its Git metadata' + ); + } catch (error) { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + throw error; + } + const instanceObjects = await realpath(join(instanceCommon, 'objects')); + if (!isInside(canonicalPath, instanceObjects)) { + throw new Error('Conversation worktree does not own its Git objects'); + } + try { + await lstat(join(instanceObjects, 'info', 'alternates')); + throw new Error( + 'Conversation worktree must not use external Git objects' + ); + } catch (error) { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) { + throw error; + } + } + return { + id: instanceId, + identity: await directoryIdentity(canonicalPath), + root: canonicalPath, + sourceWorkspaceId, + }; + } + + private async validateExisting( + sourceWorkspaceId: string, + instanceId: string, + path: string, + source: WorkspaceRootIdentity, + sourceGit: string, + signal?: AbortSignal + ): Promise { + if (!(await this.hasCompletionMarker(path, source, sourceGit))) { + const error = new Error('Conversation worktree is incomplete'); + Object.assign(error, { code: 'EINCOMPLETE' }); + throw error; + } + return await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + } + + private async createLocked( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + if (!WORKTREE_INSTANCE_PATTERN.test(instanceId)) { + throw new Error( + 'Conversation worktree identity must be a SHA-256 digest' + ); + } + const source = this.options.sources.get(sourceWorkspaceId); + if (!source) throw new Error('Conversation worktree source is unavailable'); + const sourceRoot = await this.admittedSourceRoot(source); + const snapshot = await this.sourceSnapshot(sourceRoot, source); + const path = await this.instancePath(sourceWorkspaceId, instanceId); + try { + return await this.validateExisting( + sourceWorkspaceId, + instanceId, + path, + source.identity, + snapshot.fingerprint, + signal + ); + } catch (error) { + if (!(error instanceof Error) || !('code' in error)) { + throw error; + } + if (error.code === 'EINCOMPLETE') { + await rm(path, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + } else if (error.code !== 'ENOENT') { + throw error; + } + } + if ((await this.countInstances()) >= this.options.maxCount) { + throw new Error('Conversation worktree capacity is exhausted'); + } + await mkdir(resolve(path, '..'), { mode: 0o700, recursive: true }); + const branch = this.branch(sourceWorkspaceId, instanceId); + let instance: GitWorktreeInstance | undefined; + const staging = `${path}.source`; + try { + // Reserve before launching any writer. A worker crash may leave a Git + // child or setup executor alive after the parent's kernel lock releases. + // Recovery must not sweep or reuse that uncertain directory. + await this.writeCompletionMarker( + path, + source.identity, + snapshot.fingerprint, + true + ); + const cloneSignal = AbortSignal.any([ + ...(signal ? [signal] : []), + AbortSignal.timeout( + this.options.cloneTimeoutMs ?? DEFAULT_CLONE_TIMEOUT_MS + ), + ]); + await snapshot.copyTo(staging, cloneSignal); + const remote = await sourceConfig( + staging, + 'remote.origin.url', + cloneSignal + ); + const objectFormat = await sourceConfig( + staging, + 'extensions.objectformat', + cloneSignal + ); + if ( + objectFormat && + objectFormat !== 'sha1' && + objectFormat !== 'sha256' + ) { + throw new Error('Unsupported source Git object format'); + } + if (objectFormat === 'sha256') { + await writeFile( + join(staging, 'config'), + '[core]\nrepositoryformatversion = 1\nbare = true\n[extensions]\nobjectformat = sha256\n', + { mode: 0o600 } + ); + } + await git( + resolve(path, '..'), + ['clone', '--local', '--no-checkout', '--no-tags', staging, path], + cloneSignal, + this.options.cloneTimeoutMs ?? DEFAULT_CLONE_TIMEOUT_MS + ); + await rm(staging, { recursive: true, force: true }); + const sourceHasHead = await hasCommittedHead(path, signal); + if (remote) { + await git(path, ['remote', 'set-url', 'origin', remote], signal); + } else { + await git(path, ['remote', 'remove', 'origin'], signal); + } + await git( + path, + sourceHasHead + ? ['checkout', '--force', '-b', branch, 'HEAD'] + : ['checkout', '--orphan', branch], + signal + ); + instance = await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + await this.options.prepareInstance?.(instance, signal); + signal?.throwIfAborted(); + if (!(await matchesWorkspaceRoot(instance.root, instance.identity))) { + throw new Error('Conversation worktree changed during setup'); + } + await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + await this.admittedSourceRoot(source); + await snapshot.validate(); + await this.writeCompletionMarker( + path, + source.identity, + snapshot.fingerprint + ); + return instance; + } catch (error) { + if (instance) { + // The reservation remains until executor cleanup is confirmed. + await this.options.discardInstance?.(instance); + } + await rm(path, { recursive: true, force: true }); + await rm(staging, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + throw error; + } + } + + private async create( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + return await this.withProvisioningLock( + () => this.createLocked(sourceWorkspaceId, instanceId, signal), + signal + ); + } + + async resolve( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted(); + const key = this.key(sourceWorkspaceId, instanceId); + const cached = this.instances.get(key); + if (cached) { + await this.root(); + const source = this.options.sources.get(sourceWorkspaceId)!; + await this.sourceSnapshot(await this.admittedSourceRoot(source), source); + if (!(await matchesWorkspaceRoot(cached.root, cached.identity))) { + this.instances.delete(key); + throw new Error('Conversation worktree changed after admission'); + } + await this.validateRepository( + sourceWorkspaceId, + instanceId, + cached.root, + signal + ); + return cached; + } + // The same kernel lock coordinates callers and processes. Keep cancellation + // attached through setup and cleanup; never release a lease while detached + // provisioning is still mutating the checkout. + const instance = await this.create(sourceWorkspaceId, instanceId, signal); + this.instances.set(key, instance); + return instance; + } +} diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index 736b7216..33f3d082 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -2,7 +2,10 @@ import { afterEach, expect, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { RedisBridgeStore } from './store'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { + BRIDGE_PROTOCOL_VERSION, + workspaceIsolationKey, +} from '../../../packages/code/src/protocol'; import type { CodeBridgeAssignment } from './store'; const redis = new RedisMock() as unknown as Redis; @@ -26,13 +29,20 @@ async function register(workspaceLeaseSlots = 2) { workspaceTools: { protocolVersion: BRIDGE_PROTOCOL_VERSION, operations: ['read_file'], - workspaces: [{ id: 'a' }, { id: 'b' }], + workspaces: [ + { id: 'a', workspaceInstances: ['git_worktree'] }, + { id: 'b' }, + ], }, }, }); await store.confirmReady(workerId, incarnationId, generation); } -function dispatch(workspaceId: string, signal = new AbortController().signal) { +function dispatch( + workspaceId: string, + signal = new AbortController().signal, + workspaceInstanceId?: string, +) { const promise = store.dispatchWorkspaceTool({ workerId, signal, @@ -41,6 +51,7 @@ function dispatch(workspaceId: string, signal = new AbortController().signal) { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'read_file', workspaceId, + ...(workspaceInstanceId == null ? {} : { workspaceInstanceId }), path: 'file.txt', }, }); @@ -346,6 +357,52 @@ test('same-root work waits while another root progresses', async () => { await Promise.all([nextA, b]); }); +test('conversation worktrees on one source use independent capacity lanes', async () => { + await register(); + const firstId = 'a'.repeat(64); + const secondId = 'b'.repeat(64); + const firstPending = dispatch('a', undefined, firstId); + const first = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + const samePending = dispatch('a', undefined, firstId); + const secondPending = dispatch('a', undefined, secondId); + const second = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 1, + ))!; + expect(second.request).toMatchObject({ + workspaceId: 'a', + workspaceInstanceId: secondId, + }); + await settle(first); + await firstPending; + const same = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + expect(same.request).toMatchObject({ + workspaceId: 'a', + workspaceInstanceId: firstId, + }); + await settle(second); + await settle(same); + await Promise.all([samePending, secondPending]); +}); + test('queued cancellation never leases and does not block another root', async () => { await register(); const a = dispatch('a'); @@ -548,7 +605,11 @@ test('post-settlement fences are authenticated, idempotent, and invalidated by r await expect(dispatch('a')).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED', }); - await store.resetWorkspace(workerId, incarnationId, 'native-workspace:a'); + await store.resetWorkspace( + workerId, + incarnationId, + `native-workspace:${workspaceIsolationKey('a')}`, + ); await expect( store.settle( workerId, diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 73870772..5b319733 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -430,6 +430,7 @@ router.post( supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], supportedWorkspaceListFileFeatures: ['after_path'], supportedWorkspaceProgrammaticLanguages: ['bash'], + supportedWorkspaceInstanceTypes: ['git_worktree'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/store.test.ts b/service/src/bridge/store.test.ts index 9d271551..354108af 100644 --- a/service/src/bridge/store.test.ts +++ b/service/src/bridge/store.test.ts @@ -4,7 +4,7 @@ import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import type * as t from '../types'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; -import { RedisBridgeStore } from './store'; +import { RedisBridgeStore, workspaceAdmissionId } from './store'; import type { RegisteredBridgeWorker } from './store'; @@ -27,6 +27,16 @@ afterEach(async () => { }); describe('RedisBridgeStore', () => { + test('uses disjoint admission identities for roots and worktree instances', () => { + const instanceId = 'a'.repeat(64); + expect( + workspaceAdmissionId(`foo:git-worktree:${instanceId}`), + ).not.toBe(workspaceAdmissionId('foo', instanceId)); + expect(workspaceAdmissionId('foo')).not.toBe( + workspaceAdmissionId('workspace:foo'), + ); + }); + test('reports an atomic, capability-limited worker status snapshot', async () => { const store = new RedisBridgeStore(redis); const capabilities = { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 4eaabd7f..2ffeec66 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -15,8 +15,9 @@ import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, - isWorkspaceToolRequest, - isWorkspaceToolResult, + isWorkspaceToolRequest, + isWorkspaceToolResult, + workspaceIsolationKey, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; import { BridgeAdmissionQueue } from './admission'; @@ -124,6 +125,12 @@ function supportsWorkspaceTool( if (!supportsOperation) { return supportsOperation; } + if ( + request.workspaceInstanceId !== undefined && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + return false; + } if (request.operation === 'list_files' && request.afterPath !== undefined) { return capabilities?.listFileFeatures?.includes('after_path') === true; } @@ -156,6 +163,7 @@ function supportsWorkspaceProgrammatic( registration: RegisteredBridgeWorker, workspaceId: string, language: string, + workspaceInstanceId?: string, ): boolean { const capabilities = registration.capabilities.workspaceTools; const workspace = capabilities?.workspaces.find( @@ -163,6 +171,8 @@ function supportsWorkspaceProgrammatic( ); return ( workspace != null && + (workspaceInstanceId === undefined || + workspace.workspaceInstances?.includes('git_worktree') === true) && capabilities?.operations.includes('execute_command') === true && (workspace.operations == null || workspace.operations.includes('execute_command')) && @@ -172,6 +182,25 @@ function supportsWorkspaceProgrammatic( ); } +function workspaceInstanceId(body: t.PayloadBody): string | undefined { + if ( + typeof body === 'object' && + body != null && + 'workspace_instance_id' in body && + typeof body.workspace_instance_id === 'string' + ) { + return body.workspace_instance_id; + } + return undefined; +} + +export function workspaceAdmissionId( + workspaceId: string, + instanceId?: string, +): string { + return workspaceIsolationKey(workspaceId, instanceId); +} + function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } @@ -807,6 +836,7 @@ export class RedisBridgeStore { registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), ) ) { throw new BridgeStoreError( @@ -843,6 +873,15 @@ export class RedisBridgeStore { let workspaceLeaseSlot: number | undefined; const selectedWorkspaceId = args.workspaceRequest?.workspaceId ?? args.workspaceId; + const selectedWorkspaceInstanceId = + args.workspaceRequest?.workspaceInstanceId ?? workspaceInstanceId(args.body); + const selectedWorkspaceAdmissionId = + selectedWorkspaceId == null + ? undefined + : workspaceAdmissionId( + selectedWorkspaceId, + selectedWorkspaceInstanceId, + ); const workspaceSlots = selectedWorkspaceId != null && (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 @@ -864,7 +903,7 @@ export class RedisBridgeStore { args.deadlineAtMs, workspaceSlots == null ? undefined - : selectedWorkspaceId, + : selectedWorkspaceAdmissionId, ), args, 'Bridge admission enqueue', @@ -899,7 +938,7 @@ export class RedisBridgeStore { workerId: args.workerId, incarnationId: lockIncarnationId, assignmentId, - workspaceId: selectedWorkspaceId!, + workspaceId: selectedWorkspaceAdmissionId!, capacity: registration.capabilities.workspaceLeaseSlots!, expiresAtMs: Date.now() + ttlSeconds * 1000, }), @@ -961,6 +1000,7 @@ export class RedisBridgeStore { current.registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), )) ) { throw new BridgeStoreError( @@ -986,14 +1026,14 @@ export class RedisBridgeStore { generation, leaseToken, leaseTokenHash: tokenHash(leaseToken), - ...(selectedWorkspaceId == null ? {} : { - workspaceFence: `native-workspace:${selectedWorkspaceId}`, + ...(selectedWorkspaceAdmissionId == null ? {} : { + workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId}`, }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${selectedWorkspaceId!}`, + workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -1082,6 +1122,7 @@ export class RedisBridgeStore { replacement.registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), ) ) { throw new BridgeStoreError( diff --git a/service/src/bridge/workspace-instance.test.ts b/service/src/bridge/workspace-instance.test.ts new file mode 100644 index 00000000..44e93300 --- /dev/null +++ b/service/src/bridge/workspace-instance.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'bun:test'; +import { principalWorkspaceInstanceId } from './workspace-instance'; + +describe('principalWorkspaceInstanceId', () => { + it('keeps principal components distinct even when identifiers contain delimiters', () => { + const instanceId = 'a'.repeat(64); + expect(principalWorkspaceInstanceId({ instanceId, tenantId: 'tenant\0user', principalId: 'a' })) + .not.toBe(principalWorkspaceInstanceId({ instanceId, tenantId: 'tenant', principalId: 'user\0a' })); + }); + it('is stable only within the same authenticated principal', () => { + const instanceId = 'a'.repeat(64); + const first = principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-a', + }); + expect(first).toMatch(/^[a-f0-9]{64}$/); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-a', + }) + ).toBe(first); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-b', + }) + ).not.toBe(first); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'other', + principalId: 'user-a', + }) + ).not.toBe(first); + }); +}); diff --git a/service/src/bridge/workspace-instance.ts b/service/src/bridge/workspace-instance.ts new file mode 100644 index 00000000..4c49afbb --- /dev/null +++ b/service/src/bridge/workspace-instance.ts @@ -0,0 +1,17 @@ +import { createHash } from 'node:crypto'; + +/** Bind a caller-selected conversation identity to the authenticated principal. */ +export function principalWorkspaceInstanceId(args: { + instanceId: string; + tenantId: string; + principalId: string; +}): string { + return createHash('sha256') + .update(JSON.stringify([ + 'codeapi-workspace-instance-v1', + args.tenantId, + args.principalId, + args.instanceId, + ])) + .digest('hex'); +} diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 1063fbfe..a646b9ad 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -44,6 +44,7 @@ import { SessionKeyResolutionError, } from '../session-key'; import { getCredentialId, getPrincipalOrReject } from '../auth/principal'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; import { getExecutionIdentity } from '../execution-identity'; import { PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION } from '../runtime-session/job-policy'; import { @@ -81,6 +82,7 @@ import { authorizeRequestedFiles, } from './file-authorization'; import { + bindReplayWorkspaceInstance, buildReplayExecutionState, resolveReplayStateSandboxBackend, } from './programmatic-state'; @@ -335,7 +337,7 @@ function buildReplayPayload( state: ExecutionState, history: Record, ): t.PayloadBody { - return createProgrammaticPayload({ + const payload = createProgrammaticPayload({ req, session_id: state.session_id, execution_id: state.execution_id, @@ -347,6 +349,7 @@ function buildReplayPayload( filesOverride: state.files, language: state.language ?? 'python', }); + return bindReplayWorkspaceInstance(payload, state); } async function runReplayIteration( @@ -501,10 +504,17 @@ async function handleReplayInitial( userId: string; bridgeWorkerId?: string; workspaceId?: string; + workspaceInstanceId?: string; }, cancellation: ReplayRequestCancellation, ): Promise { - const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; + const { + apiKeyId, + userId, + bridgeWorkerId, + workspaceId, + workspaceInstanceId, + } = params; const { code, tools, user_id, files } = req.body as t.ProgrammaticRequestBody; let timeout: number; @@ -660,6 +670,7 @@ async function handleReplayInitial( language, bridgeWorkerId, workspaceId, + workspaceInstanceId, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -1279,6 +1290,7 @@ router.post( const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; + let workspaceInstanceId: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -1312,6 +1324,23 @@ router.post( } workspaceId = requestedWorkspaceId; } + const requestedWorkspaceInstanceId = rawBody.workspace_instance_id; + if (requestedWorkspaceInstanceId !== undefined) { + if ( + workspaceId == null || + typeof requestedWorkspaceInstanceId !== 'string' || + !/^[a-f0-9]{64}$/.test(requestedWorkspaceInstanceId) + ) { + return res.status(400).json({ + error: 'Invalid code workspace instance ID', + }); + } + workspaceInstanceId = principalWorkspaceInstanceId({ + instanceId: requestedWorkspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }); + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { return res @@ -1428,6 +1457,7 @@ router.post( userId, bridgeWorkerId, workspaceId, + workspaceInstanceId, }, cancellation); } if (workspaceId != null) { diff --git a/service/src/service/programmatic-state.test.ts b/service/src/service/programmatic-state.test.ts index 81405021..b0bff413 100644 --- a/service/src/service/programmatic-state.test.ts +++ b/service/src/service/programmatic-state.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import type { CodeApiAuthContext, RequestFile } from '../types'; import type { LCTool } from '../preamble'; import { + bindReplayWorkspaceInstance, buildReplayExecutionState, resolveReplayStateSandboxBackend, } from './programmatic-state'; @@ -84,6 +85,7 @@ describe('buildReplayExecutionState', () => { authContext, bridgeWorkerId: 'code-user_123', workspaceId: 'project-a', + workspaceInstanceId: 'a'.repeat(64), sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -104,6 +106,7 @@ describe('buildReplayExecutionState', () => { apiKeyId: 'key_legacy', bridgeWorkerId: 'code-user_123', workspaceId: 'project-a', + workspaceInstanceId: 'a'.repeat(64), sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -119,6 +122,19 @@ describe('buildReplayExecutionState', () => { }); }); + test('binds a selected conversation checkout into every replay payload', () => { + const payload = { language: 'bash', version: '5.2', files: [] }; + expect( + bindReplayWorkspaceInstance(payload, { + workspaceInstanceId: 'b'.repeat(64), + }), + ).toEqual({ + ...payload, + workspace_instance_id: 'b'.repeat(64), + }); + expect(bindReplayWorkspaceInstance(payload, {})).toBe(payload); + }); + test('falls back to JWT identity only when no managed auth context exists', () => { const state = build({ authContext: undefined, userId: 'user_api_key' }); diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index e6bda59a..38c72486 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -39,6 +39,7 @@ export interface BuildReplayExecutionStateParams { language: 'python' | 'bash'; bridgeWorkerId?: string; workspaceId?: string; + workspaceInstanceId?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -68,6 +69,7 @@ export function buildReplayExecutionState( apiKeyId: params.apiKeyId, bridgeWorkerId: params.bridgeWorkerId, workspaceId: params.workspaceId, + workspaceInstanceId: params.workspaceInstanceId, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, @@ -83,3 +85,13 @@ export function buildReplayExecutionState( language: params.language, }; } + +/** Bind the authenticated conversation checkout to every replay iteration. */ +export function bindReplayWorkspaceInstance( + payload: t.PayloadBody, + state: Pick, +): t.PayloadBody { + return state.workspaceInstanceId == null + ? payload + : { ...payload, workspace_instance_id: state.workspaceInstanceId }; +} diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 3b65cedf..fd4e90e3 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -118,6 +118,8 @@ export interface ExecutionState { bridgeWorkerId?: string; /** Selected workspace retained and bound across every replay iteration. */ workspaceId?: string; + /** Selected conversation checkout retained across every replay iteration. */ + workspaceInstanceId?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 0404ad16..91c3ed89 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -204,6 +204,8 @@ export type PayloadFileRef = { export interface PayloadBody { language: string; version: string; + /** Opaque conversation checkout selected and authenticated by the API. */ + workspace_instance_id?: string; /** Stable identity shared by all replay iterations of one execution. */ execution_id?: string; replay_tool_count?: number; @@ -393,6 +395,8 @@ export interface ProgrammaticRequestBody { * legacy `/exec` sandbox body), so the router accepts either key and * normalizes to `language`. If both are present, `language` wins. */ lang?: 'python' | 'bash'; + /** Opaque conversation checkout binding for a selected native workspace. */ + workspace_instance_id?: string; } export interface ProgrammaticToolCall { diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 04738b8a..a52b3f0b 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -13,6 +13,7 @@ import { executionProfileMiddleware } from '../middleware/execution-profile'; import { hostedAppPreviewGateway } from '../hosted-app/preview-gateway'; import { applyPrincipal } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; import { bridgeStoreStatus, createWorkspaceToolsRouter } from './router'; import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; @@ -36,6 +37,36 @@ test('maps invalid worker results to an upstream failure', () => { expect(bridgeStoreStatus(new BridgeStoreError('WORKER_QUEUE_FULL', 'queue full'))).toBe(429); }); +test('binds instance admission to the authenticated tenant and user while preserving legacy requests', async () => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); + next(); + }); + const dispatched: WorkspaceToolRequest[] = []; + app.use(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, + store: { async dispatchWorkspaceTool(args) { + dispatched.push(args.request); + return { protocolVersion: 1, generation: 1, leaseToken: 'lease', incarnationId: 'incarnation', status: 'rejected', error: 'fixture' }; + } }, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + for (const workspaceInstanceId of ['a'.repeat(64), undefined]) { + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', workspaceInstanceId, path: 'README.md' }), + }); + await response.json(); + } + expect(dispatched[0]?.workspaceInstanceId).toBe(principalWorkspaceInstanceId({ instanceId: 'a'.repeat(64), tenantId: 'tenant-1', principalId: 'user-1' })); + expect(dispatched[1]?.workspaceInstanceId).toBeUndefined(); +}); + test.each<[WorkspaceToolRequest, number, number?]>([ [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined], [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined], diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index eb89370e..3e1f9fee 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -18,6 +18,7 @@ import { BridgeWorkerSelectionError, resolveBridgeWorkerSelection, } from '../bridge/selection'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; interface WorkspaceToolsRouterOptions { store: Pick; @@ -82,12 +83,22 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) return; } outcome.operation = req.body.operation; - const request: WorkspaceToolRequest = req.body.operation === 'execute_command' - ? { ...req.body, timeoutMs: Math.min( - req.body.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null + ? req.body + : { + ...req.body, + workspaceInstanceId: principalWorkspaceInstanceId({ + instanceId: req.body.workspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }), + }; + const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' + ? { ...principalRequest, timeoutMs: Math.min( + principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, options.timeoutMs ?? Number.MAX_SAFE_INTEGER, ) } - : req.body; + : principalRequest; const executionBudgetMs = request.operation === 'execute_command' ? request.timeoutMs! + 5_000 : Math.min(options.timeoutMs ?? 30_000, 30_000); From c8b3e1490416753b6f633e4cdbf9100e93aeb849 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 20 Sep 2026 05:44:11 -0400 Subject: [PATCH 41/75] fix: Close native scratch directory streams (#240) --- docs/remote-bridge/README.md | 5 ++++- packages/code/src/native-scratch.ts | 30 +++++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index f66499b2..efe84b84 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -235,7 +235,10 @@ execution. - Remote bridge deployments use backend-specific BullMQ queues and serialize the expected backend on every new job, preventing Lambda or HTTP consumers from accepting attached-worker executions. -- Code API permits one active assignment per worker. +- Code API negotiates a bounded number of active workspace assignments per + worker. The lower API or worker slot ceiling wins, and assignments sharing + the same workspace isolation key remain serialized while independent + conversation worktrees may run concurrently. - Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are diff --git a/packages/code/src/native-scratch.ts b/packages/code/src/native-scratch.ts index 0ecd330e..8394c1ae 100644 --- a/packages/code/src/native-scratch.ts +++ b/packages/code/src/native-scratch.ts @@ -151,19 +151,25 @@ export async function restoreScratchTraversal( if (directoryFd === undefined) continue; try { const directory = await opendir(descriptorPath(directoryFd)); - for await (const entry of directory) { - entriesInspected += 1; - if (entriesInspected > MAX_SCRATCH_ENTRIES) { - throw new Error('Native sandbox scratch cleanup exceeded its entry limit'); + try { + while (true) { + const entry = await directory.read(); + if (entry === null) break; + entriesInspected += 1; + if (entriesInspected > MAX_SCRATCH_ENTRIES) { + throw new Error('Native sandbox scratch cleanup exceeded its entry limit'); + } + if (!entry.isDirectory()) continue; + if (components.length >= MAX_SCRATCH_DEPTH) { + throw new Error('Native sandbox scratch cleanup exceeded its depth limit'); + } + if (pending.length >= MAX_SCRATCH_DIRECTORIES) { + throw new Error('Native sandbox scratch cleanup exceeded its directory limit'); + } + pending.push([...components, entry.name]); } - if (!entry.isDirectory()) continue; - if (components.length >= MAX_SCRATCH_DEPTH) { - throw new Error('Native sandbox scratch cleanup exceeded its depth limit'); - } - if (pending.length >= MAX_SCRATCH_DIRECTORIES) { - throw new Error('Native sandbox scratch cleanup exceeded its directory limit'); - } - pending.push([...components, entry.name]); + } finally { + await directory.close(); } } finally { if (directoryFd !== root.fd) closeDirectory(directoryFd); From 67d75d859aee891923c40cde1db073489d74a424 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 23 Sep 2026 19:56:56 -0400 Subject: [PATCH 42/75] feat(code): route trusted VM GitHub App tokens by checkout (#248) * feat(code): route trusted VM GitHub App tokens by checkout * fix(code): resolve checkout credentials within canonical root * fix(code): reject cross-root credential aliases --- docs/remote-bridge/worker-runbook.md | 19 ++++-- packages/code/README.md | 26 +++++++-- packages/code/src/cli.test.ts | 86 +++++++++++++++++++++++++--- packages/code/src/cli.ts | 39 ++++++++++--- packages/code/src/github.test.ts | 79 +++++++++++++++++++++++++ packages/code/src/github.ts | 47 ++++++++++++--- 6 files changed, 261 insertions(+), 35 deletions(-) diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 16c24742..3043c541 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -327,12 +327,19 @@ Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/ ``` Install the same App separately on every personal account or organization the -worker is allowed to use. The trusted worker resolves the correct installation -from the repository containing each command's working directory, then mints and -caches a repository-scoped token. Cross-repository work therefore does not -require changing an installation ID or restarting the worker. Set -`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation -fallback. +worker is allowed to use. By default, the worker binds each admitted workspace +root to its repository at startup, then mints and caches repository-scoped +tokens. Different admitted roots can use different installations without +restarting the worker. For a trusted VM with multiple checkouts under one root, +set `LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING=checkout` and use the `trusted-vm` +command policy. This opt-in resolves the local `origin` URL of each command's +current checkout, including linked worktrees. It remains inside the admitted +filesystem root, but anyone able to alter a checkout's remote can select any +repository where the App is installed; keep the App's installation scope narrow. +Pass the checkout as the command working directory; changing directories only +inside the shell cannot change the token chosen before command launch. +Set `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy +fixed-installation fallback; it cannot be combined with checkout routing. Sandboxed commands receive masked Git/`gh` credentials only for the configured GitHub hosts; the token is not written to the repository, remote URL, or Git diff --git a/packages/code/README.md b/packages/code/README.md index e30b80cb..5a06b92a 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -293,9 +293,23 @@ installation tokens. At startup, the worker binds each explicitly admitted workspace root to its Git repository. Commands in those independent roots can use simultaneous installations on personal accounts and organizations without being restarted or reconfigured, while a command cannot gain access by changing -its workspace's remote URL. Tokens are scoped and cached per repository. For -compatibility with deployments -that intentionally bind a worker to one installation, set the optional legacy +its workspace's remote URL. Tokens are scoped and cached per repository. + +For trusted VMs that intentionally work in multiple Git checkouts beneath one +admitted root, opt in to `--github-repository-routing checkout` (or +`LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING=checkout`) together with the +`trusted-vm` command policy. Each command then uses the repository identified +by its current checkout's local `origin` URL, including linked worktrees. +The command must set its working directory to that checkout; a shell `cd` +inside a command does not change which credential was selected before launch. +This does not widen the admitted filesystem roots, but a command able to alter +a checkout's remote can obtain a token for **any repository where the App is +installed**. Use this mode only where the machine operator trusts the VM and +the App's installation scope; the default `admitted` mode keeps the startup +binding. Checkout routing requires an App without a fixed installation ID. + +For compatibility with deployments that intentionally bind a worker to one +installation, set the optional legacy `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. App-authenticated commits use the GitHub App bot's canonical no-reply identity, @@ -758,8 +772,10 @@ Also archive any adjacent `.source` staging directory. Pre-release version-1 completion records are deliberately preserved but not admitted by this version; they do not contain the required source Git identity binding. -GitHub App routing is inherited from the operator-admitted source repository; -commands cannot select a different installation by rewriting a worktree remote. +By default, GitHub App routing is inherited from the operator-admitted source +repository; commands cannot select a different installation by rewriting a +worktree remote. On trusted VMs, the opt-in checkout routing mode above instead +uses the current worktree's local `origin` URL, within the admitted root. Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f39b28d9..f95d7eee 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { generateKeyPairSync } from 'node:crypto'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -242,6 +242,8 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in t.after(() => rm(directory, { recursive: true, force: true })); const privateKeyPath = join(directory, 'app.pem'); const preload = join(directory, 'fetch.mjs'); + const workspace = join(directory, 'workspace'); + await mkdir(workspace); const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); await writeFile( privateKeyPath, @@ -251,13 +253,8 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in await writeFile( preload, ` - globalThis.fetch = async (input) => { - const url = String(input); - if (url.endsWith('/app')) return Response.json({ slug: 'lia-by-librechat' }); - if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { - return Response.json({ id: 328778573, login: 'lia-by-librechat[bot]', type: 'Bot' }); - } - throw new Error('test stopped after GitHub App validation'); + globalThis.fetch = async () => { + throw new Error('test reached GitHub App validation'); }; `, ); @@ -276,7 +273,7 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', - LIBRECHAT_CODE_WORKER_DIR: directory, + LIBRECHAT_CODE_WORKER_DIR: workspace, LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', LIBRECHAT_CODE_GITHUB_TOKEN: undefined, LIBRECHAT_CODE_GITHUB_APP_ID: '123', @@ -288,6 +285,77 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in assert.notEqual(result.status, 0); assert.doesNotMatch(result.stderr, /GitHub App authentication requires/); assert.doesNotMatch(result.stderr, /installation ID/i); + assert.match(result.stderr, /test reached GitHub App validation/); + + const checkout = spawnSync( + process.execPath, + ['--import', preload, fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + timeout: 10_000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: workspace, + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: privateKeyPath, + LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'checkout', + }, + }, + ); + assert.notEqual(checkout.status, 0); + assert.doesNotMatch( + checkout.stderr, + /Checkout GitHub repository routing requires/, + ); + assert.match(checkout.stderr, /test reached GitHub App validation/); +}); + +test('CLI rejects checkout routing outside a trusted VM or without repository-scoped App auth', () => { + const base = { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: process.cwd(), + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: '/does/not/matter', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'checkout', + }; + const cli = fileURLToPath(new URL('./cli.js', import.meta.url)); + const restricted = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: base, + }); + assert.notEqual(restricted.status, 0); + assert.match(restricted.stderr, /requires the trusted-vm command policy/); + + const fixed = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { + ...base, + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + }, + }); + assert.notEqual(fixed.status, 0); + assert.match(fixed.stderr, /without a fixed installation ID/); + + const invalid = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'unknown' }, + }); + assert.notEqual(invalid.status, 0); + assert.match(invalid.stderr, /must be admitted or checkout/); }); test('CLI requires a runtime image for Docker supervision', () => { diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 26c60e9a..841164fb 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -49,7 +49,7 @@ import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, GitHubAppCredentialProvider, - gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForCommand, gitHubRepositoryForDirectory, gitHubCommandCredentialEnvironment, gitHubMaskedCredentialVariables, @@ -150,12 +150,13 @@ function nonEmpty(value: string | undefined): string | undefined { return value?.trim().length ? value : undefined; } -function githubCredentials(): { +function githubCredentials(args: string[]): { provider?: GitHubCredentialProvider; host: string; privateKeyPath?: string; mode?: 'app' | 'token'; repositoryRouting?: boolean; + checkoutRouting?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -163,6 +164,13 @@ function githubCredentials(): { const installationId = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_INSTALLATION_ID, ); + const routing = + option(args, '--github-repository-routing')?.trim().toLowerCase() ?? + process.env.LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING?.trim().toLowerCase() ?? + 'admitted'; + if (routing !== 'admitted' && routing !== 'checkout') { + throw new Error('GitHub repository routing must be admitted or checkout'); + } const privateKeyPath = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE, ); @@ -178,6 +186,11 @@ function githubCredentials(): { 'Configure either GitHub App authentication or a GitHub token, not both', ); } + if (routing === 'checkout' && (!hasApp || installationId)) { + throw new Error( + 'Checkout GitHub repository routing requires a GitHub App without a fixed installation ID', + ); + } const configuredHostValue = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_HOST, ); @@ -211,12 +224,13 @@ function githubCredentials(): { host, mode: 'app', repositoryRouting: !installationId, + checkoutRouting: routing === 'checkout', policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, appId, installationId, - }), + }) + (routing === 'checkout' ? ':routing:checkout' : ''), privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, @@ -530,9 +544,11 @@ async function run( } const github = runtimeSessionId == null - ? githubCredentials() + ? githubCredentials(args) : { host: 'github.com', + repositoryRouting: false, + checkoutRouting: false, policyIdentity: gitHubAuthenticationPolicyIdentity({ host: 'github.com', }), @@ -547,6 +563,11 @@ async function run( 'GitHub authentication currently requires the native-srt command sandbox', ); } + if (github.checkoutRouting && commandPolicy.preset !== 'trusted-vm') { + throw new Error( + 'Checkout GitHub repository routing requires the trusted-vm command policy', + ); + } const githubDomains = github.provider ? github.host === 'github.com' ? [...GITHUB_ALLOWED_DOMAINS] @@ -759,9 +780,8 @@ async function run( }), ]), ); - // Bind credentials to immutable, explicitly admitted roots. The repository - // remote is operator input at startup, never an authorization input that a - // sandboxed command may change for its next invocation. + // Keep an admission boundary even when trusted-VM checkout routing uses a + // nested repository's remote for the current command. const admittedGitHubRepositories = github.provider && github.repositoryRouting ? new Map( await Promise.all( @@ -998,9 +1018,12 @@ async function run( ), async resolve(signal?: AbortSignal, cwd?: string) { const repository = cwd && admittedGitHubRepositories - ? gitHubRepositoryForAdmittedDirectory( + ? await gitHubRepositoryForCommand( cwd, admittedGitHubRepositories, + github.checkoutRouting ? 'checkout' : 'admitted', + github.host, + signal, ) : undefined; if (!repository && github.repositoryRouting) { diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 352bf1c3..b13d70f7 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -4,6 +4,7 @@ import { chmod, mkdtemp, mkdir, + realpath, rm, symlink, writeFile, @@ -26,6 +27,7 @@ import { GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForCommand, gitHubRepositoryForDirectory, normalizeGitHubHost, wrapGitHubCredentialCommand, @@ -459,6 +461,83 @@ test('keeps repository authorization bound to the admitted workspace root', asyn ); }); +test('checkout routing follows nested repositories only inside an admitted root', async (t) => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-github-checkout-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const nested = join(directory, 'worktrees', 'other'); + await mkdir(nested, { recursive: true }); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', directory, 'remote', 'add', 'origin', 'git@github.com:acme/outer.git', + ]); + execFileSync('git', ['init', nested]); + execFileSync('git', [ + '-C', nested, 'remote', 'add', 'origin', 'git@github.com:acme/inner.git', + ]); + const admitted = new Map([[directory, 'acme/outer']]); + + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'admitted'), + 'acme/outer', + ); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout'), + 'acme/inner', + ); + execFileSync('git', [ + '-C', nested, 'remote', 'set-url', 'origin', 'git@github.com:acme/changed.git', + ]); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout'), + 'acme/changed', + ); + assert.equal( + await gitHubRepositoryForCommand(dirname(directory), admitted, 'checkout'), + undefined, + ); + const outside = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-github-outside-')), + ); + t.after(() => rm(outside, { recursive: true, force: true })); + execFileSync('git', ['init', outside]); + execFileSync('git', [ + '-C', outside, 'remote', 'add', 'origin', 'git@github.com:acme/outside.git', + ]); + const escaped = join(directory, 'worktrees', 'escaped'); + await symlink(outside, escaped); + assert.equal( + await gitHubRepositoryForCommand(escaped, admitted, 'checkout'), + undefined, + ); + admitted.set(outside, 'acme/outside'); + assert.equal( + await gitHubRepositoryForCommand(escaped, admitted, 'checkout'), + undefined, + ); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout', 'github.example.test'), + undefined, + ); + + const linked = join(directory, 'worktrees', 'linked'); + execFileSync('git', [ + '-C', nested, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', + 'commit', '--allow-empty', '-m', 'initial', + ]); + execFileSync('git', ['-C', nested, 'worktree', 'add', '--detach', linked]); + assert.equal( + await gitHubRepositoryForCommand(linked, admitted, 'checkout'), + 'acme/changed', + ); + admitted.set(linked, 'acme/linked'); + assert.equal( + await gitHubRepositoryForCommand(linked, admitted, 'admitted'), + 'acme/linked', + ); +}); + test('uses the configured GHES host for the App bot no-reply identity', async (t) => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-ghes-identity-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index fb6b6fdc..19b8e639 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -1,7 +1,7 @@ import { constants } from 'node:fs'; import { execFile } from 'node:child_process'; import { createHash, createPrivateKey, sign } from 'node:crypto'; -import { open } from 'node:fs/promises'; +import { open, realpath } from 'node:fs/promises'; import { dirname, isAbsolute, relative, sep } from 'node:path'; import { promisify } from 'node:util'; import { projectRemote } from './projects.js'; @@ -137,21 +137,54 @@ export async function gitHubRepositoryForDirectory( return repository; } -/** Return the startup-bound repository for the admitted root containing cwd. */ -export function gitHubRepositoryForAdmittedDirectory( +function admittedRepositoryEntry( cwd: string, repositories: ReadonlyMap, -): string | undefined { - for (const [root, repository] of repositories) { +): readonly [string, string | undefined] | undefined { + let closest: readonly [string, string | undefined] | undefined; + for (const entry of repositories) { + const [root] = entry; const path = relative(root, cwd); if ( path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) ) { - return repository; + if (!closest || root.length > closest[0].length) closest = entry; } } - return undefined; + return closest; +} + +/** Return the startup-bound repository for the admitted root containing cwd. */ +export function gitHubRepositoryForAdmittedDirectory( + cwd: string, + repositories: ReadonlyMap, +): string | undefined { + return admittedRepositoryEntry(cwd, repositories)?.[1]; +} + +/** Resolve a checkout repository only when its cwd remains inside an admitted root. */ +export async function gitHubRepositoryForCommand( + cwd: string, + repositories: ReadonlyMap, + routing: 'admitted' | 'checkout', + host = 'github.com', + signal?: AbortSignal, +): Promise { + const admitted = admittedRepositoryEntry(cwd, repositories); + if (!admitted) return undefined; + if (routing === 'admitted') return admitted[1]; + let canonicalCwd: string; + try { + canonicalCwd = await realpath(cwd); + } catch { + signal?.throwIfAborted(); + return undefined; + } + if (admittedRepositoryEntry(canonicalCwd, repositories)?.[0] !== admitted[0]) { + return undefined; + } + return gitHubRepositoryForDirectory(canonicalCwd, host, signal); } function base64UrlJson(value: unknown): string { From cf0e6683ccad320c7467a10d55cf7090a1bbfb88 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 23:13:38 -0400 Subject: [PATCH 43/75] =?UTF-8?q?=F0=9F=A7=B9=20fix:=20Await=20Job=20Clean?= =?UTF-8?q?up=20and=20Measure=20Retained=20Resources=20(#249)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧹 fix: Await Job Cleanup and Measure Retained Resources * 📡 fix: Scrape Sandbox Runner Metrics in Helm --------- Co-authored-by: Lia --- .github/workflows/ci.yml | 14 ++ api/README.md | 67 ++++++++ api/src/api/v2-cleanup.test.ts | 189 +++++++++++++++++++++ api/src/api/v2.ts | 4 + api/src/cleanup-metrics.test.ts | 110 ++++++++++++ api/src/cleanup-metrics.ts | 185 ++++++++++++++++++++ api/src/cleanup.integration.test.ts | 184 ++++++++++++++++++++ api/src/job-cleanup-metrics.test.ts | 118 +++++++++++++ api/src/job.ts | 83 +++++---- api/src/nsjail.ts | 4 +- helm/codeapi/Chart.yaml | 2 +- helm/codeapi/README.md | 37 ++++ helm/codeapi/templates/network-policy.yaml | 10 ++ helm/codeapi/templates/podmonitor.yaml | 22 +++ helm/codeapi/values.yaml | 5 + tests/sandbox_runner_metrics.sh | 147 ++++++++++++++++ 16 files changed, 1143 insertions(+), 38 deletions(-) create mode 100644 api/src/api/v2-cleanup.test.ts create mode 100644 api/src/cleanup-metrics.test.ts create mode 100644 api/src/cleanup-metrics.ts create mode 100644 api/src/cleanup.integration.test.ts create mode 100644 api/src/job-cleanup-metrics.test.ts create mode 100755 tests/sandbox_runner_metrics.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b645f4b8..4076d8f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: Sandbox-runner liveness checks run: tests/sandbox_runner_healthcheck.sh + - name: Sandbox-runner metrics discovery and network policy + run: tests/sandbox_runner_metrics.sh + - name: Bridge pairing rollout safety run: tests/bridge_pairing_rollout.sh @@ -121,6 +124,17 @@ jobs: - name: Bun tests run: bun run test + - name: File-heavy workspace cleanup on tmpfs + run: | + docker run --rm --user 0 \ + --tmpfs /tmp:rw,size=1g \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --workdir /work/api \ + --env SANDBOX_CLEANUP_TMPFS_TEST=1 \ + --env SANDBOX_LOG_LEVEL=error \ + oven/bun:1.3.14-debian \ + bun test src/cleanup.integration.test.ts + service-unit-tests: name: Service Unit Tests runs-on: ubuntu-latest diff --git a/api/README.md b/api/README.md index 38e4cd98..34b32145 100644 --- a/api/README.md +++ b/api/README.md @@ -139,3 +139,70 @@ validation behavior. Roll out this sandbox behavior before enabling timeout forwarding in the service's plain `/exec` handler. Older sandboxes reject caps above their local runtime limit; older services remain compatible with updated sandboxes. + +### Runner memory after cleanup + +`Job.cleanup()` emits `Post-cleanup resources` after workspace removal and UID +release (or quarantine). `/execute` now waits for cleanup before sending success +or execution-failure responses. Artifact upload still finishes before cleanup. +Persistent session workspaces and their pinned UIDs are intentionally preserved; +failed disposable cleanup still quarantines the directory and retains its UID +for the existing retry path. This change does not alter those policies. + +The following metrics use only fixed, low-cardinality labels: + +| Metric suffix (prefix `codeapi_sandbox_`) | Meaning | +| --- | --- | +| `post_cleanup_memory_bytes{kind}` | `current`, `anon`, `file`, `shmem` at the visible cgroup v2 mount root | +| `post_cleanup_tmp_used{resource}` | `/tmp` allocated `bytes`, allocated `inodes`, and `is_tmpfs` (0 or 1) | +| `post_cleanup_workspaces{kind}` | Remaining `disposable`, `session`, and `other` entries in `/tmp/sandbox` | +| `cleanup_total{mode,outcome}` | Disposable/session cleanup attempts: `removed`, `preserved`, `retained`, `error` | +| `post_cleanup_sample_success{source}` | Whether `memory`, `tmp`, or `workspaces` was readable on the last sample | +| `post_cleanup_timestamp_seconds` | When the last cleanup sample was taken | + +These are runner-wide **last-cleanup samples**, not live gauges or per-job +memory attribution. Other jobs can still be running. Reaper-only changes are +visible on the next job cleanup, not immediately. Interpret the workspace counts +alongside active executions and the sample timestamp. Unavailable sources are +reported as unavailable and their old gauge values are removed, never replaced +with a healthy-looking zero. The structured log also includes active UID slots +and the number of retained cleanup retries. Synthetic jobs update metrics while +suppressing successful per-job logs as before. + +The visible cgroup root covers the API and sibling NsJail cgroups. The old +`Post-execution memory` log instead samples `/proc/self/cgroup` **before** job +cleanup, so it can have a different scope. `file` includes `shmem`; do not add +them or assume all `file` memory is reclaimable page cache. + +`statfs` measures allocation, including deleted-but-open files on the sampled +mount, without walking user files. The runner's 1 GiB `/tmp` mount is distinct +from each jail's 20 MiB `/tmp` mount. Its ceiling does not cap all container +memory. If workspace counts fall but shmem does not, investigate descendant +processes and retained mounts as well as files. A stable API-process FD count +alone cannot exclude those cases. Memory requests and HPA settings are unchanged. + +#### Cleanup stress tests + +Ordinary unit tests cover metrics, unavailable sources, session/disposable +classification and response ordering. The opt-in stress tests must run in an +**isolated test container**, never on an active runner: + +```bash +# From api/, with /tmp mounted as tmpfs and per-job chown available: +SANDBOX_CLEANUP_TMPFS_TEST=1 bun test src/cleanup.integration.test.ts + +# Additionally requires the runner's NsJail binary, spec-guard, config, +# and normal namespace/cgroup permissions. Run this file alone. +NSJAIL_CONFIG=/sandbox_api/config/sandbox.cfg \ +SANDBOX_CLEANUP_NSJAIL_TEST=1 bun test src/cleanup.integration.test.ts +``` + +The first test repeats real Job priming and cleanup with large payloads and many +small files, checking tmpfs bytes/inodes and UID slots after every iteration. +CI runs it in a dedicated Bun container with a 1 GiB `/tmp` tmpfs. It does not +execute NsJail. The second test uses real NsJail jobs covering normal completion, +timeout and output overflow, including a detached child holding an unlinked file. +It asserts no processes remain under the job UID before removing its workspace, +then checks workspace removal and post-cleanup tmpfs allocation. This test is +skipped unless explicitly enabled; a unit-test pass is not proof of namespace +teardown on the production kernel. diff --git a/api/src/api/v2-cleanup.test.ts b/api/src/api/v2-cleanup.test.ts new file mode 100644 index 00000000..d1efee5d --- /dev/null +++ b/api/src/api/v2-cleanup.test.ts @@ -0,0 +1,189 @@ +import { afterAll, afterEach, beforeAll, expect, test } from 'bun:test'; +import express from 'express'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import type { Server } from 'http'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { config } from '../config'; +import { Job } from '../job'; +import { loadPackage } from '../runtime'; +import { ValidationError } from '../validation'; +import router from './v2'; + +let server: Server; +let url: string; +let directory: string; +const language = 'cleanup-order-test'; +const originalPrime = Job.prototype.prime; +const originalExecute = Job.prototype.execute; +const originalUpload = Job.prototype.uploadGeneratedFiles; +const originalCleanup = Job.prototype.cleanup; +const requireManifest = config.require_execution_manifest; + +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'cleanup-order-')); + await writeFile( + join(directory, 'pkg-info.json'), + JSON.stringify({ language, version: '1.0.0', aliases: [] }) + ); + loadPackage(directory); + const app = express(); + app.use(router); + await new Promise(resolve => { + server = app.listen(0, '127.0.0.1', () => resolve()); + }); + const address = server.address(); + url = `http://127.0.0.1:${ + typeof address === 'object' && address ? address.port : 0 + }/execute`; +}); +afterAll(async () => { + await new Promise(resolve => server.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); +}); +afterEach(() => { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.uploadGeneratedFiles = originalUpload; + Job.prototype.cleanup = originalCleanup; + config.require_execution_manifest = requireManifest; +}); + +for (const outcome of [ + 'success', + 'prime_failure', + 'execution_failure', + 'validation_failure', +] as const) { + test(`${outcome} waits for cleanup before sending a response and cleans exactly once`, async () => { + config.require_execution_manifest = false; + const events: string[] = []; + let releaseCleanup!: () => void; + const cleanupGate = new Promise(resolve => { + releaseCleanup = resolve; + }); + let markCleanupStarted!: () => void; + const cleanupStarted = new Promise(resolve => { + markCleanupStarted = resolve; + }); + Job.prototype.prime = async function () { + events.push('prime'); + if (outcome === 'prime_failure') + throw new Error('scripted prime failure'); + }; + Job.prototype.execute = async function () { + events.push('execute'); + if (outcome === 'execution_failure') + throw new Error('scripted execution failure'); + if (outcome === 'validation_failure') + throw new ValidationError('scripted validation failure'); + return { files: [{ id: 'artifact', name: 'result.txt' }] } as Awaited< + ReturnType + >; + }; + Job.prototype.uploadGeneratedFiles = async function () { + events.push('upload'); + return new Set(['artifact']); + }; + Job.prototype.cleanup = async function () { + events.push('cleanup-start'); + markCleanupStarted(); + await cleanupGate; + events.push('cleanup-end'); + }; + let responseArrived = false; + const pending = fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + language, + version: '1.0.0', + files: [{ name: 'main.txt', content: 'test' }], + }), + }).then(response => { + responseArrived = true; + return response; + }); + try { + await Promise.race([ + cleanupStarted, + pending.then(response => { + throw new Error(`Response preceded cleanup: ${response.status}`); + }), + ]); + // Give an incorrectly early res.json() time to reach the client. + await new Promise(resolve => setTimeout(resolve, 30)); + expect(responseArrived).toBe(false); + } finally { + releaseCleanup(); + const response = await pending; + expect(response.status).toBe( + outcome === 'success' + ? 200 + : outcome === 'validation_failure' + ? 400 + : 500 + ); + await response.text(); + } + expect(events.filter(event => event === 'cleanup-start')).toHaveLength(1); + expect(events[events.length - 1]).toBe('cleanup-end'); + if (outcome === 'success') + expect(events).toEqual([ + 'prime', + 'execute', + 'upload', + 'cleanup-start', + 'cleanup-end', + ]); + }); +} + +test('client disconnect does not clean a workspace while execution is still running', async () => { + config.require_execution_manifest = false; + let releaseExecution!: () => void; + const executionGate = new Promise(resolve => { + releaseExecution = resolve; + }); + let markExecuting!: () => void; + const executing = new Promise(resolve => { + markExecuting = resolve; + }); + let markCleaned!: () => void; + const cleaned = new Promise(resolve => { + markCleaned = resolve; + }); + let cleanupCount = 0; + Job.prototype.prime = async function () {}; + Job.prototype.execute = async function () { + markExecuting(); + await executionGate; + return {} as Awaited>; + }; + Job.prototype.cleanup = async function () { + cleanupCount++; + markCleaned(); + }; + const controller = new AbortController(); + const pending = fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + language, + version: '1.0.0', + files: [{ name: 'main.txt', content: 'test' }], + }), + }).catch(() => undefined); + try { + await executing; + controller.abort(); + await pending; + await new Promise(resolve => setTimeout(resolve, 30)); + expect(cleanupCount).toBe(0); + } finally { + releaseExecution(); + await cleaned; + } + expect(cleanupCount).toBe(1); +}); diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index e2f252ce..121cf2e3 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -588,9 +588,13 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn } } + /* Upload must finish before cleanup, and cleanup must settle before + * acknowledging completion. Failed removals retain a quarantined UID. */ + await cleanupHandler(); metricsOutcome = 'success'; return res.status(200).json(result); } catch (error) { + await cleanupHandler(); /* Deliberately BEFORE the ValidationError branch below: once priming has * completed, the workspace has been written to, so any later failure — * including a validation one — leaves state the next execute must not diff --git a/api/src/cleanup-metrics.test.ts b/api/src/cleanup-metrics.test.ts new file mode 100644 index 00000000..e1f61b22 --- /dev/null +++ b/api/src/cleanup-metrics.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + postCleanupMemory, + postCleanupTmp, + postCleanupWorkspaces, + readCleanupResources, + recordCleanupResources, +} from './cleanup-metrics'; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, { recursive: true, force: true }); +}); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cleanup-metrics-')); + roots.push(root); + const cgroupRoot = path.join(root, 'cgroup'); + const workspaceRoot = path.join(root, 'workspaces'); + fs.mkdirSync(cgroupRoot); + fs.mkdirSync(workspaceRoot); + fs.writeFileSync(path.join(cgroupRoot, 'memory.current'), '1048576\n'); + fs.writeFileSync( + path.join(cgroupRoot, 'memory.stat'), + 'anon 262144\nfile 786432\nshmem 524288\n' + ); + return { cgroupRoot, workspaceRoot, tmpRoot: root }; +} + +describe('post-cleanup resources', () => { + test('reads byte-accurate cgroup counters without adding shmem to file', () => { + const paths = fixture(); + // The API child cgroup excludes sibling jails. Sample the visible root. + fs.mkdirSync(path.join(paths.cgroupRoot, 'sandbox_api')); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'sandbox_api/memory.current'), + '1' + ); + const snapshot = readCleanupResources(paths); + expect(snapshot.memory).toEqual({ + current: 1048576, + anon: 262144, + file: 786432, + shmem: 524288, + }); + expect(snapshot.tmp?.bytes).toBeGreaterThanOrEqual(0); + expect(snapshot.tmp?.inodes).toBeGreaterThanOrEqual(0); + expect(snapshot.errors).toEqual({}); + }); + + test('separates disposable, persistent and unexpected entries without following symlinks', () => { + const paths = fixture(); + fs.mkdirSync(path.join(paths.workspaceRoot, 'ws_active')); + fs.mkdirSync(path.join(paths.workspaceRoot, 'session')); + fs.mkdirSync(path.join(paths.workspaceRoot, 'unknown')); + fs.symlinkSync(paths.tmpRoot, path.join(paths.workspaceRoot, 'ws_symlink')); + expect(readCleanupResources(paths).workspaces).toEqual({ + disposable: 1, + session: 1, + other: 2, + }); + }); + + test('does not turn missing or malformed statistics into a healthy zero', () => { + const paths = fixture(); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'memory.stat'), + 'anon 1\nfile 2\n' + ); + expect(readCleanupResources(paths).memory).toBeNull(); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'memory.current'), + 'not-a-number' + ); + const snapshot = readCleanupResources(paths); + expect(snapshot.memory).toBeNull(); + expect(snapshot.tmp).not.toBeNull(); + expect(snapshot.workspaces).not.toBeNull(); + expect(snapshot.errors.memory).toBe('unavailable'); + }); + + test('unreadable sources remove stale gauges without failing cleanup', async () => { + const paths = fixture(); + const context = { + job: 'test', + mode: 'disposable', + outcome: 'removed', + suppressSuccessLogs: true, + } as const; + recordCleanupResources(context, readCleanupResources(paths)); + expect((await postCleanupMemory.get()).values).toHaveLength(4); + expect((await postCleanupTmp.get()).values).toHaveLength(3); + expect((await postCleanupWorkspaces.get()).values).toHaveLength(3); + fs.rmSync(paths.tmpRoot, { recursive: true }); + const snapshot = readCleanupResources(paths); + expect(snapshot.errors).toEqual({ + memory: 'ENOENT', + tmp: 'ENOENT', + workspaces: 'ENOENT', + }); + expect(() => recordCleanupResources(context, snapshot)).not.toThrow(); + expect((await postCleanupMemory.get()).values).toHaveLength(0); + expect((await postCleanupTmp.get()).values).toHaveLength(0); + expect((await postCleanupWorkspaces.get()).values).toHaveLength(0); + }); +}); diff --git a/api/src/cleanup-metrics.ts b/api/src/cleanup-metrics.ts new file mode 100644 index 00000000..7444960d --- /dev/null +++ b/api/src/cleanup-metrics.ts @@ -0,0 +1,185 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { Counter, Gauge } from 'prom-client'; +import { logger } from './logger'; +import { + SANDBOX_WORKSPACE_ROOT, + SESSION_WORKSPACE_ID, + WORKSPACE_ID_PREFIX, + retainedWorkspaceCleanupCount, + sandboxJobUidPool, +} from './workspace-isolation'; + +const memoryKinds = ['current', 'anon', 'file', 'shmem'] as const; +const workspaceKinds = ['disposable', 'session', 'other'] as const; +type MemoryKind = typeof memoryKinds[number]; +type WorkspaceKind = typeof workspaceKinds[number]; + +export const postCleanupMemory = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_memory_bytes', + help: 'Last post-cleanup cgroup-root memory sample; file includes shmem, not additive', + labelNames: ['kind'] as const, +}); +export const postCleanupTmp = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_tmp_used', + help: 'Last post-cleanup /tmp filesystem allocation, including unlinked open files', + labelNames: ['resource'] as const, +}); +export const postCleanupWorkspaces = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_workspaces', + help: 'Remaining workspace-root entries after cleanup; includes other active jobs', + labelNames: ['kind'] as const, +}); +const cleanupAttempts = new Counter({ + name: 'codeapi_sandbox_cleanup_total', + help: 'Job cleanup attempts; session workspaces are intentionally preserved', + labelNames: ['mode', 'outcome'] as const, +}); +const sampleSuccess = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_sample_success', + help: 'Whether each source was readable on the last post-cleanup sample', + labelNames: ['source'] as const, +}); +const sampleTimestamp = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_timestamp_seconds', + help: 'Timestamp of the last job cleanup resource sample, not a live usage gauge', +}); + +export interface CleanupResourceSnapshot { + memory: Record | null; + tmp: { bytes: number; inodes: number; isTmpfs: boolean } | null; + workspaces: Record | null; + errors: Partial>; +} + +/** Read only fixed kernel files and the workspace root, never recurse into + * untrusted workspace trees or follow their symlinks. Synchronous sampling + * avoids older, slower samples overwriting newer ones during parallel cleanup. + * The visible cgroup mount root includes the API AND sibling NsJail cgroups; + * /proc/self/cgroup may point only at the delegated sandbox_api child instead. + * These are runner-wide observations, not memory attributable to one job. */ +export function readCleanupResources( + paths: { + cgroupRoot?: string; + tmpRoot?: string; + workspaceRoot?: string; + } = {} +): CleanupResourceSnapshot { + const snapshot: CleanupResourceSnapshot = { + memory: null, + tmp: null, + workspaces: null, + errors: {}, + }; + function sample( + source: keyof CleanupResourceSnapshot['errors'], + read: () => T + ): T | null { + try { + return read(); + } catch (error) { + snapshot.errors[source] = + (error as NodeJS.ErrnoException).code ?? 'unavailable'; + return null; + } + } + snapshot.memory = sample('memory', () => { + const root = paths.cgroupRoot ?? '/sys/fs/cgroup'; + const current = fs + .readFileSync(path.join(root, 'memory.current'), 'utf8') + .trim(); + const stat = fs.readFileSync(path.join(root, 'memory.stat'), 'utf8'); + const values = Object.fromEntries( + stat + .trim() + .split('\n') + .map(line => line.trim().split(/\s+/)) + ); + const result = {} as Record; + for (const kind of memoryKinds) { + const raw = kind === 'current' ? current : values[kind]; + if (!raw || !/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw))) { + throw new Error('Invalid memory statistic'); + } + result[kind] = Number(raw); + } + return result; + }); + snapshot.tmp = sample('tmp', () => { + const stat = fs.statfsSync(paths.tmpRoot ?? '/tmp'); + return { + bytes: (stat.blocks - stat.bfree) * stat.bsize, + inodes: stat.files - stat.ffree, + isTmpfs: stat.type === 0x01021994, + }; + }); + snapshot.workspaces = sample('workspaces', () => { + const counts = { disposable: 0, session: 0, other: 0 }; + for (const entry of fs.readdirSync( + paths.workspaceRoot ?? SANDBOX_WORKSPACE_ROOT, + { withFileTypes: true } + )) { + const kind = + entry.isDirectory() && entry.name === SESSION_WORKSPACE_ID + ? 'session' + : entry.isDirectory() && entry.name.startsWith(WORKSPACE_ID_PREFIX) + ? 'disposable' + : 'other'; + counts[kind]++; + } + return counts; + }); + return snapshot; +} + +export function recordCleanupResources( + context: { + job: string; + mode: 'disposable' | 'session'; + outcome: 'removed' | 'preserved' | 'retained' | 'error'; + suppressSuccessLogs?: boolean; + }, + snapshot = readCleanupResources() +): void { + cleanupAttempts.inc({ mode: context.mode, outcome: context.outcome }); + sampleTimestamp.set(Date.now() / 1000); + for (const source of ['memory', 'tmp', 'workspaces'] as const) { + sampleSuccess.set({ source }, snapshot[source] === null ? 0 : 1); + } + for (const kind of memoryKinds) { + if (snapshot.memory) postCleanupMemory.set({ kind }, snapshot.memory[kind]); + else postCleanupMemory.remove({ kind }); + } + for (const resource of ['bytes', 'inodes', 'is_tmpfs'] as const) { + if (snapshot.tmp) { + postCleanupTmp.set( + { resource }, + resource === 'is_tmpfs' + ? Number(snapshot.tmp.isTmpfs) + : snapshot.tmp[resource] + ); + } else postCleanupTmp.remove({ resource }); + } + for (const kind of workspaceKinds) { + if (snapshot.workspaces) + postCleanupWorkspaces.set({ kind }, snapshot.workspaces[kind]); + else postCleanupWorkspaces.remove({ kind }); + } + if ( + !context.suppressSuccessLogs || + context.outcome === 'retained' || + context.outcome === 'error' + ) { + logger.info( + { + job: context.job, + mode: context.mode, + outcome: context.outcome, + ...snapshot, + activeUidSlots: sandboxJobUidPool.activeCount(), + retainedCleanups: retainedWorkspaceCleanupCount(), + }, + 'Post-cleanup resources' + ); + } +} diff --git a/api/src/cleanup.integration.test.ts b/api/src/cleanup.integration.test.ts new file mode 100644 index 00000000..0d08e804 --- /dev/null +++ b/api/src/cleanup.integration.test.ts @@ -0,0 +1,184 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as fsp from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { config } from './config'; +import { Job } from './job'; +import type { Runtime } from './runtime'; +import { readCleanupResources } from './cleanup-metrics'; +import { prepareWorkspaceRoot, sandboxJobUidPool } from './workspace-isolation'; + +// Run only in a dedicated Linux test container with /tmp mounted as tmpfs. +// The real-jail variant additionally needs the runner image's namespace/cgroup +// permissions, nsjail, spec-guard and NSJAIL_CONFIG. Never target a live runner. +const tmpfsEnabled = process.env.SANDBOX_CLEANUP_TMPFS_TEST === '1'; +const nsjailEnabled = process.env.SANDBOX_CLEANUP_NSJAIL_TEST === '1'; + +function runtime(pkgdir: string): Runtime { + return { + language: 'bash', + version: new semver.SemVer('5.2.0'), + aliases: [], + pkgdir, + compiled: false, + env_vars: {}, + timeouts: { compile: 1000, run: 1000 }, + cpu_times: { compile: 1000, run: 1000 }, + memory_limits: { compile: 128 * 1048576, run: 128 * 1048576 }, + max_process_count: 64, + max_open_files: 64, + max_file_size: 16 * 1048576, + output_max_size: 4096, + }; +} + +function jobFor(rt: Runtime, content: string): Job { + return new Job({ + session_id: 'cleanup-stress', + runtime: rt, + files: [{ name: 'main.sh', content }], + args: [], + stdin: '', + timeouts: rt.timeouts, + cpu_times: rt.cpu_times, + memory_limits: rt.memory_limits, + is_synthetic: true, + }); +} + +function workspace(job: Job): string { + return (job as unknown as { submissionDir: string }).submissionDir; +} + +function assertTmpfsBaseline( + baseline: ReturnType +): void { + const after = readCleanupResources(); + expect(after.tmp?.isTmpfs).toBe(true); + // Allow a few metadata pages, not one retained 8 MiB payload per execution. + expect(after.tmp!.bytes).toBeLessThanOrEqual(baseline.tmp!.bytes + 64 * 1024); + expect(after.tmp!.inodes).toBeLessThanOrEqual(baseline.tmp!.inodes + 4); + expect(after.workspaces).toEqual(baseline.workspaces); +} + +test.skipIf(!tmpfsEnabled)( + 'repeated file-heavy Job cleanup returns tmpfs bytes and inodes near baseline', + async () => { + expect(process.platform).toBe('linux'); + await prepareWorkspaceRoot(); + const baseline = readCleanupResources(); + expect(baseline.tmp?.isTmpfs).toBe(true); + const activeBefore = sandboxJobUidPool.activeCount(); + const payload = Buffer.alloc(8 * 1048576, 0x61); + for (let iteration = 0; iteration < 24; iteration++) { + const job = jobFor(runtime('/tmp'), 'echo test'); + let dir = ''; + try { + await job.prime(); + dir = workspace(job); + await fsp.writeFile(path.join(dir, 'payload'), payload); + for (let i = 0; i < 64; i++) + await fsp.writeFile(path.join(dir, `small-${i}`), 'x'); + expect(readCleanupResources().tmp!.bytes).toBeGreaterThan( + baseline.tmp!.bytes + payload.length + ); + } finally { + await job.cleanup(); + } + expect(fs.existsSync(dir)).toBe(false); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + assertTmpfsBaseline(baseline); + } + }, + 30_000 +); + +function processesForUid(uid: number): string[] { + return fs.readdirSync('/proc').filter(pid => { + if (!/^\d+$/.test(pid)) return false; + try { + return ( + fs + .readFileSync(`/proc/${pid}/status`, 'utf8') + .match(/^Uid:\s+(\d+)/m)?.[1] === String(uid) + ); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code === 'ENOENT' || + (error as NodeJS.ErrnoException).code === 'ESRCH' + ) + return false; + throw error; + } + }); +} + +test.skipIf(!nsjailEnabled)( + 'real NsJail releases detached descendants and file-heavy workspaces on success, timeout and overflow', + async () => { + expect(process.platform).toBe('linux'); + expect(config.per_job_uids).toBe(true); + expect(process.getuid?.()).toBe(0); + await prepareWorkspaceRoot(); + const pkgdir = await fsp.mkdtemp( + path.join(os.tmpdir(), 'cleanup-runtime-') + ); + await fsp.chmod(pkgdir, 0o755); + await fsp.writeFile(path.join(pkgdir, 'run'), 'exec /bin/bash "$@"\n', { + mode: 0o644, + }); + const baseline = readCleanupResources(); + expect(baseline.tmp?.isTmpfs).toBe(true); + const activeBefore = sandboxJobUidPool.activeCount(); + try { + for (let iteration = 0; iteration < 12; iteration++) { + const ending = iteration % 3; + const job = jobFor( + runtime(pkgdir), + `set -eu +# Both linked workspace data and an unlinked open file held by a detached child. +dd if=/dev/zero of=payload bs=1048576 count=8 status=none +# Per-jail /tmp is a different mount from the runner's /tmp. +dd if=/dev/zero of=/tmp/held bs=1048576 count=8 status=none +exec 3 /mnt/data/child-ready; exec sleep 30' >/dev/null 2>&1 & +while [ ! -f child-ready ]; do sleep 0.01; done +${ + ending === 0 + ? 'echo complete' + : ending === 1 + ? 'sleep 30' + : "while :; do printf '%1024s' x; done" +} +` + ); + let dir = ''; + let uid = -1; + try { + await job.prime(); + dir = workspace(job); + uid = (job as unknown as { jobIdentity: { uid: number } }).jobIdentity + .uid; + const result = await job.execute(); + expect(result.run?.status).toBe( + ending === 0 ? null : ending === 1 ? 'TO' : 'OL' + ); + if (ending === 0) expect(result.run?.stdout).toContain('complete'); + // Check before workspace removal can conceal a surviving writer. + expect(processesForUid(uid)).toEqual([]); + } finally { + await job.cleanup(); + } + expect(fs.existsSync(dir)).toBe(false); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + assertTmpfsBaseline(baseline); + } + } finally { + await fsp.rm(pkgdir, { recursive: true, force: true }); + } + }, + 60_000 +); diff --git a/api/src/job-cleanup-metrics.test.ts b/api/src/job-cleanup-metrics.test.ts new file mode 100644 index 00000000..6806f637 --- /dev/null +++ b/api/src/job-cleanup-metrics.test.ts @@ -0,0 +1,118 @@ +import { afterEach, expect, spyOn, test } from 'bun:test'; +import * as fsp from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { register } from 'prom-client'; +import { Job } from './job'; +import type { Runtime } from './runtime'; +import type { SessionWorkspace } from './session-workspace'; +import { + clearRetainedWorkspaceCleanupsForTest, + retainedWorkspaceCleanupCount, + retryRetainedWorkspaceCleanups, + sandboxJobUidPool, +} from './workspace-isolation'; + +import type { + SandboxJobIdentity, + SandboxWorkspaceLease, +} from './workspace-isolation'; + +interface Internals { + submissionDir: string; + workspaceLease?: SandboxWorkspaceLease; + jobIdentity?: SandboxJobIdentity; +} +const roots: string[] = []; +afterEach(async () => { + await retryRetainedWorkspaceCleanups(); + clearRetainedWorkspaceCleanupsForTest(); + for (const root of roots.splice(0)) + await fsp.rm(root, { recursive: true, force: true }); +}); + +async function fixture(persistent = false) { + const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'job-cleanup-metric-')); + roots.push(dir); + const identity = sandboxJobUidPool.acquire()!; + expect(identity).not.toBeNull(); + const lease = { dir, workspaceId: path.basename(dir), identity }; + const rt: Runtime = { + language: 'bash', + version: new semver.SemVer('5.2.0'), + aliases: [], + pkgdir: '/tmp', + compiled: false, + env_vars: {}, + timeouts: { compile: 1000, run: 1000 }, + cpu_times: { compile: 1000, run: 1000 }, + memory_limits: { compile: -1, run: -1 }, + max_process_count: 64, + max_open_files: 64, + max_file_size: 1000, + output_max_size: 1000, + }; + const job = new Job({ + runtime: rt, + files: [], + args: [], + stdin: '', + timeouts: rt.timeouts, + cpu_times: rt.cpu_times, + memory_limits: rt.memory_limits, + is_synthetic: true, + session: persistent ? ({} as SessionWorkspace) : undefined, + }); + Object.assign(job as unknown as Internals, { + submissionDir: dir, + workspaceLease: lease, + jobIdentity: identity, + }); + return { job, dir, identity }; +} + +async function counter(mode: string, outcome: string): Promise { + const metric = register.getSingleMetric('codeapi_sandbox_cleanup_total')!; + return ( + (await metric.get()).values.find( + value => value.labels.mode === mode && value.labels.outcome === outcome + )?.value ?? 0 + ); +} + +test('failed removal is measured as retained and does not release its UID before retry', async () => { + const activeBefore = sandboxJobUidPool.activeCount(); + const retainedBefore = await counter('disposable', 'retained'); + const { job, dir } = await fixture(); + const rm = spyOn(fsp, 'rm').mockRejectedValueOnce( + Object.assign(new Error('test busy workspace'), { code: 'EBUSY' }) + ); + try { + await job.cleanup(); + expect(await counter('disposable', 'retained')).toBe(retainedBefore + 1); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore + 1); + expect(retainedWorkspaceCleanupCount()).toBe(1); + expect((await fsp.lstat(dir)).isDirectory()).toBe(true); + } finally { + rm.mockRestore(); + await retryRetainedWorkspaceCleanups(); + } + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + expect(await fsp.lstat(dir).catch(() => null)).toBeNull(); +}); + +test('session cleanup is measured as preserved and retains both files and pinned UID', async () => { + const activeBefore = sandboxJobUidPool.activeCount(); + const preservedBefore = await counter('session', 'preserved'); + const { job, dir, identity } = await fixture(true); + try { + await fsp.writeFile(path.join(dir, 'state'), 'keep'); + await job.cleanup(); + expect(await counter('session', 'preserved')).toBe(preservedBefore + 1); + expect(await fsp.readFile(path.join(dir, 'state'), 'utf8')).toBe('keep'); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore + 1); + } finally { + sandboxJobUidPool.release(identity); + } +}); diff --git a/api/src/job.ts b/api/src/job.ts index 0939de08..5c2fddef 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -14,6 +14,7 @@ import { logger as rootLogger } from './logger'; import { getRuntimes } from './runtime'; import { execute } from './nsjail'; import { config } from './config'; +import { recordCleanupResources } from './cleanup-metrics'; import { internalServiceHeaders } from './internal-service-auth'; import { EGRESS_GRANT_HEADER, EGRESS_ERROR_CODE_HEADER } from './egress'; import { injectTraceHeaders } from './telemetry'; @@ -2716,49 +2717,61 @@ export class Job { this.log.info('Cleaning up'); } - /* Session mode: the workspace and pinned UID belong to the long-lived - * session, not this job. Keep both so the next call sees prior files; - * teardown happens on the /terminate hook (or explicit session reset). */ - if (this.session) { - this.workspaceLease = undefined; - this.submissionDir = ''; - this.jobIdentity = undefined; - return; - } - - let workspaceRemoved = true; - const workspaceLease = this.workspaceLease; - const jobIdentity = this.jobIdentity; - - if (workspaceLease) { - try { - workspaceRemoved = await cleanupSandboxWorkspace(workspaceLease); - } catch (error) { - workspaceRemoved = false; - this.log.error({ submissionDir: this.submissionDir, err: error }, 'Failed to clean up'); - } finally { + let outcome: 'removed' | 'preserved' | 'retained' | 'error' = 'error'; + try { + /* Session mode: the workspace and pinned UID belong to the long-lived + * session, not this job. Keep both so the next call sees prior files; + * teardown happens on the /terminate hook (or explicit session reset). */ + if (this.session) { this.workspaceLease = undefined; this.submissionDir = ''; + this.jobIdentity = undefined; + outcome = 'preserved'; + return; } - } - if (jobIdentity) { - if (!workspaceLease || workspaceRemoved) { - releaseJobIdentity(jobIdentity); - } else { - retainWorkspaceCleanupUntilRemoved(workspaceLease, () => { + let workspaceRemoved = true; + const workspaceLease = this.workspaceLease; + const jobIdentity = this.jobIdentity; + + if (workspaceLease) { + try { + workspaceRemoved = await cleanupSandboxWorkspace(workspaceLease); + } catch (error) { + workspaceRemoved = false; + this.log.error({ submissionDir: this.submissionDir, err: error }, 'Failed to clean up'); + } finally { + this.workspaceLease = undefined; + this.submissionDir = ''; + } + } + + if (jobIdentity) { + if (!workspaceLease || workspaceRemoved) { releaseJobIdentity(jobIdentity); - this.log.info( + } else { + retainWorkspaceCleanupUntilRemoved(workspaceLease, () => { + releaseJobIdentity(jobIdentity); + this.log.info( + { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, + 'Released retained sandbox job UID slot after workspace cleanup', + ); + }); + this.log.error( { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, - 'Released retained sandbox job UID slot after workspace cleanup', + 'Retaining sandbox job UID slot after failed workspace cleanup', ); - }); - this.log.error( - { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, - 'Retaining sandbox job UID slot after failed workspace cleanup', - ); + } + this.jobIdentity = undefined; } - this.jobIdentity = undefined; + outcome = workspaceRemoved ? 'removed' : 'retained'; + } finally { + recordCleanupResources({ + job: this.uuid, + mode: this.session ? 'session' : 'disposable', + outcome, + suppressSuccessLogs: this.isSynthetic, + }); } } } diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 87a2b791..a9f02931 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -523,10 +523,10 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = }); const wallTime = Date.now() - startTime; - // Log memory metrics after each execution to track potential leaks. + // Pre-cleanup diagnostic only: Job.cleanup records post-cleanup resources. // Reads from /proc/self/cgroup to find the actual cgroup path, then reads // memory.current and memory.stat from that cgroup. - // Distinguishes real usage (anon) from reclaimable kernel page cache (file). + // `file` includes shmem/tmpfs and is not all reclaimable page cache. try { const cgroupLine = fs.readFileSync('/proc/self/cgroup', 'utf8').trim(); // cgroup v2 format: "0::" diff --git a/helm/codeapi/Chart.yaml b/helm/codeapi/Chart.yaml index 4e122bbf..dfcdd45b 100644 --- a/helm/codeapi/Chart.yaml +++ b/helm/codeapi/Chart.yaml @@ -3,7 +3,7 @@ apiVersion: v2 name: codeapi description: A Helm chart for Code Interpreter API - scalable code execution service type: application -version: 0.3.1 # Chart version (bump this when you change the chart) +version: 0.3.2 # Chart version (bump this when you change the chart) appVersion: "2.0.0" # App version (bump this when you change the app) # Keywords for searching diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index a063e597..f63950ae 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -55,6 +55,43 @@ platform rather than templated here: external ingress/service mesh, KEDA-style queue-depth autoscaling, and cloud-IAM secret delivery (the env hooks below cover all of them). +**Prometheus scraping.** With `metrics.enabled=true` and +`workerSandbox.enabled=true`, separate PodMonitors scrape the service-worker's `health` port and the sandbox-runner's +`sandbox` port at `/metrics`. Only the latter exports +`codeapi_sandbox_post_cleanup_*` metrics. Both inherit `metrics.interval` and +`metrics.scrapeTimeout`. Install the Prometheus Operator PodMonitor CRD and +configure Prometheus to discover this release's PodMonitors and namespace. + +The default NetworkPolicy allows runner ingress only from service-worker pods. +To permit scraping with `networkPolicy.enabled=true`, explicitly match your +trusted Prometheus namespace **and** pod labels in one peer, for example: + +```yaml +metrics: + enabled: true + sandboxRunner: + ingressFrom: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus +``` + +Replace the example labels with those of your scraper. The default empty list +adds no access, even when metrics are enabled. This rule is rendered only while +metrics and the worker/sandbox tier are enabled. It uses the configured runner +port and does not change sandbox egress. Any policies on the Prometheus side +must also allow the scrape connection. When chart network policies are disabled, +allow scraping through your platform's policies instead. + +NetworkPolicy cannot restrict access to an HTTP path. These peers can reach the +runner's execution endpoint as well, so do not use unrestricted peers such as +`{}` or disable execution-manifest verification to enable scraping. No extra +Service or externally exposed port is created. Chart upgrades do not modify +runner images or require changes to stored session state for this scrape wiring. + **Pairing-fence rollbacks.** Do not use a direct `helm rollback` from a chart revision containing the bridge pairing fence to an older revision. Helm runs rollback hooks from the target revision, so a pre-fence target cannot stop its diff --git a/helm/codeapi/templates/network-policy.yaml b/helm/codeapi/templates/network-policy.yaml index 2184e9f9..fe24123f 100644 --- a/helm/codeapi/templates/network-policy.yaml +++ b/helm/codeapi/templates/network-policy.yaml @@ -3,6 +3,7 @@ Network Policies for CodeAPI. The important boundary is sandbox-runner: - service-worker may call sandbox-runner. + - explicitly configured metrics peers may reach the same runner port. - sandbox-runner may call only egress-gateway (plus DNS if required). - egress-gateway delegates only to file-server and tool-call-server. */}} @@ -70,6 +71,15 @@ spec: ports: - protocol: TCP port: {{ .Values.workerSandbox.sandbox.port }} + {{- if and .Values.metrics.enabled .Values.workerSandbox.enabled }} + {{- with (.Values.metrics.sandboxRunner | default dict).ingressFrom }} + - from: + {{- toYaml . | nindent 6 }} + ports: + - protocol: TCP + port: {{ $.Values.workerSandbox.sandbox.port }} + {{- end }} + {{- end }} egress: - to: - namespaceSelector: {} diff --git a/helm/codeapi/templates/podmonitor.yaml b/helm/codeapi/templates/podmonitor.yaml index 405e3bc3..aee33f44 100644 --- a/helm/codeapi/templates/podmonitor.yaml +++ b/helm/codeapi/templates/podmonitor.yaml @@ -43,6 +43,28 @@ spec: matchLabels: {{- include "codeapi.serviceWorker.selectorLabels" . | nindent 6 }} {{- end }} +{{- if .Values.workerSandbox.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "codeapi.fullname" . }}-sandbox-runner + labels: + {{- include "codeapi.sandboxRunner.labels" . | nindent 4 }} +spec: + podMetricsEndpoints: + - path: /metrics + port: sandbox + {{- with .Values.metrics.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.metrics.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "codeapi.sandboxRunner.selectorLabels" . | nindent 6 }} +{{- end }} {{- if .Values.egressGateway.enabled }} --- apiVersion: monitoring.coreos.com/v1 diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index bafed305..0a242639 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -512,6 +512,11 @@ metrics: enabled: false # Set to true to create PodMonitors for Prometheus scraping interval: "30s" scrapeTimeout: "10s" + sandboxRunner: + # Explicit NetworkPolicy peers allowed to scrape the runner when metrics and + # networkPolicy are enabled. Empty preserves worker-only ingress. This port + # also serves /execute, so select only trusted Prometheus pods (see README). + ingressFrom: [] # ============================================================================= # NETWORK POLICIES diff --git a/tests/sandbox_runner_metrics.sh b/tests/sandbox_runner_metrics.sh new file mode 100755 index 00000000..df7c9ff3 --- /dev/null +++ b/tests/sandbox_runner_metrics.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT +command -v helm >/dev/null + +# Like block_root_package_delivery.sh, render only this chart's templates so +# tests do not download Redis or MinIO dependencies or contact a cluster. +mkdir "$TMP_DIR/chart" +cp "$ROOT/helm/codeapi/values.yaml" "$TMP_DIR/chart/values.yaml" +cp -R "$ROOT/helm/codeapi/templates" "$TMP_DIR/chart/templates" +awk '/^dependencies:/{exit} {print}' "$ROOT/helm/codeapi/Chart.yaml" > "$TMP_DIR/chart/Chart.yaml" + +render() { + helm template telemetry "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test "$@" +} + +resource() { + local file="$1" kind="$2" name="$3" + awk -v kind="$kind" -v name="$name" ' + function emit() { if (foundKind && foundName) printf "%s", document } + /^---$/ { emit(); document=""; foundKind=0; foundName=0; next } + { document=document $0 "\n" } + $0 == "kind: " kind { foundKind=1 } + $0 == " name: " name { foundName=1 } + END { emit() } + ' "$file" +} + +contains() { + if ! grep -Fq -- "$2" "$1"; then + echo "Missing '$2' in $(basename "$1")" >&2 + exit 1 + fi +} + +check_monitor() { + local rendered="$1" component="$2" port="$3" + local monitor="$TMP_DIR/$component-monitor.yaml" + resource "$rendered" PodMonitor "telemetry-codeapi-$component" > "$monitor" + contains "$monitor" ' - path: /metrics' + contains "$monitor" " port: $port" + contains "$monitor" ' interval: 17s' + contains "$monitor" ' scrapeTimeout: 7s' + # Exact selectors must match the corresponding Deployment's pod labels, + # including this release, and not accidentally select the service worker. + sed -n '/^ selector:/,$p' "$monitor" > "$TMP_DIR/selector.yaml" + contains "$TMP_DIR/selector.yaml" ' app.kubernetes.io/name: codeapi' + contains "$TMP_DIR/selector.yaml" ' app.kubernetes.io/instance: telemetry' + contains "$TMP_DIR/selector.yaml" " app.kubernetes.io/component: $component" + resource "$rendered" Deployment "telemetry-codeapi-$component" > "$TMP_DIR/deployment.yaml" + contains "$TMP_DIR/deployment.yaml" " app.kubernetes.io/component: $component" + contains "$TMP_DIR/deployment.yaml" " - name: $port" +} + +render > "$TMP_DIR/default.yaml" +if grep -q '^kind: PodMonitor$' "$TMP_DIR/default.yaml"; then + echo 'metrics.enabled=false must not create PodMonitors' >&2 + exit 1 +fi + +render --set metrics.enabled=true --set metrics.interval=17s --set metrics.scrapeTimeout=7s \ + --set workerSandbox.sandbox.port=2345 > "$TMP_DIR/enabled.yaml" +check_monitor "$TMP_DIR/enabled.yaml" sandbox-runner sandbox +check_monitor "$TMP_DIR/enabled.yaml" service-worker health +contains "$TMP_DIR/deployment.yaml" ' app.kubernetes.io/instance: telemetry' +resource "$TMP_DIR/enabled.yaml" Deployment telemetry-codeapi-sandbox-runner > "$TMP_DIR/runner.yaml" +contains "$TMP_DIR/runner.yaml" ' containerPort: 2345' + +# Enabling metrics must not silently weaken sandbox isolation. +resource "$TMP_DIR/default.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/default-policy.yaml" +resource "$TMP_DIR/enabled.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/enabled-policy.yaml" +# Use the same configured runner port when comparing policies. +sed 's/port: 2345/port: 2000/' "$TMP_DIR/enabled-policy.yaml" > "$TMP_DIR/normalized-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/normalized-policy.yaml" + +cat > "$TMP_DIR/scraper.yaml" <<'YAML' +metrics: + sandboxRunner: + ingressFrom: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: metrics-test + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-test +YAML +render --set metrics.enabled=true --set workerSandbox.sandbox.port=2345 \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/scraper-render.yaml" +resource "$TMP_DIR/scraper-render.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/scraper-policy.yaml" +contains "$TMP_DIR/scraper-policy.yaml" 'kubernetes.io/metadata.name: metrics-test' +contains "$TMP_DIR/scraper-policy.yaml" 'app.kubernetes.io/name: prometheus-test' +contains "$TMP_DIR/scraper-policy.yaml" 'app.kubernetes.io/component: service-worker' +# The namespace and pod selectors must be ANDed in ONE peer, not two ORed peers. +contains "$TMP_DIR/scraper-policy.yaml" ' - namespaceSelector:' +contains "$TMP_DIR/scraper-policy.yaml" ' podSelector:' +[[ "$(grep -Fc ' port: 2345' "$TMP_DIR/scraper-policy.yaml")" == 2 ]] +# Scraping changes only ingress, never the sandbox egress boundary. +sed -n '/^ egress:/,$p' "$TMP_DIR/scraper-policy.yaml" > "$TMP_DIR/scraper-egress.yaml" +sed -n '/^ egress:/,$p' "$TMP_DIR/default-policy.yaml" > "$TMP_DIR/default-egress.yaml" +cmp "$TMP_DIR/default-egress.yaml" "$TMP_DIR/scraper-egress.yaml" + +render -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/disabled-scraper.yaml" +resource "$TMP_DIR/disabled-scraper.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/disabled-scraper-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/disabled-scraper-policy.yaml" + +render --set metrics.enabled=true --set workerSandbox.enabled=false \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/no-workers.yaml" +[[ -z "$(resource "$TMP_DIR/no-workers.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +[[ -z "$(resource "$TMP_DIR/no-workers.yaml" PodMonitor telemetry-codeapi-service-worker)" ]] +resource "$TMP_DIR/no-workers.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/no-workers-policy.yaml" +if grep -q prometheus-test "$TMP_DIR/no-workers-policy.yaml"; then + echo 'disabled workerSandbox must not enable scrape ingress' >&2 + exit 1 +fi + +# Older values may omit the new subtree; this must remain worker-only ingress. +render --set metrics.enabled=true --set metrics.sandboxRunner=null > "$TMP_DIR/old-values.yaml" +resource "$TMP_DIR/old-values.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/old-values-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/old-values-policy.yaml" + +render --set metrics.enabled=true --set api.enabled=false > "$TMP_DIR/no-api.yaml" +[[ -n "$(resource "$TMP_DIR/no-api.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +[[ -z "$(resource "$TMP_DIR/no-api.yaml" PodMonitor telemetry-codeapi-api)" ]] + +render --set metrics.enabled=true --set metrics.interval= --set metrics.scrapeTimeout= > "$TMP_DIR/no-timing.yaml" +resource "$TMP_DIR/no-timing.yaml" PodMonitor telemetry-codeapi-sandbox-runner > "$TMP_DIR/no-timing-monitor.yaml" +if grep -Eq 'interval:|scrapeTimeout:' "$TMP_DIR/no-timing-monitor.yaml"; then + echo 'empty scrape timing overrides must defer to Prometheus defaults' >&2 + exit 1 +fi + +render --set metrics.enabled=true --set networkPolicy.enabled=false \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/no-policy.yaml" +[[ -n "$(resource "$TMP_DIR/no-policy.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +if grep -q '^kind: NetworkPolicy$' "$TMP_DIR/no-policy.yaml"; then + echo 'networkPolicy.enabled=false must not create NetworkPolicies' >&2 + exit 1 +fi + +helm lint "$TMP_DIR/chart" --set executionManifest.privateKey=test --set executionManifest.publicKey=test \ + --set metrics.enabled=true -f "$TMP_DIR/scraper.yaml" +echo 'Sandbox runner metrics chart regressions passed' From b03582ae8740e6b56d12945e95f57cd658048fd8 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 27 Sep 2026 11:41:22 -0400 Subject: [PATCH 44/75] feat: Allow Installation-Scoped GitHub Tokens for Trusted Workers (#263) * feat(code): allow installation-scoped GitHub tokens on trusted workers * fix(code): revalidate GitHub installation token grants --------- Co-authored-by: Lia --- docs/remote-bridge/worker-runbook.md | 9 + packages/code/README.md | 11 ++ packages/code/src/cli.test.ts | 29 +++ packages/code/src/cli.ts | 23 ++- packages/code/src/github.test.ts | 266 +++++++++++++++++++++++++++ packages/code/src/github.ts | 159 ++++++++++++---- 6 files changed, 459 insertions(+), 38 deletions(-) diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 3043c541..493eb000 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -338,6 +338,15 @@ filesystem root, but anyone able to alter a checkout's remote can select any repository where the App is installed; keep the App's installation scope narrow. Pass the checkout as the command working directory; changing directories only inside the shell cannot change the token chosen before command launch. +For a trusted VM that needs to switch among repositories in the same installed +account or organization inside one command, set +`LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE=installation`. The resolved installation +token covers only repositories and permissions GitHub granted to that App +installation. It refreshes after two minutes so newly approved permissions +become available without a worker restart. The default is `repository`. +Commands spanning different accounts or organizations must start in a checkout +from the target account or organization; a shell `cd` cannot switch the +installation chosen at command launch. Set `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation fallback; it cannot be combined with checkout routing. diff --git a/packages/code/README.md b/packages/code/README.md index 5a06b92a..a87c87df 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -308,6 +308,17 @@ installed**. Use this mode only where the machine operator trusts the VM and the App's installation scope; the default `admitted` mode keeps the startup binding. Checkout routing requires an App without a fixed installation ID. +On a trusted VM, `--github-token-scope installation` (or +`LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE=installation`) mints one token for all +repositories GitHub grants to the resolved App installation. This lets a +command started in one checkout push to another repository in the same account +or organization, including through `cd` or `git -C`, and use organization +Projects. GitHub still enforces the installation's selected repositories and +permissions. Tokens are shared by installation, refreshed after two minutes, +and kept out of the sandbox's readable environment. The default remains +`repository`. A command crossing to another account or organization still +needs to start in a checkout belonging to that account or organization. + For compatibility with deployments that intentionally bind a worker to one installation, set the optional legacy `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f95d7eee..f6737d5f 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -358,6 +358,35 @@ test('CLI rejects checkout routing outside a trusted VM or without repository-sc assert.match(invalid.stderr, /must be admitted or checkout/); }); +test('CLI permits installation-scoped GitHub tokens only for a trusted VM with routed App auth', () => { + const cli = fileURLToPath(new URL('./cli.js', import.meta.url)); + const base = { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: process.cwd(), + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: '/does/not/matter', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE: 'installation', + }; + const restricted = spawnSync(process.execPath, [cli], { encoding: 'utf8', env: base }); + assert.match(restricted.stderr, /Installation-scoped GitHub tokens require the trusted-vm/); + const trusted = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm' }, + }); + assert.doesNotMatch(trusted.stderr, /Installation-scoped GitHub tokens require/); + const fixed = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456' }, + }); + assert.match(fixed.stderr, /without a fixed installation ID/); +}); + test('CLI requires a runtime image for Docker supervision', () => { const result = spawnSync( process.execPath, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 841164fb..6e3a0062 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -157,6 +157,7 @@ function githubCredentials(args: string[]): { mode?: 'app' | 'token'; repositoryRouting?: boolean; checkoutRouting?: boolean; + installationTokenScope?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -191,6 +192,18 @@ function githubCredentials(args: string[]): { 'Checkout GitHub repository routing requires a GitHub App without a fixed installation ID', ); } + const tokenScope = + option(args, '--github-token-scope')?.trim().toLowerCase() ?? + process.env.LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE?.trim().toLowerCase() ?? + 'repository'; + if (tokenScope !== 'repository' && tokenScope !== 'installation') { + throw new Error('GitHub token scope must be repository or installation'); + } + if (tokenScope === 'installation' && (!hasApp || installationId)) { + throw new Error( + 'Installation-scoped GitHub tokens require a GitHub App without a fixed installation ID', + ); + } const configuredHostValue = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_HOST, ); @@ -225,16 +238,19 @@ function githubCredentials(args: string[]): { mode: 'app', repositoryRouting: !installationId, checkoutRouting: routing === 'checkout', + installationTokenScope: tokenScope === 'installation', policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, appId, installationId, - }) + (routing === 'checkout' ? ':routing:checkout' : ''), + }) + (routing === 'checkout' ? ':routing:checkout' : '') + + (tokenScope === 'installation' ? ':scope:installation' : ''), privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, installationId, + tokenScope, privateKeyPath: privateKeyPath!, host, apiUrl, @@ -568,6 +584,11 @@ async function run( 'Checkout GitHub repository routing requires the trusted-vm command policy', ); } + if (github.installationTokenScope && commandPolicy.preset !== 'trusted-vm') { + throw new Error( + 'Installation-scoped GitHub tokens require the trusted-vm command policy', + ); + } const githubDomains = github.provider ? github.host === 'github.com' ? [...GITHUB_ALLOWED_DOMAINS] diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index b13d70f7..5d401515 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -239,6 +239,272 @@ test('routes and scopes GitHub App tokens per repository installation', async (t ); }); +test('installation scope shares one token across repositories in an organization and refreshes grants', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-installation-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let now = new Date('2030-01-01T00:00:00Z'); + const minted: Array<{ installation: string; body?: string }> = []; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async (input, init) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (/\/repos\/LibreChat-AI\/(agents|LibreChat)\/installation$/.test(url)) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/repos/ClickHouse/Agents/installation')) { + return Response.json({ id: 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + minted.push({ + installation, + body: typeof init?.body === 'string' ? init.body : undefined, + }); + return Response.json({ + token: `ghs_${installation}_${minted.length}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const [agents, librechat] = await Promise.all([ + provider.getCredential(undefined, 'LibreChat-AI/agents'), + provider.getCredential(undefined, 'LibreChat-AI/LibreChat'), + ]); + assert.equal(agents.value, librechat.value); + assert.deepEqual(minted, [{ installation: '111', body: undefined }]); + const clickhouse = await provider.getCredential(undefined, 'ClickHouse/Agents'); + assert.notEqual(clickhouse.value, librechat.value); + assert.deepEqual(minted.map(entry => entry.installation), ['111', '222']); + now = new Date('2030-01-01T00:02:01Z'); + const refreshed = await provider.getCredential(undefined, 'LibreChat-AI/LibreChat'); + assert.notEqual(refreshed.value, librechat.value); + assert.deepEqual(minted.map(entry => entry.installation), ['111', '222', '111']); +}); + +test('installation scope shares a bounded lookup when a waiter cancels', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-shared-lookup-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let releaseLookup!: (response: Response) => void; + const lookupResponse = new Promise(resolve => { releaseLookup = resolve; }); + let lookupStarted!: () => void; + const started = new Promise(resolve => { lookupStarted = resolve; }); + const cancelled = new AbortController(); + let lookups = 0; + let mints = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input, init) => { + const url = String(input); + if (url.endsWith('/repos/acme/project/installation')) { + lookups += 1; + assert.ok(init?.signal); + assert.notEqual(init.signal, cancelled.signal); + lookupStarted(); + return lookupResponse; + } + if (url.endsWith('/app/installations/111/access_tokens')) { + mints += 1; + return Response.json({ + token: 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const first = provider.getCredential(cancelled.signal, 'acme/project'); + const others = Array.from({ length: 12 }, () => + provider.getCredential(undefined, 'acme/project')); + await started; + cancelled.abort(new Error('first command cancelled')); + await assert.rejects(first, /first command cancelled/); + releaseLookup(Response.json({ id: 111 })); + const credentials = await Promise.all(others); + assert.equal(lookups, 1); + assert.equal(mints, 1); + assert.equal(new Set(credentials.map(credential => credential.value)).size, 1); +}); + +test('installation scope times out a stalled lookup without a caller signal', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-lookup-timeout-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + const timeout = new AbortController(); + t.mock.method(AbortSignal, 'timeout', (ms: number) => { + assert.equal(ms, 30_000); + return timeout.signal; + }); + let lookupStarted!: () => void; + const started = new Promise(resolve => { lookupStarted = resolve; }); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + fetch: (async (input, init) => { + if (String(input).endsWith('/repos/acme/project/installation')) { + assert.equal(init?.signal, timeout.signal); + lookupStarted(); + return new Promise((_resolve, reject) => { + timeout.signal.addEventListener('abort', () => reject(timeout.signal.reason), { once: true }); + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const pending = provider.getCredential(undefined, 'acme/project'); + await started; + timeout.abort(new DOMException('Installation lookup timed out', 'TimeoutError')); + await assert.rejects(pending, { name: 'TimeoutError' }); +}); + +test('installation replacement checks every shared waiter’s repository grant', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-replaced-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let now = new Date('2030-01-01T00:00:00Z'); + let replaced = false; + let bLookups = 0; + let oldMints = 0; + let releaseOldMint!: () => void; + const oldMintResponse = new Promise(resolve => { releaseOldMint = resolve; }); + let oldMintStarted!: () => void; + const oldMintPending = new Promise(resolve => { oldMintStarted = resolve; }); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async input => { + const url = String(input); + if (url.endsWith('/repos/acme/a/installation')) { + return Response.json({ id: replaced ? 222 : 111 }); + } + if (url.endsWith('/repos/acme/b/installation')) { + bLookups += 1; + return replaced ? Response.json({}, { status: 404 }) : Response.json({ id: 111 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + oldMints += 1; + if (replaced) { + oldMintStarted(); + await oldMintResponse; + return Response.json({}, { status: 404 }); + } + return Response.json({ + token: 'ghs_old_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T00:06:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app/installations/222/access_tokens')) { + return Response.json({ + token: 'ghs_new_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const old = await provider.getCredential(undefined, 'acme/a'); + assert.equal((await provider.getCredential(undefined, 'acme/b')).value, old.value); + replaced = true; + now = new Date('2030-01-01T00:01:01Z'); // Token needs refresh; both mappings are still live. + const forA = provider.getCredential(undefined, 'acme/a'); + await oldMintPending; + const forB = provider.getCredential(undefined, 'acme/b'); + await new Promise(resolve => setImmediate(resolve)); // Join A's stale installation mint. + releaseOldMint(); + const [a, b] = await Promise.allSettled([forA, forB]); + assert.equal(a.status, 'fulfilled'); + if (a.status === 'fulfilled') assert.equal(a.value.value, 'ghs_new_abcdefghijklmnopqrstuvwxyz'); + assert.equal(b.status, 'rejected'); + if (b.status === 'rejected') assert.match(String(b.reason), /not installed for acme\/b/); + assert.equal(oldMints, 2); // One initial mint and just one shared failed refresh. + assert.equal(bLookups, 2); // B must revalidate instead of receiving A's new token. +}); + +test('installation scope follows a repository transfer after the lookup expires', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-transfer-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let now = new Date('2030-01-01T00:00:00Z'); + let lookupCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + lookupCount += 1; + return Response.json({ id: lookupCount === 1 ? 111 : 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + return Response.json({ + token: `ghs_${installation}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + assert.equal((await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_111_abcdefghijklmnopqrstuvwxyz'); + now = new Date('2030-01-01T00:02:01Z'); + assert.equal((await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_222_abcdefghijklmnopqrstuvwxyz'); + assert.equal(lookupCount, 2); +}); + test('keeps a shared token refresh alive when one waiter is cancelled', async (t) => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-cancel-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 19b8e639..8519938f 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -41,6 +41,8 @@ export interface GitHubAppCredentialProviderOptions { appId: string; /** Legacy fixed installation. Omit to resolve the installation per repository. */ installationId?: string; + /** Opt in to all repositories granted to the resolved installation. */ + tokenScope?: 'repository' | 'installation'; privateKeyPath: string; apiUrl?: string; /** Git HTTPS hostname; non-public hosts default to the GHES /api/v3 base. */ @@ -52,6 +54,13 @@ export interface GitHubAppCredentialProviderOptions { const execFileAsync = promisify(execFile); const GITHUB_SHARED_REQUEST_TIMEOUT_MS = 30_000; +const GITHUB_INSTALLATION_CACHE_MS = 2 * 60_000; + +class InstallationChangedError extends Error { + constructor(readonly installationId: string) { + super('GitHub App installation changed while issuing a token'); + } +} async function waitForShared( promise: Promise, @@ -248,8 +257,14 @@ function createAppJwt(appId: string, privateKey: string, now: Date): string { export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private readonly cached = new Map(); + private readonly cachedAt = new Map(); private readonly inFlight = new Map>(); private readonly installationIds = new Map(); + private readonly installationIdCachedAt = new Map(); + private readonly installationIdInFlight = new Map< + string, + Promise<{ id: string; jwt: string }> + >(); private appLogin?: string; private appLoginInFlight?: Promise; private actor?: GitHubCredential['actor']; @@ -423,32 +438,80 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private async resolveInstallationId( repository: string, - jwt: string, signal?: AbortSignal, - ): Promise { - if (this.options.installationId) return this.options.installationId; + jwt?: string, + ): Promise<{ id: string; jwt?: string }> { + if (this.options.installationId) { + return { id: this.options.installationId, jwt }; + } const cached = this.installationIds.get(repository); - if (cached) return cached; - const { owner, name } = repositoryName(repository); - const response = await this.request( - `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, - jwt, - signal, - ); - if (!response.ok) { - throw new Error( - response.status === 404 - ? `GitHub App is not installed for ${repository}` - : `GitHub App installation lookup failed with status ${response.status}`, + const now = (this.options.now ?? (() => new Date()))().getTime(); + if ( + cached && + now - (this.installationIdCachedAt.get(repository) ?? 0) < GITHUB_INSTALLATION_CACHE_MS + ) return { id: cached, jwt }; + const existing = this.installationIdInFlight.get(repository); + if (existing) return waitForShared(existing, signal); + const pending = (async () => { + const sharedSignal = AbortSignal.timeout(GITHUB_SHARED_REQUEST_TIMEOUT_MS); + const lookupJwt = jwt ?? await this.appJwt( + (this.options.now ?? (() => new Date()))(), ); + const { owner, name } = repositoryName(repository); + const response = await this.request( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, + lookupJwt, + sharedSignal, + ); + if (!response.ok) { + throw new Error( + response.status === 404 + ? `GitHub App is not installed for ${repository}` + : `GitHub App installation lookup failed with status ${response.status}`, + ); + } + const body = (await response.json()) as { id?: unknown }; + if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { + throw new Error('GitHub App installation response is invalid'); + } + const id = String(body.id); + this.installationIds.set(repository, id); + this.installationIdCachedAt.set( + repository, + (this.options.now ?? (() => new Date()))().getTime(), + ); + return { id, jwt: lookupJwt }; + })(); + this.installationIdInFlight.set(repository, pending); + const clearPending = () => { + if (this.installationIdInFlight.get(repository) === pending) { + this.installationIdInFlight.delete(repository); + } + }; + void pending.then(clearPending, clearPending); + return waitForShared(pending, signal); + } + + private async waitForCredential( + pending: Promise, + signal: AbortSignal | undefined, + repository: string | undefined, + retried: boolean, + ): Promise { + try { + return await waitForShared(pending, signal); + } catch (error) { + if (!(error instanceof InstallationChangedError) || retried || !repository) { + throw error; + } + // A shared mint can serve several repositories. Never pass its replacement + // installation's token to a repository whose grant has not been checked. + if (this.installationIds.get(repository) === error.installationId) { + this.installationIds.delete(repository); + this.installationIdCachedAt.delete(repository); + } + return this.getCredentialAttempt(signal, repository, true); } - const body = (await response.json()) as { id?: unknown }; - if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { - throw new Error('GitHub App installation response is invalid'); - } - const installationId = String(body.id); - this.installationIds.set(repository, installationId); - return installationId; } async getCredential( @@ -462,32 +525,46 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ); } if (repository) repositoryName(repository); + return this.getCredentialAttempt(signal, repository, false); + } + + private async getCredentialAttempt( + signal: AbortSignal | undefined, + repository: string | undefined, + retried: boolean, + ): Promise { + signal?.throwIfAborted(); + const installationScope = this.options.tokenScope === 'installation'; + const installation = installationScope + ? await this.resolveInstallationId(repository ?? '', signal) + : undefined; const now = (this.options.now ?? (() => new Date()))(); - const key = this.options.installationId ?? repository!; + const key = installation?.id ?? this.options.installationId ?? repository!; const cached = this.cached.get(key); if ( cached?.expiresAt != null && - cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 + cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 && + (!installationScope || now.getTime() - (this.cachedAt.get(key) ?? 0) < GITHUB_INSTALLATION_CACHE_MS) ) { return cached; } const existing = this.inFlight.get(key); - if (existing) return waitForShared(existing, signal); + if (existing) return this.waitForCredential(existing, signal, repository, retried); const pending = (async () => { const sharedSignal = AbortSignal.timeout( GITHUB_SHARED_REQUEST_TIMEOUT_MS, ); - const jwt = await this.appJwt(now); - const scopedRepository = repository + const jwt = installation?.jwt ?? await this.appJwt(now); + const scopedRepository = !installationScope && repository ? repositoryName(repository).name : undefined; - const installationId = await this.resolveInstallationId( + const resolvedInstallationId = installation?.id ?? (await this.resolveInstallationId( repository ?? '', - jwt, sharedSignal, - ); + jwt, + )).id; let response = await this.request( - `/app/installations/${installationId}/access_tokens`, + `/app/installations/${resolvedInstallationId}/access_tokens`, jwt, sharedSignal, { @@ -495,18 +572,20 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { headers: { 'Content-Type': 'application/json', }, - ...(this.options.installationId + ...(this.options.installationId || installationScope ? {} : { body: JSON.stringify({ repositories: [scopedRepository] }) }), }, ); if (!this.options.installationId && response.status === 404) { + if (installationScope) throw new InstallationChangedError(resolvedInstallationId); this.installationIds.delete(repository!); - const refreshedInstallationId = await this.resolveInstallationId( + this.installationIdCachedAt.delete(repository!); + const refreshedInstallationId = (await this.resolveInstallationId( repository!, - jwt, sharedSignal, - ); + jwt, + )).id; response = await this.request( `/app/installations/${refreshedInstallationId}/access_tokens`, jwt, @@ -516,7 +595,9 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ repositories: [scopedRepository] }), + ...(installationScope + ? {} + : { body: JSON.stringify({ repositories: [scopedRepository] }) }), }, ); } @@ -554,6 +635,10 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { actor, }; this.cached.set(key, credential); + this.cachedAt.set( + key, + (this.options.now ?? (() => new Date()))().getTime(), + ); return credential; })(); this.inFlight.set(key, pending); @@ -561,7 +646,7 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { if (this.inFlight.get(key) === pending) this.inFlight.delete(key); }; void pending.then(clearPending, clearPending); - return waitForShared(pending, signal); + return this.waitForCredential(pending, signal, repository, retried); } } From 67329888d66060b7e21416f539f6f0ddf56fff28 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 15:19:05 -0400 Subject: [PATCH 45/75] =?UTF-8?q?=E2=8F=B3=20fix:=20Let=20BYOM=20Workspace?= =?UTF-8?q?=20Calls=20Queue=20Past=20Thirty=20Seconds=20(#264)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ⏳ fix: Let BYOM Workspace Calls Queue Past Thirty Seconds * fix: Recompute BYOM Reservation TTLs After Admission --------- Co-authored-by: Lia --- docs/byom-worker-admission.md | 48 +++++++++------ docs/remote-bridge/README.md | 6 ++ packages/code/README.md | 10 +++- service/src/bridge/concurrent-store.test.ts | 45 ++++++++++++++ service/src/bridge/store.ts | 14 +++-- service/src/bridge/worker-admission.test.ts | 66 +++++++++++++++++++++ service/src/workspace-tools/router.test.ts | 41 +++++++++---- service/src/workspace-tools/router.ts | 28 +++++---- 8 files changed, 212 insertions(+), 46 deletions(-) diff --git a/docs/byom-worker-admission.md b/docs/byom-worker-admission.md index 2c2b354b..2fcd9f9f 100644 --- a/docs/byom-worker-admission.md +++ b/docs/byom-worker-admission.md @@ -5,14 +5,17 @@ The limit is 32 admitted requests per worker, including the active request. When the limit is reached, the workspace endpoint returns HTTP 429 with `WORKER_QUEUE_FULL`. A different worker has an independent admission queue. -The workspace HTTP endpoint allows at most 30 seconds for admission. After -admission and worker validation, a separate execution deadline starts. Commands -receive their requested timeout (30 seconds by default, up to five minutes), -capped by the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. -Read/search/list operations receive up to 30 seconds, also capped by `JOB_TIMEOUT`. -Disconnecting or cancelling removes the waiting -request without cancelling the active assignment. Expired entries are pruned; -Redis key expiry also bounds state left by a crashed API process. +The Code API workspace HTTP endpoint allows up to the smaller of `JOB_TIMEOUT` +and five minutes for admission while the caller stays connected. After admission +and worker validation, a separate execution deadline starts. Commands receive +their requested timeout (30 seconds by default, up to five minutes), capped by +the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. Other +operations receive up to 30 seconds, also capped by `JOB_TIMEOUT`. +Disconnecting or cancelling removes a waiting request without cancelling the +active assignment. Expired entries are pruned; Redis key expiry also bounds +state left by a crashed API process. Reservations derive their TTL at acquisition +from the remaining absolute deadline or a fresh execution budget; enqueued +assignment records use the final execution deadline, not the elapsed queue budget. After admission, the API revalidates the worker incarnation, identity, tenant binding and workspace operation. A waiting request cannot migrate to a replacement @@ -26,13 +29,24 @@ Existing workers still execute one assignment at a time. Parallel execution acro workspaces requires separate lease claims and isolated native sandbox contexts; this admission change does not advertise that capability. -LibreChat must allow queue time plus execution/settlement time and five seconds -for HTTP delivery: 65 seconds for reads, 70 seconds for default commands, and -340 seconds for five-minute commands. Either side can be upgraded first. Older -clients still cancel at their earlier deadline; newer clients preserve errors from -older servers without retrying mutations. Both updates are needed for the full -waiting budget. Any reverse proxy request timeout must accommodate these totals. -The worker package does not need an update for the deadline change. +Clients and reverse proxies must allow queue time plus execution/settlement time +and five seconds for HTTP delivery. With the default five-minute `JOB_TIMEOUT`, +that is at least 335 seconds for non-command tools, 340 seconds for default +commands, and 610 seconds for five-minute commands. With a smaller `JOB_TIMEOUT`, +use `min(JOB_TIMEOUT, 300s)` for the queue, plus `min(JOB_TIMEOUT, 30s)` for other +operations or `min(JOB_TIMEOUT, requested command timeout) + 5s` for commands, +plus five seconds for delivery. -Focused regression coverage lives in `service/src/bridge/admission.test.ts` and -`service/src/bridge/worker-admission.test.ts`. +At the time of this change, LibreChat's `getWorkspaceToolTimeoutMs` still budgets +only 30 seconds for a single admission attempt (65/70/340 seconds in total). +Its `maxQueueWaitMs` is a retry horizon after a typed capacity rejection, **not** +a per-attempt HTTP timeout. Updating Code API alone therefore does not guarantee +the full wait. An earlier client, tool, or proxy timeout disconnects the request; +if work was already admitted, a mutation may have run and must not be blindly +retried. Match LibreChat's per-attempt timeout and each intermediary to the new +budget before relying on it. Existing workers do not need an update. + +Focused regression coverage lives in `service/src/bridge/admission.test.ts`, +`service/src/bridge/worker-admission.test.ts`, +`service/src/bridge/concurrent-store.test.ts`, and +`service/src/workspace-tools/router.test.ts`. diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index efe84b84..c5bc573c 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -239,6 +239,12 @@ execution. worker. The lower API or worker slot ceiling wins, and assignments sharing the same workspace isolation key remain serialized while independent conversation worktrees may run concurrently. +- Workspace tool admission waits for capacity up to the smaller of `JOB_TIMEOUT` + and five minutes while the HTTP caller remains connected. Disconnects cancel + waiting, and admitted work receives a separate execution budget. A shorter + client or proxy timeout can end the wait sooner; Code API does not receive an + absolute caller deadline. See [BYOM worker admission](../byom-worker-admission.md) + for the caller and proxy timeout requirements. - Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are diff --git a/packages/code/README.md b/packages/code/README.md index a87c87df..87a6fe61 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -791,11 +791,17 @@ Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. -Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP -503, `Retry-After: 1`) means the operation was not assigned or started; wait for +On an updated Code API, admission waits up to the smaller of `JOB_TIMEOUT` and +five minutes while the HTTP caller remains connected; older Code API versions +waited at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, +`Retry-After: 1`) means the operation was not assigned or started; wait for capacity before submitting it again. This is distinct from `ASSIGNMENT_EXPIRED` or a transport timeout after dispatch, where execution may have occurred and mutations must not be blindly retried. No automatic retry is added by this policy. +Align the client's per-attempt timeout and any proxy with the queue **plus** +execution budget before relying on the longer wait. See the +[BYOM worker admission guide](../../docs/byom-worker-admission.md) for the +current client limitation and the timeout calculations. Keep the existing URL, pairing/identity, and network policy configuration. The primary root keeps its configured workspace ID (default `primary`). Repeat diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index 33f3d082..6bdfde88 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -357,6 +357,51 @@ test('same-root work waits while another root progresses', async () => { await Promise.all([nextA, b]); }); +test('a long same-root queue allowance does not extend slot or assignment TTLs', async () => { + await register(); + const active = dispatch('a'); + const activeAssignment = await store.lease( + workerId, incarnationId, 1000, undefined, undefined, 0, + ); + const controller = new AbortController(); + const queued = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + 300_000, + executionTimeoutMs: 305_000, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'a', path: 'second.txt' }, + }); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 150)); + await settle(activeAssignment!); + await active; + + const assignment = await store.lease( + workerId, incarnationId, 1000, undefined, undefined, 0, + ); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'second.txt' }); + try { + const expiresAtMs = Date.parse(assignment.expiresAt); + const slotExpiresAtMs = Number(await redis.hget( + `codeapi:bridge:v1:worker:${workerId}:workspace-slots`, 'e:0', + )); + expect(slotExpiresAtMs).toBeGreaterThan(expiresAtMs + 28_000); + expect(slotExpiresAtMs).toBeLessThan(expiresAtMs + 32_000); + const assignmentTtlMs = await redis.pttl(`codeapi:bridge:v1:assignment:${assignment.assignmentId}`); + const remainingMs = expiresAtMs - Date.now(); + expect(assignmentTtlMs).toBeGreaterThan(remainingMs + 28_000); + expect(assignmentTtlMs).toBeLessThan(remainingMs + 32_000); + } finally { + await settle(assignment); + await queued; + } +}); + test('conversation worktrees on one source use independent capacity lanes', async () => { await register(); const firstId = 'a'.repeat(64); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 2ffeec66..3d9d7e1c 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -863,10 +863,6 @@ export class RedisBridgeStore { const assignmentId = randomBytes(18).toString('base64url'); const leaseToken = randomBytes(32).toString('base64url'); - // The lock is acquired before admission finishes; it must outlive the later execution deadline. - const ttlSeconds = assignmentTtlSeconds( - args.deadlineAtMs + (args.executionTimeoutMs ?? 0), - ); const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; let enqueueAttempted = false; @@ -931,6 +927,14 @@ export class RedisBridgeStore { ); continue; } + // A queued request may have waited nearly its full admission budget. + // Reserve for the *remaining* absolute deadline or a fresh execution + // budget, not the original queue window plus execution again. + const ttlSeconds = assignmentTtlSeconds( + args.executionTimeoutMs === undefined + ? args.deadlineAtMs + : Date.now() + args.executionTimeoutMs, + ); if (workspaceSlots != null) { workspaceLeaseSlot = await this.dispatchCommand( () => @@ -1070,7 +1074,7 @@ export class RedisBridgeStore { enqueueAttempted = true; return this.enqueueForActiveIncarnation( assignment!, - ttlSeconds, + assignmentTtlSeconds(Date.parse(assignment!.expiresAt)), readyToken, ); }, diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index 655748b1..98363c34 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -161,6 +161,72 @@ test('execution receives a fresh budget after waiting and the lock covers long c await second; }); +test('a long queue allowance does not extend serial lock or assignment TTLs after admission', async () => { + await register(); + const active = dispatch('first'); + const activeAssignment = await store.lease(workerId, incarnationId, 1000); + const controller = new AbortController(); + const queued = dispatch('second', controller, 300_000, 305_000); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 150)); + await settle(activeAssignment); + await active; + + const assignment = await store.lease(workerId, incarnationId, 1000); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'second' }); + try { + const expiresAtMs = Date.parse(assignment.expiresAt); + for (const key of [ + `codeapi:bridge:v1:worker:${workerId}:lock`, + `codeapi:bridge:v1:worker:${workerId}:lock:incarnation`, + `codeapi:bridge:v1:assignment:${assignment.assignmentId}`, + ]) { + const ttlMs = await redis.pttl(key); + const remainingMs = expiresAtMs - Date.now(); + expect(ttlMs).toBeGreaterThan(remainingMs + 28_000); + expect(ttlMs).toBeLessThan(remainingMs + 32_000); + } + } finally { + await settle(assignment); + await queued; + } +}); + +test('absolute-deadline callers keep only their remaining deadline in the serial lock TTL', async () => { + await register(); + const active = dispatch('first'); + const activeAssignment = await store.lease(workerId, incarnationId, 1000); + const controller = new AbortController(); + const queued = dispatch('absolute', controller, 2_500); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 1450)); + await settle(activeAssignment); + await active; + + const assignment = await store.lease(workerId, incarnationId, 1000); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'absolute' }); + try { + const remainingMs = Date.parse(assignment.expiresAt) - Date.now(); + expect(remainingMs).toBeGreaterThan(0); + const ttlMs = await redis.pttl(`codeapi:bridge:v1:worker:${workerId}:lock`); + expect(ttlMs).toBeGreaterThan(remainingMs + 28_000); + expect(ttlMs).toBeLessThan(remainingMs + 31_000); + } finally { + await settle(assignment); + await queued; + } +}); + test('execution expires independently of an unused queue allowance', async () => { await register(); const completion = dispatch('short', new AbortController(), 5000, 150); diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index a52b3f0b..18d24a76 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -67,13 +67,16 @@ test('binds instance admission to the authenticated tenant and user while preser expect(dispatched[1]?.workspaceInstanceId).toBeUndefined(); }); -test.each<[WorkspaceToolRequest, number, number?]>([ - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000], -])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling) => { +test.each<[WorkspaceToolRequest, number, number?, number?]>([ + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 90_000 }, 95_000, 125_000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, 5000], +])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling, queueTimeoutMs) => { const app = express(); app.use(json()); app.use((req, _res, next) => { @@ -86,6 +89,7 @@ test.each<[WorkspaceToolRequest, number, number?]>([ app.use(createWorkspaceToolsRouter({ backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, timeoutMs: ceiling, + queueTimeoutMs, store: { async dispatchWorkspaceTool(args) { executionBudget = args.executionTimeoutMs; if (args.request.operation === 'execute_command') commandTimeout = args.request.timeoutMs; @@ -103,8 +107,19 @@ test.each<[WorkspaceToolRequest, number, number?]>([ await response.json(); expect(executionBudget).toBe(expectedExecution); if (request.operation === 'execute_command') expect(commandTimeout).toBe(expectedExecution - 5000); - expect(queueRemaining).toBeGreaterThan(29_000); - expect(queueRemaining).toBeLessThanOrEqual(30_000); + const expectedQueueBudget = queueTimeoutMs ?? Math.min(ceiling ?? 30_000, 300_000); + expect(queueRemaining).toBeGreaterThan(expectedQueueBudget - 1000); + expect(queueRemaining).toBeLessThanOrEqual(expectedQueueBudget); +}); + +test.each([0, 300_001, Number.POSITIVE_INFINITY])('rejects an unbounded queue override (%s)', (queueTimeoutMs) => { + expect(() => createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'user-worker', + dynamicWorkers: false, + queueTimeoutMs, + store: { async dispatchWorkspaceTool() { throw new Error('must not dispatch'); } }, + })).toThrow('Workspace queue timeout must be between 1 and 300000 milliseconds'); }); test('rejects new workspace dispatches while the service is shutting down', async () => { @@ -404,7 +419,7 @@ test.each([ status: expectedStatus, errorCode, outcome: 'completed', - deadlineBudgetMs: 60_000, + deadlineBudgetMs: 330_000, dispatchDurationMs: expect.any(Number), }), ); @@ -486,6 +501,7 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => const closed = Promise.withResolvers(); const settlementGate = Promise.withResolvers(); let dispatchAborted = false; + let queueRemaining: number | undefined; let closeConnection = (): void => { throw new Error('connection not ready'); }; app.use(json()); app.use((req, res, next) => { @@ -499,8 +515,10 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: true, + timeoutMs: 125_000, store: { - async dispatchWorkspaceTool({ signal }) { + async dispatchWorkspaceTool({ deadlineAtMs, signal }) { + queueRemaining = deadlineAtMs - Date.now(); started.resolve(); return await new Promise((_resolve, reject) => { signal.addEventListener( @@ -531,6 +549,7 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => closeConnection(); await expect(response).rejects.toThrow(); await closed.promise; + expect(queueRemaining).toBeGreaterThan(120_000); expect(dispatchAborted).toBe(true); expect(logSpy).not.toHaveBeenCalled(); settlementGate.resolve(); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 3e1f9fee..51e47f26 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -20,6 +20,8 @@ import { } from '../bridge/selection'; import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; +const MAX_WORKSPACE_QUEUE_WAIT_MS = 5 * 60_000; + interface WorkspaceToolsRouterOptions { store: Pick; backend: 'http' | 'lambda-microvm' | 'remote-bridge'; @@ -50,15 +52,19 @@ export function bridgeStoreStatus(error: BridgeStoreError): number { } export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions): Router { - const queueBudgetMs = options.queueTimeoutMs ?? 30_000; - if (!Number.isSafeInteger(queueBudgetMs) || queueBudgetMs < 1 || queueBudgetMs > 30_000) { - throw new RangeError('Workspace queue timeout must be between 1 and 30000 milliseconds'); - } if (options.timeoutMs !== undefined && ( !Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1 )) { throw new RangeError('Workspace execution timeout must be a positive safe integer'); } + // The HTTP disconnect cancels waiting; this bounds admission while the caller remains connected. + const queueBudgetMs = options.queueTimeoutMs ?? Math.min( + options.timeoutMs ?? 30_000, + MAX_WORKSPACE_QUEUE_WAIT_MS, + ); + if (!Number.isSafeInteger(queueBudgetMs) || queueBudgetMs < 1 || queueBudgetMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + throw new RangeError('Workspace queue timeout must be between 1 and 300000 milliseconds'); + } const router = Router(); router.post( @@ -86,13 +92,13 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null ? req.body : { - ...req.body, - workspaceInstanceId: principalWorkspaceInstanceId({ - instanceId: req.body.workspaceInstanceId, - tenantId: principal.tenantId, - principalId: principal.userId, - }), - }; + ...req.body, + workspaceInstanceId: principalWorkspaceInstanceId({ + instanceId: req.body.workspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }), + }; const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' ? { ...principalRequest, timeoutMs: Math.min( principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, From 2a310027800a6a4936ae277ddf6cb0179d2b74b2 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 07:54:34 -0400 Subject: [PATCH 46/75] =?UTF-8?q?=F0=9F=94=90=20feat:=20Recover=20Code=20A?= =?UTF-8?q?PI=20Machine=20Credentials=20After=20Outages=20(#260)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔐 feat: Recover Code API Machine Credentials After Outages * 🔐 fix: Protect Recovery Budgets and Persist Compose Enrollment * 🛡️ fix: Limit Untrusted Code API Recovery Traffic --------- Co-authored-by: Lia --- docker-compose.yaml | 17 +- docs/adr/001-stateful-code-environments.md | 7 +- docs/remote-bridge/README.md | 71 ++- docs/remote-bridge/worker-runbook.md | 15 +- packages/code/package.json | 4 + packages/code/src/identity.test.ts | 43 ++ packages/code/src/identity.ts | 94 ++++ packages/code/src/protocol.ts | 26 ++ service/src/bridge/index.ts | 18 +- service/src/bridge/pairing.ts | 480 ++++++++++++++++++- service/src/bridge/recovery-router.test.ts | 213 +++++++++ service/src/bridge/recovery.test.ts | 518 +++++++++++++++++++++ service/src/bridge/router.ts | 109 ++++- service/src/config.ts | 16 + tests/compose-bridge-config.cjs | 24 +- 15 files changed, 1634 insertions(+), 21 deletions(-) create mode 100644 service/src/bridge/recovery-router.test.ts create mode 100644 service/src/bridge/recovery.test.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index ab37fd24..1d7d28cf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,6 +15,12 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} + - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} + - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} + - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} + - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} + - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} + - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_ALLOW_AUTH_PROVIDER_NONE=${CODEAPI_ALLOW_AUTH_PROVIDER_NONE:-} - CODEAPI_JWT_ISSUER=${CODEAPI_JWT_ISSUER:-} @@ -64,6 +70,12 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} + - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} + - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} + - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} + - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} + - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} + - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_JWT_SINGLE_TENANT_ID=${CODEAPI_JWT_SINGLE_TENANT_ID:-} - CODEAPI_TENANT_ISOLATION_STRICT=${CODEAPI_TENANT_ISOLATION_STRICT:-} @@ -213,9 +225,11 @@ services: redis: image: redis:7-alpine container_name: redis - command: redis-server --requirepass localdev + command: redis-server --requirepass localdev --appendonly yes ports: - ${CODEAPI_REDIS_PORT:-16379}:6379 + volumes: + - redis_data:/data minio: image: quay.io/minio/minio @@ -232,3 +246,4 @@ services: volumes: minio_data: + redis_data: diff --git a/docs/adr/001-stateful-code-environments.md b/docs/adr/001-stateful-code-environments.md index cfa44bf4..7992ded8 100644 --- a/docs/adr/001-stateful-code-environments.md +++ b/docs/adr/001-stateful-code-environments.md @@ -52,8 +52,11 @@ worker replacement; the UI and operator documentation must not imply otherwise. - The VM requires no inbound internet listener. - Code API, not the worker, authenticates LibreChat users and normalizes work. - A stolen short-lived credential is insufficient without the worker private - key; a stolen private key is insufficient after credential expiry or - revocation. + key. In the original pairing-only model, a stolen private key is insufficient + after credential expiry or revocation. With optional durable machine + enrollment and signed credential recovery, the private key itself remains + a revocable long-lived credential: access-credential expiry alone does not + protect against theft of that key. Revocation invalidates both. - Pairing codes and credentials are stored by digest where lookup permits. - One configured worker has at most one active fenced assignment. - Sandbox isolation and default-deny egress remain the mandatory default; diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index c5bc573c..96241ad7 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -31,6 +31,42 @@ CODEAPI_BRIDGE_TOKEN= CODEAPI_BRIDGE_AUTH_MODE=paired ``` +To opt in to durable machine authorization on every Code API replica, set a +single stable public **Code API** origin (not the LibreChat URL): + +```dotenv +CODEAPI_BRIDGE_RECOVERY_SERVER_ID=https://code.example.com +# 0 (default): enrolled machine keys remain authorized until revoked. +# CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=0 +# CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=60 +# CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=12 +# CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=30 +# CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=240 +``` + +Omitting the server ID retains the existing pairing and refresh behavior and +hides the recovery routes. Deploy the compatible Code API version to **all** +replicas before setting this value and enrolling workers again. Older Code API +replicas can still pair or refresh a worker but do not write durable enrollment; +they must not serve device login or recovery requests. Only a pairing redeemed +after this option is enabled has a recoverable key. Updating Code API alone +does not make old workers reconnect automatically: the CLI must also implement +this recovery protocol in the later worker release. + +Store Redis state durably across restarts. The primary `docker-compose.yaml` +now uses Redis AOF and a named `/data` volume; preserve that volume when +recreating the stack. If upgrading a running stack with an in-memory Redis, +migrate its state before recreating the container: mounting an empty volume +does **not** preserve active assignments, fences, or earlier revocations. Other +deployments must provide equivalent durable Redis (for example, a managed +persistent Redis service and backups). Revocation and +machine enrollment share that state across replicas; do not configure eviction +of authorization keys. If enrollment state is missing, credentials minted under +that enrollment fail closed, and the worker must be explicitly enrolled again. +Restoring a backup from *before* a revocation can revive trust; reconcile +revocations after recovery from backup. Use a distinct server ID for each Code +API deployment and keep it stable when the endpoint changes behind a proxy. + Use `strict` instead of `affinity` if every request must include a runtime session hint. In hardened mode, startup requires the bridge token to be at least 32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a @@ -226,7 +262,40 @@ execution. atomically on their first redemption attempt. - Worker credentials expire after fifteen minutes and are bound to an Ed25519 public key. Exact-request signatures include the HTTP method, path, body - digest, timestamp, nonce, and credential. + digest, timestamp, nonce, and credential. With recovery enabled, redeeming a + pairing also persists a separate machine authorization and its public key in + Redis without a TTL by default; an operator can instead set a bounded + enrollment lifetime. +- `POST /v1/bridge/workers/:workerId/credentials/challenge` does not require + an administrator token or an existing access credential, but **does** require + the enrolled key. Its JSON body contains `protocolVersion: 1`, + `operation: "credential.challenge"`, the configured `serverId`, the matching + `workerId`, a fresh UTC ISO `timestamp`, a random 32-byte base64url `nonce`, + and `signature` computed with `signBridgeRecoveryStart(privateKey, fields)` + from `@librechat/code/identity`. Code API verifies the signed fields and + consumes the nonce once before charging the machine's shared challenge + budget; a fabricated request cannot exhaust another worker's budget. +- The response is a short-lived, single-use challenge with the server ID, + worker ID, enrollment generation, operation and expiry. Sign those fields + with `signBridgeRecovery(privateKey, challenge)` and send the fields plus + `signature` to `POST .../credentials/recover` to obtain a new short-lived + credential. Invalid proofs are limited per high-entropy challenge; only + successfully verified, unused proofs consume the machine's shared recovery + budget. Separately, both recovery endpoints limit all incoming requests per + connection peer *before* key verification, including well-formed JSON with + malformed or forged proofs; forged headers and worker IDs cannot bypass + that limit or consume the signed machine budget. All limits live in shared + Redis; HTTP 429 means back off. When a reverse proxy connects to Code API, + its clients share that peer's limit. Restrict direct backend access and apply + client-IP and global + abuse limits at the trusted ingress to keep one proxy peer from becoming a + shared bottleneck; do not trust an arbitrary `X-Forwarded-For` on Code API. +- Recovery and revocation are atomic Redis transitions across API replicas. + A missing, revoked, expired or superseded enrollment never creates new + credentials. Recovery only restores transport authentication. It does not + clear assignment fences, worker or workspace quarantine, or uncertain + execution state. The worker private key is a durable, revocable credential; + expiry of an access credential alone does **not** protect against key theft. - Accepted proof nonces cannot be replayed, credentials rotate before expiry, and an administrator can revoke the active worker identity immediately. - Assignment leases bind to a stable paired identity rather than an individual diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 493eb000..aeeef4c4 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -488,11 +488,16 @@ it still advertises named environments. ### Expired bridge credential -A running worker refreshes its short-lived credential automatically. If a -machine is offline long enough that refresh can no longer authenticate, issue -a fresh one-time pairing for the same worker ID and redeem it with a newly -generated keypair. Reusing the worker ID preserves the LibreChat environment -record and its agent assignments; creating a new ID creates a new environment. +A running worker refreshes its short-lived credential automatically. With +Code API durable enrollment enabled, a worker that still has its enrolled +private key can request a short-lived challenge and recover a new access +credential without manual re-pairing. The current CLI does **not** yet invoke +that endpoint automatically; update it when worker reconnect support ships. +Until then, or if enrollment is missing or revoked, use the one-time operator +pairing fallback. A new pairing replaces the Code API worker identity and may +require LibreChat environment reauthorization; reusing a worker ID alone does +not guarantee preservation of its LibreChat environment or agent assignments. +Never clear quarantine or workspace fences as part of credential recovery. ### Failed environment setup or uncertain mutation diff --git a/packages/code/package.json b/packages/code/package.json index b00ca807..d3cf714b 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -15,6 +15,10 @@ "types": "./dist/protocol.d.ts", "import": "./dist/protocol.js" }, + "./identity": { + "types": "./dist/identity.d.ts", + "import": "./dist/identity.js" + }, "./worker": { "types": "./dist/worker.d.ts", "import": "./dist/worker.js" diff --git a/packages/code/src/identity.test.ts b/packages/code/src/identity.test.ts index 4e92b91f..bef3b673 100644 --- a/packages/code/src/identity.test.ts +++ b/packages/code/src/identity.test.ts @@ -1,9 +1,14 @@ import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; import test from 'node:test'; import { createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, signBridgeRequest, + verifyBridgeRecovery, + verifyBridgeRecoveryStart, verifyBridgeRequest, } from './identity.js'; @@ -33,3 +38,41 @@ test('worker identity proves possession for the exact HTTP request', () => { false, ); }); + +test('signed recovery starts bind operation, server, worker, time and nonce', () => { + const identity = createBridgeIdentity(); + const start = { + operation: 'credential.challenge' as const, + serverId: 'https://code.example.test', + workerId: 'vm-1', + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + const signature = signBridgeRecoveryStart(identity.privateKey, start); + assert.equal(verifyBridgeRecoveryStart(identity.publicKey, start, signature), true); + for (const modified of [ + { ...start, operation: 'credential.recover' as 'credential.challenge' }, + { ...start, serverId: 'https://other.example.test' }, + { ...start, workerId: 'vm-2' }, + { ...start, timestamp: new Date(Date.now() + 60_000).toISOString() }, + { ...start, nonce: randomBytes(32).toString('base64url') }, + ]) { + assert.equal(verifyBridgeRecoveryStart(identity.publicKey, modified, signature), false); + } + + const challenge = { + operation: 'credential.recover' as const, + serverId: start.serverId, + workerId: start.workerId, + enrollmentGeneration: randomBytes(18).toString('base64url'), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + assert.equal(verifyBridgeRecovery(identity.publicKey, challenge, signature), false); + assert.equal( + verifyBridgeRecoveryStart( + identity.publicKey, start, signBridgeRecovery(identity.privateKey, challenge), + ), + false, + ); +}); diff --git a/packages/code/src/identity.ts b/packages/code/src/identity.ts index ab11c865..3556520c 100644 --- a/packages/code/src/identity.ts +++ b/packages/code/src/identity.ts @@ -19,6 +19,25 @@ export interface BridgeRequestProofInput { body: string; } +/** Signed without an access credential before requesting a server recovery challenge. */ +export interface BridgeRecoveryStartProofInput { + operation: 'credential.challenge'; + serverId: string; + workerId: string; + timestamp: string; + nonce: string; +} + +/** Signed separately from access-credential requests, so neither proof can be reused for the other. */ +export interface BridgeRecoveryProofInput { + operation: 'credential.recover'; + serverId: string; + workerId: string; + enrollmentGeneration: string; + challenge: string; + expiresAt: string; +} + export function createBridgeIdentity(): BridgeIdentity { const { publicKey, privateKey } = generateKeyPairSync('ed25519', { publicKeyEncoding: { type: 'spki', format: 'pem' }, @@ -64,3 +83,78 @@ export function verifyBridgeRequest( return false; } } + +function canonicalBridgeRecoveryStart(input: BridgeRecoveryStartProofInput): string { + return [ + 'librechat-code:bridge-recovery-start:v1', + input.operation, + input.serverId, + input.workerId, + input.timestamp, + input.nonce, + ].join('\n'); +} + +export function signBridgeRecoveryStart( + privateKey: string, + input: BridgeRecoveryStartProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRecoveryStart(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRecoveryStart( + publicKey: string, + input: BridgeRecoveryStartProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRecoveryStart(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} + +function canonicalBridgeRecovery(input: BridgeRecoveryProofInput): string { + return [ + 'librechat-code:bridge-recovery:v1', + input.operation, + input.serverId, + input.workerId, + input.enrollmentGeneration, + input.challenge, + input.expiresAt, + ].join('\n'); +} + +export function signBridgeRecovery( + privateKey: string, + input: BridgeRecoveryProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRecovery(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRecovery( + publicKey: string, + input: BridgeRecoveryProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRecovery(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c678a1af..7657fc9c 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -759,6 +759,32 @@ export interface BridgeWorkerCredentialResponse { expiresAt: string; } +/** The enrolled machine signs this request before Code API issues a challenge. */ +export interface BridgeRecoveryChallengeRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'credential.challenge'; + serverId: string; + workerId: string; + timestamp: string; + nonce: string; + signature: string; +} + +/** A short-lived, single-use challenge for an already enrolled machine key. */ +export interface BridgeRecoveryChallengeResponse { + protocolVersion: BridgeProtocolVersion; + operation: 'credential.recover'; + serverId: string; + workerId: string; + enrollmentGeneration: string; + challenge: string; + expiresAt: string; +} + +export interface BridgeRecoveryRequest extends BridgeRecoveryChallengeResponse { + signature: string; +} + export interface BridgeSandboxRequest { body: TBody; headers: Record; diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index 16857fad..b04de55f 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -11,7 +11,23 @@ export const bridgeStore = new RedisBridgeStore( undefined, env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, ); -export const bridgePairings = new RedisBridgePairingStore(connection); +export const bridgePairings = new RedisBridgePairingStore( + connection, + undefined, + undefined, + undefined, + undefined, + env.BRIDGE_RECOVERY_SERVER_ID + ? { + serverId: env.BRIDGE_RECOVERY_SERVER_ID, + enrollmentTtlSeconds: env.BRIDGE_ENROLLMENT_TTL_SECONDS, + challengeTtlSeconds: env.BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, + maxChallengesPerMinute: env.BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, + maxAttemptsPerMinute: env.BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, + maxUntrustedRequestsPerMinute: env.BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, + } + : undefined, +); export default createBridgeRouter({ enabled: isBridgeEnabled(), diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index 9eb6d76c..b8a32162 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -6,13 +6,26 @@ import { import type Redis from 'ioredis'; -import { verifyBridgeRequest } from '../../../packages/code/src/identity'; +import { + verifyBridgeRecovery, + verifyBridgeRecoveryStart, + verifyBridgeRequest, +} from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryProofInput, + BridgeRecoveryStartProofInput, +} from '../../../packages/code/src/identity'; const PREFIX = 'codeapi:bridge:v1'; const DEFAULT_PAIRING_TTL_SECONDS = 10 * 60; const DEFAULT_CREDENTIAL_TTL_SECONDS = 15 * 60; const PROOF_NONCE_TTL_SECONDS = 2 * 60; const PROOF_CLOCK_SKEW_MS = 60_000; +const DEFAULT_CHALLENGE_TTL_SECONDS = 60; +const DEFAULT_CHALLENGES_PER_MINUTE = 12; +const DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE = 30; +const DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE = 240; +const RECOVERY_START_NONCE_TTL_SECONDS = 3 * 60; const LEGACY_SCAN_CLAIM_TTL_MS = 5_000; const LEGACY_SCAN_POLL_INTERVAL_MS = 25; const LEGACY_SCAN_PENDING = 'pending'; @@ -41,6 +54,7 @@ elseif (generation or '0') ~= ARGV[2] then redis.call('DEL', KEYS[1]) return 0 end +if ARGV[7] ~= '' and (generation or '0') ~= ARGV[9] then return 0 end redis.call('DEL', KEYS[1]) if redis.call('GET', KEYS[5]) == KEYS[1] then redis.call('DEL', KEYS[5]) @@ -48,9 +62,26 @@ end redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) redis.call('SET', KEYS[6], ARGV[6], 'EX', ARGV[4]) +if ARGV[7] ~= '' then + if tonumber(ARGV[8]) > 0 then + redis.call('SET', KEYS[7], ARGV[7], 'EX', ARGV[8]) + else + redis.call('SET', KEYS[7], ARGV[7]) + end + redis.call('SET', KEYS[8], ARGV[10]) +else + redis.call('DEL', KEYS[7], KEYS[8]) +end return 1 `; const ROTATE_CREDENTIAL_SCRIPT = ` +if ARGV[6] ~= '' then + if redis.call('GET', KEYS[5]) ~= ARGV[6] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[7] or redis.call('GET', KEYS[7]) ~= ARGV[8] then + return 0 + end +elseif redis.call('EXISTS', KEYS[7]) == 1 then + return 0 +end local activeDigest = redis.call('GET', KEYS[1]) local previous = redis.call('GET', KEYS[2]) if not activeDigest or not previous then @@ -82,12 +113,45 @@ if credential then redis.call('DEL', KEYS[3]) redis.call('DEL', ARGV[1] .. credential) end -redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7]) +redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8]) if activeIncarnation then redis.call('SET', ARGV[2] .. activeIncarnation .. ':fenced', '1') end return 1 `; +const CREATE_RECOVERY_CHALLENGE_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] or (redis.call('GET', KEYS[4]) or '0') ~= ARGV[4] or redis.call('GET', KEYS[5]) ~= ARGV[6] then + return 0 +end +if redis.call('EXISTS', KEYS[6]) == 1 then return -2 end +local count = redis.call('INCR', KEYS[2]) +if count == 1 then redis.call('EXPIRE', KEYS[2], 60) end +if count > tonumber(ARGV[2]) then return -1 end +if redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') ~= 'OK' then return 0 end +redis.call('SET', KEYS[6], '1', 'EX', ARGV[7]) +return 1 +`; +const LIMIT_RECOVERY_ATTEMPTS_SCRIPT = ` +local count = redis.call('INCR', KEYS[1]) +if count == 1 then redis.call('EXPIRE', KEYS[1], 60) end +if count > tonumber(ARGV[1]) then return 0 end +return 1 +`; +const COMPLETE_RECOVERY_CHALLENGE_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] or redis.call('GET', KEYS[2]) ~= ARGV[2] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[3] or redis.call('GET', KEYS[7]) ~= ARGV[8] then + return 0 +end +local stableIdentity = redis.call('GET', KEYS[5]) +if stableIdentity and stableIdentity ~= ARGV[4] then return 0 end +local count = redis.call('INCR', KEYS[8]) +if count == 1 then redis.call('EXPIRE', KEYS[8], 60) end +if count > tonumber(ARGV[9]) then return -1 end +redis.call('DEL', KEYS[2]) +redis.call('SET', KEYS[3], ARGV[5], 'EX', ARGV[6]) +redis.call('SET', KEYS[4], ARGV[7], 'EX', ARGV[6]) +redis.call('SET', KEYS[5], ARGV[4], 'EX', ARGV[6]) +return 1 +`; const RELEASE_LEGACY_SCAN_CLAIM_SCRIPT = ` if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) @@ -147,8 +211,34 @@ interface StoredCredential { publicKey: string; expiresAt: string; binding?: BridgeWorkerBinding; + /** Present only for credentials backed by durable machine enrollment. */ + enrollmentGeneration?: string; +} + +interface StoredEnrollment { + workerId: string; + serverId: string; + identityId: string; + generation: string; + pairingGeneration: number; + publicKey: string; + binding?: BridgeWorkerBinding; +} + +export interface BridgeRecoveryOptions { + /** Stable HTTPS origin of this Code API deployment, identical on every replica. */ + serverId: string; + /** Zero (the default) keeps enrollment until explicit revocation. */ + enrollmentTtlSeconds?: number; + challengeTtlSeconds?: number; + maxChallengesPerMinute?: number; + maxAttemptsPerMinute?: number; + /** Shared per-connection-peer cap, separate from machine-signed budgets. */ + maxUntrustedRequestsPerMinute?: number; } +export type BridgeRecoveryChallenge = BridgeRecoveryProofInput; + export interface BridgePairing { workerId: string; code: string; @@ -168,7 +258,10 @@ export class BridgePairingError extends Error { | 'PUBLIC_KEY_INVALID' | 'CREDENTIAL_INVALID' | 'PROOF_INVALID' - | 'PROOF_REPLAYED', + | 'PROOF_REPLAYED' + | 'ENROLLMENT_INVALID' + | 'CHALLENGE_INVALID' + | 'RECOVERY_RATE_LIMITED', message: string, ) { super(message); @@ -196,6 +289,39 @@ function workerStableIdentityKey(workerId: string): string { return `${PREFIX}:stable-identity:${workerId}`; } +function workerEnrollmentKey(workerId: string): string { + return `${PREFIX}:enrollment:${workerId}`; +} + +function workerEnrollmentRequiredKey(workerId: string): string { + return `${PREFIX}:enrollment-required:${workerId}`; +} + +function recoveryChallengeKey(challenge: string): string { + return `${PREFIX}:recovery:challenge:${digest(challenge)}`; +} + +function recoveryStartNonceKey(workerId: string, nonce: string): string { + return `${PREFIX}:recovery:nonce:${workerId}:${digest(nonce)}`; +} + +function recoveryChallengeAttemptKey(challenge: string): string { + return `${PREFIX}:recovery:attempt:${digest(challenge)}`; +} + +function untrustedRecoveryRateKey(peer: string, operation: 'challenge' | 'recover'): string { + // Never partition by a caller-supplied worker ID or an untrusted forwarded IP. + return `${PREFIX}:recovery:untrusted:${operation}:${digest(peer)}`; +} + +function recoveryRateKey( + workerId: string, + enrollmentGeneration: string, + operation: 'start' | 'complete', +): string { + return `${PREFIX}:recovery:rate:${operation}:${workerId}:${enrollmentGeneration}`; +} + function workerPairingGenerationKey(workerId: string): string { return `${PREFIX}:pairing-generation:${workerId}`; } @@ -236,8 +362,120 @@ export class RedisBridgePairingStore { private readonly credentialTtlSeconds = DEFAULT_CREDENTIAL_TTL_SECONDS, private readonly legacyScanClaimTtlMs = LEGACY_SCAN_CLAIM_TTL_MS, private readonly rollbackEpoch = - process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', - ) {} + process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', + private readonly recovery?: BridgeRecoveryOptions, + ) { + if (recovery == null) return; + let server: URL; + try { + server = new URL(recovery.serverId); + } catch { + throw new Error('Bridge recovery requires a stable HTTPS server origin'); + } + if ( + server.protocol !== 'https:' || + server.username !== '' || + server.password !== '' || + server.pathname !== '/' || + server.search !== '' || + server.hash !== '' || + server.origin !== recovery.serverId + ) { + throw new Error('Bridge recovery requires a stable HTTPS server origin'); + } + for (const [value, max] of [ + [recovery.enrollmentTtlSeconds ?? 0, 10 * 365 * 24 * 3600], + [recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS, 300], + [recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE, 120], + [recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE, 120], + [recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE, 1200], + ]) { + if (!Number.isSafeInteger(value) || value < 0 || value > max) { + throw new RangeError('Invalid bridge recovery lifetime or rate limit'); + } + } + if ( + (recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) === 0 || + (recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE) === 0 || + (recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE) === 0 || + (recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE) === 0 + ) { + throw new RangeError('Bridge recovery challenge lifetime and rate limits must be positive'); + } + } + + get recoveryEnabled(): boolean { + return this.recovery != null; + } + + async limitUntrustedRecovery( + peer: string, + operation: 'challenge' | 'recover', + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const allowed = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + untrustedRecoveryRateKey(peer, operation), + String(this.recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE), + ); + if (allowed !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many recovery requests from this peer'); + } + } + + private checkedEnrollment( + workerId: string, + raw: string | null, + pairingGeneration: string | null, + requiredGeneration: string | null, + ): StoredEnrollment { + let parsed: unknown; + try { + parsed = raw == null ? undefined : JSON.parse(raw); + } catch { + parsed = undefined; + } + const enrollment = ( + typeof parsed === 'object' && parsed != null && !Array.isArray(parsed) + ? parsed + : {} + ) as Partial; + if ( + this.recovery == null || + enrollment.workerId !== workerId || + enrollment.serverId !== this.recovery.serverId || + typeof enrollment.identityId !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.identityId) || + typeof enrollment.generation !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.generation) || + enrollment.generation !== requiredGeneration || + typeof enrollment.publicKey !== 'string' || + !validEd25519PublicKey(enrollment.publicKey) || + !Number.isSafeInteger(enrollment.pairingGeneration) || + String(enrollment.pairingGeneration) !== (pairingGeneration ?? '0') + ) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine enrollment is unavailable or revoked'); + } + return enrollment as StoredEnrollment; + } + + private checkEnrolledCredential( + credential: StoredCredential, + enrollment: StoredEnrollment, + ): void { + if ( + credential.workerId !== enrollment.workerId || + credential.identityId !== enrollment.identityId || + credential.publicKey !== enrollment.publicKey || + credential.enrollmentGeneration !== enrollment.generation || + JSON.stringify(credential.binding ?? null) !== JSON.stringify(enrollment.binding ?? null) + ) { + throw new BridgePairingError('CREDENTIAL_INVALID', 'Worker credential does not match its machine enrollment'); + } + } async issue( workerId: string, @@ -314,28 +552,49 @@ export class RedisBridgePairingStore { Date.now() + this.credentialTtlSeconds * 1000, ).toISOString(); const identityId = randomBytes(18).toString('base64url'); + const pairingGeneration = pairing.generation ?? Number( + (await this.redis.get(workerPairingGenerationKey(args.workerId))) ?? '0', + ); + const enrollment: StoredEnrollment | undefined = this.recovery == null + ? undefined + : { + workerId: args.workerId, + serverId: this.recovery.serverId, + identityId, + generation: randomBytes(18).toString('base64url'), + pairingGeneration, + publicKey: args.publicKey, + binding: pairing.binding, + }; const stored: StoredCredential = { workerId: args.workerId, identityId, publicKey: args.publicKey, expiresAt, binding: pairing.binding, + ...(enrollment == null ? {} : { enrollmentGeneration: enrollment.generation }), }; const accepted = await this.redis.eval( REDEEM_PAIRING_SCRIPT, - 6, + 8, codeKey, workerPairingGenerationKey(pairing.workerId), credentialDigestKey(credentialDigest), workerIdentityKey(args.workerId), workerPairingIndexKey(args.workerId), workerStableIdentityKey(args.workerId), + workerEnrollmentKey(args.workerId), + workerEnrollmentRequiredKey(args.workerId), raw, pairing.generation == null ? '' : String(pairing.generation), JSON.stringify(stored), String(this.credentialTtlSeconds), credentialDigest, identityId, + enrollment == null ? '' : JSON.stringify(enrollment), + String(this.recovery?.enrollmentTtlSeconds ?? 0), + String(pairingGeneration), + enrollment?.generation ?? '', ); if (accepted !== 1) { throw new BridgePairingError( @@ -374,10 +633,12 @@ export class RedisBridgePairingStore { ); } const credentialDigest = digest(args.credential); - const [raw, activeDigest, pairingGeneration] = await this.redis.mget( + const [raw, activeDigest, pairingGeneration, enrollmentRaw, requiredGeneration] = await this.redis.mget( credentialDigestKey(credentialDigest), workerIdentityKey(args.workerId), workerPairingGenerationKey(args.workerId), + workerEnrollmentKey(args.workerId), + workerEnrollmentRequiredKey(args.workerId), ); if (raw == null || activeDigest == null) { throw new BridgePairingError( @@ -386,6 +647,12 @@ export class RedisBridgePairingStore { ); } const stored = JSON.parse(raw) as StoredCredential; + if (stored.enrollmentGeneration != null || requiredGeneration != null || enrollmentRaw != null) { + this.checkEnrolledCredential( + stored, + this.checkedEnrollment(args.workerId, enrollmentRaw, pairingGeneration, requiredGeneration), + ); + } if (activeDigest !== credentialDigest) { const activeRaw = await this.redis.get( credentialDigestKey(activeDigest), @@ -439,6 +706,177 @@ export class RedisBridgePairingStore { }; } + async createRecoveryChallenge( + workerId: string, + request: BridgeRecoveryStartProofInput, + signature: string, + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const [raw, pairingGeneration, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + const enrollment = this.checkedEnrollment(workerId, raw, pairingGeneration, requiredGeneration); + const proofTime = Date.parse(request.timestamp); + if ( + String(request.operation) !== 'credential.challenge' || + request.serverId !== enrollment.serverId || + request.workerId !== workerId || + !/^[A-Za-z0-9_-]{43}$/.test(request.nonce) || + !Number.isFinite(proofTime) || + Math.abs(Date.now() - proofTime) > PROOF_CLOCK_SKEW_MS || + !verifyBridgeRecoveryStart(enrollment.publicKey, request, signature) + ) { + throw new BridgePairingError('PROOF_INVALID', 'Machine recovery challenge proof is invalid'); + } + const challenge: BridgeRecoveryChallenge = { + operation: 'credential.recover', + serverId: enrollment.serverId, + workerId, + enrollmentGeneration: enrollment.generation, + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date( + Date.now() + (this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) * 1000, + ).toISOString(), + }; + const created = await this.redis.eval( + CREATE_RECOVERY_CHALLENGE_SCRIPT, + 6, + workerEnrollmentKey(workerId), + recoveryRateKey(workerId, enrollment.generation, 'start'), + recoveryChallengeKey(challenge.challenge), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + recoveryStartNonceKey(workerId, request.nonce), + raw!, + String(this.recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE), + JSON.stringify(challenge), + String(enrollment.pairingGeneration), + String(this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS), + enrollment.generation, + String(RECOVERY_START_NONCE_TTL_SECONDS), + ); + if (created === -2) { + throw new BridgePairingError('PROOF_REPLAYED', 'Machine recovery challenge proof was already used'); + } + if (created === -1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery challenges'); + } + if (created !== 1) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine enrollment is unavailable or revoked'); + } + return challenge; + } + + async recoverCredential( + workerId: string, + proof: BridgeRecoveryChallenge, + signature: string, + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const challengeKey = recoveryChallengeKey(proof.challenge); + const [enrollmentRaw, challengeRaw, pairingGeneration, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + challengeKey, + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + const enrollment = this.checkedEnrollment( + workerId, enrollmentRaw, pairingGeneration, requiredGeneration, + ); + let parsed: unknown; + try { + parsed = challengeRaw == null ? undefined : JSON.parse(challengeRaw); + } catch { + parsed = undefined; + } + const saved = ( + typeof parsed === 'object' && parsed != null && !Array.isArray(parsed) + ? parsed + : {} + ) as Partial; + if ( + saved.operation !== 'credential.recover' || + saved.serverId !== enrollment.serverId || + saved.workerId !== workerId || + saved.enrollmentGeneration !== enrollment.generation || + saved.challenge !== proof.challenge || + saved.expiresAt !== proof.expiresAt || + String(proof.operation) !== saved.operation || + saved.serverId !== proof.serverId || + saved.workerId !== proof.workerId || + saved.enrollmentGeneration !== proof.enrollmentGeneration || + typeof saved.expiresAt !== 'string' || + !Number.isFinite(Date.parse(saved.expiresAt)) || + Date.parse(saved.expiresAt) <= Date.now() + ) { + throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); + } + // Invalid signatures can only exhaust the high-entropy challenge they know, + // never the enrolled worker's shared quota. The worker-wide counter is + // charged inside completion, after the key proof and replay checks. + const attempts = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + recoveryChallengeAttemptKey(proof.challenge), + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (attempts !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery attempts'); + } + if (!verifyBridgeRecovery(enrollment.publicKey, saved as BridgeRecoveryChallenge, signature)) { + throw new BridgePairingError('PROOF_INVALID', 'Machine recovery signature is invalid'); + } + + const credential = randomBytes(32).toString('base64url'); + const credentialDigest = digest(credential); + const expiresAt = new Date(Date.now() + this.credentialTtlSeconds * 1000).toISOString(); + const stored: StoredCredential = { + workerId, + identityId: enrollment.identityId, + publicKey: enrollment.publicKey, + expiresAt, + binding: enrollment.binding, + enrollmentGeneration: enrollment.generation, + }; + // The same Redis decision consumes the proof and issues the credential. + // A revoke or replacement on another replica wins by invalidating the + // enrollment/generation comparison, with no window to recreate trust. + const issued = await this.redis.eval( + COMPLETE_RECOVERY_CHALLENGE_SCRIPT, + 8, + workerEnrollmentKey(workerId), + challengeKey, + credentialDigestKey(credentialDigest), + workerIdentityKey(workerId), + workerStableIdentityKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + recoveryRateKey(workerId, enrollment.generation, 'complete'), + enrollmentRaw!, + challengeRaw!, + String(enrollment.pairingGeneration), + enrollment.identityId, + JSON.stringify(stored), + String(this.credentialTtlSeconds), + credentialDigest, + enrollment.generation, + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (issued === -1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recoveries'); + } + if (issued !== 1) { + throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); + } + return { workerId, credential, expiresAt }; + } + async revoke(workerId: string): Promise { await this.removeLegacyPairings(workerId); // Fence redemption and consume the currently indexed code atomically. An @@ -446,7 +884,7 @@ export class RedisBridgePairingStore { // that linearizes afterward installs a distinct generation and code. await this.redis.eval( REVOKE_PAIRING_SCRIPT, - 7, + 8, workerPairingIndexKey(workerId), workerPairingGenerationKey(workerId), workerIdentityKey(workerId), @@ -454,6 +892,7 @@ export class RedisBridgePairingStore { `${PREFIX}:worker:${encodeURIComponent(workerId)}`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:ready`, + workerEnrollmentKey(workerId), `${PREFIX}:credential:`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:`, ); @@ -652,12 +1091,26 @@ export class RedisBridgePairingStore { ); } const previous = JSON.parse(previousRaw) as StoredCredential; + let enrollment: StoredEnrollment | undefined; + let enrollmentRaw: string | null = null; + const [raw, generation, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + if (previous.enrollmentGeneration != null || requiredGeneration != null || raw != null) { + enrollment = this.checkedEnrollment(workerId, raw, generation, requiredGeneration); + this.checkEnrolledCredential(previous, enrollment); + enrollmentRaw = raw; + } return await this.issueCredential( workerId, previous.publicKey, previousDigest, previous.binding, previous.identityId ?? null, + enrollment, + enrollmentRaw, ); } @@ -667,6 +1120,8 @@ export class RedisBridgePairingStore { previousDigest?: string, binding?: BridgeWorkerBinding, identityId?: string | null, + enrollment?: StoredEnrollment, + enrollmentRaw?: string | null, ): Promise { const credential = randomBytes(32).toString('base64url'); const credentialDigest = digest(credential); @@ -683,20 +1138,27 @@ export class RedisBridgePairingStore { publicKey, expiresAt, binding, + ...(enrollment == null ? {} : { enrollmentGeneration: enrollment.generation }), }; if (previousDigest !== undefined) { const rotated = await this.redis.eval( ROTATE_CREDENTIAL_SCRIPT, - 4, + 7, workerIdentityKey(workerId), credentialDigestKey(previousDigest), credentialDigestKey(credentialDigest), workerStableIdentityKey(workerId), + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), previousDigest, credentialDigest, JSON.stringify(stored), String(this.credentialTtlSeconds), stableIdentityId ?? '', + enrollmentRaw ?? '', + String(enrollment?.pairingGeneration ?? ''), + enrollment?.generation ?? '', ); if (rotated !== 1) { throw new BridgePairingError( diff --git a/service/src/bridge/recovery-router.test.ts b/service/src/bridge/recovery-router.test.ts new file mode 100644 index 00000000..28ea4b1e --- /dev/null +++ b/service/src/bridge/recovery-router.test.ts @@ -0,0 +1,213 @@ +import { randomBytes } from 'crypto'; +import { createServer, type Server } from 'http'; + +import { afterEach, describe, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; +import type { BridgeRecoveryChallengeRequest, BridgeRecoveryChallengeResponse } from '../../../packages/code/src/protocol'; + +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, +} from '../../../packages/code/src/identity'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { RedisBridgePairingStore } from './pairing'; +import { createBridgeRouter } from './router'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const workerId = 'http-recovery-worker'; +const serverId = 'https://code.example.test'; +const servers: Server[] = []; + +function signedStart(privateKey: string): BridgeRecoveryChallengeRequest { + const request = { + operation: 'credential.challenge' as const, + serverId, + workerId, + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + ...request, + signature: signBridgeRecoveryStart(privateKey, request), + }; +} + +async function startRouter(pairings: RedisBridgePairingStore): Promise { + const app = express(); + app.set('trust proxy', 1); + app.use(json()); + app.use('/v1/bridge', createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings, + authMode: 'paired', + adminToken: 'operator-only', + configuredWorkerId: workerId, + })); + const server = createServer(app); + servers.push(server); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + return `http://127.0.0.1:${address.port}/v1/bridge/workers/${workerId}/credentials`; +} + +function post( + url: string, + body: object, + headers: Record = {}, +): Promise { + return fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...headers }, + body: JSON.stringify(body), + }); +} + +afterEach(async () => { + await Promise.all(servers.splice(0).map(server => new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve()); + }))); + await redis.flushall(); +}); + +describe('machine credential recovery HTTP API', () => { + test('does not expose recovery until the server identity is enabled', async () => { + const baseUrl = await startRouter(new RedisBridgePairingStore(redis)); + const response = await post(`${baseUrl}/challenge`, { protocolVersion: BRIDGE_PROTOCOL_VERSION }); + expect(response.status).toBe(404); + }); + + test('recovers using the stored key without administrator authentication and never leaks the binding', async () => { + const pairings = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, + }); + const identity = createBridgeIdentity(); + const pairing = await pairings.issue(workerId, { + tenantId: 'tenant-one', principal: { type: 'user', id: 'owner-one' }, + }); + await pairings.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const baseUrl = await startRouter(pairings); + const unsigned = await post(`${baseUrl}/challenge`, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + }); + expect(unsigned.status).toBe(400); + const outsider = createBridgeIdentity(); + const forged = await post(`${baseUrl}/challenge`, signedStart(outsider.privateKey)); + expect(forged.status).toBe(401); + await expect(forged.json()).resolves.toMatchObject({ code: 'PROOF_INVALID' }); + + const challengeResponse = await post(`${baseUrl}/challenge`, signedStart(identity.privateKey)); + expect(challengeResponse.status).toBe(200); + const challenge = (await challengeResponse.json()) as BridgeRecoveryChallengeResponse; + expect(challenge).toMatchObject({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + serverId, + workerId, + operation: 'credential.recover', + }); + expect(JSON.stringify(challenge)).not.toMatch(/tenant-one|owner-one|privateKey/); + + const signature = signBridgeRecovery(identity.privateKey, challenge); + const rejected = await post(`${baseUrl}/recover`, { + ...challenge, + serverId: 'https://unrelated.example.test', + signature, + }); + expect(rejected.status).toBe(401); + await expect(rejected.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); + + const recovered = await post(`${baseUrl}/recover`, { ...challenge, signature }); + expect(recovered.status).toBe(200); + const credential = (await recovered.json()) as { workerId: string; credential: string }; + expect(credential.workerId).toBe(workerId); + expect(credential.credential.length).toBeGreaterThanOrEqual(32); + const replay = await post(`${baseUrl}/recover`, { ...challenge, signature }); + expect(replay.status).toBe(401); + await expect(replay.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('limits forged starts before signature work across replicas despite spoofed proxy headers', async () => { + const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, maxUntrustedRequestsPerMinute: 2, + }); + const second = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, maxUntrustedRequestsPerMinute: 2, + }); + const identity = createBridgeIdentity(); + const attacker = createBridgeIdentity(); + const pairing = await first.issue(workerId); + await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const firstUrl = await startRouter(first); + const secondUrl = await startRouter(second); + let signatureChecks = 0; + for (const store of [first, second]) { + const original = store.createRecoveryChallenge.bind(store); + store.createRecoveryChallenge = async ( + ...args + ): ReturnType => { + signatureChecks += 1; + return original(...args); + }; + } + + const forged = async (url: string, suffix: number): Promise => post( + `${url}/challenge`, signedStart(attacker.privateKey), + { 'X-Forwarded-For': `198.51.100.${suffix}` }, + ); + const firstResponse = await forged(firstUrl, 1); + const secondResponse = await forged(secondUrl, 2); + const blocked = await forged( + firstUrl.replace(`/workers/${workerId}/`, '/workers/attacker-selected-worker/'), + 3, + ); + expect([firstResponse.status, secondResponse.status, blocked.status]).toEqual([401, 401, 429]); + await expect(blocked.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + expect(Number(blocked.headers.get('retry-after'))).toBeGreaterThan(0); + expect(signatureChecks).toBe(2); + expect(await redis.keys('codeapi:bridge:v1:recovery:rate:start:*')).toEqual([]); + + // Bogus completions use another untrusted bucket, not the worker's signed quota. + const fake = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'credential.recover', + serverId, + workerId, + enrollmentGeneration: 'a'.repeat(24), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + signature: 'a'.repeat(86), + }; + const rejected = await post(`${secondUrl}/recover`, fake); + expect(rejected.status).toBe(401); + expect(await redis.keys('codeapi:bridge:v1:recovery:rate:complete:*')).toEqual([]); + const recovered = await post(`${secondUrl}/recover`, fake); + expect(recovered.status).toBe(401); + const blockedCompletion = await post(`${secondUrl}/recover`, fake); + expect(blockedCompletion.status).toBe(429); + await expect(blockedCompletion.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('limits recovery challenges across two API routers sharing Redis', async () => { + const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', maxChallengesPerMinute: 1, + }); + const second = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', maxChallengesPerMinute: 1, + }); + const identity = createBridgeIdentity(); + const pairing = await first.issue(workerId); + await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const baseUrl = await startRouter(second); + const initial = signedStart(identity.privateKey); + await first.createRecoveryChallenge(workerId, initial, initial.signature); + const denied = await post(`${baseUrl}/challenge`, signedStart(identity.privateKey)); + expect(denied.status).toBe(429); + await expect(denied.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); +}); diff --git a/service/src/bridge/recovery.test.ts b/service/src/bridge/recovery.test.ts new file mode 100644 index 00000000..443e7277 --- /dev/null +++ b/service/src/bridge/recovery.test.ts @@ -0,0 +1,518 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash, randomBytes } from 'crypto'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; +import type { + BridgeRecoveryProofInput, + BridgeRecoveryStartProofInput, +} from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryChallenge, + BridgeRecoveryOptions, + BridgeWorkerBinding, + BridgeWorkerCredential, +} from './pairing'; + +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, + signBridgeRequest, +} from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const serverId = 'https://code.example.test'; +const workerId = 'durable-worker'; +const binding: BridgeWorkerBinding = { + tenantId: 'tenant-one', + principal: { type: 'user', id: 'owner-one' }, +}; + +function recoverableStore(options: Partial = {}): RedisBridgePairingStore { + return new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, + ...options, + }); +} + +function authorizedRequest( + privateKey: string, + credential: string, + nonce: string, +): Parameters[0] { + const proof = { + credential, + method: 'POST', + path: '/v1/bridge/workers/register', + timestamp: new Date().toISOString(), + nonce, + body: JSON.stringify({ protocolVersion: 1, workerId }), + }; + return { + ...proof, + workerId, + signature: signBridgeRequest(privateKey, proof), + }; +} + +async function enroll( + store: RedisBridgePairingStore, + publicKey: string, +): Promise { + const pairing = await store.issue(workerId, binding); + return store.redeem({ workerId, code: pairing.code, publicKey }); +} + +function recoveryStart( + privateKey: string, + nonce = randomBytes(32).toString('base64url'), + overrides: Partial = {}, +): { proof: BridgeRecoveryStartProofInput; signature: string } { + const proof: BridgeRecoveryStartProofInput = { + operation: 'credential.challenge', + serverId, + workerId, + timestamp: new Date().toISOString(), + nonce, + ...overrides, + }; + return { proof, signature: signBridgeRecoveryStart(privateKey, proof) }; +} + +async function challengeFor( + store: RedisBridgePairingStore, + privateKey: string, +): Promise { + const { proof, signature } = recoveryStart(privateKey); + return store.createRecoveryChallenge(workerId, proof, signature); +} + +async function recover( + store: RedisBridgePairingStore, + privateKey: string, +): Promise<{ challenge: BridgeRecoveryChallenge; credential: BridgeWorkerCredential }> { + const challenge = await challengeFor(store, privateKey); + const credential = await store.recoverCredential( + workerId, + challenge, + signBridgeRecovery(privateKey, challenge), + ); + return { challenge, credential }; +} + +afterEach(async () => { + await redis.flushall(); +}); + +describe('durable bridge enrollment', () => { + test('keeps legacy pairing and refresh compatible until recovery is enabled', async () => { + const legacy = new RedisBridgePairingStore(redis); + const identity = createBridgeIdentity(); + const issued = await enroll(legacy, identity.publicKey); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + + const replica = recoverableStore(); + const { proof, signature } = recoveryStart(identity.privateKey); + await expect(replica.createRecoveryChallenge(workerId, proof, signature)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const rotated = await replica.rotate(workerId); + await expect( + replica.authorize(authorizedRequest(identity.privateKey, rotated.credential, 'legacy-proof')), + ).resolves.toMatchObject({ workerId, binding }); + expect(issued.credential).not.toBe(rotated.credential); + }); + + test('recovers after access expiry and restart with the same identity and binding', async () => { + const identity = createBridgeIdentity(); + const firstReplica = recoverableStore(); + const issued = await enroll(firstReplica, identity.publicKey); + const original = await firstReplica.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'before-outage'), + ); + const digest = createHash('sha256').update(issued.credential).digest('hex'); + await redis.del( + `codeapi:bridge:v1:credential:${digest}`, + `codeapi:bridge:v1:identity:${workerId}`, + `codeapi:bridge:v1:stable-identity:${workerId}`, + ); + await expect(firstReplica.rotate(workerId)).rejects.toMatchObject({ + code: 'CREDENTIAL_INVALID', + }); + + const restartedReplica = recoverableStore(); + const { challenge, credential } = await recover(restartedReplica, identity.privateKey); + expect(challenge).toMatchObject({ + operation: 'credential.recover', + serverId, + workerId, + }); + const restored = await firstReplica.authorize( + authorizedRequest(identity.privateKey, credential.credential, 'after-outage'), + ); + expect(restored).toMatchObject({ identityId: original.identityId, binding }); + expect(credential.expiresAt).toBeString(); + const rotated = await restartedReplica.rotate(workerId, restored.credentialId); + await expect(firstReplica.authorize( + authorizedRequest(identity.privateKey, rotated.credential, 'after-rotation'), + )).resolves.toMatchObject({ identityId: original.identityId, binding }); + }); + + test('requires the enrolled key and binds proofs to server, worker, generation, operation, and expiry', async () => { + const enrolled = createBridgeIdentity(); + const outsider = createBridgeIdentity(); + const store = recoverableStore(); + await enroll(store, enrolled.publicKey); + const challenge = await challengeFor(store, enrolled.privateKey); + + await expect(store.recoverCredential( + workerId, challenge, signBridgeRecovery(outsider.privateKey, challenge), + )).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + for (const modified of [ + { ...challenge, serverId: 'https://other.example.test' }, + { ...challenge, workerId: 'another-worker' }, + { ...challenge, enrollmentGeneration: '0'.repeat(24) }, + { ...challenge, operation: 'credential.recover-other' as 'credential.recover' }, + { ...challenge, expiresAt: new Date(Date.now() + 120_000).toISOString() }, + ]) { + await expect(store.recoverCredential( + workerId, + modified, + signBridgeRecovery(enrolled.privateKey, modified as BridgeRecoveryProofInput), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + } + const signature = signBridgeRecovery(enrolled.privateKey, challenge); + await store.recoverCredential(workerId, challenge, signature); + await expect(store.recoverCredential(workerId, challenge, signature)) + .rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('rejects a signed challenge after its short-lived Redis window expires', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore({ challengeTtlSeconds: 1 }); + await enroll(store, identity.publicKey); + const challenge = await challengeFor(store, identity.privateKey); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await expect(store.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('retries a lost recovery response without changing the enrolled identity', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const originalCredential = await enroll(first, identity.publicKey); + const original = await first.authorize( + authorizedRequest(identity.privateKey, originalCredential.credential, 'before-lost-response'), + ); + await recover(first, identity.privateKey); // The caller lost this credential response. + const second = recoverableStore(); + const retried = await recover(second, identity.privateKey); + const auth = await second.authorize( + authorizedRequest(identity.privateKey, retried.credential.credential, 'after-lost-response'), + ); + expect(auth.identityId).toBe(original.identityId); + expect(auth.binding).toEqual(binding); + const enrolled = JSON.parse((await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`))!) as { + generation: string; + }; + expect(await redis.get(`codeapi:bridge:v1:enrollment-required:${workerId}`)) + .toBe(enrolled.generation); + }); + + test('rejects expired and missing enrollment even when an access credential remains live', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore({ enrollmentTtlSeconds: 1 }); + const issued = await enroll(store, identity.publicKey); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.rotate(workerId)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'expired-enrollment'), + )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('never treats an unmarked access token as legacy after authorization state is lost', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore(); + const issued = await enroll(store, identity.publicKey); + const credentialKey = `codeapi:bridge:v1:credential:${createHash('sha256').update(issued.credential).digest('hex')}`; + const raw = JSON.parse((await redis.get(credentialKey))!) as { enrollmentGeneration?: string }; + delete raw.enrollmentGeneration; // Simulate a refresh by a pre-recovery replica. + await redis.set(credentialKey, JSON.stringify(raw), 'EX', 300); + await redis.del(`codeapi:bridge:v1:enrollment:${workerId}`); + + expect(await redis.get(`codeapi:bridge:v1:enrollment-required:${workerId}`)).not.toBeNull(); + await expect(store.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'lost-authorization'), + )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.rotate(workerId)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await redis.set(`codeapi:bridge:v1:enrollment:${workerId}`, 'null'); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('re-pairing explicitly supersedes a previous key and binds to the new principal', async () => { + const first = createBridgeIdentity(); + const second = createBridgeIdentity(); + const store = recoverableStore(); + const initial = await enroll(store, first.publicKey); + const pending = await challengeFor(store, first.privateKey); + const replacement = await store.issue(workerId, { + tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' }, + }); + await store.redeem({ workerId, code: replacement.code, publicKey: second.publicKey }); + + await expect(store.recoverCredential( + workerId, pending, signBridgeRecovery(first.privateKey, pending), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + await expect(store.authorize( + authorizedRequest(first.privateKey, initial.credential, 'superseded-key'), + )).rejects.toMatchObject({ code: 'CREDENTIAL_INVALID' }); + const { credential } = await recover(store, second.privateKey); + await expect(store.authorize( + authorizedRequest(second.privateKey, credential.credential, 'new-owner'), + )).resolves.toMatchObject({ + binding: { tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' } }, + }); + }); + + test('concurrent replicas consume a recovery challenge and count it only once', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 2 }); + const second = recoverableStore({ maxAttemptsPerMinute: 2 }); + await enroll(first, identity.publicKey); + const challenge = await challengeFor(first, identity.privateKey); + const signature = signBridgeRecovery(identity.privateKey, challenge); + const attempts = await Promise.allSettled([ + first.recoverCredential(workerId, challenge, signature), + second.recoverCredential(workerId, challenge, signature), + ]); + const issued = attempts.filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled', + ); + const rejected = attempts.filter( + (result): result is PromiseRejectedResult => result.status === 'rejected', + ); + expect(issued).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect(rejected[0]).toMatchObject({ reason: { code: 'CHALLENGE_INVALID' } }); + const digest = createHash('sha256').update(issued[0].value.credential).digest('hex'); + expect(await redis.get(`codeapi:bridge:v1:identity:${workerId}`)).toBe(digest); + expect(await redis.get(`codeapi:bridge:v1:recovery:rate:complete:${workerId}:${challenge.enrollmentGeneration}`)) + .toBe('1'); + }); + + test('revocation beats a signed recovery pending on a different replica', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const second = recoverableStore(); + await enroll(first, identity.publicKey); + const challenge = await challengeFor(first, identity.privateKey); + const originalEval = redis.eval.bind(redis); + let release!: () => void; + let enter!: () => void; + const paused = new Promise((resolve) => { enter = resolve; }); + const resume = new Promise((resolve) => { release = resolve; }); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes('local stableIdentity = redis.call')) { + enter(); + await resume; + } + return (originalEval as (...evalArgs: unknown[]) => Promise)(script, ...args); + }) as Redis['eval']; + try { + const pending = first.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + ); + await paused; + await second.revoke(workerId); + release(); + await expect(pending).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + expect(await redis.get(`codeapi:bridge:v1:identity:${workerId}`)).toBeNull(); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + } finally { + redis.eval = originalEval as Redis['eval']; + release(); + } + }); + + test('isolates untrusted recovery limits by socket peer and endpoint from signed machine quotas', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ + maxUntrustedRequestsPerMinute: 1, maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1, + }); + const second = recoverableStore({ + maxUntrustedRequestsPerMinute: 1, maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1, + }); + await enroll(first, identity.publicKey); + await first.limitUntrustedRecovery('192.0.2.1', 'challenge'); + await expect(second.limitUntrustedRecovery('192.0.2.1', 'challenge')) + .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + await expect(second.limitUntrustedRecovery('192.0.2.2', 'challenge')) + .resolves.toBeUndefined(); + await expect(second.limitUntrustedRecovery('192.0.2.1', 'recover')) + .resolves.toBeUndefined(); + expect((await redis.keys('codeapi:bridge:v1:recovery:rate:start:*')).length).toBe(0); + const { credential } = await recover(second, identity.privateKey); + expect(credential.workerId).toBe(workerId); + }); + + test('only a signed, fresh, unused start request consumes the worker challenge budget', async () => { + const identity = createBridgeIdentity(); + const outsider = createBridgeIdentity(); + const first = recoverableStore({ maxChallengesPerMinute: 2 }); + const second = recoverableStore({ maxChallengesPerMinute: 2 }); + await enroll(first, identity.publicKey); + + for (let index = 0; index < 20; index += 1) { + const forged = recoveryStart(outsider.privateKey); + await expect(first.createRecoveryChallenge(workerId, forged.proof, forged.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + } + const otherServer = recoveryStart(identity.privateKey, undefined, { + serverId: 'https://unrelated.example.test', + }); + await expect(first.createRecoveryChallenge(workerId, otherServer.proof, otherServer.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + const stale = recoveryStart(identity.privateKey, undefined, { + timestamp: new Date(Date.now() - 5 * 60_000).toISOString(), + }); + await expect(first.createRecoveryChallenge(workerId, stale.proof, stale.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + + const valid = recoveryStart(identity.privateKey); + await first.createRecoveryChallenge(workerId, valid.proof, valid.signature); + await expect(second.createRecoveryChallenge(workerId, valid.proof, valid.signature)) + .rejects.toMatchObject({ code: 'PROOF_REPLAYED' }); + await challengeFor(second, identity.privateKey); + await expect(challengeFor(first, identity.privateKey)) + .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('fabricated completions never spend the worker budget or block a fresh signed recovery', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const real = await challengeFor(first, identity.privateKey); + + for (let index = 0; index < 35; index += 1) { + const fake = { ...real, challenge: randomBytes(32).toString('base64url') }; + await expect(first.recoverCredential(workerId, fake, 'forged')) + .rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + } + await first.recoverCredential(workerId, real, signBridgeRecovery(identity.privateKey, real)); + const next = await challengeFor(second, identity.privateKey); + await expect(second.recoverCredential( + workerId, next, signBridgeRecovery(identity.privateKey, next), + )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('invalid signatures exhaust only their own challenge, not the enrolled worker', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const attacked = await challengeFor(first, identity.privateKey); + await expect(first.recoverCredential(workerId, attacked, 'forged')) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(second.recoverCredential( + workerId, attacked, signBridgeRecovery(identity.privateKey, attacked), + )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + const fresh = await challengeFor(second, identity.privateKey); + await expect(first.recoverCredential( + workerId, fresh, signBridgeRecovery(identity.privateKey, fresh), + )).resolves.toMatchObject({ workerId }); + }); + + test('key replacement starts new signed challenge and recovery budgets', async () => { + const oldKey = createBridgeIdentity(); + const newKey = createBridgeIdentity(); + const store = recoverableStore({ maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1 }); + await enroll(store, oldKey.publicKey); + const oldChallenge = await challengeFor(store, oldKey.privateKey); + await store.recoverCredential( + workerId, oldChallenge, signBridgeRecovery(oldKey.privateKey, oldChallenge), + ); + const replacement = await store.issue(workerId, binding); + await store.redeem({ workerId, code: replacement.code, publicKey: newKey.publicKey }); + + await expect(challengeFor(store, oldKey.privateKey)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + const newChallenge = await challengeFor(store, newKey.privateKey); + await expect(store.recoverCredential( + workerId, newChallenge, signBridgeRecovery(newKey.privateKey, newChallenge), + )).resolves.toMatchObject({ workerId }); + }); + + test('revocation fences a previously verified start proof before a challenge is written', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const second = recoverableStore(); + await enroll(first, identity.publicKey); + const { proof, signature } = recoveryStart(identity.privateKey); + const originalEval = redis.eval.bind(redis); + let release!: () => void; + let enter!: () => void; + const paused = new Promise((resolve) => { enter = resolve; }); + const resume = new Promise((resolve) => { release = resolve; }); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes('KEYS[6]) == 1 then return -2')) { + enter(); + await resume; + } + return (originalEval as (...evalArgs: unknown[]) => Promise)(script, ...args); + }) as Redis['eval']; + try { + const pending = first.createRecoveryChallenge(workerId, proof, signature); + await paused; + await second.revoke(workerId); + release(); + await expect(pending).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.keys('codeapi:bridge:v1:recovery:challenge:*')).toEqual([]); + } finally { + redis.eval = originalEval as Redis['eval']; + release(); + } + }); + + test('recovering credentials never clears worker or workspace quarantine', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore(); + await enroll(store, identity.publicKey); + const workerQuarantine = `codeapi:bridge:v1:worker:${workerId}:incarnation:incarnation-00000001:quarantined`; + const workspaceQuarantine = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('session-one').digest('hex')}:quarantined`; + await redis.set(workerQuarantine, '1'); + await redis.set(workspaceQuarantine, '1'); + const { credential } = await recover(store, identity.privateKey); + expect(await redis.get(workerQuarantine)).toBe('1'); + expect(await redis.get(workspaceQuarantine)).toBe('1'); + const auth = await store.authorize( + authorizedRequest(identity.privateKey, credential.credential, 'quarantined-machine'), + ); + await expect(new RedisBridgeStore(redis).register({ + protocolVersion: 1, + workerId, + incarnationId: 'incarnation-00000001', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'] }, + }, auth)).rejects.toMatchObject({ code: 'WORKER_QUARANTINED' }); + }); + + test('refuses non-HTTPS or non-origin deployment identity and unbounded recovery policy', () => { + for (const invalid of ['http://code.example.test', 'https://code.example.test/path', 'https://user@code.example.test']) { + expect(() => recoverableStore({ serverId: invalid })).toThrow(); + } + expect(() => recoverableStore({ maxAttemptsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ maxUntrustedRequestsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ maxUntrustedRequestsPerMinute: 1201 })).toThrow(); + expect(() => recoverableStore({ challengeTtlSeconds: 301 })).toThrow(); + }); +}); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 5b319733..b3d549b1 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -310,6 +310,114 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { } })); + const untrustedRecoveryLimit = (operation: 'challenge' | 'recover'): RequestHandler => + (req, res, next) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + next(); + return; + } + // req.ip trusts X-Forwarded-For in our server. Use the connection peer so + // untrusted headers and arbitrary worker IDs cannot create new buckets. + void options.pairings.limitUntrustedRecovery(req.socket.remoteAddress ?? '', operation) + .then(() => next(), (error: unknown) => { + if (error instanceof BridgePairingError && error.code === 'RECOVERY_RATE_LIMITED') { + res.set('Retry-After', '60').status(429).json({ error: error.message, code: error.code }); + return; + } + next(error); + }); + }; + + router.post('/workers/:workerId/credentials/challenge', untrustedRecoveryLimit('challenge'), asyncRoute(async (req, res) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + res.status(404).json({ error: 'Machine recovery is disabled' }); + return; + } + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || !configuredWorker(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + body.operation !== 'credential.challenge' || + typeof body.serverId !== 'string' || body.serverId.length > 256 || + body.workerId !== workerId || + typeof body.timestamp !== 'string' || body.timestamp.length > 64 || + typeof body.nonce !== 'string' || !/^[A-Za-z0-9_-]{43}$/.test(body.nonce) || + typeof body.signature !== 'string' || !/^[A-Za-z0-9_-]{86}$/.test(body.signature) + ) { + res.status(400).json({ error: 'Invalid machine recovery challenge request' }); + return; + } + try { + const challenge = await options.pairings.createRecoveryChallenge( + workerId, + { + operation: 'credential.challenge', + serverId: body.serverId, + workerId, + timestamp: body.timestamp, + nonce: body.nonce, + }, + body.signature, + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...challenge }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(error.code === 'RECOVERY_RATE_LIMITED' ? 429 : 401) + .json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + + router.post('/workers/:workerId/credentials/recover', untrustedRecoveryLimit('recover'), asyncRoute(async (req, res) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + res.status(404).json({ error: 'Machine recovery is disabled' }); + return; + } + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || !configuredWorker(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + body.operation !== 'credential.recover' || + typeof body.serverId !== 'string' || body.serverId.length > 256 || + typeof body.enrollmentGeneration !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(body.enrollmentGeneration) || + typeof body.challenge !== 'string' || + !/^[A-Za-z0-9_-]{43}$/.test(body.challenge) || + typeof body.expiresAt !== 'string' || body.expiresAt.length > 64 || + typeof body.signature !== 'string' || + !/^[A-Za-z0-9_-]{86}$/.test(body.signature) + ) { + res.status(400).json({ error: 'Invalid machine recovery proof' }); + return; + } + try { + const credential = await options.pairings.recoverCredential( + workerId, + { + operation: 'credential.recover', + serverId: body.serverId, + workerId, + enrollmentGeneration: body.enrollmentGeneration, + challenge: body.challenge, + expiresAt: body.expiresAt, + }, + body.signature, + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...credential }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(error.code === 'RECOVERY_RATE_LIMITED' ? 429 : 401) + .json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + router.post( '/workers/:workerId/revoke', adminAuth, @@ -768,6 +876,5 @@ router.post( }), ); - return router; } diff --git a/service/src/config.ts b/service/src/config.ts index 55570dfe..29ae854d 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -426,6 +426,22 @@ export const env = { BRIDGE_AUTH_MODE: bridgeAuthMode, /** Enrollment and lease credential shared only with the configured worker. */ BRIDGE_TOKEN: process.env.CODEAPI_BRIDGE_TOKEN ?? '', + /** Opt-in stable HTTPS origin of this Code API deployment (shared by all replicas). */ + BRIDGE_RECOVERY_SERVER_ID: process.env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? '', + /** Zero preserves machine authorization until explicit revocation. */ + BRIDGE_ENROLLMENT_TTL_SECONDS: Number(process.env.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS ?? 0), + BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? 60, + ), + BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? 12, + ), + BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? 30, + ), + BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? 240, + ), /** * Runtime session affinity for stateful sandbox backends. * - `stateless` (default): no runtime sessions; `runtime_session_hint` ignored. diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index de14b553..8b3abd1d 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -15,6 +15,12 @@ function render(overrides) { CODEAPI_BRIDGE_WORKER_ID: '', CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', + CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', + CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', + CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', + CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '', + CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '', + CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: '', ...overrides, }, })); @@ -28,6 +34,12 @@ for (const overrides of [ CODEAPI_BRIDGE_DYNAMIC_WORKERS: 'false', CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', + CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', + CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', + CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', + CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '8', + CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '16', + CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: '400', }, ]) { const config = render(overrides); @@ -39,10 +51,20 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, overrides.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? '1'); assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, overrides.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? ''); + assert.equal(env.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS, overrides.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS ?? '0'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, overrides.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? '60'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? '12'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? '30'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? '240'); } for (const name of ['egress_gateway', 'sandbox-runner']) { assert.equal(config.services[name].environment.CODEAPI_BRIDGE_TOKEN, undefined); + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, undefined); + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, undefined); } + assert.match(JSON.stringify(config.services.redis.command), /--appendonly.*yes/); + assert.ok(config.services.redis.volumes.some(volume => volume.target === '/data' && volume.type === 'volume')); } assert.equal(render({}).services.api.environment.CODEAPI_BRIDGE_TOKEN, ''); -console.log('Compose bridge configuration passed (dynamic/fixed pairing, no default secret).'); +console.log('Compose bridge configuration passed (pairing, opt-in recovery, durable Redis).'); From 337ddd60a8ea2207459a52f00c33fdeb05899988 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 28 Sep 2026 08:04:06 -0400 Subject: [PATCH 47/75] =?UTF-8?q?=E2=8F=B3=20fix:=20Honor=20Workspace=20Qu?= =?UTF-8?q?eue=20Budgets=20Per=20Request=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: bound workspace queue admission per request * fix: classify pre-enqueue admission expiry and document queue budgets --- docs/byom-worker-admission.md | 42 +++++++----- docs/remote-bridge/README.md | 14 ++-- packages/code/README.md | 17 +++-- service/src/bridge/store.ts | 72 +++++++++++++-------- service/src/bridge/worker-admission.test.ts | 26 ++++++++ service/src/workspace-tools/router.test.ts | 59 +++++++++++++---- service/src/workspace-tools/router.ts | 27 ++++++-- 7 files changed, 183 insertions(+), 74 deletions(-) diff --git a/docs/byom-worker-admission.md b/docs/byom-worker-admission.md index 2fcd9f9f..0df8f17c 100644 --- a/docs/byom-worker-admission.md +++ b/docs/byom-worker-admission.md @@ -5,8 +5,13 @@ The limit is 32 admitted requests per worker, including the active request. When the limit is reached, the workspace endpoint returns HTTP 429 with `WORKER_QUEUE_FULL`. A different worker has an independent admission queue. -The Code API workspace HTTP endpoint allows up to the smaller of `JOB_TIMEOUT` -and five minutes for admission while the caller stays connected. After admission +The Code API workspace HTTP endpoint allows 30 seconds for admission when the +request has no `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can +advertise a positive integer allowance in milliseconds through that header, +up to five minutes and any configured server queue ceiling. An invalid or +out-of-range value is rejected before dispatch. This queue budget is +independent of `JOB_TIMEOUT`; a shorter client or proxy deadline still ends +the wait. After admission and worker validation, a separate execution deadline starts. Commands receive their requested timeout (30 seconds by default, up to five minutes), capped by the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. Other @@ -29,22 +34,27 @@ Existing workers still execute one assignment at a time. Parallel execution acro workspaces requires separate lease claims and isolated native sandbox contexts; this admission change does not advertise that capability. -Clients and reverse proxies must allow queue time plus execution/settlement time -and five seconds for HTTP delivery. With the default five-minute `JOB_TIMEOUT`, -that is at least 335 seconds for non-command tools, 340 seconds for default -commands, and 610 seconds for five-minute commands. With a smaller `JOB_TIMEOUT`, -use `min(JOB_TIMEOUT, 300s)` for the queue, plus `min(JOB_TIMEOUT, 30s)` for other +Clients and reverse proxies must allow the admitted queue budget plus +execution/settlement time and five seconds for HTTP delivery. With the default +five-minute `JOB_TIMEOUT` and no queue header, that is at least 65 seconds for +non-command tools, 70 seconds for default commands, and 340 seconds for +five-minute commands. At the maximum advertised five-minute queue allowance, +those totals become 335, 340, and 610 seconds respectively. With a smaller +`JOB_TIMEOUT`, use the advertised allowance (or 30 seconds without a header), +bounded by the server queue ceiling, plus `min(JOB_TIMEOUT, 30s)` for other operations or `min(JOB_TIMEOUT, requested command timeout) + 5s` for commands, -plus five seconds for delivery. +plus five seconds for delivery. The caller should advertise only the queue time +left after reserving execution, settlement, and delivery under its own HTTP +deadline; Code API does not receive that absolute deadline. -At the time of this change, LibreChat's `getWorkspaceToolTimeoutMs` still budgets -only 30 seconds for a single admission attempt (65/70/340 seconds in total). -Its `maxQueueWaitMs` is a retry horizon after a typed capacity rejection, **not** -a per-attempt HTTP timeout. Updating Code API alone therefore does not guarantee -the full wait. An earlier client, tool, or proxy timeout disconnects the request; -if work was already admitted, a mutation may have run and must not be blindly -retried. Match LibreChat's per-attempt timeout and each intermediary to the new -budget before relying on it. Existing workers do not need an update. +LibreChat's `maxQueueWaitMs` is a retry horizon after a typed capacity +rejection, **not** a per-attempt HTTP timeout. Without its opt-in +`maxRequestTimeoutMs`, LibreChat keeps a 30-second admission allowance per +attempt. Enabling a longer client budget requires LibreChat's header support on +every API replica and a timed canary through each intermediary; changing Code +API alone does not guarantee the full wait. An earlier client, tool, or proxy +timeout disconnects the request; if work was already admitted, a mutation may +have run and must not be blindly retried. Existing workers do not need an update. Focused regression coverage lives in `service/src/bridge/admission.test.ts`, `service/src/bridge/worker-admission.test.ts`, diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 96241ad7..464e4763 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -308,12 +308,14 @@ execution. worker. The lower API or worker slot ceiling wins, and assignments sharing the same workspace isolation key remain serialized while independent conversation worktrees may run concurrently. -- Workspace tool admission waits for capacity up to the smaller of `JOB_TIMEOUT` - and five minutes while the HTTP caller remains connected. Disconnects cancel - waiting, and admitted work receives a separate execution budget. A shorter - client or proxy timeout can end the wait sooner; Code API does not receive an - absolute caller deadline. See [BYOM worker admission](../byom-worker-admission.md) - for the caller and proxy timeout requirements. +- Workspace tool admission waits for capacity up to 30 seconds without a + `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can advertise a longer + per-request allowance, bounded by five minutes and any server queue ceiling. + Disconnects cancel waiting, and admitted work receives a separate execution + budget capped by `JOB_TIMEOUT`. A shorter client or proxy timeout can end the + wait sooner; Code API does not receive an absolute caller deadline. See + [BYOM worker admission](../byom-worker-admission.md) for the total-request + and proxy timeout requirements. - Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are diff --git a/packages/code/README.md b/packages/code/README.md index 87a6fe61..95d86485 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -791,17 +791,22 @@ Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. -On an updated Code API, admission waits up to the smaller of `JOB_TIMEOUT` and -five minutes while the HTTP caller remains connected; older Code API versions -waited at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, +On an updated Code API, admission waits up to 30 seconds without the +`X-LibreChat-Workspace-Queue-Wait-Ms` request header. A caller may advertise a +positive integer millisecond allowance up to five minutes, capped by any server +queue ceiling. This allowance is separate from the `JOB_TIMEOUT` execution +budget and cannot outlast a shorter client or proxy timeout. A +`WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, `Retry-After: 1`) means the operation was not assigned or started; wait for capacity before submitting it again. This is distinct from `ASSIGNMENT_EXPIRED` or a transport timeout after dispatch, where execution may have occurred and mutations must not be blindly retried. No automatic retry is added by this policy. -Align the client's per-attempt timeout and any proxy with the queue **plus** -execution budget before relying on the longer wait. See the +Align the client's per-attempt timeout and every proxy with the queue **plus** +execution, settlement, and delivery budget before relying on a longer wait. +LibreChat keeps the 30-second admission allowance unless its longer total HTTP +budget is explicitly enabled and the live ingress path is verified. See the [BYOM worker admission guide](../../docs/byom-worker-admission.md) for the -current client limitation and the timeout calculations. +timeout calculations. Keep the existing URL, pairing/identity, and network policy configuration. The primary root keeps its configured workspace ID (default `primary`). Repeat diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 3d9d7e1c..0ca75566 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -65,6 +65,26 @@ export class BridgeStoreError extends Error { } } +function classifyPreEnqueueExpiry( + error: unknown, + args: { workspaceRequest?: WorkspaceToolRequest; workspaceId?: string; signal: AbortSignal }, +): unknown { + // A queue deadline reached before the assignment enqueue attempt is a + // definite non-execution, including expiry during the initial Redis reads. + if ( + (args.workspaceRequest != null || args.workspaceId != null) && + !args.signal.aborted && + error instanceof BridgeStoreError && + error.code === 'ASSIGNMENT_EXPIRED' + ) { + return new BridgeStoreError( + 'WORKSPACE_QUEUE_TIMEOUT', + 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', + ); + } + return error; +} + interface StoredAssignment extends CodeBridgeAssignment { leaseTokenHash: string; workerIdentityId?: string; @@ -778,12 +798,17 @@ export class RedisBridgeStore { 'Invalid workspace execution budget', ); } - this.assertDispatchActive(args.signal, args.deadlineAtMs); - const dispatchable = await this.dispatchCommand( - () => this.dispatchableRegistration(args.workerId), - args, - 'Bridge worker registration read', - ); + let dispatchable: Awaited>; + try { + this.assertDispatchActive(args.signal, args.deadlineAtMs); + dispatchable = await this.dispatchCommand( + () => this.dispatchableRegistration(args.workerId), + args, + 'Bridge worker registration read', + ); + } catch (error) { + throw classifyPreEnqueueExpiry(error, args); + } if (dispatchable == null) { throw new BridgeStoreError( 'WORKER_OFFLINE', @@ -844,17 +869,20 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} does not advertise programmatic execution for the selected workspace`, ); } - if ( - args.runtimeSessionId !== undefined && - (await this.dispatchCommand( - () => - this.redis.exists( - workspaceQuarantineKey(args.workerId, args.runtimeSessionId ?? ''), - ), - args, - 'Bridge workspace fence read', - )) === 1 - ) { + let quarantined = false; + const runtimeSessionId = args.runtimeSessionId; + if (runtimeSessionId !== undefined) { + try { + quarantined = (await this.dispatchCommand( + () => this.redis.exists(workspaceQuarantineKey(args.workerId, runtimeSessionId)), + args, + 'Bridge workspace fence read', + )) === 1; + } catch (error) { + throw classifyPreEnqueueExpiry(error, args); + } + } + if (quarantined) { throw new BridgeStoreError( 'WORKSPACE_QUARANTINED', 'Bridge workspace is quarantined after an incomplete result commit', @@ -1181,15 +1209,7 @@ export class RedisBridgeStore { } } catch (error) { // Once enqueue starts, even a lost Redis response may hide execution. - if ( - admission != null && !enqueueAttempted && !args.signal.aborted && - error instanceof BridgeStoreError && error.code === 'ASSIGNMENT_EXPIRED' - ) { - throw new BridgeStoreError( - 'WORKSPACE_QUEUE_TIMEOUT', - 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', - ); - } + if (!enqueueAttempted) throw classifyPreEnqueueExpiry(error, args); throw error; } finally { if (admission != null) { diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index 98363c34..c77ac19e 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -124,6 +124,32 @@ test('an expired queued call never reaches the worker and does not strand later await third; }); +test('an already-expired workspace deadline is a definite queue timeout before registration', async () => { + await register(); + await expect(dispatch('expired-before-read', new AbortController(), -1)).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + expect(await redis.zcard(`codeapi:bridge:v1:worker:${workerId}:admission`)).toBe(0); + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +}); + +test('expiry during the registration read never becomes an ambiguous assignment error', async () => { + await register(); + const registrationRead = spyOn(redis, 'mget').mockImplementation(async () => { + await new Promise(resolve => setTimeout(resolve, 25)); + throw new Error('registration read outlived its queue budget'); + }); + try { + await expect(dispatch('expired-during-read', new AbortController(), 1)).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + } finally { + registrationRead.mockRestore(); + } + expect(await redis.zcard(`codeapi:bridge:v1:worker:${workerId}:admission`)).toBe(0); + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +}); + test('a queued request is rejected if the worker withdraws its capability', async () => { await register(); const first = dispatch('first'); diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 18d24a76..d3cb032e 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -67,16 +67,17 @@ test('binds instance admission to the authenticated tenant and user while preser expect(dispatched[1]?.workspaceInstanceId).toBeUndefined(); }); -test.each<[WorkspaceToolRequest, number, number?, number?]>([ - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined], - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 90_000 }, 95_000, 125_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000, undefined], - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, 5000], -])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling, queueTimeoutMs) => { +test.each<[WorkspaceToolRequest, number, number?, number?, number?]>([ + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 90_000 }, 95_000, 125_000, undefined, 90_000], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, 5000, 90_000], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined, 300_000], +])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling, queueTimeoutMs, advertisedQueueWaitMs) => { const app = express(); app.use(json()); app.use((req, _res, next) => { @@ -102,12 +103,15 @@ test.each<[WorkspaceToolRequest, number, number?, number?]>([ const address = server.address(); if (address == null || typeof address === 'string') throw new Error('Missing listener'); const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(request), + method: 'POST', headers: { + 'Content-Type': 'application/json', + ...(advertisedQueueWaitMs === undefined ? {} : { 'X-LibreChat-Workspace-Queue-Wait-Ms': String(advertisedQueueWaitMs) }), + }, body: JSON.stringify(request), }); await response.json(); expect(executionBudget).toBe(expectedExecution); if (request.operation === 'execute_command') expect(commandTimeout).toBe(expectedExecution - 5000); - const expectedQueueBudget = queueTimeoutMs ?? Math.min(ceiling ?? 30_000, 300_000); + const expectedQueueBudget = Math.min(advertisedQueueWaitMs ?? 30_000, queueTimeoutMs ?? 300_000); expect(queueRemaining).toBeGreaterThan(expectedQueueBudget - 1000); expect(queueRemaining).toBeLessThanOrEqual(expectedQueueBudget); }); @@ -122,6 +126,32 @@ test.each([0, 300_001, Number.POSITIVE_INFINITY])('rejects an unbounded queue ov })).toThrow('Workspace queue timeout must be between 1 and 300000 milliseconds'); }); +test.each(['0', '-1', '300001', '1.5', '01', '1, 2', '999999999999999999999'])('rejects invalid per-request queue allowance %j before dispatch', async (queueWait) => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); + next(); + }); + let dispatched = false; + app.use(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, + store: { async dispatchWorkspaceTool() { dispatched = true; throw new Error('must not dispatch'); } }, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Queue-Wait-Ms': queueWait }, + body: JSON.stringify({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }), + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + expect(dispatched).toBe(false); +}); + test('rejects new workspace dispatches while the service is shutting down', async () => { let dispatched = false; const app = express(); @@ -419,7 +449,7 @@ test.each([ status: expectedStatus, errorCode, outcome: 'completed', - deadlineBudgetMs: 330_000, + deadlineBudgetMs: 60_000, dispatchDurationMs: expect.any(Number), }), ); @@ -549,7 +579,8 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => closeConnection(); await expect(response).rejects.toThrow(); await closed.promise; - expect(queueRemaining).toBeGreaterThan(120_000); + expect(queueRemaining).toBeGreaterThan(29_000); + expect(queueRemaining).toBeLessThanOrEqual(30_000); expect(dispatchAborted).toBe(true); expect(logSpy).not.toHaveBeenCalled(); settlementGate.resolve(); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 51e47f26..78dbd1e7 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -21,6 +21,8 @@ import { import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; const MAX_WORKSPACE_QUEUE_WAIT_MS = 5 * 60_000; +const DEFAULT_WORKSPACE_QUEUE_WAIT_MS = 30_000; +const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms'; interface WorkspaceToolsRouterOptions { store: Pick; @@ -57,12 +59,10 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) )) { throw new RangeError('Workspace execution timeout must be a positive safe integer'); } - // The HTTP disconnect cancels waiting; this bounds admission while the caller remains connected. - const queueBudgetMs = options.queueTimeoutMs ?? Math.min( - options.timeoutMs ?? 30_000, - MAX_WORKSPACE_QUEUE_WAIT_MS, - ); - if (!Number.isSafeInteger(queueBudgetMs) || queueBudgetMs < 1 || queueBudgetMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + // A configured queue timeout is a ceiling. Legacy callers get 30 seconds even + // when their execution timeout is longer; only the per-request header opts in. + const queueCeilingMs = options.queueTimeoutMs ?? MAX_WORKSPACE_QUEUE_WAIT_MS; + if (!Number.isSafeInteger(queueCeilingMs) || queueCeilingMs < 1 || queueCeilingMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { throw new RangeError('Workspace queue timeout must be between 1 and 300000 milliseconds'); } const router = Router(); @@ -88,6 +88,21 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) }); return; } + const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); + if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const requestedQueueWaitMs = advertisedQueueWait === undefined + ? DEFAULT_WORKSPACE_QUEUE_WAIT_MS + : Number(advertisedQueueWait); + if (!Number.isSafeInteger(requestedQueueWaitMs) || requestedQueueWaitMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const queueBudgetMs = Math.min(requestedQueueWaitMs, queueCeilingMs); outcome.operation = req.body.operation; const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null ? req.body From 5b71e8c14d8843e725467174b2684debdf31ab58 Mon Sep 17 00:00:00 2001 From: proseer-lars Date: Mon, 28 Sep 2026 13:15:47 -0400 Subject: [PATCH 48/75] fix: deduplicate files by destination before sandbox validation (#247) * fix: deduplicate files by destination before sandbox validation Some callers (e.g. LibreChat) send the same file more than once per exec request when a user re-uploads a file in the same conversation. The sandbox's validateExecuteFiles() rejects duplicate destinations, which causes every tool call in that conversation to fail with: Error from sandbox: [bad_request] files contains duplicate destination "filename.pdf" This surfaces to the end user as a confusing "sandbox is down" error when the sandbox itself is fine - it is just rejecting the duplicate file list. Fix: add deduplicateFilesByDestination() and call it in getJob() before validateExecuteFiles(). Keeps the first file for each destination, silently drops subsequent duplicates, and logs a warning. * fix: preserve validated destinations when deduplicating input files * fix: protect submitted code from uploaded file collisions --------- Co-authored-by: Lia --- api/src/api/v2-files.test.ts | 103 ++++++++++++++++- api/src/api/v2-session-binding.test.ts | 86 ++++++++++++++ api/src/api/v2.ts | 151 ++++++++++++++++--------- 3 files changed, 284 insertions(+), 56 deletions(-) diff --git a/api/src/api/v2-files.test.ts b/api/src/api/v2-files.test.ts index 540b529b..83918884 100644 --- a/api/src/api/v2-files.test.ts +++ b/api/src/api/v2-files.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from 'bun:test'; import { config } from '../config'; +import { collectExecuteRequestInputFiles } from '../execution-manifest-request'; import type { TFile } from '../job'; -import { validateExecuteArguments, validateExecuteFiles } from './v2'; +import { validateExecuteArguments, validateExecuteFiles, deduplicateFilesByDestination } from './v2'; function messageOf(fn: () => void): string { try { @@ -85,4 +86,104 @@ describe('execute file validation', () => { ]; expect(() => validateExecuteFiles(files)).not.toThrow(); }); + + test('deduplicateFilesByDestination keeps the latest upload in the original destination order', () => { + const files: TFile[] = [ + { name: 'data.csv', id: 'first', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'second', storage_session_id: 'uploads' }, + { name: 'other.csv', id: 'unique', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'third', storage_session_id: 'latest-uploads' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result.map(file => file.name)).toEqual(['data.csv', 'other.csv']); + expect(result[0].id).toBe('third'); + expect(result[0].storage_session_id).toBe('latest-uploads'); + }); + + test('keeps a by-reference program in the entrypoint position when its upload is refreshed', () => { + const files: TFile[] = [ + { name: 'run.py', id: 'old-script', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'data', storage_session_id: 'uploads' }, + { name: 'run.py', id: 'new-script', storage_session_id: 'uploads' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result.map(file => file.name)).toEqual(['run.py', 'data.csv']); + expect(result[0].id).toBe('new-script'); + expect(() => validateExecuteFiles(result)).not.toThrow(); + }); + + test('rejects an upload that targets inline source, regardless of file order', () => { + const source: TFile = { name: 'main.py', content: 'print("submitted program")' }; + const upload: TFile = { name: 'main.py', id: 'uploaded-source', storage_session_id: 'uploads' }; + for (const files of [[source, upload], [upload, source]]) { + const message = messageOf(() => deduplicateFilesByDestination(files)); + expect(message).toContain('duplicate destination "main.py"'); + expect(message).toContain('inline content'); + } + expect(messageOf(() => deduplicateFilesByDestination([ + { content: 'unnamed source' } as TFile, + { name: 'file0.code', id: 'uploaded-source', storage_session_id: 'uploads' }, + ]))).toContain('duplicate destination "file0.code"'); + expect(messageOf(() => deduplicateFilesByDestination([ + source, { name: 'main.py', content: 'different source' }, + ]))).toContain('inline content'); + }); + + test('deduplicateFilesByDestination returns the same array when there are no duplicates', () => { + const files: TFile[] = [ + { name: 'a.csv', content: 'a' }, + { name: 'b.csv', content: 'b' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result).toEqual(files); + }); + + test('preserves the original destination of an unnamed file reference after deduplication', () => { + const files: TFile[] = [ + { name: 'main.py', content: 'print(1)' }, + { name: 'data.csv', id: 'old', storage_session_id: 'storage-session' }, + { name: 'data.csv', id: 'new', storage_session_id: 'storage-session' }, + { id: 'file-ref', storage_session_id: 'storage-session' } as TFile, + ]; + const deduped = deduplicateFilesByDestination(files); + expect(deduped.map(file => file.name)).toEqual(['main.py', 'data.csv', 'file3.code']); + expect(deduped[1].id).toBe('new'); + const signedDestination = collectExecuteRequestInputFiles({ files }) + .find(file => file.id === 'file-ref'); + expect(collectExecuteRequestInputFiles({ files: deduped }) + .find(file => file.id === 'file-ref')).toEqual(signedDestination); + expect(() => validateExecuteFiles(deduped)).not.toThrow(); + }); + + test('rejects malformed files even if another entry owns their destination', () => { + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'file1.code', content: 'source' }, + null as unknown as TFile, + ]))).toContain('files[1] must be an object'); + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'data.csv', content: 'old', encoding: 'invalid' as TFile['encoding'] }, + { name: 'data.csv', content: 'new' }, + ]))).toContain('files[0].encoding'); + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'data.csv', content: 'old' }, + { name: 'data.csv', id: 'file-ref' }, + ]))).toContain('files[1].storage_session_id'); + }); + + test('caps raw input count before dropping duplicates', () => { + const files = Array.from({ length: config.max_input_files + 1 }, () => ({ + name: 'data.csv', content: 'duplicate', + })); + expect(messageOf(() => deduplicateFilesByDestination(files))).toContain('cannot contain more than'); + }); + + test('deduplicateFilesByDestination allows validateExecuteFiles to accept duplicate uploads', () => { + const files: TFile[] = [ + { name: 'data.csv', id: 'first', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'second', storage_session_id: 'uploads' }, + ]; + const deduped = deduplicateFilesByDestination(files); + expect(deduped[0].id).toBe('second'); + expect(() => validateExecuteFiles(deduped)).not.toThrow(); + }); }); diff --git a/api/src/api/v2-session-binding.test.ts b/api/src/api/v2-session-binding.test.ts index 32a245f6..0987a68a 100644 --- a/api/src/api/v2-session-binding.test.ts +++ b/api/src/api/v2-session-binding.test.ts @@ -281,6 +281,92 @@ describe('per-request session binding', () => { } }); + test('uses the latest upload without renumbering later unnamed files', async () => { + config.session_workspace_enabled = false; + config.require_execution_manifest = false; + + const originalPrime = Job.prototype.prime; + const originalExecute = Job.prototype.execute; + const originalCleanup = Job.prototype.cleanup; + + let primedFiles: Array<{ name: string; id?: string; content?: string }> = []; + Job.prototype.prime = async function captureFiles(): Promise { + primedFiles = this.files; + }; + Job.prototype.execute = async function executeWithoutSandbox() { + return {} as Awaited>; + }; + Job.prototype.cleanup = async function cleanupWithoutFilesystem(): Promise {}; + + try { + const response = await fetch(`${baseUrl}/api/v2/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + language: testLanguage, + version: testVersion, + files: [ + { name: 'main.txt', content: 'source' }, + { name: 'data.csv', id: 'old-upload', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'new-upload', storage_session_id: 'uploads' }, + { content: 'unnamed source' }, + ], + }), + }); + + expect(response.status).toBe(200); + expect(primedFiles.map(file => file.name)).toEqual(['main.txt', 'data.csv', 'file3.code']); + expect(primedFiles[0].content).toBe('source'); + expect(primedFiles[1].id).toBe('new-upload'); + expect(primedFiles[2].content).toBe('unnamed source'); + } finally { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.cleanup = originalCleanup; + } + }); + + test('rejects an upload targeting the submitted source before binding or priming', async () => { + config.session_workspace_enabled = true; + config.require_execution_manifest = false; + + const originalPrime = Job.prototype.prime; + const originalExecute = Job.prototype.execute; + const originalCleanup = Job.prototype.cleanup; + let primed = false; + Job.prototype.prime = async function trackPrime(): Promise { primed = true; }; + Job.prototype.execute = async function executeWithoutSandbox() { + return {} as Awaited>; + }; + Job.prototype.cleanup = async function cleanupWithoutFilesystem(): Promise {}; + + try { + const response = await fetch(`${baseUrl}/api/v2/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Runtime-Session-Id': 'rt_source_collision' }, + body: JSON.stringify({ + language: testLanguage, + version: testVersion, + files: [ + { name: 'main.txt', content: 'submitted program' }, + { name: 'data.csv', id: 'data', storage_session_id: 'uploads' }, + { name: 'main.txt', id: 'uploaded-program', storage_session_id: 'uploads' }, + ], + }), + }); + + expect(response.status).toBe(400); + expect((await response.json() as { message: string }).message) + .toContain('duplicate destination "main.txt"'); + expect(primed).toBe(false); + expect(getBoundSessionWorkspace()).toBeUndefined(); + } finally { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.cleanup = originalCleanup; + } + }); + test('a post-prime failure still reports the workspace as dirty', async () => { config.session_workspace_enabled = true; config.require_execution_manifest = false; diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index 121cf2e3..3ace08cf 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -46,72 +46,111 @@ import { const router = express.Router(); const SYNTHETIC_PRINCIPAL_SOURCE = 'synthetic_test'; +/** + * Deduplicate files by destination name. Some callers (e.g. LibreChat) + * send the same file more than once per request when a user re-uploads a + * file in the same conversation. The sandbox rejects duplicate destinations, + * which surfaces to the end user as a confusing "sandbox is down" error. + * Keeps the latest uploaded file at each destination without moving its + * original position (the first runnable file is the program entrypoint). + * Never replace inline source code with an uploaded file. + */ +export function deduplicateFilesByDestination(files: TFile[]): TFile[] { + if (files.length > config.max_input_files) { + throw { message: `files cannot contain more than ${config.max_input_files} destinations` }; + } + const byDestination = new Map(); + for (const [i, file] of files.entries()) { + // Validate even entries that would otherwise be replaced by a later upload. + const destination = validateExecuteFile(file, i); + const previous = byDestination.get(destination); + if (previous && (typeof previous.content === 'string' || typeof file.content === 'string')) { + throw { message: `files contains duplicate destination "${destination}" involving inline content` }; + } + // Updating a Map value preserves its first position for Job.execute's entrypoint. + // Manifest claims use original indices; Job would otherwise renumber unnamed files. + byDestination.set(destination, file.name ? file : { ...file, name: destination }); + } + if (byDestination.size < files.length) { + logger.warn( + { original: files.length, deduped: byDestination.size }, + 'Deduplicated file list before validation', + ); + } + return [...byDestination.values()]; +} + function existingDestinationConflictMessage(existing: string, destination: string): string { return existing === destination ? `files contains duplicate destination "${destination}"` : `files contains conflicting destinations "${existing}" and "${destination}"`; } +function validateExecuteFile(value: TFile, i: number): string { + if (value == null || typeof value !== 'object' || Array.isArray(value)) { + throw { message: `files[${i}] must be an object` }; + } + const file = value as TFile; + const inline = typeof file.content === 'string'; + const byRef = typeof file.id === 'string' && file.id.length > 0; + if (inline === byRef) { + throw { + message: `files[${i}] must contain exactly one of non-empty id or string content`, + }; + } + if (file.id !== undefined && !byRef) { + throw { message: `files[${i}].id must be a non-empty string if provided` }; + } + if (byRef) { + if (typeof file.storage_session_id !== 'string' || file.storage_session_id.length === 0) { + throw { message: `files[${i}].storage_session_id is required as a non-empty string for file refs` }; + } + } else if (file.storage_session_id !== undefined || file.input_cache_key !== undefined) { + throw { + message: `files[${i}] inline content cannot include storage_session_id or input_cache_key`, + }; + } + if (file.name !== undefined && typeof file.name !== 'string') { + throw { message: `files[${i}].name must be a string if provided` }; + } + if ( + file.encoding !== undefined + && !(['base64', 'hex', 'utf8'] as const).includes(file.encoding) + ) { + throw { message: `files[${i}].encoding must be base64, hex, or utf8 if provided` }; + } + if (file.entity_id !== undefined && typeof file.entity_id !== 'string') { + throw { message: `files[${i}].entity_id must be a string if provided` }; + } + if ( + file.input_cache_key !== undefined && + ( + typeof file.input_cache_key !== 'string' || + !/^[0-9a-f]{64}$/.test(file.input_cache_key) + ) + ) { + throw { message: `files[${i}].input_cache_key must be a 64-character lowercase hex digest` }; + } + const destination = file.name || `file${i}.code`; + try { + validateFilePath(destination, '/tmp/codeapi-request-validation'); + } catch (error) { + throw { + message: error instanceof Error + ? `files[${i}].name is invalid: ${error.message}` + : `files[${i}].name is invalid`, + }; + } + return destination; +} + export function validateExecuteFiles(files: TFile[]): void { if (files.length > config.max_input_files) { throw { message: `files cannot contain more than ${config.max_input_files} destinations` }; } const destinations = new Set(); for (const [i, value] of files.entries()) { - if (value == null || typeof value !== 'object' || Array.isArray(value)) { - throw { message: `files[${i}] must be an object` }; - } - const file = value as TFile; - const inline = typeof file.content === 'string'; - const byRef = typeof file.id === 'string' && file.id.length > 0; - if (inline === byRef) { - throw { - message: `files[${i}] must contain exactly one of non-empty id or string content`, - }; - } - if (file.id !== undefined && !byRef) { - throw { message: `files[${i}].id must be a non-empty string if provided` }; - } - if (byRef) { - if (typeof file.storage_session_id !== 'string' || file.storage_session_id.length === 0) { - throw { message: `files[${i}].storage_session_id is required as a non-empty string for file refs` }; - } - } else if (file.storage_session_id !== undefined || file.input_cache_key !== undefined) { - throw { - message: `files[${i}] inline content cannot include storage_session_id or input_cache_key`, - }; - } - if (file.name !== undefined && typeof file.name !== 'string') { - throw { message: `files[${i}].name must be a string if provided` }; - } - if ( - file.encoding !== undefined - && !(['base64', 'hex', 'utf8'] as const).includes(file.encoding) - ) { - throw { message: `files[${i}].encoding must be base64, hex, or utf8 if provided` }; - } - if (file.entity_id !== undefined && typeof file.entity_id !== 'string') { - throw { message: `files[${i}].entity_id must be a string if provided` }; - } - if ( - file.input_cache_key !== undefined && - ( - typeof file.input_cache_key !== 'string' || - !/^[0-9a-f]{64}$/.test(file.input_cache_key) - ) - ) { - throw { message: `files[${i}].input_cache_key must be a 64-character lowercase hex digest` }; - } - const destination = file.name || `file${i}.code`; - try { - validateFilePath(destination, '/tmp/codeapi-request-validation'); - } catch (error) { - throw { - message: error instanceof Error - ? `files[${i}].name is invalid: ${error.message}` - : `files[${i}].name is invalid`, - }; - } + const destination = validateExecuteFile(value, i); const conflict = [...destinations].find( existing => existing === destination || @@ -251,12 +290,13 @@ function getJob( runtimeSessionHeader?: string | string[], ): Job { const { - session_id, language, version, args, stdin, files, + session_id, language, version, args, stdin, files: rawFiles, compile_memory_limit, run_memory_limit, compile_timeout, run_cpu_time, compile_cpu_time, env_vars, } = body; + let files = rawFiles; if (!language || typeof language !== 'string') { throw { message: 'language is required as a string' }; @@ -271,6 +311,7 @@ function getJob( throw { message: 'tool_call_socket must be a boolean if specified' }; } validateExecuteArguments(args, stdin); + files = deduplicateFilesByDestination(files); validateExecuteFiles(files); const rt = getLatestRuntimeMatchingLanguageVersion(language, version); From 1634027e185ca2d558327c02335d8e827059a550 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 29 Sep 2026 08:13:45 -0400 Subject: [PATCH 49/75] =?UTF-8?q?=F0=9F=9B=A4=EF=B8=8F=20feat:=20Schedule?= =?UTF-8?q?=20Linked=20Worktrees=20as=20Their=20Own=20Workspace=20Lanes=20?= =?UTF-8?q?(#270)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Schedule Linked Worktrees as Their Own Workspace Lanes * fix: Refuse quarantined-parent lanes before enqueue, forward lane policy, reset lane fences - Check the parent checkout fence before the assignment is stored or queued. - Forward the linked worktree policy to the forked native sandbox. - Protect shared Git config, hooks and info even before they exist. - Reset a lane fence with --reset-workspace-worktree. * fix: Grant lanes an explicit Git write allowlist, hold checkouts on quarantined lanes, release stale lanes - A lane writes only shared objects, refs, ref logs, LFS storage and its own worktree metadata; the checkout HEAD, index, operation state, config and hooks stay read-only without per-path denies. - A checkout assignment waits on any quarantined lane beneath it. - Lane registrations are released when their worktree disappears and capped at 32, dropping command roots and credential routes. * fix: Refuse a checkout while a lane keeps a stuck fence; let lane probes read shared Git - Code API indexes the lane fences enqueued beneath each checkout. A checkout reaches enqueue only once no lane holds a slot, so an indexed fence that still exists is stuck and the checkout is refused, surviving worker restarts. - Replay probes of a lane may read its common Git directory (read-only). * fix: Guard linked-worktree Git grants and unresolved actions --------- Co-authored-by: Lia --- docs/remote-bridge/README.md | 5 +- docs/remote-bridge/projects.md | 6 +- packages/code/README.md | 56 +++ packages/code/src/cli.ts | 106 +++++- packages/code/src/linked-worktrees.test.ts | 294 ++++++++++++++++ packages/code/src/linked-worktrees.ts | 386 +++++++++++++++++++++ packages/code/src/native-process.test.ts | 16 + packages/code/src/native-process.ts | 2 + packages/code/src/native-sandbox.test.ts | 61 ++++ packages/code/src/native-sandbox.ts | 77 +++- packages/code/src/protocol.test.ts | 61 ++++ packages/code/src/protocol.ts | 79 ++++- packages/code/src/worker-slots.test.ts | 201 +++++++++++ packages/code/src/worker.ts | 175 +++++++++- packages/code/src/workspace-cli.test.ts | 41 +++ service/src/bridge/linked-worktree.test.ts | 197 +++++++++++ service/src/bridge/router.ts | 1 + service/src/bridge/slots.test.ts | 62 +++- service/src/bridge/slots.ts | 38 +- service/src/bridge/store.ts | 83 ++++- service/src/service/programmatic-router.ts | 20 +- service/src/service/programmatic-state.ts | 21 +- service/src/service/replay-state.ts | 2 + service/src/types/service.ts | 4 + 24 files changed, 1944 insertions(+), 50 deletions(-) create mode 100644 packages/code/src/linked-worktrees.test.ts create mode 100644 packages/code/src/linked-worktrees.ts create mode 100644 service/src/bridge/linked-worktree.test.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 464e4763..a37745a2 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -307,7 +307,10 @@ execution. - Code API negotiates a bounded number of active workspace assignments per worker. The lower API or worker slot ceiling wins, and assignments sharing the same workspace isolation key remain serialized while independent - conversation worktrees may run concurrently. + conversation worktrees may run concurrently. A linked-worktree lane + (`worktree: `) nests beneath its checkout's key: sibling lanes run + concurrently, while a lane and its checkout exclude each other, and a lane + cannot start while its checkout is quarantined. - Workspace tool admission waits for capacity up to 30 seconds without a `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can advertise a longer per-request allowance, bounded by five minutes and any server queue ceiling. diff --git a/docs/remote-bridge/projects.md b/docs/remote-bridge/projects.md index 987ce112..26887ee9 100644 --- a/docs/remote-bridge/projects.md +++ b/docs/remote-bridge/projects.md @@ -41,7 +41,11 @@ workspace registration remains available for independent project directories. filesystem boundary and coordination for the common Git directory. - A worktree beneath its parent checkout overlaps that checkout. Either use disjoint execution roots under a discovery grant or explicitly exclude and - coordinate descendant worktrees before relaxing root exclusion. + coordinate descendant worktrees before relaxing root exclusion. Linked + worktree lanes (`--linked-worktree-lanes`) take the second approach for + `.worktrees/`: hierarchical admission keeps a lane and its checkout + exclusive, and each lane's sandbox can write only its worktree, shared + object and ref storage, and its own worktree metadata. - Setup and dependency links must remain within the execution policy. Sharing writable dependency directories between supposedly isolated worktrees reintroduces overlap and requires an explicit operator decision. diff --git a/packages/code/README.md b/packages/code/README.md index 95d86485..e3051930 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -791,6 +791,57 @@ Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. +#### Linked worktree lanes + +Agents that keep one checkout and give each task its own linked worktree +(`git worktree add .worktrees/`) can run those tasks concurrently: + +```sh +librechat-code run \ + --worker-dir /projects/LibreChat \ + --workspace-lease-slots 4 \ + --linked-worktree-lanes \ + --allow-workspace-writes \ + --allow-workspace-commands +``` + +`LIBRECHAT_CODE_LINKED_WORKTREE_LANES=true` is the environment equivalent. The +worker then advertises `workspaceScopes: ['git_linked_worktree']` for each +registered root, and a request that names `worktree: ` runs in its own +lane at `/.worktrees/`, with `cwd` and file paths relative to that +worktree. Sibling lanes run concurrently up to the negotiated slot count. A +lane and its checkout never run at the same time: requests without a +`worktree`, including `git worktree add` or `remove` run at the root, wait for +every lane beneath the checkout, and a waiting root request holds back newer +lanes so it cannot be starved. + +Before admission the worker verifies, without running Git, that the directory +is a real linked worktree of that checkout: no symlinks on the path, a `.git` +file pointing at `/.git/worktrees/`, and metadata whose `commondir` +and `gitdir` point back. Shared Git storage paths granted for writes must be +real directories, not symlinks into sibling metadata; optional log and LFS +paths may be absent. A lane's sandbox can write only its worktree, the +shared object and ref storage (`.git/objects`, `.git/refs`, `.git/logs/refs`, +`.git/lfs`) and its own `.git/worktrees/` metadata. Everything else in +`.git` stays read-only: configuration, hooks and `info`, the checkout's own +`HEAD`, index and merge or rebase state, and sibling metadata. Automatic `gc` +and maintenance are disabled, and `git gc` itself cannot run in a lane (it +needs to write `.git/gc.pid` and `packed-refs`); run storage maintenance from +the checkout. Explicit `git prune --expire now`, `git repack -ad`, or LFS +pruning can still delete objects another lane is writing; the sandbox does +**not** prevent this race. Do not enable lanes for agents that run destructive +maintenance until those commands are blocked or serialized. Deleting a branch +or tag also needs the checkout, because Git locks `packed-refs` for every ref +deletion. Each lane has its own durable quarantine guard. A lane +cannot start while its checkout is quarantined, and a checkout cannot start +while any lane beneath it is. + +Lanes require native-srt commands and at least two lease slots, and cannot yet +be combined with conversation worktrees. Code API must advertise +`supportedWorkspaceScopes`; older deployments do not, and the worker omits the +scope for them. Deploy consumers that read worker status (such as LibreChat) +with support for `workspaceScopes` before enabling lanes on a worker. + On an updated Code API, admission waits up to 30 seconds without the `X-LibreChat-Workspace-Queue-Wait-Ms` request header. A caller may advertise a positive integer millisecond allowance up to five minutes, capped by any server @@ -850,6 +901,11 @@ To recover a quarantined native root: 3. Run the normal worker command with all its root/slot options plus `--reset-workspace-quarantine second`. This verifies the local guard is cleared, resets the server fence, then exits. 4. Restart the normal worker command without the reset option. +A linked-worktree lane keeps its own guard and fence. Inspect or restore +`/.worktrees/`, clear its guard with +`--worker-dir /.worktrees/`, then add +`--reset-workspace-worktree ` to the reset command in step 3. + The workspace selector in LibreChat must preserve these registered IDs. Adding roots here does not grant a principal access or change an agent's selected root. # Named project environments diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 6e3a0062..135b9f8e 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { realpath, stat } from 'node:fs/promises'; -import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; +import { readdir, realpath, stat } from 'node:fs/promises'; +import { basename, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { discoverProjects } from './projects.js'; @@ -37,6 +37,7 @@ import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; +import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; import { GitWorktreeManager } from './worktrees.js'; import { captureWorkspaceRootIdentity } from './root-identity.js'; import { @@ -66,6 +67,7 @@ import { BridgeProtocolError, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isValidLinkedWorktreeName, workspaceIsolationKey, } from './protocol.js'; @@ -514,6 +516,11 @@ async function run( (args.includes('--allow-workspace-commands') || process.env.LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS?.trim().toLowerCase() === 'true'); + const linkedWorktreeLanes = + runtimeSessionId == null && + (args.includes('--linked-worktree-lanes') || + process.env.LIBRECHAT_CODE_LINKED_WORKTREE_LANES?.trim().toLowerCase() === + 'true'); const commandSandboxMode = option(args, '--command-sandbox') ?? process.env.LIBRECHAT_CODE_COMMAND_SANDBOX?.trim().toLowerCase() ?? @@ -783,6 +790,21 @@ async function run( 'Conversation worktrees require native-srt commands and at least two workspace lease slots', ); } + if ( + linkedWorktreeLanes && + (!allowWorkspaceCommands || + commandSandboxMode !== 'native-srt' || + workspaceLeaseSlots < 2) + ) { + throw new Error( + 'Linked worktree lanes require native-srt commands and at least two workspace lease slots', + ); + } + if (linkedWorktreeLanes && conversationWorktreeRoot) { + throw new Error( + 'Linked worktree lanes cannot be combined with conversation worktrees', + ); + } if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() @@ -1073,7 +1095,7 @@ async function run( }; const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot + ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot || linkedWorktreeLanes ? new NativeWorkspaceCommandPool( new Map( roots.map(root => [ @@ -1199,6 +1221,41 @@ async function run( ), }); workspaceTools = conversationWorkspaceTools; + } + let linkedWorktreeTools: LinkedWorktreeWorkspaceTools | undefined; + if (linkedWorktreeLanes && workspaceTools) { + if (!(nativeCommandSandbox instanceof NativeWorkspaceCommandPool)) { + throw new Error('Linked worktree lanes require a native command pool'); + } + linkedWorktreeTools = new LinkedWorktreeWorkspaceTools({ + commandPool: nativeCommandSandbox, + delegate: workspaceTools, + programmaticDelegate: conversationWorkspaceTools ?? nativeCommandSandbox, + onResolve(workspaceId, root) { + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + root, + repositoriesByWorkspace?.get(workspaceId), + ); + } + }, + onRelease(root) { + admittedGitHubRepositories?.delete(root); + }, + sources: new Map( + roots.map((root) => [ + root.id, + { + root: root.root, + identity: root.identity, + command: nativeOptions, + repositoryInstructions: args.includes('--repository-instructions'), + writable: root.writable ?? false, + }, + ]), + ), + }); + workspaceTools = linkedWorktreeTools; } if (workspaceTools && environments.length) { workspaceTools = new EnvironmentWorkspaceTools( @@ -1304,7 +1361,7 @@ async function run( ...(nativeProgrammaticEnabled && nativeCommandSandbox ? { workspaceProgrammatic: - conversationWorkspaceTools ?? nativeCommandSandbox, + linkedWorktreeTools ?? conversationWorkspaceTools ?? nativeCommandSandbox, } : {}), ...(conversationWorktrees @@ -1331,6 +1388,40 @@ async function run( ), } : {}), + ...(linkedWorktreeTools + ? { + linkedWorktreeQuarantineResolver: ( + selectedWorkspaceId: string, + worktree: string, + ) => { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) { + throw new BridgeProtocolError('Linked worktree source is not registered'); + } + return workspaceMutationGuard( + defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: join(source.root, LINKED_WORKTREE_DIRECTORY, worktree), + }), + workerId, + workspaceIsolationKey(selectedWorkspaceId, undefined, worktree), + incarnationId, + ); + }, + linkedWorktreeNames: async (selectedWorkspaceId: string) => { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) return []; + try { + const entries = await readdir(join(source.root, LINKED_WORKTREE_DIRECTORY)); + return entries.filter(isValidLinkedWorktreeName); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + }, + } + : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( @@ -1445,15 +1536,20 @@ async function run( args, '--reset-workspace-instance', ); + const resetWorkspaceWorktree = option( + args, + '--reset-workspace-worktree', + ); await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); await worker.resetNativeWorkspace( resetNativeRoot, controller.signal, resetWorkspaceInstance, + resetWorkspaceWorktree, ); process.stdout.write( - `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}\n`, + `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}${resetWorkspaceWorktree ? ` worktree ${resetWorkspaceWorktree}` : ''}\n`, ); return; } diff --git a/packages/code/src/linked-worktrees.test.ts b/packages/code/src/linked-worktrees.test.ts new file mode 100644 index 00000000..53ec156e --- /dev/null +++ b/packages/code/src/linked-worktrees.test.ts @@ -0,0 +1,294 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { + LinkedWorktreeWorkspaceTools, + linkedWorktreeWorkspaceId, + verifyLinkedWorktree, +} from './linked-worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceToolRequest } from './protocol.js'; + +const execFileAsync = promisify(execFile); + +async function git(cwd: string, ...args: string[]): Promise { + await execFileAsync( + 'git', + ['-c', 'user.name=test', '-c', 'user.email=test@example.com', '-c', 'commit.gpgsign=false', ...args], + { cwd }, + ); +} + +async function checkout(): Promise<{ parent: string; root: string }> { + const parent = await realpath(await mkdtemp(join(tmpdir(), 'linked-worktree-'))); + const root = join(parent, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'README.md'), 'root\n'); + await git(root, 'add', '.'); + await git(root, 'commit', '-q', '-m', 'init'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '-b', 'task-a', '.worktrees/task-a'); + return { parent, root }; +} + +async function rejects(promise: Promise, code = 'INVALID_REQUEST'): Promise { + await assert.rejects(promise, (error: unknown) => error instanceof WorkspaceToolError && error.code === code); +} + +test('verifies a linked worktree of the checkout and lists the only Git paths it may write', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + + const lane = await verifyLinkedWorktree(root, 'task-a'); + + assert.equal(lane.root, join(root, '.worktrees', 'task-a')); + assert.equal(lane.checkoutRoot, root); + assert.equal(lane.commonGitDir, join(root, '.git')); + assert.deepEqual(lane.writableGitPaths, [ + join(root, '.git', 'objects'), + join(root, '.git', 'refs'), + join(root, '.git', 'logs', 'refs'), + join(root, '.git', 'lfs'), + join(root, '.git', 'worktrees', 'task-a'), + ]); +}); + + +test('rejects directories that are not linked worktrees of this checkout', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + + await mkdir(join(root, '.worktrees', 'plain')); + await rejects(verifyLinkedWorktree(root, 'plain')); + + await mkdir(join(root, '.worktrees', 'borrowed')); + await writeFile( + join(root, '.worktrees', 'borrowed', '.git'), + `gitdir: ${join(root, '.git', 'worktrees', 'task-a')}\n`, + ); + await rejects(verifyLinkedWorktree(root, 'borrowed')); + + await symlink(join(root, '.worktrees', 'task-a'), join(root, '.worktrees', 'alias')); + await rejects(verifyLinkedWorktree(root, 'alias')); + + const other = join(parent, 'other'); + await mkdir(other); + await git(other, 'init', '-q', '-b', 'main'); + await writeFile(join(other, 'file'), 'x\n'); + await git(other, 'add', '.'); + await git(other, 'commit', '-q', '-m', 'other'); + await git(other, 'worktree', 'add', '-q', '-b', 'foreign', join(root, '.worktrees', 'foreign')); + await rejects(verifyLinkedWorktree(root, 'foreign')); + + for (const name of ['..', '.git', 'a/b', 'task.lock']) { + await rejects(verifyLinkedWorktree(root, name)); + } + await rejects(verifyLinkedWorktree(root, 'missing')); +}); + +test('rejects a checkout whose .worktrees directory is a symlink', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const elsewhere = join(parent, 'elsewhere'); + await git(root, 'worktree', 'move', '.worktrees/task-a', elsewhere); + await rm(join(root, '.worktrees'), { recursive: true }); + await symlink(parent, join(root, '.worktrees')); + await rejects(verifyLinkedWorktree(root, 'elsewhere')); +}); + +test('rejects shared Git storage symlinks into sibling worktree metadata', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + const commonGitDir = join(root, '.git'); + const siblingMetadata = join(commonGitDir, 'worktrees', 'task-b'); + await rm(join(commonGitDir, 'objects'), { recursive: true }); + await symlink(siblingMetadata, join(commonGitDir, 'objects')); + + await rejects(verifyLinkedWorktree(root, 'task-a')); +}); + +function recordingPool(): { + pool: NativeWorkspaceCommandPool; + calls: Array<{ action: string; id: string; options?: NativeProcessSandboxOptions }>; +} { + const calls: Array<{ action: string; id: string; options?: NativeProcessSandboxOptions }> = []; + const pool = { + async registerRoot(id: string, options: NativeProcessSandboxOptions) { + calls.push({ action: 'register', id, options }); + }, + async unregisterRoot(id: string) { + calls.push({ action: 'unregister', id }); + }, + async execute(request: WorkspaceToolRequest) { + calls.push({ action: 'execute', id: request.workspaceId }); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + async executeProgrammatic(id: string) { + calls.push({ action: 'programmatic', id }); + return {}; + }, + } as unknown as NativeWorkspaceCommandPool; + return { pool, calls }; +} + +async function laneTools( + root: string, + pool?: NativeWorkspaceCommandPool, + onRelease?: (root: string) => void, +) { + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + return new LinkedWorktreeWorkspaceTools({ + commandPool: pool, + delegate, + onRelease, + sources: new Map([ + [ + 'repo', + { + root, + command: { workspaceRoot: root } as NativeProcessSandboxOptions, + repositoryInstructions: false, + writable: true, + }, + ], + ]), + }); +} + +test('lane file tools are confined to the worktree and report the public workspace', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const tools = await laneTools(root); + + assert.deepEqual(tools.capabilities.workspaces[0]?.workspaceScopes, ['git_linked_worktree']); + const written = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'lane.txt', + content: 'from the lane\n', + }); + assert.equal(written.workspaceId, 'repo'); + assert.equal(await readFile(join(root, '.worktrees', 'task-a', 'lane.txt'), 'utf8'), 'from the lane\n'); + await assert.rejects(stat(join(root, 'lane.txt'))); + + const read = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + path: 'README.md', + }); + assert.equal(read.operation === 'read_file' && read.content.trimEnd(), 'root'); + + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + workspaceInstanceId: 'a'.repeat(64), + path: 'README.md', + }), + ); +}); + +test('lane commands register a confined root once, whatever siblings come and go', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const { pool, calls } = recordingPool(); + const tools = await laneTools(root, pool); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree: 'task-a', + command: 'git status', + }; + const id = linkedWorktreeWorkspaceId('repo', 'task-a'); + + const result = await tools.execute(command); + await tools.execute(command); + assert.equal(result.workspaceId, 'repo'); + assert.deepEqual( + calls.map((call) => call.action), + ['register', 'execute', 'execute'], + ); + const registered = calls[0]!.options!; + assert.equal(calls[0]!.id, id); + assert.equal(registered.workspaceRoot, join(root, '.worktrees', 'task-a')); + assert.equal(registered.linkedWorktree?.commonGitDir, join(root, '.git')); + assert.equal(registered.linkedWorktree?.checkoutRoot, root); + + assert.ok(!registered.linkedWorktree?.writableGitPaths.includes(join(root, '.git'))); + + await rejects(tools.execute({ + ...command, + environmentAction: { name: 'unresolved', fingerprint: 'a'.repeat(64) }, + })); + assert.equal(calls.length, 3, 'an unresolved action must never reach the command pool'); + + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + await tools.execute(command); + assert.deepEqual( + calls.slice(3).map((call) => call.action), + ['execute'], + ); +}); + +test('a removed lane releases its command root and credential route', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const { pool, calls } = recordingPool(); + const released: string[] = []; + const tools = await laneTools(root, pool, (lane) => released.push(lane)); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree: 'task-a', + command: 'git status', + }; + await tools.execute(command); + + await git(root, 'worktree', 'remove', '.worktrees/task-a'); + await rejects(tools.execute(command)); + + assert.deepEqual( + calls.map((call) => call.action), + ['register', 'execute', 'unregister'], + ); + assert.deepEqual(released, [join(root, '.worktrees', 'task-a')]); +}); diff --git a/packages/code/src/linked-worktrees.ts b/packages/code/src/linked-worktrees.ts new file mode 100644 index 00000000..e049aa6e --- /dev/null +++ b/packages/code/src/linked-worktrees.ts @@ -0,0 +1,386 @@ +import { createHash } from 'node:crypto'; +import { lstat, open, realpath } from 'node:fs/promises'; +import { isAbsolute, join, resolve } from 'node:path'; + +import { isValidLinkedWorktreeName } from './protocol.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, + WorkspaceToolRequest, + WorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +/** Linked worktrees are only admitted from this directory beneath a checkout. */ +export const LINKED_WORKTREE_DIRECTORY = '.worktrees'; +/** + * Shared Git storage a lane may write beneath the common Git directory. Every + * other path there (config, hooks, the checkout's own HEAD, index and + * operation state, sibling metadata) stays read-only. + */ +const LINKED_WORKTREE_SHARED_GIT_PATHS = ['objects', 'refs', join('logs', 'refs'), 'lfs']; +/** Lane registrations kept per worker; the least recently used idle lanes are released first. */ +const LINKED_WORKTREE_LANE_LIMIT = 32; +/** Git pointer files are a single line; anything larger is not one. */ +const GIT_POINTER_MAX_BYTES = 4096; + +export interface LinkedWorktreeSource { + root: string; + identity?: WorkspaceRootIdentity; + command?: NativeProcessSandboxOptions; + repositoryInstructions: boolean; + writable: boolean; +} + +export interface VerifiedLinkedWorktree { + root: string; + identity: WorkspaceRootIdentity; + checkoutRoot: string; + commonGitDir: string; + /** Shared object and ref storage plus the lane's own metadata; nothing else in the common Git directory. */ + writableGitPaths: string[]; +} + +export interface LinkedWorktreeWorkspaceToolsOptions { + commandPool?: NativeWorkspaceCommandPool; + delegate: WorkspaceToolExecutor; + /** Called with each verified lane root, e.g. to route credentials for its repository. */ + onResolve?: (workspaceId: string, root: string) => void; + /** Called when a lane root is released, e.g. to drop its credential route. */ + onRelease?: (root: string) => void; + programmaticDelegate?: { + executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise; + }; + sources: ReadonlyMap; +} + +export function linkedWorktreeWorkspaceId(workspaceId: string, worktree: string): string { + return `lane-${createHash('sha256').update(`${workspaceId}\0${worktree}`).digest('hex')}`; +} + +function rejected(message: string): WorkspaceToolError { + return new WorkspaceToolError(message, 'INVALID_REQUEST'); +} + +async function realDirectory(path: string): Promise { + try { + const status = await lstat(path); + return status.isDirectory() && !status.isSymbolicLink(); + } catch { + return false; + } +} + +/** Optional Git paths may be absent, but cannot redirect a write grant into sibling metadata. */ +async function safeSharedGitStorage(commonGitDir: string): Promise { + for (const path of ['objects', 'refs', 'logs', join('logs', 'refs'), 'lfs']) { + try { + const status = await lstat(join(commonGitDir, path)); + if (!status.isDirectory() || status.isSymbolicLink()) return false; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT' || + path === 'objects' || path === 'refs') return false; + } + } + return true; +} + +async function readPointer(path: string): Promise { + let handle; + try { + const status = await lstat(path); + if (!status.isFile() || status.isSymbolicLink() || status.size > GIT_POINTER_MAX_BYTES) { + return undefined; + } + handle = await open(path, 'r'); + const buffer = Buffer.alloc(GIT_POINTER_MAX_BYTES + 1); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (bytesRead > GIT_POINTER_MAX_BYTES) return undefined; + return buffer.subarray(0, bytesRead).toString('utf8'); + } catch { + return undefined; + } finally { + await handle?.close().catch(() => undefined); + } +} + +async function canonicalOrUndefined(path: string): Promise { + try { + return await realpath(path); + } catch { + return undefined; + } +} + +function singleLine(value: string | undefined): string | undefined { + const line = value?.replace(/\r?\n$/, ''); + return line == null || line.length === 0 || /[\r\n\0]/.test(line) ? undefined : line; +} + +/** + * Verify, without running Git, that `/.worktrees/` is a linked + * worktree of that checkout: a real directory whose `.git` file points at + * `/.git/worktrees/`, whose metadata points back at it, and + * whose common directory is the checkout's own `.git`. Neither the lane path + * nor a writable shared Git storage directory may be a symlink, so a forged + * worktree cannot borrow a lane or redirect its Git write grants. + */ +export async function verifyLinkedWorktree( + checkoutRoot: string, + name: string, + checkoutIdentity?: WorkspaceRootIdentity, +): Promise { + if (!isValidLinkedWorktreeName(name)) { + throw rejected('Invalid linked worktree name'); + } + const checkout = await canonicalOrUndefined(checkoutRoot); + if (checkout == null || !(await realDirectory(checkout))) { + throw new WorkspaceToolError('Selected project is unavailable', 'REGISTRATION_INVALID'); + } + if (checkoutIdentity != null && !(await matchesWorkspaceRoot(checkout, checkoutIdentity))) { + throw new WorkspaceToolError('Selected project changed before lane admission', 'REGISTRATION_INVALID'); + } + const commonGitDir = join(checkout, '.git'); + const worktreesDirectory = join(checkout, LINKED_WORKTREE_DIRECTORY); + const root = join(worktreesDirectory, name); + const metadata = join(commonGitDir, 'worktrees', name); + if ( + !(await realDirectory(commonGitDir)) || + !(await realDirectory(worktreesDirectory)) || + !(await realDirectory(root)) || + (await canonicalOrUndefined(root)) !== root || + !(await realDirectory(metadata)) || + (await canonicalOrUndefined(metadata)) !== metadata || + !(await safeSharedGitStorage(commonGitDir)) + ) { + throw rejected(`No linked worktree named ${name} in ${LINKED_WORKTREE_DIRECTORY}`); + } + const dotGit = join(root, '.git'); + const pointer = singleLine(await readPointer(dotGit))?.match(/^gitdir: (.+)$/)?.[1]; + const gitDir = pointer == null ? undefined : isAbsolute(pointer) ? pointer : resolve(root, pointer); + const commonPointer = singleLine(await readPointer(join(metadata, 'commondir'))); + const backPointer = singleLine(await readPointer(join(metadata, 'gitdir'))); + if ( + gitDir == null || + (await canonicalOrUndefined(gitDir)) !== metadata || + commonPointer == null || + (await canonicalOrUndefined(resolve(metadata, commonPointer))) !== commonGitDir || + backPointer == null || + (await canonicalOrUndefined(resolve(metadata, backPointer))) !== dotGit + ) { + throw rejected(`${LINKED_WORKTREE_DIRECTORY}/${name} is not a linked worktree of this project`); + } + let identity: WorkspaceRootIdentity; + try { + identity = await captureWorkspaceRootIdentity(root); + } catch { + throw rejected(`${LINKED_WORKTREE_DIRECTORY}/${name} is unavailable`); + } + const writableGitPaths = [ + ...LINKED_WORKTREE_SHARED_GIT_PATHS.map((path) => join(commonGitDir, path)), + metadata, + ]; + return { root, identity, checkoutRoot: checkout, commonGitDir, writableGitPaths }; +} + +function publicResult(result: WorkspaceToolResult, workspaceId: string): WorkspaceToolResult { + return { ...result, workspaceId }; +} + +/** + * Route requests that name a linked worktree into their own isolated executor. + * Code API schedules each `.worktrees/` as its own lane beneath the + * checkout, so file tools and commands here run confined to that worktree while + * sibling lanes run concurrently. Requests without a worktree pass through. + */ +export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly executors = new Map< + string, + { fingerprint: string; value: Promise } + >(); + private readonly commandRoots = new Map(); + /** Verified lane roots by internal ID, least recently used first. */ + private readonly lanes = new Map(); + + constructor(private readonly options: LinkedWorktreeWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + this.capabilities = { + ...options.delegate.capabilities, + workspaces: options.delegate.capabilities.workspaces.map((workspace) => ({ + ...workspace, + ...(options.sources.has(workspace.id) + ? { workspaceScopes: ['git_linked_worktree' as const] } + : {}), + })), + }; + } + + private async resolveLane( + workspaceId: string, + worktree: string, + workspaceInstanceId: string | undefined, + ): Promise<{ lane: VerifiedLinkedWorktree; source: LinkedWorktreeSource; internalId: string }> { + if (workspaceInstanceId != null) { + throw rejected('Linked worktree lanes are not available inside conversation worktrees'); + } + const source = this.options.sources.get(workspaceId); + if (!source) { + throw rejected('Workspace does not allow linked worktree lanes'); + } + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + let lane: VerifiedLinkedWorktree; + try { + lane = await verifyLinkedWorktree(source.root, worktree, source.identity); + } catch (error) { + await this.release(internalId); + throw error; + } + this.lanes.delete(internalId); + this.lanes.set(internalId, lane.root); + this.options.onResolve?.(workspaceId, lane.root); + await this.releaseIdleLanes(internalId); + return { lane, source, internalId }; + } + + /** Forget a lane's executors and routes; a lane still running a command is kept. */ + private async release(internalId: string): Promise { + const root = this.lanes.get(internalId); + if (root == null) return; + if (this.commandRoots.has(internalId)) { + try { + await this.options.commandPool?.unregisterRoot(internalId); + } catch { + return; + } + this.commandRoots.delete(internalId); + } + this.executors.delete(internalId); + this.lanes.delete(internalId); + this.options.onRelease?.(root); + } + + private async releaseIdleLanes(current: string): Promise { + for (const internalId of [...this.lanes.keys()]) { + if (this.lanes.size <= LINKED_WORKTREE_LANE_LIMIT) return; + if (internalId !== current) await this.release(internalId); + } + } + + private async fileExecutor( + internalId: string, + lane: VerifiedLinkedWorktree, + source: LinkedWorktreeSource, + ): Promise { + const fingerprint = `${lane.identity.path}\0${lane.identity.dev}\0${lane.identity.ino}`; + let cached = this.executors.get(internalId); + if (cached == null || cached.fingerprint !== fingerprint) { + cached = { + fingerprint, + value: LocalWorkspaceTools.create({ + repositoryInstructions: source.repositoryInstructions, + workspaces: [ + { id: internalId, identity: lane.identity, root: lane.root, writable: source.writable }, + ], + }), + }; + this.executors.set(internalId, cached); + } + return await cached.value; + } + + /** Register the lane's command root, replacing it when its identity or writable Git paths changed. */ + private async registerCommandRoot( + internalId: string, + lane: VerifiedLinkedWorktree, + source: LinkedWorktreeSource, + ): Promise { + const pool = this.options.commandPool; + if (!source.command || !pool) { + throw new WorkspaceToolError('Linked worktree commands are unavailable', 'COMMAND_DISABLED'); + } + const fingerprint = JSON.stringify([ + lane.identity.path, + lane.identity.dev, + lane.identity.ino, + lane.writableGitPaths, + ]); + const registered = this.commandRoots.get(internalId); + if (registered !== fingerprint) { + if (registered != null) await pool.unregisterRoot(internalId); + await pool.registerRoot(internalId, { + ...source.command, + workspaceIdentity: lane.identity, + workspaceRoot: lane.root, + linkedWorktree: { + checkoutRoot: lane.checkoutRoot, + commonGitDir: lane.commonGitDir, + writableGitPaths: lane.writableGitPaths, + }, + }); + this.commandRoots.set(internalId, fingerprint); + } + return pool; + } + + async execute(request: WorkspaceToolRequest, signal?: AbortSignal): Promise { + if (request.worktree == null) { + return await this.options.delegate.execute(request, signal); + } + if (request.operation === 'execute_command' && request.environmentAction) { + throw rejected('Environment action was not resolved by this worker'); + } + const { worktree, ...baseRequest } = request; + const { lane, source, internalId } = await this.resolveLane( + request.workspaceId, + worktree, + request.workspaceInstanceId, + ); + const laneRequest = { ...baseRequest, workspaceId: internalId } as WorkspaceToolRequest; + if (request.operation === 'execute_command') { + const pool = await this.registerCommandRoot(internalId, lane, source); + return publicResult( + await pool.execute(laneRequest as WorkspaceExecuteCommandRequest, signal), + request.workspaceId, + ); + } + const executor = await this.fileExecutor(internalId, lane, source); + return publicResult(await executor.execute(laneRequest, signal), request.workspaceId); + } + + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const worktree = request.body.workspace_worktree; + if (worktree == null) { + if (!this.options.programmaticDelegate) { + throw new WorkspaceToolError('Workspace programmatic execution is unavailable', 'COMMAND_DISABLED'); + } + return await this.options.programmaticDelegate.executeProgrammatic(workspaceId, request, signal); + } + const { lane, source, internalId } = await this.resolveLane( + workspaceId, + worktree, + request.body.workspace_instance_id, + ); + const pool = await this.registerCommandRoot(internalId, lane, source); + const { workspace_worktree: _worktree, ...body } = request.body; + return await pool.executeProgrammatic(internalId, { ...request, body }, signal); + } +} diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 5f7eedbb..8abf263c 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -150,6 +150,22 @@ test('executor bootstrap excludes bridge credentials and Node injection variable await sandbox.close(); }); +test('executor forwards the linked worktree policy to the sandbox process', async () => { + const fake = fixture(); + const linkedWorktree = { + checkoutRoot: '/checkout', + commonGitDir: '/checkout/.git', + writableGitPaths: ['/checkout/.git/objects', '/checkout/.git/refs'], + }; + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/checkout/.worktrees/task-a', linkedWorktree }, + fake.fork, + ); + await sandbox.prepare(); + assert.deepEqual(fake.messages[0].options.linkedWorktree, linkedWorktree); + await sandbox.close(); +}); + test('executor forwards the resolved command policy without worker credentials', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index ba411042..77fe8269 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -343,6 +343,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan homeDirectory, shellPath, programmaticFileUpstream, + linkedWorktree, } = this.options; await this.rpc( 'prepare', @@ -356,6 +357,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan homeDirectory, shellPath, programmaticFileUpstream, + linkedWorktree, variables: this.options.maskedEnvironment?.variables, }, }, diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index eab72966..7ecb06f3 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1598,3 +1598,64 @@ test('cleans allocated command state exactly once on every execution exit', asyn } } }); + +test('a linked worktree lane may write only shared Git storage and its own metadata', async t => { + const checkoutRoot = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-lane-'))); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const writableGitPaths = [ + join(commonGitDir, 'objects'), + join(commonGitDir, 'refs'), + join(commonGitDir, 'worktrees', 'task-a'), + ]; + await Promise.all( + [lane, ...writableGitPaths].map(path => mkdir(path, { recursive: true })), + ); + const prepare = async (paths: string[]) => { + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths: paths }, + environment: { PATH: '/usr/bin' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + return fake.config!; + }; + + const config = await prepare(writableGitPaths); + assert.deepEqual(config.filesystem.allowWrite.slice(0, 4), [lane, ...writableGitPaths]); + assert.ok(!config.filesystem.allowWrite.includes(commonGitDir)); + assert.ok(config.filesystem.allowRead?.includes(commonGitDir)); + + const probed = fakeManager(); + const prober = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths }, + environment: { PATH: '/usr/bin' }, + manager: probed.manager, + }); + t.after(() => prober.close()); + const dataDirectory = await prober.createExecutionDirectory(); + await prober.executeProgrammatic(request, dataDirectory, undefined, { probe: true }); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.allowRead?.includes(commonGitDir)); + assert.ok(!probed.customConfigSeenDuringWrap?.filesystem?.allowWrite?.includes(commonGitDir)); + + await assert.rejects( + prepare([commonGitDir]), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', + ); + + const siblingMetadata = join(commonGitDir, 'worktrees', 'task-b'); + await mkdir(siblingMetadata, { recursive: true }); + await rm(join(commonGitDir, 'objects'), { recursive: true }); + await symlink(siblingMetadata, join(commonGitDir, 'objects')); + await assert.rejects( + prepare(writableGitPaths), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', + ); +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 5bf61c50..6ac51acc 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -106,10 +106,15 @@ const TRUSTED_GIT_ENVIRONMENT = { GIT_CONFIG_KEY_3: 'filter.lfs.required', GIT_CONFIG_VALUE_3: 'true', } as const; -const { - GIT_CONFIG_COUNT: TRUSTED_GIT_CONFIG_COUNT, - ...TRUSTED_GIT_CONFIG_ENTRIES -} = TRUSTED_GIT_ENVIRONMENT; +/** Sibling lanes share object storage, so a lane never starts automatic gc or maintenance. */ +const LINKED_WORKTREE_GIT_ENVIRONMENT = { + ...TRUSTED_GIT_ENVIRONMENT, + GIT_CONFIG_COUNT: '6', + GIT_CONFIG_KEY_4: 'gc.auto', + GIT_CONFIG_VALUE_4: '0', + GIT_CONFIG_KEY_5: 'maintenance.auto', + GIT_CONFIG_VALUE_5: 'false', +} as const; const NATIVE_SANDBOX_SCRATCH_PREFIX = 'librechat-code-srt-'; // SRT grants these shared compatibility paths by default. A worker-specific @@ -162,6 +167,15 @@ type SpawnCommand = ( export interface NativeSrtWorkspaceCommandSandboxOptions { workspaceIdentity?: WorkspaceRootIdentity; workspaceRoot: string; + /** Present when `workspaceRoot` is a verified linked worktree lane of a checkout. */ + linkedWorktree?: { + /** The checkout that owns the worktree; trusted as a Git safe directory. */ + checkoutRoot: string; + /** `/.git`: readable, but writable only at `writableGitPaths`. */ + commonGitDir: string; + /** Shared objects and refs plus the lane's own metadata beneath `commonGitDir`. */ + writableGitPaths: string[]; + }; commandPolicy?: NativeSrtCommandPolicy; /** Trusted worker files that must never become workspace-readable or writable. */ protectedPaths?: string[]; @@ -295,9 +309,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private readonly platform: NodeJS.Platform; private initialized?: Promise; private canonicalRoot?: string; + /** A lane's shared Git directory; replay probes of the lane must read it too. */ + private canonicalCommonGitDir?: string; private runtimeConfig?: SandboxRuntimeConfig; private denyReadPaths: string[] = []; private denyWritePaths: string[] = []; + private get gitEnvironment(): + | typeof TRUSTED_GIT_ENVIRONMENT + | typeof LINKED_WORKTREE_GIT_ENVIRONMENT { + return this.options.linkedWorktree + ? LINKED_WORKTREE_GIT_ENVIRONMENT + : TRUSTED_GIT_ENVIRONMENT; + } private scratchDirectory?: string; private scratchHandle?: FileHandle; private execution?: Promise; @@ -424,6 +447,33 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } } + const lane = this.options.linkedWorktree; + const commonGitDir = lane ? await canonicalPath(lane.commonGitDir) : undefined; + const checkoutRoot = lane ? await canonicalPath(lane.checkoutRoot) : undefined; + const writableGitPaths = lane + ? await Promise.all(lane.writableGitPaths.map(canonicalPath)) + : []; + if ( + lane && + (commonGitDir == null || + checkoutRoot == null || + !isWithin(checkoutRoot, commonGitDir) || + !isWithin(checkoutRoot, root) || + isWithin(commonGitDir, home) || + protectedPaths.some(path => isWithin(commonGitDir, path)) || + writableGitPaths.some( + (path, index) => + path !== resolve(lane.writableGitPaths[index]!) || + path === commonGitDir || + !isWithin(commonGitDir, path), + )) + ) { + throw new WorkspaceToolError( + 'Linked worktree Git storage is outside its checkout', + 'REGISTRATION_INVALID', + ); + } + const laneGitPaths = commonGitDir ? [commonGitDir] : []; const canonicalScratchDirectory = await this.createScratchDirectory(sharedScratchPaths); if ( @@ -459,17 +509,22 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], allowRead: [ root, + ...laneGitPaths, ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), ], allowWrite: [ root, + ...writableGitPaths, ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), ], - denyWrite: [...protectedPaths, ...deniedInheritedWritablePaths], + denyWrite: [ + ...protectedPaths, + ...deniedInheritedWritablePaths, + ], allowGitConfig: false, }, credentials: { @@ -520,13 +575,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox allowAppleEvents: false, enableWeakerNestedSandbox: false, enableWeakerNetworkIsolation: false, - git: { safeDirectories: [root] }, + git: { safeDirectories: checkoutRoot ? [root, checkoutRoot] : [root] }, }; await this.manager.initialize( config, unrestrictedNetwork ? async () => true : undefined, ); this.canonicalRoot = root; + this.canonicalCommonGitDir = commonGitDir; this.runtimeConfig = config; this.denyReadPaths = [ home, @@ -758,6 +814,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox allowRead: [ canonicalWorkspaceRoot ?? this.canonicalRoot!, canonicalDataDirectory, + ...(this.canonicalCommonGitDir + ? [this.canonicalCommonGitDir] + : []), ], allowWrite: [ ...(canonicalWorkspaceRoot != null @@ -894,7 +953,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : request.command; wrapped = await this.withTemporaryHostEnvironment( { - ...TRUSTED_GIT_ENVIRONMENT, + ...this.gitEnvironment, ...(credentialEnvironment ?? {}), ...this.scratchSelectorEnvironment(sandboxScratchDirectory), }, @@ -999,10 +1058,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ...wrapped.env, ...this.scratchEnvironment(), ...trustedEnvironment, - ...TRUSTED_GIT_CONFIG_ENTRIES, + ...this.gitEnvironment, GIT_CONFIG_COUNT: wrapped.env.GIT_CONFIG_COUNT ?? - TRUSTED_GIT_CONFIG_COUNT, + this.gitEnvironment.GIT_CONFIG_COUNT, GIT_CONFIG_GLOBAL: this.platform === 'win32' ? 'NUL' diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 5c26c462..729b20e2 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -11,6 +11,8 @@ import { isWorkspaceToolRequest, isWorkspaceToolResult, workspaceIsolationKey, + workspaceIsolationKeysConflict, + workspaceIsolationParent, } from './protocol.js'; import type { WorkspaceEditFileRequest, @@ -816,3 +818,62 @@ test('bridge artifact policy and media types match the hardened gateway contract assert.equal(bridgeArtifactMediaType('reports/result.json'), 'application/json'); assert.equal(bridgeArtifactMediaType('Dockerfile'), 'application/octet-stream'); }); + +test('linked-worktree lanes nest beneath their checkout key and conflict only with it', () => { + const instanceId = 'a'.repeat(64); + const lane = workspaceIsolationKey('repo', undefined, 'task-a'); + const sibling = workspaceIsolationKey('repo', undefined, 'task-b'); + const nested = workspaceIsolationKey('repo', instanceId, 'task-a'); + assert.notEqual(lane, workspaceIsolationKey('repo')); + assert.notEqual(lane, workspaceIsolationKey('repo\0task-a')); + assert.notEqual(nested, lane); + assert.equal(workspaceIsolationParent(lane), 'repo'); + assert.equal(workspaceIsolationParent(nested), workspaceIsolationKey('repo', instanceId)); + assert.equal(workspaceIsolationParent('repo'), undefined); + assert.equal(workspaceIsolationParent(workspaceIsolationKey('repo', instanceId)), undefined); + assert.equal(workspaceIsolationKeysConflict(lane, 'repo'), true); + assert.equal(workspaceIsolationKeysConflict('repo', lane), true); + assert.equal(workspaceIsolationKeysConflict(lane, lane), true); + assert.equal(workspaceIsolationKeysConflict(lane, sibling), false); + assert.equal(workspaceIsolationKeysConflict(nested, 'repo'), false); + assert.equal(workspaceIsolationKeysConflict(nested, workspaceIsolationKey('repo', instanceId)), true); +}); + +test('workspace requests accept only a single safe worktree name', () => { + const base = { protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }; + for (const worktree of ['task-a', 'fix_16464', 'v2.0']) { + assert.equal(isWorkspaceToolRequest({ ...base, worktree }), true, worktree); + } + for (const worktree of ['', '.hidden', '..', 'a/b', 'a\\b', 'task.lock', 'x'.repeat(129), 7, null]) { + assert.equal(isWorkspaceToolRequest({ ...base, worktree }), false, String(worktree)); + } + const programmatic = (workspace_worktree: string) => ({ + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'true' }], + workspace_worktree, + }, + }); + assert.equal(isBridgeWorkspaceProgrammaticRequest(programmatic('task-a')), true); + assert.equal(isBridgeWorkspaceProgrammaticRequest(programmatic('../x')), false); +}); + +test('workspace capabilities advertise linked-worktree scopes exactly', () => { + const capabilities = (workspaceScopes: unknown) => ({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes }], + }, + }); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['git_linked_worktree'])), true); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['git_worktree'])), false); + assert.equal(isValidBridgeWorkerCapabilities(capabilities([])), false); + assert.equal(isValidBridgeWorkerCapabilities(capabilities('git_linked_worktree')), false); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 7657fc9c..b23ad327 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -259,14 +259,51 @@ export function bridgeArtifactMediaType(name: string): string { export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; -/** Collision-free identity shared by scheduling and worker quarantine state. */ +/** One path segment naming a linked worktree at `/.worktrees/`. */ +export const BRIDGE_LINKED_WORKTREE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +export function isValidLinkedWorktreeName(value: unknown): value is string { + return ( + typeof value === 'string' && + BRIDGE_LINKED_WORKTREE_NAME_PATTERN.test(value) && + !value.endsWith('.lock') + ); +} + +const LINKED_WORKTREE_KEY_PREFIX = '\0linked-worktree\0'; + +/** Collision-free identity shared by scheduling and worker quarantine state. + * A linked worktree lane nests beneath the key of the checkout that owns it. */ export function workspaceIsolationKey( workspaceId: string, instanceId?: string, + worktree?: string, ): string { - return instanceId === undefined + const base = instanceId === undefined ? workspaceId : `\0git-worktree\0${workspaceId}\0${instanceId}`; + return worktree === undefined + ? base + : `${LINKED_WORKTREE_KEY_PREFIX}${base}\0${worktree}`; +} + +/** The checkout key a linked-worktree lane nests beneath, or undefined for a root key. + * Scheduling treats a lane and its parent as conflicting; sibling lanes do not. */ +export function workspaceIsolationParent(key: string): string | undefined { + if (!key.startsWith(LINKED_WORKTREE_KEY_PREFIX)) return undefined; + const separator = key.lastIndexOf('\0'); + return separator <= LINKED_WORKTREE_KEY_PREFIX.length + ? undefined + : key.slice(LINKED_WORKTREE_KEY_PREFIX.length, separator); +} + +/** Two isolation keys may not execute concurrently when either nests the other. */ +export function workspaceIsolationKeysConflict(left: string, right: string): boolean { + return ( + left === right || + workspaceIsolationParent(left) === right || + workspaceIsolationParent(right) === left + ); } export type BridgeWorkspaceToolOperation = @@ -292,6 +329,9 @@ export interface BridgeWorkspaceDescriptor { operations?: BridgeWorkspaceToolOperation[]; /** Worker-owned isolation schemes available beneath this selected root. */ workspaceInstances?: ['git_worktree']; + /** Scheduling scopes available beneath this root. `git_linked_worktree` gives each + * verified `.worktrees/` linked worktree its own lane. */ + workspaceScopes?: ['git_linked_worktree']; environment?: { fingerprint: string; repo?: string; @@ -321,6 +361,8 @@ export interface WorkspaceReadFileRequest { operation: 'read_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; startLine?: number; maxLines?: number; @@ -364,6 +406,8 @@ export interface WorkspaceSearchTextRequest { operation: 'search_text'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; query: string; path?: string; maxResults?: number; @@ -389,6 +433,8 @@ export interface WorkspaceListFilesRequest { operation: 'list_files'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path?: string; maxResults?: number; /** Continue strictly after this canonical path from a previous page. */ @@ -410,6 +456,8 @@ export interface WorkspaceWriteFileRequest { operation: 'write_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; content: string; /** False requires an atomic create and refuses to replace an existing file. */ @@ -430,6 +478,8 @@ interface WorkspaceEditFileRequestBase { operation: 'edit_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; /** Refuses the mutation unless current file bytes match this preview revision. */ expectedBaseSha256?: string; @@ -475,6 +525,8 @@ interface WorkspacePreviewEditRequestBase { operation: 'preview_edit'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; } @@ -513,6 +565,8 @@ export interface WorkspaceExecuteCommandRequest { operation: 'execute_command'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; /** Shell source evaluated only inside the selected sandbox runtime. */ command: string; /** Portable path relative to the workspace root; defaults to '.'. */ @@ -558,6 +612,7 @@ const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'startLine', 'maxLines', @@ -567,6 +622,7 @@ const WORKSPACE_SEARCH_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'query', 'path', 'maxResults', @@ -576,6 +632,7 @@ const WORKSPACE_LIST_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'maxResults', 'afterPath', @@ -585,6 +642,7 @@ const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'content', 'overwrite', @@ -594,6 +652,7 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'oldText', 'newText', @@ -605,6 +664,7 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'oldText', 'newText', @@ -617,6 +677,7 @@ const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'command', 'cwd', 'timeoutMs', @@ -730,6 +791,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; /** Workspace isolation schemes this Code API understands and can route. */ supportedWorkspaceInstanceTypes?: ['git_worktree']; + /** Scheduling scopes this Code API can admit as independent lanes. */ + supportedWorkspaceScopes?: ['git_linked_worktree']; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -803,6 +866,8 @@ export interface BridgeWorkspaceProgrammaticBody { language: 'bash'; version: string; workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; /** Stable identity shared by every replay iteration of one execution. */ execution_id?: string; /** Declared replay tools; zero allows the worker to skip the probe pass. */ @@ -970,6 +1035,8 @@ export function isBridgeWorkspaceProgrammaticRequest( (body.workspace_instance_id !== undefined && (typeof body.workspace_instance_id !== 'string' || !/^[a-f0-9]{64}$/.test(body.workspace_instance_id))) || + (body.workspace_worktree !== undefined && + !isValidLinkedWorktreeName(body.workspace_worktree)) || (body.execution_id !== undefined && (typeof body.execution_id !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || @@ -1222,7 +1289,8 @@ export function isWorkspaceToolRequest( !isValidBridgeWorkerId(request.workspaceId) || (request.workspaceInstanceId !== undefined && (typeof request.workspaceInstanceId !== 'string' || - !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) + !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) || + (request.worktree !== undefined && !isValidLinkedWorktreeName(request.worktree)) ) { return false; } @@ -1676,6 +1744,7 @@ export function isValidBridgeWorkspaceToolCapabilities( key !== 'name' && key !== 'operations' && key !== 'workspaceInstances' && + key !== 'workspaceScopes' && key !== 'instructions' && key !== 'environment', ) || @@ -1686,6 +1755,10 @@ export function isValidBridgeWorkspaceToolCapabilities( (!Array.isArray(descriptor.workspaceInstances) || descriptor.workspaceInstances.length !== 1 || descriptor.workspaceInstances[0] !== 'git_worktree')) || + (descriptor.workspaceScopes !== undefined && + (!Array.isArray(descriptor.workspaceScopes) || + descriptor.workspaceScopes.length !== 1 || + descriptor.workspaceScopes[0] !== 'git_linked_worktree')) || (descriptor.instructions !== undefined && (!Array.isArray(descriptor.instructions) || descriptor.instructions.length > 1 || !descriptor.instructions.every(isRepositoryInstructionDescriptor))) || (descriptor.environment !== undefined && !isValidCodeEnvironmentDescriptor(descriptor.environment)) || diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index fda6c659..72052974 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -377,3 +377,204 @@ test('programmatic work on an independent workspace bypasses another root cleanu 'previous', ); }); + +test('sibling linked-worktree lanes run past each other, but a lane and its checkout wait for one another', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + const internals = worker as unknown as { + activeWorkspaceAssignments: Map }>; + executeOwned: (assignment: BridgeAssignment) => Promise; + }; + const executed: string[] = []; + internals.executeOwned = async (assignment) => { + executed.push(assignment.assignmentId); + }; + const assignment = (assignmentId: string, worktree?: string) => + ({ + assignmentId, + executionKind: 'workspace_tool', + remainingMs: 1_000, + request: { + protocolVersion: 1, + workspaceId: 'repo', + operation: 'read_file', + path: 'README.md', + ...(worktree ? { worktree } : {}), + }, + }) as BridgeAssignment; + let releaseLane!: () => void; + internals.activeWorkspaceAssignments.set(workspaceIsolationKey('repo', undefined, 'task-a'), { + id: 'lane-a', + done: new Promise((resolve) => { + releaseLane = resolve; + }), + }); + + await worker.executeAndSettle(assignment('lane-b', 'task-b')); + assert.deepEqual(executed, ['lane-b']); + + const checkout = worker.executeAndSettle(assignment('checkout')); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.deepEqual(executed, ['lane-b']); + internals.activeWorkspaceAssignments.delete(workspaceIsolationKey('repo', undefined, 'task-a')); + releaseLane(); + await checkout; + assert.deepEqual(executed, ['lane-b', 'checkout']); + + let releaseCheckout!: () => void; + internals.activeWorkspaceAssignments.set(workspaceIsolationKey('repo'), { + id: 'root', + done: new Promise((resolve) => { + releaseCheckout = resolve; + }), + }); + const lane = worker.executeAndSettle(assignment('lane-c', 'task-c')); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.deepEqual(executed, ['lane-b', 'checkout']); + internals.activeWorkspaceAssignments.delete(workspaceIsolationKey('repo')); + releaseCheckout(); + await lane; + assert.deepEqual(executed, ['lane-b', 'checkout', 'lane-c']); +}); + +test('a linked worktree lane fence is reset through its own guard and isolation key', async () => { + const bodies: Record[] = []; + const guarded: string[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + incarnationId: 'incarnation-reset', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 2, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes: ['git_linked_worktree'] }, { id: 'plain' }], + }, + }, + workspaceQuarantines: new Map( + ['repo', 'plain'].map((id) => [ + id, + { + assertAvailable: async () => undefined, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }, + ]), + ), + workspaceTools: { + capabilities: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes: ['git_linked_worktree'] }, { id: 'plain' }], + }, + async execute() { + throw new Error('must not execute'); + }, + }, + linkedWorktreeQuarantineResolver: (workspaceId, worktree) => { + guarded.push(`${workspaceId}/${worktree}`); + return { + assertAvailable: async () => undefined, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }; + }, + fetchImpl: async (url, init) => { + assert.ok(new URL(String(url)).pathname.endsWith('/workspaces/reset')); + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ protocolVersion: 1, reset: true }); + }, + }); + + await worker.resetNativeWorkspace('repo', undefined, undefined, 'task-a'); + + assert.deepEqual(guarded, ['repo/task-a']); + assert.equal( + bodies[0]?.runtimeSessionId, + `native-workspace:${workspaceIsolationKey('repo', undefined, 'task-a')}`, + ); + await assert.rejects( + worker.resetNativeWorkspace('plain', undefined, undefined, 'task-a'), + /worktree lanes/, + ); + await assert.rejects( + worker.resetNativeWorkspace('repo', undefined, 'a'.repeat(64), 'task-a'), + /worktree lanes/, + ); + await assert.rejects( + worker.resetNativeWorkspace('repo', undefined, undefined, '../task-a'), + /worktree lanes/, + ); + assert.equal(bodies.length, 1); +}); + +test('a checkout waits on quarantined linked worktrees beneath it', async () => { + const quarantinedLanes = new Set(['task-b']); + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + linkedWorktreeQuarantineResolver: (_workspaceId, worktree) => ({ + assertAvailable: async () => { + if (quarantinedLanes.has(worktree)) throw new Error(`lane ${worktree} is quarantined`); + }, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }), + linkedWorktreeNames: async () => ['task-a', 'task-b'], + }); + const internals = worker as unknown as { + quarantinedWorkspaces: Set; + assertLaneFamilyAvailable: (key: string, assignment: BridgeAssignment) => Promise; + }; + const assignment = (worktree?: string) => + ({ + assignmentId: 'probe', + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + workspaceId: 'repo', + operation: 'read_file', + path: 'README.md', + ...(worktree ? { worktree } : {}), + }, + }) as BridgeAssignment; + const check = (worktree?: string) => + internals.assertLaneFamilyAvailable( + workspaceIsolationKey('repo', undefined, worktree), + assignment(worktree), + ); + + await assert.rejects(check(), /task-b is quarantined/); + await check('task-a'); + + quarantinedLanes.clear(); + await check(); + internals.quarantinedWorkspaces.add(workspaceIsolationKey('repo', undefined, 'task-c')); + await assert.rejects(check(), /linked worktree/); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 2a5896fd..37223b34 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -6,8 +6,11 @@ import { BridgeProtocolError, bridgeWorkerPath, isBridgeWorkspaceProgrammaticRequest, + isValidLinkedWorktreeName, isWorkspaceToolResult, workspaceIsolationKey, + workspaceIsolationKeysConflict, + workspaceIsolationParent, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; @@ -62,6 +65,15 @@ export interface BridgeWorkerOptions { ) => | WorkspaceMutationQuarantine | Promise; + /** Resolve a durable guard for a linked-worktree lane beneath a registered root. */ + linkedWorktreeQuarantineResolver?: ( + workspaceId: string, + worktree: string, + ) => + | WorkspaceMutationQuarantine + | Promise; + /** Names of the linked worktrees currently beneath a registered root. */ + linkedWorktreeNames?: (workspaceId: string) => Promise; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; @@ -223,7 +235,13 @@ function workspaceCapabilitiesMatch( (instanceType, instanceIndex) => instanceType === executor.workspaces[index]?.workspaceInstances?.[instanceIndex], - ) ?? executor.workspaces[index]?.workspaceInstances == null), + ) ?? executor.workspaces[index]?.workspaceInstances == null) && + workspace.workspaceScopes?.length === + executor.workspaces[index]?.workspaceScopes?.length && + (workspace.workspaceScopes?.every( + (scope, scopeIndex) => + scope === executor.workspaces[index]?.workspaceScopes?.[scopeIndex], + ) ?? executor.workspaces[index]?.workspaceScopes == null), ) ); } @@ -239,7 +257,9 @@ function registrationCompatibleCapabilities( ) && workspaceTools.workspaces.every( (workspace) => - workspace.operations == null && workspace.workspaceInstances == null, + workspace.operations == null && + workspace.workspaceInstances == null && + workspace.workspaceScopes == null, )) ) { return capabilities; @@ -263,6 +283,7 @@ function registrationCompatibleCapabilities( const { operations: _operations, workspaceInstances: _workspaceInstances, + workspaceScopes: _workspaceScopes, ...compatibleWorkspace } = workspace; return [{ ...compatibleWorkspace, ...(workspace.environment ? { @@ -355,6 +376,10 @@ function supportedWorkspaceCapabilities( ) ? { workspaceInstances: workspace.workspaceInstances } : { workspaceInstances: undefined }), + ...(workspace.workspaceScopes != null && + registration.supportedWorkspaceScopes?.includes('git_linked_worktree') + ? { workspaceScopes: workspace.workspaceScopes } + : { workspaceScopes: undefined }), ...(workspace.operations ? { operations: workspaceOperations } : {}), ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { environment: { ...workspace.environment, actions: [] }, @@ -506,7 +531,8 @@ export class BridgeWorker { ) === true && options.workspaceMutationQuarantine == null && options.workspaceQuarantines == null && - options.workspaceQuarantineResolver == null + options.workspaceQuarantineResolver == null && + options.linkedWorktreeQuarantineResolver == null ) { throw new BridgeProtocolError( 'Workspace mutation capabilities require durable quarantine storage', @@ -522,6 +548,17 @@ export class BridgeWorker { 'Workspace instance capabilities require a durable quarantine resolver', ); } + if ( + options.capabilities.workspaceTools?.workspaces.some( + (root) => (root.workspaceScopes?.length ?? 0) > 0, + ) && + (options.linkedWorktreeQuarantineResolver == null || + (options.capabilities.workspaceLeaseSlots ?? 1) < 2) + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes require a durable quarantine resolver and at least two lease slots', + ); + } if ((options.capabilities.workspaceLeaseSlots ?? 1) > 1) { if ( options.capabilities.requiresReadyConfirmation !== true || @@ -823,18 +860,33 @@ export class BridgeWorker { workspaceId: string, signal?: AbortSignal, workspaceInstanceId?: string, + worktree?: string, ): Promise { const workspace = this.options.capabilities.workspaceTools?.workspaces.find( (root) => root.id === workspaceId, ); - const key = workspaceIsolationKey(workspaceId, workspaceInstanceId); - const guard = - workspaceInstanceId == null - ? this.options.workspaceQuarantines?.get(workspaceId) - : await this.options.workspaceQuarantineResolver?.( - workspaceId, - workspaceInstanceId, - ); + if ( + worktree != null && + (workspaceInstanceId != null || + !isValidLinkedWorktreeName(worktree) || + workspace?.workspaceScopes?.includes('git_linked_worktree') !== true) + ) { + throw new BridgeProtocolError( + 'Linked worktree reset requires a registered checkout with worktree lanes', + ); + } + const key = workspaceIsolationKey(workspaceId, workspaceInstanceId, worktree); + let guard: WorkspaceMutationQuarantine | undefined; + if (worktree != null) { + guard = await this.options.linkedWorktreeQuarantineResolver?.(workspaceId, worktree); + } else if (workspaceInstanceId != null) { + guard = await this.options.workspaceQuarantineResolver?.( + workspaceId, + workspaceInstanceId, + ); + } else { + guard = this.options.workspaceQuarantines?.get(workspaceId); + } if ( !guard || this.activeWorkspaceAssignments.size > 0 || @@ -1313,8 +1365,11 @@ export class BridgeWorker { ): Promise { const root = this.assignmentWorkspaceId(assignment); const waitingAt = Date.now(); - while (root != null && this.activeWorkspaceAssignments.has(root)) { - const active = this.activeWorkspaceAssignments.get(root)!; + for ( + let active = root == null ? undefined : this.conflictingActiveAssignment(root); + root != null && active != null; + active = this.conflictingActiveAssignment(root) + ) { if (active.id === assignment.assignmentId) throw new BridgeProtocolError( 'Code API replayed an active workspace assignment', @@ -1386,6 +1441,50 @@ export class BridgeWorker { } } + /** An active assignment on the same key, the key's parent checkout, or a lane beneath it. */ + private conflictingActiveAssignment( + key: string, + ): { id: string; done: Promise } | undefined { + for (const [activeKey, active] of this.activeWorkspaceAssignments) { + if (workspaceIsolationKeysConflict(activeKey, key)) return active; + } + return undefined; + } + + /** A lane may not run while the checkout it belongs to is quarantined. */ + /** + * A lane may not start while its checkout is quarantined, and a checkout may + * not start while any linked worktree beneath it is: root commands can reach + * `.worktrees/*`, including `git worktree remove`. + */ + private async assertLaneFamilyAvailable( + workspaceKey: string, + assignment: BridgeAssignment, + ): Promise { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + const parent = workspaceIsolationParent(workspaceKey); + if (parent != null) { + if (this.quarantinedWorkspaces.has(parent)) { + throw new Error('Parent workspace requires an explicit quarantine reset'); + } + if (workspaceId != null && parent === workspaceId) { + await this.options.workspaceQuarantines?.get(workspaceId)?.assertAvailable(); + } + return; + } + const resolveLaneGuard = this.options.linkedWorktreeQuarantineResolver; + if (workspaceId == null || workspaceKey !== workspaceId || resolveLaneGuard == null) return; + for (const key of this.quarantinedWorkspaces) { + if (workspaceIsolationParent(key) === workspaceKey) { + throw new Error('A linked worktree in this workspace requires an explicit quarantine reset'); + } + } + const names = (await this.options.linkedWorktreeNames?.(workspaceId)) ?? []; + await Promise.all( + names.map(async (name) => (await resolveLaneGuard(workspaceId, name)).assertAvailable()), + ); + } + private workspaceGuard( assignment: BridgeAssignment, ): @@ -1394,6 +1493,10 @@ export class BridgeWorker { | undefined { const workspaceId = this.assignmentBaseWorkspaceId(assignment); const instanceId = this.assignmentWorkspaceInstanceId(assignment); + const worktree = this.assignmentWorktree(assignment); + if (workspaceId != null && worktree != null) { + return this.options.linkedWorktreeQuarantineResolver?.(workspaceId, worktree); + } if (workspaceId != null && instanceId != null) { return this.options.workspaceQuarantineResolver?.( workspaceId, @@ -1412,7 +1515,27 @@ export class BridgeWorker { const workspaceId = this.assignmentBaseWorkspaceId(assignment); if (workspaceId == null) return undefined; const instanceId = this.assignmentWorkspaceInstanceId(assignment); - return workspaceIsolationKey(workspaceId, instanceId); + return workspaceIsolationKey( + workspaceId, + instanceId, + this.assignmentWorktree(assignment), + ); + } + + private assignmentWorktree(assignment: BridgeAssignment): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.worktree; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + return assignment.request.body.workspace_worktree; + } + return undefined; } private assignmentBaseWorkspaceId( @@ -1580,9 +1703,11 @@ export class BridgeWorker { } if ( this.options.workspaceQuarantines != null || - this.options.workspaceQuarantineResolver != null + this.options.workspaceQuarantineResolver != null || + this.options.linkedWorktreeQuarantineResolver != null ) { await guard?.assertAvailable(); + await this.assertLaneFamilyAvailable(workspaceKey, assignment); } } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1615,6 +1740,14 @@ export class BridgeWorker { 'Workspace instance type is not advertised', ); } + if ( + workspaceRequest.worktree != null && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes are not advertised for workspace', + ); + } if ( workspace.operations != null && !workspace.operations.includes(workspaceRequest.operation) @@ -1739,9 +1872,11 @@ export class BridgeWorker { } if ( this.options.workspaceQuarantines != null || - this.options.workspaceQuarantineResolver != null + this.options.workspaceQuarantineResolver != null || + this.options.linkedWorktreeQuarantineResolver != null ) { await guard?.assertAvailable(); + await this.assertLaneFamilyAvailable(workspaceKey, assignment); } } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1772,6 +1907,14 @@ export class BridgeWorker { 'Workspace instance type is not advertised', ); } + if ( + assignment.request.body.workspace_worktree != null && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes are not advertised for workspace', + ); + } this.mutationGuardArmed = true; try { this.armedWorkspaces.add(workspaceKey); diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index 1de68818..52743bae 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -147,6 +147,47 @@ test('CLI requires concurrent native slots for conversation worktrees', async (t assert.match(result.stderr, /at least two workspace lease slots/i); }); +test('CLI requires concurrent native slots and no conversation worktrees for linked worktree lanes', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-lanes-')); + const workspaceRoot = join(root, 'workspace'); + await mkdir(workspaceRoot); + t.after(() => rm(root, { recursive: true, force: true })); + const run = (...extra: string[]) => + spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspaceRoot, + '--allow-workspace-writes', + '--allow-workspace-commands', + '--linked-worktree-lanes', + ...extra, + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }, + }, + ); + const serial = run(); + assert.notEqual(serial.status, 0); + assert.match(serial.stderr, /Linked worktree lanes require .*at least two workspace lease slots/i); + const combined = run( + '--workspace-lease-slots', + '2', + '--conversation-worktree-root', + join(root, 'conversations'), + ); + assert.notEqual(combined.status, 0); + assert.match(combined.stderr, /cannot be combined with conversation worktrees/i); +}); + test('CLI advertises explicitly enabled writes without exposing the workspace root', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); diff --git a/service/src/bridge/linked-worktree.test.ts b/service/src/bridge/linked-worktree.test.ts new file mode 100644 index 00000000..5a286726 --- /dev/null +++ b/service/src/bridge/linked-worktree.test.ts @@ -0,0 +1,197 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment } from './store'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; + +const redisUrl = process.env.BRIDGE_TEST_REDIS_URL; +const incarnationId = 'lane-incarnation'; + +function fenceKey(workerId: string, fence: string): string { + const hash = createHash('sha256').update(fence).digest('hex'); + return `codeapi:bridge:v1:worker:${encodeURIComponent(workerId)}:workspace:${hash}:quarantined`; +} + +async function withLaneWorker( + run: (context: { + store: RedisBridgeStore; + redis: Redis; + workerId: string; + dispatch: ( + request: Partial, + budgetMs?: number, + ) => Promise; + lease: (slot: number) => Promise; + settle: (assignment: CodeBridgeAssignment) => Promise; + }) => Promise, + scopes = true, +): Promise { + const redis = new Redis(redisUrl!); + const store = new RedisBridgeStore(redis, 60, 1000, 3); + const workerId = `lanes-${randomUUID()}`; + const controller = new AbortController(); + const pending: Promise[] = []; + try { + const generation = await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 3, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [ + { + id: 'repo', + ...(scopes ? { workspaceScopes: ['git_linked_worktree' as const] } : {}), + }, + ], + }, + }, + }); + await store.confirmReady(workerId, incarnationId, generation); + await run({ + store, + redis, + workerId, + dispatch(request, budgetMs = 3000) { + const result = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + budgetMs, + executionTimeoutMs: 5000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + path: 'probe', + ...request, + } as WorkspaceToolRequest, + }); + void result.catch(() => undefined); + pending.push(result); + return result; + }, + lease: (slot) => store.lease(workerId, incarnationId, 1000, undefined, undefined, slot), + async settle(assignment) { + await store.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'probe complete', + }); + await store.confirmWorkspaceCleanup(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'local cleanup confirmed', + }); + }, + }); + } finally { + controller.abort(); + await Promise.allSettled(pending); + await redis.quit(); + } +} + +test.skipIf(!redisUrl)( + 'sibling linked-worktree lanes run together while their checkout waits for both', + async () => { + await withLaneWorker(async ({ dispatch, lease, settle }) => { + const laneA = dispatch({ worktree: 'task-a' }); + const laneB = dispatch({ worktree: 'task-b' }); + const first = await lease(0); + const second = await lease(1); + expect( + [first?.request, second?.request].map( + (request) => (request as WorkspaceToolRequest).worktree, + ).sort(), + ).toEqual(['task-a', 'task-b']); + + const checkout = dispatch({}, 400); + expect(await lease(2)).toBeUndefined(); + await expect(checkout).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); + + await settle(first!); + await settle(second!); + await Promise.allSettled([laneA, laneB]); + }); + }, +); + +test.skipIf(!redisUrl)('a busy checkout holds every lane beneath it', async () => { + await withLaneWorker(async ({ dispatch, lease, settle }) => { + const checkout = dispatch({}); + const held = await lease(0); + expect((held?.request as WorkspaceToolRequest).worktree).toBeUndefined(); + + const lane = dispatch({ worktree: 'task-a' }); + expect(await lease(1)).toBeUndefined(); + + await settle(held!); + await Promise.allSettled([checkout]); + const admitted = await lease(0); + expect((admitted?.request as WorkspaceToolRequest).worktree).toBe('task-a'); + await settle(admitted!); + await Promise.allSettled([lane]); + }); +}); + +test.skipIf(!redisUrl)('a lane may not start while its checkout is quarantined', async () => { + await withLaneWorker(async ({ redis, workerId, dispatch }) => { + await redis.set(fenceKey(workerId, 'native-workspace:repo'), 'quarantined:earlier'); + await expect(dispatch({ worktree: 'task-a' })).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + expect(await redis.exists(fenceKey(workerId, 'native-workspace:\0linked-worktree\0repo\0task-a'))).toBe(0); + }); +}); + +test.skipIf(!redisUrl)('a checkout is refused while a lane beneath it keeps a stuck fence', async () => { + await withLaneWorker(async ({ redis, workerId, dispatch, lease, settle }) => { + const settleNext = async (request: Partial) => { + const pending = dispatch(request); + const assignment = await lease(0); + expect((assignment?.request as WorkspaceToolRequest).worktree).toBe(request.worktree); + await settle(assignment!); + await Promise.allSettled([pending]); + }; + await settleNext({ worktree: 'task-a' }); + await settleNext({}); + + // A lane whose cleanup was never acknowledged keeps its fence after its slot is released. + await settleNext({ worktree: 'task-a' }); + await redis.set( + fenceKey(workerId, 'native-workspace:\0linked-worktree\0repo\0task-a'), + 'quarantined:cleanup-unacknowledged', + ); + + await expect(dispatch({})).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + }); +}); + +test.skipIf(!redisUrl)('a lane is refused unless the worker advertises linked-worktree scopes', async () => { + await withLaneWorker(async ({ dispatch }) => { + await expect(dispatch({ worktree: 'task-a' })).rejects.toMatchObject({ + code: 'WORKER_MISMATCH', + }); + }, false); +}); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index b3d549b1..962c6242 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -539,6 +539,7 @@ router.post( supportedWorkspaceListFileFeatures: ['after_path'], supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], + supportedWorkspaceScopes: ['git_linked_worktree'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/slots.test.ts b/service/src/bridge/slots.test.ts index dfe89495..1efe3b97 100644 --- a/service/src/bridge/slots.test.ts +++ b/service/src/bridge/slots.test.ts @@ -3,6 +3,7 @@ import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { BridgeAdmissionQueue } from './admission'; import { BridgeWorkspaceSlots } from './slots'; +import { workspaceIsolationKey } from '../../../packages/code/src/protocol'; const redis = new RedisMock() as unknown as Redis; const admission = new BridgeAdmissionQueue(redis); @@ -13,20 +14,27 @@ const prefix = `codeapi:bridge:v1:worker:${workerId}`; afterEach(async () => { await redis.flushall(); }); -async function enqueue(assignmentId: string, workspaceId: string) { +async function enqueue(assignmentId: string, workspaceId: string, capacity = 2) { await redis.set(`${prefix}:incarnation`, incarnationId); - await redis.set(`${prefix}:workspace-slot-capacity`, '2'); + await redis.set(`${prefix}:workspace-slot-capacity`, String(capacity)); await admission.enter(workerId, assignmentId, Date.now() + 5000, workspaceId); return { workerId, incarnationId, assignmentId, workspaceId, - capacity: 2, + capacity, expiresAtMs: Date.now() + 10000, }; } +async function finish(assignmentId: string) { + await slots.release(workerId, incarnationId, assignmentId); + await admission.leave(workerId, assignmentId); +} + +const lane = (name: string, parent = 'repo') => workspaceIsolationKey(parent, undefined, name); + test('slots admit independent workspaces, skip a busy root, and bound capacity', async () => { const a = await enqueue('a', 'root-a'); const a2 = await enqueue('a2', 'root-a'); @@ -91,3 +99,51 @@ test('releasing a long slot shortens the aggregate expiry to remaining work', as await slots.release(workerId, incarnationId, 'a'); expect(await redis.pttl(`${prefix}:lock`)).toBeLessThanOrEqual(3000); }); + +test('sibling linked-worktree lanes share their checkout, which waits for both', async () => { + const a = await enqueue('a', lane('task-a'), 3); + const b = await enqueue('b', lane('task-b'), 3); + const root = await enqueue('root', 'repo', 3); + expect(await slots.reserve(a)).toBe(0); + expect(await slots.reserve(b)).toBe(1); + expect(await slots.reserve(root)).toBeUndefined(); + await finish('a'); + expect(await slots.reserve(root)).toBeUndefined(); + await finish('b'); + expect(await slots.reserve(root)).toBe(0); +}); + +test('a busy checkout holds every lane beneath it but not other checkouts', async () => { + const root = await enqueue('root', 'repo', 3); + const a = await enqueue('a', lane('task-a'), 3); + const other = await enqueue('other', lane('task-a', 'other-repo'), 3); + expect(await slots.reserve(root)).toBe(0); + expect(await slots.reserve(a)).toBeUndefined(); + expect(await slots.reserve(other)).toBe(1); + await finish('root'); + expect(await slots.reserve(a)).toBe(0); +}); + +test('a waiting checkout holds back newer lanes so root work cannot starve', async () => { + const a = await enqueue('a', lane('task-a'), 3); + expect(await slots.reserve(a)).toBe(0); + const root = await enqueue('root', 'repo', 3); + const b = await enqueue('b', lane('task-b'), 3); + const other = await enqueue('other', 'other-repo', 3); + expect(await slots.reserve(root)).toBeUndefined(); + expect(await slots.reserve(b)).toBeUndefined(); + expect(await slots.reserve(other)).toBe(1); + await finish('a'); + expect(await slots.reserve(b)).toBeUndefined(); + expect(await slots.reserve(root)).toBe(0); +}); + +test('lanes nest beneath a conversation checkout, not the source root', async () => { + const instance = workspaceIsolationKey('repo', 'c'.repeat(64)); + const conversation = await enqueue('conversation', instance, 3); + const nested = await enqueue('nested', workspaceIsolationKey('repo', 'c'.repeat(64), 'task-a'), 3); + const source = await enqueue('source', lane('task-a'), 3); + expect(await slots.reserve(conversation)).toBe(0); + expect(await slots.reserve(nested)).toBeUndefined(); + expect(await slots.reserve(source)).toBe(1); +}); diff --git a/service/src/bridge/slots.ts b/service/src/bridge/slots.ts index 9c9e7d3b..81f60ebe 100644 --- a/service/src/bridge/slots.ts +++ b/service/src/bridge/slots.ts @@ -6,6 +6,12 @@ export const MAX_WORKSPACE_LEASE_SLOTS = 8; /** Reservations share the legacy admission queue and aggregate worker lock. * Thus a serial dispatcher or replacement incarnation cannot race active slots. * Workspace mutation uncertainty is fenced separately by the assignment store. + * + * Keys are hierarchical: a linked-worktree lane (`\0linked-worktree\0\0`) + * conflicts with itself and with its parent checkout, and a checkout conflicts + * with every lane beneath it. Sibling lanes run concurrently. A waiting checkout + * request also holds back newer lanes beneath it, so root operations such as + * `git worktree add` cannot be starved by a steady stream of lane work. */ export class BridgeWorkspaceSlots { constructor(private readonly redis: Redis) {} @@ -47,10 +53,26 @@ export class BridgeWorkspaceSlots { [ "if redis.call('GET', KEYS[4]) ~= ARGV[1] then return -2 end", "if (redis.call('GET', KEYS[8]) or '1') ~= ARGV[4] then return -2 end", + // Parent of a linked-worktree lane key; nil for a checkout key. + 'local lanePrefix = "\\0linked-worktree\\0"', + 'local function parentOf(key)', + ' if string.sub(key, 1, #lanePrefix) ~= lanePrefix then return nil end', + ' local last = nil', + ' local cursor = #lanePrefix + 1', + ' while true do', + ' local found = string.find(key, "\\0", cursor, true)', + ' if not found then break end', + ' last = found', + ' cursor = found + 1', + ' end', + ' if last == nil or last <= #lanePrefix + 1 then return nil end', + ' return string.sub(key, #lanePrefix + 1, last - 1)', + 'end', "local lock = redis.call('GET', KEYS[2])", "local owner = 'workspace-slots:' .. ARGV[1]", 'if lock and lock ~= owner then return -1 end', 'local busy = {}', + 'local busyChildren = {}', 'local free = nil', 'local latest = tonumber(ARGV[5])', `for slot = 0, ${MAX_WORKSPACE_LEASE_SLOTS - 1} do`, @@ -63,12 +85,20 @@ export class BridgeWorkspaceSlots { ' else', ' if entry[1] == ARGV[2] then return slot end', ' busy[entry[3]] = true', + ' local busyParent = parentOf(entry[3])', + ' if busyParent then busyChildren[busyParent] = true end', ' latest = math.max(latest, tonumber(entry[4]))', ' end', ' end', ' if not occupied and free == nil and slot < tonumber(ARGV[4]) then free = slot end', 'end', - 'if free == nil or busy[ARGV[3]] then return -1 end', + 'local function conflicts(key)', + ' if busy[key] then return true end', + ' local parent = parentOf(key)', + ' if parent then return busy[parent] == true end', + ' return busyChildren[key] == true', + 'end', + 'if free == nil or conflicts(ARGV[3]) then return -1 end', // Expiry removes queue metadata, never a workspace uncertainty fence. "local expired = redis.call('ZRANGEBYSCORE', KEYS[6], '-inf', ARGV[6])", 'for _, id in ipairs(expired) do', @@ -78,11 +108,15 @@ export class BridgeWorkspaceSlots { 'end', "local pending = redis.call('ZRANGE', KEYS[5], 0, 31)", 'local selected = nil', + 'local waitingCheckouts = {}', 'for _, id in ipairs(pending) do', " local workspace = redis.call('HGET', KEYS[7], id)", // An older serial request remains a barrier until its dispatcher finishes. ' if not workspace then return -1 end', - ' if not busy[workspace] then selected = id; break end', + ' local parent = parentOf(workspace)', + ' local held = conflicts(workspace) or (parent ~= nil and waitingCheckouts[parent] == true)', + ' if not held then selected = id; break end', + ' if parent == nil then waitingCheckouts[workspace] = true end', 'end', 'if selected ~= ARGV[2] then return -1 end', "redis.call('HSET', KEYS[1], 'a:' .. free, ARGV[2], 'i:' .. free, ARGV[1], 'w:' .. free, ARGV[3], 'e:' .. free, ARGV[5])", diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 0ca75566..123e7ee2 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -18,6 +18,7 @@ import { isWorkspaceToolRequest, isWorkspaceToolResult, workspaceIsolationKey, + workspaceIsolationParent, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; import { BridgeAdmissionQueue } from './admission'; @@ -105,6 +106,25 @@ type AssignmentOwnership = Pick< | 'runtimeSessionId' >; +const NATIVE_WORKSPACE_FENCE_PREFIX = 'native-workspace:'; + +/** Fence keys of the linked-worktree lanes enqueued beneath a checkout fence. */ +function workspaceLaneFenceIndexKey(checkoutFenceKey: string): string { + return `${checkoutFenceKey}:lanes`; +} + +/** The fence of the checkout a linked-worktree lane nests beneath. While that + * checkout is quarantined, its lanes may not start either. */ +function workspaceFenceParent(fence: string): string | undefined { + if (!fence.startsWith(NATIVE_WORKSPACE_FENCE_PREFIX)) return undefined; + const parent = workspaceIsolationParent( + fence.slice(NATIVE_WORKSPACE_FENCE_PREFIX.length), + ); + return parent === undefined + ? undefined + : `${NATIVE_WORKSPACE_FENCE_PREFIX}${parent}`; +} + function workspaceFenceReceiptKey(assignmentId: string): string { return `${assignmentKey(assignmentId)}:workspace-fence-owner`; } @@ -151,6 +171,12 @@ function supportsWorkspaceTool( ) { return false; } + if ( + request.worktree !== undefined && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + return false; + } if (request.operation === 'list_files' && request.afterPath !== undefined) { return capabilities?.listFileFeatures?.includes('after_path') === true; } @@ -184,6 +210,7 @@ function supportsWorkspaceProgrammatic( workspaceId: string, language: string, workspaceInstanceId?: string, + worktree?: string, ): boolean { const capabilities = registration.capabilities.workspaceTools; const workspace = capabilities?.workspaces.find( @@ -193,6 +220,8 @@ function supportsWorkspaceProgrammatic( workspace != null && (workspaceInstanceId === undefined || workspace.workspaceInstances?.includes('git_worktree') === true) && + (worktree === undefined || + workspace.workspaceScopes?.includes('git_linked_worktree') === true) && capabilities?.operations.includes('execute_command') === true && (workspace.operations == null || workspace.operations.includes('execute_command')) && @@ -214,11 +243,24 @@ function workspaceInstanceId(body: t.PayloadBody): string | undefined { return undefined; } +function workspaceWorktree(body: t.PayloadBody): string | undefined { + if ( + typeof body === 'object' && + body != null && + 'workspace_worktree' in body && + typeof body.workspace_worktree === 'string' + ) { + return body.workspace_worktree; + } + return undefined; +} + export function workspaceAdmissionId( workspaceId: string, instanceId?: string, + worktree?: string, ): string { - return workspaceIsolationKey(workspaceId, instanceId); + return workspaceIsolationKey(workspaceId, instanceId, worktree); } function workerKey(workerId: string): string { @@ -862,6 +904,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), ) ) { throw new BridgeStoreError( @@ -899,12 +942,15 @@ export class RedisBridgeStore { args.workspaceRequest?.workspaceId ?? args.workspaceId; const selectedWorkspaceInstanceId = args.workspaceRequest?.workspaceInstanceId ?? workspaceInstanceId(args.body); + const selectedWorktree = + args.workspaceRequest?.worktree ?? workspaceWorktree(args.body); const selectedWorkspaceAdmissionId = selectedWorkspaceId == null ? undefined : workspaceAdmissionId( selectedWorkspaceId, selectedWorkspaceInstanceId, + selectedWorktree, ); const workspaceSlots = selectedWorkspaceId != null && @@ -1033,6 +1079,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), )) ) { throw new BridgeStoreError( @@ -1059,13 +1106,13 @@ export class RedisBridgeStore { leaseToken, leaseTokenHash: tokenHash(leaseToken), ...(selectedWorkspaceAdmissionId == null ? {} : { - workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId}`, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${selectedWorkspaceAdmissionId}`, }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId!}`, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${selectedWorkspaceAdmissionId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -1155,6 +1202,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), ) ) { throw new BridgeStoreError( @@ -2185,6 +2233,13 @@ export class RedisBridgeStore { "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", 'if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[6]) ~= ARGV[7] then return 0 end', "if #KEYS >= 7 and redis.call('EXISTS', KEYS[7]) == 1 then return -1 end", + // A lane refuses a quarantined checkout. A checkout reaches enqueue only once no + // lane beneath it holds a slot, so any lane fence still indexed here is stuck. + "if ARGV[9] == 'lane' and redis.call('EXISTS', KEYS[10]) == 1 then return -1 end", + "if ARGV[9] == 'checkout' then", + " for i = 11, #KEYS do if redis.call('EXISTS', KEYS[i]) == 1 then return -1 end end", + " for i = 11, #KEYS do redis.call('SREM', KEYS[10], KEYS[i]) end", + 'end', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"EX\", ARGV[3])', "redis.call('RPUSH', KEYS[3], ARGV[4])", "redis.call('EXPIRE', KEYS[3], ARGV[3])", @@ -2193,7 +2248,8 @@ export class RedisBridgeStore { : []), 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', "if #KEYS >= 7 then redis.call('SET', KEYS[7], ARGV[4]) end", - 'if #KEYS == 9 then', + "if ARGV[9] == 'lane' then redis.call('SADD', KEYS[11], KEYS[7]) end", + 'if #KEYS >= 9 then', " local epoch = redis.call('GET', KEYS[9])", " if type(epoch) ~= 'string' then epoch = '0'; redis.call('SET', KEYS[9], epoch, 'EX', ARGV[3]) end", " if redis.call('PTTL', KEYS[9]) < tonumber(ARGV[3]) * 1000 then redis.call('EXPIRE', KEYS[9], ARGV[3]) end", @@ -2233,11 +2289,29 @@ export class RedisBridgeStore { workerIdentityId: assignment.workerIdentityId, expiresAt: assignment.expiresAt, }; + let fenceScope: '' | 'lane' | 'checkout' = ''; if (assignment.workspaceLeaseSlot !== undefined) { keys.push( workspaceFenceReceiptKey(assignment.assignmentId), `${workspaceQuarantineKey(assignment.workerId, assignment.workspaceFence!)}:epoch`, ); + const parent = workspaceFenceParent(assignment.workspaceFence!); + if (parent !== undefined) { + const parentFence = workspaceQuarantineKey(assignment.workerId, parent); + keys.push(parentFence, workspaceLaneFenceIndexKey(parentFence)); + fenceScope = 'lane'; + } else { + const index = workspaceLaneFenceIndexKey( + workspaceQuarantineKey(assignment.workerId, assignment.workspaceFence!), + ); + const laneFences = await boundedCommand( + this.redis.smembers(index), + this.redisCommandTimeoutMs, + 'Bridge linked worktree fence index read', + ); + keys.push(index, ...laneFences); + fenceScope = 'checkout'; + } } const result = await this.redis.eval( script, @@ -2251,6 +2325,7 @@ export class RedisBridgeStore { String(Date.parse(assignment.expiresAt)), readyToken ?? '', JSON.stringify(receipt), + fenceScope, ); if (Number(result) === -1) { throw new BridgeStoreError( diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index a646b9ad..1e2a367c 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -92,7 +92,11 @@ import { CODEAPI_BRIDGE_WORKSPACE_HEADER, resolveBridgeWorkerSelection, } from '../bridge/selection'; -import { isValidBridgeWorkerId, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES } from '../../../packages/code/src/protocol'; +import { + isValidBridgeWorkerId, + isValidLinkedWorktreeName, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES, +} from '../../../packages/code/src/protocol'; import logger from '../logger'; import { type ExecutionState, @@ -505,6 +509,7 @@ async function handleReplayInitial( bridgeWorkerId?: string; workspaceId?: string; workspaceInstanceId?: string; + workspaceWorktree?: string; }, cancellation: ReplayRequestCancellation, ): Promise { @@ -514,6 +519,7 @@ async function handleReplayInitial( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, } = params; const { code, tools, user_id, files } = req.body as t.ProgrammaticRequestBody; @@ -671,6 +677,7 @@ async function handleReplayInitial( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -1291,6 +1298,7 @@ router.post( let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; let workspaceInstanceId: string | undefined; + let workspaceWorktree: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -1341,6 +1349,15 @@ router.post( principalId: principal.userId, }); } + const requestedWorktree = rawBody.workspace_worktree; + if (requestedWorktree !== undefined) { + if (workspaceId == null || !isValidLinkedWorktreeName(requestedWorktree)) { + return res.status(400).json({ + error: 'Invalid code workspace worktree', + }); + } + workspaceWorktree = requestedWorktree; + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { return res @@ -1458,6 +1475,7 @@ router.post( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, }, cancellation); } if (workspaceId != null) { diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index 38c72486..056fcbb5 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -40,6 +40,7 @@ export interface BuildReplayExecutionStateParams { bridgeWorkerId?: string; workspaceId?: string; workspaceInstanceId?: string; + workspaceWorktree?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -70,6 +71,7 @@ export function buildReplayExecutionState( bridgeWorkerId: params.bridgeWorkerId, workspaceId: params.workspaceId, workspaceInstanceId: params.workspaceInstanceId, + workspaceWorktree: params.workspaceWorktree, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, @@ -86,12 +88,21 @@ export function buildReplayExecutionState( }; } -/** Bind the authenticated conversation checkout to every replay iteration. */ +/** Bind the authenticated conversation checkout and linked-worktree lane to every replay iteration. */ export function bindReplayWorkspaceInstance( payload: t.PayloadBody, - state: Pick, + state: Pick, ): t.PayloadBody { - return state.workspaceInstanceId == null - ? payload - : { ...payload, workspace_instance_id: state.workspaceInstanceId }; + if (state.workspaceInstanceId == null && state.workspaceWorktree == null) { + return payload; + } + return { + ...payload, + ...(state.workspaceInstanceId == null + ? {} + : { workspace_instance_id: state.workspaceInstanceId }), + ...(state.workspaceWorktree == null + ? {} + : { workspace_worktree: state.workspaceWorktree }), + }; } diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index fd4e90e3..003ce6b5 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -120,6 +120,8 @@ export interface ExecutionState { workspaceId?: string; /** Selected conversation checkout retained across every replay iteration. */ workspaceInstanceId?: string; + /** Selected linked-worktree lane retained across every replay iteration. */ + workspaceWorktree?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 91c3ed89..b0375b38 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -206,6 +206,8 @@ export interface PayloadBody { version: string; /** Opaque conversation checkout selected and authenticated by the API. */ workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; /** Stable identity shared by all replay iterations of one execution. */ execution_id?: string; replay_tool_count?: number; @@ -397,6 +399,8 @@ export interface ProgrammaticRequestBody { lang?: 'python' | 'bash'; /** Opaque conversation checkout binding for a selected native workspace. */ workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; } export interface ProgrammaticToolCall { From dbb84779312d473a3f3eee763e36c7f92804aab3 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:05:10 -0400 Subject: [PATCH 50/75] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20fix:=20Guard=20Li?= =?UTF-8?q?nked-Worktree=20Lanes=20Against=20Accidental=20Git=20Pruning=20?= =?UTF-8?q?(#272)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Schedule Linked Worktrees as Their Own Workspace Lanes * fix: Refuse quarantined-parent lanes before enqueue, forward lane policy, reset lane fences - Check the parent checkout fence before the assignment is stored or queued. - Forward the linked worktree policy to the forked native sandbox. - Protect shared Git config, hooks and info even before they exist. - Reset a lane fence with --reset-workspace-worktree. * fix: Grant lanes an explicit Git write allowlist, hold checkouts on quarantined lanes, release stale lanes - A lane writes only shared objects, refs, ref logs, LFS storage and its own worktree metadata; the checkout HEAD, index, operation state, config and hooks stay read-only without per-path denies. - A checkout assignment waits on any quarantined lane beneath it. - Lane registrations are released when their worktree disappears and capped at 32, dropping command roots and credential routes. * fix: Refuse a checkout while a lane keeps a stuck fence; let lane probes read shared Git - Code API indexes the lane fences enqueued beneath each checkout. A checkout reaches enqueue only once no lane holds a slot, so an indexed fence that still exists is stuck and the checkout is refused, surviving worker restarts. - Replay probes of a lane may read its common Git directory (read-only). * fix: Guard linked-worktree Git grants and unresolved actions * fix: Guard linked-worktree lanes against accidental Git pruning * fix: Refuse Git Aliases and Allow Pathspec Flags in Worktree Lanes * fix: Guard Worktree Git Against Indirect Pruning * fix: Refuse Git Repository Dispatchers in Worktree Lanes --------- Co-authored-by: Danny Avila Co-authored-by: Lia --- packages/code/README.md | 37 ++- packages/code/src/cli.ts | 3 + .../src/linked-worktree-git-guard.test.ts | 276 ++++++++++++++++++ .../code/src/linked-worktree-git-guard.ts | 123 ++++++++ packages/code/src/native-sandbox.test.ts | 81 +++++ packages/code/src/native-sandbox.ts | 46 ++- 6 files changed, 556 insertions(+), 10 deletions(-) create mode 100644 packages/code/src/linked-worktree-git-guard.test.ts create mode 100644 packages/code/src/linked-worktree-git-guard.ts diff --git a/packages/code/README.md b/packages/code/README.md index e3051930..a38523a3 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -826,15 +826,34 @@ shared object and ref storage (`.git/objects`, `.git/refs`, `.git/logs/refs`, `.git` stays read-only: configuration, hooks and `info`, the checkout's own `HEAD`, index and merge or rebase state, and sibling metadata. Automatic `gc` and maintenance are disabled, and `git gc` itself cannot run in a lane (it -needs to write `.git/gc.pid` and `packed-refs`); run storage maintenance from -the checkout. Explicit `git prune --expire now`, `git repack -ad`, or LFS -pruning can still delete objects another lane is writing; the sandbox does -**not** prevent this race. Do not enable lanes for agents that run destructive -maintenance until those commands are blocked or serialized. Deleting a branch -or tag also needs the checkout, because Git locks `packed-refs` for every ref -deletion. Each lane has its own durable quarantine guard. A lane -cannot start while its checkout is quarantined, and a checkout cannot start -while any lane beneath it is. +needs to write `.git/gc.pid` and `packed-refs`). A lane's `PATH` starts with +a read-only Git wrapper that refuses `prune`, `gc`, `repack`, `prune-packed`, +`maintenance`, `multi-pack-index`, `for-each-repo` and `git lfs prune`, +including after Git options such as `-C` or `-c`. `for-each-repo` is refused +entirely because Git dispatches its child commands without re-entering the +wrapper. It rejects `git lfs fetch` or `pull` with +`--prune`/`-p`, and `git fetch` or `pull` with `--auto-maintenance`/`--auto-gc`. +It enforces `maintenance.auto=false`, `gc.auto=0` and `help.autocorrect=0` +after caller-supplied Git options, preventing those overrides from restoring +automatic maintenance or correcting a misspelled command to `prune`. +It also refuses **all configured Git aliases**, +including harmless ones: aliases can hide destructive maintenance through +local config, `-c`, `--config-env`, includes or shell commands. Call the +underlying Git command instead, or run the alias from the checkout. Git's +safe pathspec options (`--literal-pathspecs`, `--glob-pathspecs`, +`--noglob-pathspecs`, `--icase-pathspecs`) remain usable in lanes. Run storage +maintenance from the checkout. + +This is a guardrail against accidental commands, **not** a filesystem +boundary: invoking Git by an absolute path, resetting `PATH`, or using a +script that does either bypasses it. Such commands can still delete a sibling +lane's unpublished objects. Do not enable concurrent lanes for agents or +scripts that deliberately bypass the wrapper. The guard requires `/bin/bash` +and a trusted system Git executable; without either, lane commands are not +admitted. Deleting a branch or tag also needs the checkout, because Git locks +`packed-refs` for every ref deletion. Each lane has its own durable quarantine +guard. A lane cannot start while its checkout is quarantined, and a checkout +cannot start while any lane beneath it is. Lanes require native-srt commands and at least two lease slots, and cannot yet be combined with conversation worktrees. Code API must advertise diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 135b9f8e..410caf25 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -805,6 +805,9 @@ async function run( 'Linked worktree lanes cannot be combined with conversation worktrees', ); } + if (linkedWorktreeLanes && process.platform === 'win32') { + throw new Error('Linked worktree Git guard requires a POSIX host'); + } if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() diff --git a/packages/code/src/linked-worktree-git-guard.test.ts b/packages/code/src/linked-worktree-git-guard.test.ts new file mode 100644 index 00000000..96f16b4a --- /dev/null +++ b/packages/code/src/linked-worktree-git-guard.test.ts @@ -0,0 +1,276 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +import { writeLinkedWorktreeGitGuard } from './linked-worktree-git-guard.js'; + +const execFileAsync = promisify(execFile); + +test('lane Git guard prevents destructive maintenance after global options without breaking ordinary Git', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-git-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const wrapper = join(bin, 'git'); + assert.equal((await stat(wrapper)).mode & 0o222, 0, 'the script is not writable'); + assert.match(await readFile(wrapper, 'utf8'), /^#!\/bin\/bash/); + await execFileAsync('/bin/bash', ['-n', wrapper]); + assert.match(await readFile(wrapper, 'utf8'), /run storage maintenance from the checkout/); + + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + assert.match((await run(['--version'])).stdout, /git version/); + assert.equal((await run(['-C', repo, '-c', 'core.quotepath=false', 'status', '--short'])).stdout, ''); + for (const option of ['--literal-pathspecs', '--glob-pathspecs', '--noglob-pathspecs', '--icase-pathspecs']) { + assert.equal((await run([option, '-C', repo, 'status', '--short'])).stdout, '', option); + } + await writeFile(join(repo, 'literal[abc].txt'), 'literal\n'); + await run(['--literal-pathspecs', 'add', '--', 'literal[abc].txt']); + assert.match((await run(['status', '--short'])).stdout, /A literal\[abc\]\.txt/); + + // Git has created this object, but no ref points to it yet. Another lane can be + // in exactly this window between writing objects and updating a ref. + await writeFile(join(repo, 'unpublished.txt'), 'from a second lane\n'); + const object = (await run(['hash-object', '-w', 'unpublished.txt'])).stdout.trim(); + const blocked = [ + ['prune', '--expire', 'now'], + ['-C', repo, 'prune', '--expire=now'], + ['-C' + repo, '-c', 'gc.auto=0', '--git-dir=' + join(repo, '.git'), 'prune', '--expire=now'], + ['--no-pager', '-cgc.auto=0', 'gc', '--force'], + ['repack', '-ad'], + ['prune-packed'], + ['maintenance', 'run', '--task=gc'], + ['multi-pack-index', 'expire'], + ['lfs', 'prune'], + ['--literal-pathspecs', 'prune', '--expire', 'now'], + ]; + for (const args of blocked) { + await assert.rejects(run(args), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /run storage maintenance from the checkout/.test(result.stderr ?? ''); + }, `must reject git ${args.join(' ')}`); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + await assert.rejects(run(['--unrecognized-option', 'prune']), /unsupported global option/); + assert.match((await run(['-C', repo, 'status', '--short'])).stdout, /\?\? unpublished\.txt\n/); + + // The guard is deliberately not a sandbox boundary: resetting PATH or using + // an absolute Git binary can still prune shared objects. + await execFileAsync('git', ['prune', '--expire', 'now'], { cwd: repo }); + await assert.rejects(run(['cat-file', '-e', object])); +}); + +test('lane Git guard rejects aliases from checkout config, -c, include.path and --config-env', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-alias-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + const otherRepo = join(parent, 'other'); + await Promise.all([mkdir(bin), mkdir(repo), mkdir(otherRepo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[], extraEnv: Record = {}) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, ...extraEnv, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await run(['config', 'alias.cleanup', 'prune --expire now']); + await run(['config', 'alias.inspect', 'status --short']); + await run(['config', 'alias.indirect', 'cleanup']); + await run(['-C', otherRepo, 'init', '-q', '-b', 'main']); + await run(['-C', otherRepo, 'config', 'alias.othercleanup', 'prune --expire now']); + await writeFile(join(parent, 'aliases.cfg'), '[alias]\n frominclude = prune --expire now\n'); + await writeFile(join(repo, 'unpublished.txt'), 'a different lane wrote this object\n'); + const object = (await run(['hash-object', '-w', 'unpublished.txt'])).stdout.trim(); + const blocked: Array<[string[], Record?]> = [ + [['cleanup']], + [['-C', repo, 'cleanup']], + [['-C', otherRepo, 'othercleanup']], + [['--git-dir=' + join(otherRepo, '.git'), 'othercleanup']], + [['inspect']], + [['indirect']], + [['-c', 'alias.fromoption=prune --expire now', 'fromoption']], + [['-c', 'alias.fromshell=!git prune --expire now', 'fromshell']], + [['--git-dir=' + join(repo, '.git'), '-c', 'alias.fromgitdir=prune --expire now', 'fromgitdir']], + [['-c', `include.path=${join(parent, 'aliases.cfg')}`, 'frominclude']], + [['--config-env=alias.fromenv=LANE_TEST_GIT_ALIAS', 'fromenv'], { LANE_TEST_GIT_ALIAS: 'prune --expire now' }], + ]; + for (const [args, environment] of blocked) { + await assert.rejects(run(args, environment), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /Git aliases are unavailable in linked worktree lanes/.test(result.stderr ?? ''); + }, `must reject Git alias in ${JSON.stringify(args)}`); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + assert.match((await run(['status', '--short'])).stdout, /unpublished\.txt/); +}); + +test('a lane rejects a checkout-defined maintenance alias before pruning another lane’s unpublished object', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-shared-alias-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const checkout = join(parent, 'checkout'); + const bin = join(parent, 'bin'); + await Promise.all([mkdir(checkout), mkdir(bin)]); + await execFileAsync('git', ['init', '-q', '-b', 'main'], { cwd: checkout }); + await execFileAsync('git', [ + '-c', 'user.name=test', '-c', 'user.email=test@example.test', + '-c', 'commit.gpgsign=false', 'commit', '--allow-empty', '-q', '-m', 'seed', + ], { cwd: checkout }); + await mkdir(join(checkout, '.worktrees')); + const lane = join(checkout, '.worktrees', 'task'); + await execFileAsync('git', ['worktree', 'add', '-q', '-b', 'task', lane], { cwd: checkout }); + await execFileAsync('git', ['config', 'alias.cleanup', 'prune --expire now'], { cwd: checkout }); + await writeLinkedWorktreeGitGuard(bin); + await writeFile(join(lane, 'pending.txt'), 'another lane has not updated its ref\n'); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: lane, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + const object = (await run(['hash-object', '-w', 'pending.txt'])).stdout.trim(); + await assert.rejects(run(['cleanup']), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /Git aliases are unavailable in linked worktree lanes/.test(result.stderr ?? ''); + }); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); +}); + +test('a lane never lets a misspelled maintenance command autocorrect to prune', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-autocorrect-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await run(['config', 'help.autocorrect', 'immediate']); + await writeFile(join(repo, 'unpublished'), 'a sibling has not made this blob reachable'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['prun', '--expire', 'now'], + ['-c', 'help.autocorrect=immediate', 'prun', '--expire=now'], + ]) { + await assert.rejects(run(args), /not a git command|not a Git command|unknown Git command/i); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } +}); + +test('Git options cannot re-enable automatic maintenance for lane commands', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-maintenance-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + const remote = join(parent, 'remote.git'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[], additionalEnv: NodeJS.ProcessEnv = {}) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, ...additionalEnv, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await execFileAsync('git', ['init', '-q', '--bare', remote]); + for (const [key, supplied, enforced] of [ + ['maintenance.auto', 'true', 'false'], + ['gc.auto', '1', '0'], + ['help.autocorrect', 'immediate', '0'], + ]) { + const { stdout } = await run(['-c', `${key}=${supplied}`, 'config', '--get', key]); + assert.equal(stdout.trim(), enforced, `${key} must be enforced after caller-supplied -c`); + } + const { stdout } = await run(['config', '--get', 'maintenance.auto'], { + GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'maintenance.auto', GIT_CONFIG_VALUE_0: 'true', + }); + assert.equal(stdout.trim(), 'false', 'inherited configuration must not re-enable maintenance'); + const configuredGc = await run(['--config-env=gc.auto=LANE_TEST_GC_AUTO', 'config', '--get', 'gc.auto'], { + LANE_TEST_GC_AUTO: '1', + }); + assert.equal(configuredGc.stdout.trim(), '0', 'config-env cannot override the enforced setting'); + await writeFile(join(repo, 'unpublished'), 'not referenced yet'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['fetch', '--auto-maintenance', remote], + ['fetch', '--auto-maintenance=true', remote], + ['-c', 'maintenance.auto=true', 'fetch', '--auto-gc', remote], + ['fetch', '--auto-gc=true', remote], + ['-c', 'gc.auto=1', 'fetch', '--auto-maintenance', remote], + ['pull', '--auto-maintenance', remote], + ]) { + await assert.rejects(run(args), /run storage maintenance from the checkout/); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + await run(['-c', 'maintenance.auto=true', '-c', 'gc.auto=1', 'fetch', '--no-auto-maintenance', remote, 'refs/heads/*:refs/remotes/origin/*']); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); +}); + +test('for-each-repo cannot dispatch maintenance behind the lane Git guard', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-for-each-repo-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await writeFile(join(repo, 'pending'), 'a sibling has not published this object yet\n'); + const object = (await run(['hash-object', '-w', 'pending'])).stdout.trim(); + const dispatcher = ['-c', `maintenance.repo=${repo}`, 'for-each-repo', '--config=maintenance.repo']; + for (const args of [ + [...dispatcher, 'prune', '--expire', 'now'], + [...dispatcher, 'status', '--short'], + ]) { + await assert.rejects(run(args), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /run storage maintenance from the checkout/.test(result.stderr ?? ''); + }, 'Git dispatchers cannot be inspected safely inside a lane'); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + assert.match((await run(['status', '--short'])).stdout, /pending/); +}); + +test('Git LFS fetch and pull cannot request pruning in a lane', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-lfs-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await writeFile(join(repo, 'unpublished'), 'an LFS sibling has no ref yet'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['lfs', 'fetch', '--prune'], + ['lfs', 'fetch', '-p'], + ['lfs', 'fetch', '-rp'], + ['lfs', 'fetch', '--prune', '--recent'], + ['lfs', 'pull', '--prune'], + ['-C', repo, 'lfs', 'fetch', '--prune'], + ]) { + await assert.rejects(run(args), /run storage maintenance from the checkout/); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } +}); diff --git a/packages/code/src/linked-worktree-git-guard.ts b/packages/code/src/linked-worktree-git-guard.ts new file mode 100644 index 00000000..227bfd3f --- /dev/null +++ b/packages/code/src/linked-worktree-git-guard.ts @@ -0,0 +1,123 @@ +import { constants as fsConstants } from 'node:fs'; +import { access, realpath, stat, writeFile } from 'node:fs/promises'; +import { join, sep } from 'node:path'; + +import { WorkspaceToolError } from './workspace.js'; + +// Match the system-only executable search used for native programmatic tools. +const GIT_BIN_DIRS = [ + '/opt/homebrew/bin', + '/usr/local/bin', + '/usr/bin', + '/bin', + '/home/linuxbrew/.linuxbrew/bin', +]; +const TRUSTED_GIT_DIRS = [ + ...GIT_BIN_DIRS, + '/opt/homebrew/Cellar', + '/usr/local/Cellar', + '/home/linuxbrew/.linuxbrew/Cellar', + '/usr/lib/git-core', +]; + +async function systemGitExecutable(): Promise { + for (const directory of GIT_BIN_DIRS) { + try { + const executable = await realpath(join(directory, 'git')); + if (!TRUSTED_GIT_DIRS.some(path => executable.startsWith(`${path}${sep}`))) continue; + if (!(await stat(executable)).isFile()) continue; + await access(executable, fsConstants.X_OK); + return executable; + } catch { + // A missing system installation is not a reason to run Git from the workspace or PATH. + } + } + throw new WorkspaceToolError('Trusted Git executable is unavailable for linked worktree lanes', 'COMMAND_UNAVAILABLE'); +} + +/** This guards accidental `git` calls, not absolute executable paths or caller shell aliases. */ +export async function writeLinkedWorktreeGitGuard(directory: string): Promise { + try { + await access('/bin/bash', fsConstants.X_OK); + } catch { + throw new WorkspaceToolError('Linked worktree Git guard requires /bin/bash', 'COMMAND_UNAVAILABLE'); + } + const git = await systemGitExecutable(); + const quotedGit = `'${git.replaceAll("'", "'\\''")}'`; + const script = [ + // Bash arrays preserve global option arguments when checking the same + // effective Git configuration without eval or interpreting alias contents. + '#!/bin/bash', + 'check() {', + ' local -a globals=() forwarded=()', + ' local aliasStatus=0', + ' while [ "$#" -gt 0 ]; do', + ' case "$1" in', + ' -C|-c|--git-dir|--work-tree|--namespace|--config-env)', + ' if [ "$#" -lt 2 ]; then echo "git: missing global option argument" >&2; return 2; fi', + ' globals+=("$1" "$2"); forwarded+=("$1" "$2"); shift 2 ;;', + ' -C?*|-c?*|--git-dir=*|--work-tree=*|--namespace=*|--config-env=*)', + ' globals+=("$1"); forwarded+=("$1"); shift ;;', + ' --bare)', + ' globals+=("$1"); forwarded+=("$1"); shift ;;', + ' -p|-P|--paginate|--no-pager|--no-replace-objects|--no-optional-locks|--exec-path=*)', + ' forwarded+=("$1"); shift ;;', + ' --literal-pathspecs|--glob-pathspecs|--noglob-pathspecs|--icase-pathspecs)', + ' forwarded+=("$1"); shift ;;', + ' --) forwarded+=("$1"); shift; break ;;', + ' -v|--version|-h|--help|--exec-path|--html-path|--man-path|--info-path)', + ' return 0 ;;', + ' -*) echo "git: unsupported global option in linked worktree lane" >&2; return 2 ;;', + ' *) break ;;', + ' esac', + ' done', + ' case "${1:-}" in', + ' prune|gc|repack|prune-packed|maintenance|multi-pack-index|for-each-repo)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' fetch|pull)', + ' for option in "${@:2}"; do', + ' case "$option" in', + ' --auto-maintenance|--auto-maintenance=*|--auto-gc|--auto-gc=*)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' esac', + ' done ;;', + ' lfs)', + ' if [ "${2:-}" = prune ]; then', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1', + ' fi', + ' if [ "${2:-}" = fetch ] || [ "${2:-}" = pull ]; then', + ' for option in "${@:3}"; do', + ' case "$option" in', + ' --prune|--prune=*|-p|-p?*|-[!-]*p*)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' esac', + ' done', + ' fi ;;', + ' esac', + ' if [ "$#" -eq 0 ]; then return 0; fi', + // Git itself loads local config, -C, -c, includes and --config-env, so an + // alias from any of those sources is rejected even when it hides a prune. + // Reject all aliases, including harmless ones: Git shell aliases and nested + // aliases cannot be proven safe by inspecting the first token. + ` ${quotedGit} "\${globals[@]}" config --get "alias.\${1}" >/dev/null 2>&1 || aliasStatus=$?`, + ' case "$aliasStatus" in', + ' 0) echo "git: Git aliases are unavailable in linked worktree lanes; run the underlying command or use the checkout" >&2; return 1 ;;', + ' 1) ;;', + ' *) echo "git: cannot verify Git aliases in linked worktree lane" >&2; return 2 ;;', + ' esac', + // Command-line -c values override both Git config files and GIT_CONFIG_COUNT. + // Add these after every caller global option: fetch can start maintenance + // internally without invoking the PATH guard a second time. + ` exec ${quotedGit} "\${forwarded[@]}" -c maintenance.auto=false -c gc.auto=0 -c help.autocorrect=0 "$@"`, + '}', + 'check "$@" || exit "$?"', + `exec ${quotedGit} "$@"`, + '', + ].join('\n'); + await writeFile(join(directory, 'git'), script, { flag: 'wx', mode: 0o500 }); + await access(join(directory, 'git'), fsConstants.X_OK); +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 7ecb06f3..b6fb25bc 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1629,6 +1629,11 @@ test('a linked worktree lane may write only shared Git storage and its own metad assert.deepEqual(config.filesystem.allowWrite.slice(0, 4), [lane, ...writableGitPaths]); assert.ok(!config.filesystem.allowWrite.includes(commonGitDir)); assert.ok(config.filesystem.allowRead?.includes(commonGitDir)); + const gitGuard = config.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(gitGuard, 'lane Git guard must be readable'); + assert.ok(!config.filesystem.allowWrite.includes(gitGuard)); + assert.ok(config.filesystem.denyWrite.includes(gitGuard)); + assert.equal((await stat(join(gitGuard, 'git'))).mode & 0o222, 0); const probed = fakeManager(); const prober = new NativeSrtWorkspaceCommandSandbox({ @@ -1642,6 +1647,10 @@ test('a linked worktree lane may write only shared Git storage and its own metad await prober.executeProgrammatic(request, dataDirectory, undefined, { probe: true }); assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.allowRead?.includes(commonGitDir)); assert.ok(!probed.customConfigSeenDuringWrap?.filesystem?.allowWrite?.includes(commonGitDir)); + const probeGuard = probed.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(probeGuard); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.allowRead?.includes(probeGuard)); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.denyWrite?.includes(probeGuard)); await assert.rejects( prepare([commonGitDir]), @@ -1659,3 +1668,75 @@ test('a linked worktree lane may write only shared Git storage and its own metad error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', ); }); + +test('lane commands put the read-only Git guard ahead of the ordinary PATH', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const checkoutRoot = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-guard-'))); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const writableGitPaths = [join(commonGitDir, 'objects'), join(commonGitDir, 'worktrees', 'task-a')]; + await Promise.all([lane, ...writableGitPaths].map(path => mkdir(path, { recursive: true }))); + const fake = fakeManager(); + let commandPath: string | undefined; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths }, + environment: { PATH: '/usr/bin:/bin' }, + manager: fake.manager, + spawnCommand(command, args, options) { + commandPath = options.env?.PATH; + return spawn(command, args, options); + }, + }); + t.after(() => sandbox.close()); + const result = await sandbox.execute({ ...request, command: 'git --version' }); + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /git version/); + const guard = fake.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(guard); + assert.ok(commandPath?.startsWith(`${guard}:`)); + await sandbox.close(); + await assert.rejects(stat(guard), { code: 'ENOENT' }); +}); + +test('linked worktree Git guard is removed when sandbox initialization fails', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const checkoutRoot = await mkdtemp(join(tmpdir(), 'librechat-code-failed-guard-')); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const objects = join(commonGitDir, 'objects'); + await Promise.all([lane, objects].map(path => mkdir(path, { recursive: true }))); + const fake = fakeManager({ initializeError: new Error('init failed') }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths: [objects] }, + manager: fake.manager, + }); + await assert.rejects(sandbox.prepare(), /init failed/); + const guard = fake.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(guard); + await assert.rejects(stat(guard), { code: 'ENOENT' }); + await sandbox.close(); +}); + +test('linked worktree Git guard refuses Windows rather than admitting an unguarded lane', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-win-guard-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + linkedWorktree: { + checkoutRoot: root, + commonGitDir: join(root, '.git'), + writableGitPaths: [join(root, '.git', 'objects')], + }, + platform: 'win32', + manager: fakeManager().manager, + }); + await assert.rejects(sandbox.prepare(), (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'COMMAND_UNAVAILABLE' && + /POSIX host/.test(error.message), + ); + await sandbox.close(); +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 6ac51acc..e635f8e7 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -32,6 +32,7 @@ import { } from './private-storage.js'; import { WorkspaceToolError } from './workspace.js'; import { restoreScratchTraversal } from './native-scratch.js'; +import { writeLinkedWorktreeGitGuard } from './linked-worktree-git-guard.js'; import type { ChildProcessWithoutNullStreams, @@ -323,6 +324,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } private scratchDirectory?: string; private scratchHandle?: FileHandle; + /** A private sibling of scratch, readable but never writable by lane commands. */ + private gitGuardDirectory?: string; private execution?: Promise; private closing?: Promise; private resetFailed = false; @@ -361,6 +364,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox await this.manager.reset().catch(() => { this.resetFailed = true; }); + await this.removeLinkedWorktreeGitGuard().catch(() => undefined); await this.removeScratchDirectory().catch(() => undefined); if (!this.resetFailed) managerOwners.delete(this.manager); this.initialized = undefined; @@ -448,6 +452,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } } const lane = this.options.linkedWorktree; + if (lane && this.platform === 'win32') { + throw new WorkspaceToolError('Linked worktree Git guard requires a POSIX host', 'COMMAND_UNAVAILABLE'); + } const commonGitDir = lane ? await canonicalPath(lane.commonGitDir) : undefined; const checkoutRoot = lane ? await canonicalPath(lane.checkoutRoot) : undefined; const writableGitPaths = lane @@ -485,6 +492,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'REGISTRATION_INVALID', ); } + if (lane && !canonicalScratchDirectory) { + throw new WorkspaceToolError('Linked worktree Git guard requires private scratch storage', 'COMMAND_UNAVAILABLE'); + } + const gitGuardDirectory = lane + ? await this.createLinkedWorktreeGitGuard(canonicalScratchDirectory!) + : undefined; const commandPolicy = normalizeNativeSrtCommandPolicy( this.options.commandPolicy, ); @@ -510,6 +523,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox allowRead: [ root, ...laneGitPaths, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), @@ -524,6 +538,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox denyWrite: [ ...protectedPaths, ...deniedInheritedWritablePaths, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), ], allowGitConfig: false, }, @@ -593,6 +608,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.denyWritePaths = [ ...protectedPaths, ...deniedInheritedWritablePaths, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), ]; } @@ -817,6 +833,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ...(this.canonicalCommonGitDir ? [this.canonicalCommonGitDir] : []), + ...(this.gitGuardDirectory + ? [this.gitGuardDirectory] + : []), ], allowWrite: [ ...(canonicalWorkspaceRoot != null @@ -1067,6 +1086,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ? 'NUL' : '/dev/null', GIT_CONFIG_NOSYSTEM: '1', + ...(this.gitGuardDirectory ? { + PATH: `${this.gitGuardDirectory}:${wrapped.env.PATH || this.environment.PATH || '/usr/bin:/bin'}`, + } : {}), }, detached: this.platform !== 'win32', shell: false, @@ -1297,6 +1319,24 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } + private async createLinkedWorktreeGitGuard(scratchDirectory: string): Promise { + if (this.gitGuardDirectory) { + throw new Error('Linked worktree Git guard cleanup is still pending'); + } + // Unlike scratch, this sibling directory is not in filesystem.allowWrite. + const directory = await mkdtemp(join(dirname(scratchDirectory), 'librechat-code-git-')); + this.gitGuardDirectory = directory; + await assertPrivateStorageAncestors(directory); + await writeLinkedWorktreeGitGuard(directory); + return directory; + } + + private async removeLinkedWorktreeGitGuard(): Promise { + if (!this.gitGuardDirectory) return; + await rm(this.gitGuardDirectory, { recursive: true, force: true }); + this.gitGuardDirectory = undefined; + } + private async removeScratchDirectory(): Promise { const scratchDirectory = this.scratchDirectory; if (!scratchDirectory) return; @@ -1340,6 +1380,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } this.initialized = undefined; this.canonicalRoot = undefined; - await this.removeScratchDirectory(); + try { + await this.removeLinkedWorktreeGitGuard(); + } finally { + await this.removeScratchDirectory(); + } } } From 3189cfd8ee7f81cf90671f9b91b1d52ae9d3693d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 29 Sep 2026 15:47:33 -0400 Subject: [PATCH 51/75] =?UTF-8?q?=F0=9F=94=93=20fix:=20Let=20Linked-Worktr?= =?UTF-8?q?ee=20Lanes=20Commit=20When=20the=20Checkout=20Lives=20in=20the?= =?UTF-8?q?=20Worker=20Home=20(#274)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: Grant lane Git reads entry by entry so write binds survive a read-denied home A read grant on the whole common Git directory masked the write binds beneath it whenever the checkout lived inside the worker home, which SRT re-binds under a tmpfs (writes first, then reads). Lanes could not create their own index.lock or ref locks. Grant each entry off the writable paths instead, and add a real-SRT regression test with a Linux CI job. * fix: Re-bind lane Git writes after the common read bind instead of granting entries Per-entry read grants left the common Git directory as a writable tmpfs placeholder inside the worker home (pack-refs could strand refs there) and pinned replaced files to stale inodes. Grant the whole directory read-only again and, on Linux, list each existing writable Git directory as a deeper read-deny so SRT re-binds its write mount after the ancestor read bind. --- .github/workflows/ci.yml | 24 ++++ .../code/src/linked-worktrees-live.test.ts | 108 ++++++++++++++++++ packages/code/src/native-sandbox.test.ts | 13 ++- packages/code/src/native-sandbox.ts | 20 ++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 packages/code/src/linked-worktrees-live.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4076d8f6..f7393284 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -235,6 +235,30 @@ jobs: LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' run: node --test dist/environment-live.test.js + linux-native-sandbox-tests: + name: Linux Native Sandbox Tests + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: packages/code + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: 24.16.0 + - name: Install bubblewrap + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq bubblewrap socat ripgrep + # Ubuntu 24.04 blocks unprivileged user namespaces through AppArmor; bwrap needs them. + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - run: npm ci + - run: npm run build + - name: Linked worktree lane containment + env: + LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' + run: node --test dist/linked-worktrees-live.test.js + lambda-microvm-provisioning: name: Lambda MicroVM Provisioning runs-on: ubuntu-latest diff --git a/packages/code/src/linked-worktrees-live.test.ts b/packages/code/src/linked-worktrees-live.test.ts new file mode 100644 index 00000000..c9052a35 --- /dev/null +++ b/packages/code/src/linked-worktrees-live.test.ts @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { verifyLinkedWorktree } from './linked-worktrees.js'; +import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +const execFileAsync = promisify(execFile); +const IDENTITY = ['-c', 'user.name=lane', '-c', 'user.email=lane@example.com', '-c', 'commit.gpgsign=false']; + +async function git(cwd: string, ...args: string[]): Promise { + return (await execFileAsync('git', [...IDENTITY, ...args], { cwd })).stdout.trim(); +} + +async function snapshot(paths: string[]): Promise> { + const entries = await Promise.all( + paths.map(async (path) => [path, await readFile(path, 'utf8').catch(() => null)] as const), + ); + return Object.fromEntries(entries); +} + +/** + * The worker's home is read-denied, so a checkout beneath it exercises the + * sandbox's tmpfs re-binding; one beneath the system temporary directory does not. + */ +for (const [location, parent] of [ + ['beneath the worker home', homedir()], + ['outside the worker home', tmpdir()], +] as const) { + test(`real SRT lets a linked worktree lane commit while checkout and sibling Git metadata stay intact (${location})`, { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 60_000, + }, async (t) => { + const base = await realpath(await mkdtemp(join(parent, 'lane-live-'))); + t.after(() => rm(base, { recursive: true, force: true })); + const root = join(base, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'tracked.txt'), 'checkout\n'); + await git(root, 'add', 'tracked.txt'); + await git(root, 'commit', '-qm', 'init'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '-b', 'task-a', '.worktrees/task-a'); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + + const commonGitDir = join(root, '.git'); + const protectedFiles = [ + join(commonGitDir, 'HEAD'), + join(commonGitDir, 'index'), + join(commonGitDir, 'config'), + join(commonGitDir, 'MERGE_HEAD'), + join(commonGitDir, 'packed-refs'), + join(commonGitDir, 'hooks', 'pre-commit'), + join(commonGitDir, 'worktrees', 'task-b', 'HEAD'), + join(root, 'tracked.txt'), + ]; + const before = await snapshot(protectedFiles); + const lane = await verifyLinkedWorktree(root, 'task-a'); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane.root, + workspaceIdentity: lane.identity, + linkedWorktree: { + checkoutRoot: lane.checkoutRoot, + commonGitDir: lane.commonGitDir, + writableGitPaths: lane.writableGitPaths, + }, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + environment: { PATH: process.env.PATH, LANG: 'C.UTF-8' }, + }); + t.after(() => sandbox.close()); + const run = (command: string) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'lane', + command, + timeoutMs: 30_000, + maxOutputBytes: 16_384, + }); + const gitInLane = `git ${IDENTITY.join(' ')}`; + + const committed = await run( + `printf lane > lane.txt && ${gitInLane} add lane.txt && ${gitInLane} commit -qm lane && ${gitInLane} branch lane-extra`, + ); + assert.equal(committed.exitCode, 0, committed.stderr); + + for (const path of protectedFiles) { + await run(`printf tampered > '${path}'`); + } + // Packing must not move refs into storage that vanishes with the sandbox. + await run(`${gitInLane} pack-refs --all`); + await sandbox.close(); + + for (const branch of ['main', 'task-a', 'task-b', 'lane-extra']) { + assert.ok(await git(root, 'rev-parse', '--verify', '-q', `refs/heads/${branch}`), branch); + } + + assert.equal(await git(root, 'log', '-1', '--format=%s', 'task-a'), 'lane'); + assert.deepEqual(await snapshot(protectedFiles), before); + assert.equal(await git(root, 'status', '--porcelain', '--untracked-files=no'), ''); + await assert.rejects(stat(join(commonGitDir, 'MERGE_HEAD')), { code: 'ENOENT' }); + }); +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index b6fb25bc..5b79aa1a 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1612,13 +1612,14 @@ test('a linked worktree lane may write only shared Git storage and its own metad await Promise.all( [lane, ...writableGitPaths].map(path => mkdir(path, { recursive: true })), ); - const prepare = async (paths: string[]) => { + const prepare = async (paths: string[], platform: NodeJS.Platform = 'linux') => { const fake = fakeManager(); const sandbox = new NativeSrtWorkspaceCommandSandbox({ workspaceRoot: lane, linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths: paths }, environment: { PATH: '/usr/bin' }, manager: fake.manager, + platform, }); t.after(() => sandbox.close()); await sandbox.prepare(); @@ -1629,6 +1630,16 @@ test('a linked worktree lane may write only shared Git storage and its own metad assert.deepEqual(config.filesystem.allowWrite.slice(0, 4), [lane, ...writableGitPaths]); assert.ok(!config.filesystem.allowWrite.includes(commonGitDir)); assert.ok(config.filesystem.allowRead?.includes(commonGitDir)); + // Deeper read denies make SRT re-bind each writable Git directory after the + // read-only bind of the common directory (Linux tmpfs re-binding order). + for (const path of writableGitPaths) { + assert.ok(config.filesystem.denyRead.includes(path), path); + } + assert.ok(!config.filesystem.denyRead.includes(join(commonGitDir, 'lfs'))); + const darwin = await prepare(writableGitPaths, 'darwin'); + for (const path of writableGitPaths) { + assert.ok(!darwin.filesystem.denyRead.includes(path), path); + } const gitGuard = config.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); assert.ok(gitGuard, 'lane Git guard must be readable'); assert.ok(!config.filesystem.allowWrite.includes(gitGuard)); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index e635f8e7..49fe6ecf 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -481,6 +481,25 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const laneGitPaths = commonGitDir ? [commonGitDir] : []; + /** + * On Linux, SRT hides a read-denied directory (such as the worker home) under a + * tmpfs and then re-binds writes before reads, so the read-only bind of the whole + * common Git directory would mask its writable descendants. SRT processes read + * denies shallow-first, re-binding each one's writes on top, so listing every + * existing writable Git directory as a deeper deny restores its write bind after + * the ancestor read bind. The common directory stays a live, read-only host + * directory: nothing can be created at its top level. + */ + const laneWriteRebinds = + this.platform === 'linux' + ? ( + await Promise.all( + writableGitPaths.map(async path => + (await stat(path).catch(() => undefined))?.isDirectory() ? path : undefined, + ), + ) + ).filter((path): path is string => path != null) + : []; const canonicalScratchDirectory = await this.createScratchDirectory(sharedScratchPaths); if ( @@ -519,6 +538,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ...sharedScratchPaths.filter(path => deniedInheritedWritablePaths.includes(path), ), + ...laneWriteRebinds, ], allowRead: [ root, From 1e683ce749278788288a541364688077ebc595de Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 30 Sep 2026 06:30:16 -0400 Subject: [PATCH 52/75] =?UTF-8?q?=F0=9F=8E=AF=20feat:=20Diagnose=20Every?= =?UTF-8?q?=20Failing=20Workspace=20Edit=20and=20Negotiate=20Tolerant=20Ma?= =?UTF-8?q?tching=20(#271)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🎯 feat: Diagnose Every Failing Workspace Edit and Negotiate Tolerant Matching A rejected edit batch now reports every failing edit by position: missing edits name the nearest candidate line and flag elided, line-numbered, whitespace-only or CRLF mismatches, and ambiguous edits give their match count and line numbers. Two negotiated edit features add tolerant matching (line-trimmed, indentation-flexible, whitespace-normalized) and replaceAll, with per-edit match reporting only for requests that opt in. * fix: Honor Edit Boundaries and Negotiated Capabilities * fix: Preserve Matched Line Endings in Tolerant Edits * fix: Reject Unrepresentable Tolerant Edit Boundaries * fix: Require Tolerant Edit Boundary Line Breaks * fix: Bound Edit Matching and Redact Edit-Only Diagnostics * fix: Stream Workspace Replace-All Results Within Memory Bounds * fix: Preserve Extra Blank Lines in Tolerant Edits * fix: Honor Full Edit Wrappers and Linear Candidate Budgets * fix: Retain Every Failed Edit Position in Bounded Diagnostics --------- Co-authored-by: Lia --- docs/remote-bridge/README.md | 5 +- packages/code/README.md | 48 +- packages/code/src/edits.test.ts | 721 +++++++++++++++++++++ packages/code/src/edits.ts | 712 ++++++++++++++++++++ packages/code/src/protocol.test.ts | 158 +++++ packages/code/src/protocol.ts | 119 +++- packages/code/src/worker.ts | 39 +- packages/code/src/workspace-worker.test.ts | 136 ++++ packages/code/src/workspace.test.ts | 390 ++++++++++- packages/code/src/workspace.ts | 56 +- service/src/bridge/router.test.ts | 6 +- service/src/bridge/router.ts | 3 +- service/src/bridge/store.ts | 15 +- service/src/bridge/workspace-store.test.ts | 128 +++- 14 files changed, 2493 insertions(+), 43 deletions(-) create mode 100644 packages/code/src/edits.test.ts create mode 100644 packages/code/src/edits.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index a37745a2..b2d33db7 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -165,7 +165,10 @@ backslashes, symlink escapes, unexpected fields, and host roots are rejected. Workspace mutation remains disabled unless the operator starts the worker with `--allow-workspace-writes` (or `LIBRECHAT_CODE_ALLOW_WORKSPACE_WRITES=true`). That adds bounded `write_file` -and exact-match `edit_file` operations. Writes are limited to 1 MiB of UTF-8 +and `edit_file` operations. `edit_file` matches exactly by default; the +negotiated `tolerant_match` and `replace_all` edit features add +whitespace-tolerant matching and multi-location replacement (see +`packages/code/README.md`). Writes are limited to 1 MiB of UTF-8 text, require an existing in-workspace parent directory, reject symlinks, and commit atomically. The worker capability is an enforcement boundary; LibreChat should still route every mutation through its configurable tool-approval hooks. diff --git a/packages/code/README.md b/packages/code/README.md index a38523a3..ba7ec4d7 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -590,6 +590,48 @@ installed as one atomic mutation. Code API dispatches the batch form only after the worker and server negotiate `batch` in `editFileModes`. Revision-fenced edits likewise require the negotiated `expected_base_sha256` entry in `editFileFeatures`. + +Edits apply in order, each to the text the earlier ones produced. When any edit +fails, the worker still checks the rest and rejects the whole batch with one +`EDIT_CONFLICT` whose message lists every failing edit by position. Large +batch messages shorten reasons and source excerpts to stay within the bound, +but never omit failing edit positions. A missing edit names the nearest +candidate line and flags elided (`...`) or line-numbered `oldText`, a +whitespace-only difference, or CRLF line endings. +An ambiguous edit gives its match count and line numbers. Overlapping +occurrences count as separate locations. Detailed source-line excerpts require +`read_file` or `preview_edit` on the same workspace; edit-only workers return a +generic failure and its error code without revealing file contents. A preview +itself exposes the resulting file text, so it is read-capable. + +Two optional features change matching, each negotiated in `editFileFeatures` +before Code API dispatches it: + +- `tolerant_match`: a request-level `matching: 'tolerant'` falls back from an + exact match to, in order, `line-trimmed` (ignores trailing whitespace and + CRLF), `indentation-flexible` (a uniformly shifted block, with `newText` + moved to the file's indentation) and `whitespace-normalized` (any whitespace + run between complete whitespace-delimited tokens, never a prefix or suffix + of another token). Without `replaceAll`, a match must still be unique; + replacements use the matched line's ending even in mixed-ending files. The + whitespace-normalized tier peels the complete shared newline-and-indentation + wrapper from `newText` even when CRLF/LF or nearby spaces differ, without + removing intentional extra line breaks or duplicating the source line ending. + Boundary whitespace claimed by `oldText`, including the number of line breaks, + must exist beside the matched tokens in the source; an attempt to remove it + with a token-only fallback fails rather than silently preserving it. Exact and + line-window matches can still replace terminators. + Excessively repetitive indentation candidates fail closed with a request + for more context rather than scanning every long window. Dense files reuse a + compact newline index for matching and diagnostics; overlapping exact matches + are counted without restarting a scan at each offset. +- `replace_all`: a batch edit's `replaceAll: true` replaces every + non-overlapping match instead of requiring exactly one, and still fails when + nothing matches. + +A request that sets `matching` or any `replaceAll` receives `matches`, one +`{ strategy, occurrences }` entry per edit. Requests that set neither receive +exactly the legacy result. Only IDs, names, protocol version, supported operations, and negotiated write modes appear in worker capabilities; absolute host paths remain local to the worker process. @@ -655,8 +697,10 @@ non-regular files, and commit through an owner-only temporary file followed by an atomic rename. The worker syncs the containing directory and verifies that the installed inode still contains the requested bytes before reporting success. Edits replace text only when the requested old text occurs exactly -once and reject if the file changes before commit. These operations do not -create directories or execute commands. +once (unless negotiated `replaceAll` selects every non-overlapping match) and +reject if the file changes before commit. Intermediate replacements are bounded +before construction, including `replaceAll`; these operations do not create +directories or execute commands. Register one directory already present on the worker machine with the worker-directory option: diff --git a/packages/code/src/edits.test.ts b/packages/code/src/edits.test.ts new file mode 100644 index 00000000..3ef9428d --- /dev/null +++ b/packages/code/src/edits.test.ts @@ -0,0 +1,721 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + applyTextEdits, + EDIT_DIAGNOSTIC_MAX_CHARS, + WorkspaceEditMatchError, + WorkspaceEditOutputLimitError, +} from './edits.js'; + +function rejection(run: () => unknown): WorkspaceEditMatchError { + try { + run(); + } catch (error) { + assert.ok(error instanceof WorkspaceEditMatchError); + return error; + } + assert.fail('expected the edits to be rejected'); +} + +test('exact matching keeps its single-location contract', () => { + const applied = applyTextEdits('alpha\nbeta\n', [{ oldText: 'beta', newText: 'gamma' }]); + assert.deepEqual(applied, { + text: 'alpha\ngamma\n', + matches: [{ strategy: 'exact', occurrences: 1 }], + }); +}); + +test('overlapping occurrences make an exact edit ambiguous', () => { + const error = rejection(() => applyTextEdits('aaa', [{ oldText: 'aa', newText: 'b' }])); + assert.match(error.message, /matched 2 locations/); + const applied = applyTextEdits('aaaa', [{ oldText: 'aa', newText: 'b', replaceAll: true }]); + assert.deepEqual(applied, { text: 'bb', matches: [{ strategy: 'exact', occurrences: 2 }] }); +}); + +test('later edits see the result of earlier ones', () => { + const applied = applyTextEdits('one', [ + { oldText: 'one', newText: 'two' }, + { oldText: 'two', newText: 'three' }, + ]); + assert.equal(applied.text, 'three'); +}); + +test('an ambiguous edit names every location instead of a bare conflict', () => { + const text = 'const a = 1;\nreturn a;\nconst b = 2;\nreturn a;\n'; + const error = rejection(() => + applyTextEdits(text, [{ oldText: 'return a;', newText: 'return b;' }]), + ); + assert.match(error.message, /matched 2 locations at lines 2, 4/); + assert.match(error.message, /include more surrounding lines/); + assert.match(error.message, /nothing was written/); +}); + +test('a batch reports every failing edit with its position in one rejection', () => { + const text = 'a\nb\nc\nb\n'; + const error = rejection(() => + applyTextEdits(text, [ + { oldText: 'a', newText: 'A' }, + { oldText: 'missing', newText: 'x' }, + { oldText: 'c', newText: 'C' }, + { oldText: 'b', newText: 'B' }, + ]), + ); + assert.deepEqual( + error.failures.map((failure) => failure.index), + [1, 3], + ); + assert.match(error.message, /^2 of 4 workspace edits did not apply/); + assert.match(error.message, /\nEdit 2: old_text was not found/); + assert.match(error.message, /\nEdit 4: old_text matched 2 locations at lines 2, 4/); +}); + +test('a missing edit points at the line it most likely meant', () => { + const text = 'function load(user) {\n return fetchUser(user.id);\n}\n'; + const error = rejection(() => + applyTextEdits(text, [ + { oldText: 'function load(user) {\n return fetchUser(user);\n}', newText: 'x' }, + ]), + ); + assert.match(error.message, /its first line appears at line 1, but the lines after it differ/); +}); + +test('a missing edit explains elided and line-numbered old_text', () => { + const text = 'start\nmiddle\nend\n'; + const elided = rejection(() => + applyTextEdits(text, [{ oldText: 'start\n// ...\nend', newText: 'x' }]), + ); + assert.match(elided.message, /elision placeholder/); + const numbered = rejection(() => + applyTextEdits(text, [{ oldText: '1 | start\n2 | middle', newText: 'x' }]), + ); + assert.match(numbered.message, /line-number prefixes/); +}); + +test('exact mode says when only the whitespace differs', () => { + const shifted = rejection(() => + applyTextEdits('class A {\n a();\n b();\n}\n', [{ oldText: 'a();\nb();', newText: 'x' }]), + ); + assert.match(shifted.message, /exists at line 2 with different whitespace \(indentation-flexible\)/); + const reflowed = rejection(() => + applyTextEdits('if (ready) {\n run();\n}\n', [ + { oldText: 'if (ready) {\n run();\n}', newText: 'x' }, + ]), + ); + assert.match(reflowed.message, /exists at line 1 with different whitespace \(whitespace-normalized\)/); +}); + +test('exact-mode diagnostics detect normalized whitespace with both boundary wrappers intact', () => { + for (const [source, oldText] of [ + ['foo bar\n', 'foo bar\n'], + ['header\n foo bar\n', '\nfoo bar'], + ['header\r\n foo bar\r\nnext', '\r\nfoo bar\r\n'], + [' foo bar \n', ' foo bar '], + ]) { + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: '' }])); + assert.match(error.message, /exists at line \d+ with different whitespace \(whitespace-normalized\)/, + JSON.stringify({ source, oldText })); + } + const missingBoundary = rejection(() => applyTextEdits('foo bar', [ + { oldText: 'foo bar\n', newText: 'unchanged' }, + ])); + assert.doesNotMatch(missingBoundary.message, /exists at.*different whitespace/); +}); + +test('tolerant matching ignores trailing whitespace and keeps CRLF line endings', () => { + const text = 'first \r\nsecond\t\r\nthird\r\n'; + const applied = applyTextEdits( + text, + [{ oldText: 'first\nsecond', newText: 'one\ntwo' }], + 'tolerant', + ); + assert.equal(applied.text, 'one\r\ntwo\r\nthird\r\n'); + assert.deepEqual(applied.matches, [{ strategy: 'line-trimmed', occurrences: 1 }]); +}); + +test('a trailing newline in old_text matches through the line terminator', () => { + const text = 'keep\ndrop\nkeep too\n'; + const applied = applyTextEdits(text, [{ oldText: 'drop \n', newText: '' }], 'tolerant'); + assert.equal(applied.text, 'keep\nkeep too\n'); +}); + +test('indentation-flexible matches move new_text to the file indentation', () => { + const text = 'class A {\n method() {\n return 1;\n }\n}\n'; + const applied = applyTextEdits( + text, + [ + { + oldText: 'method() {\n return 1;\n}', + newText: 'method() {\n const value = 2;\n return value;\n}', + }, + ], + 'tolerant', + ); + assert.equal( + applied.text, + 'class A {\n method() {\n const value = 2;\n return value;\n }\n}\n', + ); + assert.deepEqual(applied.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); +}); + +test('whitespace-normalized matches do not splice prefixes or suffixes of tokens', () => { + for (const [source, oldText] of [ + ['prereturn value;\n', 'return value;'], + ['return valueSuffix\n', 'return value'], + ['return value;postfix\n', 'return value;'], + ]) { + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant')); + assert.match(error.message, /old_text was not found/); + } + const valid = applyTextEdits('return value;\n', [ + { oldText: 'return value;', newText: 'return changed;' }, + ], 'tolerant'); + assert.equal(valid.text, 'return changed;\n'); + assert.deepEqual(valid.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('large whitespace-normalized edits match without compiling a request-sized regular expression', () => { + const oldText = `head ${'part '.repeat(16_000)}tail`; + const source = `before ${oldText.replace(/ /g, '\t')} after`; + const applied = applyTextEdits(source, [{ oldText, newText: 'result' }], 'tolerant'); + assert.equal(applied.text, 'before result after'); + assert.deepEqual(applied.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matches count overlapping token sequences but replaceAll does not overlap', () => { + const source = 'a b a\tb a'; + const edit = { oldText: 'a b a', newText: 'x' }; + const error = rejection(() => applyTextEdits(source, [edit], 'tolerant')); + assert.match(error.message, /matched 2 locations/); + const replaced = applyTextEdits(source, [{ ...edit, replaceAll: true }], 'tolerant'); + assert.equal(replaced.text, 'x\tb a'); + assert.deepEqual(replaced.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +for (const fileEnding of ['\n', '\r\n'] as const) { + for (const [oldEnding, newEnding] of [['\r\n', '\n'], ['\n', '\r\n']] as const) { + test(`whitespace-normalized matching peels ${JSON.stringify(oldEnding)} to ${JSON.stringify(newEnding)} in ${JSON.stringify(fileEnding)} files`, () => { + const source = `foo bar${fileEnding}next`; + const result = applyTextEdits(source, [{ + oldText: `foo bar${oldEnding}`, + newText: `baz qux${newEnding}`, + }], 'tolerant'); + assert.equal(result.text, `baz qux${fileEnding}next`); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + }); + } +} + +test('whitespace-normalized matching peels leading and trailing CRLF/LF wrappers in a batch', () => { + const result = applyTextEdits('header\r\n \tfoo bar\r\nnext\r\n', [ + { oldText: '\n foo bar\r\n', newText: '\r\n baz qux\n' }, + { oldText: 'next', newText: 'done' }, + ], 'tolerant'); + assert.equal(result.text, 'header\r\n \tbaz qux\r\ndone\r\n'); + assert.deepEqual(result.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); +}); + +test('whitespace-normalized matching peels a trailing newline even if adjacent spaces differ', () => { + const result = applyTextEdits('foo bar \r\nnext', [{ + oldText: 'foo bar \r\n', newText: 'baz qux\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux \r\nnext'); +}); + +test('whitespace-normalized matching does not prepend new indentation beside preserved source indentation', () => { + const source = 'header\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\n baz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('newline-only boundary prefixes peel the complete equivalent indentation run', () => { + for (const newline of ['\n', '\r\n']) { + const result = applyTextEdits(`header${newline} foo bar${newline}next`, [{ + oldText: '\nfoo bar\n', newText: '\n\tbaz qux\t\n', + }], 'tolerant'); + assert.equal(result.text, `header${newline} baz qux${newline}next`); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + } +}); + +test('wrapper peeling preserves source whitespace across complete prefixes and extra-newline combinations', () => { + for (const newline of ['\n', '\r\n']) { + for (const oldIndent of ['', ' ', '\t']) { + for (const newIndent of ['', ' ', '\t']) { + for (let extra = 0; extra < 3; extra++) { + const extraLines = `${newline}${newIndent}`.repeat(extra); + const leading = applyTextEdits(`header${newline} foo bar${newline}`, [{ + oldText: `${newline}${oldIndent}foo bar`, + newText: `${newline}${newIndent}${extraLines}baz qux`, + }], 'tolerant'); + assert.equal(leading.text, `header${newline} ${extraLines}baz qux${newline}`); + const trailing = applyTextEdits(`foo bar${newline} next`, [{ + oldText: `foo bar${newline}${oldIndent}`, + newText: `baz qux${extraLines}${newline}${newIndent}`, + }], 'tolerant'); + assert.equal(trailing.text, `baz qux${extraLines}${newline} next`); + } + } + } + } +}); + +test('horizontal boundary prefixes do not silently duplicate mismatched spaces', () => { + const error = rejection(() => applyTextEdits(' foo bar', [ + { oldText: ' foo bar', newText: ' baz qux' }, + ], 'tolerant')); + assert.match(error.message, /old_text was not found/); +}); + +test('whitespace-normalized matching preserves extra leading blank lines when indentation changes', () => { + const source = 'header\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\t\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\n \n\tbaz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized replaceAll preserves extra leading blank lines independently', () => { + const source = 'one\n foo bar\ntwo\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\t\n\tbaz qux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'one\n \n\tbaz qux\ntwo\n \n\tbaz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized matching preserves extra lines after an identical leading wrapper', () => { + const result = applyTextEdits('header\r\n foo bar', [{ + oldText: '\r\n foo bar', newText: '\r\n \r\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n \r\n\tbaz qux'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves multiple extra leading blank lines', () => { + const source = 'header\r\n \r\n foo bar\r\n'; + const result = applyTextEdits(source, [{ + oldText: '\n \n foo bar', newText: '\r\n\t\r\n\t\r\n\t\r\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n \r\n \r\n\t\r\n\tbaz qux\r\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves extra trailing blank lines when boundary indentation changes', () => { + const source = 'foo bar\r\n next'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar\n ', newText: 'baz qux\n\t\n\t', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\t\r\n next'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching leaves intentional indentation on an extra blank line', () => { + const result = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\n', newText: 'baz qux\n\t\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\t\r\nnext'); +}); + +test('whitespace-normalized matching does not duplicate differing trailing spaces', () => { + const result = applyTextEdits('foo bar \nnext', [{ + oldText: 'foo bar \n', newText: 'baz qux\t\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux \nnext'); +}); + +test('whitespace-normalized matching peels a leading newline even if adjacent spaces differ', () => { + const result = applyTextEdits('header\r\n foo bar\r\nnext', [{ + oldText: '\n foo bar', newText: '\nchanged', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n changed\r\nnext'); +}); + +test('whitespace-normalized matching rejects absent required boundary whitespace', () => { + const cases = [ + { source: 'foo bar', oldText: 'foo bar\r\n', newText: 'baz qux\n' }, + { source: 'foo bar next', oldText: 'foo bar\n', newText: 'baz qux\n' }, + { source: 'previous foo bar', oldText: '\nfoo bar', newText: '\nbaz qux' }, + { source: 'foo bar', oldText: ' foo bar', newText: ' baz qux' }, + { source: 'foo bar', oldText: 'foo bar ', newText: 'baz qux ' }, + { source: 'foo bar\nnext', oldText: 'foo bar\n\n', newText: 'baz qux\n\n' }, + { source: 'header\nfoo bar\nnext', oldText: '\n\nfoo bar', newText: '\n\nbaz qux' }, + { source: 'header\n \n foo bar', oldText: '\n \n foo bar', newText: '\n\tbaz qux' }, + ]; + for (const edit of cases) { + const { source, ...request } = edit; + const error = rejection(() => applyTextEdits(source, [request], 'tolerant')); + assert.match(error.message, /old_text was not found/, `unmatched boundary: ${JSON.stringify(edit)}`); + } +}); + +test('whitespace-normalized scanning skips boundary-invalid matches and keeps later valid ones', () => { + const source = 'header foo bar header\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: '\nfoo bar\n', newText: '\nbaz qux\n', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'header foo bar header\nbaz qux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching rejects boundary removal outside its token range', () => { + const cases = [ + { source: 'foo bar\r\nnext', oldText: 'foo bar\r\n', newText: 'baz qux' }, + { source: 'header\r\n foo bar', oldText: '\n foo bar', newText: 'baz qux' }, + { source: ' foo bar', oldText: ' foo bar', newText: 'baz qux' }, + { source: 'foo bar ', oldText: 'foo bar ', newText: 'baz qux' }, + { source: 'header\nfoo bar\nnext', oldText: '\nfoo bar\n', newText: '\n' }, + { source: 'foo bar\n\nnext', oldText: 'foo bar\n\n', newText: 'baz qux\n' }, + ]; + for (const edit of cases) { + const { source, ...request } = edit; + const error = rejection(() => applyTextEdits(source, [request], 'tolerant')); + assert.match(error.message, /old_text was not found/, `unremovable boundary: ${JSON.stringify(edit)}`); + } + const exact = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\r\n', newText: 'baz qux', + }], 'tolerant'); + assert.equal(exact.text, 'baz quxnext'); + assert.deepEqual(exact.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves two present boundary line breaks', () => { + const result = applyTextEdits('foo bar\n\nnext', [{ + oldText: 'foo bar\r\n\r\n', newText: 'baz qux\n\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\n\nnext'); +}); + +test('whitespace-normalized matching preserves an intentional extra line break', () => { + const result = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\r\n', newText: 'baz qux\n\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\r\nnext'); +}); + +test('line-trimmed replacements adopt the matched line ending in mixed files', () => { + const source = 'header\r\nfoo \nbar \nnext\n'; + const result = applyTextEdits(source, [{ oldText: 'foo\nbar', newText: 'baz\nqux' }], 'tolerant'); + assert.equal(result.text, 'header\r\nbaz\nqux\nnext\n'); + assert.deepEqual(result.matches, [{ strategy: 'line-trimmed', occurrences: 1 }]); +}); + +test('indentation-flexible replacements keep the selected line ending in mixed files', () => { + const source = 'header\r\n method() {\n return 1;\n }\nend\n'; + const result = applyTextEdits(source, [{ + oldText: 'method() {\n return 1;\n}', + newText: 'method() {\r\n return 2;\r\n}', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n method() {\n return 2;\n }\nend\n'); + assert.deepEqual(result.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); +}); + +test('line-trimmed replaceAll keeps each local ending in mixed files', () => { + const source = 'foo \r\nbar \r\nfoo \nbar \nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo\nbar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\r\nqux\r\nbaz\nqux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'line-trimmed', occurrences: 2 }]); +}); + +test('exact replacements preserve caller line endings rather than normalizing them', () => { + const source = 'foo bar\r\nnext\r\n'; + const result = applyTextEdits(source, [{ oldText: 'foo bar\r\n', newText: 'baz qux\n' }], 'tolerant'); + assert.equal(result.text, 'baz qux\nnext\r\n'); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('whitespace-normalized replacement adopts the matched line ending in mixed files', () => { + const original = 'header\r\nfoo bar\nnext\n'; + const result = applyTextEdits(original, [{ + oldText: 'foo bar\n', newText: 'baz\nqux\n', + }], 'tolerant'); + assert.equal(result.text, 'header\r\nbaz\nqux\nnext\n'); +}); + +test('whitespace-normalized replaceAll peels boundary newlines on every mixed-ending match', () => { + const source = 'foo bar\r\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar\r\n', newText: 'baz qux\n', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\nbaz qux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized replaceAll uses each matched line ending in mixed files', () => { + const source = 'foo bar\r\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\r\nqux\r\nbaz\nqux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized replaceAll finds many same-line matches without rescanning newline tails', () => { + const source = 'foo bar '.repeat(16_000); + const start = performance.now(); + const result = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\nqux '.repeat(16_000)); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 16_000 }]); + assert.ok(performance.now() - start < 2_000, 'line endings must not be searched from each match'); +}); + +test('whitespace-normalized matches do not indent new_text twice', () => { + const text = ' total = price *\n quantity;\n'; + const applied = applyTextEdits( + text, + [{ oldText: ' total = price * quantity;', newText: ' total = price * quantity * rate;' }], + 'tolerant', + ); + assert.equal(applied.text, ' total = price * quantity * rate;\n'); + assert.deepEqual(applied.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('tolerant matching still refuses an ambiguous edit', () => { + const error = rejection(() => + applyTextEdits('x = 1 \ny = 2\nx = 1\n', [{ oldText: 'x = 1', newText: 'x = 3' }], 'tolerant'), + ); + assert.match(error.message, /matched 2 locations at lines 1, 3/); +}); + +test('exact overlap counts UTF-16 offsets while replaceAll consumes whole characters', () => { + const error = rejection(() => applyTextEdits('😀😀😀', [ + { oldText: '😀😀', newText: 'x' }, + ])); + assert.match(error.message, /matched 2 locations/); + const result = applyTextEdits('😀😀😀', [{ oldText: '😀😀', newText: 'x', replaceAll: true }]); + assert.equal(result.text, 'x😀'); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('long overlapping exact matches are counted without repeatedly rescanning the input', () => { + const text = 'a'.repeat(450_000); + const oldText = 'a'.repeat(220_000); + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'b' }])); + assert.match(error.message, /matched 230001 locations/); + assert.ok(performance.now() - started < 3_000, 'overlapping ambiguity must be counted in linear time'); +}); + +test('highly repeated exact matches report the count with bounded line samples', () => { + const error = rejection(() => applyTextEdits('z'.repeat(200_000), [ + { oldText: 'z', newText: 'y' }, + ])); + assert.match(error.message, /matched 200000 locations at lines 1, 1, 1, 1, 1 and 199995 more/); +}); + +test('replaceAll replaces every location and reports the count', () => { + const applied = applyTextEdits('foo(); bar(); foo();', [ + { oldText: 'foo()', newText: 'baz()', replaceAll: true }, + ]); + assert.equal(applied.text, 'baz(); bar(); baz();'); + assert.deepEqual(applied.matches, [{ strategy: 'exact', occurrences: 2 }]); +}); + +test('replaceAll over line windows never overlaps its own matches', () => { + const applied = applyTextEdits( + 'a\na\na\na\n', + [{ oldText: 'a\na', newText: 'b', replaceAll: true }], + 'tolerant', + ); + assert.equal(applied.text, 'b\nb\n'); + assert.deepEqual(applied.matches, [{ strategy: 'exact', occurrences: 2 }]); +}); + +test('replaceAll bounds memory for a million identical and changed matches', () => { + const source = 'x'.repeat(1024 * 1024); + const noop = applyTextEdits(source, [ + { oldText: 'x', newText: 'x', replaceAll: true }, + { oldText: 'x', newText: 'y', replaceAll: true }, + ]); + assert.equal(noop.text, 'y'.repeat(source.length)); + assert.deepEqual(noop.matches, [ + { strategy: 'exact', occurrences: source.length }, + { strategy: 'exact', occurrences: source.length }, + ]); +}); + +test('replaceAll preserves non-overlapping literal edits when identical hits precede changed hits', () => { + for (const [source, oldText, newText] of [ + ['aaaaa', 'aa', 'a'], + ['foo foo foo', 'foo', 'bar'], + ['😀😀😀', '😀😀', 'x'], + ['x\nx\nx', 'x', ''], + ['aba aba', 'aba', 'aba'], + ]) { + const result = applyTextEdits(source, [{ oldText, newText, replaceAll: true }]); + assert.equal(result.text, source.split(oldText).join(newText)); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: source.split(oldText).length - 1 }]); + } + const source = 'foo\tbar foo bar'; + const tolerant = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'foo\tbar', replaceAll: true, + }], 'tolerant'); + assert.equal(tolerant.text, 'foo\tbar foo\tbar'); + assert.deepEqual(tolerant.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('replaceAll handles repeated tolerant and non-overlapping line-window matches', () => { + const lines = 'foo \r\nbar \r\n'.repeat(4000); + const result = applyTextEdits(lines, [ + { oldText: 'foo\nbar', newText: 'baz\nqux', replaceAll: true }, + { oldText: 'baz qux', newText: 'updated', replaceAll: true }, + ], 'tolerant'); + assert.equal(result.text, 'updated\r\n'.repeat(4000)); + assert.deepEqual(result.matches, [ + { strategy: 'line-trimmed', occurrences: 4000 }, + { strategy: 'whitespace-normalized', occurrences: 4000 }, + ]); +}); + +test('replaceAll rejects oversized intermediate output before constructing it', () => { + assert.throws( + () => applyTextEdits('x'.repeat(1_000_000), [ + { oldText: 'x', newText: 'y'.repeat(100_000), replaceAll: true }, + ]), + WorkspaceEditOutputLimitError, + ); +}); + +test('replaceAll still fails when nothing matches', () => { + const error = rejection(() => + applyTextEdits('abc', [{ oldText: 'xyz', newText: '', replaceAll: true }]), + ); + assert.match(error.message, /old_text was not found/); +}); + +test('line diagnostic index follows earlier successful batch edits', () => { + const error = rejection(() => applyTextEdits('alpha\nbeta\n', [ + { oldText: 'not present', newText: 'skip' }, + { oldText: 'alpha', newText: 'first\nsecond' }, + { oldText: 'beta\nmissing', newText: 'nope' }, + ])); + assert.match(error.message, /Edit 3: old_text was not found; its first line appears at line 3/); +}); + +test('newline-dense files do not materialize millions of line objects across failed batch edits', () => { + const source = 'a\n'.repeat(450_000); + const edits = Array.from({ length: 80 }, () => ({ oldText: 'missing\ntext', newText: 'other' })); + const started = performance.now(); + const error = rejection(() => applyTextEdits(source, edits)); + assert.equal(error.failures.length, edits.length); + assert.ok(performance.now() - started < 10_000, 'a bounded file must not re-index every failed diagnostic'); +}); + +test('first-line hints report bounded samples even when the line repeats throughout a file', () => { + const error = rejection(() => applyTextEdits('a\n'.repeat(100_000), [ + { oldText: 'a\nmissing', newText: 'replacement' }, + ])); + assert.match(error.message, /its first line appears at lines 1, 2, 3, 4, 5 and 99995 more/); +}); + +test('one uniquely anchored indentation candidate is allowed on both sides of the verification threshold', () => { + for (const length of [100_000, 100_001]) { + const oldText = ['anchor', ...Array.from({ length: length - 2 }, () => 'x'), 'end'].join('\n'); + const source = ` ${oldText.replaceAll('\n', '\n ')}\n`; + const result = applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant'); + assert.equal(result.text, ' changed\n'); + assert.deepEqual(result.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); + } +}); + +test('large indentation verification still bounds repeated candidates', () => { + const oldText = `${'x\n'.repeat(100_000)}x`; + const source = ' x\n'.repeat(100_002); + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant')); + assert.match(error.message, /too many repetitive line-window candidates/); +}); + +test('repetitive indentation candidates fail closed after a bounded comparison budget', () => { + const error = rejection(() => applyTextEdits(' a\n'.repeat(1_200), [ + { oldText: `${'a\n'.repeat(199)}a`, newText: 'changed' }, + ], 'tolerant')); + assert.match(error.message, /too many repetitive line-window candidates/); +}); + +test('repetitive long line windows return a missing-edit diagnosis without quadratic scans', () => { + const text = 'line\n'.repeat(24_000); + const oldText = `${'line\n'.repeat(6_000)}missing`; + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'replacement' }])); + assert.match(error.message, /did not apply and nothing was written/); + assert.ok(performance.now() - started < 2_000, 'a bounded edit must not compare every long window'); +}); + +test('a hundred failing multi-line edits on a large file are diagnosed quickly', () => { + const text = Array.from({ length: 30_000 }, (_, index) => ` line ${index} = value;`).join('\n'); + const edits = Array.from({ length: 100 }, (_, index) => ({ + oldText: `line ${index} = value;\n other ${index};\n more ${index};`, + newText: 'y', + })); + const started = performance.now(); + rejection(() => applyTextEdits(text, edits)); + assert.ok(performance.now() - started < 5_000); +}); + +test('diagnostics for a hundred failing edits stay within the settlement bound', () => { + const edits = Array.from({ length: 100 }, (_, index) => ({ + oldText: `missing ${'x'.repeat(200)} ${index}`, + newText: 'y', + })); + const error = rejection(() => applyTextEdits('content\n'.repeat(1000), edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.match(error.message, /^100 of 100 workspace edits did not apply/); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 100 }, (_, index) => index + 1), + ); + assert.doesNotMatch(error.message, /failing edits not shown/); +}); + +test('bounded batch diagnostics preserve sparse failing positions rather than only their count', () => { + const source = `KEEP\nconst shared = '${'x'.repeat(180)}';\n`; + const edits = Array.from({ length: 60 }, (_, index) => index % 2 === 0 + ? { oldText: `const shared missing_${index}`, newText: 'replacement' } + : { oldText: 'KEEP', newText: 'KEEP' }); + const error = rejection(() => applyTextEdits(source, edits)); + assert.equal(error.failures.length, 30); + assert.match(error.message, /^30 of 60 workspace edits did not apply/); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 30 }, (_, index) => 2 * index + 1), + ); + assert.doesNotMatch(error.message, /failing edits not shown/); +}); + +test('bounded diagnostics keep all positions and well-formed Unicode with mixed reason lengths', () => { + for (const count of [2, 30, 100]) { + const failures = Array.from({ length: count }, (_, index) => ({ + index, + reason: index % 3 === 0 ? 'missing' : `source excerpt ${'😀'.repeat(300)}`, + })); + const error = new WorkspaceEditMatchError(failures, count); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: count }, (_, index) => index + 1), + ); + } +}); + +test('short batch diagnostic reasons remain complete when the message fits', () => { + const error = rejection(() => applyTextEdits('a\nb\nb\n', [ + { oldText: 'missing', newText: 'x' }, + { oldText: 'b', newText: 'B' }, + ])); + for (const failure of error.failures) { + assert.ok(error.message.includes(`\nEdit ${failure.index + 1}: ${failure.reason}.`)); + } +}); diff --git a/packages/code/src/edits.ts b/packages/code/src/edits.ts new file mode 100644 index 00000000..a6d54269 --- /dev/null +++ b/packages/code/src/edits.ts @@ -0,0 +1,712 @@ +import { BRIDGE_WORKSPACE_WRITE_MAX_BYTES } from './protocol.js'; + +import type { + WorkspaceEditMatch, + WorkspaceEditMatching, + WorkspaceEditMatchStrategy, + WorkspaceTextEdit, +} from './protocol.js'; + +/** + * The Code API rejects a settlement whose error exceeds 4096 characters, and + * callers prefix their own context, so diagnostics stay well below that. + */ +export const EDIT_DIAGNOSTIC_MAX_CHARS = 3000; +const MAX_REPORTED_LINES = 5; +const MAX_SNIPPET_CHARS = 120; +/** A highly repetitive indentation candidate must not monopolize the worker. */ +const MAX_LINE_WINDOW_VERIFICATIONS = 100_000; +const MAX_REPLACEMENT_CHUNK_CHARS = 16 * 1024; + +interface MatchedRange { + start: number; + end: number; + /** Replacement for this range, already adapted to its indentation and line endings. */ + replacement: string; +} + +type MatchOutcome = + | { status: 'matched'; strategy: WorkspaceEditMatchStrategy; ranges: MatchedRange[]; occurrences: number; updated?: string } + | { status: 'ambiguous'; strategy: WorkspaceEditMatchStrategy; count: number; starts: number[] } + | { status: 'limit' } + | { status: 'none' }; + +interface CollectedMatches { + count: number; + source: string; + projectedLength: number; + /** Only the first range is needed for a unique edit or an exact-mode hint. */ + ranges: MatchedRange[]; + starts: number[]; + output?: { chunks: string[]; pending: string; cursor: number }; +} + +export interface EditFailure { + /** Zero-based position of the edit in the request. */ + index: number; + reason: string; +} + +export class WorkspaceEditMatchError extends Error { + constructor( + readonly failures: EditFailure[], + readonly editCount: number, + ) { + super(formatEditFailures(failures, editCount)); + this.name = 'WorkspaceEditMatchError'; + } +} + +export class WorkspaceEditOutputLimitError extends Error { + constructor() { + super('Workspace file exceeds write limit'); + this.name = 'WorkspaceEditOutputLimitError'; + } +} + +export interface AppliedEdits { + text: string; + matches: WorkspaceEditMatch[]; +} + +/** + * Applies ordered replacements to `text`. Each edit sees the result of the + * edits before it. Every edit is attempted even after a failure, so a caller + * learns about all unmatched or ambiguous edits from one rejection; nothing is + * returned unless every edit applied. + */ +export function applyTextEdits( + text: string, + edits: readonly WorkspaceTextEdit[], + matching: WorkspaceEditMatching = 'exact', +): AppliedEdits { + let working = text; + let lineIndex: LineIndex | undefined; + const lines = (): LineIndex => (lineIndex ??= new LineIndex(working)); + const matches: WorkspaceEditMatch[] = []; + const failures: EditFailure[] = []; + edits.forEach((edit, index) => { + const outcome = findEditMatch(working, edit, matching, lines); + if (outcome.status === 'matched') { + working = outcome.updated ?? replaceRanges(working, outcome.ranges); + lineIndex = undefined; + matches.push({ strategy: outcome.strategy, occurrences: outcome.occurrences }); + return; + } + failures.push({ + index, + reason: + outcome.status === 'ambiguous' + ? describeAmbiguous(working, outcome.strategy, outcome.count, outcome.starts) + : outcome.status === 'limit' + ? 'old_text has too many repetitive line-window candidates; include more surrounding lines or use an exact match' + : describeMissing(working, edit.oldText, matching, lines), + }); + }); + if (failures.length > 0) { + throw new WorkspaceEditMatchError(failures, edits.length); + } + return { text: working, matches }; +} + +function findEditMatch( + text: string, + edit: WorkspaceTextEdit, + matching: WorkspaceEditMatching, + lines: () => LineIndex, +): MatchOutcome { + const strategies = matching === 'tolerant' ? TOLERANT_STRATEGIES : EXACT_STRATEGIES; + for (const find of strategies) { + const outcome = find(text, edit, lines); + if (outcome.status !== 'none') return outcome; + } + return { status: 'none' }; +} + +function collectedMatches(text: string, replaceAll?: boolean): CollectedMatches { + return { + count: 0, source: text, projectedLength: text.length, starts: [], ranges: [], + ...(replaceAll === true ? { output: { chunks: [], pending: '', cursor: 0 } } : {}), + }; +} + +function appendReplacement(output: NonNullable, part: string): void { + if (!part) return; + output.pending += part; + if (output.pending.length >= MAX_REPLACEMENT_CHUNK_CHARS) { + output.chunks.push(output.pending); + output.pending = ''; + } +} + +function finishReplacement(matches: CollectedMatches): string { + const output = matches.output!; + if (output.cursor === 0) return matches.source; + appendReplacement(output, matches.source.slice(output.cursor)); + return output.chunks.join('') + output.pending; +} + +function collectMatch( + matches: CollectedMatches, + start: number, + end: number, + replacement: string, + replaceAll?: boolean, +): void { + if (replaceAll === true) { + matches.projectedLength += replacement.length - (end - start); + // Even deleting every remaining source character cannot bring this + // intermediate below the 1 MiB limit. Do not retain more matches. + if (matches.projectedLength - (matches.source.length - end) > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceEditOutputLimitError(); + } + // Leave unchanged ranges in the source suffix rather than constructing a + // million identical output pieces for a no-op replaceAll edit. + if (replacement !== matches.source.slice(start, end)) { + const output = matches.output!; + appendReplacement(output, matches.source.slice(output.cursor, start)); + appendReplacement(output, replacement); + output.cursor = end; + } + } + matches.count++; + if (matches.starts.length < MAX_REPORTED_LINES) matches.starts.push(start); + if (matches.count === 1) matches.ranges.push({ start, end, replacement }); +} + +function resolve( + strategy: WorkspaceEditMatchStrategy, + matches: CollectedMatches, + replaceAll: boolean | undefined, +): MatchOutcome { + if (matches.count === 0) return { status: 'none' }; + if (matches.count === 1 || replaceAll === true) { + return { + status: 'matched', strategy, ranges: matches.ranges, occurrences: matches.count, + ...(replaceAll === true ? { updated: finishReplacement(matches) } : {}), + }; + } + return { status: 'ambiguous', strategy, count: matches.count, starts: matches.starts }; +} + +/** + * Overlapping occurrences count toward ambiguity (`aa` in `aaa` is two + * locations), while `replaceAll` replaces non-overlapping occurrences. + */ +function findExact(text: string, edit: WorkspaceTextEdit): MatchOutcome { + if (edit.oldText.length === 0) return { status: 'none' }; + const matches = collectedMatches(text, edit.replaceAll); + const prefix = prefixTable(edit.oldText); + let matched = 0; + for (let index = 0; index < text.length; index++) { + while (matched > 0 && text[index] !== edit.oldText[matched]) matched = prefix[matched - 1]; + if (text[index] === edit.oldText[matched]) matched++; + if (matched !== edit.oldText.length) continue; + collectMatch(matches, index - matched + 1, index + 1, edit.newText, edit.replaceAll); + // Ambiguity counts overlaps, but replaceAll must consume disjoint ranges. + matched = edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + return resolve('exact', matches, edit.replaceAll); +} + +interface Line { + /** Offset of the first character of the line. */ + start: number; + /** Offset just past the line's content, excluding `\r\n` or `\n`. */ + end: number; + /** Offset just past the line terminator, or `undefined` for an unterminated last line. */ + next: number | undefined; + text: string; +} + +/** One bounded index per source revision, shared by matching and diagnostics. */ +class LineIndex { + private readonly newlines: Uint32Array; + readonly length: number; + + constructor(private readonly source: string) { + this.newlines = new Uint32Array(newlineCount(source)); + let index = 0; + for (let offset = source.indexOf('\n'); offset >= 0; offset = source.indexOf('\n', offset + 1)) { + this.newlines[index++] = offset; + } + // A trailing terminator leaves an empty final line, just as before. + this.length = this.newlines.length + 1; + } + + start(index: number): number { + return index === 0 ? 0 : this.newlines[index - 1] + 1; + } + + next(index: number): number | undefined { + return index < this.newlines.length ? this.newlines[index] + 1 : undefined; + } + + end(index: number): number { + const start = this.start(index); + const end = index < this.newlines.length ? this.newlines[index] : this.source.length; + return end > start && this.source[end - 1] === '\r' ? end - 1 : end; + } + + text(index: number): string { + return this.source.slice(this.start(index), this.end(index)); + } + + at(index: number): Line { + return { start: this.start(index), end: this.end(index), next: this.next(index), text: this.text(index) }; + } +} + +/** + * Lines a line-window strategy must match. A trailing line break means "through + * the end of the last line", not "followed by an empty line". + */ +function neededLines(oldText: string): { lines: string[]; throughTerminator: boolean } { + const normalized = oldText.replace(/\r\n/g, '\n'); + const throughTerminator = normalized.endsWith('\n'); + return { + lines: (throughTerminator ? normalized.slice(0, -1) : normalized).split('\n'), + throughTerminator, + }; +} + +/** Fallback when neither the matched line nor its neighbors have a terminator. */ +function fileLineEnding(text: string): '\r\n' | '\n' { + return text.includes('\r\n') ? '\r\n' : '\n'; +} + +function lineEndingAt(text: string, newline: number): '\r\n' | '\n' { + return newline > 0 && text[newline - 1] === '\r' ? '\r\n' : '\n'; +} + +function leadingWhitespace(line: string): string { + return /^[ \t]*/.exec(line)?.[0] ?? ''; +} + +function commonIndent(lines: Iterable): string { + let common: string | undefined; + for (const line of lines) { + if (line.trim().length === 0) continue; + const indent = leadingWhitespace(line); + if (common === undefined) { + common = indent; + continue; + } + let shared = 0; + while (shared < common.length && shared < indent.length && common[shared] === indent[shared]) { + shared++; + } + common = common.slice(0, shared); + } + return common ?? ''; +} + +function withLineEnding(value: string, ending: '\r\n' | '\n'): string { + const normalized = value.replace(/\r\n/g, '\n'); + return ending === '\r\n' ? normalized.replace(/\n/g, '\r\n') : normalized; +} + +function prefixTable(values: ArrayLike): Uint32Array { + const prefix = new Uint32Array(values.length); + for (let index = 1, matched = 0; index < values.length; index++) { + while (matched > 0 && values[index] !== values[matched]) matched = prefix[matched - 1]; + if (values[index] === values[matched]) matched++; + prefix[index] = matched; + } + return prefix; +} + +/** + * Line-window strategies compare whole lines, so a match always spans from the + * start of its first line to the end of its last line's content. The file's + * own line terminators are kept, and the replacement adopts them. + */ +function findLineWindows( + text: string, + edit: WorkspaceTextEdit, + strategy: 'line-trimmed' | 'indentation-flexible', + getLines: () => LineIndex, +): MatchOutcome { + const { lines: needle, throughTerminator } = neededLines(edit.oldText); + if (needle.every((line) => line.trim().length === 0)) return { status: 'none' }; + const lines = getLines(); + const ending = fileLineEnding(text); + const needleIndent = strategy === 'indentation-flexible' ? commonIndent(needle) : ''; + const normalizedNeedle = needle.map((line) => + (strategy === 'indentation-flexible' ? stripIndent(line, needleIndent) : line).trimEnd(), + ); + if (needle.length > lines.length) return { status: 'none' }; + // For line-trimmed matching, the normalized lines are the complete comparison. + // For indentation-flexible matching, whole-line content is a linear-time + // prefilter; only complete candidates need their relative indent verified. + const sought = strategy === 'line-trimmed' + ? normalizedNeedle + : normalizedNeedle.map((line) => line.trimStart()); + const prefix = prefixTable(sought); + const collected = collectedMatches(text, edit.replaceAll); + let matched = 0; + let verifications = 0; + // Allow one complete linear verification even for a large valid block. + const verificationBudget = Math.max(MAX_LINE_WINDOW_VERIFICATIONS, needle.length); + for (let index = 0; index < lines.length; index++) { + const value = strategy === 'line-trimmed' ? lines.text(index).trimEnd() : lines.text(index).trim(); + while (matched > 0 && value !== sought[matched]) matched = prefix[matched - 1]; + if (value === sought[matched]) matched++; + if (matched !== sought.length) continue; + + const first = index - sought.length + 1; + const end = throughTerminator ? lines.next(index) : lines.end(index); + let windowIndent = ''; + let valid = end !== undefined; + if (valid && strategy === 'indentation-flexible') { + if (verifications + needle.length > verificationBudget) return { status: 'limit' }; + verifications += needle.length; + windowIndent = commonIndent((function* () { + for (let offset = 0; offset < needle.length; offset++) yield lines.text(first + offset); + })()); + valid = normalizedNeedle.every((expected, offset) => + stripIndent(lines.text(first + offset), windowIndent).trimEnd() === expected, + ); + } + if (valid) { + const replacement = strategy === 'indentation-flexible' + ? reindent(edit.newText, needleIndent, windowIndent) + : edit.newText; + const lineFeed = lines.next(first) ?? (first > 0 ? lines.next(first - 1) : undefined); + const localEnding = lineFeed === undefined ? ending : lineEndingAt(text, lineFeed - 1); + collectMatch(collected, lines.start(first), end!, withLineEnding(replacement, localEnding), edit.replaceAll); + } + // A replacement cannot consume overlapping lines; ambiguity still counts them. + matched = valid && edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + return resolve(strategy, collected, edit.replaceAll); +} + +function stripIndent(line: string, indent: string): string { + return line.startsWith(indent) ? line.slice(indent.length) : line.trimStart(); +} + +/** + * Moves `value` from the indentation the caller wrote to the file's own. Lines + * the caller indented less than its `old_text` keep their indentation as written. + */ +function reindent(value: string, from: string, to: string): string { + if (from === to) return value; + return value + .replace(/\r\n/g, '\n') + .split('\n') + .map((line) => + line.trim().length > 0 && line.startsWith(from) ? to + line.slice(from.length) : line, + ) + .join('\n'); +} + +function findLineTrimmed(text: string, edit: WorkspaceTextEdit, lines: () => LineIndex): MatchOutcome { + return findLineWindows(text, edit, 'line-trimmed', lines); +} + +function findIndentationFlexible(text: string, edit: WorkspaceTextEdit, lines: () => LineIndex): MatchOutcome { + return findLineWindows(text, edit, 'indentation-flexible', lines); +} + +function newlineCount(text: string): number { + let count = 0; + for (let index = text.indexOf('\n'); index >= 0; index = text.indexOf('\n', index + 1)) count++; + return count; +} + +/** A boundary supplied by oldText must exist beside the candidate tokens. */ +function hasBoundaryWhitespace( + text: string, + position: number, + direction: -1 | 1, + requiredNewlines: number, +): boolean { + for (let index = position; index >= 0 && index < text.length; index += direction) { + const char = text[index]; + if (!/\s/.test(char)) break; + if (char === '\n') requiredNewlines--; + if (requiredNewlines <= 0) return true; + } + return false; +} + +/** A token-only match leaves the source's outer whitespace untouched. */ +function peelBoundaryWhitespace( + text: string, + boundary: string, + side: 'leading' | 'trailing', +): string | undefined { + if (!boundary) return text; + const whitespace = (side === 'leading' ? /^\s*/ : /\s*$/).exec(text)?.[0] ?? ''; + if (whitespace === boundary) { + return side === 'leading' ? text.slice(boundary.length) : text.slice(0, -boundary.length); + } + + const requiredNewlines = newlineCount(boundary); + if (!requiredNewlines) return undefined; + if (newlineCount(whitespace) < requiredNewlines) return undefined; + + if (side === 'leading') { + let end = 0; + for (let count = 0; count < requiredNewlines; count++) { + end = whitespace.indexOf('\n', end) + 1; + } + // Consume the equivalent indentation, stopping before the first extra + // newline. Extra blank lines belong to the replacement, not the wrapper. + while (end < whitespace.length && whitespace[end] !== '\n') end++; + return text.slice(end); + } + + if (newlineCount(whitespace) === requiredNewlines) { + return text.slice(0, text.length - whitespace.length); + } + let start = whitespace.length; + for (let count = 0; count < requiredNewlines; count++) { + start = whitespace.lastIndexOf('\n', start - 1); + } + // Indentation before this newline belongs to the extra blank line. + return text.slice(0, text.length - (whitespace.length - start)); +} + +/** + * Tolerates any run of whitespace, including line breaks, between tokens. The + * match starts and ends on a token, so whitespace the caller wrapped around + * `old_text` is also peeled off `new_text` rather than inserted twice. + */ +function findWhitespaceNormalized(text: string, edit: WorkspaceTextEdit): MatchOutcome { + const oldText = edit.oldText.replace(/\r\n/g, '\n'); + const tokens = oldText.trim().split(/\s+/).filter(Boolean); + if (tokens.length < 2) return { status: 'none' }; + // The matched range contains tokens only; leave the file's boundary whitespace + // outside it. Normalize the caller's line endings *before* peeling equivalent + // wrappers, otherwise CRLF/LF differences insert a second line break. + const leading = /^\s*/.exec(oldText)?.[0] ?? ''; + const trailing = /\s*$/.exec(oldText)?.[0] ?? ''; + const leadingNewlines = newlineCount(leading); + const trailingNewlines = newlineCount(trailing); + const normalizedNewText = edit.newText.replace(/\r\n/g, '\n'); + const withoutLeading = peelBoundaryWhitespace(normalizedNewText, leading, 'leading'); + if (withoutLeading === undefined) return { status: 'none' }; + const newText = peelBoundaryWhitespace(withoutLeading, trailing, 'trailing'); + if (newText === undefined) return { status: 'none' }; + const lfReplacement = newText; + const crlfReplacement = withLineEnding(newText, '\r\n'); + // Match entire whitespace-delimited tokens, never an identifier prefix or + // suffix. A fixed-size regex tokenizes the file; KMP keeps repetitive input + // linear without compiling user-provided text as a regular expression. + const prefix = prefixTable(tokens); + const tokenStarts = new Uint32Array(tokens.length); + const collected = collectedMatches(text, edit.replaceAll); + const words = /\S+/g; + let matched = 0; + let tokenIndex = 0; + // Advance the newline cursor only forwards. replaceAll may encounter many + // matches on one long line, so searching from each match would be quadratic. + let nextNewline = text.indexOf('\n'); + let previousNewline = -1; + for (let word = words.exec(text); word != null; word = words.exec(text)) { + tokenStarts[tokenIndex % tokens.length] = word.index; + while (matched > 0 && word[0] !== tokens[matched]) matched = prefix[matched - 1]; + if (word[0] === tokens[matched]) matched++; + if (matched === tokens.length) { + const start = tokenStarts[(tokenIndex + 1) % tokens.length]; + const end = word.index + word[0].length; + const boundariesMatch = + (leading.length === 0 || hasBoundaryWhitespace(text, start - 1, -1, leadingNewlines)) && + (trailing.length === 0 || hasBoundaryWhitespace(text, end, 1, trailingNewlines)); + if (boundariesMatch) { + while (nextNewline >= 0 && nextNewline < start) { + previousNewline = nextNewline; + nextNewline = text.indexOf('\n', nextNewline + 1); + } + const nearestNewline = nextNewline >= 0 ? nextNewline : previousNewline; + const replacement = nearestNewline >= 0 && lineEndingAt(text, nearestNewline) === '\r\n' + ? crlfReplacement + : lfReplacement; + collectMatch(collected, start, end, replacement, edit.replaceAll); + } + matched = boundariesMatch && edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + tokenIndex++; + } + return resolve('whitespace-normalized', collected, edit.replaceAll); +} + +type Strategy = (text: string, edit: WorkspaceTextEdit, lines: () => LineIndex) => MatchOutcome; + +/** + * Loosest last. Line-window strategies run before whitespace normalization + * because they can carry `new_text` over to the file's indentation; a + * whitespace-normalized match would insert it exactly as written. + */ +const RELAXED_STRATEGIES: readonly Strategy[] = [ + findLineTrimmed, + findIndentationFlexible, + findWhitespaceNormalized, +]; +const EXACT_STRATEGIES: readonly Strategy[] = [findExact]; +const TOLERANT_STRATEGIES: readonly Strategy[] = [findExact, ...RELAXED_STRATEGIES]; + +function replaceRanges(text: string, ranges: readonly MatchedRange[]): string { + const length = ranges.reduce( + (total, range) => total + range.replacement.length - (range.end - range.start), + text.length, + ); + // Every valid UTF-8 string has at least this many encoded bytes. Reject a + // pathological replaceAll before allocating an unbounded intermediate string. + if (length > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) throw new WorkspaceEditOutputLimitError(); + let result = ''; + let cursor = 0; + for (const range of ranges) { + result += text.slice(cursor, range.start) + range.replacement; + cursor = range.end; + } + return result + text.slice(cursor); +} + +function lineNumberAt(text: string, offset: number): number { + let line = 1; + for (let index = text.indexOf('\n'); index >= 0 && index < offset; index = text.indexOf('\n', index + 1)) { + line++; + } + return line; +} + +function formatLineList(text: string, starts: readonly number[], count = starts.length): string { + const shown = starts.slice(0, MAX_REPORTED_LINES).map((start) => lineNumberAt(text, start)); + const more = count - shown.length; + return `line${shown.length === 1 ? '' : 's'} ${shown.join(', ')}${more > 0 ? ` and ${more} more` : ''}`; +} + +function describeAmbiguous( + text: string, + strategy: WorkspaceEditMatchStrategy, + count: number, + starts: readonly number[], +): string { + const how = strategy === 'exact' ? '' : ` (${strategy})`; + return `old_text matched ${count} locations${how} at ${formatLineList(text, starts, count)}; include more surrounding lines so it matches exactly one`; +} + +function snippet(line: string): string { + const trimmed = line.trim(); + return JSON.stringify( + trimmed.length > MAX_SNIPPET_CHARS ? `${trimmed.slice(0, MAX_SNIPPET_CHARS)}…` : trimmed, + ); +} + +const ELISION_LINE = /^\s*(?:(?:\/\/|#|--|\/\*|\*|))?\s*$/; +const LINE_NUMBER_PREFIX = /^\s*\d+\s*(?:\||:|\t)/; + +function describeMissing( + text: string, + oldText: string, + matching: WorkspaceEditMatching, + lines: () => LineIndex, +): string { + const hints: string[] = []; + const nonBlank = neededLines(oldText).lines.filter((line) => line.trim().length > 0); + if (nonBlank.some((line) => ELISION_LINE.test(line))) { + hints.push('it contains an elision placeholder ("..."); copy the exact lines instead of abbreviating'); + } + if (nonBlank.length > 0 && nonBlank.every((line) => LINE_NUMBER_PREFIX.test(line))) { + hints.push('it appears to include line-number prefixes from read_file output; remove them'); + } + if (matching === 'exact') { + let tolerant: MatchOutcome | undefined; + for (const find of RELAXED_STRATEGIES) { + // Diagnose match existence, not whether the requested replacement can + // preserve its out-of-range wrappers. A no-op keeps those wrappers valid. + tolerant = find(text, { oldText, newText: oldText }, lines); + if (tolerant.status !== 'none') break; + } + if (tolerant?.status === 'matched') { + hints.push( + `the same text exists at ${formatLineList(text, [tolerant.ranges[0].start])} with different whitespace (${tolerant.strategy}); copy that whitespace exactly`, + ); + } else if (text.includes('\r\n') && !oldText.includes('\r\n') && oldText.includes('\n')) { + hints.push('the file uses CRLF line endings'); + } + } + const nearest = nearestLine(nonBlank[0], lines); + if (nearest != null && hints.length === 0) { + hints.push( + nearest.exact + ? `its first line appears at ${formatLineList(text, nearest.starts, nearest.count)}, but the lines after it differ` + : `the closest line is ${formatLineList(text, nearest.starts)}: ${snippet(nearest.text)}`, + ); + } + return `old_text was not found${hints.length > 0 ? `; ${hints.join('; ')}` : ''}`; +} + +/** Finds where the first line of a failed edit most likely belongs. */ +function nearestLine( + firstLine: string | undefined, + getLines: () => LineIndex, +): { exact: boolean; count: number; starts: number[]; text: string } | undefined { + const target = firstLine?.trim(); + if (!target) return undefined; + const lines = getLines(); + const starts: number[] = []; + let count = 0; + let firstMatch = ''; + for (let index = 0; index < lines.length; index++) { + const content = lines.text(index); + if (content.trim() !== target) continue; + if (count++ === 0) firstMatch = content; + if (starts.length < MAX_REPORTED_LINES) starts.push(lines.start(index)); + } + if (count > 0) { + return { exact: true, count, starts, text: firstMatch }; + } + const tokens = new Set(target.split(/\W+/).filter((token) => token.length > 1)); + if (tokens.size < 2) return undefined; + let best: Line | undefined; + let bestScore = 0; + for (let index = 0; index < lines.length; index++) { + const content = lines.text(index); + let score = 0; + for (const token of new Set(content.split(/\W+/))) { + if (tokens.has(token)) score++; + } + if (score > bestScore) { + best = { start: lines.start(index), end: lines.end(index), next: lines.next(index), text: content }; + bestScore = score; + } + } + return best != null && bestScore / tokens.size >= 0.5 + ? { exact: false, count: 1, starts: [best.start], text: best.text } + : undefined; +} + +function formatEditFailures(failures: readonly EditFailure[], editCount: number): string { + if (editCount === 1) { + return `Workspace edit did not apply and nothing was written: ${failures[0]?.reason ?? 'no match'}.`.slice( + 0, + EDIT_DIAGNOSTIC_MAX_CHARS, + ); + } + const header = `${failures.length} of ${editCount} workspace edits did not apply, so nothing was written. Every other edit matched.`; + const footer = failures.some((failure) => failure.index > 0) + ? '\nLine numbers account for the earlier edits in this batch.' + : ''; + const prefixes = failures.map((failure) => `\nEdit ${failure.index + 1}: `); + // Labels and punctuation for every failing position take priority over long + // reasons or source excerpts. Never leave callers guessing which edits failed. + const available = EDIT_DIAGNOSTIC_MAX_CHARS - header.length - footer.length - + prefixes.reduce((total, prefix) => total + prefix.length + 1, 0); + const reasonLength = failures.reduce((total, failure) => total + failure.reason.length, 0); + const reasonLimit = reasonLength <= available ? Infinity : Math.floor(available / failures.length); + const details = failures.map((failure, index) => { + let reason = failure.reason; + if (reason.length > reasonLimit) { + let end = reasonLimit - 1; + // Do not split a surrogate pair in a shortened source excerpt. + if (end > 0 && /[\uD800-\uDBFF]/.test(reason[end - 1]) && /[\uDC00-\uDFFF]/.test(reason[end])) end--; + reason = `${reason.slice(0, end)}…`; + } + return `${prefixes[index]}${reason}.`; + }); + return header + details.join('') + footer; +} diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 729b20e2..340ddfdf 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -545,6 +545,164 @@ test('workspace mutations accept bounded UTF-8 requests and exact result shapes' ); }); +test('edit matching and replaceAll are opt-in and reported only when requested', () => { + const tolerant = { + protocolVersion: 1 as const, + operation: 'edit_file' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: 'hello', newText: 'goodbye' }, + { oldText: 'world', newText: 'BYOM', replaceAll: true }, + ], + }; + const result = { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + replacements: 2, + bytesWritten: 12, + }; + const matches = [ + { strategy: 'line-trimmed', occurrences: 1 }, + { strategy: 'exact', occurrences: 3 }, + ]; + assert.equal(isWorkspaceToolRequest(tolerant), true); + assert.equal(isWorkspaceToolRequest({ ...tolerant, operation: 'preview_edit' }), true); + assert.equal(isWorkspaceToolRequest({ ...tolerant, matching: 'fuzzy' }), false); + assert.equal( + isWorkspaceToolRequest({ + ...tolerant, + edits: [{ oldText: 'a', newText: 'b', replaceAll: 'yes' }], + }), + false, + ); + assert.equal( + isWorkspaceToolRequest({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'a', + newText: 'b', + replaceAll: true, + }), + false, + 'replaceAll is only accepted per batch edit', + ); + + assert.equal(isWorkspaceToolResult(tolerant, { ...result, matches }), true); + assert.equal(isWorkspaceToolResult(tolerant, result), false, 'an opted-in request must report matches'); + assert.equal( + isWorkspaceToolResult(tolerant, { ...result, matches: matches.slice(0, 1) }), + false, + ); + assert.equal( + isWorkspaceToolResult(tolerant, { + ...result, + matches: [matches[0], { strategy: 'exact', occurrences: 0 }], + }), + false, + ); + assert.equal( + isWorkspaceToolResult(tolerant, { + ...result, + matches: [{ strategy: 'exact', occurrences: 2 }, matches[1]], + }), + false, + 'only replaceAll edits may replace more than one location', + ); + + const exactReplaceAll = { + ...tolerant, + matching: undefined, + edits: [{ oldText: 'a', newText: 'b', replaceAll: true }], + }; + delete exactReplaceAll.matching; + assert.equal( + isWorkspaceToolResult(exactReplaceAll, { + ...result, + replacements: 1, + matches: [{ strategy: 'line-trimmed', occurrences: 2 }], + }), + false, + 'without tolerant matching every edit matches exactly', + ); + + const legacy = { ...tolerant, edits: [{ oldText: 'a', newText: 'b' }] }; + delete (legacy as { matching?: string }).matching; + assert.equal( + isWorkspaceToolResult(legacy, { + ...result, + replacements: 1, + matches: [{ strategy: 'exact', occurrences: 1 }], + }), + false, + 'legacy requests never receive a matches field', + ); +}); + +test('edit features advertise any unique subset of the known features', () => { + const valid = { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'edit_file'], + workspaces: [{ id: 'primary' }], + editFileModes: ['single', 'batch'], + }, + }; + const withFeatures = (editFileFeatures: unknown) => ({ + ...valid, + workspaceTools: { ...valid.workspaceTools, editFileFeatures }, + }); + for (const features of [ + ['expected_base_sha256'], + ['tolerant_match'], + ['expected_base_sha256', 'tolerant_match', 'replace_all'], + ]) { + assert.equal(isValidBridgeWorkerCapabilities(withFeatures(features)), true, features.join(',')); + } + for (const features of [[], ['fuzzy'], ['replace_all', 'replace_all']]) { + assert.equal(isValidBridgeWorkerCapabilities(withFeatures(features)), false, features.join(',')); + } +}); + +test('preview-only workers may advertise tolerant matching and replace-all, but not edit-only hashes', () => { + const previewOnly = { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'preview_edit'], + workspaces: [{ id: 'primary' }], + editFileModes: ['single', 'batch'], + }, + }; + for (const features of [ + ['tolerant_match'], + ['replace_all'], + ['tolerant_match', 'replace_all'], + ]) { + assert.equal(isValidBridgeWorkerCapabilities({ + ...previewOnly, + workspaceTools: { ...previewOnly.workspaceTools, editFileFeatures: features }, + }), true, features.join(',')); + } + assert.equal(isValidBridgeWorkerCapabilities({ + ...previewOnly, + workspaceTools: { + ...previewOnly.workspaceTools, + editFileFeatures: ['expected_base_sha256', 'tolerant_match'], + }, + }), false); +}); + test('workspace commands require bounded sandbox inputs and outputs', () => { const request = { protocolVersion: 1 as const, diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index b23ad327..26111273 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -317,7 +317,29 @@ export type BridgeWorkspaceToolOperation = export type WorkspaceWriteFileMode = 'replace' | 'create'; export type WorkspaceEditFileMode = 'single' | 'batch'; -export type WorkspaceEditFileFeature = 'expected_base_sha256'; +export type WorkspaceEditFileFeature = + | 'expected_base_sha256' + | 'tolerant_match' + | 'replace_all'; +/** Every edit feature this protocol version defines, for capability validation. */ +export const WORKSPACE_EDIT_FILE_FEATURES: readonly WorkspaceEditFileFeature[] = [ + 'expected_base_sha256', + 'tolerant_match', + 'replace_all', +]; +/** `tolerant` falls back from exact matching to whitespace-tolerant strategies. */ +export type WorkspaceEditMatching = 'exact' | 'tolerant'; +export type WorkspaceEditMatchStrategy = + | 'exact' + | 'line-trimmed' + | 'whitespace-normalized' + | 'indentation-flexible'; +const WORKSPACE_EDIT_MATCH_STRATEGIES = new Set([ + 'exact', + 'line-trimmed', + 'whitespace-normalized', + 'indentation-flexible', +]); export type WorkspaceListFileFeature = 'after_path'; export type WorkspaceProgrammaticLanguage = 'bash'; @@ -483,6 +505,8 @@ interface WorkspaceEditFileRequestBase { path: string; /** Refuses the mutation unless current file bytes match this preview revision. */ expectedBaseSha256?: string; + /** Requires the `tolerant_match` edit feature. Omitted means `exact`. */ + matching?: WorkspaceEditMatching; } export interface WorkspaceSingleEditFileRequest @@ -509,6 +533,15 @@ export type WorkspaceEditFileRequest = export interface WorkspaceTextEdit { oldText: string; newText: string; + /** Replaces every match instead of requiring exactly one. Requires `replace_all`. */ + replaceAll?: boolean; +} + +/** How one edit matched. Present only when the request set `matching` or `replaceAll`. */ +export interface WorkspaceEditMatch { + strategy: WorkspaceEditMatchStrategy; + /** Locations replaced; always 1 unless the edit set `replaceAll`. */ + occurrences: number; } export interface WorkspaceEditFileResult { @@ -516,8 +549,10 @@ export interface WorkspaceEditFileResult { operation: 'edit_file'; workspaceId: string; path: string; + /** Number of edits applied, which is always the number requested. */ replacements: number; bytesWritten: number; + matches?: WorkspaceEditMatch[]; } interface WorkspacePreviewEditRequestBase { @@ -528,6 +563,8 @@ interface WorkspacePreviewEditRequestBase { /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ worktree?: string; path: string; + /** Requires the `tolerant_match` edit feature. Omitted means `exact`. */ + matching?: WorkspaceEditMatching; } export interface WorkspaceSinglePreviewEditRequest @@ -558,6 +595,7 @@ export interface WorkspacePreviewEditResult { baseSha256: string; replacements: number; bytesWritten: number; + matches?: WorkspaceEditMatch[]; } export interface WorkspaceExecuteCommandRequest { @@ -658,6 +696,7 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'newText', 'edits', 'expectedBaseSha256', + 'matching', ]); const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', @@ -669,8 +708,10 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'oldText', 'newText', 'edits', + 'matching', ]); -const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText']); +const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText', 'replaceAll']); +const WORKSPACE_EDIT_MATCH_KEYS = new Set(['strategy', 'occurrences']); const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'environmentAction', 'protocolVersion', @@ -724,6 +765,7 @@ const WORKSPACE_EDIT_RESULT_KEYS = new Set([ 'path', 'replacements', 'bytesWritten', + 'matches', ]); const WORKSPACE_PREVIEW_EDIT_RESULT_KEYS = new Set([ 'protocolVersion', @@ -735,6 +777,7 @@ const WORKSPACE_PREVIEW_EDIT_RESULT_KEYS = new Set([ 'baseSha256', 'replacements', 'bytesWritten', + 'matches', ]); const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'protocolVersion', @@ -1212,6 +1255,44 @@ function isWithinRequestedPath(candidate: string, requested?: string): boolean { ); } +/** Whether a request asked for per-edit match reporting (and so must receive it). */ +export function workspaceEditRequestReportsMatches( + request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, +): boolean { + return ( + request.matching !== undefined || + (request.edits?.some((edit) => edit.replaceAll !== undefined) ?? false) + ); +} + +function isValidWorkspaceEditMatches( + request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, + matches: unknown, +): boolean { + if (!workspaceEditRequestReportsMatches(request)) return matches === undefined; + const edits: WorkspaceTextEdit[] = request.edits ?? [ + { oldText: request.oldText ?? '', newText: request.newText ?? '' }, + ]; + return ( + Array.isArray(matches) && + matches.length === edits.length && + matches.every((match: unknown, index) => { + if (typeof match !== 'object' || match === null) return false; + const candidate = match as Record; + return ( + hasOnlyKeys(candidate, WORKSPACE_EDIT_MATCH_KEYS) && + WORKSPACE_EDIT_MATCH_STRATEGIES.has( + candidate.strategy as WorkspaceEditMatchStrategy, + ) && + (request.matching === 'tolerant' || candidate.strategy === 'exact') && + Number.isSafeInteger(candidate.occurrences) && + Number(candidate.occurrences) >= 1 && + (edits[index]?.replaceAll === true || candidate.occurrences === 1) + ); + }) + ); +} + function isValidWorkspaceEditRequest( request: Record, ): boolean { @@ -1222,6 +1303,13 @@ function isValidWorkspaceEditRequest( ) { return false; } + if ( + request.matching !== undefined && + request.matching !== 'exact' && + request.matching !== 'tolerant' + ) { + return false; + } const edits = hasBatch ? request.edits : [{ oldText: request.oldText, newText: request.newText }]; @@ -1252,7 +1340,9 @@ function isValidWorkspaceEditRequest( candidate.oldText || typeof candidate.newText !== 'string' || Buffer.from(candidate.newText).toString('utf8') !== - candidate.newText + candidate.newText || + (candidate.replaceAll !== undefined && + typeof candidate.replaceAll !== 'boolean') ) { return false; } @@ -1561,7 +1651,8 @@ export function isWorkspaceToolResult( result.replacements === replacements && Number.isSafeInteger(result.bytesWritten) && Number(result.bytesWritten) >= 0 && - Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES + Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES && + isValidWorkspaceEditMatches(request, result.matches) ); } @@ -1582,7 +1673,8 @@ export function isWorkspaceToolResult( Number(result.bytesWritten) === new TextEncoder().encode(content).byteLength + (result.hasUtf8Bom ? 3 : 0) && - Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES + Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES && + isValidWorkspaceEditMatches(request, result.matches) ); } @@ -1706,9 +1798,20 @@ export function isValidBridgeWorkspaceToolCapabilities( if ( capabilities.editFileFeatures !== undefined && (!Array.isArray(capabilities.editFileFeatures) || - capabilities.editFileFeatures.length !== 1 || - !capabilities.operations.includes('edit_file') || - capabilities.editFileFeatures[0] !== 'expected_base_sha256') + capabilities.editFileFeatures.length < 1 || + capabilities.editFileFeatures.length > + WORKSPACE_EDIT_FILE_FEATURES.length || + (!capabilities.operations.includes('edit_file') && + !capabilities.operations.includes('preview_edit')) || + !capabilities.editFileFeatures.every((feature: unknown) => + WORKSPACE_EDIT_FILE_FEATURES.includes( + feature as WorkspaceEditFileFeature, + ) && + (feature !== 'expected_base_sha256' || + (capabilities.operations as string[]).includes('edit_file')), + ) || + new Set(capabilities.editFileFeatures).size !== + capabilities.editFileFeatures.length) ) { return false; } diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 37223b34..217ca2c7 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -173,6 +173,19 @@ function errorCode(value: object): string | undefined { return undefined; } +/** Edit conflicts can include source excerpts, so disclosure needs read or full-preview access. */ +function workspaceCanReadSource( + capabilities: BridgeWorkerCapabilities['workspaceTools'], + workspaceId: string, +): boolean { + const workspace = capabilities?.workspaces.find((entry) => entry.id === workspaceId); + if (!workspace || !capabilities) return false; + return (['read_file', 'preview_edit'] as const).some( + (operation) => capabilities.operations.includes(operation) && + (workspace.operations == null || workspace.operations.includes(operation)), + ); +} + function workspaceCapabilitiesMatch( advertised: NonNullable, executor: NonNullable, @@ -388,7 +401,8 @@ function supportedWorkspaceCapabilities( }); if (workspaces.length === 0) return undefined; const editFileFeatures = desired.editFileFeatures?.filter((feature) => - registration.supportedWorkspaceEditFileFeatures?.includes(feature), + registration.supportedWorkspaceEditFileFeatures?.includes(feature) && + (feature !== 'expected_base_sha256' || operations.includes('edit_file')), ); const listFileFeatures = desired.listFileFeatures?.filter((feature) => registration.supportedWorkspaceListFileFeatures?.includes(feature), @@ -417,7 +431,7 @@ function supportedWorkspaceCapabilities( ...(supportsEditRequests && editFileModes?.length ? { editFileModes } : {}), - ...(operations.includes('edit_file') && editFileFeatures?.length + ...(supportsEditRequests && editFileFeatures?.length ? { editFileFeatures } : {}), ...(operations.includes('list_files') && listFileFeatures?.length @@ -1785,9 +1799,13 @@ export class BridgeWorker { ); } if ( - workspaceRequest.operation === 'edit_file' && - workspaceRequest.expectedBaseSha256 !== undefined && - !advertised.editFileFeatures?.includes('expected_base_sha256') + (workspaceRequest.operation === 'edit_file' && + workspaceRequest.expectedBaseSha256 !== undefined && + !advertised.editFileFeatures?.includes('expected_base_sha256')) || + (workspaceRequest.matching !== undefined && + !advertised.editFileFeatures?.includes('tolerant_match')) || + (workspaceRequest.edits?.some((edit) => edit.replaceAll !== undefined) === true && + !advertised.editFileFeatures?.includes('replace_all')) ) { throw new BridgeProtocolError( 'Workspace edit feature is not advertised', @@ -2069,9 +2087,14 @@ export class BridgeWorker { error instanceof WorkspaceToolError ? { errorCode: error.code } : {}), - error: (error instanceof Error - ? error.message - : 'Sandbox execution failed' + error: (assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + assignment.request.operation === 'edit_file' && + !workspaceCanReadSource(this.activeCapabilities.workspaceTools, assignment.request.workspaceId) + ? 'Workspace edit failed; source diagnostics require read access' + : error instanceof Error + ? error.message + : 'Sandbox execution failed' ).slice(0, MAX_SETTLEMENT_ERROR_LENGTH), }; } diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 96a5d1f0..71ef2cd3 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { BridgeProtocolError } from './protocol.js'; +import type { WorkspaceToolRequest } from './protocol.js'; import { BridgeWorker, BridgeWorkspaceQuarantinedError } from './worker.js'; import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; @@ -659,6 +660,49 @@ test('worker advertises only edit modes and features negotiated by Code API', as ]); }); +test('worker negotiates preview-only tolerant features and drops edit-only hashes', async () => { + const registrations: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'preview_edit' as const, 'edit_file' as const], + workspaces: [{ id: 'primary' }], + editFileModes: ['single' as const, 'batch' as const], + editFileFeatures: ['expected_base_sha256' as const, 'tolerant_match' as const, 'replace_all' as const], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'native-srt', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { throw new Error('must not execute'); }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + const capabilities = JSON.parse(String(init?.body)).capabilities.workspaceTools; + registrations.push(capabilities); + return Response.json({ + protocolVersion: 1, workerId: 'vm-1', incarnationId, + registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'preview_edit'], + supportedWorkspaceEditFileModes: ['single', 'batch'], + supportedWorkspaceEditFileFeatures: ['tolerant_match', 'replace_all', 'expected_base_sha256'], + }); + }, + }); + await worker.register(); + assert.deepEqual(registrations.at(-1)?.operations, ['read_file', 'preview_edit']); + assert.deepEqual(registrations.at(-1)?.editFileFeatures, ['tolerant_match', 'replace_all']); +}); + test('worker drops file operations when no request mode is compatible', async () => { const registrations: Array> = []; const workspaceCapabilities = { @@ -3165,6 +3209,98 @@ test('worker rejects workspace operations outside its advertised capability', as assert.match(String(settlement?.error), /operation is not advertised/i); }); +for (const [operation, extras] of [ + ['preview_edit', { matching: 'tolerant', oldText: 'a', newText: 'b' }], + ['preview_edit', { matching: 'exact', oldText: 'a', newText: 'b' }], + ['preview_edit', { edits: [{ oldText: 'a', newText: 'b', replaceAll: false }] }], + ['edit_file', { edits: [{ oldText: 'a', newText: 'b', replaceAll: true }] }], +] as const) { + test(`worker refuses unnegotiated ${operation} feature before dispatch`, async () => { + let executions = 0; + let armed = 0; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['preview_edit' as const, 'edit_file' as const], + editFileModes: ['single' as const, 'batch' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'native-srt', runtimes: ['bash'], + workspaceTools: workspaceCapabilities }, + workspaceTools: { capabilities: workspaceCapabilities, async execute() { + executions++; + throw new Error('must not execute'); + } }, + workspaceMutationQuarantine: mutationQuarantine(undefined, () => armed++), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: `assignment-${operation}-${String('matching' in extras ? extras.matching : 'replace')}`, + workerId: 'vm-1', incarnationId, generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', ...extras } as WorkspaceToolRequest, + }); + assert.equal(executions, 0); + assert.equal(armed, 0); + assert.equal(settlement?.status, 'rejected'); + assert.match(String(settlement?.error), /edit feature is not advertised/i); + }); +} + +for (const [label, operations, allowed, mayRead] of [ + ['edit-only worker', ['edit_file'], ['edit_file'], false], + ['read denied by workspace', ['read_file', 'edit_file'], ['edit_file'], false], + ['read permitted by workspace', ['read_file', 'edit_file'], ['read_file', 'edit_file'], true], + ['preview permits full file reads', ['preview_edit', 'edit_file'], ['preview_edit', 'edit_file'], true], +] as const) { + test(`edit diagnostics respect ${label} authorization`, async () => { + const source = 'confidential-source-line'; + let settlement: Record | undefined; + const capabilities = { + protocolVersion: 1 as const, + operations: [...operations], + editFileModes: ['single' as const], + workspaces: [{ id: 'primary', operations: [...allowed] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], + workspaceTools: capabilities }, + workspaceTools: { capabilities, mutationFailuresAreAtomic: true, async execute() { + throw new WorkspaceToolError(`old_text was not found; closest line: "${source}"`, 'EDIT_CONFLICT'); + } }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_url, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'edit-source-authorization', + workerId: 'vm-1', incarnationId, generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { protocolVersion: 1, operation: 'edit_file', workspaceId: 'primary', + path: 'notes.txt', oldText: 'missing', newText: 'replacement' }, + }); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'EDIT_CONFLICT'); + assert.equal(String(settlement?.error).includes(source), mayRead); + }); +} + test('worker rejects legacy replacement writes outside its advertised mode', async () => { let executions = 0; let settlement: Record | undefined; diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 3dd7d2f4..4a0ecb3e 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -939,7 +939,7 @@ test('writable workspaces create, replace, and exactly edit files', async (t) => ], writeFileModes: ['replace', 'create'], editFileModes: ['single', 'batch'], - editFileFeatures: ['expected_base_sha256'], + editFileFeatures: ['expected_base_sha256', 'tolerant_match', 'replace_all'], listFileFeatures: ['after_path'], }); await tools.execute({ @@ -1351,6 +1351,392 @@ test('exact edits reject missing or repeated text without changing the file', as assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'aaa'); }); +test('a failed edit batch reports every failing edit and writes nothing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'alpha\nbeta\nbeta\ngamma\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + edits: [ + { oldText: 'alpha', newText: 'ALPHA' }, + { oldText: 'beta', newText: 'BETA' }, + { oldText: 'delta', newText: 'DELTA' }, + ], + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EDIT_CONFLICT' && + /^2 of 3 workspace edits did not apply/.test(error.message) && + /Edit 2: old_text matched 2 locations at lines 2, 3/.test(error.message) && + /Edit 3: old_text was not found/.test(error.message), + ); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); +}); + +test('long preview and edit batch errors expose every failing edit position without writing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-batch-positions-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = `KEEP\nconst shared = '${'x'.repeat(180)}';\n`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const edits = Array.from({ length: 60 }, (_, index) => index % 2 === 0 + ? { oldText: `const shared missing_${index}`, newText: 'replacement' } + : { oldText: 'KEEP', newText: 'KEEP' }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', edits, + }), (error: unknown) => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'EDIT_CONFLICT'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.ok(error.message.length <= 3_000); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 30 }, (_, index) => 2 * index + 1), + ); + return true; + }); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant edits and replaceAll report how each edit matched', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'app.ts'), 'if (ok) { \r\n run();\r\n}\r\nrun();\r\n'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + const preview = await tools.execute({ + protocolVersion: 1, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'app.ts', + matching: 'tolerant', + edits: [{ oldText: 'if (ok) {\n run();\n}', newText: 'if (ok) {\n go();\n}' }], + }); + assert.equal(preview.operation, 'preview_edit'); + assert.deepEqual(preview.operation === 'preview_edit' && preview.matches, [ + { strategy: 'line-trimmed', occurrences: 1 }, + ]); + + const edit = await tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'app.ts', + matching: 'tolerant', + edits: [ + { oldText: 'if (ok) {\n run();\n}', newText: 'if (ok) {\n go();\n}' }, + { oldText: 'run();', newText: 'stop();', replaceAll: true }, + ], + }); + assert.deepEqual(edit, { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'app.ts', + replacements: 2, + bytesWritten: 32, + matches: [ + { strategy: 'line-trimmed', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ], + }); + assert.equal( + await readFile(join(root, 'app.ts'), 'utf8'), + 'if (ok) {\r\n go();\r\n}\r\nstop();\r\n', + ); +}); + +test('preview and edit peel equivalent newline wrappers without doubling line breaks or losing BOM', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-newlines-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n \tfoo bar\r\nnext\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { + protocolVersion: 1 as const, + operation: 'preview_edit' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: '\n foo bar\r\n', newText: '\r\n baz qux\n' }, + { oldText: 'next', newText: 'done' }, + ], + }; + const preview = await tools.execute(request); + assert.equal(preview.operation, 'preview_edit'); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n \tbaz qux\r\ndone\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(preview.bytesWritten, Buffer.byteLength(`\uFEFF${preview.content}`)); + assert.equal(isWorkspaceToolResult(request, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { + ...request, + operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256, + }; + const edited = await tools.execute(editRequest); + assert.equal(edited.operation, 'edit_file'); + if (edited.operation !== 'edit_file') assert.fail('expected edit result'); + assert.deepEqual(edited.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('tolerant previews and edits keep source indentation when newText uses a different indent', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-indent-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'header\n foo bar\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const edit = { + protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText: '\n foo bar', newText: '\n\tbaz qux', + }; + const preview = await tools.execute({ ...edit, operation: 'preview_edit' }); + assert.equal(preview.operation === 'preview_edit' && preview.content, 'header\n baz qux\n'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const result = await tools.execute({ ...edit, operation: 'edit_file' }); + assert.equal(result.operation === 'edit_file' && result.matches?.[0]?.strategy, 'whitespace-normalized'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'header\n baz qux\n'); +}); + +test('tolerant previews and edits retain an extra leading blank line across a fenced batch', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-extra-line-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n foo bar\r\nend\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { + protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: '\n foo bar', newText: '\r\n\t\r\n\tbaz qux' }, + { oldText: 'end', newText: 'done' }, + ], + }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n \r\n\tbaz qux\r\ndone\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + if (edited.operation !== 'edit_file') assert.fail('expected edit result'); + assert.deepEqual(edited.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('exact preview and edit failures retain whitespace hints without writing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-probe-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFfoo bar\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + oldText: 'foo bar\n', newText: '', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT' && + /different whitespace \(whitespace-normalized\)/.test(error.message) && !error.mutationMayHaveCommitted); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant preview and edit peel newline-only wrappers and preserve source indentation', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-newline-only-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n foo bar\r\nnext\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText: '\nfoo bar\n', newText: '\n\tbaz qux\t\n' }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n baz qux\r\nnext\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, + [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('preview and edit verify a uniquely anchored indentation block larger than the repeat budget', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-large-indent-')); + t.after(() => rm(root, { recursive: true, force: true })); + const oldText = ['anchor', ...Array.from({ length: 99_999 }, () => 'x'), 'end'].join('\n'); + const original = ` ${oldText.replaceAll('\n', '\n ')}\n`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText, newText: 'changed' }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, ' changed\n'); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, + [{ strategy: 'indentation-flexible', occurrences: 1 }]); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), preview.content); +}); + +test('tolerant previews and edits reject missing source boundary newlines without changing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-missing-newline-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'foo bar'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'foo bar\r\n', newText: 'baz qux\n', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant previews and edits reject boundary removal outside the token match', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-remove-newline-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'foo bar\r\nnext'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'foo bar\r\n', newText: 'baz qux', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant previews and edits refuse partial tokens without changing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-boundary-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'prereturn value;\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'return value;', newText: 'return changed;', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('large whitespace-only differences work for preview and edit_file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-large-')); + t.after(() => rm(root, { recursive: true, force: true })); + const oldText = `head ${'part '.repeat(16_000)}tail`; + const original = `before ${oldText.replace(/ /g, '\t')} after`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const base = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText, newText: 'result' }; + const preview = await tools.execute({ ...base, operation: 'preview_edit' }); + assert.equal(preview.operation === 'preview_edit' && preview.content, 'before result after'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const edited = await tools.execute({ ...base, operation: 'edit_file' }); + assert.equal(edited.operation === 'edit_file' && edited.matches?.[0]?.strategy, 'whitespace-normalized'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'before result after'); +}); + +test('large replaceAll previews and edits return bounded content and accurate match counts', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-stream-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'x'.repeat(256 * 1024); + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + edits: [{ oldText: 'x', newText: 'y', replaceAll: true }] }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'y'.repeat(original.length)); + assert.equal(preview.bytesWritten, original.length); + assert.deepEqual(preview.matches, [{ strategy: 'exact', occurrences: original.length }]); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(edited.operation === 'edit_file' && edited.bytesWritten, original.length); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, preview.matches); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), preview.content); +}); + +test('oversized replaceAll previews and edits fail before writing the source file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-limit-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'x'.repeat(10_000); + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, + operation, + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'x', newText: 'y'.repeat(100_000), replaceAll: true }], + }), (error: unknown) => error instanceof WorkspaceToolError && + error.code === 'WRITE_LIMIT_EXCEEDED' && !error.mutationMayHaveCommitted); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + test('writes reject symlink targets and missing parent directories', async (t) => { const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); t.after(() => rm(parent, { recursive: true, force: true })); @@ -1930,6 +2316,8 @@ test('composes sandboxed commands without exposing them on unconfigured workspac assert.deepEqual(tools.capabilities.editFileModes, ['single', 'batch']); assert.deepEqual(tools.capabilities.editFileFeatures, [ 'expected_base_sha256', + 'tolerant_match', + 'replace_all', ]); assert.deepEqual(tools.capabilities.listFileFeatures, ['after_path']); assert.deepEqual( diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 65757d39..0b9328d0 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -20,10 +20,19 @@ import { isValidBridgeWorkspaceToolCapabilities, isWorkspaceToolRequest, isWorkspaceToolResult, + WORKSPACE_EDIT_FILE_FEATURES, + workspaceEditRequestReportsMatches, } from './protocol.js'; +import { + applyTextEdits, + WorkspaceEditMatchError, + WorkspaceEditOutputLimitError, +} from './edits.js'; +import type { AppliedEdits } from './edits.js'; import type { BridgeWorkspaceDescriptor, + WorkspaceEditMatch, BridgeWorkspaceToolCapabilities, WorkspaceReadFileRequest, WorkspaceReadFileResult, @@ -722,7 +731,10 @@ async function editWorkspaceFile( 'EDIT_CONFLICT', ); } - const { updated, replacements } = applyWorkspaceEdits(original, request); + const { updated, replacements, matches } = applyWorkspaceEdits( + original, + request, + ); await atomicWriteConfinedFile( root, request.path, @@ -741,6 +753,7 @@ async function editWorkspaceFile( path: request.path, replacements, bytesWritten: updated.byteLength, + ...(matches ? { matches } : {}), }; } catch (error) { if (error instanceof WorkspaceToolError) throw error; @@ -753,7 +766,7 @@ async function editWorkspaceFile( function applyWorkspaceEdits( original: Buffer, request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, -): { updated: Buffer; replacements: number } { +): { updated: Buffer; replacements: number; matches?: WorkspaceEditMatch[] } { const hasBom = original[0] === 0xef && original[1] === 0xbb && original[2] === 0xbf; const body = hasBom ? original.subarray(3) : original; @@ -767,20 +780,19 @@ function applyWorkspaceEdits( const edits = request.edits ?? [ { oldText: request.oldText ?? '', newText: request.newText ?? '' }, ]; - let updatedText = text; - for (const edit of edits) { - const first = updatedText.indexOf(edit.oldText); - if (first < 0 || updatedText.indexOf(edit.oldText, first + 1) >= 0) { - throw new WorkspaceToolError( - 'Workspace edit must match exactly once', - 'EDIT_CONFLICT', - ); + let applied: AppliedEdits; + try { + applied = applyTextEdits(text, edits, request.matching ?? 'exact'); + } catch (error) { + if (error instanceof WorkspaceEditMatchError) { + throw new WorkspaceToolError(error.message, 'EDIT_CONFLICT'); + } + if (error instanceof WorkspaceEditOutputLimitError) { + throw new WorkspaceToolError(error.message, 'WRITE_LIMIT_EXCEEDED'); } - updatedText = - updatedText.slice(0, first) + - edit.newText + - updatedText.slice(first + edit.oldText.length); + throw error; } + const updatedText = applied.text; const updatedBody = Buffer.from(updatedText, 'utf8'); const updated = hasBom ? Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), updatedBody]) @@ -791,7 +803,13 @@ function applyWorkspaceEdits( 'WRITE_LIMIT_EXCEEDED', ); } - return { updated, replacements: edits.length }; + return { + updated, + replacements: edits.length, + ...(workspaceEditRequestReportsMatches(request) + ? { matches: applied.matches } + : {}), + }; } async function previewWorkspaceEdit( @@ -806,7 +824,10 @@ async function previewWorkspaceEdit( 'EXECUTION_ABORTED', ); } - const { updated, replacements } = applyWorkspaceEdits(original, request); + const { updated, replacements, matches } = applyWorkspaceEdits( + original, + request, + ); if (signal?.aborted) { throw new WorkspaceToolError( 'Workspace tool execution aborted', @@ -825,6 +846,7 @@ async function previewWorkspaceEdit( baseSha256: createHash('sha256').update(original).digest('hex'), replacements, bytesWritten: updated.byteLength, + ...(matches ? { matches } : {}), }; } @@ -1497,7 +1519,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ...(anyWritable ? { writeFileModes: ['replace', 'create'] } : {}), ...(anyWritable ? { editFileModes: ['single', 'batch'] } : {}), ...(anyWritable - ? { editFileFeatures: ['expected_base_sha256'] } + ? { editFileFeatures: [...WORKSPACE_EDIT_FILE_FEATURES] } : {}), listFileFeatures: ['after_path'], }; diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 63640111..28035847 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -432,7 +432,11 @@ describe('paired bridge HTTP API', () => { ], supportedWorkspaceWriteFileModes: ['replace', 'create'], supportedWorkspaceEditFileModes: ['single', 'batch'], - supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], + supportedWorkspaceEditFileFeatures: [ + 'expected_base_sha256', + 'tolerant_match', + 'replace_all', + ], supportedWorkspaceListFileFeatures: ['after_path'], }); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 962c6242..8fdc700e 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -12,6 +12,7 @@ import { isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, isWorkspaceToolErrorCode, + WORKSPACE_EDIT_FILE_FEATURES, } from '../../../packages/code/src/protocol'; import { BridgePairingError, RedisBridgePairingStore } from './pairing'; import { BridgeStoreError, RedisBridgeStore } from './store'; @@ -535,7 +536,7 @@ router.post( ], supportedWorkspaceWriteFileModes: ['replace', 'create'], supportedWorkspaceEditFileModes: ['single', 'batch'], - supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], + supportedWorkspaceEditFileFeatures: [...WORKSPACE_EDIT_FILE_FEATURES], supportedWorkspaceListFileFeatures: ['after_path'], supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 123e7ee2..f64e748a 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -194,12 +194,21 @@ function supportsWorkspaceTool( const mode = request.edits === undefined ? 'single' : 'batch'; const modes = capabilities?.editFileModes; const supportsMode = modes == null ? mode === 'single' : modes.includes(mode); - if (request.operation === 'preview_edit') return supportsMode; + const features = capabilities?.editFileFeatures ?? []; + const supportsMatching = + request.matching === undefined || features.includes('tolerant_match'); + const supportsReplaceAll = + request.edits?.some((edit) => edit.replaceAll !== undefined) !== true || + features.includes('replace_all'); + if (request.operation === 'preview_edit') { + return supportsMode && supportsMatching && supportsReplaceAll; + } return ( supportsMode && + supportsMatching && + supportsReplaceAll && (request.expectedBaseSha256 === undefined || - capabilities?.editFileFeatures?.includes('expected_base_sha256') === - true) + features.includes('expected_base_sha256')) ); } return true; diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index f248aa40..90f68da3 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -2,8 +2,9 @@ import { afterEach, expect, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities } from '../../../packages/code/src/protocol'; import { RedisBridgeStore } from './store'; const redis = new RedisMock() as unknown as Redis; @@ -643,6 +644,131 @@ test('rejects fenced edits from workers without the negotiated feature', async ( expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); }); +const featureGatedEdits: Array<[string, WorkspaceToolRequest]> = [ + [ + 'tolerant matching', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant', + edits: [{ oldText: 'before', newText: 'after' }], + }, + ], + [ + 'tolerant previews', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant', + edits: [{ oldText: 'before', newText: 'after' }], + }, + ], + [ + 'replaceAll edits', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'before', newText: 'after', replaceAll: true }], + }, + ], +]; + +test.each(featureGatedEdits)( + 'rejects %s from workers without the negotiated feature', + async (_label, request) => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit', 'edit_file'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['expected_base_sha256'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); + }, +); + +test('preview-only workers can dispatch negotiated tolerant and replace-all previews', async () => { + expect(isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['tolerant_match', 'replace_all'], + workspaces: [{ id: 'primary' }], + }, + })).toBe(true); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['tolerant_match', 'replace_all'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const request = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'preview_edit' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [{ oldText: 'before', newText: 'after', replaceAll: true }], + }; + const completion = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + expect(assignment).toMatchObject({ executionKind: 'workspace_tool', request }); + await store.settle('workspace-worker', assignment!.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment!.generation, + leaseToken: assignment!.leaseToken, + incarnationId, + status: 'rejected', + error: 'preview tested', + }); + await expect(completion).resolves.toMatchObject({ status: 'rejected', error: 'preview tested' }); +}); + test('rejects batch previews from workers without the negotiated mode', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, From 836d001319015072f9493df5d7710fc371403962 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 30 Sep 2026 08:27:56 -0400 Subject: [PATCH 53/75] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20fix:=20Deny=20a?= =?UTF-8?q?=20Non-Lane=20Root's=20Own=20Git=20Metadata=20in=20the=20Native?= =?UTF-8?q?=20SRT=20Sandbox=20(#276)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🛡️ fix: Deny a Non-Lane Root's Own Git Metadata in the Native SRT Sandbox A trusted-vm command in a native (non-lane) SRT workspace could write the registered root's own `.git/hooks/*` and `.git/config`, planting a hook or a `core.fsmonitor` / `filter.*` entry that then runs unsandboxed the next time the user — or trusted worker code — invokes Git in that checkout. The non-lane config relied on SRT's mandatory `.git/hooks` and `.git/config` denies, which on Linux are computed from the worker *process's* current directory (see `sandbox/linux-sandbox-utils.js`). The worker's cwd is its home (the systemd user unit sets no WorkingDirectory), never the registered root, so those denies landed elsewhere and the root's `.git` was writable. macOS was unaffected because it applies global `**/.git/hooks/**` and `**/.git/config` Seatbelt patterns. `NativeSrtWorkspaceCommandSandbox` now adds explicit `denyWrite` entries for the registered root's own Git metadata when `/.git` is a directory (lanes are untouched — they already keep the whole common Git directory read-only): - `/.git/hooks` and `/.git/config` unconditionally; - `/.git/config.worktree` and `/.git/commondir` where they exist; - the same files for every submodule Git directory under `.git/modules/**` (including nested submodules) and every `.git/worktrees/*`. `commondir` and `config.worktree` are denied only where present: Git reads them strictly and SRT would otherwise mask an absent target with an empty `/dev/null` bind Git cannot parse, breaking ordinary commands. Denied files that exist are re-bound read-only, so `.git/config` stays readable (remotes keep working) while writes fail; a benign `git commit` in the workspace is unaffected. Tests: a live SRT test gated on `LIBRECHAT_CODE_LIVE_SRT_TESTS=1` runs the sandbox with `process.cwd()` outside the root and asserts hooks, config, an existing per-worktree config, a submodule config, and a linked-worktree commondir cannot be tampered while a normal commit still succeeds; wired into the Linux native-sandbox CI job. A fast unit test asserts the computed `denyWrite` set. `packages/code/README.md` documents the guarantee and the residual writable-`.git` vectors (whole-`.git` replacement, a fresh top-level commondir, a nested repo) that the personal-machine SRT trust model accepts. * 🔁 fix: Refresh Root Git Metadata Denies per Command and Fail Closed The root's Git metadata denies were a snapshot taken at sandbox initialization, but the worker is long-lived: a repository, submodule, or linked worktree created on the host afterwards stayed writable until restart. Ordinary root commands now recompute the set and pass it as a per-command `denyWrite` override (every other filesystem field is the session policy), the way SRT recomputes its own mandatory denies on every wrap. Native Windows keeps the initialization snapshot because srt-win rejects per-command allowRead/allowWrite. Inspecting the metadata now treats only ENOENT/ENOTDIR as absence; any other failure propagates, so an unreadable `.git` fails the command closed (COMMAND_UNAVAILABLE) instead of silently dropping a deny. Corrects the `config.worktree` comment: in a repository that already enables `worktreeConfig`, a missing per-worktree config stays creatable, because denying an absent file makes every Git command fail. The README lists it with the other writable-workspace residuals. --- .github/workflows/ci.yml | 7 +- packages/code/README.md | 28 +++ packages/code/src/native-sandbox.test.ts | 122 ++++++++++++ packages/code/src/native-sandbox.ts | 178 +++++++++++++++++- .../code/src/root-git-denies-live.test.ts | 140 ++++++++++++++ 5 files changed, 471 insertions(+), 4 deletions(-) create mode 100644 packages/code/src/root-git-denies-live.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7393284..e6a23b61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,10 +254,13 @@ jobs: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - run: npm ci - run: npm run build - - name: Linked worktree lane containment + - name: Linked worktree lane and root Git metadata containment env: LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' - run: node --test dist/linked-worktrees-live.test.js + run: >- + node --test + dist/linked-worktrees-live.test.js + dist/root-git-denies-live.test.js lambda-microvm-provisioning: name: Lambda MicroVM Provisioning diff --git a/packages/code/README.md b/packages/code/README.md index ba7ec4d7..d421e444 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -213,12 +213,40 @@ SRT with: worker's private scratch directory; - read access denied to the worker's home directory except for that workspace; - paired identity and mutation-quarantine files explicitly denied; +- the registered workspace's own Git metadata denied writes — `.git/hooks` + and `.git/config` always, plus `.git/config.worktree`, `.git/commondir`, + and the equivalent files under `.git/modules/*` and `.git/worktrees/*` + where they already exist — so a sandboxed command cannot plant a hook or a + `filter`/`fsmonitor`/`diff` config entry that would run unsandboxed the + next time Git runs in the checkout; - `LIBRECHAT_CODE_*` and nonessential inherited environment variables removed; - network egress denied by default, local binding denied, and Unix sockets denied; and - bounded time and aggregate output, with best-effort process-group termination on cancellation, timeout, and completion. +The Git-metadata denies are applied to the registered root directly rather +than relying on SRT's Linux mandatory denies, which are derived from the +worker process's own current directory — the worker home, not the workspace — +and so never covered the registered root; macOS already enforced the +equivalent through global Seatbelt patterns, and this keeps the guarantee +identical on both platforms regardless of the worker's cwd. The set is +recomputed before every command, so a repository, submodule, or linked +worktree created on the host after the worker starts is covered from the next +command, and metadata the worker cannot inspect fails the command closed. +`.git/commondir` and `.git/config.worktree` are denied only where they already +exist, because Git reads them strictly and SRT would otherwise mask an absent +one with a stub Git cannot open, failing every Git command. Because the whole +workspace stays writable, a sandboxed command can still stage Git +configuration Git will honor later by other means — for example replacing +the entire `.git` directory, writing a new `commondir` that redirects the +common directory, creating a missing `config.worktree` in a repository that +already enables the `worktreeConfig` extension, or initializing a fresh +nested repository. Denying those safely would require +making the workspace's Git storage structurally read-only, which the +personal-machine SRT trust model does not; use the Docker/NsJail backend or a +dedicated VM boundary when a workspace command must be treated as adversarial. + SRT restrictions remain inherited by descendants. Windows additionally uses a kill-on-close Job Object. Native macOS does not provide an equivalent hard process-lifetime boundary: a deliberately daemonized descendant can outlive diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 5b79aa1a..9001102f 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1751,3 +1751,125 @@ test('linked worktree Git guard refuses Windows rather than admitting an unguard ); await sandbox.close(); }); + +test('non-lane roots deny their own executable Git metadata at initialization', async t => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const gitDir = join(root, '.git'); + await mkdir(join(gitDir, 'hooks'), { recursive: true }); + await writeFile(join(gitDir, 'config'), '[core]\n'); + await writeFile(join(gitDir, 'config.worktree'), '[core]\n'); + // A submodule Git directory (identified by its HEAD file) and a nested one. + // This submodule has no config.worktree, so it is not denied one. + await mkdir(join(gitDir, 'modules', 'sub', 'modules', 'inner'), { recursive: true }); + await writeFile(join(gitDir, 'modules', 'sub', 'HEAD'), 'ref: refs/heads/main\n'); + await writeFile(join(gitDir, 'modules', 'sub', 'config'), '[core]\n'); + await writeFile( + join(gitDir, 'modules', 'sub', 'modules', 'inner', 'HEAD'), + 'ref: refs/heads/main\n', + ); + // A linked worktree's per-worktree metadata: an existing commondir and + // config.worktree are both denied. + await mkdir(join(gitDir, 'worktrees', 'wt'), { recursive: true }); + await writeFile(join(gitDir, 'worktrees', 'wt', 'commondir'), '../..\n'); + await writeFile(join(gitDir, 'worktrees', 'wt', 'config.worktree'), '[core]\n'); + + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + + const denyWrite = fake.config?.filesystem.denyWrite ?? []; + for (const relativePath of [ + '.git/hooks', + '.git/config', + '.git/config.worktree', + '.git/modules/sub/hooks', + '.git/modules/sub/config', + '.git/modules/sub/modules/inner/hooks', + '.git/modules/sub/modules/inner/config', + '.git/worktrees/wt/config.worktree', + '.git/worktrees/wt/commondir', + ]) { + assert.ok(denyWrite.includes(join(root, relativePath)), relativePath); + } + // Deny-if-exists paths are skipped when absent: masking them with an empty + // bind would make Git read a broken redirect or config from every command. + assert.ok(!denyWrite.includes(join(gitDir, 'commondir'))); + assert.ok(!denyWrite.includes(join(gitDir, 'modules', 'sub', 'config.worktree'))); +}); + +test('ordinary root commands deny Git metadata created after initialization', async t => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-late-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + const session = fake.config!.filesystem; + const gitDir = join(root, '.git'); + assert.ok(!session.denyWrite.some(path => path.startsWith(gitDir))); + + // The host initializes the repository and adds a submodule and a linked + // worktree while the worker keeps running. + await mkdir(join(gitDir, 'hooks'), { recursive: true }); + await writeFile(join(gitDir, 'config'), '[core]\n'); + await mkdir(join(gitDir, 'modules', 'late'), { recursive: true }); + await writeFile(join(gitDir, 'modules', 'late', 'HEAD'), 'ref: refs/heads/main\n'); + await mkdir(join(gitDir, 'worktrees', 'late'), { recursive: true }); + await writeFile(join(gitDir, 'worktrees', 'late', 'commondir'), '../..\n'); + + const result = await sandbox.execute(request); + assert.equal(result.exitCode, 0); + const filesystem = fake.customConfigSeenDuringWrap?.filesystem; + for (const relativePath of [ + '.git/hooks', + '.git/config', + '.git/modules/late/hooks', + '.git/modules/late/config', + '.git/worktrees/late/commondir', + ]) { + assert.ok(filesystem?.denyWrite.includes(join(root, relativePath)), relativePath); + } + // Only denyWrite differs from the session filesystem policy. + assert.deepEqual({ ...filesystem, denyWrite: session.denyWrite }, session); +}); + +test('a root command fails closed when its Git metadata cannot be inspected', async t => { + if (process.getuid?.() === 0) { + t.skip('directory permissions do not restrict root'); + return; + } + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-denied-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const modules = join(root, '.git', 'modules'); + await mkdir(modules, { recursive: true }); + await writeFile(join(root, '.git', 'config'), '[core]\n'); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + + await chmod(modules, 0o000); + try { + await assert.rejects( + sandbox.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'COMMAND_UNAVAILABLE', + ); + } finally { + await chmod(modules, 0o700); + } +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 49fe6ecf..5d39d405 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -11,7 +11,8 @@ import { sep, } from 'node:path'; import { constants as fsConstants } from 'node:fs'; -import { access, mkdtemp, open, realpath, rm, stat } from 'node:fs/promises'; +import type { Dirent, Stats } from 'node:fs'; +import { access, mkdtemp, open, readdir, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { matchesWorkspaceRoot } from './root-identity.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; @@ -234,6 +235,130 @@ async function canonicalPath(path: string): Promise { } } +/** + * Paths inside a repository's own Git directory that Git treats as executable + * configuration: hooks run on the next Git invocation, and `config` / + * an existing `config.worktree` can define filter, diff, or fsmonitor commands + * that later run unsandboxed. An existing `commondir` is denied too, since it + * redirects the whole common Git directory to attacker-controlled config. + * + * A sandboxed command may write anywhere in the workspace root, so these are + * denied explicitly for the registered root's `.git`. SRT's Linux mandatory + * denies derive equivalents from the worker process's current directory, which + * is not the registered root, so they never cover it; macOS instead applies + * global `.git/hooks` and `.git/config` Seatbelt patterns. Denying the root's + * own metadata directly keeps the guarantee identical on both platforms and + * independent of the worker's cwd. Lane workspaces keep the whole common Git + * directory read-only already, so this augments only the non-lane root. + * + * The set is recomputed before every ordinary root command, so a repository, + * submodule, or linked worktree created after initialization is covered from + * the next command on. Only submodule and linked-worktree Git directories + * that exist at that moment are enumerated: masking a non-existent + * `.git/modules` would block a later `git submodule add` from creating it + * during the same command. `.git/info` + * is intentionally omitted — its attributes reference filter/diff drivers by + * name, but the commands those names resolve to live in the denied config, so + * `info` alone cannot introduce a new executable. + */ +async function collectGitMetadataDenies(root: string): Promise { + const gitDir = join(root, '.git'); + if (!(await statIfPresent(gitDir))?.isDirectory()) return []; + const denies = [join(gitDir, 'hooks'), join(gitDir, 'config')]; + await pushExistingDenies(denies, gitDir); + for (const submoduleGitDir of await collectSubmoduleGitDirs( + join(gitDir, 'modules'), + )) { + denies.push(join(submoduleGitDir, 'hooks'), join(submoduleGitDir, 'config')); + await pushExistingDenies(denies, submoduleGitDir); + } + const worktreesDir = join(gitDir, 'worktrees'); + for (const entry of await readdirIfPresent(worktreesDir)) { + if (entry.isDirectory()) { + await pushExistingDenies(denies, join(worktreesDir, entry.name)); + } + } + return denies; +} + +/** + * `commondir` and `config.worktree` are read strictly by Git when present — + * `commondir` at every startup, `config.worktree` whenever `worktreeConfig` + * is enabled — and SRT masks a non-existent deny target with an empty + * `/dev/null` bind that Git cannot open, failing every Git command. Denying + * them only where they already exist keeps the mask a read-only bind of the + * real file. `config` and `hooks` stay unconditional: they always exist in a + * real repository. In a repository that already enables `worktreeConfig`, a + * missing `config.worktree` therefore stays creatable, like a new + * `commondir`; the package README lists both with the other residuals of a + * writable workspace. + */ +async function pushExistingDenies(paths: string[], gitDir: string): Promise { + for (const name of ['config.worktree', 'commondir']) { + const candidate = join(gitDir, name); + if (await statIfPresent(candidate)) paths.push(candidate); + } +} + +/** + * Enumerate every submodule Git directory beneath `.git/modules`, following the + * `/modules/` nesting Git uses for recursive submodules. A + * directory holding a `HEAD` file is a Git directory; any other directory is an + * intermediate segment of a submodule path that itself contains slashes. + */ +async function collectSubmoduleGitDirs(modulesDir: string): Promise { + const found: string[] = []; + const stack = [modulesDir]; + while (stack.length > 0) { + const dir = stack.pop()!; + const entries = await readdirIfPresent(dir); + if (entries.some(entry => entry.isFile() && entry.name === 'HEAD')) { + found.push(dir); + if ( + entries.some(entry => entry.isDirectory() && entry.name === 'modules') + ) { + stack.push(join(dir, 'modules')); + } + continue; + } + for (const entry of entries) { + if (entry.isDirectory()) stack.push(join(dir, entry.name)); + } + } + return found; +} + +/** + * Absence is an expected answer while inspecting Git metadata. Any other + * failure (for example `EACCES`) propagates, so metadata that cannot be + * inspected fails the command closed instead of silently dropping a deny. + */ +function isMissing(error: unknown): boolean { + return ( + error instanceof Error && + 'code' in error && + (error.code === 'ENOENT' || error.code === 'ENOTDIR') + ); +} + +async function statIfPresent(path: string): Promise { + try { + return await stat(path); + } catch (error) { + if (isMissing(error)) return undefined; + throw error; + } +} + +async function readdirIfPresent(path: string): Promise { + try { + return await readdir(path, { withFileTypes: true }); + } catch (error) { + if (isMissing(error)) return []; + throw error; + } +} + function boundedUtf8(buffer: Buffer, budget: number): string { let end = Math.min(buffer.byteLength, budget); while (end > 0) { @@ -315,6 +440,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private runtimeConfig?: SandboxRuntimeConfig; private denyReadPaths: string[] = []; private denyWritePaths: string[] = []; + /** Worker-owned write denies that precede each command's live Git metadata denies. */ + private baseDenyWritePaths: string[] = []; private get gitEnvironment(): | typeof TRUSTED_GIT_ENVIRONMENT | typeof LINKED_WORKTREE_GIT_ENVIRONMENT { @@ -517,6 +644,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const gitGuardDirectory = lane ? await this.createLinkedWorktreeGitGuard(canonicalScratchDirectory!) : undefined; + // A lane already keeps the shared common Git directory read-only; only + // the non-lane root exposes a writable `.git` whose executable metadata + // SRT's cwd-derived mandatory denies do not reach. + const rootGitMetadataDenies = lane + ? [] + : await collectGitMetadataDenies(root); const commandPolicy = normalizeNativeSrtCommandPolicy( this.options.commandPolicy, ); @@ -558,6 +691,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox denyWrite: [ ...protectedPaths, ...deniedInheritedWritablePaths, + ...rootGitMetadataDenies, ...(gitGuardDirectory ? [gitGuardDirectory] : []), ], allowGitConfig: false, @@ -625,9 +759,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox deniedInheritedWritablePaths.includes(path), ), ]; + this.baseDenyWritePaths = [ + ...protectedPaths, + ...deniedInheritedWritablePaths, + ]; this.denyWritePaths = [ ...protectedPaths, ...deniedInheritedWritablePaths, + ...rootGitMetadataDenies, ...(gitGuardDirectory ? [gitGuardDirectory] : []), ]; } @@ -937,6 +1076,40 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } } + /** + * The session policy snapshots the root's Git metadata at initialization, but + * the worker outlives that snapshot: a repository, submodule, or linked + * worktree created on the host afterwards must be protected without a + * restart, the way SRT recomputes its own mandatory denies on every wrap. + * Only `denyWrite` differs from the session filesystem policy. Lanes keep + * the whole common Git directory read-only already. Native Windows keeps the + * initialization snapshot, because `srt-win` rejects the per-command + * `allowRead`/`allowWrite` a full filesystem override carries. + */ + private async rootCommandConfig(): Promise< + Partial | undefined + > { + const config = this.runtimeConfig; + const root = this.canonicalRoot; + if ( + !config || + !root || + this.options.linkedWorktree || + this.platform === 'win32' + ) { + return undefined; + } + return { + filesystem: { + ...config.filesystem, + denyWrite: [ + ...this.baseDenyWritePaths, + ...(await collectGitMetadataDenies(root)), + ], + }, + }; + } + private async executeBound( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, @@ -981,6 +1154,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ReturnType >; try { + const commandConfig = customConfig ?? (await this.rootCommandConfig()); const credentialEnvironment = await this.options.maskedEnvironment?.resolve(signal, cwd); const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand @@ -1002,7 +1176,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.platform === 'win32' ? undefined : (this.options.shellPath ?? '/bin/bash'), - customConfig, + commandConfig, signal, cwd, { commandId, commandText: request.command }, diff --git a/packages/code/src/root-git-denies-live.test.ts b/packages/code/src/root-git-denies-live.test.ts new file mode 100644 index 00000000..12dc820c --- /dev/null +++ b/packages/code/src/root-git-denies-live.test.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +const execFileAsync = promisify(execFile); +const IDENTITY = ['-c', 'user.name=t', '-c', 'user.email=t@example.com', '-c', 'commit.gpgsign=false']; + +async function git(cwd: string, ...args: string[]): Promise { + return (await execFileAsync('git', [...IDENTITY, ...args], { cwd })).stdout.trim(); +} + +async function snapshot(paths: string[]): Promise> { + const entries = await Promise.all( + paths.map(async (path) => [path, await readFile(path, 'utf8').catch(() => null)] as const), + ); + return Object.fromEntries(entries); +} + +async function exists(path: string): Promise { + return stat(path).then(() => true, () => false); +} + +/** + * SRT's Linux mandatory denies for `.git/hooks` and `.git/config` are computed + * from the worker process's own current directory, which is never the registered + * workspace root. This exercises the sandbox from a cwd outside the root and + * asserts the root's executable Git metadata is unwritable anyway — hooks, + * config, an existing per-worktree config, and a submodule's config cannot be + * tampered, so nothing an attacker writes runs the next time git is invoked on + * the host. + */ +test('real SRT denies a non-lane root its own Git hooks and config even when the worker cwd is elsewhere', { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 90_000, +}, async (t) => { + const outside = await realpath(await mkdtemp(join(tmpdir(), 'git-denies-cwd-'))); + const base = await realpath(await mkdtemp(join(tmpdir(), 'git-denies-'))); + t.after(() => rm(base, { recursive: true, force: true })); + t.after(() => rm(outside, { recursive: true, force: true })); + + // A standalone repo used only as a local submodule source. + const upstream = join(base, 'sub-src'); + await mkdir(upstream); + await git(upstream, 'init', '-q', '-b', 'main'); + await writeFile(join(upstream, 's.txt'), 'sub\n'); + await git(upstream, 'add', 's.txt'); + await git(upstream, 'commit', '-qm', 'sub'); + + const root = join(base, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'tracked.txt'), 'checkout\n'); + await git(root, 'add', 'tracked.txt'); + await git(root, 'commit', '-qm', 'init'); + await git(root, '-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', upstream, 'sub'); + await git(root, 'commit', '-qm', 'add submodule'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '.worktrees/wt', '-b', 'wt'); + // An existing per-worktree config for the linked worktree; --worktree needs + // the repository extension enabled first. + await git(root, 'config', 'extensions.worktreeConfig', 'true'); + await git(join(root, '.worktrees', 'wt'), 'config', '--worktree', 'core.testflag', 'true'); + + const gitDir = join(root, '.git'); + const submoduleGitDir = join(gitDir, 'modules', 'sub'); + const worktreeGitDir = join(gitDir, 'worktrees', 'wt'); + const guarded = [ + join(gitDir, 'hooks', 'post-checkout'), + join(gitDir, 'config'), + join(submoduleGitDir, 'config'), + join(submoduleGitDir, 'hooks', 'post-checkout'), + join(worktreeGitDir, 'config.worktree'), + join(worktreeGitDir, 'commondir'), + ]; + for (const path of guarded) { + assert.ok(await exists(path) || path.endsWith('post-checkout'), `precondition: ${path}`); + } + const before = await snapshot(guarded); + + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + environment: { PATH: process.env.PATH, LANG: 'C.UTF-8' }, + }); + t.after(() => sandbox.close()); + const run = (command: string) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'root', + command, + timeoutMs: 30_000, + maxOutputBytes: 16_384, + }); + + // The worker process runs from outside the registered root, the way the + // systemd user unit does (its cwd is the worker home, not the workspace). + const previousCwd = process.cwd(); + process.chdir(outside); + t.after(() => process.chdir(previousCwd)); + + for (const path of guarded) { + const attempt = await run(`printf tampered > '${path}'`); + assert.notEqual(attempt.exitCode, 0, `expected a write to ${path} to be denied`); + } + // Planting a fresh hook file (not just overwriting one) must also be blocked. + const plantHook = await run( + `printf '#!/bin/sh\\ntouch ${join(base, 'PWNED')}\\n' > .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit`, + ); + assert.notEqual(plantHook.exitCode, 0); + + // Metadata the host creates while the worker runs is covered from the next + // command, not only what existed when the sandbox initialized. + await git(root, 'worktree', 'add', '-q', '.worktrees/late', '-b', 'late'); + const lateCommondir = join(gitDir, 'worktrees', 'late', 'commondir'); + const lateBefore = await readFile(lateCommondir, 'utf8'); + const lateAttempt = await run(`printf tampered > '${lateCommondir}'`); + assert.notEqual(lateAttempt.exitCode, 0, 'expected the new worktree metadata to be denied'); + assert.equal(await readFile(lateCommondir, 'utf8'), lateBefore); + + // A benign commit in the workspace must still succeed: the guard makes Git + // metadata read-only, not the checkout or the writable object store. + const committed = await run( + `printf work > work.txt && git ${IDENTITY.join(' ')} add work.txt && git ${IDENTITY.join(' ')} commit -qm work`, + ); + assert.equal(committed.exitCode, 0, committed.stderr); + + await sandbox.close(); + + assert.equal(await exists(join(base, 'PWNED')), false); + assert.deepEqual(await snapshot(guarded), before); + assert.equal(await git(root, 'log', '-1', '--format=%s'), 'work'); +}); From 05bad85d10197832b0fa472e00687b699beec680 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:52:30 -0400 Subject: [PATCH 54/75] =?UTF-8?q?=F0=9F=A7=B7=20test:=20Retain=20the=20Con?= =?UTF-8?q?tention=20Lock=20Owner=20Through=20Garbage=20Collection=20(#283?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lia --- packages/code/src/process-lock.test.ts | 51 ++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/packages/code/src/process-lock.test.ts b/packages/code/src/process-lock.test.ts index 3594a1a9..77091db3 100644 --- a/packages/code/src/process-lock.test.ts +++ b/packages/code/src/process-lock.test.ts @@ -17,25 +17,60 @@ test( const child = spawn( process.execPath, [ + '--expose-gc', '--input-type=module', '-e', ` + import { setTimeout as delay } from 'node:timers/promises'; import { withProcessLock } from ${JSON.stringify(moduleUrl)}; await withProcessLock(${JSON.stringify(path)}, async () => { - process.stdout.write('locked'); - await new Promise(() => { setInterval(() => {}, 1000); }); + setImmediate(() => { + global.gc(); + setImmediate(() => process.send('locked')); + }); + // The timer retains its resolver and the lock-owning async frame. + await delay(60_000); }); `, ], - { stdio: ['ignore', 'pipe', 'pipe'] } + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] } ); + let stdout = ''; + let stderr = ''; + child.stdout!.setEncoding('utf8').on('data', (chunk: string) => { + stdout = (stdout + chunk).slice(-4096); + }); + child.stderr!.setEncoding('utf8').on('data', (chunk: string) => { + stderr = (stderr + chunk).slice(-4096); + }); const closed = once(child, 'close'); t.after(async () => { child.kill('SIGKILL'); - await closed; - await rm(directory, { recursive: true, force: true }); + try { + await closed; + } finally { + t.diagnostic( + JSON.stringify({ + stdout, + stderr, + exitCode: child.exitCode, + signal: child.signalCode, + }) + ); + await rm(directory, { recursive: true, force: true }); + } }); - await once(child.stdout!, 'data'); + await Promise.race([ + once(child, 'message', { signal: t.signal }).then(([marker]) => { + assert.equal(marker, 'locked'); + t.diagnostic('Lock owner ready: locked (after GC)'); + }), + closed.then(([code, signal]) => { + throw new Error( + `Lock owner exited before readiness: ${code}/${signal}` + ); + }), + ]); let entered = false; await assert.rejects( withProcessLock( @@ -47,8 +82,10 @@ test( ) ); assert.equal(entered, false); + assert.equal(child.exitCode, null); + assert.equal(child.signalCode, null); child.kill('SIGKILL'); - await closed; + assert.deepEqual(await closed, [null, 'SIGKILL']); await withProcessLock( path, async () => { From e46fca4ceb175c7cb9d46f8e3fca9d4224e39ccd Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:48:41 -0400 Subject: [PATCH 55/75] =?UTF-8?q?=F0=9F=AA=9F=20feat:=20Preserve=20Command?= =?UTF-8?q?=20Output=20Heads=20and=20Tails=20(#275)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🪟 feat: Preserve Command Output Heads and Tails * 🪟 fix: Retain Logical Command Output Suffixes --------- Co-authored-by: Lia --- packages/code/README.md | 15 + packages/code/src/native-sandbox.test.ts | 142 ++++++++- packages/code/src/native-sandbox.ts | 57 +--- packages/code/src/output.test.ts | 353 +++++++++++++++++++++++ packages/code/src/output.ts | 182 ++++++++++++ 5 files changed, 702 insertions(+), 47 deletions(-) create mode 100644 packages/code/src/output.test.ts create mode 100644 packages/code/src/output.ts diff --git a/packages/code/README.md b/packages/code/README.md index d421e444..acae9bb8 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -225,6 +225,21 @@ SRT with: - bounded time and aggregate output, with best-effort process-group termination on cancellation, timeout, and completion. +Native command output keeps a prefix and rolling suffix for each stream, so late +summaries and errors survive truncation. Each stream stores at most +`maxOutputBytes` of copied raw bytes while the command runs, independent of output +volume or chunk count. When both streams are noisy they split the existing combined +response budget equally, with the odd byte reserved for stderr; a quiet stream gives +its unused allowance to the other based on rendered UTF-8 bytes, including replacement +characters for malformed input. Sandbox violation annotations enter the same +stderr window before rendering. UTF-8 boundaries and inline +`[... N bytes omitted ...]` markers count toward the combined byte limit. The count +reports omitted raw bytes for that stream, including annotation bytes. If a stream's +allowance cannot fit its marker, only the retained text and the existing `truncated` +flag are returned. Truncation does not stop execution. Exit codes, timeout/signal +fields, and cancellation errors keep their existing semantics, and no new request, +result, or capability keys are introduced. + The Git-metadata denies are applied to the registered root directly rather than relying on SRT's Linux mandatory denies, which are derived from the worker process's own current directory — the worker home, not the workspace — diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 9001102f..90215156 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -33,6 +33,7 @@ import { } from './native-sandbox.js'; import { restoreScratchTraversal } from './native-scratch.js'; import { WorkspaceToolError } from './workspace.js'; +import { isWorkspaceToolResult } from './protocol.js'; const request = { protocolVersion: 1 as const, @@ -1355,14 +1356,151 @@ test('executes in the canonical workspace and bounds aggregate output', async t operation: 'execute_command', workspaceId: 'primary', exitCode: 0, - stdout: '1234567890', - stderr: 'ab', + stdout: '123890', + stderr: 'abchij', truncated: true, timedOut: false, }, ); }); +test('retains real command summaries on both streams under the legacy combined budget', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const commandRequest = { + ...request, + maxOutputBytes: 256, + command: + "printf 'OUT\\n'; printf '%20000d' 0; printf '\\n42 tests passed\\n'; printf 'ERR\\n' >&2; printf '%20000d' 0 >&2; printf '\\nlate stderr summary\\n' >&2", + }; + const result = await sandbox.execute(commandRequest); + assert.equal(result.exitCode, 0); + assert.equal(result.timedOut, false); + assert.equal(result.truncated, true); + assert.ok(result.stdout.startsWith('OUT\n')); + assert.ok(result.stderr.startsWith('ERR\n')); + assert.ok(result.stdout.endsWith('\n42 tests passed\n')); + assert.ok(result.stderr.endsWith('\nlate stderr summary\n')); + assert.ok(result.stdout.includes('bytes omitted')); + assert.ok(result.stderr.includes('bytes omitted')); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + +test('partly filled command buffers retain final summaries regardless of stream order', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const summary = 'late stderr summary\n'; + const quiet = `printf '%109d' 0; printf 'late stderr summary\\n'`; + const noisy = `printf '%20000d' 0`; + for (const [command, stream] of [ + [`{ ${quiet}; } >&2; ${noisy}`, 'stderr'], + [`${noisy}; { ${quiet}; } >&2`, 'stderr'], + [`{ ${quiet}; }; ${noisy} >&2`, 'stdout'], + [`${noisy} >&2; { ${quiet}; }`, 'stdout'], + ] as const) { + const commandRequest = { ...request, command, maxOutputBytes: 256 }; + const result = await sandbox.execute(commandRequest); + assert.ok(result[stream].endsWith(summary)); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, 0); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + } +}); + +test('quiet malformed command output is retained beside a noisy stream', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + for (const [command, stream] of [ + ["printf '%20000d' 0; printf '\\377\\377' >&2", 'stderr'], + ["printf '\\377\\377'; printf '%20000d' 0 >&2", 'stdout'], + ] as const) { + const commandRequest = { ...request, command, maxOutputBytes: 32 }; + const result = await sandbox.execute(commandRequest); + assert.equal(result[stream], '\ufffd\ufffd'); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, 0); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + } +}); + +test('sandbox annotations survive full stdout and remain bounded when stderr is noisy', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + fake.manager.annotateStderrWithSandboxFailures = (_commandId, stderr) => + stderr + '\n\ndenied write\n'; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + for (const stderrCommand of ['', "printf '%20000d' 0 >&2;"]) { + const commandRequest = { + ...request, + maxOutputBytes: 512, + command: `printf '%20000d' 0; ${stderrCommand} true`, + }; + const result = await sandbox.execute(commandRequest); + assert.ok( + result.stderr.endsWith( + '\ndenied write\n' + ) + ); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + assert.equal(result.truncated, true); + } + fake.manager.annotateStderrWithSandboxFailures = () => { + throw new Error('annotation unavailable'); + }; + const result = await sandbox.execute({ + ...request, + maxOutputBytes: 128, + command: "printf '%20000d' 0; printf 'child failure' >&2", + }); + assert.equal(result.stderr, 'child failure'); +}); + +test('timeout settlement keeps late diagnostics, signal, and truncation without widening the result', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const commandRequest = { + ...request, + maxOutputBytes: 128, + timeoutMs: 100, + command: + "printf 'START\\n'; printf '%20000d' 0; printf '\\nlast progress\\n'; printf 'last failure' >&2; sleep 30", + }; + const result = await sandbox.execute(commandRequest); + assert.ok(result.stdout.startsWith('START\n')); + assert.ok(result.stdout.endsWith('\nlast progress\n')); + assert.equal(result.stderr, 'last failure'); + assert.equal(result.timedOut, true); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, null); + assert.equal(result.signal, 'SIGKILL'); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + test('rejects an escaping or unavailable command working directory', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 5d39d405..26c848e5 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -32,6 +32,7 @@ import { removePrivateStorageAcl, } from './private-storage.js'; import { WorkspaceToolError } from './workspace.js'; +import { OutputBuffer, renderCommandOutput } from './output.js'; import { restoreScratchTraversal } from './native-scratch.js'; import { writeLinkedWorktreeGitGuard } from './linked-worktree-git-guard.js'; @@ -1301,28 +1302,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } let settled = false; let timedOut = false; - let outputBytes = 0; - let truncated = false; - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - const append = (target: Buffer[], chunk: Buffer): void => { - const remaining = outputLimit - outputBytes; - if (remaining <= 0) { - truncated = true; - return; - } - const accepted = chunk.subarray(0, remaining); - target.push(accepted); - outputBytes += accepted.byteLength; - if (accepted.byteLength !== chunk.byteLength) - truncated = true; - }; - child.stdout.on('data', (chunk: Buffer) => - append(stdout, chunk), - ); - child.stderr.on('data', (chunk: Buffer) => - append(stderr, chunk), - ); + const stdout = new OutputBuffer(outputLimit); + const stderr = new OutputBuffer(outputLimit); + child.stdout.on('data', (chunk: Buffer) => stdout.append(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.append(chunk)); const abort = (): void => { if (settled) return; this.killCommandTree(child); @@ -1371,29 +1354,17 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); return; } - const stdoutValue = boundedUtf8( - Buffer.concat(stdout), - outputLimit, - ); - const stderrBudget = Math.max( - 0, - outputLimit - Buffer.byteLength(stdoutValue), - ); - const rawStderr = Buffer.concat(stderr).toString('utf8'); - let annotatedStderr = rawStderr; try { - annotatedStderr = - this.manager.annotateStderrWithSandboxFailures( - commandId, - rawStderr, + // SRT appends violations to its input. Capture them in the same bounded stderr window. + stderr.append( + Buffer.from( + this.manager.annotateStderrWithSandboxFailures(commandId, ''), + ), ); } catch { // Preserve the bounded child error if optional violation annotation fails. } - const stderrValue = boundedUtf8( - Buffer.from(annotatedStderr), - stderrBudget, - ); + const output = renderCommandOutput(stdout, stderr, outputLimit); resolvePromise({ protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'execute_command', @@ -1403,11 +1374,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ? null : this.protocolExitCode(code), ...(childSignal ? { signal: childSignal } : {}), - stdout: stdoutValue, - stderr: stderrValue, - truncated: - truncated || - Buffer.byteLength(annotatedStderr) > stderrBudget, + ...output, timedOut, }); }); diff --git a/packages/code/src/output.test.ts b/packages/code/src/output.test.ts new file mode 100644 index 00000000..f78b0993 --- /dev/null +++ b/packages/code/src/output.test.ts @@ -0,0 +1,353 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { OutputBuffer, renderCommandOutput } from './output.js'; + +function collect( + content: Buffer, + budget: number, + chunkSize = content.length || 1 +): OutputBuffer { + const output = new OutputBuffer(budget); + for (let offset = 0; offset < content.length; offset += chunkSize) { + output.append(content.subarray(offset, offset + chunkSize)); + } + return output; +} + +function assertBounded(text: string, budget: number): void { + assert.ok(Buffer.byteLength(text) <= budget); + assert.equal(Buffer.from(text).toString('utf8'), text); +} + +test('small output stays byte-for-byte unchanged, including split UTF-8 and exact limits', () => { + for (const content of ['', '1234567', 'hello\n世界🌍\n']) { + const bytes = Buffer.from(content); + const budget = Math.max(1, bytes.length); + for (const chunkSize of [1, 2, 7, 64]) { + assert.deepEqual(collect(bytes, budget, chunkSize).render(budget), { + text: content, + truncated: false, + }); + } + } +}); + +test('a rolling suffix keeps the last summary and reports the exact dropped-byte count', () => { + const budget = 128; + const content = Buffer.from( + 'START\n' + 'x'.repeat(10_000) + '\n42 tests passed\n' + ); + for (const chunkSize of [1, 3, 63, 64, 65, 127, 1024, content.length]) { + const result = collect(content, budget, chunkSize).render(budget); + assert.equal(result.truncated, true); + assert.ok(result.text.startsWith('START\n')); + assert.ok(result.text.endsWith('\n42 tests passed\n')); + const marker = /\n\[\.\.\. (\d+) bytes omitted \.\.\.\]\n/.exec( + result.text + ); + assert.ok(marker); + assert.equal( + Number(marker[1]), + content.length - + Buffer.byteLength(result.text.replace(marker[0], '')) + ); + assertBounded(result.text, budget); + } +}); + +test('either quiet stream donates its budget, and stdout cannot starve stderr', () => { + const budget = 256; + const empty = collect(Buffer.alloc(0), budget); + const exact = collect(Buffer.alloc(budget, 'a'), budget, 1); + assert.deepEqual(renderCommandOutput(exact, empty, budget), { + stdout: 'a'.repeat(budget), + stderr: '', + truncated: false, + }); + assert.deepEqual(renderCommandOutput(empty, exact, budget), { + stdout: '', + stderr: 'a'.repeat(budget), + truncated: false, + }); + const stdout = collect( + Buffer.from('OUT\n' + 'x'.repeat(4096) + '\nstdout summary'), + budget + ); + const stderr = collect( + Buffer.from('ERR\n' + 'y'.repeat(4096) + '\nstderr summary'), + budget + ); + const result = renderCommandOutput(stdout, stderr, budget); + assert.ok(result.stdout.startsWith('OUT\n')); + assert.ok(result.stderr.startsWith('ERR\n')); + assert.ok(result.stdout.endsWith('stdout summary')); + assert.ok(result.stderr.endsWith('stderr summary')); + assert.ok(result.stdout.includes('bytes omitted')); + assert.ok(result.stderr.includes('bytes omitted')); + assert.ok( + Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr) <= + budget + ); + assert.equal(result.truncated, true); + + const shortError = collect(Buffer.from('late failure'), budget); + assert.equal( + renderCommandOutput(stdout, shortError, budget).stderr, + 'late failure' + ); + assert.equal( + renderCommandOutput(shortError, stdout, budget).stdout, + 'late failure' + ); +}); + +test('partly filled buffers retain final summaries on either stream', () => { + const capacity = 256; + const summary = 'late stderr summary\n'; + const content = Buffer.from('y'.repeat(129 - summary.length) + summary); + for (const chunkSize of [1, 7, 128, content.length]) { + const quiet = collect(content, capacity, chunkSize); + const noisy = collect(Buffer.alloc(20_000, 'x'), capacity, chunkSize); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput(stdout, stderr, capacity); + assert.ok(result[stream].endsWith(summary)); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + capacity + ); + } + } +}); + +test('rendered windows match logical stream ends across capture and allowance boundaries', () => { + for (const capacity of [32, 63, 64, 127, 128, 255, 256, 257]) { + const half = Math.floor(capacity / 2); + for (const length of [ + 0, + 1, + half - 1, + half, + half + 1, + capacity - 1, + capacity, + capacity + 1, + capacity * 4, + ]) { + const content = Buffer.from( + Array.from({ length }, (_, index) => + String.fromCharCode(65 + (index % 26)) + ).join('') + ); + for (const budget of [ + 0, + 1, + Math.floor(half / 2), + half, + capacity - 1, + capacity, + ]) { + let expected = content.toString('utf8'); + if (length > budget) { + const marker = (bytes: number): string => + `\n[... ${bytes} bytes omitted ...]\n`; + const markerBytes = Buffer.byteLength(marker(length)); + const includeMarker = budget >= markerBytes; + const remaining = includeMarker + ? budget - markerBytes + : budget; + const prefix = Math.floor(remaining / 2); + const suffix = Math.ceil(remaining / 2); + expected = + content.subarray(0, prefix).toString('utf8') + + (includeMarker + ? marker(length - prefix - suffix) + : '') + + content.subarray(length - suffix).toString('utf8'); + } + for (const chunkSize of [1, half, capacity + 7]) { + assert.deepEqual( + collect(content, capacity, chunkSize).render(budget), + { + text: expected, + truncated: length > budget, + }, + `capacity ${capacity}, length ${length}, budget ${budget}, chunk ${chunkSize}` + ); + } + } + } + } +}); + +test('UTF-8 summaries survive capture transitions and split code-point chunks', () => { + const summary = 'end 世界🌍\n'; + for (const capacity of [128, 255, 256, 257]) { + const half = Math.floor(capacity / 2); + for (const length of [ + half - 1, + half, + half + 1, + capacity - 1, + capacity, + capacity + 1, + ]) { + const content = Buffer.from( + 'x'.repeat(length - Buffer.byteLength(summary)) + summary + ); + for (const chunkSize of [1, 2, content.length]) { + const quiet = collect(content, capacity, chunkSize); + const noisy = collect( + Buffer.alloc(capacity * 4, 'y'), + capacity, + chunkSize + ); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput( + stdout, + stderr, + capacity + ); + assert.ok(result[stream].endsWith(summary)); + assert.ok(!result[stream].includes('\ufffd')); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + capacity + ); + } + } + } + } +}); + +test('budgeting includes UTF-8 boundaries and marker overhead for every small limit', () => { + const content = Buffer.from('世界🌍 café Ελληνικά\n'.repeat(100)); + for (let budget = 1; budget <= 256; budget += 1) { + const result = collect(content, budget, 1).render(budget); + assertBounded(result.text, budget); + assert.equal(result.truncated, true); + assert.ok( + !result.text.includes('\ufffd'), + `split code point at budget ${budget}` + ); + const marker = /\n\[\.\.\. (\d+) bytes omitted \.\.\.\]\n/.exec( + result.text + ); + if (marker) { + assert.equal( + Number(marker[1]), + content.length - + Buffer.byteLength(result.text.replace(marker[0], '')) + ); + } + const streams = renderCommandOutput( + collect(content, budget, 7), + collect(content, budget, 3), + budget + ); + assert.ok( + Buffer.byteLength(streams.stdout) + + Buffer.byteLength(streams.stderr) <= + budget + ); + assert.ok( + !streams.stdout.includes('\ufffd') && + !streams.stderr.includes('\ufffd') + ); + } +}); + +test('malformed output cannot expand past the byte budget during decoding', () => { + for (const content of [ + Buffer.alloc(1024, 0xff), + Buffer.from([0xf0, 0x80, 0xff, 0xc2, 0xa2, 0xe0, 0xa0]), + Buffer.from('a\ufffdb'.repeat(300)), + ]) { + for (let budget = 1; budget <= 128; budget += 1) { + const result = collect(content, budget, 1).render(budget); + assertBounded(result.text, budget); + if (content.length > budget) assert.equal(result.truncated, true); + } + } +}); + +test('short decoded output is preserved when replacement characters fit the combined budget', () => { + const budget = 32; + assert.deepEqual( + renderCommandOutput( + collect(Buffer.from([0xff, 0xff]), budget), + collect(Buffer.from('child failure'), budget), + budget + ), + { stdout: '\ufffd\ufffd', stderr: 'child failure', truncated: false } + ); +}); + +test('quiet-stream donation counts decoded bytes beside noisy output', () => { + for (const bytes of [ + Buffer.from([0xff, 0xff]), + Buffer.from([0x80, 0x80]), + Buffer.from([0xf0, 0x9f]), + Buffer.from([0x61, 0xff, 0x62]), + ]) { + const decoded = bytes.toString('utf8'); + for (const budget of [16, 31, 32, 63, 64, 128]) { + const quiet = collect(bytes, budget, 1); + const noisy = collect(Buffer.alloc(budget * 4, 'x'), budget, 7); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput(stdout, stderr, budget); + assert.equal(result[stream], decoded); + assert.equal(result.truncated, true); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + budget + ); + } + } + } +}); + +test('tiny budgets still flag truncation and give stderr the odd byte', () => { + for (let budget = 1; budget < 32; budget += 1) { + const result = renderCommandOutput( + collect(Buffer.from('a'.repeat(256)), budget), + collect(Buffer.from('b'.repeat(256)), budget), + budget + ); + assert.equal(result.truncated, true); + assert.ok(result.stderr.length > 0); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + budget + ); + } +}); + +test('copies bounded windows instead of retaining or repeatedly concatenating source chunks', () => { + const budget = 128; + const output = new OutputBuffer(budget); + const oversized = Buffer.alloc(1024 * 1024, 'a'); + output.append(oversized); + oversized.fill('z'); + const byte = Buffer.from('b'); + for (let i = 0; i < 100_000; i += 1) output.append(byte); + byte[0] = 0x7a; + const result = output.render(budget); + assert.ok(result.text.startsWith('a'.repeat(40))); + assert.ok(result.text.endsWith('b'.repeat(40))); + assert.ok(!result.text.includes('z')); + assert.equal(output.bytes, 1024 * 1024 + 100_000); + assertBounded(result.text, budget); +}); diff --git a/packages/code/src/output.ts b/packages/code/src/output.ts new file mode 100644 index 00000000..4cb34211 --- /dev/null +++ b/packages/code/src/output.ts @@ -0,0 +1,182 @@ +interface OutputWindow { + text: string; + bytes: number; +} + +function utf8Window( + buffer: Buffer, + length: number, + tail: boolean +): OutputWindow { + let start = tail ? buffer.length - length : 0; + let end = tail ? buffer.length : length; + if (tail) { + while (start < end && (buffer[start] & 0xc0) === 0x80) start += 1; + } else if (end > 0) { + let last = end - 1; + while (last > 0 && (buffer[last] & 0xc0) === 0x80) last -= 1; + const lead = buffer[last]; + const width = + lead >= 0xc2 && lead <= 0xdf + ? 2 + : lead >= 0xe0 && lead <= 0xef + ? 3 + : lead >= 0xf0 && lead <= 0xf4 + ? 4 + : 1; + if (end - last < width) end = last; + } + return { + text: buffer.subarray(start, end).toString('utf8'), + bytes: end - start, + }; +} + +function boundedWindow( + buffer: Buffer, + budget: number, + tail: boolean +): OutputWindow { + const length = Math.min(buffer.length, budget); + const window = utf8Window(buffer, length, tail); + if (Buffer.byteLength(window.text) <= budget) return window; + + // Malformed bytes expand to replacement characters. Valid UTF-8 takes the fast path. + let low = 0; + let high = length; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if ( + Buffer.byteLength(utf8Window(buffer, middle, tail).text) <= budget + ) { + low = middle; + } else { + high = middle - 1; + } + } + return utf8Window(buffer, low, tail); +} + +/** Copies a prefix and rolling suffix, never retaining child-process chunk buffers. */ +export class OutputBuffer { + private readonly headCapacity: number; + private readonly tailCapacity: number; + private head?: Buffer; + private tail?: Buffer; + private headLength = 0; + private tailLength = 0; + private tailOffset = 0; + private totalBytes = 0; + + constructor(capacity: number) { + this.headCapacity = Math.floor(capacity / 2); + this.tailCapacity = capacity - this.headCapacity; + } + + get bytes(): number { + return this.totalBytes; + } + + append(chunk: Buffer): void { + this.totalBytes += chunk.length; + const headBytes = Math.min( + chunk.length, + this.headCapacity - this.headLength + ); + if (headBytes > 0) { + this.head ??= Buffer.allocUnsafe(this.headCapacity); + chunk.copy(this.head, this.headLength, 0, headBytes); + this.headLength += headBytes; + } + const remaining = chunk.length - headBytes; + if (remaining === 0) return; + this.tail ??= Buffer.allocUnsafe(this.tailCapacity); + if (remaining >= this.tailCapacity) { + chunk.copy(this.tail, 0, chunk.length - this.tailCapacity); + this.tailOffset = 0; + this.tailLength = this.tailCapacity; + return; + } + const first = Math.min(remaining, this.tailCapacity - this.tailOffset); + chunk.copy(this.tail, this.tailOffset, headBytes, headBytes + first); + chunk.copy(this.tail, 0, headBytes + first); + this.tailOffset = (this.tailOffset + remaining) % this.tailCapacity; + this.tailLength = Math.min( + this.tailCapacity, + this.tailLength + remaining + ); + } + + private suffix(): Buffer { + if (!this.tail) return Buffer.alloc(0); + if (this.tailLength < this.tailCapacity) + return this.tail.subarray(0, this.tailLength); + if (this.tailOffset === 0) return this.tail; + return Buffer.concat([ + this.tail.subarray(this.tailOffset), + this.tail.subarray(0, this.tailOffset), + ]); + } + + render(budget: number): { text: string; truncated: boolean } { + const head = this.head?.subarray(0, this.headLength) ?? Buffer.alloc(0); + const tail = this.suffix(); + // Before overflow the buffers are adjacent; either window may cross their boundary. + const contiguous = + this.totalBytes === head.length + tail.length + ? tail.length === 0 + ? head + : Buffer.concat([head, tail]) + : undefined; + if (contiguous && this.totalBytes <= budget) { + const text = contiguous.toString('utf8'); + if (Buffer.byteLength(text) <= budget) + return { text, truncated: false }; + } + const marker = (bytes: number): string => + `\n[... ${bytes} bytes omitted ...]\n`; + const markerBytes = Buffer.byteLength(marker(this.totalBytes)); + const includeMarker = budget >= markerBytes; + const contentBudget = includeMarker ? budget - markerBytes : budget; + const prefix = boundedWindow( + contiguous ?? head, + Math.floor(contentBudget / 2), + false + ); + const suffix = boundedWindow( + contiguous ?? tail, + Math.ceil(contentBudget / 2), + true + ); + const omitted = this.totalBytes - prefix.bytes - suffix.bytes; + return { + text: + prefix.text + + (includeMarker ? marker(omitted) : '') + + suffix.text, + truncated: omitted > 0, + }; + } +} + +/** Each stream can use the whole budget when the other is quiet; stderr cannot be starved. */ +export function renderCommandOutput( + stdout: OutputBuffer, + stderr: OutputBuffer, + budget: number +): { + stdout: string; + stderr: string; + truncated: boolean; +} { + let err = stderr.render(Math.ceil(budget / 2)); + const out = stdout.render(budget - Buffer.byteLength(err.text)); + if (!out.truncated) { + err = stderr.render(budget - Buffer.byteLength(out.text)); + } + return { + stdout: out.text, + stderr: err.text, + truncated: out.truncated || err.truncated, + }; +} From 3db258469569658480b51e618d9c078b2dda8509 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 1 Oct 2026 23:07:09 -0400 Subject: [PATCH 56/75] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20feat:=20Reuse=20Veri?= =?UTF-8?q?fied=20Environment=20Preparation=20(#286)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Reuse Verified Environment Preparation * fix: Preserve Environment Setup Recovery Diagnostics --- docs/remote-bridge/worker-runbook.md | 5 +- packages/code/README.md | 51 ++- packages/code/src/cli.ts | 68 ++-- .../code/src/environment-preparation.test.ts | 367 ++++++++++++++++++ packages/code/src/environment-preparation.ts | 150 +++++++ packages/code/src/environment.ts | 30 +- packages/code/src/storage.ts | 46 ++- 7 files changed, 688 insertions(+), 29 deletions(-) create mode 100644 packages/code/src/environment-preparation.test.ts create mode 100644 packages/code/src/environment-preparation.ts diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index aeeef4c4..8cbeda98 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -255,7 +255,10 @@ Important semantics: existing ID such as `primary` to preserve agent/conversation bindings. - `repo` and `ref` are labels. They do not clone, fetch, or check out anything. - `root` must already exist. Relative roots resolve from the definition file. -- Setup runs before registration on every worker start. It must be idempotent. +- Setup runs before registration on every worker start by default. It must be + idempotent. Optional `setup.reuse` declares fingerprint inputs and a sandboxed + readiness check to avoid reinstalling an unchanged, still-ready checkout. See + [preparation reuse](../../packages/code/README.md#reusing-a-prepared-checkout). - A setup failure or timeout prevents registration and leaves a durable quarantine marker for operator inspection. - Actions are fixed operator commands. The model selects only the action name diff --git a/packages/code/README.md b/packages/code/README.md index acae9bb8..47771057 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -1050,7 +1050,7 @@ worker runs. This inspection happens at startup, not on the command hot path. Setup is an operator-authorized startup command under the configured native sandbox policy. It requires commands to be enabled, runs once per worker startup before -registration, and must be idempotent for restarts. Its timeout is bounded to five +registration by default, and must be idempotent for restarts. Its timeout is bounded to five minutes and captured output to 8 KiB. Setup failure prevents registration. A nonzero exit, timeout, crash or uncertain termination retains the workspace quarantine marker; inspect the workspace before running `librechat-code clear-workspace-quarantine @@ -1058,6 +1058,55 @@ inspect the workspace before running `librechat-code clear-workspace-quarantine deployment and identity configuration. Only use the separate `--reset-workspace-quarantine ` run option afterward if a server fence also needs clearing. Only successful setup automatically clears its marker. + +## Reusing a prepared checkout + +Opt in to checkout-local preparation reuse when setup is an installation rather +than work that must run on every startup: + +```yaml +setup: + command: npm ci + timeoutMs: 300000 + reuse: + inputs: + - package.json + - package-lock.json + - packages/api/package.json + - packages/data-provider/package.json + checkCommand: test -f node_modules/.package-lock.json && test -x node_modules/.bin/tsc + checkTimeoutMs: 10000 +``` + +Declare **all** relevant manifests, installation configuration and lifecycle-script +inputs. There is no globbing or automatic monorepo discovery. Inputs must be +existing root-confined regular files: at most 32, 8 MiB per file and 32 MiB total. +Include an operator-maintained toolchain/version file if installation uses tools +other than the worker's Node runtime. Lockfile equality alone does not prove that +arbitrary postinstall scripts are reusable. + +The worker fingerprints declared file bytes, the setup recipe, checkout inode, +Node version/ABI, platform/architecture and native command policy. A matching +worker-owned receipt runs the readiness check instead of setup. A nonzero check +reruns setup; a timed-out, signalled or aborted check fails without starting a +replacement command. After successful setup, the check must pass and inputs must +remain unchanged before the worker publishes a receipt. Receipts are bounded, +owner-only files alongside the identity, outside all registered roots and denied +to native commands. Deleting one causes setup to run again; it never clears a +quarantine. Checks are operator commands under the same sandbox and mutation guard +as setup, not unsandboxed host scripts. Keep them cheap and non-mutating. + +Existing definitions without `reuse` retain the startup behavior. Fresh conversation +instances use the same preparation contract, with independent checkout receipts. +This does not attach another checkout's `node_modules`, provision linked lanes on +command admission, or recheck existing instances on every command. It does not +deduplicate installed dependencies between worktrees or enforce disk quotas. + +The next resource-store slice must explicitly grant shared cache paths under SRT, +keep monorepo links and mutable outputs checkout-local, and bound retention. Do not +work around that missing grant by broadening the sandbox root or symlinking another +branch's full installation. Shared download caches alone do not reduce installed +`node_modules` copies. No setup output is sent to the model. Named actions are fixed commands without model-supplied substitution. The bridge diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 410caf25..054074ed 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -2,7 +2,7 @@ import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { readdir, realpath, stat } from 'node:fs/promises'; -import { basename, join, resolve, relative, isAbsolute, sep } from 'node:path'; +import { basename, dirname, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { discoverProjects } from './projects.js'; @@ -12,6 +12,7 @@ import { assertEnvironmentDefinitionsOutsideRoots, EnvironmentWorkspaceTools, } from './environment.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -26,6 +27,7 @@ import { loadWorkspaceMutationQuarantine, saveBridgeIdentity, saveWorkspaceMutationQuarantine, + prepareEnvironmentPreparationDirectory, } from './storage.js'; import { BridgeWorker } from './worker.js'; import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; @@ -826,6 +828,17 @@ async function run( }), ]), ); + const preparationDirectory = join(dirname(identityPath ?? defaultBridgeIdentityPath(workerId)), 'environment-preparation'); + if (environments.some(environment => environment.definition.setup?.reuse)) { + const sourceParents = await prepareEnvironmentPreparationDirectory(preparationDirectory); + // Reuse the mount/ancestor isolation checks for this worker-owned state directory. + await assertEnvironmentDefinitionsOutsideRoots([{ + path: preparationDirectory, sourceParents, + definition: { name: 'preparation-state', root: preparationDirectory }, fingerprint: '', + }], roots); + } + const preparationReceipt = (root: string) => join(preparationDirectory, + `${createHash('sha256').update(JSON.stringify([codeApiUrl, workerId, root])).digest('hex')}.json`); // Keep an admission boundary even when trusted-VM checkout routing uses a // nested repository's remote for the current command. const admittedGitHubRepositories = github.provider && github.repositoryRouting @@ -1048,6 +1061,7 @@ async function run( commandPolicy, protectedPaths: [ identityPath, + ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), github.privateKeyPath, @@ -1151,22 +1165,20 @@ async function run( workspaceIdentity: instance.identity, workspaceRoot: instance.root, }); - const result = await nativeCommandSandbox.execute( - { + await prepareCodeEnvironment({ + root: instance.root, identity: instance.identity, setup, + receiptPath: preparationReceipt(instance.root), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity]), + signal, + execute: (command, timeoutMs) => nativeCommandSandbox.execute({ protocolVersion: 1, operation: 'execute_command', workspaceId: id, - command: setup.command, - timeoutMs: setup.timeoutMs, + command, + timeoutMs, maxOutputBytes: 8192, - }, - signal, - ); - if (result.exitCode !== 0 || result.timedOut) { - throw new Error( - `Environment ${instance.sourceWorkspaceId} setup failed for its conversation worktree`, - ); - } + }, signal), + }); }, discardInstance: async (instance) => { await nativeCommandSandbox.unregisterRoot( @@ -1323,26 +1335,36 @@ async function run( incarnationId, ); await guard.assertAvailable(); - await guard.arm('Environment setup did not settle', 'setup'); - const result = await nativeCommandSandbox.execute( - { + let armed = false; + const preparation = await prepareCodeEnvironment({ + root: environment.definition.root, + identity: roots.find(root => root.id === id)!.identity!, + setup, receiptPath: preparationReceipt(environment.definition.root), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity]), + signal: controller.signal, + execute: async (command, timeoutMs) => { + if (!armed) { + await guard.arm('Environment preparation did not settle', 'setup'); + armed = true; + } + return nativeCommandSandbox.execute({ protocolVersion: 1, operation: 'execute_command', workspaceId: id, - command: setup.command, - timeoutMs: setup.timeoutMs, + command, + timeoutMs, maxOutputBytes: 8192, + }, controller.signal); }, - controller.signal, - ); - if (result.exitCode !== 0 || result.timedOut) { + }).catch(error => { throw new Error( `Environment ${id} setup failed; inspect the workspace and use clear-workspace-quarantine with its root and workspace ID before restarting`, + { cause: error }, ); - } + }); await guard.clear('setup'); process.stdout.write( - `librechat-code: environment ${id} prepared\n`, + `librechat-code: environment ${id} ${preparation}\n`, ); } } catch (error) { diff --git a/packages/code/src/environment-preparation.test.ts b/packages/code/src/environment-preparation.test.ts new file mode 100644 index 00000000..51149441 --- /dev/null +++ b/packages/code/src/environment-preparation.test.ts @@ -0,0 +1,367 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + chmod, + mkdtemp, + mkdir, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { parseCodeEnvironment } from './environment.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { prepareEnvironmentPreparationDirectory } from './storage.js'; +import type { EnvironmentPreparationOptions } from './environment-preparation.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; + +async function fixture(t: test.TestContext) { + const directory = await mkdtemp(join(tmpdir(), 'code-preparation-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'checkout'); + const state = join(directory, 'private'); + await mkdir(root); + await prepareEnvironmentPreparationDirectory(state); + await writeFile(join(root, 'package-lock.json'), 'lock-v1'); + const commands: string[] = []; + const options: EnvironmentPreparationOptions = { + root: (await captureWorkspaceRootIdentity(root)).path, + identity: await captureWorkspaceRootIdentity(root), + receiptPath: join(state, 'receipt.json'), + context: 'policy-v1', + setup: { + command: 'install', + timeoutMs: 1000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'check', + checkTimeoutMs: 100, + }, + }, + async execute(command) { + commands.push(command); + return { exitCode: 0, timedOut: false }; + }, + }; + return { root, state, commands, options }; +} + +test('unchanged checkout checks readiness without repeating installation, including after restart', async t => { + const { options, commands } = await fixture(t); + assert.equal(await prepareCodeEnvironment(options), 'prepared'); + assert.equal(await prepareCodeEnvironment({ ...options }), 'reused'); + assert.deepEqual(commands, ['install', 'check', 'check']); + assert.equal( + (await readFile(options.receiptPath, 'utf8')).includes('lock-v1'), + false, + ); +}); + +test('changed inputs, recipe and toolchain/policy context invalidate preparation', async t => { + const { options, root, commands } = await fixture(t); + await prepareCodeEnvironment(options); + await writeFile(join(root, 'package-lock.json'), 'lock-v2'); + await prepareCodeEnvironment(options); + await prepareCodeEnvironment({ ...options, context: 'policy-v2' }); + await prepareCodeEnvironment({ + ...options, + context: 'policy-v2', + setup: { ...options.setup, command: 'install-v2' }, + }); + assert.deepEqual(commands, [ + 'install', + 'check', + 'install', + 'check', + 'install', + 'check', + 'install-v2', + 'check', + ]); +}); + +test('missing installed artifacts trigger repair; a failed setup is never stamped', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + let installed = false; + await prepareCodeEnvironment({ + ...options, + async execute(command) { + commands.push(command); + if (command === 'install') installed = true; + return { exitCode: installed ? 0 : 1, timedOut: false }; + }, + }); + assert.deepEqual(commands, [ + 'install', + 'check', + 'check', + 'install', + 'check', + ]); + const failed = { + ...options, + receiptPath: join(options.receiptPath, '..', 'failed.json'), + execute: async () => ({ exitCode: 1, timedOut: false }), + }; + await assert.rejects(prepareCodeEnvironment(failed), /setup failed/); + await assert.rejects(readFile(failed.receiptPath), { code: 'ENOENT' }); +}); + +test('timed-out readiness and cancellation do not fall through to another setup', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async command => { + commands.push(command); + return { exitCode: 0, timedOut: true }; + }, + }), + /did not settle/, + ); + assert.deepEqual(commands, ['install', 'check', 'check']); + const controller = new AbortController(); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + signal: controller.signal, + execute: async () => { + controller.abort(); + return { exitCode: 0, timedOut: false }; + }, + }), + { name: 'AbortError' }, + ); +}); + +test('changing inputs during setup or readiness never publishes success', async t => { + const { options, root } = await fixture(t); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async () => { + await writeFile(join(root, 'package-lock.json'), 'changed'); + return { exitCode: 0, timedOut: false }; + }, + }), + /inputs changed/, + ); + await assert.rejects(readFile(options.receiptPath), { code: 'ENOENT' }); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async command => { + if (command === 'check') + await writeFile( + join(root, 'package-lock.json'), + 'changed-again', + ); + return { exitCode: 0, timedOut: false }; + }, + }), + /inputs changed/, + ); +}); + +test('input hashing is bounded and refuses symlink traversal and non-files', async t => { + const { options, root } = await fixture(t); + const outside = join(root, '..', 'outside'); + await mkdir(outside); + await writeFile(join(outside, 'secret'), 'do-not-read'); + await symlink(outside, join(root, 'escape')); + for (const path of ['escape/secret', 'escape']) { + await assert.rejects( + prepareCodeEnvironment({ + ...options, + setup: { + ...options.setup, + reuse: { ...options.setup.reuse!, inputs: [path] }, + }, + }), + ); + } + await writeFile(join(root, 'big'), Buffer.alloc(8 * 1024 * 1024 + 1)); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + setup: { + ...options.setup, + reuse: { ...options.setup.reuse!, inputs: ['big'] }, + }, + }), + /bounded regular-file/, + ); +}); + +test('legacy setup always runs and does not create preparation state', async t => { + const { options, commands } = await fixture(t); + const legacy = { + ...options, + setup: { command: 'install', timeoutMs: 1000 }, + }; + await prepareCodeEnvironment(legacy); + await prepareCodeEnvironment(legacy); + assert.deepEqual(commands, ['install', 'install']); + await assert.rejects(readFile(options.receiptPath), { code: 'ENOENT' }); +}); + +test('a foreign checkout receipt is not installation evidence and exposed state fails closed', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + const other = join(options.root, '..', 'other'); + await mkdir(other); + await writeFile(join(other, 'package-lock.json'), 'lock-v1'); + await prepareCodeEnvironment({ + ...options, + root: (await captureWorkspaceRootIdentity(other)).path, + identity: await captureWorkspaceRootIdentity(other), + }); + assert.deepEqual(commands, ['install', 'check', 'install', 'check']); + await chmod(options.receiptPath, 0o644); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + root: (await captureWorkspaceRootIdentity(other)).path, + identity: await captureWorkspaceRootIdentity(other), + }), + /owner-only/, + ); +}); + +test('reuse YAML requires bounded explicit inputs and readiness check', () => { + assert.ok( + parseCodeEnvironment( + 'name: app\nroot: app\nsetup:\n command: npm ci\n reuse:\n inputs: [package.json, package-lock.json]\n checkCommand: test -d node_modules\n', + ).setup?.reuse, + ); + for (const reuse of [ + 'inputs: []\ncheckCommand: ready', + 'inputs: [../outside]\ncheckCommand: ready', + 'inputs: [package.json, package.json]\ncheckCommand: ready', + 'inputs: [package.json]', + 'inputs: [package.json]\ncheckCommand: ready\ncheckTimeoutMs: 0', + 'inputs: [package.json]\ncheckCommand: ready\nunknown: true', + ]) + assert.throws(() => + parseCodeEnvironment( + `name: app\nroot: app\nsetup:\n command: install\n reuse:\n${reuse + .split('\n') + .map(line => ` ${line}`) + .join('\n')}\n`, + ), + ); +}); + +test('real local shell canary: installation reused, deleted output repaired, changed lockfile reinstalled', async t => { + const { options } = await fixture(t); + const run = promisify(execFile); + const ready = join(options.root, 'installed'); + const configured: EnvironmentPreparationOptions = { + ...options, + setup: { + command: 'printf "install\\n" >> installs.log; mkdir -p installed', + timeoutMs: 1000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -d installed', + checkTimeoutMs: 1000, + }, + }, + async execute(command, timeout) { + try { + await run('/bin/sh', ['-c', command], { + cwd: options.root, + timeout, + }); + return { exitCode: 0, timedOut: false }; + } catch (error) { + return { + exitCode: 1, + timedOut: Boolean((error as { killed?: boolean }).killed), + }; + } + }, + }; + await prepareCodeEnvironment(configured); + assert.equal(await prepareCodeEnvironment(configured), 'reused'); + await rm(ready, { recursive: true }); + await prepareCodeEnvironment(configured); + await writeFile(resolve(options.root, 'package-lock.json'), 'new-lock'); + await prepareCodeEnvironment(configured); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\ninstall\ninstall\n', + ); +}); + +test( + 'real native SRT preparation reuse keeps receipts inaccessible to checkout commands', + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, + }, + async t => { + const { options, state } = await fixture(t); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: options.root, + workspaceIdentity: options.identity, + protectedPaths: [state], + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + const execute: EnvironmentPreparationOptions['execute'] = ( + command, + timeoutMs, + ) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs, + maxOutputBytes: 1024, + }); + const configured = { + ...options, + execute, + setup: { + command: + 'printf "install\\n" >> installs.log; mkdir -p installed', + timeoutMs: 5000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -d installed', + checkTimeoutMs: 5000, + }, + }, + }; + assert.equal(await prepareCodeEnvironment(configured), 'prepared'); + assert.equal(await prepareCodeEnvironment(configured), 'reused'); + assert.notEqual( + (await execute(`cat '${options.receiptPath}'`, 5000)).exitCode, + 0, + ); + assert.notEqual( + (await execute(`printf forged > '${options.receiptPath}'`, 5000)) + .exitCode, + 0, + ); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\n', + ); + await rm(join(options.root, 'installed'), { recursive: true }); + assert.equal(await prepareCodeEnvironment(configured), 'prepared'); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\ninstall\n', + ); + }, +); diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts new file mode 100644 index 00000000..c2597f3b --- /dev/null +++ b/packages/code/src/environment-preparation.ts @@ -0,0 +1,150 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { resolve } from 'node:path'; +import { open, withWorkspaceRoot } from './root-access.js'; +import { + loadEnvironmentPreparationKey, + saveEnvironmentPreparationKey, +} from './storage.js'; +import type { CodeEnvironmentDefinition } from './environment.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +// Safety bounds on operator-declared hashing, not a dependency-store quota. +const MAX_INPUT_BYTES = 8 * 1024 * 1024; +const MAX_TOTAL_INPUT_BYTES = 32 * 1024 * 1024; + +export interface EnvironmentPreparationOptions { + root: string; + identity: WorkspaceRootIdentity; + setup: NonNullable; + receiptPath: string; + /** Includes the worker's policy and toolchain configuration. */ + context: string; + execute( + command: string, + timeoutMs: number, + ): Promise<{ exitCode: number | null; timedOut: boolean }>; + signal?: AbortSignal; +} + +/** Checkout-local reuse. Never transfers mutable installations between worktrees. */ +export async function prepareCodeEnvironment( + options: EnvironmentPreparationOptions, +): Promise<'prepared' | 'reused'> { + options.signal?.throwIfAborted(); + const key = await preparationKey(options); + if ( + key && + (await loadEnvironmentPreparationKey(options.receiptPath)) === key + ) { + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode === null) + throw new Error('Environment readiness check did not settle'); + if (check.exitCode === 0 && (await preparationKey(options)) === key) + return 'reused'; + } + const result = await options.execute( + options.setup.command, + options.setup.timeoutMs, + ); + options.signal?.throwIfAborted(); + if (result.timedOut || result.exitCode !== 0) + throw new Error('Environment setup failed'); + if (key) { + // Do not stamp an installation against inputs that changed during setup. + if ((await preparationKey(options)) !== key) + throw new Error( + 'Environment preparation inputs changed during setup', + ); + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode !== 0) + throw new Error('Environment readiness check failed after setup'); + if ((await preparationKey(options)) !== key) + throw new Error( + 'Environment preparation inputs changed during readiness check', + ); + await saveEnvironmentPreparationKey(options.receiptPath, key); + } + return 'prepared'; +} + +async function preparationKey( + options: EnvironmentPreparationOptions, +): Promise { + if (!options.setup.reuse) return undefined; + return withWorkspaceRoot(options.root, options.identity, async () => { + const hash = createHash('sha256').update( + JSON.stringify({ + version: 1, + root: options.identity, + setup: options.setup, + context: options.context, + node: process.version, + abi: process.versions.modules, + platform: process.platform, + arch: process.arch, + }), + ); + let total = 0; + for (const path of options.setup.reuse!.inputs) { + options.signal?.throwIfAborted(); + const handle = await open( + resolve(options.root, path), + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK, + ); + try { + const before = await handle.stat(); + if ( + !before.isFile() || + before.size > MAX_INPUT_BYTES || + total + before.size > MAX_TOTAL_INPUT_BYTES + ) + throw new Error( + 'Environment preparation inputs exceed the bounded regular-file contract', + ); + const buffer = Buffer.alloc(before.size + 1); + let length = 0; + while (length < buffer.length) { + options.signal?.throwIfAborted(); + const read = await handle.read( + buffer, + length, + buffer.length - length, + length, + ); + if (!read.bytesRead) break; + length += read.bytesRead; + } + const after = await handle.stat(); + if ( + length !== before.size || + after.size !== before.size || + after.mtimeMs !== before.mtimeMs || + after.ctimeMs !== before.ctimeMs + ) + throw new Error( + 'Environment preparation input changed while hashing', + ); + total += length; + hash.update(JSON.stringify([path, length])).update( + buffer.subarray(0, length), + ); + } finally { + await handle.close(); + } + } + return hash.digest('hex'); + }); +} diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index c2b2cf1f..406e3850 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -10,6 +10,7 @@ import { import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, + isSafePortableRelativePath, } from './protocol.js'; import type { LocalWorkspaceConfig } from './workspace.js'; import { WorkspaceToolError } from './workspace.js'; @@ -25,7 +26,12 @@ export interface CodeEnvironmentDefinition { root: string; repo?: string; ref?: string; - setup?: { command: string; timeoutMs: number }; + setup?: { + command: string; + timeoutMs: number; + /** Explicit readiness contract; absent preserves startup setup behavior. */ + reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number }; + }; actions?: { name: string; command: string; timeoutMs: number }[]; } @@ -89,7 +95,7 @@ export function parseCodeEnvironment( if ( !record(value.setup) || Object.keys(value.setup).some( - key => !['command', 'timeoutMs'].includes(key), + key => !['command', 'timeoutMs', 'reuse'].includes(key), ) || !text(value.setup.command, 16_384) || Buffer.byteLength(value.setup.command) > @@ -109,7 +115,25 @@ export function parseCodeEnvironment( `Environment setup timeout must be between 1 and ${BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS} ms`, ); } - setup = { command: value.setup.command, timeoutMs }; + let reuse: NonNullable['reuse']; + if (value.setup.reuse !== undefined) { + const candidate = value.setup.reuse; + if ( + !record(candidate) || + Object.keys(candidate).some(key => !['inputs', 'checkCommand', 'checkTimeoutMs'].includes(key)) || + !Array.isArray(candidate.inputs) || + candidate.inputs.length < 1 || candidate.inputs.length > 32 || + candidate.inputs.some(path => typeof path !== 'string' || !isSafePortableRelativePath(path) || path === '.') || + new Set(candidate.inputs).size !== candidate.inputs.length || + !text(candidate.checkCommand, 16_384) || + Buffer.byteLength(candidate.checkCommand) > BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) throw new Error('Invalid environment setup reuse'); + const checkTimeoutMs = candidate.checkTimeoutMs ?? 10_000; + if (typeof checkTimeoutMs !== 'number' || !Number.isSafeInteger(checkTimeoutMs) || checkTimeoutMs < 1 || checkTimeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS) + throw new Error('Invalid environment readiness check timeout'); + reuse = { inputs: candidate.inputs as string[], checkCommand: candidate.checkCommand, checkTimeoutMs }; + } + setup = { command: value.setup.command, timeoutMs, ...(reuse ? { reuse } : {}) }; } let actions: CodeEnvironmentDefinition['actions']; if (value.actions !== undefined) { diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index 1cbb7093..904a8562 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -1,4 +1,5 @@ import { createHash, randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; import { chmod, lstat, @@ -201,11 +202,15 @@ async function assertOwnedByWorker(path: string): Promise { async function readGuardedFile( path: string, exposed: (mode: string) => string, + maxBytes = Infinity, ): Promise { await assertReadPathPrivate(path); const handle = await open(path, 'r'); try { const stats = await handle.stat(); + if (!stats.isFile() || stats.size > maxBytes) { + throw new BridgeProtocolError('Invalid private storage file size or type'); + } const self = process.getuid?.(); if (self !== undefined && !isTrustedOwner(stats.uid, self)) { throw new BridgeProtocolError( @@ -423,6 +428,44 @@ export async function saveBridgeIdentity( path: string, identity: PairedBridgeWorkerIdentity, ): Promise { + return savePrivateJson(path, identity); +} + +/** Preparation receipts are worker state, never a workspace-authored success marker. */ +export async function prepareEnvironmentPreparationDirectory(path: string): Promise { + await assertPrivateStorageAncestors(path, true); + await ensureDurableDirectory(path); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await handle.stat(); + if (!metadata.isDirectory() || (metadata.mode & 0o077) !== 0) + throw new BridgeProtocolError('Environment preparation directory must be owner-only'); + await assertPrivateStorageAcl(handle, path, true); + } finally { await handle.close(); } + return assertPrivateStorageAncestors(path); +} + +export async function loadEnvironmentPreparationKey(path: string): Promise { + let content: string; + try { + content = await readGuardedFile(path, () => 'Environment preparation receipt must be owner-only', 256); + } catch (error) { + if (isMissingPathError(error)) return undefined; + throw error; + } + try { + const value: unknown = JSON.parse(content); + if (isRecord(value) && value.version === 1 && typeof value.key === 'string' && /^[a-f0-9]{64}$/.test(value.key)) return value.key; + } catch { /* Invalid receipts are cache misses, not readiness evidence. */ } + return undefined; +} + +export async function saveEnvironmentPreparationKey(path: string, key: string): Promise { + if (!/^[a-f0-9]{64}$/.test(key)) throw new Error('Invalid environment preparation key'); + await savePrivateJson(path, { version: 1, key }); +} + +async function savePrivateJson(path: string, value: unknown): Promise { assertPrivateStorageSupported(); await assertWriteContainerPrivate(path); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); @@ -435,13 +478,14 @@ export async function saveBridgeIdentity( await removePrivateStorageAcl(file, path); await file.chmod(0o600); await assertOwnerOnlyFile(file, path); - await file.writeFile(`${JSON.stringify(identity, null, 2)}\n`, 'utf8'); + await file.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8'); await file.sync(); } finally { await file.close(); } await rename(temporaryPath, path); await chmod(path, 0o600); + await syncParentDirectory(path); } catch (error) { await rm(temporaryPath, { force: true }); throw error; From 42a7fcd7cb02596cbb9ee861e1e24315b1163a24 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 07:24:03 -0400 Subject: [PATCH 57/75] =?UTF-8?q?=F0=9F=A7=B0=20feat:=20Share=20Explicit?= =?UTF-8?q?=20Environment=20Tool=20Resources=20(#287)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Grant Explicit Shared Tool Cache Resources * fix: Reject Replaced Tool Stores Before Dispatch * test: Verify Protected Receipt Contents Across Native Platforms --- packages/code/README.md | 46 +++- packages/code/src/cli.ts | 31 ++- .../code/src/environment-preparation.test.ts | 24 +- .../code/src/environment-resources.test.ts | 210 ++++++++++++++++++ packages/code/src/environment-resources.ts | 164 ++++++++++++++ packages/code/src/environment.ts | 8 +- packages/code/src/native-process.ts | 2 + packages/code/src/native-sandbox.ts | 23 ++ 8 files changed, 482 insertions(+), 26 deletions(-) create mode 100644 packages/code/src/environment-resources.test.ts create mode 100644 packages/code/src/environment-resources.ts diff --git a/packages/code/README.md b/packages/code/README.md index 47771057..47d3c3a0 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -1102,11 +1102,47 @@ This does not attach another checkout's `node_modules`, provision linked lanes o command admission, or recheck existing instances on every command. It does not deduplicate installed dependencies between worktrees or enforce disk quotas. -The next resource-store slice must explicitly grant shared cache paths under SRT, -keep monorepo links and mutable outputs checkout-local, and bound retention. Do not -work around that missing grant by broadening the sandbox root or symlinking another -branch's full installation. Shared download caches alone do not reduce installed -`node_modules` copies. +Shared tool cache grants below do not attach another checkout's installed dependency +tree. Keep monorepo links and mutable outputs checkout-local. Do not broaden the +sandbox root or symlink another branch's full installation. Shared download caches +alone do not reduce installed `node_modules` copies. + +## Shared tool and download resources + +Explicit operator-managed stores can live outside the checkouts: + +```yaml +resources: + - kind: npm-cache + path: /srv/lia-resources/npm + access: read-write + - kind: uv-cache + path: /srv/lia-resources/uv + access: read-write + - kind: playwright-browsers + path: /srv/lia-resources/playwright + access: read-only +``` + +Create each directory as the worker service account with mode `0700`, then populate +browser binaries using the matching Playwright version and `PLAYWRIGHT_BROWSERS_PATH` +outside the coding session. Roots must exist and may not be symlinks or overlap any +registered workspace or worker control state. Linux mount-alias checks cover both +directions. Keep the mount namespace stable while the worker runs. + +The worker grants only these paths and injects `npm_config_cache`, `UV_CACHE_DIR` +and `PLAYWRIGHT_BROWSERS_PATH` from the loaded definition, including in linked lanes +and fresh conversation worktrees. An undeclared process environment variable never +grants filesystem access. Each store root is inode-bound and revalidated before +commands. Read-only stores stay read-only, and speculative programmatic probes +cannot write any shared store. + +Sharing is explicit within one worker's trust domain. Do not share mutable caches +between unrelated principals, store credentials in them, or treat their contents +as trusted worker code. Separate package caches from browsers and mutable browser +profiles, Redis data, build output and test state. Stores currently have no automatic +eviction; the storage-lifecycle slice adds that separately. This shares downloads +and browser binaries, not installed `node_modules` trees. No setup output is sent to the model. Named actions are fixed commands without model-supplied substitution. The bridge diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 054074ed..04d94aaa 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -13,6 +13,7 @@ import { EnvironmentWorkspaceTools, } from './environment.js'; import { prepareCodeEnvironment } from './environment-preparation.js'; +import { assertEnvironmentResourceIsolation } from './environment-resources.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -837,6 +838,16 @@ async function run( definition: { name: 'preparation-state', root: preparationDirectory }, fingerprint: '', }], roots); } + // Cache contents may be shared explicitly; their grants must never authorize source or control state. + const resourceRoots = environments.flatMap(environment => (environment.resources ?? []).map(resource => ({ + path: resource.path, sourceParents: resource.controlPaths, + definition: { name: 'shared-resource', root: resource.path }, fingerprint: '', + }))); + await assertEnvironmentResourceIsolation(environments.flatMap(environment => environment.resources ?? []), roots.map(root => root.root), [ + identityPath, preparationDirectory, github.privateKeyPath, + ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), + ].filter((path): path is string => path != null)); + await assertEnvironmentDefinitionsOutsideRoots(resourceRoots, roots); const preparationReceipt = (root: string) => join(preparationDirectory, `${createHash('sha256').update(JSON.stringify([codeApiUrl, workerId, root])).digest('hex')}.json`); // Keep an admission boundary even when trusted-VM checkout routing uses a @@ -1110,6 +1121,10 @@ async function run( } : {}), }; + const nativeOptionsForWorkspace = (workspaceId: string): NativeProcessSandboxOptions => ({ + ...nativeOptions, + resources: environments.find(environment => environment.definition.name === workspaceId)?.resources, + }); const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot || linkedWorktreeLanes @@ -1117,12 +1132,12 @@ async function run( new Map( roots.map(root => [ root.id, - { ...nativeOptions, workspaceRoot: root.root, workspaceIdentity: root.identity }, + { ...nativeOptionsForWorkspace(root.id), workspaceRoot: root.root, workspaceIdentity: root.identity }, ]), ), workspaceLeaseSlots, ) - : new NativeProcessWorkspaceCommandSandbox(nativeOptions) + : new NativeProcessWorkspaceCommandSandbox(nativeOptionsForWorkspace(roots[0].id)) : undefined; if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ @@ -1161,14 +1176,15 @@ async function run( } const id = internalWorkspaceId(instance.sourceWorkspaceId, instance.id); await nativeCommandSandbox.registerRoot(id, { - ...nativeOptions, + ...nativeOptionsForWorkspace(instance.sourceWorkspaceId), workspaceIdentity: instance.identity, workspaceRoot: instance.root, }); await prepareCodeEnvironment({ root: instance.root, identity: instance.identity, setup, receiptPath: preparationReceipt(instance.root), - context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity]), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, + nativeOptionsForWorkspace(instance.sourceWorkspaceId).resources]), signal, execute: (command, timeoutMs) => nativeCommandSandbox.execute({ protocolVersion: 1, @@ -1225,7 +1241,7 @@ async function run( root.id, { command: { - ...nativeOptions, + ...nativeOptionsForWorkspace(root.id), workspaceIdentity: root.identity, workspaceRoot: root.root, }, @@ -1263,7 +1279,7 @@ async function run( { root: root.root, identity: root.identity, - command: nativeOptions, + command: nativeOptionsForWorkspace(root.id), repositoryInstructions: args.includes('--repository-instructions'), writable: root.writable ?? false, }, @@ -1340,7 +1356,8 @@ async function run( root: environment.definition.root, identity: roots.find(root => root.id === id)!.identity!, setup, receiptPath: preparationReceipt(environment.definition.root), - context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity]), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, + nativeOptionsForWorkspace(id).resources]), signal: controller.signal, execute: async (command, timeoutMs) => { if (!armed) { diff --git a/packages/code/src/environment-preparation.test.ts b/packages/code/src/environment-preparation.test.ts index 51149441..55186064 100644 --- a/packages/code/src/environment-preparation.test.ts +++ b/packages/code/src/environment-preparation.test.ts @@ -25,7 +25,7 @@ async function fixture(t: test.TestContext) { t.after(() => rm(directory, { recursive: true, force: true })); const root = join(directory, 'checkout'); const state = join(directory, 'private'); - await mkdir(root); + await mkdir(root, { mode: 0o700 }); await prepareEnvironmentPreparationDirectory(state); await writeFile(join(root, 'package-lock.json'), 'lock-v1'); const commands: string[] = []; @@ -316,9 +316,9 @@ test( }); t.after(() => sandbox.close()); await sandbox.prepare(); - const execute: EnvironmentPreparationOptions['execute'] = ( - command, - timeoutMs, + const execute = ( + command: string, + timeoutMs: number, ) => sandbox.execute({ protocolVersion: 1, @@ -344,15 +344,13 @@ test( }; assert.equal(await prepareCodeEnvironment(configured), 'prepared'); assert.equal(await prepareCodeEnvironment(configured), 'reused'); - assert.notEqual( - (await execute(`cat '${options.receiptPath}'`, 5000)).exitCode, - 0, - ); - assert.notEqual( - (await execute(`printf forged > '${options.receiptPath}'`, 5000)) - .exitCode, - 0, - ); + const receipt = await readFile(options.receiptPath, 'utf8'); + const protectedRead = await execute(`cat '${options.receiptPath}'`, 5000); + // Linux SRT may mask denied reads with an empty file. Exit status alone + // is not evidence that the protected receipt was exposed. + assert.ok(!protectedRead.stdout?.includes(receipt)); + await execute(`printf forged > '${options.receiptPath}'`, 5000); + assert.equal(await readFile(options.receiptPath, 'utf8'), receipt); assert.equal( await readFile(join(options.root, 'installs.log'), 'utf8'), 'install\n', diff --git a/packages/code/src/environment-resources.test.ts b/packages/code/src/environment-resources.test.ts new file mode 100644 index 00000000..32d1d390 --- /dev/null +++ b/packages/code/src/environment-resources.test.ts @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict'; +import { + chmod, + mkdir, + mkdtemp, + readFile, + rename, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + parseCodeEnvironment, + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, +} from './environment.js'; +import { + parseEnvironmentResources, + loadEnvironmentResource, + assertEnvironmentResourcesStable, +} from './environment-resources.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; + +test('shared resources are explicit bounded known cache kinds, not arbitrary environment or filesystem grants', () => { + assert.equal( + parseCodeEnvironment( + 'name: app\nroot: app\nresources:\n - kind: npm-cache\n path: /cache/npm\n access: read-write\n', + ).resources?.[0].kind, + 'npm-cache', + ); + for (const value of [ + [], + [{ kind: 'secrets', path: '/cache', access: 'read-write' }], + [{ kind: 'npm-cache', path: '../cache', access: 'read-write' }], + [{ kind: 'uv-cache', path: '/cache', access: 'all' }], + [ + { + kind: 'uv-cache', + path: '/cache', + access: 'read-only', + env: 'GH_TOKEN', + }, + ], + [1], + [ + { kind: 'uv-cache', path: '/cache', access: 'read-only' }, + { kind: 'uv-cache', path: '/other', access: 'read-only' }, + ], + ]) + assert.throws(() => parseEnvironmentResources(value)); +}); + +test('resource roots must be stable private directories and outside source grants', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-resources-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'root'); + const cache = join(directory, 'cache'); + await mkdir(root, { mode: 0o700 }); + await mkdir(cache, { mode: 0o700 }); + const resource = { + kind: 'npm-cache' as const, + path: cache, + access: 'read-write' as const, + }; + const loaded = await loadEnvironmentResource(resource); + await assertEnvironmentResourcesStable([loaded]); + await chmod(cache, 0o755); + await assert.rejects(loadEnvironmentResource(resource), /owner-only/); + await chmod(cache, 0o700); + await symlink(cache, join(directory, 'alias')); + await assert.rejects( + loadEnvironmentResource({ + ...resource, + path: join(directory, 'alias'), + }), + /symlink/, + ); + await rename(cache, join(directory, 'old')); + await mkdir(cache, { mode: 0o700 }); + await assert.rejects(assertEnvironmentResourcesStable([loaded]), /changed/); + const file = join(directory, 'app.yaml'); + await writeFile( + file, + `name: app\nroot: ${root}\nresources:\n - kind: npm-cache\n path: ${cache}\n access: read-write\n`, + { mode: 0o600 }, + ); + const environment = await loadCodeEnvironment(file); + const [cacheRoot] = environment.resources!; + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots( + [ + { + path: cacheRoot.path, + sourceParents: cacheRoot.controlPaths, + fingerprint: '', + definition: { name: 'cache', root: cacheRoot.path }, + }, + ], + [{ id: 'root', root: directory }], + ), + /workspace|root|definition/i, + ); +}); + +test( + 'real native SRT shares only declared stores across independent checkouts and refuses readonly writes', + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, + }, + async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-resources-live-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const cache = join(directory, 'cache'); + const browser = join(directory, 'browsers'); + await mkdir(cache, { mode: 0o700 }); + await mkdir(browser, { mode: 0o700 }); + await writeFile(join(browser, 'version'), 'pinned-browser'); + const resources = await Promise.all([ + loadEnvironmentResource({ + kind: 'npm-cache', + path: cache, + access: 'read-write', + }), + loadEnvironmentResource({ + kind: 'playwright-browsers', + path: browser, + access: 'read-only', + }), + ]); + const sandboxes = await Promise.all( + ['a', 'b'].map(async name => { + const root = join(directory, name); + await mkdir(root); + const identity = await captureWorkspaceRootIdentity(root); + await writeFile(join(root, 'private'), 'other-checkout'); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: identity.path, + workspaceIdentity: identity, + homeDirectory: directory, + resources, + }); + await sandbox.prepare(); + return sandbox; + }), + ); + t.after(async () => { + for (const sandbox of sandboxes) await sandbox.close(); + }); + const execute = (index: number, command: string) => + sandboxes[index].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs: 5000, + }); + const results = await Promise.all( + sandboxes.map((_, i) => + execute( + i, + 'printf "%s\\n" "$npm_config_cache"; printf once >> "$npm_config_cache/marker"; cat "$PLAYWRIGHT_BROWSERS_PATH/version"', + ), + ), + ); + for (const result of results) { + assert.equal(result.exitCode, 0, result.stderr); + assert.match(result.stdout, /pinned-browser/); + } + assert.equal(await readFile(join(cache, 'marker'), 'utf8'), 'onceonce'); + assert.notEqual( + ( + await execute( + 0, + 'printf forbidden > "$PLAYWRIGHT_BROWSERS_PATH/version"', + ) + ).exitCode, + 0, + ); + assert.equal( + await readFile(join(browser, 'version'), 'utf8'), + 'pinned-browser', + ); + assert.notEqual( + (await execute(0, `cat '${join(directory, 'b', 'private')}'`)) + .exitCode, + 0, + ); + await rename(cache, join(directory, 'old-cache')); + await mkdir(cache, { mode: 0o700 }); + await assert.rejects( + execute(0, 'printf not-started > side-effect'), + error => { + assert.equal( + (error as { mutationMayHaveCommitted: boolean }) + .mutationMayHaveCommitted, + false, + ); + return /resource changed/.test((error as Error).message); + }, + ); + await assert.rejects(readFile(join(directory, 'a', 'side-effect')), { + code: 'ENOENT', + }); + }, +); diff --git a/packages/code/src/environment-resources.ts b/packages/code/src/environment-resources.ts new file mode 100644 index 00000000..1bb57440 --- /dev/null +++ b/packages/code/src/environment-resources.ts @@ -0,0 +1,164 @@ +import { lstat, open, realpath } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { isAbsolute, relative, sep } from 'node:path'; +import { + assertPrivateStorageAcl, + assertPrivateStorageAncestors, +} from './private-storage.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + createEnvironmentMountIsolation, + readEnvironmentMountTable, +} from './environment-mount.js'; + +export type EnvironmentResourceKind = + | 'npm-cache' + | 'uv-cache' + | 'playwright-browsers'; +export interface EnvironmentResource { + kind: EnvironmentResourceKind; + path: string; + access: 'read-only' | 'read-write'; +} +export interface LoadedEnvironmentResource extends EnvironmentResource { + identity: WorkspaceRootIdentity; + controlPaths: string[]; +} +export const RESOURCE_ENVIRONMENT_NAMES: Record< + EnvironmentResourceKind, + string +> = { + 'npm-cache': 'npm_config_cache', + 'uv-cache': 'UV_CACHE_DIR', + 'playwright-browsers': 'PLAYWRIGHT_BROWSERS_PATH', +}; + +export function parseEnvironmentResources( + value: unknown, +): EnvironmentResource[] { + if (!Array.isArray(value) || value.length < 1 || value.length > 3) + throw new Error('Invalid environment resources'); + const kinds = new Set(); + return value.map(raw => { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) + throw new Error('Invalid environment resource'); + const entry = raw as Record; + if ( + Object.keys(entry).some( + key => !['kind', 'path', 'access'].includes(key), + ) || + typeof entry.kind !== 'string' || + !Object.hasOwn(RESOURCE_ENVIRONMENT_NAMES, entry.kind) || + kinds.has(entry.kind) || + typeof entry.path !== 'string' || + !isAbsolute(entry.path) || + entry.path.length > 4096 || + /[\0\r\n]/.test(entry.path) || + (entry.access !== 'read-only' && entry.access !== 'read-write') + ) + throw new Error('Invalid environment resource'); + kinds.add(entry.kind); + return { + kind: entry.kind as EnvironmentResourceKind, + path: entry.path, + access: entry.access, + }; + }); +} + +/** Operator-owned roots only. Resource contents are still untrusted tool data. */ +export async function loadEnvironmentResource( + resource: EnvironmentResource, +): Promise { + const controlPaths = await assertPrivateStorageAncestors(resource.path); + if ((await lstat(resource.path)).isSymbolicLink()) + throw new Error('Environment resource root cannot be a symlink'); + const path = await realpath(resource.path); + const identity = await captureWorkspaceRootIdentity(path); + const handle = await open( + path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat({ bigint: true }); + if ( + (metadata.mode & 0o077n) !== 0n || + metadata.dev.toString() !== identity.dev || + metadata.ino.toString() !== identity.ino + ) + throw new Error( + 'Environment resource directory must be stable and owner-only', + ); + await assertPrivateStorageAcl(handle, path, true); + } finally { + await handle.close(); + } + return { ...resource, path, identity, controlPaths }; +} + +export async function assertEnvironmentResourcesStable( + resources: readonly LoadedEnvironmentResource[], +): Promise { + for (const resource of resources) { + if (!(await matchesWorkspaceRoot(resource.path, resource.identity))) + throw new Error( + 'Environment resource root changed after admission', + ); + } +} + +export function environmentResourceVariables( + resources: readonly EnvironmentResource[], +): Record { + return Object.fromEntries( + resources.map(resource => [ + RESOURCE_ENVIRONMENT_NAMES[resource.kind], + resource.path, + ]), + ); +} + +export function assertEnvironmentResourceSeparation( + resources: readonly EnvironmentResource[], + roots: readonly string[], +): void { + const contains = (parent: string, child: string) => { + const path = relative(parent, child); + return ( + path === '' || + (!isAbsolute(path) && path !== '..' && !path.startsWith(`..${sep}`)) + ); + }; + for (const resource of resources) { + if ( + roots.some( + root => + contains(root, resource.path) || + contains(resource.path, root), + ) + ) + throw new Error( + 'Environment resources must not overlap any registered workspace', + ); + } +} + +export async function assertEnvironmentResourceIsolation( + resources: readonly EnvironmentResource[], + roots: readonly string[], + controls: readonly string[], +): Promise { + if (!resources.length) return; + assertEnvironmentResourceSeparation(resources, [...roots, ...controls]); + const table = await readEnvironmentMountTable(); + if (table !== undefined) { + const check = createEnvironmentMountIsolation(table); + const stores = resources.map(resource => resource.path); + check(stores, roots); + check([...roots, ...controls], stores); + } +} diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 406e3850..976ffe6d 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -3,6 +3,8 @@ import { constants } from 'node:fs'; import { open, realpath, stat } from 'node:fs/promises'; import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import { parseDocument } from 'yaml'; +import { parseEnvironmentResources, loadEnvironmentResource } from './environment-resources.js'; +import type { EnvironmentResource, LoadedEnvironmentResource } from './environment-resources.js'; import { assertPrivateStorageAcl, assertPrivateStorageAncestors, @@ -33,6 +35,7 @@ export interface CodeEnvironmentDefinition { reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number }; }; actions?: { name: string; command: string; timeoutMs: number }[]; + resources?: EnvironmentResource[]; } export interface LoadedCodeEnvironment { @@ -41,6 +44,7 @@ export interface LoadedCodeEnvironment { rootPaths?: string[]; definition: CodeEnvironmentDefinition; fingerprint: string; + resources?: LoadedEnvironmentResource[]; } function record(value: unknown): value is Record { @@ -73,7 +77,7 @@ export function parseCodeEnvironment( !record(value) || Object.keys(value).some( key => - !['name', 'root', 'repo', 'ref', 'setup', 'actions'].includes( + !['name', 'root', 'repo', 'ref', 'setup', 'actions', 'resources'].includes( key, ), ) || @@ -173,6 +177,7 @@ export function parseCodeEnvironment( ...(typeof value.ref === 'string' ? { ref: value.ref } : {}), ...(setup ? { setup } : {}), ...(actions ? { actions } : {}), + ...(value.resources !== undefined ? { resources: parseEnvironmentResources(value.resources) } : {}), }; } @@ -328,6 +333,7 @@ export async function loadCodeEnvironment( sourceParents, rootPaths, definition, + ...(definition.resources ? { resources: await Promise.all(definition.resources.map(loadEnvironmentResource)) } : {}), fingerprint: createHash('sha256') .update(JSON.stringify(definition)) .digest('hex'), diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 77fe8269..588f4c33 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -344,6 +344,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan shellPath, programmaticFileUpstream, linkedWorktree, + resources, } = this.options; await this.rpc( 'prepare', @@ -358,6 +359,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan shellPath, programmaticFileUpstream, linkedWorktree, + resources, variables: this.options.maskedEnvironment?.variables, }, }, diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 26c848e5..2701fe44 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -15,6 +15,7 @@ import type { Dirent, Stats } from 'node:fs'; import { access, mkdtemp, open, readdir, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { matchesWorkspaceRoot } from './root-identity.js'; +import { assertEnvironmentResourcesStable, environmentResourceVariables } from './environment-resources.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; import { withWorkspaceRoot, WorkspaceRootAccessError, spawnWithinWorkspace, realpath as rootedRealpath, stat as rootedStat } from './root-access.js'; @@ -168,6 +169,8 @@ type SpawnCommand = ( ) => ChildProcessWithoutNullStreams; export interface NativeSrtWorkspaceCommandSandboxOptions { + /** Explicit operator-approved stores, never inferred from process environment. */ + resources?: import('./environment-resources.js').LoadedEnvironmentResource[]; workspaceIdentity?: WorkspaceRootIdentity; workspaceRoot: string; /** Present when `workspaceRoot` is a verified linked worktree lane of a checkout. */ @@ -530,6 +533,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const protectedPaths = await Promise.all( (this.options.protectedPaths ?? []).map(canonicalPath), ); + const resources = this.options.resources ?? []; + await assertEnvironmentResourcesStable(resources); + for (const resource of resources) { + if (isWithin(root, resource.path) || isWithin(resource.path, root) || + protectedPaths.some(path => isWithin(resource.path, path) || isWithin(path, resource.path))) { + throw new WorkspaceToolError('Environment resource overlaps a workspace or worker control path', 'REGISTRATION_INVALID'); + } + } if (protectedPaths.some(path => isWithin(root, path))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain worker control files', @@ -676,6 +687,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], allowRead: [ root, + ...resources.map(resource => resource.path), ...laneGitPaths, ...(gitGuardDirectory ? [gitGuardDirectory] : []), ...(canonicalScratchDirectory @@ -684,6 +696,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], allowWrite: [ root, + ...resources.filter(resource => resource.access === 'read-write').map(resource => resource.path), ...writableGitPaths, ...(canonicalScratchDirectory ? [canonicalScratchDirectory] @@ -691,6 +704,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], denyWrite: [ ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), ...deniedInheritedWritablePaths, ...rootGitMetadataDenies, ...(gitGuardDirectory ? [gitGuardDirectory] : []), @@ -762,10 +776,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ]; this.baseDenyWritePaths = [ ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), ...deniedInheritedWritablePaths, ]; this.denyWritePaths = [ ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), ...deniedInheritedWritablePaths, ...rootGitMetadataDenies, ...(gitGuardDirectory ? [gitGuardDirectory] : []), @@ -988,6 +1004,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ? { filesystem: { allowRead: [ + ...(this.options.resources ?? []).map(resource => resource.path), canonicalWorkspaceRoot ?? this.canonicalRoot!, canonicalDataDirectory, ...(this.canonicalCommonGitDir @@ -1005,6 +1022,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], denyRead: this.denyReadPaths, denyWrite: [ + ...(this.options.resources ?? []).map(resource => resource.path), this.canonicalRoot!, ...this.denyWritePaths, ], @@ -1138,6 +1156,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } await this.initialize(); + await assertEnvironmentResourcesStable(this.options.resources ?? []).catch(() => { + throw new WorkspaceToolError('Environment resource changed before command dispatch', 'REGISTRATION_INVALID'); + }); const root = workspaceRoot ?? this.canonicalRoot!; let cwd: string; try { @@ -1167,6 +1188,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : request.command; wrapped = await this.withTemporaryHostEnvironment( { + ...environmentResourceVariables(this.options.resources ?? []), ...this.gitEnvironment, ...(credentialEnvironment ?? {}), ...this.scratchSelectorEnvironment(sandboxScratchDirectory), @@ -1270,6 +1292,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox cwd, env: { ...wrapped.env, + ...environmentResourceVariables(this.options.resources ?? []), ...this.scratchEnvironment(), ...trustedEnvironment, ...this.gitEnvironment, From cdb457d024cde52967a01b96a39d7733dc7dc135 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 07:41:15 -0400 Subject: [PATCH 58/75] =?UTF-8?q?=F0=9F=93=A6=20feat:=20Restore=20Isolated?= =?UTF-8?q?=20Dependency=20Snapshots=20(#288)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Restore Isolated Dependency Snapshots * fix: Reject Dependency Hardlink and Control Path Aliases --- packages/code/README.md | 48 +- packages/code/src/cli.ts | 20 +- .../code/src/dependency-snapshots.test.ts | 267 ++++++++++ packages/code/src/dependency-snapshots.ts | 457 ++++++++++++++++++ packages/code/src/environment-preparation.ts | 77 ++- packages/code/src/environment.ts | 11 +- packages/code/src/root-access.ts | 67 ++- 7 files changed, 936 insertions(+), 11 deletions(-) create mode 100644 packages/code/src/dependency-snapshots.test.ts create mode 100644 packages/code/src/dependency-snapshots.ts diff --git a/packages/code/README.md b/packages/code/README.md index 47d3c3a0..a861efd3 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -1100,13 +1100,59 @@ Existing definitions without `reuse` retain the startup behavior. Fresh conversa instances use the same preparation contract, with independent checkout receipts. This does not attach another checkout's `node_modules`, provision linked lanes on command admission, or recheck existing instances on every command. It does not -deduplicate installed dependencies between worktrees or enforce disk quotas. +deduplicate installed dependencies between worktrees unless snapshots below are +configured, or enforce disk quotas. Shared tool cache grants below do not attach another checkout's installed dependency tree. Keep monorepo links and mutable outputs checkout-local. Do not broaden the sandbox root or symlink another branch's full installation. Shared download caches alone do not reduce installed `node_modules` copies. +## Copy-on-write installed dependencies + +For matching fresh checkouts on the **same clone-capable filesystem**, add a +private snapshot store to the readiness contract: + +```yaml +setup: + command: npm ci + timeoutMs: 300000 + reuse: + inputs: [package.json, package-lock.json] + checkCommand: test -x node_modules/.bin/tsc + snapshot: + store: /srv/lia-state/dependency-snapshots + paths: [node_modules] + maxBytes: 4294967296 + maxFiles: 200000 +``` + +Create the external store as the worker account with mode `0700`. It must not +overlap any source root, definition, credential or shared tool cache. Commands +cannot read or write it. Use a separate store per project/trust domain. The +portable key includes project identity, declared input bytes, recipe, policy, +Node ABI and platform, but not the checkout inode. Kernel locks serialize setup +for one key; different keys remain independent. Incomplete clones are never +published. Cancellation is checked during bounded traversal. + +Matching snapshots restore only when **every** declared `node_modules` directory +is missing. The sandboxed readiness check must pass before accepting the restore. +Existing directories are never replaced by restoration; ordinary setup handles +repair. Include nested workspace installations explicitly. Relative checkout-local +package links are preserved; absolute/escaping links, hard-linked files, links into +Git/worktree control paths and special files are rejected. Hardlink-based package +manager layouts need a different adapter; this snapshot mode targets npm copies. +Changing one restored installation cannot modify the snapshot or another checkout. + +This requires APFS clones or Linux reflinks (for example a suitably configured +XFS/Btrfs volume). The worker verifies cloning before installation and rejects +unsupported filesystems instead of silently making full copies or writable hard +links. Ordinary ext4 workers should leave snapshots disabled and can still use +checkout-local preparation receipts and shared downloads. This is **not** Python +virtualenv relocation, automatic preparation of manually created linked lanes, +or a hard quota on arbitrary commands. Validate all install/postinstall inputs +and path-independent artifacts before opting in. + ## Shared tool and download resources Explicit operator-managed stores can live outside the checkouts: diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 04d94aaa..4c64c0ae 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -848,6 +848,13 @@ async function run( ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), ].filter((path): path is string => path != null)); await assertEnvironmentDefinitionsOutsideRoots(resourceRoots, roots); + const snapshotStores = environments.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); + await assertEnvironmentResourceIsolation(snapshotStores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), roots.map(root => root.root), [ + identityPath, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), + ...rootQuarantinePaths.values(), ...environments.flatMap(environment => (environment.resources ?? []).map(resource => resource.path)), + ].filter((path): path is string => path != null)); + await assertEnvironmentDefinitionsOutsideRoots(snapshotStores.map(store => ({ path: store.store, sourceParents: store.controlPaths, + definition: { name: 'dependency-store', root: store.store }, fingerprint: '' })), roots); const preparationReceipt = (root: string) => join(preparationDirectory, `${createHash('sha256').update(JSON.stringify([codeApiUrl, workerId, root])).digest('hex')}.json`); // Keep an admission boundary even when trusted-VM checkout routing uses a @@ -1073,6 +1080,7 @@ async function run( protectedPaths: [ identityPath, ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), + ...snapshotStores.map(store => store.store), ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), github.privateKeyPath, @@ -1163,10 +1171,11 @@ async function run( ...(nativeCommandSandbox instanceof NativeWorkspaceCommandPool ? { prepareInstance: async (instance, signal) => { - const setup = environments.find( + const environment = environments.find( (environment) => environment.definition.name === instance.sourceWorkspaceId, - )?.definition.setup; + ); + const setup = environment?.definition.setup; if (!setup) return; if (admittedGitHubRepositories) { admittedGitHubRepositories.set( @@ -1181,6 +1190,8 @@ async function run( workspaceRoot: instance.root, }); await prepareCodeEnvironment({ + snapshotStore: environment!.snapshotStore, + snapshotScope: environment!.definition.repo ?? environment!.definition.name, root: instance.root, identity: instance.identity, setup, receiptPath: preparationReceipt(instance.root), context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, @@ -1353,6 +1364,11 @@ async function run( await guard.assertAvailable(); let armed = false; const preparation = await prepareCodeEnvironment({ + snapshotStore: environment.snapshotStore, + snapshotScope: environment.definition.repo ?? environment.definition.name, + beforeMutation: async () => { + if (!armed) { await guard.arm('Dependency restoration did not settle', 'setup'); armed = true; } + }, root: environment.definition.root, identity: roots.find(root => root.id === id)!.identity!, setup, receiptPath: preparationReceipt(environment.definition.root), diff --git a/packages/code/src/dependency-snapshots.test.ts b/packages/code/src/dependency-snapshots.test.ts new file mode 100644 index 00000000..b23ae70f --- /dev/null +++ b/packages/code/src/dependency-snapshots.test.ts @@ -0,0 +1,267 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + mkdtemp, + mkdir, + writeFile, + readFile, + lstat, + link, + realpath, + rm, + symlink, + rename, + readdir, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + parseDependencySnapshot, + loadDependencySnapshot, + publishDependencySnapshot, + restoreDependencySnapshot, + withDependencySnapshot, +} from './dependency-snapshots.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import type { EnvironmentPreparationOptions } from './environment-preparation.js'; + +async function fixture(t: test.TestContext) { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'dependency-snapshot-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const a = join(directory, 'a'), + b = join(directory, 'b'), + storePath = join(directory, 'store'); + for (const path of [a, b, storePath]) await mkdir(path, { mode: 0o700 }); + const store = await loadDependencySnapshot( + parseDependencySnapshot({ store: storePath, paths: ['node_modules'] }), + ); + const ai = await captureWorkspaceRootIdentity(a), + bi = await captureWorkspaceRootIdentity(b); + return { directory, a, b, store, ai, bi, key: 'a'.repeat(64) }; +} + +test('snapshot schema restricts installed directories, overlapping paths and finite bounds', () => { + const valid = { + store: '/private/store', + paths: ['node_modules', 'packages/api/node_modules'], + }; + assert.equal(parseDependencySnapshot(valid).maxFiles, 200_000); + for (const paths of [ + ['.git'], + ['../node_modules'], + ['node_modules', 'node_modules/pkg/node_modules'], + ['node_modules', 'node_modules'], + ]) + assert.throws(() => parseDependencySnapshot({ ...valid, paths })); + assert.throws(() => + parseDependencySnapshot({ ...valid, maxBytes: Infinity }), + ); + assert.throws(() => + parseDependencySnapshot({ ...valid, store: 'relative' }), + ); +}); + +// These assertions exercise real filesystem cloning, not a mocked successful copy. +test( + 'real clone snapshots isolate writable files, preserve checkout-local links and never replace installations', + { + skip: + process.platform !== 'darwin' && + process.env.LIBRECHAT_CODE_LIVE_SNAPSHOT_TESTS !== '1', + }, + async t => { + const { a, b, store, ai, bi, key } = await fixture(t); + await mkdir(join(a, 'node_modules')); + await writeFile( + join(a, 'node_modules', 'package.js'), + 'module.exports = 42', + ); + await mkdir(join(a, 'packages')); + await mkdir(join(b, 'packages')); + await symlink('../packages', join(a, 'node_modules', 'local')); + await withDependencySnapshot(store, key, () => + publishDependencySnapshot(store, key, a, ai), + ); + assert.equal( + await withDependencySnapshot(store, key, () => + restoreDependencySnapshot(store, key, b, bi), + ), + true, + ); + assert.notEqual( + (await lstat(join(a, 'node_modules', 'package.js'))).ino, + (await lstat(join(b, 'node_modules', 'package.js'))).ino, + ); + await writeFile(join(b, 'node_modules', 'package.js'), 'changed'); + assert.equal( + await readFile(join(a, 'node_modules', 'package.js'), 'utf8'), + 'module.exports = 42', + ); + assert.equal( + await readFile(join(store.store, key, '0', 'package.js'), 'utf8'), + 'module.exports = 42', + ); + assert.equal(await restoreDependencySnapshot(store, key, b, bi), false); + assert.equal( + await readFile(join(b, 'node_modules', 'package.js'), 'utf8'), + 'changed', + ); + }, +); + +test('unsafe links and bounded traversal never publish a partial snapshot', async t => { + const { a, store, ai, key } = await fixture(t); + await mkdir(join(a, 'node_modules')); + await symlink('/etc/passwd', join(a, 'node_modules', 'escape')); + await assert.rejects( + publishDependencySnapshot(store, key, a, ai), + /checkout-local/, + ); + assert.deepEqual(await readdir(store.store), []); + await rm(join(a, 'node_modules', 'escape')); + await writeFile(join(a, 'node_modules', 'large'), 'large'); + await assert.rejects( + publishDependencySnapshot({ ...store, maxBytes: 1 }, key, a, ai), + /maxBytes/, + ); + assert.deepEqual(await readdir(store.store), []); +}); + +test('hardlinks cannot copy outside file contents into a readable dependency snapshot', async t => { + const { a, store, ai, key, directory } = await fixture(t); + await mkdir(join(a, 'node_modules')); + const outside = join(directory, 'private-input'); + await writeFile(outside, 'private'); + await link(outside, join(a, 'node_modules', 'alias')); + await assert.rejects( + publishDependencySnapshot(store, key, a, ai), + /hard-linked/, + ); + assert.deepEqual(await readdir(store.store), []); +}); + +test('kernel lock serializes publishers, supports cancellation and rejects a replaced store', async t => { + const { store, key } = await fixture(t); + let release!: () => void; + const blocked = new Promise(resolve => { + release = resolve; + }); + let entered!: () => void; + const started = new Promise(resolve => { + entered = resolve; + }); + const first = withDependencySnapshot(store, key, async () => { + entered(); + await blocked; + }); + await started; + const controller = new AbortController(); + const second = withDependencySnapshot( + store, + key, + async () => assert.fail('cancelled lock ran'), + controller.signal, + ); + controller.abort(); + await assert.rejects(second); + release(); + await first; + await rename(store.store, `${store.store}-old`); + await mkdir(store.store, { mode: 0o700 }); + await assert.rejects( + withDependencySnapshot(store, key, async () => {}), + /changed after admission/, + ); +}); + +test( + 'real native preparation shares dependencies between checkouts, invalidates changed inputs and denies store access', + { + skip: + process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1' || + (process.platform !== 'darwin' && + process.env.LIBRECHAT_CODE_LIVE_SNAPSHOT_TESTS !== '1'), + }, + async t => { + const { a, b, store, directory, ai, bi } = await fixture(t); + for (const root of [a, b]) + await writeFile(join(root, 'package-lock.json'), 'version-one'); + const commands: string[] = []; + const setup: EnvironmentPreparationOptions['setup'] = { + command: + "mkdir -p node_modules; printf 'module.exports = 42' > node_modules/package.js", + timeoutMs: 5000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -f node_modules/package.js', + checkTimeoutMs: 3000, + snapshot: { + store: store.store, + paths: store.paths, + maxBytes: store.maxBytes, + maxFiles: store.maxFiles, + }, + }, + }; + const sandboxes = [ai, bi].map( + identity => + new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: identity.path, + workspaceIdentity: identity, + homeDirectory: directory, + protectedPaths: [store.store], + allowedDomains: [], + }), + ); + t.after(async () => { + for (const sandbox of sandboxes) await sandbox.close(); + }); + const run = (index: number) => { + const identity = [ai, bi][index]; + return prepareCodeEnvironment({ + root: identity.path, + identity, + setup, + context: 'policy', + snapshotStore: store, + snapshotScope: 'project', + receiptPath: join(store.store, `receipt-${index}.json`), + execute: async (command, timeoutMs) => { + commands.push(command); + return sandboxes[index].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs, + maxOutputBytes: 8192, + }); + }, + }); + }; + assert.equal(await run(0), 'prepared'); + assert.equal(await run(1), 'restored'); + assert.equal( + commands.filter(command => command === setup.command).length, + 1, + ); + await writeFile(join(b, 'package-lock.json'), 'version-two'); + assert.equal(await run(1), 'prepared'); + assert.equal( + commands.filter(command => command === setup.command).length, + 2, + ); + const denied = await sandboxes[0].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: `ls '${store.store}'`, + timeoutMs: 3000, + }); + assert.notEqual(denied.exitCode, 0); + }, +); diff --git a/packages/code/src/dependency-snapshots.ts b/packages/code/src/dependency-snapshots.ts new file mode 100644 index 00000000..5c9ab173 --- /dev/null +++ b/packages/code/src/dependency-snapshots.ts @@ -0,0 +1,457 @@ +import { randomUUID } from 'node:crypto'; +import { constants } from 'node:fs'; +import * as fs from 'node:fs/promises'; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, + sep, +} from 'node:path'; +import { createRequire } from 'node:module'; +import * as rooted from './root-access.js'; +import { withProcessLock } from './process-lock.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import { loadEnvironmentResource } from './environment-resources.js'; +import { isSafePortableRelativePath } from './protocol.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +export interface DependencySnapshotConfig { + store: string; + paths: string[]; + maxBytes: number; + maxFiles: number; +} +export interface DependencySnapshotStore extends DependencySnapshotConfig { + identity: WorkspaceRootIdentity; + controlPaths: string[]; +} +export function parseDependencySnapshot( + value: unknown, +): DependencySnapshotConfig { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid dependency snapshot'); + const v = value as Record; + const maxBytes = v.maxBytes ?? 4 * 1024 ** 3; + const maxFiles = v.maxFiles ?? 200_000; + if ( + Object.keys(v).some( + k => !['store', 'paths', 'maxBytes', 'maxFiles'].includes(k), + ) || + typeof v.store !== 'string' || + !isAbsolute(v.store) || + v.store.length > 4096 || + /[\0\r\n]/.test(v.store) || + !Array.isArray(v.paths) || + !v.paths.length || + v.paths.length > 32 || + v.paths.some( + p => + typeof p !== 'string' || + !isSafePortableRelativePath(p) || + basename(p) !== 'node_modules' || + p.split('/').some(c => c === '.git' || c === '.worktrees'), + ) || + new Set(v.paths).size !== v.paths.length || + v.paths.some(p => + (v.paths as unknown[]).some( + q => p !== q && (p as string).startsWith(`${q}/`), + ), + ) || + !Number.isSafeInteger(maxBytes) || + (maxBytes as number) < 1 || + !Number.isSafeInteger(maxFiles) || + (maxFiles as number) < 1 + ) + throw new Error('Invalid dependency snapshot'); + return { + store: v.store, + paths: v.paths as string[], + maxBytes: maxBytes as number, + maxFiles: maxFiles as number, + }; +} +export async function loadDependencySnapshot( + config: DependencySnapshotConfig, +): Promise { + const resource = await loadEnvironmentResource({ + kind: 'npm-cache', + path: config.store, + access: 'read-only', + }); + return { + ...config, + store: resource.path, + identity: resource.identity, + controlPaths: resource.controlPaths, + }; +} + +let native: + | { clone: (...args: any[]) => number; errno: () => number } + | undefined; +/** No copy fallback and no hardlinks: failure means the host cannot meet the contract. */ +async function cloneFile(source: fs.FileHandle, target: string): Promise { + if (!native) { + const koffi = createRequire(import.meta.url)('koffi'); + const library = koffi.load(null); + native = { + errno: () => koffi.errno(), + clone: + process.platform === 'darwin' + ? library.func( + 'int fclonefileat(int srcfd, int dstfd, const char *name, unsigned int flags)', + ) + : library.func( + 'int ioctl(int fd, unsigned long request, ...)', + ), + }; + } + if (process.platform === 'darwin') { + const parent = await rooted.open( + dirname(target), + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + if (native.clone(source.fd, parent.fd, basename(target), 0) !== 0) + throw new Error( + `Copy-on-write dependency cloning unavailable (errno ${native.errno()})`, + ); + } finally { + await parent.close(); + } + } else if (process.platform === 'linux') { + const output = await rooted.open( + target, + constants.O_CREAT | + constants.O_EXCL | + constants.O_WRONLY | + constants.O_NOFOLLOW, + 0o600, + ); + try { + if (native.clone(output.fd, 0x40049409, 'int', source.fd) !== 0) + throw new Error( + `Copy-on-write dependency cloning unavailable (errno ${native.errno()})`, + ); + } finally { + await output.close(); + } + } else + throw new Error( + 'Dependency snapshots require a clone-capable POSIX filesystem', + ); +} + +function inside(root: string, path: string): boolean { + const p = relative(root, path); + return !isAbsolute(p) && p !== '..' && !p.startsWith(`..${sep}`); +} +async function removeTree(path: string): Promise { + const stat = await rooted.lstat(path).catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (!stat) return; + if (!stat.isDirectory()) return rooted.unlink(path); + for (const name of await rooted.readdir(path)) + await removeTree(join(path, name)); + await rooted.rmdir(path); +} +async function cloneTree( + sourceRoot: string, + sourceIdentity: WorkspaceRootIdentity, + targetRoot: string, + targetIdentity: WorkspaceRootIdentity, + paths: string[], + config: DependencySnapshotConfig, + checkout: string, + signal?: AbortSignal, +): Promise { + let bytes = 0, + files = 0; + const target = (operation: () => Promise) => + rooted.withWorkspaceRoot(targetRoot, targetIdentity, operation); + await rooted.withWorkspaceRoot(sourceRoot, sourceIdentity, async () => { + const visit = async ( + path: string, + out: string, + checkoutPath: string, + ): Promise => { + signal?.throwIfAborted(); + if (++files > config.maxFiles) + throw new Error('Dependency snapshot exceeds maxFiles'); + const before = await rooted.lstat(path); + if (before.isDirectory()) { + await target(() => rooted.mkdir(out)); + for (const name of await rooted.readdir( + path, + config.maxFiles - files, + )) + await visit( + join(path, name), + join(out, name), + join(checkoutPath, name), + ); + const after = await rooted.lstat(path); + if ( + before.ino !== after.ino || + before.dev !== after.dev || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) + throw new Error( + 'Dependencies changed during snapshot traversal', + ); + } else if (before.isSymbolicLink()) { + const link = await rooted.readlink(path); + const resolved = resolve(dirname(checkoutPath), link); + if ( + isAbsolute(link) || + !inside(checkout, resolved) || + relative(checkout, resolved).split(sep).includes('.git') || + relative(checkout, resolved) + .split(sep) + .includes('.worktrees') + ) + throw new Error( + 'Dependency snapshot link must remain checkout-local', + ); + await target(() => rooted.symlink(link, out)); + } else if (before.isFile()) { + if (before.nlink !== 1) + throw new Error( + 'Dependency snapshots reject hard-linked files', + ); + bytes += Number(before.size); + if (bytes > config.maxBytes) + throw new Error('Dependency snapshot exceeds maxBytes'); + const handle = await rooted.open( + path, + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK, + ); + try { + const opened = await handle.stat(); + if ( + !opened.isFile() || + opened.nlink !== 1 || + opened.ino !== before.ino || + opened.dev !== before.dev + ) + throw new Error( + 'Dependency entry changed before cloning', + ); + await target(async () => { + await cloneFile(handle, out); + const output = await rooted.open( + out, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + await output.chmod(opened.mode & 0o777); + } finally { + await output.close(); + } + }); + const after = await handle.stat(); + if ( + opened.size !== after.size || + after.nlink !== 1 || + opened.mtimeMs !== after.mtimeMs || + opened.ctimeMs !== after.ctimeMs + ) + throw new Error( + 'Dependency file changed during cloning', + ); + } finally { + await handle.close(); + } + } else + throw new Error( + 'Dependency snapshots accept regular files, directories and relative links only', + ); + }; + for (let i = 0; i < paths.length; i++) + await visit( + join(sourceRoot, paths[i]), + join(targetRoot, String(i)), + join(checkout, config.paths[i]), + ); + }); +} + +export async function withDependencySnapshot( + store: DependencySnapshotStore, + key: string, + operation: () => Promise, + signal?: AbortSignal, + verifyCloneSupport = false, +): Promise { + if (!/^[a-f0-9]{64}$/.test(key)) + throw new Error('Invalid dependency snapshot key'); + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error('Dependency snapshot store changed after admission'); + return withProcessLock( + join(store.store, `${key}.lock`), + async () => { + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error( + 'Dependency snapshot store changed while waiting', + ); + signal?.throwIfAborted(); + // Fail before installation, rather than discovering unsupported reflinks after npm ci. + if (verifyCloneSupport) { + const probe = await fs.mkdtemp(join(store.store, '.probe-')); + try { + await fs.writeFile(join(probe, 'source'), 'clone-support', { + mode: 0o600, + }); + const input = await fs.open( + join(probe, 'source'), + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + await rooted.withWorkspaceRoot( + store.store, + store.identity, + () => cloneFile(input, join(probe, 'target')), + ); + } finally { + await input.close(); + } + } finally { + await fs.rm(probe, { recursive: true, force: true }); + } + } + signal?.throwIfAborted(); + return operation(); + }, + signal, + ); +} +/** Restores only when every declared installation is absent. Existing files are never replaced. */ +export async function restoreDependencySnapshot( + store: DependencySnapshotStore, + key: string, + checkout: string, + identity: WorkspaceRootIdentity, + signal?: AbortSignal, +): Promise { + return rooted.withWorkspaceRoot(checkout, identity, async () => { + for (const path of store.paths) { + const exists = await rooted.lstat(join(checkout, path)).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ); + if (exists) return false; + } + const source = join(store.store, key); + const metadata = await fs + .lstat(source) + .catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (!metadata) return false; + if (!metadata.isDirectory() || metadata.isSymbolicLink()) + throw new Error('Invalid dependency snapshot entry'); + const staging = join( + checkout, + `.librechat-dependencies-${randomUUID()}`, + ); + await rooted.mkdir(staging); + const installed: string[] = []; + try { + const stagingIdentity = await captureWorkspaceRootIdentity(staging); + await cloneTree( + source, + await captureWorkspaceRootIdentity(source), + staging, + stagingIdentity, + store.paths.map((_, i) => String(i)), + store, + checkout, + signal, + ); + for (let i = 0; i < store.paths.length; i++) { + signal?.throwIfAborted(); + const destination = join(checkout, store.paths[i]); + // rename is descriptor-anchored; reject a newly appeared entry instead of replacing it. + if ( + await rooted.lstat(destination).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ) + ) + throw new Error( + 'Dependency destination appeared during restore', + ); + await rooted.renameExclusive( + join(staging, String(i)), + destination, + ); + installed.push(destination); + } + return true; + } catch (error) { + // A partial restore is inspection-worthy; do not delete a destination another process may have changed. + if (installed.length) + throw new Error( + 'Dependency restore partially committed; inspect the checkout', + { cause: error }, + ); + throw error; + } finally { + await removeTree(staging); + } + }); +} +export async function publishDependencySnapshot( + store: DependencySnapshotStore, + key: string, + checkout: string, + identity: WorkspaceRootIdentity, + signal?: AbortSignal, +): Promise { + const destination = join(store.store, key); + if ( + await fs.lstat(destination).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ) + ) + return; + const staging = await fs.mkdtemp(join(store.store, '.staging-')); + try { + await cloneTree( + checkout, + identity, + staging, + await captureWorkspaceRootIdentity(staging), + store.paths, + store, + checkout, + signal, + ); + signal?.throwIfAborted(); + await fs.rename(staging, destination); + } finally { + await fs.rm(staging, { recursive: true, force: true }); + } +} diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts index c2597f3b..73df85f3 100644 --- a/packages/code/src/environment-preparation.ts +++ b/packages/code/src/environment-preparation.ts @@ -8,6 +8,12 @@ import { } from './storage.js'; import type { CodeEnvironmentDefinition } from './environment.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + withDependencySnapshot, + restoreDependencySnapshot, + publishDependencySnapshot, +} from './dependency-snapshots.js'; +import type { DependencySnapshotStore } from './dependency-snapshots.js'; // Safety bounds on operator-declared hashing, not a dependency-store quota. const MAX_INPUT_BYTES = 8 * 1024 * 1024; @@ -25,12 +31,40 @@ export interface EnvironmentPreparationOptions { timeoutMs: number, ): Promise<{ exitCode: number | null; timedOut: boolean }>; signal?: AbortSignal; + snapshotStore?: DependencySnapshotStore; + snapshotScope?: string; + beforeMutation?(): Promise; } /** Checkout-local reuse. Never transfers mutable installations between worktrees. */ export async function prepareCodeEnvironment( options: EnvironmentPreparationOptions, -): Promise<'prepared' | 'reused'> { +): Promise<'prepared' | 'reused' | 'restored'> { + if (options.snapshotStore) { + if (!options.setup.reuse?.snapshot || !options.snapshotScope) + throw new Error( + 'Dependency snapshot requires an explicit preparation scope', + ); + if (options.snapshotStore.identity.dev !== options.identity.dev) + throw new Error( + 'Dependency snapshots and checkouts must use the same clone-capable filesystem', + ); + const portable = await preparationKey(options, true); + return withDependencySnapshot( + options.snapshotStore, + portable!, + () => prepareInLock(options, portable), + options.signal, + true, + ); + } + return prepareInLock(options); +} + +async function prepareInLock( + options: EnvironmentPreparationOptions, + portable?: string, +): Promise<'prepared' | 'reused' | 'restored'> { options.signal?.throwIfAborted(); const key = await preparationKey(options); if ( @@ -48,6 +82,36 @@ export async function prepareCodeEnvironment( if (check.exitCode === 0 && (await preparationKey(options)) === key) return 'reused'; } + if (options.snapshotStore && portable) { + await options.beforeMutation?.(); + if ( + await restoreDependencySnapshot( + options.snapshotStore, + portable, + options.root, + options.identity, + options.signal, + ) + ) { + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode === null) + throw new Error( + 'Restored dependency readiness check did not settle', + ); + if ( + check.exitCode === 0 && + (await preparationKey(options)) === key + ) { + await saveEnvironmentPreparationKey(options.receiptPath, key!); + return 'restored'; + } + } + } const result = await options.execute( options.setup.command, options.setup.timeoutMs, @@ -74,19 +138,28 @@ export async function prepareCodeEnvironment( 'Environment preparation inputs changed during readiness check', ); await saveEnvironmentPreparationKey(options.receiptPath, key); + if (options.snapshotStore && portable) + await publishDependencySnapshot( + options.snapshotStore, + portable, + options.root, + options.identity, + options.signal, + ); } return 'prepared'; } async function preparationKey( options: EnvironmentPreparationOptions, + portable = false, ): Promise { if (!options.setup.reuse) return undefined; return withWorkspaceRoot(options.root, options.identity, async () => { const hash = createHash('sha256').update( JSON.stringify({ version: 1, - root: options.identity, + root: portable ? options.snapshotScope : options.identity, setup: options.setup, context: options.context, node: process.version, diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 976ffe6d..ca859a2e 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -4,6 +4,8 @@ import { open, realpath, stat } from 'node:fs/promises'; import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import { parseDocument } from 'yaml'; import { parseEnvironmentResources, loadEnvironmentResource } from './environment-resources.js'; +import { parseDependencySnapshot, loadDependencySnapshot } from './dependency-snapshots.js'; +import type { DependencySnapshotConfig, DependencySnapshotStore } from './dependency-snapshots.js'; import type { EnvironmentResource, LoadedEnvironmentResource } from './environment-resources.js'; import { assertPrivateStorageAcl, @@ -32,7 +34,7 @@ export interface CodeEnvironmentDefinition { command: string; timeoutMs: number; /** Explicit readiness contract; absent preserves startup setup behavior. */ - reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number }; + reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number; snapshot?: DependencySnapshotConfig }; }; actions?: { name: string; command: string; timeoutMs: number }[]; resources?: EnvironmentResource[]; @@ -45,6 +47,7 @@ export interface LoadedCodeEnvironment { definition: CodeEnvironmentDefinition; fingerprint: string; resources?: LoadedEnvironmentResource[]; + snapshotStore?: DependencySnapshotStore; } function record(value: unknown): value is Record { @@ -124,7 +127,7 @@ export function parseCodeEnvironment( const candidate = value.setup.reuse; if ( !record(candidate) || - Object.keys(candidate).some(key => !['inputs', 'checkCommand', 'checkTimeoutMs'].includes(key)) || + Object.keys(candidate).some(key => !['inputs', 'checkCommand', 'checkTimeoutMs', 'snapshot'].includes(key)) || !Array.isArray(candidate.inputs) || candidate.inputs.length < 1 || candidate.inputs.length > 32 || candidate.inputs.some(path => typeof path !== 'string' || !isSafePortableRelativePath(path) || path === '.') || @@ -135,7 +138,8 @@ export function parseCodeEnvironment( const checkTimeoutMs = candidate.checkTimeoutMs ?? 10_000; if (typeof checkTimeoutMs !== 'number' || !Number.isSafeInteger(checkTimeoutMs) || checkTimeoutMs < 1 || checkTimeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS) throw new Error('Invalid environment readiness check timeout'); - reuse = { inputs: candidate.inputs as string[], checkCommand: candidate.checkCommand, checkTimeoutMs }; + reuse = { inputs: candidate.inputs as string[], checkCommand: candidate.checkCommand, checkTimeoutMs, + ...(candidate.snapshot !== undefined ? { snapshot: parseDependencySnapshot(candidate.snapshot) } : {}) }; } setup = { command: value.setup.command, timeoutMs, ...(reuse ? { reuse } : {}) }; } @@ -334,6 +338,7 @@ export async function loadCodeEnvironment( rootPaths, definition, ...(definition.resources ? { resources: await Promise.all(definition.resources.map(loadEnvironmentResource)) } : {}), + ...(definition.setup?.reuse?.snapshot ? { snapshotStore: await loadDependencySnapshot(definition.setup.reuse.snapshot) } : {}), fingerprint: createHash('sha256') .update(JSON.stringify(definition)) .digest('hex'), diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts index 4aeb6100..b06de8d0 100644 --- a/packages/code/src/root-access.ts +++ b/packages/code/src/root-access.ts @@ -62,10 +62,16 @@ const openAt = nativeOpenAt const renameAt = bind( 'int renameat(int fromfd, const char *from, int tofd, const char *to)', ); +const renameExclusiveAt = bind(process.platform === 'darwin' + ? 'int renameatx_np(int fromfd, const char *from, int tofd, const char *to, unsigned int flags)' + : 'int renameat2(int fromfd, const char *from, int tofd, const char *to, unsigned int flags)'); const linkAt = bind( 'int linkat(int fromfd, const char *from, int tofd, const char *to, int flags)', ); const unlinkAt = bind('int unlinkat(int dirfd, const char *path, int flags)'); +const mkdirAt = bind('int mkdirat(int dirfd, const char *path, unsigned int mode)'); +const symlinkAt = bind('int symlinkat(const char *target, int dirfd, const char *path)'); +const readlinkAt = bind('int readlinkat(int dirfd, const char *path, void *buffer, size_t size)'); const getPath = process.platform === 'darwin' ? bind('int fcntl(int fd, int command, ...)') @@ -328,13 +334,14 @@ export class WorkspaceRootAccess { } } - install(from: string, to: string, link: boolean): void { + install(from: string, to: string, link: boolean, exclusive = false): void { const source = this.parent(from); let target: ReturnType | undefined; try { target = this.parent(to); const result = link ? linkAt!(source.fd, source.name, target.fd, target.name, 0) + : exclusive ? renameExclusiveAt!(source.fd, source.name, target.fd, target.name, process.platform === 'darwin' ? 4 : 1) : renameAt!(source.fd, source.name, target.fd, target.name); if (result !== 0) throw nativeError(); } finally { @@ -343,10 +350,27 @@ export class WorkspaceRootAccess { } } - unlink(path: string): void { + entry(path: string, operation: 'mkdir' | 'symlink' | 'readlink', target?: string): string | void { const parent = this.parent(path); try { - if (unlinkAt!(parent.fd, parent.name, 0) !== 0) throw nativeError(); + if (operation === 'readlink') { + const buffer = Buffer.alloc(4097); + const length = readlinkAt!(parent.fd, parent.name, buffer, buffer.length); + if (length < 0) throw nativeError(); + if (length === buffer.length) throw new Error('Workspace link exceeds its bounded contract'); + return buffer.subarray(0, length).toString(); + } + const result = operation === 'mkdir' + ? mkdirAt!(parent.fd, parent.name, 0o700) + : symlinkAt!(target!, parent.fd, parent.name); + if (result !== 0) throw nativeError(); + } finally { closeSync(parent.fd); } + } + + unlink(path: string, directory = false): void { + const parent = this.parent(path); + try { + if (unlinkAt!(parent.fd, parent.name, directory ? (process.platform === 'darwin' ? 0x80 : 0x200) : 0) !== 0) throw nativeError(); } finally { closeSync(parent.fd); } @@ -448,6 +472,11 @@ export const rename = async (from: string, to: string): Promise => { if (access) access.install(from, to, false); else await fs.rename(from, to); }; +export const renameExclusive = async (from: string, to: string): Promise => { + const access = context.getStore(); + if (!access) throw new Error('Exclusive installation requires a selected root'); + access.install(from, to, false, true); +}; export const link = async (from: string, to: string): Promise => { const access = context.getStore(); if (access) access.install(from, to, true); @@ -458,3 +487,35 @@ export const unlink = async (path: string): Promise => { if (access) access.unlink(path); else await fs.unlink(path); }; + +/** Enumeration and entry mutation retain the selected root descriptor. */ +export const readdir = async (path: string, maxEntries = 200_000): Promise => { + const access = context.getStore(); + const handle = access ? await access.openFile(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW) : undefined; + try { + const entries: string[] = []; + const directory = await fs.opendir(handle ? descriptorPath(handle.fd) : path); + for await (const entry of directory) { + if (entries.length >= maxEntries) throw new Error('Dependency directory exceeds maxFiles'); + entries.push(entry.name); + } + return entries; + } finally { await handle?.close(); } +}; +export const readlink = async (path: string): Promise => + context.getStore()?.entry(path, 'readlink') as string ?? fs.readlink(path); +export const mkdir = async (path: string): Promise => { + const access = context.getStore(); + if (access) access.entry(path, 'mkdir'); + else await fs.mkdir(path, { mode: 0o700 }); +}; +export const symlink = async (target: string, path: string): Promise => { + const access = context.getStore(); + if (access) access.entry(path, 'symlink', target); + else await fs.symlink(target, path); +}; +export const rmdir = async (path: string): Promise => { + const access = context.getStore(); + if (access) access.unlink(path, true); + else await fs.rmdir(path); +}; From 572ccbacf88de56f5257184942e17cd6d9e1b97c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 07:47:50 -0400 Subject: [PATCH 59/75] feat: Bound Managed Environment Storage (#289) --- docs/remote-bridge/worker-runbook.md | 16 +- packages/code/README.md | 66 +++- packages/code/src/cli.ts | 25 ++ .../code/src/dependency-snapshots.test.ts | 17 + packages/code/src/dependency-snapshots.ts | 93 +++++- packages/code/src/environment-preparation.ts | 7 + packages/code/src/environment.ts | 6 +- packages/code/src/process-lock.ts | 11 +- packages/code/src/snapshot-lifecycle.test.ts | 226 +++++++++++++ packages/code/src/snapshot-lifecycle.ts | 316 ++++++++++++++++++ 10 files changed, 773 insertions(+), 10 deletions(-) create mode 100644 packages/code/src/snapshot-lifecycle.test.ts create mode 100644 packages/code/src/snapshot-lifecycle.ts diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 8cbeda98..51018940 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -533,7 +533,21 @@ command, cancellation, or settlement whose effects may be incomplete. - **Worker online but not ready:** check native sandbox preparation, definition validation, setup, quarantine, and readiness logs. - **Setup repeats on restart:** setup is intentionally per-start; make it - idempotent or remove it. + idempotent, remove it, or opt in to the bounded `setup.reuse` readiness + contract in the [worker package guide](../../packages/code/README.md#reusing-a-prepared-checkout). +- **Dependency copies fill the disk:** declare shared npm/uv/browser resources + instead of per-worktree downloads. On a verified clone-capable filesystem, + configure private copy-on-write snapshots and their lifecycle budget. Do not + symlink another branch's mutable `node_modules` or hardlink writable installs. +- **Managed preparation deferred for low space:** `storage.minFreeBytes` plus + `setupReserveBytes` is a soft pre-setup floor, not a hard quota. Expand the + volume or clean reproducible artifacts; do not clear quarantine as a disk fix. +- **Snapshot maintenance:** preview with `prune-environment-storage + --environment ` and use `--apply` only after reviewing its JSON. Active, + unknown and unmarked data stays intact. Include all environment definitions + for root-isolation checks. This does not archive source worktrees or prune + mutable tool caches. Keep control state on separate storage when hard + protection from arbitrary build writes is required. - **Git works on the host but not in tools:** verify the App installation, permissions, private-key mode/owner, and allowed GitHub domains. - **Repository label is present but files are absent:** `repo`/`ref` are diff --git a/packages/code/README.md b/packages/code/README.md index a861efd3..38a22d5d 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -1187,10 +1187,74 @@ Sharing is explicit within one worker's trust domain. Do not share mutable cache between unrelated principals, store credentials in them, or treat their contents as trusted worker code. Separate package caches from browsers and mutable browser profiles, Redis data, build output and test state. Stores currently have no automatic -eviction; the storage-lifecycle slice adds that separately. This shares downloads +eviction. The snapshot lifecycle below does not delete these mutable stores. This shares downloads and browser binaries, not installed `node_modules` trees. No setup output is sent to the model. +## Storage admission and snapshot lifecycle + +Use an explicit free-space floor before managed setup or restoration: + +```yaml +storage: + minFreeBytes: 5368709120 # 5 GiB left for the worker and other activity + setupReserveBytes: 2147483648 # estimated installation headroom +``` + +This works without dependency snapshots. Absent `storage`, preparation retains +today's behavior. A ready checkout can still pass its readiness check when disk +is low. When installation is needed, low space rejects it **before** starting the +setup command, with a specific remediation message. These are soft admission +checks, not allocated reservations: concurrent processes and arbitrary shell +writes can still consume space after admission. Put worker identity/quarantine +state on a separate small volume and use filesystem/project quotas for hard +containment. This option neither clears quarantine nor deletes source to recover. + +The private snapshot store can bound reproducible dependency versions: + +```yaml +setup: + # command and reuse inputs/readiness omitted here; retain the full contract above + reuse: + snapshot: + store: /srv/lia-state/dependency-snapshots + paths: [node_modules] + lifecycle: + maxStoreBytes: 21474836480 + maxEntries: 8 + retentionMs: 432000000 + scanLimit: 4096 +``` + +Cleanup runs during managed preparation and publication. Oldest inactive snapshots +are reclaimed by last-use age and logical payload/count budgets. Per-key kernel +locks protect installs, restores and publishers; maintenance never waits on or +deletes an active key. A short store-wide lock makes budget checks and publication +atomic, without serializing installations for different keys. Abandoned staging +directories require a worker ownership manifest and a free key lock before removal. +Unknown/unmarked entries are reported and preserved. Lock files are kept to avoid +splitting lock ownership. Scan limits fail closed instead of crawling unbounded data. + +The byte budget counts snapshot file lengths, not deduplicated physical blocks, +metadata overhead or working checkout copies. An otherwise valid prepared checkout +does not fail when the cache cannot fit another entry: publication is skipped with +a worker diagnostic, and later low-space admission still applies. + +Preview or explicitly apply maintenance without restarting the worker: + +```sh +librechat-code prune-environment-storage --environment /etc/librechat-code/app.yaml +librechat-code prune-environment-storage --environment /etc/librechat-code/app.yaml --apply +``` + +Pass all relevant definitions with repeated `--environment` flags so their roots +participate in isolation checks. Preview is the default and the JSON identifies +`dryRun`, proposed removals, active keys, unknown data and retained logical usage. +The command is host-operator-only, not an agent workspace action. It does not touch +source worktrees (including dirty or unpushed branches), `.verification`, build +outputs, arbitrary installations or mutable npm/uv caches. Source worktree archival +requires the separate worktree ownership/binding lifecycle, not an mtime heuristic. + Named actions are fixed commands without model-supplied substitution. The bridge advertises only their names and the definition fingerprint, never their shell source or host root. A command request can select `environmentAction: { name, fingerprint }`; diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 4c64c0ae..cf05e263 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -14,6 +14,7 @@ import { } from './environment.js'; import { prepareCodeEnvironment } from './environment-preparation.js'; import { assertEnvironmentResourceIsolation } from './environment-resources.js'; +import { pruneDependencySnapshots } from './snapshot-lifecycle.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -1191,6 +1192,7 @@ async function run( }); await prepareCodeEnvironment({ snapshotStore: environment!.snapshotStore, + storage: environment!.definition.storage, snapshotScope: environment!.definition.repo ?? environment!.definition.name, root: instance.root, identity: instance.identity, setup, receiptPath: preparationReceipt(instance.root), @@ -1365,6 +1367,7 @@ async function run( let armed = false; const preparation = await prepareCodeEnvironment({ snapshotStore: environment.snapshotStore, + storage: environment.definition.storage, snapshotScope: environment.definition.repo ?? environment.definition.name, beforeMutation: async () => { if (!armed) { await guard.arm('Dependency restoration did not settle', 'setup'); armed = true; } @@ -1687,6 +1690,28 @@ async function clearMutationQuarantine(args: string[]): Promise { async function main(): Promise { const args = process.argv.slice(2); + if (args[0] === 'prune-environment-storage') { + const paths: string[] = []; + for (let i = 1; i < args.length; i++) { + if (args[i] === '--apply') continue; + if (args[i] !== '--environment' || !args[i + 1] || args[i + 1].startsWith('--')) + throw new Error('Usage: librechat-code prune-environment-storage --environment [--environment ...] [--apply]'); + paths.push(args[++i]); + } + if (!paths.length || paths.length > 32) throw new Error('Supply between one and 32 environment definitions'); + const loaded = await Promise.all(paths.map(loadCodeEnvironment)); + const roots = loaded.map(environment => ({ id: environment.definition.name, root: environment.definition.root })); + const stores = loaded.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); + if (!stores.length) throw new Error('No dependency snapshot stores configured'); + await assertEnvironmentDefinitionsOutsideRoots(loaded, roots); + await assertEnvironmentDefinitionsOutsideRoots(stores.map(store => ({ path: store.store, sourceParents: store.controlPaths, + definition: { name: 'dependency-store', root: store.store }, fingerprint: '' })), roots); + await assertEnvironmentResourceIsolation(stores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), + roots.map(root => root.root), [...loaded.map(environment => environment.path), + ...loaded.flatMap(environment => (environment.resources ?? []).map(resource => resource.path))]); + for (const store of stores) process.stdout.write(`${JSON.stringify(await pruneDependencySnapshots(store, { dryRun: !args.includes('--apply') }))}\n`); + return; + } if (args[0] === 'projects') { const root = option(args, '--root'); if (!root || args.slice(1).some((arg, index, rest) => diff --git a/packages/code/src/dependency-snapshots.test.ts b/packages/code/src/dependency-snapshots.test.ts index b23ae70f..f7d59681 100644 --- a/packages/code/src/dependency-snapshots.test.ts +++ b/packages/code/src/dependency-snapshots.test.ts @@ -188,6 +188,12 @@ test( }, async t => { const { a, b, store, directory, ai, bi } = await fixture(t); + store.lifecycle = { + maxStoreBytes: 10000, + maxEntries: 1, + retentionMs: 5 * 24 * 60 * 60 * 1000, + scanLimit: 100, + }; for (const root of [a, b]) await writeFile(join(root, 'package-lock.json'), 'version-one'); const commands: string[] = []; @@ -204,6 +210,7 @@ test( paths: store.paths, maxBytes: store.maxBytes, maxFiles: store.maxFiles, + lifecycle: store.lifecycle, }, }, }; @@ -255,6 +262,16 @@ test( commands.filter(command => command === setup.command).length, 2, ); + assert.equal( + (await readdir(store.store)).filter(name => + /^[a-f0-9]{64}$/.test(name), + ).length, + 1, + ); + assert.equal( + await readFile(join(a, 'node_modules', 'package.js'), 'utf8'), + 'module.exports = 42', + ); const denied = await sandboxes[0].execute({ protocolVersion: 1, operation: 'execute_command', diff --git a/packages/code/src/dependency-snapshots.ts b/packages/code/src/dependency-snapshots.ts index 5c9ab173..89a76935 100644 --- a/packages/code/src/dependency-snapshots.ts +++ b/packages/code/src/dependency-snapshots.ts @@ -20,12 +20,21 @@ import { import { loadEnvironmentResource } from './environment-resources.js'; import { isSafePortableRelativePath } from './protocol.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + parseSnapshotLifecycle, + pruneDependencySnapshots, + SNAPSHOT_MANIFEST, + STAGING_MANIFEST, + SnapshotBudgetFullError, +} from './snapshot-lifecycle.js'; +import type { SnapshotLifecyclePolicy } from './snapshot-lifecycle.js'; export interface DependencySnapshotConfig { store: string; paths: string[]; maxBytes: number; maxFiles: number; + lifecycle?: SnapshotLifecyclePolicy; } export interface DependencySnapshotStore extends DependencySnapshotConfig { identity: WorkspaceRootIdentity; @@ -41,7 +50,14 @@ export function parseDependencySnapshot( const maxFiles = v.maxFiles ?? 200_000; if ( Object.keys(v).some( - k => !['store', 'paths', 'maxBytes', 'maxFiles'].includes(k), + k => + ![ + 'store', + 'paths', + 'maxBytes', + 'maxFiles', + 'lifecycle', + ].includes(k), ) || typeof v.store !== 'string' || !isAbsolute(v.store) || @@ -74,6 +90,9 @@ export function parseDependencySnapshot( paths: v.paths as string[], maxBytes: maxBytes as number, maxFiles: maxFiles as number, + ...(v.lifecycle !== undefined + ? { lifecycle: parseSnapshotLifecycle(v.lifecycle) } + : {}), }; } export async function loadDependencySnapshot( @@ -172,7 +191,7 @@ async function cloneTree( config: DependencySnapshotConfig, checkout: string, signal?: AbortSignal, -): Promise { +): Promise<{ bytes: number; files: number }> { let bytes = 0, files = 0; const target = (operation: () => Promise) => @@ -285,6 +304,7 @@ async function cloneTree( join(checkout, config.paths[i]), ); }); + return { bytes, files }; } export async function withDependencySnapshot( @@ -306,6 +326,14 @@ export async function withDependencySnapshot( 'Dependency snapshot store changed while waiting', ); signal?.throwIfAborted(); + if (store.lifecycle) + await pruneDependencySnapshots(store, { + currentKey: key, + signal, + }).catch(error => { + if (!(error instanceof SnapshotBudgetFullError)) + throw error; + }); // Fail before installation, rather than discovering unsupported reflinks after npm ci. if (verifyCloneSupport) { const probe = await fs.mkdtemp(join(store.store, '.probe-')); @@ -331,7 +359,25 @@ export async function withDependencySnapshot( } } signal?.throwIfAborted(); - return operation(); + const result = await operation(); + const manifest = await fs + .open( + join(store.store, key, SNAPSHOT_MANIFEST), + constants.O_RDONLY | constants.O_NOFOLLOW, + ) + .catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (manifest) { + try { + const now = new Date(); + await manifest.utimes(now, now); + } finally { + await manifest.close(); + } + } + return result; }, signal, ); @@ -437,9 +483,20 @@ export async function publishDependencySnapshot( ) ) return; - const staging = await fs.mkdtemp(join(store.store, '.staging-')); + const staging = await fs.mkdtemp(join(store.store, `.staging-${key}-`)); try { - await cloneTree( + await fs.writeFile( + join(staging, STAGING_MANIFEST), + JSON.stringify({ + version: 1, + key, + bytes: 0, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const measured = await cloneTree( checkout, identity, staging, @@ -450,7 +507,31 @@ export async function publishDependencySnapshot( signal, ); signal?.throwIfAborted(); - await fs.rename(staging, destination); + await fs.writeFile( + join(staging, SNAPSHOT_MANIFEST), + JSON.stringify({ + version: 1, + key, + ...measured, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const publish = () => fs.rename(staging, destination); + if (store.lifecycle) + await pruneDependencySnapshots(store, { + currentKey: key, + incomingBytes: measured.bytes, + incomingEntries: 1, + signal, + publish, + }).catch(error => { + if (!(error instanceof SnapshotBudgetFullError)) throw error; + process.stderr.write( + 'librechat-code: dependency snapshot not cached because the store budget is full; prepared checkout remains valid\n', + ); + }); + else await publish(); } finally { await fs.rm(staging, { recursive: true, force: true }); } diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts index 73df85f3..8dbb8751 100644 --- a/packages/code/src/environment-preparation.ts +++ b/packages/code/src/environment-preparation.ts @@ -14,6 +14,8 @@ import { publishDependencySnapshot, } from './dependency-snapshots.js'; import type { DependencySnapshotStore } from './dependency-snapshots.js'; +import { assertPreparationSpace } from './snapshot-lifecycle.js'; +import type { EnvironmentStoragePolicy } from './snapshot-lifecycle.js'; // Safety bounds on operator-declared hashing, not a dependency-store quota. const MAX_INPUT_BYTES = 8 * 1024 * 1024; @@ -34,6 +36,7 @@ export interface EnvironmentPreparationOptions { snapshotStore?: DependencySnapshotStore; snapshotScope?: string; beforeMutation?(): Promise; + storage?: EnvironmentStoragePolicy; } /** Checkout-local reuse. Never transfers mutable installations between worktrees. */ @@ -83,6 +86,8 @@ async function prepareInLock( return 'reused'; } if (options.snapshotStore && portable) { + if (options.storage) + await assertPreparationSpace(options.root, options.storage); await options.beforeMutation?.(); if ( await restoreDependencySnapshot( @@ -112,6 +117,8 @@ async function prepareInLock( } } } + if (options.storage) + await assertPreparationSpace(options.root, options.storage); const result = await options.execute( options.setup.command, options.setup.timeoutMs, diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index ca859a2e..576a149f 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -6,6 +6,8 @@ import { parseDocument } from 'yaml'; import { parseEnvironmentResources, loadEnvironmentResource } from './environment-resources.js'; import { parseDependencySnapshot, loadDependencySnapshot } from './dependency-snapshots.js'; import type { DependencySnapshotConfig, DependencySnapshotStore } from './dependency-snapshots.js'; +import { parseEnvironmentStorage } from './snapshot-lifecycle.js'; +import type { EnvironmentStoragePolicy } from './snapshot-lifecycle.js'; import type { EnvironmentResource, LoadedEnvironmentResource } from './environment-resources.js'; import { assertPrivateStorageAcl, @@ -38,6 +40,7 @@ export interface CodeEnvironmentDefinition { }; actions?: { name: string; command: string; timeoutMs: number }[]; resources?: EnvironmentResource[]; + storage?: EnvironmentStoragePolicy; } export interface LoadedCodeEnvironment { @@ -80,7 +83,7 @@ export function parseCodeEnvironment( !record(value) || Object.keys(value).some( key => - !['name', 'root', 'repo', 'ref', 'setup', 'actions', 'resources'].includes( + !['name', 'root', 'repo', 'ref', 'setup', 'actions', 'resources', 'storage'].includes( key, ), ) || @@ -182,6 +185,7 @@ export function parseCodeEnvironment( ...(setup ? { setup } : {}), ...(actions ? { actions } : {}), ...(value.resources !== undefined ? { resources: parseEnvironmentResources(value.resources) } : {}), + ...(value.storage !== undefined ? { storage: parseEnvironmentStorage(value.storage) } : {}), }; } diff --git a/packages/code/src/process-lock.ts b/packages/code/src/process-lock.ts index aeca561a..6e456d50 100644 --- a/packages/code/src/process-lock.ts +++ b/packages/code/src/process-lock.ts @@ -5,6 +5,7 @@ import { setTimeout as delay } from 'node:timers/promises'; const LOCK_EX = 2; const LOCK_NB = 4; const LOCK_UN = 8; +export class ProcessLockBusyError extends Error {} let binding: | Promise<{ flock: (fd: number, operation: number) => number; @@ -26,7 +27,8 @@ async function lockBinding() { export async function withProcessLock( path: string, operation: () => Promise, - signal?: AbortSignal + signal?: AbortSignal, + wait = true, ): Promise { signal?.throwIfAborted(); if (process.platform !== 'darwin' && process.platform !== 'linux') { @@ -48,6 +50,7 @@ export async function withProcessLock( `Conversation worktree lock failed with errno ${errno}` ); } + if (!wait) throw new ProcessLockBusyError('Process lock is active'); await delay(50, undefined, { signal }); } signal?.throwIfAborted(); @@ -57,3 +60,9 @@ export async function withProcessLock( await handle.close(); } } + +/** Non-blocking maintenance admission. Never masks operation errors as lock contention. */ +export async function tryWithProcessLock(path: string, operation: () => Promise, signal?: AbortSignal): Promise<{ acquired: true; value: T } | { acquired: false }> { + try { return { acquired: true, value: await withProcessLock(path, operation, signal, false) }; } + catch (error) { if (error instanceof ProcessLockBusyError) return { acquired: false }; throw error; } +} diff --git a/packages/code/src/snapshot-lifecycle.test.ts b/packages/code/src/snapshot-lifecycle.test.ts new file mode 100644 index 00000000..33e37c2d --- /dev/null +++ b/packages/code/src/snapshot-lifecycle.test.ts @@ -0,0 +1,226 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + mkdtemp, + mkdir, + realpath, + writeFile, + readFile, + rm, + lstat, + utimes, + symlink, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { + loadDependencySnapshot, + parseDependencySnapshot, +} from './dependency-snapshots.js'; +import { + assertPreparationSpace, + parseEnvironmentStorage, + parseSnapshotLifecycle, + pruneDependencySnapshots, + SNAPSHOT_MANIFEST, + STAGING_MANIFEST, +} from './snapshot-lifecycle.js'; +import { withProcessLock } from './process-lock.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; + +async function fixture(t: test.TestContext) { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'snapshot-lifecycle-')), + ); + t.after(() => rm(root, { recursive: true, force: true })); + const path = join(root, 'store'); + await mkdir(path, { mode: 0o700 }); + const store = await loadDependencySnapshot( + parseDependencySnapshot({ + store: path, + paths: ['node_modules'], + lifecycle: { maxStoreBytes: 10, maxEntries: 2, retentionMs: 1000 }, + }), + ); + const add = async (key: string, bytes = 4, age = 0) => { + const directory = join(store.store, key); + await mkdir(directory, { mode: 0o700 }); + await writeFile( + join(directory, SNAPSHOT_MANIFEST), + JSON.stringify({ + version: 1, + key, + bytes, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const lastUsed = new Date(Date.now() - age); + await utimes(join(directory, SNAPSHOT_MANIFEST), lastUsed, lastUsed); + return directory; + }; + return { + root, + store, + add, + a: 'a'.repeat(64), + b: 'b'.repeat(64), + c: 'c'.repeat(64), + }; +} + +test('storage policy is opt-in, bounded and fails before a setup command', async t => { + assert.equal(parseEnvironmentStorage({}).minFreeBytes, 5 * 1024 ** 3); + assert.throws(() => parseEnvironmentStorage({ minFreeBytes: -1 })); + assert.throws(() => parseSnapshotLifecycle({ retentionMs: Infinity })); + await assertPreparationSpace( + '/', + { minFreeBytes: 5, setupReserveBytes: 2 }, + async () => 7n, + ); + await assert.rejects( + assertPreparationSpace( + '/', + { minFreeBytes: 5, setupReserveBytes: 2 }, + async () => 6n, + ), + /no setup command was started/, + ); + const { root } = await fixture(t); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + const identity = await captureWorkspaceRootIdentity(checkout); + let executions = 0; + await assert.rejects( + prepareCodeEnvironment({ + root: checkout, + identity, + setup: { command: 'install', timeoutMs: 1000 }, + receiptPath: join(root, 'receipt'), + context: '', + storage: { + minFreeBytes: Number.MAX_SAFE_INTEGER, + setupReserveBytes: 0, + }, + execute: async () => { + executions++; + return { exitCode: 0, timedOut: false }; + }, + }), + /deferred/, + ); + assert.equal(executions, 0); +}); + +test('dry-run is non-destructive; cleanup removes only expired owned entries and retains unknown or linked data', async t => { + const { store, add, a, b, c, root } = await fixture(t); + const old = await add(a, 4, 10_000); + await add(b); + await mkdir(join(store.store, 'unrecognized')); + await writeFile(join(store.store, 'unrecognized', 'source'), 'keep'); + await symlink(root, join(store.store, c)); + const preview = await pruneDependencySnapshots(store, { dryRun: true }); + assert.deepEqual(preview.removed, [a]); + assert.equal((await lstat(old)).isDirectory(), true); + const actual = await pruneDependencySnapshots(store); + assert.deepEqual(actual.removed, [a]); + await assert.rejects(lstat(old), { code: 'ENOENT' }); + assert.equal( + await readFile(join(store.store, 'unrecognized', 'source'), 'utf8'), + 'keep', + ); + assert.equal((await lstat(join(store.store, c))).isSymbolicLink(), true); + assert.equal(actual.retainedBytes, 4); +}); + +test('cleanup preserves active keys and current preparation even when over budget', async t => { + const { store, add, a, b } = await fixture(t); + await add(a, 4, 10_000); + await add(b, 4, 10_000); + await withProcessLock(join(store.store, `${a}.lock`), async () => { + const result = await pruneDependencySnapshots(store, { currentKey: b }); + assert.deepEqual(result.removed, []); + assert.deepEqual(result.skippedActive, [a]); + await assert.rejects( + pruneDependencySnapshots(store, { + currentKey: b, + incomingBytes: 10, + }), + /budget is full/, + ); + }); + assert.equal((await lstat(join(store.store, a))).isDirectory(), true); +}); + +test('publication budgets evict LRU entries and serialize commits without deleting source', async t => { + const { store, add, a, b, c, root } = await fixture(t); + await add(a, 4, 500); + await add(b, 4, 100); + const source = join(root, 'dirty-worktree'); + await mkdir(source); + await writeFile(join(source, 'uncommitted'), 'keep'); + let published = false; + const result = await pruneDependencySnapshots(store, { + currentKey: c, + incomingBytes: 4, + incomingEntries: 1, + publish: async () => { + published = true; + }, + }); + assert.deepEqual(result.removed, [a]); + assert.equal(published, true); + assert.equal(await readFile(join(source, 'uncommitted'), 'utf8'), 'keep'); +}); + +test('operator CLI previews by default and applies only after explicit flag', async t => { + const { store, add, a, root } = await fixture(t); + await add(a, 4, 10_000); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + const config = join(root, 'environment.yaml'); + await writeFile( + config, + `name: app\nroot: ${checkout}\nsetup:\n command: npm ci\n reuse:\n inputs: [package-lock.json]\n checkCommand: test -d node_modules\n snapshot:\n store: ${store.store}\n paths: [node_modules]\n lifecycle:\n retentionMs: 1000\n`, + { mode: 0o600 }, + ); + const run = promisify(execFile); + const cli = new URL('./cli.js', import.meta.url).pathname; + const args = [cli, 'prune-environment-storage', '--environment', config]; + const preview = await run(process.execPath, args); + assert.deepEqual(JSON.parse(preview.stdout).removed, [a]); + assert.equal((await lstat(join(store.store, a))).isDirectory(), true); + const applied = await run(process.execPath, [...args, '--apply']); + assert.deepEqual(JSON.parse(applied.stdout).removed, [a]); + await assert.rejects(lstat(join(store.store, a)), { code: 'ENOENT' }); +}); + +test('abandoned staging is reclaimed only with an ownership manifest and a free key lock', async t => { + const { store, a } = await fixture(t); + const name = `.staging-${a}-abcdef`; + const directory = join(store.store, name); + await mkdir(directory, { mode: 0o700 }); + await writeFile( + join(directory, STAGING_MANIFEST), + JSON.stringify({ + version: 1, + key: a, + bytes: 0, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + await withProcessLock(join(store.store, `${a}.lock`), async () => { + const active = await pruneDependencySnapshots(store); + assert.deepEqual(active.skippedActive, [name]); + assert.deepEqual(active.removed, []); + }); + const result = await pruneDependencySnapshots(store); + assert.deepEqual(result.removed, [name]); + await assert.rejects(lstat(directory), { code: 'ENOENT' }); +}); diff --git a/packages/code/src/snapshot-lifecycle.ts b/packages/code/src/snapshot-lifecycle.ts new file mode 100644 index 00000000..dfa5926e --- /dev/null +++ b/packages/code/src/snapshot-lifecycle.ts @@ -0,0 +1,316 @@ +import { constants } from 'node:fs'; +import { lstat, open, rm, statfs } from 'node:fs/promises'; +import { join } from 'node:path'; +import { withProcessLock, tryWithProcessLock } from './process-lock.js'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import { readdir, withWorkspaceRoot } from './root-access.js'; +import type { DependencySnapshotStore } from './dependency-snapshots.js'; + +export interface SnapshotLifecyclePolicy { + maxStoreBytes: number; + maxEntries: number; + retentionMs: number; + scanLimit: number; +} +export interface EnvironmentStoragePolicy { + minFreeBytes: number; + setupReserveBytes: number; +} +function positive(value: unknown, zero = false): value is number { + return ( + typeof value === 'number' && + Number.isSafeInteger(value) && + value >= (zero ? 0 : 1) + ); +} +export function parseSnapshotLifecycle( + value: unknown, +): SnapshotLifecyclePolicy { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid snapshot lifecycle'); + const v = value as Record; + const policy = { + maxStoreBytes: v.maxStoreBytes ?? 20 * 1024 ** 3, + maxEntries: v.maxEntries ?? 8, + retentionMs: v.retentionMs ?? 5 * 24 * 60 * 60 * 1000, + scanLimit: v.scanLimit ?? 4096, + }; + if ( + Object.keys(v).some(key => !Object.hasOwn(policy, key)) || + !Object.values(policy).every(value => positive(value)) + ) + throw new Error('Invalid snapshot lifecycle'); + return policy as SnapshotLifecyclePolicy; +} +export function parseEnvironmentStorage( + value: unknown, +): EnvironmentStoragePolicy { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid environment storage policy'); + const v = value as Record; + const policy = { + minFreeBytes: v.minFreeBytes ?? 5 * 1024 ** 3, + setupReserveBytes: v.setupReserveBytes ?? 2 * 1024 ** 3, + }; + if ( + Object.keys(v).some(key => !Object.hasOwn(policy, key)) || + !Object.values(policy).every(value => positive(value, true)) || + !Number.isSafeInteger( + (policy.minFreeBytes as number) + + (policy.setupReserveBytes as number), + ) + ) + throw new Error('Invalid environment storage policy'); + return policy as EnvironmentStoragePolicy; +} +/** Soft admission for managed preparation, not a reservation or arbitrary-write quota. */ +export async function assertPreparationSpace( + root: string, + policy: EnvironmentStoragePolicy, + available: (root: string) => Promise = async path => { + const status = await statfs(path, { bigint: true }); + return status.bavail * status.bsize; + }, +): Promise { + if ( + (await available(root)) < + BigInt(policy.minFreeBytes) + BigInt(policy.setupReserveBytes) + ) + throw new Error( + 'Managed environment preparation deferred: free disk is below the configured floor plus setup reserve. Clean reproducible artifacts or expand storage; no setup command was started.', + ); +} + +export const SNAPSHOT_MANIFEST = '.snapshot.json'; +export const STAGING_MANIFEST = '.staging.json'; +export class SnapshotBudgetFullError extends Error {} +export interface SnapshotManifest { + version: 1; + key: string; + bytes: number; + files: number; + createdAt: number; +} +async function manifest( + store: DependencySnapshotStore, + key: string, + name = key, + file = SNAPSHOT_MANIFEST, +): Promise { + const directory = join(store.store, name); + const status = await lstat(directory); + if (!status.isDirectory() || status.isSymbolicLink() || status.mode & 0o077) + return; + const handle = await open( + join(directory, file), + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ).catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT' || e.code === 'ELOOP') return undefined; + throw e; + }); + if (!handle) return; + try { + const metadata = await handle.stat(); + if ( + !metadata.isFile() || + metadata.size > 4096 || + metadata.mode & 0o077 || + (process.getuid && metadata.uid !== process.getuid()) + ) + return; + const buffer = Buffer.alloc(4097); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (bytesRead > 4096) return; + let value: unknown; + try { + value = JSON.parse(buffer.subarray(0, bytesRead).toString()); + } catch { + return; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) return; + const entry = value as Record; + if ( + entry.version !== 1 || + entry.key !== key || + !positive(entry.bytes, true) || + !positive(entry.files) || + !positive(entry.createdAt) + ) + return; + return value as SnapshotManifest; + } finally { + await handle.close(); + } +} +export interface SnapshotPruneResult { + dryRun: boolean; + removed: string[]; + skippedActive: string[]; + unknown: string[]; + retainedBytes: number; + retainedEntries: number; +} + +/** Only worker-published manifests authorize deletion. Source worktrees and unknown entries never qualify. */ +export async function pruneDependencySnapshots( + store: DependencySnapshotStore, + options: { + dryRun?: boolean; + currentKey?: string; + incomingBytes?: number; + incomingEntries?: number; + now?: number; + signal?: AbortSignal; + publish?: () => Promise; + } = {}, +): Promise { + const policy = store.lifecycle; + if (!policy) throw new Error('Snapshot lifecycle is not configured'); + return withProcessLock( + join(store.store, '.maintenance.lock'), + async () => { + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error( + 'Dependency snapshot store changed before cleanup', + ); + const result: SnapshotPruneResult = { + dryRun: options.dryRun === true, + removed: [], + skippedActive: [], + unknown: [], + retainedBytes: 0, + retainedEntries: 0, + }; + const candidates: { + key: string; + data: SnapshotManifest; + lastUsed: number; + }[] = []; + const names = await withWorkspaceRoot( + store.store, + store.identity, + () => readdir(store.store, policy.scanLimit), + ); + for (const key of names) { + options.signal?.throwIfAborted(); + const staging = /^\.staging-([a-f0-9]{64})-[A-Za-z0-9]+$/.exec( + key, + ); + if (staging) { + const owner = staging[1]; + const data = await manifest( + store, + owner, + key, + STAGING_MANIFEST, + ); + if (!data) { + result.unknown.push(key); + continue; + } + if (owner === options.currentKey) continue; + const lock = await tryWithProcessLock( + join(store.store, `${owner}.lock`), + async () => { + if ( + !(await manifest( + store, + owner, + key, + STAGING_MANIFEST, + )) + ) + return; + if (!options.dryRun) + await rm(join(store.store, key), { + recursive: true, + force: false, + }); + result.removed.push(key); + }, + options.signal, + ); + if (!lock.acquired) result.skippedActive.push(key); + continue; + } + if (!/^[a-f0-9]{64}$/.test(key)) { + if ( + !/^[a-f0-9]{64}\.lock$/.test(key) && + key !== '.maintenance.lock' + ) + result.unknown.push(key); + continue; + } + const data = await manifest(store, key); + if (!data) { + result.unknown.push(key); + continue; + } + candidates.push({ + key, + data, + lastUsed: ( + await lstat(join(store.store, key, SNAPSHOT_MANIFEST)) + ).mtimeMs, + }); + result.retainedBytes += data.bytes; + result.retainedEntries++; + } + const now = options.now ?? Date.now(); + candidates.sort( + (a, b) => a.lastUsed - b.lastUsed || a.key.localeCompare(b.key), + ); + for (const candidate of candidates) { + options.signal?.throwIfAborted(); + if (candidate.key === options.currentKey) continue; + if ( + now - candidate.lastUsed < policy.retentionMs && + result.retainedBytes + (options.incomingBytes ?? 0) <= + policy.maxStoreBytes && + result.retainedEntries + (options.incomingEntries ?? 0) <= + policy.maxEntries + ) + continue; + const admission = await tryWithProcessLock( + join(store.store, `${candidate.key}.lock`), + async () => { + const fresh = await manifest(store, candidate.key); + const updated = await lstat( + join(store.store, candidate.key, SNAPSHOT_MANIFEST), + ); + if ( + !fresh || + fresh.createdAt !== candidate.data.createdAt || + updated.mtimeMs !== candidate.lastUsed + ) + return; + if (!options.dryRun) + await rm(join(store.store, candidate.key), { + recursive: true, + force: false, + }); + result.removed.push(candidate.key); + result.retainedBytes -= candidate.data.bytes; + result.retainedEntries--; + }, + options.signal, + ); + if (!admission.acquired) + result.skippedActive.push(candidate.key); + } + if ( + !options.dryRun && + (result.retainedBytes + (options.incomingBytes ?? 0) > + policy.maxStoreBytes || + result.retainedEntries + (options.incomingEntries ?? 0) > + policy.maxEntries) + ) + throw new SnapshotBudgetFullError( + 'Dependency snapshot budget is full; active or unknown data was not deleted', + ); + if (!options.dryRun) await options.publish?.(); + return result; + }, + options.signal, + ); +} From a9528bc53918d63f324cfc6376bcf120a57cbcc7 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 07:48:06 -0400 Subject: [PATCH 60/75] =?UTF-8?q?=F0=9F=94=AC=20feat:=20Begin=20the=20Open?= =?UTF-8?q?Shell=20Backend=20Evaluation=20Track=20(#282)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lia --- .github/workflows/ci.yml | 3 + README.md | 6 + docs/openshell/README.md | 189 +++++++++++++++++++ scripts/openshell/probe-policy.yaml | 11 ++ scripts/openshell/probe.mjs | 217 ++++++++++++++++++++++ scripts/openshell/probe.test.mjs | 273 ++++++++++++++++++++++++++++ 6 files changed, 699 insertions(+) create mode 100644 docs/openshell/README.md create mode 100644 scripts/openshell/probe-policy.yaml create mode 100644 scripts/openshell/probe.mjs create mode 100644 scripts/openshell/probe.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6a23b61..87aaa066 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,9 @@ jobs: - name: Compose bridge configuration run: node tests/compose-bridge-config.cjs + - name: OpenShell evaluation probe (no gateway) + run: node --test scripts/openshell/probe.test.mjs + - name: Release versioning run: tests/release-versioning.sh diff --git a/README.md b/README.md index 3856b254..b746e3c0 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,12 @@ Omitting the expected-profile header remains supported for older clients, but provides no wrong-endpoint protection. There is deliberately no silent fallback between profiles and no automatic workspace or file migration. +## OpenShell evaluation + +An opt-in OpenShell backend is under [evaluation](docs/openshell/README.md). +The initial track provides a pinned lifecycle probe and security/compatibility +gates. It does not add a backend or change execution routing. + ## Sandbox Isolation Two modes are supported: diff --git a/docs/openshell/README.md b/docs/openshell/README.md new file mode 100644 index 00000000..a139ab7c --- /dev/null +++ b/docs/openshell/README.md @@ -0,0 +1,189 @@ +# OpenShell backend evaluation + +Status: Evaluation. No OpenShell backend is registered or enabled. + +Goal: add an opt-in, operator-hosted OpenShell backend behind Code API's existing +execution contract. Keep LibreChat clients, the default HTTP/libkrun backend, +Lambda MicroVMs, and outbound BYOM workers unchanged. + +## Baseline and upstream pin + +- Code API baseline: `836d001319015072f9493df5d7710fc371403962`. +- OpenShell evaluation release: `v0.1.2`, commit + `6648bd0c290efbc41ba131ee9831ee45cd431f94`. +- Pin the CLI, gateway, supervisor, and compute driver to that release. Record + image digests, effective policy, runtime/kernel, and resource limits with each + live result. The probe checks the CLI version, not the gateway's version. +- Revalidate this track against each proposed upgrade. Do not use rolling `dev` + tags for acceptance results. + +Upstream references at the evaluation commit: +[license](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/LICENSE), +[third-party notices](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/THIRD-PARTY-NOTICES), +[SDK](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/sdk/typescript/README.md), +[policy schema](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/docs/how-it-works/policies/schema.mdx). + +## Integration boundary + +Current flow: + +```text +LibreChat -> Code API auth/authorization -> Redis job + -> worker -> signed execute request -> SandboxBackend.execute + -> runner -> Code API egress gateway -> artifacts/tools + -> worker result finalization -> client +``` + +Proposed flow changes only the sandbox backend: + +```text +worker -> OpenShell adapter -> private OpenShell gateway + -> isolated workload + compatible Code API runner + -> unchanged signed execute contract and Code API egress gateway +``` + +The backend seam is +[`SandboxBackend`](../../service/src/sandbox-backend/types.ts). Its existing +consumers already own deadlines, queue/backend fencing, artifact restoration, +and result mapping. OpenShell's lifecycle API is not a substitute for the +runner's `/api/v2/execute` protocol. + +### Options + +| Approach | Trade-off | Decision | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | +| Existing backends only | No new control plane; no OpenShell policy/lifecycle integration | Keep as defaults and rollback targets | +| Adapter retaining the signed runner contract | Reuses Code API authorization, artifacts, tools, and finalization; runner compatibility is unproven | Evaluate first | +| Execute code directly with OpenShell exec | Avoids nested NsJail; requires rebuilding file/tool delivery, manifest validation, and result semantics | Defer unless the runner approach cannot retain its guarantees | + +Do not put the LibreChat application, the BYOM identity process, or Code API +control-plane credentials inside an agent workload. The gateway remains private; +Code API authenticates users and authorizes tenant/session access. + +## Invariants and owners + +| Invariant | Owner | +| ------------------------------------------------------------------------------ | ------------------------------------------------------------ | +| Tenant/user/session authorization and quotas | Code API | +| Signed body is immutable; `inputDelivery` storage IDs never reach the workload | Worker and adapter | +| Only scoped egress grants authorize artifact and tool access | Code API egress gateway and runner | +| Queue wait, provisioning, and execution share the original job deadline | Worker and adapter | +| Abort cannot settle as success; remote execution must terminate | Adapter and compute runtime | +| Retried jobs cannot create duplicate active execution or repeat mutations | Adapter, queue fencing, and runtime registry | +| Stateful workspace reuse waits for successful `sessionResultFinalizer` | Worker and adapter | +| Filesystem, process identity, network, and resource restrictions fail closed | OpenShell policy and selected runtime, plus runner hardening | +| Cleanup is confirmed, not merely requested; orphans remain observable/reapable | Adapter and lifecycle reconciler | + +OpenShell Docker/Podman policy rejects root workloads. The current runner +entrypoint performs mounts, ownership changes, cgroup setup, and an NsJail smoke +test. Compatibility is a blocker to resolve, not permission to run privileged, +disable manifests, relax Landlock, or bypass hardened startup. OpenShell policy +alone does not establish a separate-kernel tenant boundary. + +## Work sequence + +### 1. Lifecycle spike (this slice) + +- [x] Pin an upstream release and identify the existing backend seam. +- [x] Add an explicit, dependency-free CLI probe with strict filesystem policy, + no network allowances, no attached providers, resource limits, and cleanup. +- [x] Test validation, cancellation, partial failure, redaction, and nonterminal + cleanup without a live gateway. +- [ ] Run the probe against an isolated operator-owned gateway. Record results. + +This probe runs only a fixed shell/file round trip. It does **not** exercise Code +API, verify effective policy enforcement, or prove tenant isolation. A gateway +administrator's global policy can override the supplied sandbox policy. + +### 2. Hardened runner compatibility + +- [ ] Identify the supported compute/runtime boundary and prove startup without + weakening the existing security invariants. +- [ ] Forward the private runner listener through authenticated OpenShell + transport. Never expose its execute endpoint publicly. +- [ ] Execute a real worker-built signed request unchanged. Test Python/Bash, + input/output artifacts, tool-call replay, and error/result parity. +- [ ] Reject tampered/expired manifests, unauthorized storage handles, access to + supervisor/control-plane credentials, forbidden destinations, metadata + endpoints, and cross-tenant workspace reuse. +- [ ] Verify host/kernel filesystem protection and resource enforcement with + negative tests, not only effective-policy inspection. + +If nesting cannot work securely, stop and design the direct-exec replacement's +full authorization and delivery contract before implementation. + +### 3. Opt-in adapter + +- [ ] Use the same-release SDK behind lazy backend loading. Verify package + availability and locking; this release's TypeScript SDK uses GitHub Packages. + The CLI is a spike dependency, not a per-job production transport. +- [ ] Add an explicit backend config gate, isolated queues, and producer/consumer + backend checks. Old consumers must never pick up OpenShell jobs, including + legacy jobs without a backend marker. +- [ ] Start stateless. Reject affinity/strict mode until tenant-bound session + locks, fencing, finalization, and persistence are implemented and tested. +- [ ] Cover enqueue-anchored deadlines, cancellation while provisioning/running, + retry idempotency, gateway failure, ambiguous create responses, accepted/pending + deletion, worker crashes, orphan reconciliation, and shutdown. +- [ ] Keep auth tokens and provider credentials out of logs, argv, and workload + environment. Do not silently fall back to a different backend. + +### 4. Acceptance and rollout + +- [ ] Measure concurrency, queue/provision/execute/cleanup latency, resources, + artifact throughput, and failure rates against the existing backend using the + same workloads. Record measurements; define targets before promotion. +- [ ] Exercise gateway restart, worker restart, disconnects, upgrades, cleanup + backlogs, and quota pressure. Test database/runtime topology before claiming HA. +- [ ] Pin deployment artifacts, collect required notices/SBOMs, and document + private gateway authentication, backups, upgrades, and orphan operations. +- [ ] Canary a separate endpoint and worker pool only after compatibility and + security gates pass. Roll back routing, then drain the new queues. No automatic + file/workspace migration or production-default change is part of this track. + +## Run the lifecycle probe + +Requirements: Node 20+, an OpenShell `0.1.2` CLI, a registered named evaluation +gateway using the matching release, and a digest-pinned shell/coreutils image +pullable by that gateway. Install and configure these separately on dedicated +infrastructure. This repository does not install or start a gateway. + +```bash +node --test scripts/openshell/probe.test.mjs + +node scripts/openshell/probe.mjs \ + --gateway codeapi-evaluation \ + --workspace default \ + --image 'REGISTRY/IMAGE@sha256:REPLACE_WITH_64_HEX_DIGEST' +``` + +The live command creates and deletes one uniquely named `codeapi-probe-*` +sandbox. It uses the bundled policy, one CPU, 256 MiB, manual approvals, no +credential auto-discovery, and a fixed ten-second exec timeout. Each CLI phase +has a 120-second timeout (`--timeout-ms`, maximum 300000); this is not the +production job deadline. SIGINT/SIGTERM abort work but allow an independent +bounded cleanup attempt. SIGKILL or host/process failure cannot run cleanup. + +Only the pinned CLI's terminal `Deleted` or `already deleted` acknowledgement +confirms cleanup. Accepted/pending, unfamiliar, failed, and timed-out deletion +results fail the probe. CLI stdout/stderr are not emitted, except sanitized +phase/version/timing results. The live probe's overall success requires both +the file round trip and confirmed cleanup. + +On failure, use the emitted sandbox name and the **same gateway/workspace** to +inspect and reconcile it. A timed-out create can finish after cleanup; even an +`already deleted` acknowledgement does not resolve that race. Review the +gateway's resources after any ambiguous create or process failure. Do not retry +blindly or interpret `cleanup=confirmed` on a failed run as an orphan-free +fleet. Automated reconciliation is an adapter acceptance gate. + +## License implications + +OpenShell is Apache-2.0 at the pinned commit. That permits commercial hosted use +and modification without a network-use source-disclosure obligation. It does +not grant NVIDIA trademark rights or a support/warranty commitment. + +For distributed images or on-prem packages, include the license, preserve +applicable attribution/notices, mark modified upstream files, and audit bundled +runtime/SDK/image/model dependencies separately. This slice distributes no +OpenShell binaries or SDK and introduces no paid NVIDIA service dependency. diff --git a/scripts/openshell/probe-policy.yaml b/scripts/openshell/probe-policy.yaml new file mode 100644 index 00000000..bef0a892 --- /dev/null +++ b/scripts/openshell/probe-policy.yaml @@ -0,0 +1,11 @@ +version: 1 +filesystem_policy: + include_workdir: false + read_only: [/bin, /usr, /lib, /lib64, /etc] + read_write: [/tmp, /dev/null] +landlock: + compatibility: hard_requirement +process: + run_as_user: '1000' + run_as_group: '1000' +network_policies: {} diff --git a/scripts/openshell/probe.mjs b/scripts/openshell/probe.mjs new file mode 100644 index 00000000..116beb56 --- /dev/null +++ b/scripts/openshell/probe.mjs @@ -0,0 +1,217 @@ +import { execFile } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { parseArgs } from 'node:util'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export const OPENSHELL_VERSION = '0.1.2'; +export const OPENSHELL_COMMIT = '6648bd0c290efbc41ba131ee9831ee45cd431f94'; +const POLICY = fileURLToPath(new URL('./probe-policy.yaml', import.meta.url)); +const MARKER = 'codeapi-openshell-probe-ok'; + +export class ProbeError extends Error { + constructor(stage) { + super(`OpenShell probe failed at ${stage}`); + this.stage = stage; + } +} + +export function commandRunner(binary = 'openshell') { + const env = { ...process.env, NO_COLOR: '1' }; + // Named gateway selection must not inherit an endpoint or insecure override. + delete env.OPENSHELL_GATEWAY_ENDPOINT; + delete env.OPENSHELL_GATEWAY_INSECURE; + return (args, { stage, timeoutMs, signal }) => + new Promise((resolve, reject) => { + const child = execFile( + binary, + args, + { + env, + timeout: timeoutMs, + killSignal: 'SIGKILL', + maxBuffer: 65536, + signal, + }, + (error, stdout) => { + // CLI errors can contain credentials, policy contents, or provider details. + if (error) reject(new ProbeError(stage)); + else resolve(stdout); + } + ); + child.stdin.end(); + }); +} + +export async function runProbe(options, run = commandRunner()) { + const { + gateway, + image, + workspace = 'default', + timeoutMs = 120000, + signal, + } = options; + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(gateway ?? '')) { + throw new ProbeError('gateway validation'); + } + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(workspace)) { + throw new ProbeError('workspace validation'); + } + if ( + !/^\S+@sha256:[a-f0-9]{64}$/.test(image ?? '') || + image.startsWith('-') + ) { + throw new ProbeError('image validation'); + } + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 100 || + timeoutMs > 300000 + ) { + throw new ProbeError('timeout validation'); + } + const invoke = (args, stage, useSignal = true) => + run(args, { stage, timeoutMs, signal: useSignal ? signal : undefined }); + const version = await invoke(['--version'], 'version'); + if (version.trim() !== `openshell ${OPENSHELL_VERSION}`) { + throw new ProbeError('version mismatch'); + } + if (signal?.aborted) throw new ProbeError('cancelled'); + + const name = `codeapi-probe-${randomUUID()}`; + const scope = ['--gateway', gateway, '--workspace', workspace]; + const timings = {}; + const timed = async (stage, args, useSignal = true) => { + const start = performance.now(); + try { + return await invoke([...scope, ...args], stage, useSignal); + } finally { + timings[stage] = Math.round(performance.now() - start); + } + }; + let failure; + let cleanup = 'unconfirmed'; + try { + await timed('create', [ + 'sandbox', + 'create', + '--name', + name, + '--from', + image, + '--policy', + POLICY, + '--no-auto-providers', + '--approval-mode', + 'manual', + '--cpu', + '1', + '--memory', + '256Mi', + '--keep', + '--detach', + '--no-tty', + '--output', + 'json', + '--', + '/bin/sh', + '-c', + 'sleep 300', + ]); + const output = await timed('exec', [ + 'sandbox', + 'exec', + '--name', + name, + '--timeout', + '10', + '--no-tty', + '--no-login-shell', + '--', + '/bin/sh', + '-c', + `printf '%s' '${MARKER}' > /tmp/codeapi-probe && cat /tmp/codeapi-probe`, + ]); + if (output.trim() !== MARKER) throw new ProbeError('exec output'); + } catch (error) { + failure = error instanceof ProbeError ? error.stage : 'execution'; + } finally { + try { + const output = await timed( + 'delete', + ['sandbox', 'delete', name], + false + ); + const text = output.replace(/\x1b\[[0-9;]*m/g, '').trim(); + // Exit zero also covers accepted/pending deletion. Only terminal outcomes pass. + if ( + text === `✓ Deleted sandbox ${name}` || + text === `✓ Sandbox ${name} already deleted` + ) { + cleanup = 'confirmed'; + } + } catch { + // Keep the execution failure and report cleanup independently. + } + } + if (signal?.aborted && failure === undefined) failure = 'cancelled'; + return { + ok: failure === undefined && cleanup === 'confirmed', + version: OPENSHELL_VERSION, + upstreamCommit: OPENSHELL_COMMIT, + sandbox: name, + workspace, + cleanup, + ...(failure ? { failedStage: failure } : {}), + timingsMs: timings, + }; +} + +async function main() { + const { values } = parseArgs({ + options: { + gateway: { type: 'string' }, + workspace: { type: 'string', default: 'default' }, + image: { type: 'string' }, + 'timeout-ms': { type: 'string', default: '120000' }, + help: { type: 'boolean' }, + }, + }); + if (values.help) { + console.log( + 'node scripts/openshell/probe.mjs --gateway NAME --image REGISTRY/IMAGE@sha256:DIGEST [--workspace NAME] [--timeout-ms 120000]' + ); + return; + } + const controller = new AbortController(); + const abort = () => controller.abort(); + process.on('SIGINT', abort); + process.on('SIGTERM', abort); + try { + const result = await runProbe({ + gateway: values.gateway, + workspace: values.workspace, + image: values.image, + timeoutMs: Number(values['timeout-ms']), + signal: controller.signal, + }); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.ok ? 0 : 1; + } finally { + process.off('SIGINT', abort); + process.off('SIGTERM', abort); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main().catch(error => { + console.error( + error instanceof ProbeError + ? error.message + : 'Invalid OpenShell probe invocation' + ); + process.exitCode = 1; + }); +} diff --git a/scripts/openshell/probe.test.mjs b/scripts/openshell/probe.test.mjs new file mode 100644 index 00000000..2ec95997 --- /dev/null +++ b/scripts/openshell/probe.test.mjs @@ -0,0 +1,273 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + commandRunner, + OPENSHELL_VERSION, + ProbeError, + runProbe, +} from './probe.mjs'; + +const options = { + gateway: 'evaluation', + workspace: 'codeapi-test', + image: `registry.test/shell@sha256:${'a'.repeat(64)}`, +}; +function fixture(overrides = {}) { + const calls = []; + const run = async (args, call) => { + calls.push({ args, ...call }); + if (overrides[call.stage]) return overrides[call.stage](args, call); + if (call.stage === 'version') return `openshell ${OPENSHELL_VERSION}\n`; + if (call.stage === 'exec') return 'codeapi-openshell-probe-ok'; + if (call.stage === 'delete') + return `✓ Deleted sandbox ${args.at(-1)}\n`; + return '{}'; + }; + return { run, calls }; +} + +test('scopes every operation and creates a bounded, credential-free sandbox', async () => { + const { run, calls } = fixture(); + const result = await runProbe(options, run); + assert.equal(result.ok, true); + assert.equal(result.cleanup, 'confirmed'); + assert.match(result.sandbox, /^codeapi-probe-[a-f0-9-]{36}$/); + assert.deepEqual( + calls.map(call => call.stage), + ['version', 'create', 'exec', 'delete'] + ); + for (const call of calls.slice(1)) { + assert.deepEqual(call.args.slice(0, 4), [ + '--gateway', + 'evaluation', + '--workspace', + 'codeapi-test', + ]); + assert.equal(call.timeoutMs, 120000); + assert.ok(call.args.includes(result.sandbox)); + } + const create = calls[1].args; + for (const flag of [ + '--policy', + '--no-auto-providers', + '--keep', + '--detach', + '--no-tty', + ]) + assert.ok(create.includes(flag)); + assert.equal(create.includes('--provider'), false); + assert.equal(create.includes('--env'), false); + assert.equal(create[create.indexOf('--from') + 1], options.image); + assert.equal(create[create.indexOf('--cpu') + 1], '1'); + assert.equal(create[create.indexOf('--memory') + 1], '256Mi'); + assert.equal(create[create.indexOf('--approval-mode') + 1], 'manual'); +}); + +test('rejects mutable images, ambiguous scope, and invalid timeouts before invoking the CLI', async () => { + for (const invalid of [ + { image: 'shell:latest' }, + { image: `-shell@sha256:${'a'.repeat(64)}` }, + { gateway: undefined }, + { workspace: '../default' }, + { timeoutMs: 0 }, + { timeoutMs: 300001 }, + ]) { + const { run, calls } = fixture(); + await assert.rejects( + runProbe({ ...options, ...invalid }, run), + ProbeError + ); + assert.equal(calls.length, 0); + } +}); + +test('rejects an unpinned CLI without creating or deleting a sandbox', async () => { + const { run, calls } = fixture({ version: () => 'openshell 0.1.3' }); + await assert.rejects(runProbe(options, run), /version mismatch/); + assert.equal(calls.length, 1); +}); + +for (const stage of ['create', 'exec']) { + test(`attempts cleanup after ${stage} failure without exposing upstream details`, async () => { + const { run, calls } = fixture({ + [stage]: () => { + throw new Error('secret provider credential'); + }, + }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'execution'); + assert.equal(result.cleanup, 'confirmed'); + assert.equal(calls.at(-1).stage, 'delete'); + assert.equal(JSON.stringify(result).includes('secret'), false); + }); +} + +test('does not accept unexpected command output as success', async () => { + const { run } = fixture({ exec: () => 'unexpected' }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'exec output'); + assert.equal(result.cleanup, 'confirmed'); +}); + +for (const output of [ + 'deletion accepted; cleanup is pending', + 'unsupported outcome', + '', + '✓ Deleted sandbox someone-else', +]) { + test(`does not report terminal cleanup for ${JSON.stringify( + output + )}`, async () => { + const { run } = fixture({ delete: () => output }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.cleanup, 'unconfirmed'); + }); +} + +test('accepts an already-absent sandbox as terminal cleanup', async () => { + const { run } = fixture({ + delete: args => `✓ Sandbox ${args.at(-1)} already deleted\n`, + }); + assert.equal((await runProbe(options, run)).cleanup, 'confirmed'); +}); + +test('preserves execution failure when cleanup also fails', async () => { + const { run } = fixture({ + exec: () => { + throw new ProbeError('exec'); + }, + delete: () => { + throw new Error('secret'); + }, + }); + const result = await runProbe(options, run); + assert.equal(result.failedStage, 'exec'); + assert.equal(result.cleanup, 'unconfirmed'); + assert.equal(result.ok, false); +}); + +test('cancellation before create does not launch work', async () => { + const controller = new AbortController(); + controller.abort(); + const { run, calls } = fixture(); + await assert.rejects( + runProbe({ ...options, signal: controller.signal }, run), + /cancelled/ + ); + assert.equal(calls.length, 1); +}); + +test('cancellation during create still uses an independent cleanup attempt', async () => { + const controller = new AbortController(); + const { run, calls } = fixture({ + create: () => { + controller.abort(); + throw new ProbeError('create'); + }, + }); + const result = await runProbe( + { ...options, signal: controller.signal }, + run + ); + assert.equal(result.ok, false); + assert.equal(calls[1].signal, controller.signal); + assert.equal(calls.at(-1).signal, undefined); + assert.equal(calls.at(-1).stage, 'delete'); +}); + +test('command runner redacts stdout and stderr on failure', async () => { + await assert.rejects( + commandRunner(process.execPath)( + [ + '-e', + 'console.log("secret"); console.error("private-key"); process.exit(2)', + ], + { stage: 'exec', timeoutMs: 1000 } + ), + error => { + assert.equal(error.message, 'OpenShell probe failed at exec'); + assert.equal(error.cause, undefined); + assert.equal(error.stdout, undefined); + assert.equal(error.stderr, undefined); + return true; + } + ); +}); + +test('command runner kills a stalled CLI within its timeout', async () => { + await assert.rejects( + commandRunner(process.execPath)(['-e', 'setInterval(() => {}, 1000)'], { + stage: 'create', + timeoutMs: 100, + }), + /failed at create/ + ); +}); + +test('command runner removes inherited endpoint and insecure overrides', async () => { + const savedEndpoint = process.env.OPENSHELL_GATEWAY_ENDPOINT; + const savedInsecure = process.env.OPENSHELL_GATEWAY_INSECURE; + try { + process.env.OPENSHELL_GATEWAY_ENDPOINT = 'http://wrong-gateway'; + process.env.OPENSHELL_GATEWAY_INSECURE = 'true'; + const output = await commandRunner(process.execPath)( + [ + '-e', + 'console.log(JSON.stringify([process.env.OPENSHELL_GATEWAY_ENDPOINT, process.env.OPENSHELL_GATEWAY_INSECURE]))', + ], + { stage: 'version', timeoutMs: 1000 } + ); + assert.deepEqual(JSON.parse(output), [null, null]); + } finally { + if (savedEndpoint === undefined) + delete process.env.OPENSHELL_GATEWAY_ENDPOINT; + else process.env.OPENSHELL_GATEWAY_ENDPOINT = savedEndpoint; + if (savedInsecure === undefined) + delete process.env.OPENSHELL_GATEWAY_INSECURE; + else process.env.OPENSHELL_GATEWAY_INSECURE = savedInsecure; + } +}); + +test('late cancellation cannot report success while cleanup completes', async () => { + const controller = new AbortController(); + const { run } = fixture({ + delete: args => { + controller.abort(); + return `✓ Deleted sandbox ${args.at(-1)}\n`; + }, + }); + const result = await runProbe( + { ...options, signal: controller.signal }, + run + ); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'cancelled'); + assert.equal(result.cleanup, 'confirmed'); +}); + +test('command runner rejects oversized output without exposing it', async () => { + await assert.rejects( + commandRunner(process.execPath)( + ['-e', 'process.stdout.write("x".repeat(100000))'], + { stage: 'exec', timeoutMs: 1000 } + ), + error => { + assert.equal(error.message, 'OpenShell probe failed at exec'); + assert.equal(error.stdout, undefined); + return true; + } + ); +}); + +test('command runner handles a missing CLI without exposing process details', async () => { + await assert.rejects( + commandRunner('codeapi-nonexistent-probe-binary')([], { + stage: 'version', + timeoutMs: 1000, + }), + /failed at version/ + ); +}); From 032a93fec88716a819ef33483448f403d1c17729 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 22:29:58 -0400 Subject: [PATCH 61/75] feat: Retire Stale Linked Worktrees Automatically (#299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧹 feat: Retire Stale Linked Worktrees Automatically Agents create one linked worktree per task under /.worktrees and nothing ever removed them. Each keeps its own dependencies and build output, so a trusted-VM checkout accumulated 231 worktrees and filled its disk to 100% three times, quarantining workspaces. A worker with linked worktree lanes now retires stale worktrees in the background: two minutes after start, every six hours, sooner while a backlog remains, and before managed setup would be deferred for low disk space. A worktree is retired only when it verifies as a linked worktree under .worktrees, no request uses its lane or checkout, it has been idle for seven days by both this worker's lane use and on-disk Git activity, it is clean, unlocked, outside any merge/rebase/cherry-pick/revert/bisect, unquarantined, and its work is published or merged: HEAD is contained in a remote-tracking ref, or, with no live upstream (detached, never pushed, or upstream deleted after merge), its commits are already in the remote's default branch by content (patch-equivalent per git cherry, or identical content at every path the branch changed). Squash-merged branches and abandoned review worktrees therefore qualify; only local Git is consulted. Removal is `git worktree remove` without --force followed by `git worktree prune`; branches are kept so `git worktree add` restores any worktree. Retirement is on by default. --no-worktree-retirement or LIBRECHAT_CODE_WORKTREE_RETIREMENT=false disables it, and --worktree-idle-days / LIBRECHAT_CODE_WORKTREE_IDLE_DAYS change the threshold. LIBRECHAT_CODE_LOG_LEVEL=debug logs every kept worktree with its reason. * fix: Fence Checkout Requests During Retirement and Never Cut Removal Short - Checkout requests wait for an in-progress retirement of any lane beneath them, since root commands can reach .worktrees/*. - Requests waiting on a retirement honor their own cancellation. - git worktree remove runs without a timeout; a killed removal could leave a half-deleted worktree that no later pass can verify. - Drop the repository-wide git worktree prune: remove already deletes the retired worktree's metadata, and prune would also expire other registered worktrees that are only temporarily unavailable. - Read every .worktrees entry each pass instead of the first 4096. * fix: Recheck Quarantine Under the Lane Reservation A checkout or lane request that finished between inspection and the lane reservation may have left an uncertain mutation quarantined. Re-read both quarantine markers inside the reservation, immediately before removal. * fix: Defer Retirement When the Checkout Was Used During Inspection A checkout request can write ignored files inside .worktrees/ that git status never reports, and finish before the lane reservation. Count checkout request starts and finishes, capture the count before inspecting each worktree, and keep the worktree for a later pass when it changed by the time the reservation is held. * test: Clone the Merge-Simulation Remote on main Regardless of init.defaultBranch On CI the bare remote's HEAD names master, so the hosting clone had an empty HEAD and `git merge --squash` refused to run. --- docs/remote-bridge/worker-runbook.md | 38 +- packages/code/README.md | 57 ++ packages/code/src/cli.ts | 52 ++ packages/code/src/environment-preparation.ts | 16 +- packages/code/src/linked-worktrees.ts | 198 ++++- packages/code/src/snapshot-lifecycle.test.ts | 48 ++ packages/code/src/snapshot-lifecycle.ts | 24 +- packages/code/src/worktree-retirement.test.ts | 592 +++++++++++++++ packages/code/src/worktree-retirement.ts | 697 ++++++++++++++++++ 9 files changed, 1685 insertions(+), 37 deletions(-) create mode 100644 packages/code/src/worktree-retirement.test.ts create mode 100644 packages/code/src/worktree-retirement.ts diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 51018940..4026a755 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -521,6 +521,35 @@ replace the local clear command. Never clear quarantine merely to make the worker start. It represents a setup, command, cancellation, or settlement whose effects may be incomplete. +### Disk filled by task worktrees + +Agents that create one linked worktree per task under `/.worktrees/` +leave each one behind with its own dependencies and build output. A worker with +`--linked-worktree-lanes` retires stale ones automatically: two minutes after +startup, every six hours, and before managed setup would be deferred for low +space. Only verified linked worktrees in writable roots that are idle for seven +days, clean, unlocked, outside any merge or rebase, unquarantined, and whose +`HEAD` is on a remote-tracking ref or already in the default branch by content +(a squash- or rebase-merged branch whose remote branch was deleted) are +removed, with `git worktree remove` without `--force`. Merged content is judged +against the checkout's last fetch of the default branch; the worker never +fetches. Branches are kept; restore one with +`git worktree add .worktrees/ `. The +[worker package guide](../../packages/code/README.md#retiring-stale-linked-worktrees) +lists every condition. + +Each pass logs `worktree retirement: retired N, kept M (reason counts)`. To see +why a particular worktree is kept, restart once with +`LIBRECHAT_CODE_LOG_LEVEL=debug`. Worktrees kept as `dirty` or `unpushed` hold +work nobody has published; push or discard it from the checkout before removing +them by hand, never with `rm -rf`. If the disk is already full and setup keeps +failing, free space by hand first: retirement runs only after the worker starts, +or during setup that has a `storage` floor configured. + +To keep every worktree, add `--no-worktree-retirement` (or +`LIBRECHAT_CODE_WORKTREE_RETIREMENT=false`). To keep them longer, set +`--worktree-idle-days ` (or `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS`). + ## 15. Common failures - **`--environment cannot be combined...`:** remove old workspace flags and @@ -540,8 +569,13 @@ command, cancellation, or settlement whose effects may be incomplete. configure private copy-on-write snapshots and their lifecycle budget. Do not symlink another branch's mutable `node_modules` or hardlink writable installs. - **Managed preparation deferred for low space:** `storage.minFreeBytes` plus - `setupReserveBytes` is a soft pre-setup floor, not a hard quota. Expand the - volume or clean reproducible artifacts; do not clear quarantine as a disk fix. + `setupReserveBytes` is a soft pre-setup floor, not a hard quota. A worker + with linked worktree lanes first retires stale task worktrees and measures + again. If space is still short, expand the volume or clean reproducible + artifacts; do not clear quarantine as a disk fix. +- **Many `.worktrees/*` directories remain:** see + [Disk filled by task worktrees](#disk-filled-by-task-worktrees). Count the + kept reasons in the retirement summary before removing anything by hand. - **Snapshot maintenance:** preview with `prune-environment-storage --environment ` and use `--apply` only after reviewing its JSON. Active, unknown and unmarked data stays intact. Include all environment definitions diff --git a/packages/code/README.md b/packages/code/README.md index 38a22d5d..55f5c668 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -948,6 +948,63 @@ be combined with conversation worktrees. Code API must advertise scope for them. Deploy consumers that read worker status (such as LibreChat) with support for `workspaceScopes` before enabling lanes on a worker. +##### Retiring stale linked worktrees + +Nothing else removes a task's worktree once its work is pushed, and each one +keeps its own dependencies and build output. A worker with lanes therefore +retires stale ones itself, with no configuration: a pass runs two minutes after +startup and every six hours after that, sooner while a backlog remains, and +also before managed environment setup would be deferred for low disk space. +Passes run in the background and never overlap. Only an actual removal holds +back requests, and only those for that lane or its checkout. + +A worktree in a writable registered root is retired only when **all** of these +hold; otherwise it is kept and the reason is counted: + +- It verifies as a linked worktree of the checkout under `.worktrees/`, as for + lane admission. The checkout itself and worktrees elsewhere are never + touched. +- Neither the lane nor its checkout has a request in flight or ran one while + the worktree was being inspected, and both this + worker's last use of the lane and the newest on-disk activity (the worktree + directory and its Git `HEAD`, `index`, `logs/HEAD`, `ORIG_HEAD` and + `FETCH_HEAD`) are older than the idle threshold, seven days by default. +- It has no modified tracked files and no untracked files the repository does + not ignore; no merge, rebase, cherry-pick, revert or bisect in progress; no + `git worktree lock`; and no quarantine on the lane or its checkout. +- Its `HEAD` commit, including a detached one, is contained in at least one + remote-tracking ref (`refs/remotes/*`), or its work is already in the + remote's default branch by content, as after a squash or rebase merge whose + remote branch was deleted. Content counts only when `HEAD` has no live + upstream (it is detached, never pushed, or its upstream ref is gone) and + either every commit beyond the default branch is patch-equivalent to one in + it (`git cherry` shows only `-`, with no merge commits), or the default + branch has identical content at every path the branch changed since their + merge base. The default branch is the remote's `HEAD`, else `main` or + `master`, as last fetched; the worker never fetches or calls a hosting API. + Commits beyond a live upstream are never treated as merged. + +Removal is `git worktree remove` **without** `--force`, so Git re-checks for +changes itself and deletes only that worktree's metadata. No repository-wide +`git worktree prune` runs, so other registered worktrees that are temporarily +unavailable stay registered. Removal has no timeout, because a half-deleted +worktree could no longer be recognized. Ignored files such as `node_modules`, +build output and ignored `.env` files go with the worktree; that is the space +being reclaimed. The branch is kept, so +`git worktree add .worktrees/ ` restores the worktree. Lane and +checkout requests that arrive during a removal wait for it, or for their own +cancellation; a lane request then fails as an unknown worktree. Each pass reads +every `.worktrees` entry, inspects at most 128 idle worktrees and removes at +most 32, oldest first, rotating so that worktrees kept for lasting reasons +cannot hide the rest. Each pass logs one summary line, for example +`worktree retirement: retired 3, kept 12 (dirty 2, recent 8, unpushed 2), freed +about 4.1 GiB`; set `LIBRECHAT_CODE_LOG_LEVEL=debug` to log every kept worktree +and its reason. + +Pass `--no-worktree-retirement` or set `LIBRECHAT_CODE_WORKTREE_RETIREMENT=false` +to disable retirement. Change the idle threshold with `--worktree-idle-days ` +or `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS=` (1 to 3650 days). + On an updated Code API, admission waits up to 30 seconds without the `X-LibreChat-Workspace-Queue-Wait-Ms` request header. A caller may advertise a positive integer millisecond allowance up to five minutes, capped by any server diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index cf05e263..6b450b64 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -43,6 +43,10 @@ import { NativeWorkspaceCommandPool } from './native-pool.js'; import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; import { GitWorktreeManager } from './worktrees.js'; +import { + WorktreeRetirementScheduler, + worktreeRetirementSettings, +} from './worktree-retirement.js'; import { captureWorkspaceRootIdentity } from './root-identity.js'; import { resolveNativeSrtCommandPolicy, @@ -812,6 +816,13 @@ async function run( if (linkedWorktreeLanes && process.platform === 'win32') { throw new Error('Linked worktree Git guard requires a POSIX host'); } + const worktreeRetirement = worktreeRetirementSettings({ + optOut: args.includes('--no-worktree-retirement'), + enabled: process.env.LIBRECHAT_CODE_WORKTREE_RETIREMENT, + idleDays: + option(args, '--worktree-idle-days') ?? + process.env.LIBRECHAT_CODE_WORKTREE_IDLE_DAYS, + }); if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() @@ -1301,6 +1312,41 @@ async function run( }); workspaceTools = linkedWorktreeTools; } + const retirementSources = roots.filter((root) => root.writable); + const debugLogs = + process.env.LIBRECHAT_CODE_LOG_LEVEL?.trim().toLowerCase() === 'debug'; + const worktreeRetirementScheduler = + linkedWorktreeTools && + worktreeRetirement.enabled && + retirementSources.length > 0 && + option(args, '--reset-workspace-quarantine') == null + ? new WorktreeRetirementScheduler({ + activity: linkedWorktreeTools, + idleMs: worktreeRetirement.idleMs, + sources: retirementSources.map((root) => ({ + workspaceId: root.id, + root: root.root, + identity: root.identity, + })), + async isQuarantined(selectedWorkspaceId, worktree) { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) return true; + const path = + worktree == null + ? rootQuarantinePaths.get(selectedWorkspaceId)! + : defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: join(source.root, LINKED_WORKTREE_DIRECTORY, worktree), + }); + return (await loadWorkspaceMutationQuarantine(path)) != null; + }, + log(level, message) { + if (level === 'debug' && !debugLogs) return; + process.stdout.write(`librechat-code: ${message}\n`); + }, + }) + : undefined; if (workspaceTools && environments.length) { workspaceTools = new EnvironmentWorkspaceTools( workspaceTools, @@ -1375,6 +1421,9 @@ async function run( root: environment.definition.root, identity: roots.find(root => root.id === id)!.identity!, setup, receiptPath: preparationReceipt(environment.definition.root), + reclaimSpace: worktreeRetirementScheduler + ? () => worktreeRetirementScheduler.runNow() + : undefined, context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, nativeOptionsForWorkspace(id).resources]), signal: controller.signal, @@ -1404,10 +1453,12 @@ async function run( ); } } catch (error) { + await worktreeRetirementScheduler?.stop().catch(() => undefined); await nativeCommandSandbox?.close().catch(() => undefined); await fileRelaySupervisor?.stop().catch(() => undefined); throw error; } + worktreeRetirementScheduler?.start(); try { const worker = new BridgeWorker({ instructionDescriptors: () => localWorkspaceTools?.instructionDescriptors() ?? Promise.resolve(undefined), @@ -1617,6 +1668,7 @@ async function run( await worker.run(controller.signal); } finally { try { + await worktreeRetirementScheduler?.stop(); await nativeCommandSandbox?.close(); } finally { await fileRelaySupervisor?.stop(); diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts index 8dbb8751..b17b4d32 100644 --- a/packages/code/src/environment-preparation.ts +++ b/packages/code/src/environment-preparation.ts @@ -37,6 +37,8 @@ export interface EnvironmentPreparationOptions { snapshotScope?: string; beforeMutation?(): Promise; storage?: EnvironmentStoragePolicy; + /** Frees reproducible storage when the storage floor would defer setup. */ + reclaimSpace?(): Promise; } /** Checkout-local reuse. Never transfers mutable installations between worktrees. */ @@ -87,7 +89,12 @@ async function prepareInLock( } if (options.snapshotStore && portable) { if (options.storage) - await assertPreparationSpace(options.root, options.storage); + await assertPreparationSpace( + options.root, + options.storage, + undefined, + options.reclaimSpace, + ); await options.beforeMutation?.(); if ( await restoreDependencySnapshot( @@ -118,7 +125,12 @@ async function prepareInLock( } } if (options.storage) - await assertPreparationSpace(options.root, options.storage); + await assertPreparationSpace( + options.root, + options.storage, + undefined, + options.reclaimSpace, + ); const result = await options.execute( options.setup.command, options.setup.timeoutMs, diff --git a/packages/code/src/linked-worktrees.ts b/packages/code/src/linked-worktrees.ts index e049aa6e..225abebd 100644 --- a/packages/code/src/linked-worktrees.ts +++ b/packages/code/src/linked-worktrees.ts @@ -32,6 +32,27 @@ const LINKED_WORKTREE_SHARED_GIT_PATHS = ['objects', 'refs', join('logs', 'refs' const LINKED_WORKTREE_LANE_LIMIT = 32; /** Git pointer files are a single line; anything larger is not one. */ const GIT_POINTER_MAX_BYTES = 4096; +/** Last-use timestamps kept per worker; the oldest are forgotten first. */ +const LINKED_WORKTREE_ACTIVITY_LIMIT = 4096; + +/** What this worker knows about lane use, for deciding when a worktree may be retired. */ +export interface LinkedWorktreeActivity { + /** When a request for the lane last started or finished in this process. */ + lastUsed(workspaceId: string, worktree: string): number | undefined; + /** Changes whenever a checkout request (outside any lane) starts or finishes. */ + checkoutActivity(workspaceId: string): number; + /** + * Run `task` only while neither the lane nor its checkout has a request in + * flight. Requests for the lane or its checkout that arrive meanwhile wait + * for it to finish (or for their own cancellation); a lane request then + * verifies the lane again. + */ + whileIdle( + workspaceId: string, + worktree: string, + task: () => Promise, + ): Promise<{ ran: true; value: T } | { ran: false }>; +} export interface LinkedWorktreeSource { root: string; @@ -206,7 +227,7 @@ function publicResult(result: WorkspaceToolResult, workspaceId: string): Workspa * checkout, so file tools and commands here run confined to that worktree while * sibling lanes run concurrently. Requests without a worktree pass through. */ -export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { +export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor, LinkedWorktreeActivity { readonly mutationFailuresAreAtomic?: true; readonly capabilities: WorkspaceToolExecutor['capabilities']; private readonly executors = new Map< @@ -216,6 +237,16 @@ export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { private readonly commandRoots = new Map(); /** Verified lane roots by internal ID, least recently used first. */ private readonly lanes = new Map(); + /** Requests in flight by lane internal ID. */ + private readonly laneRequests = new Map(); + /** Requests in flight by checkout workspace ID, outside any lane. */ + private readonly checkoutRequests = new Map(); + /** Last request start or finish by lane internal ID, oldest first. */ + private readonly used = new Map(); + /** Checkout request starts and finishes by workspace ID. */ + private readonly checkoutEvents = new Map(); + /** Lanes being retired by internal ID; requests for the lane or its checkout wait for them to settle. */ + private readonly retiring = new Map }>(); constructor(private readonly options: LinkedWorktreeWorkspaceToolsOptions) { this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; @@ -337,29 +368,139 @@ export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { return pool; } + lastUsed(workspaceId: string, worktree: string): number | undefined { + return this.used.get(linkedWorktreeWorkspaceId(workspaceId, worktree)); + } + + checkoutActivity(workspaceId: string): number { + return this.checkoutEvents.get(workspaceId) ?? 0; + } + + async whileIdle( + workspaceId: string, + worktree: string, + task: () => Promise, + ): Promise<{ ran: true; value: T } | { ran: false }> { + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + if ( + this.laneRequests.has(internalId) || + this.checkoutRequests.has(workspaceId) || + this.retiring.has(internalId) + ) { + return { ran: false }; + } + let settle!: () => void; + const done = new Promise((resolve) => { settle = resolve; }); + this.retiring.set(internalId, { workspaceId, done }); + try { + return { ran: true, value: await task() }; + } finally { + await this.release(internalId); + this.retiring.delete(internalId); + settle(); + } + } + + private touch(internalId: string): void { + this.used.delete(internalId); + this.used.set(internalId, Date.now()); + if (this.used.size <= LINKED_WORKTREE_ACTIVITY_LIMIT) return; + const oldest = this.used.keys().next().value; + if (oldest != null) this.used.delete(oldest); + } + + /** Count a request from its first synchronous step, so `whileIdle` never races its admission. */ + private async tracked( + requests: Map, + key: string, + task: () => Promise, + lane: boolean, + ): Promise { + const record = (): void => { + if (lane) this.touch(key); + else this.checkoutEvents.set(key, (this.checkoutEvents.get(key) ?? 0) + 1); + }; + requests.set(key, (requests.get(key) ?? 0) + 1); + record(); + try { + return await task(); + } finally { + const remaining = (requests.get(key) ?? 1) - 1; + if (remaining > 0) requests.set(key, remaining); + else requests.delete(key); + record(); + } + } + + /** Wait out retirements a request could collide with, unless the request is cancelled first. */ + private async afterRetirement( + collides: (internalId: string, workspaceId: string) => boolean, + signal: AbortSignal | undefined, + ): Promise { + const pending = [...this.retiring] + .filter(([internalId, entry]) => collides(internalId, entry.workspaceId)) + .map(([, entry]) => entry.done); + if (pending.length === 0) return; + signal?.throwIfAborted(); + await new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal?.reason); + signal?.addEventListener('abort', onAbort, { once: true }); + void Promise.all(pending).then(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }); + }); + } + + private async inCheckout(workspaceId: string, task: () => Promise, signal?: AbortSignal): Promise { + return await this.tracked(this.checkoutRequests, workspaceId, async () => { + await this.afterRetirement((_internalId, laneWorkspaceId) => laneWorkspaceId === workspaceId, signal); + return await task(); + }, false); + } + + private async inLane( + workspaceId: string, + worktree: string, + task: () => Promise, + signal?: AbortSignal, + ): Promise { + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + return await this.tracked(this.laneRequests, internalId, async () => { + await this.afterRetirement((retiringId) => retiringId === internalId, signal); + return await task(); + }, true); + } + async execute(request: WorkspaceToolRequest, signal?: AbortSignal): Promise { if (request.worktree == null) { - return await this.options.delegate.execute(request, signal); + return await this.inCheckout( + request.workspaceId, + () => this.options.delegate.execute(request, signal), + signal, + ); } if (request.operation === 'execute_command' && request.environmentAction) { throw rejected('Environment action was not resolved by this worker'); } const { worktree, ...baseRequest } = request; - const { lane, source, internalId } = await this.resolveLane( - request.workspaceId, - worktree, - request.workspaceInstanceId, - ); - const laneRequest = { ...baseRequest, workspaceId: internalId } as WorkspaceToolRequest; - if (request.operation === 'execute_command') { - const pool = await this.registerCommandRoot(internalId, lane, source); - return publicResult( - await pool.execute(laneRequest as WorkspaceExecuteCommandRequest, signal), + return await this.inLane(request.workspaceId, worktree, async () => { + const { lane, source, internalId } = await this.resolveLane( request.workspaceId, + worktree, + request.workspaceInstanceId, ); - } - const executor = await this.fileExecutor(internalId, lane, source); - return publicResult(await executor.execute(laneRequest, signal), request.workspaceId); + const laneRequest = { ...baseRequest, workspaceId: internalId } as WorkspaceToolRequest; + if (request.operation === 'execute_command') { + const pool = await this.registerCommandRoot(internalId, lane, source); + return publicResult( + await pool.execute(laneRequest as WorkspaceExecuteCommandRequest, signal), + request.workspaceId, + ); + } + const executor = await this.fileExecutor(internalId, lane, source); + return publicResult(await executor.execute(laneRequest, signal), request.workspaceId); + }, signal); } async executeProgrammatic( @@ -369,18 +510,25 @@ export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { ): Promise { const worktree = request.body.workspace_worktree; if (worktree == null) { - if (!this.options.programmaticDelegate) { + const delegate = this.options.programmaticDelegate; + if (!delegate) { throw new WorkspaceToolError('Workspace programmatic execution is unavailable', 'COMMAND_DISABLED'); } - return await this.options.programmaticDelegate.executeProgrammatic(workspaceId, request, signal); + return await this.inCheckout( + workspaceId, + () => delegate.executeProgrammatic(workspaceId, request, signal), + signal, + ); } - const { lane, source, internalId } = await this.resolveLane( - workspaceId, - worktree, - request.body.workspace_instance_id, - ); - const pool = await this.registerCommandRoot(internalId, lane, source); - const { workspace_worktree: _worktree, ...body } = request.body; - return await pool.executeProgrammatic(internalId, { ...request, body }, signal); + return await this.inLane(workspaceId, worktree, async () => { + const { lane, source, internalId } = await this.resolveLane( + workspaceId, + worktree, + request.body.workspace_instance_id, + ); + const pool = await this.registerCommandRoot(internalId, lane, source); + const { workspace_worktree: _worktree, ...body } = request.body; + return await pool.executeProgrammatic(internalId, { ...request, body }, signal); + }, signal); } } diff --git a/packages/code/src/snapshot-lifecycle.test.ts b/packages/code/src/snapshot-lifecycle.test.ts index 33e37c2d..8074fd11 100644 --- a/packages/code/src/snapshot-lifecycle.test.ts +++ b/packages/code/src/snapshot-lifecycle.test.ts @@ -116,6 +116,54 @@ test('storage policy is opt-in, bounded and fails before a setup command', async assert.equal(executions, 0); }); +test('low space reclaims once before deferring setup', async t => { + const policy = { minFreeBytes: 5, setupReserveBytes: 2 }; + let free = 6n; + let reclaims = 0; + await assertPreparationSpace('/', policy, async () => free, async () => { + reclaims++; + free = 7n; + }); + assert.equal(reclaims, 1); + await assertPreparationSpace('/', policy, async () => free, async () => { + reclaims++; + }); + assert.equal(reclaims, 1, 'enough space never triggers reclamation'); + await assert.rejects( + assertPreparationSpace('/', policy, async () => 6n, async () => { + reclaims++; + }), + /no setup command was started/, + ); + assert.equal(reclaims, 2); + + const { root } = await fixture(t); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + let executions = 0; + let reclaimed = 0; + await assert.rejects( + prepareCodeEnvironment({ + root: checkout, + identity: await captureWorkspaceRootIdentity(checkout), + setup: { command: 'install', timeoutMs: 1000 }, + receiptPath: join(root, 'receipt'), + context: '', + storage: { minFreeBytes: Number.MAX_SAFE_INTEGER, setupReserveBytes: 0 }, + reclaimSpace: async () => { + reclaimed++; + }, + execute: async () => { + executions++; + return { exitCode: 0, timedOut: false }; + }, + }), + /deferred/, + ); + assert.equal(reclaimed, 1); + assert.equal(executions, 0); +}); + test('dry-run is non-destructive; cleanup removes only expired owned entries and retains unknown or linked data', async t => { const { store, add, a, b, c, root } = await fixture(t); const old = await add(a, 4, 10_000); diff --git a/packages/code/src/snapshot-lifecycle.ts b/packages/code/src/snapshot-lifecycle.ts index dfa5926e..66ed39dd 100644 --- a/packages/code/src/snapshot-lifecycle.ts +++ b/packages/code/src/snapshot-lifecycle.ts @@ -63,7 +63,11 @@ export function parseEnvironmentStorage( throw new Error('Invalid environment storage policy'); return policy as EnvironmentStoragePolicy; } -/** Soft admission for managed preparation, not a reservation or arbitrary-write quota. */ +/** + * Soft admission for managed preparation, not a reservation or arbitrary-write + * quota. Below the floor, `reclaim` (when supplied) may free reproducible + * storage once before the space is measured again. + */ export async function assertPreparationSpace( root: string, policy: EnvironmentStoragePolicy, @@ -71,14 +75,18 @@ export async function assertPreparationSpace( const status = await statfs(path, { bigint: true }); return status.bavail * status.bsize; }, + reclaim?: () => Promise, ): Promise { - if ( - (await available(root)) < - BigInt(policy.minFreeBytes) + BigInt(policy.setupReserveBytes) - ) - throw new Error( - 'Managed environment preparation deferred: free disk is below the configured floor plus setup reserve. Clean reproducible artifacts or expand storage; no setup command was started.', - ); + const floor = + BigInt(policy.minFreeBytes) + BigInt(policy.setupReserveBytes); + if ((await available(root)) >= floor) return; + if (reclaim) { + await reclaim(); + if ((await available(root)) >= floor) return; + } + throw new Error( + 'Managed environment preparation deferred: free disk is below the configured floor plus setup reserve. Clean reproducible artifacts or expand storage; no setup command was started.', + ); } export const SNAPSHOT_MANIFEST = '.snapshot.json'; diff --git a/packages/code/src/worktree-retirement.test.ts b/packages/code/src/worktree-retirement.test.ts new file mode 100644 index 00000000..f10ed8f3 --- /dev/null +++ b/packages/code/src/worktree-retirement.test.ts @@ -0,0 +1,592 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rename, rm, stat, utimes, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; +import { + WorktreeRetirementScheduler, + describeWorktreeRetirement, + retireStaleWorktrees, + worktreeRetirementSettings, +} from './worktree-retirement.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorktreeRetirementOptions } from './worktree-retirement.js'; + +const execFileAsync = promisify(execFile); +const DAY_MS = 24 * 60 * 60 * 1000; + +async function git(cwd: string, ...args: string[]): Promise { + const { stdout } = await execFileAsync( + 'git', + ['-c', 'user.name=test', '-c', 'user.email=test@example.com', '-c', 'commit.gpgsign=false', ...args], + { cwd }, + ); + return stdout.trim(); +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +/** A checkout on `main` with an `origin` bare remote and an ignored `node_modules`. */ +async function repository(t: test.TestContext): Promise { + const parent = await realpath(await mkdtemp(join(tmpdir(), 'worktree-retirement-'))); + t.after(() => rm(parent, { recursive: true, force: true })); + const remote = join(parent, 'remote.git'); + await git(parent, 'init', '-q', '--bare', remote); + const root = join(parent, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'README.md'), 'root\n'); + await writeFile(join(root, '.gitignore'), 'node_modules/\n.worktrees/\n'); + await git(root, 'add', '.'); + await git(root, 'commit', '-q', '-m', 'init'); + await git(root, 'remote', 'add', 'origin', remote); + await git(root, 'push', '-q', 'origin', 'main'); + await mkdir(join(root, '.worktrees')); + return root; +} + +/** A task worktree with one commit of its own and ignored dependencies. */ +async function worktree(root: string, name: string, push = true): Promise { + await git(root, 'worktree', 'add', '-q', '-b', name, `.worktrees/${name}`); + const path = join(root, '.worktrees', name); + await writeFile(join(path, `${name}.txt`), `${name}\n`); + await git(path, 'add', '.'); + await git(path, 'commit', '-q', '-m', name); + if (push) await git(path, 'push', '-q', 'origin', name); + await mkdir(join(path, 'node_modules', 'dependency'), { recursive: true }); + await writeFile(join(path, 'node_modules', 'dependency', 'index.js'), 'module.exports = 1;\n'); + return path; +} + +/** Backdate every on-disk activity signal, as if nobody had touched the worktree for `days`. */ +async function age(root: string, name: string, days = 30): Promise { + const when = new Date(Date.now() - days * DAY_MS); + const metadata = join(root, '.git', 'worktrees', name); + const paths = [ + join(root, '.worktrees', name), + metadata, + ...['HEAD', 'index', join('logs', 'HEAD'), 'ORIG_HEAD', 'FETCH_HEAD'].map((path) => join(metadata, path)), + ]; + for (const path of paths) { + await utimes(path, when, when).catch(() => undefined); + } +} + +function sources(root: string): WorktreeRetirementOptions['sources'] { + return [{ workspaceId: 'repo', root }]; +} + +/** A command pool whose lane commands wait until released. */ +function blockingPool(): { + pool: NativeWorkspaceCommandPool; + started: Promise; + finish: () => void; +} { + let started!: () => void; + let finish!: () => void; + const startedPromise = new Promise((resolve) => { started = resolve; }); + const finished = new Promise((resolve) => { finish = resolve; }); + const pool = { + async registerRoot() {}, + async unregisterRoot() {}, + async execute(request: { workspaceId: string }) { + started(); + await finished; + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + } as unknown as NativeWorkspaceCommandPool; + return { pool, started: startedPromise, finish }; +} + +async function laneTools(root: string, pool?: NativeWorkspaceCommandPool): Promise { + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + return new LinkedWorktreeWorkspaceTools({ + commandPool: pool, + delegate, + sources: new Map([ + ['repo', { + root, + command: { workspaceRoot: root } as NativeProcessSandboxOptions, + repositoryInstructions: false, + writable: true, + }], + ]), + }); +} + +const laneCommand = (worktree: string) => ({ + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree, + command: 'true', +}); + +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test('retires a clean, pushed, idle worktree and keeps its branch for recovery', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'done'); + await age(root, 'done'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:done']); + assert.deepEqual(summary.kept, {}); + assert.equal(await exists(path), false, 'the worktree and its ignored files are gone'); + assert.equal(await exists(join(root, '.git', 'worktrees', 'done')), false); + assert.doesNotMatch(await git(root, 'worktree', 'list', '--porcelain'), /\.worktrees\/done/); + const branch = await git(root, 'rev-parse', '--verify', 'refs/heads/done'); + assert.equal(branch, await git(root, 'rev-parse', '--verify', 'refs/remotes/origin/done')); + + await git(root, 'worktree', 'add', '-q', '.worktrees/done', 'done'); + assert.equal(await readFile(join(path, 'done.txt'), 'utf8'), 'done\n'); + assert.equal(await git(root, 'status', '--porcelain'), '', 'the main checkout is untouched'); + assert.equal(await git(root, 'branch', '--show-current'), 'main'); +}); + +test('keeps every worktree that could lose work or is not a linked worktree', async (t) => { + const root = await repository(t); + const worktrees = join(root, '.worktrees'); + + const dirty = await worktree(root, 'dirty'); + await writeFile(join(dirty, 'dirty.txt'), 'changed\n'); + const untracked = await worktree(root, 'untracked'); + await writeFile(join(untracked, 'notes.txt'), 'not yet added\n'); + await worktree(root, 'unpushed', false); + await worktree(root, 'locked'); + await git(root, 'worktree', 'lock', '.worktrees/locked'); + const merging = await worktree(root, 'merging'); + await worktree(root, 'side'); + await git(merging, 'merge', '-q', '--no-commit', '--no-ff', 'side'); + await worktree(root, 'recent'); + const detached = await worktree(root, 'detached'); + await git(detached, 'checkout', '-q', '--detach'); + await writeFile(join(detached, 'local.txt'), 'local\n'); + await git(detached, 'add', '.'); + await git(detached, 'commit', '-q', '-m', 'detached and unpushed'); + await mkdir(join(worktrees, 'forged')); + await writeFile(join(worktrees, 'forged', '.git'), `gitdir: ${join(root, '.git', 'worktrees', 'side')}\n`); + await mkdir(join(worktrees, 'plain')); + for (const name of ['dirty', 'untracked', 'unpushed', 'locked', 'merging', 'side', 'detached']) { + await age(root, name); + } + await age(root, 'forged'); + await age(root, 'plain'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:side']); + assert.deepEqual(summary.kept, { + dirty: 2, + unpushed: 2, + locked: 1, + operation: 1, + recent: 1, + unverified: 2, + }); + for (const name of ['dirty', 'untracked', 'unpushed', 'locked', 'merging', 'recent', 'detached', 'forged', 'plain']) { + assert.ok(await exists(join(worktrees, name)), `${name} is kept`); + } + assert.equal(await readFile(join(dirty, 'dirty.txt'), 'utf8'), 'changed\n'); + assert.equal(await readFile(join(untracked, 'notes.txt'), 'utf8'), 'not yet added\n'); + assert.ok(await exists(join(root, '.git', 'worktrees', 'merging', 'MERGE_HEAD'))); + assert.equal(await git(root, 'status', '--porcelain'), ''); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'root\n'); +}); + +test('a detached HEAD is retired only when a remote-tracking ref contains it', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'review'); + await git(path, 'checkout', '-q', '--detach'); + await age(root, 'review'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:review']); + assert.equal(await exists(path), false); +}); + +test('a squash- or rebase-merged branch whose remote branch was deleted is retired; unmerged work is kept', async (t) => { + const root = await repository(t); + const parent = join(root, '..'); + const tracked = async (name: string): Promise => { + const path = await worktree(root, name); + await git(path, 'branch', '-q', '--set-upstream-to', `origin/${name}`); + return path; + }; + const squashed = await tracked('squashed'); + await writeFile(join(squashed, 'second.txt'), 'second\n'); + await git(squashed, 'add', '.'); + await git(squashed, 'commit', '-q', '-m', 'second'); + await git(squashed, 'push', '-q', 'origin', 'squashed'); + await tracked('rebased'); + await tracked('abandoned'); + const review = 'review-16677-1f35df8'; + await git(root, 'worktree', 'add', '-q', '--detach', `.worktrees/${review}`, await git(squashed, 'rev-parse', 'HEAD')); + await git(root, 'worktree', 'add', '-q', '-b', 'ahead', '.worktrees/ahead'); + const ahead = join(root, '.worktrees', 'ahead'); + await git(ahead, 'push', '-q', '-u', 'origin', 'ahead'); + await writeFile(join(ahead, 'later.txt'), 'later\n'); + await git(ahead, 'add', '.'); + await git(ahead, 'commit', '-q', '-m', 'later, also landed on main but never pushed here'); + + const hosting = join(parent, 'hosting'); + await git(parent, 'clone', '-q', '-b', 'main', join(parent, 'remote.git'), hosting); + await git(hosting, 'merge', '-q', '--squash', 'origin/squashed'); + await git(hosting, 'commit', '-q', '-m', 'squash merge'); + await git(hosting, 'cherry-pick', 'origin/rebased'); + await writeFile(join(hosting, 'later.txt'), 'later\n'); + await git(hosting, 'add', '.'); + await git(hosting, 'commit', '-q', '-m', 'later'); + await git(hosting, 'push', '-q', 'origin', 'main'); + await git(hosting, 'push', '-q', 'origin', '--delete', 'squashed', 'rebased', 'abandoned'); + await git(root, 'fetch', '-q', '--prune', 'origin'); + for (const name of ['squashed', 'rebased', 'abandoned', review, 'ahead']) await age(root, name); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual([...summary.retired].sort(), ['repo:rebased', `repo:${review}`, 'repo:squashed']); + assert.deepEqual(summary.kept, { unpushed: 2 }); + assert.ok(await exists(join(root, '.worktrees', 'abandoned')), 'unmerged work with a deleted remote is kept'); + assert.ok(await exists(ahead), 'commits beyond a live upstream are kept even when main has the same change'); + for (const branch of ['squashed', 'rebased', 'abandoned', 'ahead']) { + await git(root, 'rev-parse', '--verify', `refs/heads/${branch}`); + } +}); + +test('never touches the main checkout or worktrees outside .worktrees', async (t) => { + const root = await repository(t); + const outside = join(root, '..', 'outside'); + await git(root, 'worktree', 'add', '-q', '-b', 'outside', outside); + await git(outside, 'push', '-q', 'origin', 'outside'); + const nested = join(root, 'nested'); + await git(root, 'worktree', 'add', '-q', '-b', 'nested', nested); + await git(nested, 'push', '-q', 'origin', 'nested'); + const old = new Date(Date.now() - 30 * DAY_MS); + for (const path of [root, join(root, '.git'), join(root, '.git', 'HEAD'), join(root, '.git', 'index'), outside, nested]) { + await utimes(path, old, old); + } + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, []); + assert.ok(await exists(join(root, 'README.md'))); + assert.ok(await exists(outside)); + assert.ok(await exists(nested)); + assert.equal((await git(root, 'worktree', 'list', '--porcelain')).match(/^worktree /gm)?.length, 3); +}); + +test('this worker\'s own lane use counts as activity, and an active lane is never retired', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'busy'); + await age(root, 'busy'); + const { pool, started, finish } = blockingPool(); + const tools = await laneTools(root, pool); + + const running = tools.execute(laneCommand('busy')); + await started; + await pause(5); + const during = await retireStaleWorktrees({ sources: sources(root), activity: tools, idleMs: 1 }); + assert.deepEqual(during.kept, { active: 1 }); + assert.ok(await exists(path)); + + finish(); + await running; + const recent = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(recent.kept, { recent: 1 }, 'the default threshold counts this process\'s last use'); + assert.ok(await exists(path)); + + await pause(5); + const after = await retireStaleWorktrees({ sources: sources(root), activity: tools, idleMs: 1 }); + assert.deepEqual(after.retired, ['repo:busy']); + assert.equal(await exists(path), false); + await assert.rejects( + tools.execute(laneCommand('busy')), + (error: unknown) => error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); +}); + +test('a lane request that arrives during retirement waits, then finds the lane gone', async (t) => { + const root = await repository(t); + await worktree(root, 'leaving'); + let executions = 0; + const pool = { + async registerRoot() {}, + async unregisterRoot() {}, + async execute() { + executions += 1; + throw new Error('a retired lane must not run commands'); + }, + } as unknown as NativeWorkspaceCommandPool; + const tools = await laneTools(root, pool); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + + const retiring = tools.whileIdle('repo', 'leaving', async () => { + await gate; + await git(root, 'worktree', 'remove', '.worktrees/leaving'); + return 'removed'; + }); + let settled = false; + const request = tools.execute(laneCommand('leaving')); + request.catch(() => undefined).finally(() => { settled = true; }); + let checkoutSettled = false; + const checkout = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + checkout.finally(() => { checkoutSettled = true; }); + const cancelled = new AbortController(); + const abandoned = tools.execute(laneCommand('leaving'), cancelled.signal); + await pause(20); + assert.equal(settled, false, 'the lane request waits for retirement'); + assert.equal(checkoutSettled, false, 'a checkout request waits too: it can reach the lane being removed'); + assert.deepEqual(await tools.whileIdle('repo', 'leaving', async () => 'again'), { ran: false }); + cancelled.abort(new Error('caller gave up')); + await assert.rejects(abandoned, /caller gave up/, 'cancellation ends the wait without the retirement'); + + release(); + assert.deepEqual(await retiring, { ran: true, value: 'removed' }); + await assert.rejects( + request, + (error: unknown) => error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); + const read = await checkout; + assert.equal(read.operation === 'read_file' && read.content.trimEnd(), 'root'); + assert.equal(executions, 0); +}); + +test('retirement never prunes other registered worktrees that are temporarily missing', async (t) => { + const root = await repository(t); + const external = join(root, '..', 'external'); + await git(root, 'worktree', 'add', '-q', '-b', 'external', external); + await rename(external, `${external}.unmounted`); + await worktree(root, 'done'); + await age(root, 'done'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:done']); + assert.ok(await exists(join(root, '.git', 'worktrees', 'external')), 'the unavailable worktree stays registered'); + assert.equal(await exists(join(root, '.git', 'worktrees', 'done')), false, 'removal deletes its own metadata'); + await rename(`${external}.unmounted`, external); + assert.equal(await git(external, 'branch', '--show-current'), 'external'); +}); + +test('a checkout request in flight defers retirement of its lanes', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'quiet'); + await age(root, 'quiet'); + let release!: () => void; + let entered!: () => void; + const inside = new Promise((resolve) => { entered = resolve; }); + const gate = new Promise((resolve) => { release = resolve; }); + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + const tools = new LinkedWorktreeWorkspaceTools({ + delegate: { + capabilities: delegate.capabilities, + async execute(request, signal) { + entered(); + await gate; + return await delegate.execute(request, signal); + }, + }, + sources: new Map([['repo', { root, repositoryInstructions: false, writable: true }]]), + }); + + const reading = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + await inside; + const during = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(during.kept, { active: 1 }); + release(); + await reading; + assert.ok(await exists(path)); + + const after = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(after.retired, ['repo:quiet']); +}); + +test('a checkout request that runs during inspection defers retirement', async (t) => { + const root = await repository(t); + await worktree(root, 'touched'); + await age(root, 'touched'); + const tools = await laneTools(root); + const before = tools.checkoutActivity('repo'); + await tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + assert.equal(tools.checkoutActivity('repo'), before + 2, 'a checkout request counts at start and finish'); + + let reads = 0; + const duringInspection = { + lastUsed: (workspaceId: string, name: string) => tools.lastUsed(workspaceId, name), + // The second read, under the reservation, sees a checkout request that ran during inspection. + checkoutActivity: () => reads++, + whileIdle: (workspaceId: string, name: string, task: () => Promise) => tools.whileIdle(workspaceId, name, task), + }; + const deferred = await retireStaleWorktrees({ sources: sources(root), activity: duringInspection }); + assert.deepEqual(deferred.kept, { changed: 1 }); + assert.ok(await exists(join(root, '.worktrees', 'touched'))); + + const quiet = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(quiet.retired, ['repo:touched']); +}); + +test('quarantined lanes and checkouts are left for the operator', async (t) => { + const root = await repository(t); + await worktree(root, 'held'); + await worktree(root, 'free'); + await age(root, 'held'); + await age(root, 'free'); + + const checkoutHeld = await retireStaleWorktrees({ + sources: sources(root), + isQuarantined: async (_workspaceId, worktree) => worktree == null, + }); + assert.deepEqual(checkoutHeld, { retired: [], kept: {}, freedBytes: 0, truncated: false }); + + const laneHeld = await retireStaleWorktrees({ + sources: sources(root), + isQuarantined: async (_workspaceId, worktree) => worktree === 'held', + }); + assert.deepEqual(laneHeld.retired, ['repo:free']); + assert.deepEqual(laneHeld.kept, { quarantined: 1 }); + assert.ok(await exists(join(root, '.worktrees', 'held'))); +}); + +test('a quarantine that appears after inspection is caught under the lane reservation', async (t) => { + const root = await repository(t); + await worktree(root, 'late'); + await age(root, 'late'); + for (const quarantinedLater of [undefined, 'late']) { + const asked: string[] = []; + const summary = await retireStaleWorktrees({ + sources: sources(root), + activity: await laneTools(root), + isQuarantined: async (_workspaceId, worktree) => { + asked.push(worktree ?? ''); + // Clear for the scan and inspection; quarantined once the lane is reserved. + return asked.length > 2 && worktree === quarantinedLater; + }, + }); + assert.deepEqual(summary.kept, { quarantined: 1 }); + assert.deepEqual(asked, ['', 'late', '', ...(quarantinedLater ? ['late'] : [])]); + assert.ok(await exists(join(root, '.worktrees', 'late'))); + } +}); + +test('one worktree failing does not stop the pass', async (t) => { + const root = await repository(t); + await worktree(root, 'broken'); + await worktree(root, 'fine'); + await rm(join(root, '.git', 'worktrees', 'broken', 'HEAD')); + await age(root, 'broken'); + await age(root, 'fine'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:fine']); + assert.equal(Object.values(summary.kept).reduce((total, count) => total + (count ?? 0), 0), 1); + assert.ok(await exists(join(root, '.worktrees', 'broken'))); +}); + +test('worktrees kept for lasting reasons cannot starve the rest of a backlog', async (t) => { + const root = await repository(t); + for (const [index, name] of ['first', 'second', 'third'].entries()) { + await worktree(root, name, false); + await age(root, name, 40 - index); + } + await worktree(root, 'newest'); + await age(root, 'newest', 10); + const rotation = new Set(); + const options = { sources: sources(root), rotation, limits: { inspect: 2 } }; + + const first = await retireStaleWorktrees(options); + assert.deepEqual(first.retired, []); + assert.deepEqual(first.kept, { unpushed: 2, deferred: 2 }); + assert.equal(first.truncated, true, 'uninspected worktrees remain in this rotation'); + + const second = await retireStaleWorktrees(options); + assert.deepEqual(second.retired, ['repo:newest']); + assert.deepEqual(second.kept, { unpushed: 1, deferred: 2 }); + assert.equal(second.truncated, false, 'the rotation is complete'); + + const third = await retireStaleWorktrees(options); + assert.deepEqual(third.kept, { unpushed: 2, deferred: 1 }, 'a new rotation starts from the oldest'); + assert.equal(third.truncated, true); +}); + +test('retirement is on by default and can be disabled or retuned', () => { + assert.deepEqual(worktreeRetirementSettings({ optOut: false }), { enabled: true, idleMs: 7 * DAY_MS }); + assert.equal(worktreeRetirementSettings({ optOut: true }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: false, enabled: 'FALSE' }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: true, enabled: 'true' }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: false, enabled: ' ' }).enabled, true); + assert.equal(worktreeRetirementSettings({ optOut: false, idleDays: '14' }).idleMs, 14 * DAY_MS); + assert.throws(() => worktreeRetirementSettings({ optOut: false, enabled: 'off' }), /must be true or false/); + for (const idleDays of ['0', '-1', '1.5', 'week', '3651']) { + assert.throws(() => worktreeRetirementSettings({ optOut: false, idleDays }), /IDLE_DAYS/); + } +}); + +test('scheduled passes never overlap and stop cleanly', async (t) => { + const root = await repository(t); + await worktree(root, 'scheduled'); + await age(root, 'scheduled'); + const messages: string[] = []; + const scheduler = new WorktreeRetirementScheduler({ + sources: sources(root), + startDelayMs: 60_000, + log: (level, message) => { + if (level === 'info') messages.push(message); + }, + }); + scheduler.start(); + + const first = scheduler.runNow(); + assert.equal(scheduler.runNow(), first, 'a pass requested while one runs joins it'); + const summary = await first; + assert.deepEqual(summary?.retired, ['repo:scheduled']); + assert.equal(messages.length, 1); + assert.match(messages[0]!, /^worktree retirement: retired 1, kept 0(, freed about .+)?$/); + + await scheduler.stop(); + assert.equal(await scheduler.runNow(), undefined, 'a stopped scheduler runs nothing'); +}); + +test('summaries count reasons and note a remaining backlog', () => { + assert.equal( + describeWorktreeRetirement({ + retired: ['repo:a', 'repo:b'], + kept: { unpushed: 2, dirty: 1, recent: 4 }, + freedBytes: 3 * 1024 ** 3, + truncated: true, + }), + 'worktree retirement: retired 2, kept 7 (dirty 1, recent 4, unpushed 2), freed about 3.0 GiB, more next pass', + ); +}); diff --git a/packages/code/src/worktree-retirement.ts b/packages/code/src/worktree-retirement.ts new file mode 100644 index 00000000..39a7b60c --- /dev/null +++ b/packages/code/src/worktree-retirement.ts @@ -0,0 +1,697 @@ +import { execFile } from 'node:child_process'; +import { lstat, opendir, statfs } from 'node:fs/promises'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { LINKED_WORKTREE_DIRECTORY, verifyLinkedWorktree } from './linked-worktrees.js'; +import { isValidLinkedWorktreeName } from './protocol.js'; + +import type { LinkedWorktreeActivity, VerifiedLinkedWorktree } from './linked-worktrees.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +const execFileAsync = promisify(execFile); + +const DAY_MS = 24 * 60 * 60 * 1000; +export const DEFAULT_WORKTREE_IDLE_DAYS = 7; +const MAX_WORKTREE_IDLE_DAYS = 3650; +export const WORKTREE_RETIREMENT_INTERVAL_MS = 6 * 60 * 60 * 1000; +const WORKTREE_RETIREMENT_START_DELAY_MS = 2 * 60 * 1000; +/** A pass that hit a bound continues soon instead of waiting a full interval. */ +const WORKTREE_RETIREMENT_CONTINUATION_MS = 5 * 60 * 1000; +/** + * Directory entries read beneath one checkout's `.worktrees` per pass: a + * memory bound far above any real checkout, so every pass sees every entry. + * Git inspection is what is bounded per pass, and rotation keeps it fair. + */ +const SCAN_LIMIT = 65_536; +/** Idle worktrees whose Git state is inspected per pass, oldest first. */ +const INSPECT_LIMIT = 128; +/** Worktrees removed per pass. */ +const RETIRE_LIMIT = 32; +const GIT_TIMEOUT_MS = 30_000; +/** + * Removal deletes ignored build output too, which can take a while. It has no + * timeout and ignores cancellation: a half-deleted worktree loses its `.git` + * file and can no longer be verified, retired or recognized. + */ +const GIT_REMOVE_NO_TIMEOUT = 0; +const GIT_OUTPUT_LIMIT = 64 * 1024; +/** Path and commit lists between a stale branch and its default branch can be long. */ +const GIT_LIST_LIMIT = 16 * 1024 * 1024; +/** Per-worktree Git state that marks a merge, rebase, cherry-pick, revert or bisect in progress. */ +const OPERATION_STATE = [ + 'MERGE_HEAD', + 'CHERRY_PICK_HEAD', + 'REVERT_HEAD', + 'BISECT_LOG', + 'BISECT_START', + 'rebase-merge', + 'rebase-apply', + 'sequencer', +]; +/** Per-worktree Git metadata that Git rewrites whenever the worktree is used. */ +const ACTIVITY_STATE = ['HEAD', 'index', join('logs', 'HEAD'), 'ORIG_HEAD', 'FETCH_HEAD']; + +export type WorktreeKeptReason = + | 'unverified' + | 'recent' + | 'quarantined' + | 'locked' + | 'operation' + | 'no-commit' + | 'dirty' + | 'unpushed' + | 'active' + | 'changed' + | 'failed' + | 'deferred'; + +export interface WorktreeRetirementSource { + workspaceId: string; + root: string; + identity?: WorkspaceRootIdentity; +} + +export interface WorktreeRetirementOptions { + sources: readonly WorktreeRetirementSource[]; + /** Lane use in this worker; without it, only on-disk activity counts. */ + activity?: LinkedWorktreeActivity; + idleMs?: number; + /** A quarantined checkout or lane awaits an operator and is never retired. */ + isQuarantined?: (workspaceId: string, worktree?: string) => Promise; + log?: (level: 'debug' | 'info', message: string) => void; + /** + * Idle worktrees already inspected in the current rotation. A pass inspects + * the others first and records what it inspects, so worktrees kept for a + * lasting reason cannot starve the rest of a backlog larger than one pass. + */ + rotation?: Set; + /** Per-pass bounds; the defaults suit production. */ + limits?: { inspect?: number; retire?: number }; +} + +export interface WorktreeRetirementSummary { + /** `:` for each retired worktree. */ + retired: string[]; + kept: Partial>; + /** Free-space gain on the checkouts' filesystems across the pass; other writers make it approximate. */ + freedBytes: number; + /** Idle worktrees remain that this rotation has not inspected yet. */ + truncated: boolean; +} + +export interface WorktreeRetirementSettings { + enabled: boolean; + idleMs: number; +} + +interface Candidate { + source: WorktreeRetirementSource; + name: string; + lane: VerifiedLinkedWorktree; + metadata: string; + /** Newest on-disk activity, re-read under the reservation to detect use during inspection. */ + diskActiveAt: number; + /** This worker's own last use of the lane, re-read under the reservation. */ + usedAt?: number; + /** Checkout request activity when inspection began, re-read under the reservation. */ + checkoutAt?: number; + /** Newest of on-disk activity and this worker's own lane use. */ + activeAt: number; +} + +type Kept = { kept: WorktreeKeptReason; detail?: string }; +type Verdict = Kept | { head: string; basis: string }; + +/** + * Resolve the operator's choice. Retirement is on unless explicitly disabled; + * malformed values fail startup rather than silently choosing either way. + */ +export function worktreeRetirementSettings(input: { + optOut: boolean; + enabled?: string; + idleDays?: string; +}): WorktreeRetirementSettings { + const enabled = input.enabled?.trim().toLowerCase() ?? ''; + if (enabled !== '' && enabled !== 'true' && enabled !== 'false') { + throw new Error('LIBRECHAT_CODE_WORKTREE_RETIREMENT must be true or false'); + } + const idleDays = input.idleDays?.trim() ?? ''; + const days = idleDays === '' ? DEFAULT_WORKTREE_IDLE_DAYS : Number(idleDays); + if (!Number.isSafeInteger(days) || days < 1 || days > MAX_WORKTREE_IDLE_DAYS) { + throw new Error( + `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS must be an integer between 1 and ${MAX_WORKTREE_IDLE_DAYS}`, + ); + } + return { enabled: !input.optOut && enabled !== 'false', idleMs: days * DAY_MS }; +} + +function gitEnvironment(): NodeJS.ProcessEnv { + return { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }; +} + +async function git( + cwd: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS, + maxBuffer = GIT_OUTPUT_LIMIT, +): Promise { + const { stdout } = await execFileAsync( + 'git', + ['--no-optional-locks', '-C', cwd, '-c', 'core.fsmonitor=false', ...args], + { encoding: 'utf8', env: gitEnvironment(), maxBuffer, signal, timeout }, + ); + return stdout; +} + +/** The commit a ref names, or undefined when it does not resolve. */ +async function resolveCommit(cwd: string, ref: string, signal?: AbortSignal): Promise { + try { + const commit = (await git(cwd, ['rev-parse', '--verify', '--quiet', `${ref}^{commit}`], signal)).trim(); + return /^[0-9a-f]{40,64}$/.test(commit) ? commit : undefined; + } catch { + signal?.throwIfAborted(); + return undefined; + } +} + +function pathList(output: string): string[] { + return output.split('\0').filter(Boolean); +} + +/** The remote's default branch as last fetched: its `HEAD` symref, else `main` or `master`. */ +async function remoteDefaultBranch( + checkout: string, + remote: string, + signal?: AbortSignal, +): Promise<{ ref: string; commit: string } | undefined> { + let symbolic: string | undefined; + try { + symbolic = (await git(checkout, ['symbolic-ref', '-q', `refs/remotes/${remote}/HEAD`], signal)).trim(); + } catch { + signal?.throwIfAborted(); + } + const refs = [ + ...(symbolic?.startsWith(`refs/remotes/${remote}/`) ? [symbolic] : []), + `refs/remotes/${remote}/main`, + `refs/remotes/${remote}/master`, + ]; + for (const ref of refs) { + const commit = await resolveCommit(checkout, ref, signal); + if (commit) return { ref: ref.replace(/^refs\/remotes\//, ''), commit }; + } + return undefined; +} + +/** + * Whether a HEAD that no remote-tracking ref contains is nonetheless already + * in its remote's default branch by content, as after a squash or rebase merge + * whose remote branch was then deleted. Only a HEAD without a live upstream + * qualifies: detached, never pushed, or whose upstream ref is gone. A live + * upstream that lacks HEAD means unpushed commits. Content counts when every + * commit beyond the default branch is patch-equivalent to one in it (and none + * is a merge, which could carry its own changes), or when the default branch + * has identical content at every path the branch changed since their merge + * base. Local refs only; nothing is fetched. + */ +async function mergedByContent( + lane: VerifiedLinkedWorktree, + head: string, + signal?: AbortSignal, +): Promise { + const checkout = lane.checkoutRoot; + let branch: string | undefined; + try { + branch = (await git(lane.root, ['symbolic-ref', '-q', 'HEAD'], signal)).trim() || undefined; + } catch { + signal?.throwIfAborted(); + } + let remote = 'origin'; + if (branch?.startsWith('refs/heads/')) { + const [upstream = '', upstreamRemote = ''] = ( + await git(checkout, ['for-each-ref', '--format=%(upstream)%00%(upstream:remotename)', branch], signal) + ) + .trim() + .split('\0'); + if (upstream) { + if (await resolveCommit(checkout, upstream, signal)) return undefined; + if (/^[A-Za-z0-9._-]+$/.test(upstreamRemote)) remote = upstreamRemote; + } + } + const target = await remoteDefaultBranch(checkout, remote, signal); + if (!target) return undefined; + const merges = (await git(checkout, ['rev-list', '--count', '--merges', `${target.commit}..${head}`], signal)).trim(); + if (merges === '0') { + const cherry = (await git(checkout, ['cherry', target.commit, head], signal, GIT_TIMEOUT_MS, GIT_LIST_LIMIT)) + .split('\n') + .filter(Boolean); + if (cherry.every((line) => line.startsWith('- '))) return `patch-equivalent to ${target.ref}`; + } + let base: string; + try { + base = (await git(checkout, ['merge-base', target.commit, head], signal)).trim(); + } catch { + signal?.throwIfAborted(); + return undefined; + } + const diff = (from: string): Promise => + git( + checkout, + ['diff', '--name-only', '-z', '--no-renames', '--no-ext-diff', '--no-textconv', from, head], + signal, + GIT_TIMEOUT_MS, + GIT_LIST_LIMIT, + ); + const touched = pathList(await diff(base)); + const differing = new Set(pathList(await diff(target.commit))); + return touched.every((path) => !differing.has(path)) ? `same content as ${target.ref}` : undefined; +} + +function errorDetail(error: unknown): string { + const stderr = (error as { stderr?: unknown }).stderr; + const text = typeof stderr === 'string' && stderr.trim() ? stderr : error instanceof Error ? error.message : String(error); + return text.trim().split('\n')[0]!.slice(0, 200); +} + +/** Absent is the only answer that clears a check; anything unreadable counts as present. */ +async function present(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ENOENT'; + } +} + +async function modifiedAt(path: string): Promise { + try { + return (await lstat(path)).mtimeMs; + } catch { + return 0; + } +} + +/** The newest on-disk sign of use: the worktree directory itself and its Git metadata. */ +async function lastActivity(root: string, metadata: string): Promise { + const times = await Promise.all( + [root, metadata, ...ACTIVITY_STATE.map((path) => join(metadata, path))].map(modifiedAt), + ); + return Math.max(...times); +} + +async function worktreeNames(checkout: string): Promise<{ names: string[]; truncated: boolean }> { + let directory; + try { + directory = await opendir(join(checkout, LINKED_WORKTREE_DIRECTORY)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { names: [], truncated: false }; + throw error; + } + const names: string[] = []; + try { + for await (const entry of directory) { + if (names.length >= SCAN_LIMIT) return { names, truncated: true }; + if (isValidLinkedWorktreeName(entry.name)) names.push(entry.name); + } + } finally { + await directory.close().catch(() => undefined); + } + return { names, truncated: false }; +} + +async function freeSpace(paths: readonly string[]): Promise> { + const space = new Map(); + for (const path of paths) { + try { + const device = String((await lstat(path)).dev); + if (space.has(device)) continue; + const status = await statfs(path, { bigint: true }); + space.set(device, Number(status.bavail * status.bsize)); + } catch { + // Space reporting is informational; it never decides a retirement. + } + } + return space; +} + +function describeChanges(status: string): string { + const entries = status.split('\0').filter(Boolean); + const untracked = entries.filter((entry) => entry.startsWith('?? ')).length; + return untracked === entries.length + ? 'untracked files' + : untracked > 0 + ? 'tracked changes and untracked files' + : 'tracked changes'; +} + +function formatBytes(bytes: number): string { + const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit += 1; + } + return `${unit === 0 ? value : value.toFixed(1)} ${units[unit]}`; +} + +export function describeWorktreeRetirement(summary: WorktreeRetirementSummary): string { + const kept = Object.entries(summary.kept) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([reason, count]) => `${reason} ${count}`); + const keptTotal = Object.values(summary.kept).reduce((total, count) => total + (count ?? 0), 0); + return [ + `worktree retirement: retired ${summary.retired.length}`, + `kept ${keptTotal}${kept.length ? ` (${kept.join(', ')})` : ''}`, + ...(summary.freedBytes > 0 ? [`freed about ${formatBytes(summary.freedBytes)}`] : []), + ...(summary.truncated ? ['more next pass'] : []), + ].join(', '); +} + +/** Everything short of the worker's own activity that could make removal lose work. */ +async function inspect( + candidate: Candidate, + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const { lane, metadata, name, source } = candidate; + try { + if (await options.isQuarantined?.(source.workspaceId, name)) return { kept: 'quarantined' }; + } catch { + return { kept: 'quarantined', detail: 'quarantine state is unreadable' }; + } + if (await present(join(metadata, 'locked'))) return { kept: 'locked' }; + for (const state of OPERATION_STATE) { + if (await present(join(metadata, state))) return { kept: 'operation', detail: state }; + } + let head: string; + try { + head = (await git(lane.root, ['rev-parse', '--verify', '--quiet', 'HEAD^{commit}'], signal)).trim(); + } catch { + signal?.throwIfAborted(); + return { kept: 'no-commit' }; + } + if (!/^[0-9a-f]{40,64}$/.test(head)) return { kept: 'no-commit' }; + let status: string; + try { + status = await git( + lane.root, + ['status', '--porcelain', '-z', '--untracked-files=normal', '--ignore-submodules=none'], + signal, + ); + } catch (error) { + signal?.throwIfAborted(); + if ((error as NodeJS.ErrnoException).code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { + return { kept: 'dirty', detail: 'too many changes to list' }; + } + return { kept: 'failed', detail: `status: ${errorDetail(error)}` }; + } + if (status.length > 0) return { kept: 'dirty', detail: describeChanges(status) }; + let remote: string; + try { + remote = ( + await git( + lane.checkoutRoot, + ['for-each-ref', '--count=1', '--contains', head, '--format=%(refname)', 'refs/remotes/'], + signal, + ) + ).trim(); + } catch (error) { + signal?.throwIfAborted(); + return { kept: 'failed', detail: `remote containment: ${errorDetail(error)}` }; + } + if (remote) return { head, basis: `contained in ${remote.replace(/^refs\/remotes\//, '')}` }; + let merged: string | undefined; + try { + merged = await mergedByContent(lane, head, signal); + } catch (error) { + signal?.throwIfAborted(); + return { kept: 'failed', detail: `merged content: ${errorDetail(error)}` }; + } + return merged + ? { head, basis: merged } + : { kept: 'unpushed', detail: 'not on a remote-tracking ref or merged into the default branch' }; +} + +/** Re-checked under the lane reservation: anything that moved since inspection defers to the next pass. */ +async function remove( + candidate: Candidate, + head: string, + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const { lane, metadata, name, source } = candidate; + if (options.activity?.lastUsed(source.workspaceId, name) !== candidate.usedAt) { + return { kept: 'changed', detail: 'used during inspection' }; + } + // A checkout request can reach `.worktrees/*`, including ignored files that `git status` never reports. + if (options.activity?.checkoutActivity(source.workspaceId) !== candidate.checkoutAt) { + return { kept: 'changed', detail: 'checkout used during inspection' }; + } + // A request that finished between inspection and this reservation may have quarantined either. + try { + if ( + (await options.isQuarantined?.(source.workspaceId)) || + (await options.isQuarantined?.(source.workspaceId, name)) + ) { + return { kept: 'quarantined' }; + } + } catch { + return { kept: 'quarantined', detail: 'quarantine state is unreadable' }; + } + let current: VerifiedLinkedWorktree; + try { + current = await verifyLinkedWorktree(source.root, name, source.identity); + } catch { + return { kept: 'changed', detail: 'no longer a verified linked worktree' }; + } + if (current.identity.dev !== lane.identity.dev || current.identity.ino !== lane.identity.ino) { + return { kept: 'changed', detail: 'worktree directory was replaced' }; + } + if ((await lastActivity(lane.root, metadata)) !== candidate.diskActiveAt) { + return { kept: 'changed', detail: 'used during inspection' }; + } + if (await present(join(metadata, 'locked'))) return { kept: 'locked' }; + try { + const now = (await git(lane.root, ['rev-parse', '--verify', '--quiet', 'HEAD^{commit}'], signal)).trim(); + if (now !== head) return { kept: 'changed', detail: 'HEAD moved during inspection' }; + } catch { + signal?.throwIfAborted(); + return { kept: 'changed', detail: 'HEAD is unreadable' }; + } + signal?.throwIfAborted(); + try { + // Never --force: Git re-checks for changes and untracked files itself and refuses a dirty or locked worktree. + await git(lane.checkoutRoot, ['worktree', 'remove', lane.root], undefined, GIT_REMOVE_NO_TIMEOUT); + } catch (error) { + return { kept: 'failed', detail: `remove: ${errorDetail(error)}` }; + } + return 'retired'; +} + +/** + * Retire linked worktrees beneath each checkout's `.worktrees` that are + * verified, idle, clean, free of in-progress operations and locks, and whose + * HEAD is contained in a remote-tracking ref or already merged into the + * remote's default branch by content. Removal is `git worktree remove` + * without `--force`, which also deletes that worktree's own metadata; no + * repository-wide `git worktree prune` runs, since it would also expire other + * registered worktrees that are only temporarily unavailable. Branches are + * kept, so `git worktree add` restores any of them. Each worktree is judged + * independently; one failure never ends the pass. + */ +export async function retireStaleWorktrees( + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const now = Date.now(); + const idleMs = options.idleMs ?? DEFAULT_WORKTREE_IDLE_DAYS * DAY_MS; + const summary: WorktreeRetirementSummary = { retired: [], kept: {}, freedBytes: 0, truncated: false }; + const keep = (source: WorktreeRetirementSource, name: string, reason: WorktreeKeptReason, detail?: string): void => { + summary.kept[reason] = (summary.kept[reason] ?? 0) + 1; + options.log?.('debug', `worktree retirement kept ${source.workspaceId}:${name}: ${reason}${detail ? ` (${detail})` : ''}`); + }; + const candidates: Candidate[] = []; + for (const source of options.sources) { + signal?.throwIfAborted(); + try { + if (await options.isQuarantined?.(source.workspaceId)) { + options.log?.('debug', `worktree retirement skipped ${source.workspaceId}: checkout is quarantined`); + continue; + } + const { names, truncated } = await worktreeNames(source.root); + if (truncated) { + options.log?.('debug', `worktree retirement read only the first ${SCAN_LIMIT} entries of ${source.workspaceId}`); + } + for (const name of names) { + signal?.throwIfAborted(); + let lane: VerifiedLinkedWorktree; + try { + lane = await verifyLinkedWorktree(source.root, name, source.identity); + } catch (error) { + keep(source, name, 'unverified', errorDetail(error)); + continue; + } + const metadata = join(lane.commonGitDir, 'worktrees', name); + const diskActiveAt = await lastActivity(lane.root, metadata); + const usedAt = options.activity?.lastUsed(source.workspaceId, name); + const activeAt = Math.max(diskActiveAt, usedAt ?? 0); + if (now - activeAt < idleMs) { + keep(source, name, 'recent'); + continue; + } + candidates.push({ source, name, lane, metadata, diskActiveAt, usedAt, activeAt }); + } + } catch (error) { + signal?.throwIfAborted(); + options.log?.('debug', `worktree retirement skipped ${source.workspaceId}: ${errorDetail(error)}`); + } + } + if (candidates.length === 0) { + options.rotation?.clear(); + return summary; + } + const { rotation } = options; + const key = (candidate: Candidate): string => `${candidate.source.workspaceId}\0${candidate.name}`; + if (rotation) { + const current = new Set(candidates.map(key)); + for (const entry of rotation) { + if (!current.has(entry)) rotation.delete(entry); + } + if (rotation.size === current.size) rotation.clear(); + } + const visited = (candidate: Candidate): number => (rotation?.has(key(candidate)) ? 1 : 0); + candidates.sort((a, b) => visited(a) - visited(b) || a.activeAt - b.activeAt); + const checkouts = [...new Set(candidates.map((candidate) => candidate.lane.checkoutRoot))]; + const before = await freeSpace(checkouts); + const retiredCheckouts = new Set(); + let inspected = 0; + for (const candidate of candidates) { + signal?.throwIfAborted(); + const { source, name } = candidate; + if ( + inspected >= (options.limits?.inspect ?? INSPECT_LIMIT) || + summary.retired.length >= (options.limits?.retire ?? RETIRE_LIMIT) + ) { + summary.truncated ||= visited(candidate) === 0; + keep(source, name, 'deferred'); + continue; + } + inspected += 1; + rotation?.add(key(candidate)); + try { + candidate.checkoutAt = options.activity?.checkoutActivity(source.workspaceId); + const verdict = await inspect(candidate, options, signal); + if ('kept' in verdict) { + keep(source, name, verdict.kept, verdict.detail); + continue; + } + const retire = (): Promise => remove(candidate, verdict.head, options, signal); + const reserved = options.activity + ? await options.activity.whileIdle(source.workspaceId, name, retire) + : { ran: true as const, value: await retire() }; + if (!reserved.ran) { + keep(source, name, 'active'); + continue; + } + if (reserved.value !== 'retired') { + keep(source, name, reserved.value.kept, reserved.value.detail); + continue; + } + summary.retired.push(`${source.workspaceId}:${name}`); + retiredCheckouts.add(candidate.lane.checkoutRoot); + options.log?.('debug', `worktree retirement retired ${source.workspaceId}:${name} (${verdict.basis}); its branch is kept`); + } catch (error) { + signal?.throwIfAborted(); + keep(source, name, 'failed', errorDetail(error)); + } + } + if (retiredCheckouts.size > 0) { + const after = await freeSpace(checkouts); + for (const [device, free] of after) { + summary.freedBytes += Math.max(0, free - (before.get(device) ?? free)); + } + } + return summary; +} + +export interface WorktreeRetirementSchedulerOptions extends WorktreeRetirementOptions { + intervalMs?: number; + startDelayMs?: number; + continuationMs?: number; +} + +/** + * Runs retirement passes in the background: once shortly after start, then + * every interval, sooner while a backlog remains. Passes never overlap; a pass + * requested while one runs joins it. + */ +export class WorktreeRetirementScheduler { + private readonly controller = new AbortController(); + private readonly rotation = new Set(); + private running?: Promise; + private timer?: NodeJS.Timeout; + private started = false; + + constructor(private readonly options: WorktreeRetirementSchedulerOptions) {} + + start(): void { + if (this.started || this.controller.signal.aborted) return; + this.started = true; + if (this.running == null) this.schedule(this.options.startDelayMs ?? WORKTREE_RETIREMENT_START_DELAY_MS); + } + + /** Run a pass now, or join the one already running. */ + runNow(): Promise { + if (this.controller.signal.aborted) return Promise.resolve(undefined); + this.running ??= this.pass().finally(() => { + this.running = undefined; + }); + return this.running; + } + + /** Stop scheduling; a removal already under way finishes first. */ + async stop(): Promise { + this.controller.abort(); + clearTimeout(this.timer); + await this.running; + } + + private schedule(delayMs: number): void { + clearTimeout(this.timer); + if (!this.started || this.controller.signal.aborted) return; + this.timer = setTimeout(() => void this.runNow(), delayMs); + this.timer.unref(); + } + + private async pass(): Promise { + clearTimeout(this.timer); + let summary: WorktreeRetirementSummary | undefined; + try { + summary = await retireStaleWorktrees( + { ...this.options, rotation: this.rotation }, + this.controller.signal, + ); + this.options.log?.('info', describeWorktreeRetirement(summary)); + } catch (error) { + if (!this.controller.signal.aborted) { + this.options.log?.('info', `worktree retirement pass failed: ${errorDetail(error)}`); + } + } + this.schedule( + summary?.truncated + ? (this.options.continuationMs ?? WORKTREE_RETIREMENT_CONTINUATION_MS) + : (this.options.intervalMs ?? WORKTREE_RETIREMENT_INTERVAL_MS), + ); + return summary; + } +} From d6ac1b63b82c908db74c7c38f8358d8972d0c4fc Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 22:30:23 -0400 Subject: [PATCH 62/75] fix: Report Missing Workspace Paths as NOT_FOUND and Create Missing Parents on Write (#301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧭 fix: Report Missing Workspace Paths as NOT_FOUND and Create Missing Parents on Write A missing file, directory or search scope no longer reports INVALID_PATH. When every existing ancestor resolves to a directory inside the workspace, the worker returns NOT_FOUND ("Workspace path does not exist"); escapes, symlinks, files used as directories and other path-safety failures keep INVALID_PATH. write_file creates missing parent directories beneath the workspace or lane root, one verified level at a time, never through a symlink or an existing file, and removes the directories it created if the write then fails. Code API advertises the added code in registration and maps it to HTTP 422 like INVALID_PATH. A worker reports NOT_FOUND only to a Code API that advertised it, and resends a refused settlement with INVALID_PATH, so older Code API replicas keep accepting settlements. * fix: Anchor Parent Creation to a Held Root, Sync It, and Surface Cleanup Failure Parent directories are created only under a held root descriptor, where mkdirat cannot follow a swapped path out of the root. A pathname-only root creates nothing and reports the missing parent as NOT_FOUND. Each created directory is synced into its parent before the write is acknowledged, and a rejected write that cannot remove a directory it created now reports a possible mutation instead of an atomic failure. * fix: Recheck Ancestors Before NOT_FOUND and Sync Parent Cleanup A miss is reported as NOT_FOUND only when the walk reached it through real directories that still have the same identity afterwards; any symlink in the chain stays INVALID_PATH. Cleanup syncs each removal into its parent and treats any directory it could not durably remove as a possible mutation. * fix: Keep Created Parents as a Committed Side Effect Directories a write creates are no longer rolled back. Once one exists the write finishes despite cancellation, and any later failure is reported as a possible mutation. A raced EEXIST parent is synced too, and the NOT_FOUND ancestry recheck now includes the workspace root. --- packages/code/src/linked-worktrees.test.ts | 53 +++++ packages/code/src/protocol.test.ts | 13 ++ packages/code/src/protocol.ts | 13 ++ packages/code/src/root-access.ts | 12 +- packages/code/src/worker.ts | 31 ++- packages/code/src/workspace-worker.test.ts | 106 ++++++++++ packages/code/src/workspace.test.ts | 231 ++++++++++++++++++++- packages/code/src/workspace.ts | 206 ++++++++++++++++-- service/src/bridge/router.test.ts | 1 + service/src/bridge/router.ts | 1 + service/src/workspace-tools/router.test.ts | 2 + 11 files changed, 646 insertions(+), 23 deletions(-) diff --git a/packages/code/src/linked-worktrees.test.ts b/packages/code/src/linked-worktrees.test.ts index 53ec156e..30c5b7e6 100644 --- a/packages/code/src/linked-worktrees.test.ts +++ b/packages/code/src/linked-worktrees.test.ts @@ -225,6 +225,59 @@ test('lane file tools are confined to the worktree and report the public workspa ); }); +test('lane writes create missing parents inside the worktree and report missing files', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const tools = await laneTools(root); + const lane = join(root, '.worktrees', 'task-a'); + + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'packages/new-package/package.json', + content: '{}\n', + }); + assert.equal(await readFile(join(lane, 'packages', 'new-package', 'package.json'), 'utf8'), '{}\n'); + await assert.rejects(stat(join(root, 'packages'))); + + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + path: '.checks/tests.log', + }), + 'NOT_FOUND', + ); + + await symlink(root, join(lane, 'checkout-link')); + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'checkout-link/escaped/notes.txt', + content: 'blocked', + }), + 'INVALID_PATH', + ); + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'checkout-link/missing.txt', + }), + 'INVALID_PATH', + ); + await assert.rejects(stat(join(root, 'escaped'))); +}); + test('lane commands register a confined root once, whatever siblings come and go', async (t) => { const { parent, root } = await checkout(); t.after(() => rm(parent, { recursive: true, force: true })); diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 340ddfdf..25c00cf3 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -8,8 +8,10 @@ import { isSupportedBridgeArtifactName, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isWorkspaceToolErrorCode, isWorkspaceToolRequest, isWorkspaceToolResult, + WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, workspaceIsolationKey, workspaceIsolationKeysConflict, workspaceIsolationParent, @@ -19,6 +21,17 @@ import type { WorkspacePreviewEditRequest, } from './protocol.js'; +test('every added workspace error code falls back to a legacy code', () => { + assert.equal(isWorkspaceToolErrorCode('NOT_FOUND'), true); + assert.equal(isWorkspaceToolErrorCode('MISSING'), false); + for (const [code, legacy] of Object.entries(WORKSPACE_TOOL_ERROR_CODE_FALLBACKS)) { + assert.equal(isWorkspaceToolErrorCode(code), true); + assert.equal(isWorkspaceToolErrorCode(legacy), true); + assert.equal(legacy in WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, false); + } + assert.equal(WORKSPACE_TOOL_ERROR_CODE_FALLBACKS.NOT_FOUND, 'INVALID_PATH'); +}); + test('accepts gateway directory markers as artifacts', () => { assert.equal(isSupportedBridgeArtifactName('.dirkeep'), true); assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep'), true); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 26111273..216784a4 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -836,6 +836,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceInstanceTypes?: ['git_worktree']; /** Scheduling scopes this Code API can admit as independent lanes. */ supportedWorkspaceScopes?: ['git_linked_worktree']; + /** Added workspace tool error codes this Code API accepts in settlements. */ + supportedWorkspaceToolErrorCodes?: WorkspaceToolErrorCode[]; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -976,6 +978,7 @@ export interface BridgeRejectedSettlement { export type WorkspaceToolErrorCode = | 'INVALID_PATH' + | 'NOT_FOUND' | 'INVALID_REQUEST' | 'READ_LIMIT_EXCEEDED' | 'WRITE_LIMIT_EXCEEDED' @@ -994,6 +997,7 @@ export type WorkspaceToolErrorCode = const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'INVALID_PATH', + 'NOT_FOUND', 'INVALID_REQUEST', 'READ_LIMIT_EXCEEDED', 'WRITE_LIMIT_EXCEEDED', @@ -1011,6 +1015,15 @@ const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'COMMAND_DISABLED', ]); +/** + * Codes added after the original settlement contract, each with the legacy code + * an older Code API accepts in its place. A worker reports an added code only + * after registration advertises it, and falls back if a settlement is refused. + */ +export const WORKSPACE_TOOL_ERROR_CODE_FALLBACKS: Readonly< + Partial> +> = { NOT_FOUND: 'INVALID_PATH' }; + export function isWorkspaceToolErrorCode( value: unknown, ): value is WorkspaceToolErrorCode { diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts index b06de8d0..5bdddca9 100644 --- a/packages/code/src/root-access.ts +++ b/packages/code/src/root-access.ts @@ -350,7 +350,7 @@ export class WorkspaceRootAccess { } } - entry(path: string, operation: 'mkdir' | 'symlink' | 'readlink', target?: string): string | void { + entry(path: string, operation: 'mkdir' | 'symlink' | 'readlink', target?: string, mode = 0o700): string | void { const parent = this.parent(path); try { if (operation === 'readlink') { @@ -361,7 +361,7 @@ export class WorkspaceRootAccess { return buffer.subarray(0, length).toString(); } const result = operation === 'mkdir' - ? mkdirAt!(parent.fd, parent.name, 0o700) + ? mkdirAt!(parent.fd, parent.name, mode) : symlinkAt!(target!, parent.fd, parent.name); if (result !== 0) throw nativeError(); } finally { closeSync(parent.fd); } @@ -378,6 +378,8 @@ export class WorkspaceRootAccess { } const context = new AsyncLocalStorage(); +/** Whether filesystem adapters in this call are anchored to a held root descriptor. */ +export const holdsWorkspaceRoot = (): boolean => context.getStore() != null; export async function withWorkspaceRoot( root: string, identity: WorkspaceRootIdentity | undefined, @@ -504,10 +506,10 @@ export const readdir = async (path: string, maxEntries = 200_000): Promise => context.getStore()?.entry(path, 'readlink') as string ?? fs.readlink(path); -export const mkdir = async (path: string): Promise => { +export const mkdir = async (path: string, mode = 0o700): Promise => { const access = context.getStore(); - if (access) access.entry(path, 'mkdir'); - else await fs.mkdir(path, { mode: 0o700 }); + if (access) access.entry(path, 'mkdir', undefined, mode); + else await fs.mkdir(path, { mode }); }; export const symlink = async (target: string, path: string): Promise => { const access = context.getStore(); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 217ca2c7..17a9a6e1 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -11,6 +11,7 @@ import { workspaceIsolationKey, workspaceIsolationKeysConflict, workspaceIsolationParent, + WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; @@ -28,6 +29,7 @@ import type { BridgeWorkspaceToolOperation, BridgeWorkspaceProgrammaticRequest, RepositoryInstructionDescriptor, + WorkspaceToolErrorCode, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -464,6 +466,8 @@ export class BridgeWorker { private registrationCapabilities: BridgeWorkerCapabilities; private activeCapabilities: BridgeWorkerCapabilities; private instructionMetadataSupported = true; + /** Added error codes the current Code API registration accepts in settlements. */ + private settlementErrorCodes: ReadonlySet = new Set(); private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; private maintenanceOnly = false; @@ -803,6 +807,11 @@ export class BridgeWorker { } this.registrationTtlMs = registration.leaseTtlMs; this.activeCapabilities = this.registrationCapabilities; + this.settlementErrorCodes = new Set( + Array.isArray(registration.supportedWorkspaceToolErrorCodes) + ? registration.supportedWorkspaceToolErrorCodes + : [], + ); await this.options.onRegistered?.(registration); if ( !this.maintenanceOnly && @@ -2085,7 +2094,7 @@ export class BridgeWorker { ...((assignment.executionKind === 'workspace_tool' || assignment.executionKind === 'workspace_programmatic') && error instanceof WorkspaceToolError - ? { errorCode: error.code } + ? { errorCode: this.settlementErrorCode(error.code) } : {}), error: (assignment.executionKind === 'workspace_tool' && isWorkspaceToolRequest(assignment.request) && @@ -2474,6 +2483,12 @@ export class BridgeWorker { ); } + /** An older Code API refuses a settlement carrying a code it does not know. */ + private settlementErrorCode(code: WorkspaceToolErrorCode): WorkspaceToolErrorCode { + if (this.settlementErrorCodes.has(code)) return code; + return WORKSPACE_TOOL_ERROR_CODE_FALLBACKS[code] ?? code; + } + private async settleWithRetry( assignment: BridgeAssignment, settlement: BridgeSettlement, @@ -2525,6 +2540,20 @@ export class BridgeWorker { } catch (error) { lastError = error; if (signal?.aborted) break; + const legacyErrorCode = + settlement.status === 'rejected' && settlement.errorCode != null + ? WORKSPACE_TOOL_ERROR_CODE_FALLBACKS[settlement.errorCode] + : undefined; + if ( + settlement.status === 'rejected' && + legacyErrorCode != null && + error instanceof BridgeProtocolError && + error.status === 400 + ) { + /** A replica that predates the added code can still serve this settlement. */ + settlement = { ...settlement, errorCode: legacyErrorCode }; + continue; + } if ( error instanceof BridgeProtocolError && error.status != null && diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 71ef2cd3..99fcdd02 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -237,6 +237,112 @@ test('worker omits pagination fields until Code API negotiates them', async () = }); }); +async function settleMissingFile( + registeredCodes: string[] | undefined, + rejectSettlement: (body: Record) => boolean = () => false, +): Promise<{ settlements: Array>; failure?: unknown }> { + const settlements: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new WorkspaceToolError('Workspace path does not exist', 'NOT_FOUND'); + }, + }, + fetchImpl: async (input, init) => { + if (!String(input).endsWith('/settle')) { + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file'], + ...(registeredCodes ? { supportedWorkspaceToolErrorCodes: registeredCodes } : {}), + }); + } + const body = JSON.parse(String(init?.body)) as Record; + settlements.push(body); + if (rejectSettlement(body)) { + return Response.json({ error: 'Invalid bridge settlement' }, { status: 400 }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.register(); + const failure = await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-missing-file', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'logs/pending.log', + }, + }).then(() => undefined, (error: unknown) => error); + return { settlements, failure }; +} + +test('worker reports NOT_FOUND only to a Code API that advertised it', async () => { + const { settlements: supported } = await settleMissingFile(['NOT_FOUND']); + assert.deepEqual( + supported.map(({ status, errorCode, error }) => ({ status, errorCode, error })), + [{ status: 'rejected', errorCode: 'NOT_FOUND', error: 'Workspace path does not exist' }], + ); + + for (const registeredCodes of [undefined, [], ['SOMETHING_ELSE']]) { + const { settlements: legacy } = await settleMissingFile(registeredCodes); + assert.deepEqual( + legacy.map(({ status, errorCode, error }) => ({ status, errorCode, error })), + [{ status: 'rejected', errorCode: 'INVALID_PATH', error: 'Workspace path does not exist' }], + ); + } +}); + +test('worker resends a refused NOT_FOUND settlement with its legacy code', async () => { + const { settlements, failure } = await settleMissingFile( + ['NOT_FOUND'], + (body) => body.errorCode === 'NOT_FOUND', + ); + + assert.equal(failure, undefined); + assert.deepEqual( + settlements.map(({ errorCode }) => errorCode), + ['NOT_FOUND', 'INVALID_PATH'], + ); +}); + +test('worker does not resend a refused legacy settlement', async () => { + const { settlements, failure } = await settleMissingFile(undefined, () => true); + + assert.deepEqual(settlements.map(({ errorCode }) => errorCode), ['INVALID_PATH']); + assert.ok(failure instanceof BridgeProtocolError); + assert.equal(failure.status, 400); +}); + test('worker omits restricted workspaces that legacy registration would widen', async () => { const registrations: Array<{ operations: string[]; diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 4a0ecb3e..787fbe7d 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -15,6 +15,7 @@ import { unlink, writeFile, } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, sep } from 'node:path'; import test from 'node:test'; @@ -28,6 +29,7 @@ import { SandboxWorkspaceTools, WorkspaceToolError, } from './workspace.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; import { BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, @@ -1737,7 +1739,7 @@ test('oversized replaceAll previews and edits fail before writing the source fil } }); -test('writes reject symlink targets and missing parent directories', async (t) => { +test('writes reject symlink targets and symlinked parent directories', async (t) => { const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); t.after(() => rm(parent, { recursive: true, force: true })); const root = join(parent, 'root'); @@ -1752,8 +1754,8 @@ test('writes reject symlink targets and missing parent directories', async (t) = for (const path of [ 'link.txt', - 'missing/notes.txt', 'directory-link/notes.txt', + 'directory-link/missing/notes.txt', ]) { await assert.rejects( tools.execute({ @@ -1768,6 +1770,231 @@ test('writes reject symlink targets and missing parent directories', async (t) = ); } assert.equal(await readFile(join(parent, 'outside.txt'), 'utf8'), 'outside'); + assert.deepEqual(await readdir(join(root, 'real-directory')), []); +}); + +async function heldRootTools(root: string): Promise { + return LocalWorkspaceTools.create({ + workspaces: [ + { id: 'primary', root, identity: await captureWorkspaceRootIdentity(root), writable: true }, + ], + }); +} + +test('writes create missing parent directories inside a held workspace root', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'src')); + const tools = await heldRootTools(root); + + for (const [path, overwrite] of [ + ['src/feature/deep/new.ts', undefined], + ['fresh/created.ts', false], + ] as const) { + const result = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: 'export {};\n', + ...(overwrite === undefined ? {} : { overwrite }), + }); + assert.deepEqual(result, { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + created: true, + bytesWritten: 11, + }); + assert.equal(await readFile(join(root, ...path.split('/')), 'utf8'), 'export {};\n'); + } + assert.equal((await stat(join(root, 'src', 'feature', 'deep'))).isDirectory(), true); +}); + +test('parent creation never follows symlinks, crosses files, or leaves the workspace', async (t) => { + const parent = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')), + ); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + const outside = join(parent, 'outside'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(outside); + await writeFile(join(root, 'file.txt'), 'file'); + await symlink(outside, join(root, 'linked-outside')); + await symlink(join(root, 'src'), join(root, 'alias')); + await symlink(join(parent, 'missing-target'), join(root, 'dangling')); + const tools = await heldRootTools(root); + + for (const [path, code] of [ + ['linked-outside/new/notes.txt', 'INVALID_PATH'], + ['alias/new/notes.txt', 'INVALID_PATH'], + ['dangling/new/notes.txt', 'INVALID_PATH'], + ['file.txt/new/notes.txt', 'INVALID_PATH'], + ['../outside/new/notes.txt', 'INVALID_REQUEST'], + ['new/../../outside/notes.txt', 'INVALID_REQUEST'], + ] as const) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === code && + !error.mutationMayHaveCommitted, + path, + ); + } + assert.deepEqual(await readdir(outside), []); + assert.deepEqual(await readdir(join(root, 'src')), []); + assert.deepEqual((await readdir(root)).sort(), [ + 'alias', + 'dangling', + 'file.txt', + 'linked-outside', + 'src', + ]); + await assert.rejects(stat(join(parent, 'missing-target'))); +}); + +test('a write finishes once it has created parents, even if cancelled', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await heldRootTools(root); + /** Reports cancellation as soon as the parents exist. */ + const signal = { + get aborted() { + return existsSync(join(root, 'fresh', 'nested')); + }, + } as AbortSignal; + + const result = await tools.execute( + { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'fresh/nested/file.txt', + content: 'installed', + }, + signal, + ); + + assert.equal(result.operation === 'write_file' && result.created, true); + assert.equal(await readFile(join(root, 'fresh', 'nested', 'file.txt'), 'utf8'), 'installed'); +}); + +test('a write cancelled before creating parents leaves the workspace unchanged', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await heldRootTools(root); + const controller = new AbortController(); + controller.abort(); + + await assert.rejects( + tools.execute( + { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'fresh/nested/file.txt', + content: 'never installed', + }, + controller.signal, + ), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EXECUTION_ABORTED' && + !error.mutationMayHaveCommitted, + ); + assert.deepEqual(await readdir(root), []); +}); + +test('a pathname-only workspace root reports a missing parent instead of creating it', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'missing/notes.txt', + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'NOT_FOUND' && + error.message === 'Workspace path does not exist', + ); + assert.deepEqual(await readdir(root), []); +}); + +test('a missing workspace path is NOT_FOUND only when reached through the workspace', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + const outside = join(parent, 'outside'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(outside); + await writeFile(join(root, 'file.txt'), 'file'); + await symlink(join(root, 'src'), join(root, 'alias')); + await symlink(outside, join(root, 'linked-outside')); + await symlink(join(parent, 'missing-target'), join(root, 'dangling')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const edits = [{ oldText: 'a', newText: 'b' }]; + const requests = (path: string): WorkspaceToolRequest[] => [ + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path }, + { protocolVersion: 1, operation: 'edit_file', workspaceId: 'primary', path, edits }, + { protocolVersion: 1, operation: 'preview_edit', workspaceId: 'primary', path, edits }, + { protocolVersion: 1, operation: 'search_text', workspaceId: 'primary', query: 'x', path }, + { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary', path }, + ]; + + for (const [path, code] of [ + ['missing.log', 'NOT_FOUND'], + ['src/missing.ts', 'NOT_FOUND'], + ['not-yet/created/output.log', 'NOT_FOUND'], + ['alias/missing.ts', 'INVALID_PATH'], + ['linked-outside/missing.txt', 'INVALID_PATH'], + ['dangling', 'INVALID_PATH'], + ['file.txt/missing.txt', 'INVALID_PATH'], + ] as const) { + for (const request of requests(path)) { + await assert.rejects( + tools.execute(request), + (error: unknown) => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, code, `${request.operation} ${path}`); + assert.equal(error.message.includes(parent), false); + if (code === 'NOT_FOUND') { + assert.equal(error.message, 'Workspace path does not exist'); + } + return true; + }, + ); + } + } + for (const path of ['../outside/missing.txt', 'src/../../outside/missing.txt']) { + for (const request of requests(path)) { + await assert.rejects( + tools.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && + (error.code === 'INVALID_REQUEST' || error.code === 'INVALID_PATH'), + `${request.operation} ${path}`, + ); + } + } }); test('workspace mutations preserve existing file permissions', async (t) => { diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 0b9328d0..e48365c8 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { link, lstat, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; +import { holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; @@ -324,6 +324,9 @@ async function readConfinedFileBuffer( return buffer.subarray(0, bytesRead); } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, candidate); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } finally { await handle?.close(); @@ -369,6 +372,147 @@ async function verifyDirectoryPathHasNoSymlinks( return currentIdentity; } +function isMissingEntry(error: unknown): boolean { + return (error as NodeJS.ErrnoException | undefined)?.code === 'ENOENT'; +} + +interface DirectoryIdentity { + path: string; + dev: bigint | number; + ino: bigint | number; +} + +/** Whether each ancestor observed before a miss is still the same real directory. */ +async function ancestorsUnchanged(ancestors: readonly DirectoryIdentity[]): Promise { + for (const ancestor of ancestors) { + try { + const current = await lstat(ancestor.path); + if ( + current.isSymbolicLink() || + !current.isDirectory() || + current.dev !== ancestor.dev || + current.ino !== ancestor.ino + ) { + return false; + } + } catch { + return false; + } + } + return true; +} + +/** + * Reports an absent target as `NOT_FOUND` only when the walk from the root + * reaches the missing entry through real directories, the root included, that + * are unchanged after the miss. A symlink, a file used as a directory, or an ancestor replaced + * during the walk stays a path-safety rejection, so the distinction never + * describes a path beyond the root. + */ +async function classifyMissingWorkspacePath( + root: string, + candidate: string, +): Promise { + const invalid = new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + const segments = relative(root, candidate).split(sep).filter(Boolean); + const ancestors: DirectoryIdentity[] = []; + try { + const rootEntry = await lstat(root); + if (rootEntry.isSymbolicLink() || !rootEntry.isDirectory()) return invalid; + ancestors.push({ path: root, dev: rootEntry.dev, ino: rootEntry.ino }); + } catch { + return invalid; + } + let current = root; + for (const [index, segment] of segments.entries()) { + current = resolve(current, segment); + let entry: Awaited>; + try { + entry = await lstat(current); + } catch (error) { + return isMissingEntry(error) && (await ancestorsUnchanged(ancestors)) + ? new WorkspaceToolError('Workspace path does not exist', 'NOT_FOUND') + : invalid; + } + if (index === segments.length - 1) return invalid; + if (entry.isSymbolicLink() || !entry.isDirectory()) return invalid; + ancestors.push({ path: current, dev: entry.dev, ino: entry.ino }); + } + return invalid; +} + +/** + * Creates each missing ancestor of a write target as a real directory beneath + * the root, one verified level at a time, and syncs each new entry into its + * parent. Creation needs a held root descriptor, so `mkdirat` is anchored to + * the root; a pathname-only root creates nothing and its write reports the + * missing parent. An existing symlink or file stops the walk. + * + * Created directories are a committed side effect, never rolled back: once one + * exists, a later failure of this write is reported as a possible mutation, so + * the worker quarantines rather than claiming an atomic rejection. + */ +async function createMissingParentDirectories( + root: string, + candidate: string, +): Promise { + const created: string[] = []; + if (!holdsWorkspaceRoot()) return created; + const segments = relative(root, dirname(candidate)).split(sep).filter(Boolean); + let current = root; + try { + for (const segment of segments) { + current = resolve(current, segment); + let entry: Awaited> | undefined; + try { + entry = await lstat(current); + } catch (error) { + if (!isMissingEntry(error)) throw error; + } + let createdHere = false; + if (entry == null) { + try { + await mkdir(current, 0o777); + created.push(current); + createdHere = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + entry = await lstat(current); + /** Also after a raced `EEXIST`: the write relies on that entry too. */ + await syncWorkspaceDirectory(dirname(current)); + } + if ( + entry.isSymbolicLink() || + !entry.isDirectory() || + (createdHere && !isWithinRoot(root, await realpath(current))) + ) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + } + return created; + } catch (error) { + throw afterCreatedDirectories(created, classifyWritePathValidationError(error)); + } +} + +function afterCreatedDirectories( + created: readonly string[], + error: WorkspaceToolError, +): WorkspaceToolError { + if (created.length === 0 || error.mutationMayHaveCommitted) return error; + return new WorkspaceToolError(error.message, error.code, true); +} + +function assertWithinWriteLimit(content: Buffer): void { + if (content.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds write limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } +} + function classifyWritePathValidationError(error: unknown): WorkspaceToolError { if (error instanceof WorkspaceToolError) return error; const code = (error as NodeJS.ErrnoException).code; @@ -481,12 +625,7 @@ async function atomicWriteConfinedFile( allowOverwrite = true, ): Promise<{ created: boolean }> { throwIfAborted(signal); - if (content.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { - throw new WorkspaceToolError( - 'Workspace file exceeds write limit', - 'WRITE_LIMIT_EXCEEDED', - ); - } + assertWithinWriteLimit(content); const candidate = resolveWorkspacePath(root, requestedPath); const parent = dirname(candidate); let canonicalParent: string; @@ -498,6 +637,9 @@ async function atomicWriteConfinedFile( throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } } catch (error) { + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, parent); + } throw classifyWritePathValidationError(error); } @@ -671,14 +813,26 @@ async function writeWorkspaceFile( signal?: AbortSignal, ): Promise { const content = Buffer.from(request.content, 'utf8'); - const { created } = await atomicWriteConfinedFile( + throwIfAborted(signal); + assertWithinWriteLimit(content); + const directories = await createMissingParentDirectories( root, - request.path, - content, - signal, - undefined, - request.overwrite !== false, + resolveWorkspacePath(root, request.path), ); + let created: boolean; + try { + /** Once parents exist, finish the write rather than abandon them to a cancellation. */ + ({ created } = await atomicWriteConfinedFile( + root, + request.path, + content, + directories.length === 0 ? signal : undefined, + undefined, + request.overwrite !== false, + )); + } catch (error) { + throw afterCreatedDirectories(directories, classifyWritePathValidationError(error)); + } return { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'write_file', @@ -757,6 +911,9 @@ async function editWorkspaceFile( }; } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, candidate); + } throw classifyWritePathValidationError(error); } finally { await opened?.close().catch(() => undefined); @@ -1009,7 +1166,10 @@ async function searchWorkspace( let canonicalTarget: string; try { canonicalTarget = await realpath(target); - } catch { + } catch (error) { + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, target); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } if (!isWithinRoot(root, canonicalTarget)) @@ -1060,7 +1220,9 @@ async function searchWorkspace( } catch (error) { if ( error instanceof WorkspaceToolError && - (error.code === 'INVALID_PATH' || error.code === 'READ_LIMIT_EXCEEDED') + (error.code === 'INVALID_PATH' || + error.code === 'NOT_FOUND' || + error.code === 'READ_LIMIT_EXCEEDED') ) { continue; } @@ -1167,6 +1329,13 @@ async function listWorkspaceFiles( ); } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await withinListDeadline( + classifyMissingWorkspacePath(root, target), + signal, + deadline, + ); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } if (!isWithinRoot(root, canonicalTarget)) { @@ -1181,6 +1350,13 @@ async function listWorkspaceFiles( ).isDirectory(); } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await withinListDeadline( + classifyMissingWorkspacePath(root, target), + signal, + deadline, + ); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } const normalizedRequestedResultPath = request.path diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 28035847..360414a0 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -438,6 +438,7 @@ describe('paired bridge HTTP API', () => { 'replace_all', ], supportedWorkspaceListFileFeatures: ['after_path'], + supportedWorkspaceToolErrorCodes: ['NOT_FOUND'], }); const crossDeploymentRevoke = await fetch( diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 8fdc700e..0bc20a1e 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -541,6 +541,7 @@ router.post( supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], supportedWorkspaceScopes: ['git_linked_worktree'], + supportedWorkspaceToolErrorCodes: ['NOT_FOUND'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index d3cb032e..8a753c08 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -223,6 +223,8 @@ test.each([ ['COMMAND_TIMEOUT', 504], ['COMMAND_UNAVAILABLE', 503], ['COMMAND_DISABLED', 403], + ['INVALID_PATH', 422], + ['NOT_FOUND', 422], ] as const)('maps worker %s rejections to HTTP %i', async (errorCode, expectedStatus) => { const app = express(); app.use(json()); From 893ad3603f223f279439b8ae764f2860b8ac82bd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 23:06:38 -0400 Subject: [PATCH 63/75] feat: Quote the Current Text Where a Missing Workspace Edit Belongs (#300) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📍 feat: Quote the Current Text Where a Missing Workspace Edit Belongs A missing edit's diagnosis now ends with a bounded excerpt of the region old_text most likely meant: the alignment where most of its distinctive lines fall into place (found in one pass with bounded votes), else its unique first line or closest line. Each of at most 8 rows is `|`, with a space for identical lines, `~` for whitespace-only differences and `!` for changed text; lines are shortened to 160 characters and the excerpt to 1,600. The excerpt is a final `; the current text at lines A-B (...) is ""` hint, so hosts that parse the existing grammar drop it and keep every other hint. Excerpts are the first detail dropped to fit the 3,000-character message bound and never shorten another edit's reason. * 🧭 fix: Bound Every Alignment Offset and Anchor Excerpts Where old_text Starts - Count every inspected anchor offset against the region vote budget, and stop at the first offset that would start before the file (offsets ascend), so a repetitive old_text cannot escape the bound. - Start a first-line or closest-line excerpt at the line where old_text starts, not at its first non-blank line. - Escape U+2028/U+2029 in quoted text, which JSON.stringify leaves raw, so each batch edit stays on one line for hosts that read the message line by line. * 📏 fix: Bound Edit Excerpts by Their Encoded Error-Body Size Code API returns the diagnostic inside a JSON error body, and LibreChat reads that body through a 4,096-byte buffer, discarding a truncated one. A 3,000-character message with multibyte or escape-heavy excerpts could exceed it. Excerpts are now limited to 1,600 bytes and granted only while the whole message stays within 3,800 bytes, both measured as UTF-8 once JSON-encoded. * 📐 fix: Shorten Edit Diagnostics by Encoded Size as Well as Length Batch reasons that still exceed the bound after excerpts are declined are now shortened against both the 3,000-character limit and the 3,800-byte encoded budget, and a single edit's reason is shortened the same way instead of being cut by character count. Multibyte closest-line quotes can no longer push the Code API's error body past the 4,096 bytes hosts read. --- packages/code/README.md | 13 +- packages/code/src/edits.test.ts | 243 ++++++++++++++++++++++++++++ packages/code/src/edits.ts | 270 ++++++++++++++++++++++++++++---- 3 files changed, 491 insertions(+), 35 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index 55f5c668..636cd1a8 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -640,7 +640,18 @@ fails, the worker still checks the rest and rejects the whole batch with one batch messages shorten reasons and source excerpts to stay within the bound, but never omit failing edit positions. A missing edit names the nearest candidate line and flags elided (`...`) or line-numbered `oldText`, a -whitespace-only difference, or CRLF line endings. +whitespace-only difference, or CRLF line endings. When the file has a region +that `oldText` most likely meant (the alignment where the most of its +distinctive lines fall into place, or its unique first line or closest line), +the reason ends with `the current text at lines A-B (~ whitespace differs, ! +text differs) is ""`: a JSON string of at most 8 rows of the form +`|`, each line shortened to 160 characters, where the mark is +a space for a line identical to `oldText` at that position, `~` when only its +whitespace differs and `!` when its text differs. Excerpts are the first detail +dropped to fit the message bound, and never shorten another edit's reason. Both +the excerpt (1,600 bytes) and the message that carries it (3,800 bytes) are +measured as UTF-8 once JSON-encoded, so the Code API's error body stays within +the 4,096 bytes hosts such as LibreChat read. An ambiguous edit gives its match count and line numbers. Overlapping occurrences count as separate locations. Detailed source-line excerpts require `read_file` or `preview_edit` on the same workspace; edit-only workers return a diff --git a/packages/code/src/edits.test.ts b/packages/code/src/edits.test.ts index 3ef9428d..5bcabf49 100644 --- a/packages/code/src/edits.test.ts +++ b/packages/code/src/edits.test.ts @@ -719,3 +719,246 @@ test('short batch diagnostic reasons remain complete when the message fits', () assert.ok(error.message.includes(`\nEdit ${failure.index + 1}: ${failure.reason}.`)); } }); + +const EXCERPT = /the current text at lines? (\d+)(?:-(\d+))? \(~ whitespace differs, ! text differs\) is ("(?:[^"\\]|\\.)*")/; + +function excerptOf(message: string): { first: number; last: number; rows: string[] } | undefined { + const match = EXCERPT.exec(message); + if (!match) return undefined; + const first = Number(match[1]); + return { first, last: Number(match[2] ?? first), rows: (JSON.parse(match[3]) as string).split('\n') }; +} + +test('a missing edit quotes the current text of the region it meant, marking what differs', () => { + const text = 'export function load(user) {\n const id = user.id;\n return fetchUser(id);\n}\n'; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'export function load(user) {\n const id = user.id;\n return fetchUser(user);\n}', + newText: 'x', + }], 'tolerant')); + assert.match(error.message, /old_text was not found; its first line appears at line 1, but the lines after it differ; the current text/); + assert.deepEqual(excerptOf(error.message), { + first: 1, + last: 4, + rows: [ + '1| export function load(user) {', + '2|~ const id = user.id;', + '3|! return fetchUser(id);', + '4| }', + ], + }); + assert.match(error.message, /\.$/); +}); + +test('the region is found from later lines when the first line itself changed', () => { + const text = 'header();\nfunction save(record) {\n validate(record);\n persist(record);\n}\nfooter();\n'; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'function save(item) {\n validate(record);\n persist(record);\n}', + newText: 'x', + }])); + assert.deepEqual(excerptOf(error.message), { + first: 2, + last: 5, + rows: ['2|!function save(record) {', '3| validate(record);', '4| persist(record);', '5| }'], + }); +}); + +test('the region with the most aligned lines wins, and ties prefer the earliest', () => { + const block = (name: string) => `function ${name}() {\n prepare();\n run();\n cleanup();\n}\n`; + const text = `${block('first')}${block('second')}`; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'function second() {\n prepare();\n execute();\n cleanup();\n}', + newText: 'x', + }])); + assert.equal(excerptOf(error.message)?.first, 6); + const tied = rejection(() => applyTextEdits(text, [{ + oldText: 'function third() {\n prepare();\n execute();\n cleanup();\n}', + newText: 'x', + }])); + assert.equal(excerptOf(tied.message)?.first, 1); +}); + +test('a missing one-line edit quotes its closest line', () => { + const error = rejection(() => applyTextEdits('const total = items.length;\nreturn total;\n', [ + { oldText: 'const total = items.size;', newText: 'x' }, + ])); + assert.match(error.message, /the closest line is line 1: "const total = items.length;"/); + assert.deepEqual(excerptOf(error.message), { first: 1, last: 1, rows: ['1|!const total = items.length;'] }); +}); + +test('exact mode quotes the whitespace to copy alongside the whitespace hint', () => { + const error = rejection(() => applyTextEdits('class A {\n a();\n b();\n}\n', [ + { oldText: 'a();\nb();', newText: 'x' }, + ])); + assert.match(error.message, /exists at line 2 with different whitespace \(indentation-flexible\); copy that whitespace exactly; the current text at lines 2-3/); + assert.deepEqual(excerptOf(error.message)?.rows, ['2|~ a();', '3|~ b();']); +}); + +test('a long region is quoted from just before its first difference', () => { + const source = Array.from({ length: 40 }, (_, index) => `step(${index});`); + const needle = source.slice(5, 25); + needle[8] = 'step(changed);'; + const error = rejection(() => applyTextEdits(`${source.join('\n')}\n`, [{ oldText: needle.join('\n'), newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.equal(excerpt?.first, 13); + assert.equal(excerpt?.last, 20); + assert.equal(excerpt?.rows[1], '14|!step(13);'); + assert.ok(excerpt?.rows.every((row, index) => index === 1 || row.includes('| '))); +}); + +test('excerpts shorten long lines and stay bounded without splitting characters', () => { + const wide = `x${'😀'.repeat(400)}`; + const controls = '\u0001'.repeat(2_000); + const text = Array.from({ length: 10 }, (_, index) => `key_${index} = ${index % 2 === 0 ? wide : controls}`).join('\n'); + const oldText = Array.from({ length: 10 }, (_, index) => `key_${index} = other`).join('\n'); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.ok(excerpt != null); + assert.ok(excerpt.rows.length >= 1 && excerpt.rows.length <= 8); + assert.ok(EXCERPT.exec(error.message)![0].length <= 1_600); + for (const row of excerpt.rows) { + assert.ok(row.length <= 175, row.slice(0, 40)); + assert.equal(Buffer.from(row).toString('utf8'), row); + } +}); + +test('an edit that resembles nothing in the file gets no excerpt', () => { + const error = rejection(() => applyTextEdits('alpha\nbeta\n', [ + { oldText: 'completely different text\nwith nothing shared', newText: 'x' }, + ])); + assert.equal(excerptOf(error.message), undefined); + assert.match(error.message, /old_text was not found\.$/); +}); + +test('ambiguous and line-numbered edits keep their messages without an excerpt', () => { + const ambiguous = rejection(() => applyTextEdits('return a;\nreturn a;\n', [{ oldText: 'return a;', newText: 'x' }])); + assert.equal(excerptOf(ambiguous.message), undefined); + const numbered = rejection(() => applyTextEdits('start\nmiddle\nend\n', [{ oldText: '1 | start\n2 | middle', newText: 'x' }])); + assert.equal(excerptOf(numbered.message), undefined); +}); + +test('batch excerpts are granted in edit order only while every position and reason still fits', () => { + const lines = Array.from({ length: 400 }, (_, index) => `value_${index} = compute_${index}(input_${index}, ${'p'.repeat(120)});`); + const edits = Array.from({ length: 12 }, (_, index) => ({ + oldText: `${lines[index * 10]}\n${lines[index * 10 + 1].replace('compute', 'changed')}\n${lines[index * 10 + 2]}`, + newText: 'x', + })); + const error = rejection(() => applyTextEdits(`${lines.join('\n')}\n`, edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 12 }, (_, index) => index + 1), + ); + const quoted = [...error.message.matchAll(/\nEdit (\d+): [^\n]*the current text at/g)].map((match) => Number(match[1])); + assert.ok(quoted.length >= 1 && quoted.length < 12); + assert.deepEqual(quoted, Array.from({ length: quoted.length }, (_, index) => index + 1)); + for (const failure of error.failures) { + assert.ok(error.message.includes(`\nEdit ${failure.index + 1}: ${failure.reason}`)); + } +}); + +test('a single edit drops its excerpt rather than exceed the bound', () => { + const error = new WorkspaceEditMatchError([ + { index: 0, reason: `old_text was not found${'; x'.repeat(1_200)}`, excerpt: 'the current text at line 1 (~ whitespace differs, ! text differs) is "1|!y"' }, + ], 1); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.doesNotMatch(error.message, /the current text/); +}); + +test('region votes stay bounded on highly repetitive files', () => { + const text = 'item = value;\nother = thing;\n'.repeat(150_000); + const oldText = 'item = value;\nother = thing;\nmissing = line;\nitem = value;'; + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + assert.ok(performance.now() - started < 3_000, 'a repetitive file must not cast unbounded votes'); + assert.match(error.message, /did not apply and nothing was written/); +}); + +test('an excerpt anchored on a first line after blank lines starts where old_text starts', () => { + const error = rejection(() => applyTextEdits('header();\nconst total = items.length;\nreturn total;\n', [ + { oldText: '\nconst total = items.size;', newText: 'x' }, + ])); + assert.deepEqual(excerptOf(error.message), { + first: 1, + last: 2, + rows: ['1|!header();', '2|!const total = items.length;'], + }); +}); + +test('region votes count every inspected anchor offset against the budget', () => { + const oldText = `${'repeat_line();\n'.repeat(5_000)}missing();`; + const text = 'repeat_line();\n'.repeat(20_000); + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + assert.ok(performance.now() - started < 3_000, 'repeated anchor offsets must not escape the vote budget'); + assert.match(error.message, /did not apply and nothing was written/); +}); + +test('quoted text escapes line separators so every batch edit stays on one line', () => { + const separator = String.fromCharCode(0x2028); + const text = `const label = "first${separator}second";\nconst other = "value";\n`; + const error = rejection(() => applyTextEdits(text, [ + { oldText: `const label = "first${separator}third";`, newText: 'x' }, + { oldText: 'const other = "changed";', newText: 'y' }, + ])); + assert.ok(!error.message.includes(separator)); + const lines = error.message.split('\n'); + assert.match(lines[1], /^Edit 1: old_text was not found; the closest line is line 1: .*\\u2028.*the current text at line 1/); + assert.match(lines[2], /^Edit 2: /); + assert.deepEqual(excerptOf(lines[1])?.rows, [`1|!const label = "first${separator}second";`]); +}); + +function errorBodyBytes(message: string): number { + return Buffer.byteLength(JSON.stringify({ error: message, code: 'EDIT_CONFLICT' })); +} + +test('excerpts of multibyte source stay within the 4096-byte error body hosts read', () => { + const text = Array.from({ length: 12 }, (_, index) => `名前_${index} = "${'漢字'.repeat(80)}";`).join('\n'); + const oldText = Array.from({ length: 12 }, (_, index) => `名前_${index} = "changed";`).join('\n'); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.ok(excerpt != null && excerpt.rows.length >= 1 && excerpt.rows.length < 8); + assert.ok(Buffer.byteLength(JSON.stringify(EXCERPT.exec(error.message)![0])) - 2 <= 1_600); + assert.ok(errorBodyBytes(error.message) <= 4_096); +}); + +test('batch excerpts are granted by encoded size, so escapes cannot overflow the error body', () => { + const lines = Array.from({ length: 200 }, (_, index) => `path_${index} = "C:\\\\dir\\\\${'"q"'.repeat(48)}";`); + const edits = Array.from({ length: 6 }, (_, index) => ({ + oldText: lines.slice(index * 20, index * 20 + 8).map((line, offset) => (offset === 3 ? 'changed();' : line)).join('\n'), + newText: 'x', + })); + const error = rejection(() => applyTextEdits(`${lines.join('\n')}\n`, edits)); + assert.ok(errorBodyBytes(error.message) <= 4_096, String(errorBodyBytes(error.message))); + assert.match(error.message, /\nEdit 1: [^\n]*the current text at/); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + [1, 2, 3, 4, 5, 6], + ); +}); + +test('shortened batch reasons also respect the encoded error-body size', () => { + const emojis = '😀'.repeat(200); + const text = Array.from({ length: 20 }, (_, index) => `item_${index} emoji_${index} = "${emojis}";`).join('\n'); + const edits = Array.from({ length: 20 }, (_, index) => ({ + oldText: `item_${index} emoji_${index} = "plain";`, + newText: 'x', + })); + const error = rejection(() => applyTextEdits(text, edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.ok(errorBodyBytes(error.message) <= 4_096, String(errorBodyBytes(error.message))); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 20 }, (_, index) => index + 1), + ); +}); + +test('a single edit with a long multibyte reason stays within both bounds', () => { + const error = new WorkspaceEditMatchError([ + { index: 0, reason: `old_text was not found; the closest line is line 1: "${'😀'.repeat(1_400)}"` }, + ], 1); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.ok(errorBodyBytes(error.message) <= 4_096); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.match(error.message, /…\.$/); +}); diff --git a/packages/code/src/edits.ts b/packages/code/src/edits.ts index a6d54269..8205b1d7 100644 --- a/packages/code/src/edits.ts +++ b/packages/code/src/edits.ts @@ -12,8 +12,24 @@ import type { * callers prefix their own context, so diagnostics stay well below that. */ export const EDIT_DIAGNOSTIC_MAX_CHARS = 3000; +/** + * Code API returns the message inside a JSON error body, and hosts read that + * body through a bounded buffer (4096 bytes in LibreChat). Optional excerpts + * are granted only while the JSON-encoded UTF-8 message stays within this size. + */ +export const EDIT_DIAGNOSTIC_MAX_BODY_BYTES = 3_800; const MAX_REPORTED_LINES = 5; const MAX_SNIPPET_CHARS = 120; +/** A missing edit's current-text excerpt: a few lines, each shortened, in one bounded hint. */ +const MAX_EXCERPT_LINES = 8; +const MAX_EXCERPT_LINE_CHARS = 160; +/** Measured like the message bound: UTF-8 bytes once JSON-encoded into the error body. */ +const MAX_EXCERPT_BYTES = 1_600; +/** Alignment votes cast while locating the region a missing edit most likely meant. */ +const MAX_REGION_VOTES = 200_000; +/** Lines too generic to anchor a region on their own (`}`, `);`, blank lines). */ +const ANCHOR_LINE = /[\p{L}\p{N}_]{2}/u; +const LINE_SEPARATORS = /[\u2028\u2029]/g; /** A highly repetitive indentation candidate must not monopolize the worker. */ const MAX_LINE_WINDOW_VERIFICATIONS = 100_000; const MAX_REPLACEMENT_CHUNK_CHARS = 16 * 1024; @@ -45,6 +61,12 @@ export interface EditFailure { /** Zero-based position of the edit in the request. */ index: number; reason: string; + /** + * A final `; the current text at ...` hint quoting the region the edit most + * likely meant. It is appended to `reason` only while the whole message stays + * within its bound, and is the first detail dropped when it would not. + */ + excerpt?: string; } export class WorkspaceEditMatchError extends Error { @@ -93,14 +115,16 @@ export function applyTextEdits( matches.push({ strategy: outcome.strategy, occurrences: outcome.occurrences }); return; } + if (outcome.status === 'none') { + failures.push({ index, ...describeMissing(working, edit.oldText, matching, lines) }); + return; + } failures.push({ index, reason: outcome.status === 'ambiguous' ? describeAmbiguous(working, outcome.strategy, outcome.count, outcome.starts) - : outcome.status === 'limit' - ? 'old_text has too many repetitive line-window candidates; include more surrounding lines or use an exact match' - : describeMissing(working, edit.oldText, matching, lines), + : 'old_text has too many repetitive line-window candidates; include more surrounding lines or use an exact match', }); }); if (failures.length > 0) { @@ -589,11 +613,21 @@ function describeAmbiguous( return `old_text matched ${count} locations${how} at ${formatLineList(text, starts, count)}; include more surrounding lines so it matches exactly one`; } +/** + * JSON quoting that also escapes U+2028/U+2029, which JSON.stringify leaves + * raw: hosts read a batch diagnostic one line per edit, and a regular + * expression `.` stops at those separators. + */ +function quote(value: string): string { + return JSON.stringify(value).replace( + LINE_SEPARATORS, + (separator) => `\\u${separator.charCodeAt(0).toString(16)}`, + ); +} + function snippet(line: string): string { const trimmed = line.trim(); - return JSON.stringify( - trimmed.length > MAX_SNIPPET_CHARS ? `${trimmed.slice(0, MAX_SNIPPET_CHARS)}…` : trimmed, - ); + return quote(trimmed.length > MAX_SNIPPET_CHARS ? `${trimmed.slice(0, MAX_SNIPPET_CHARS)}…` : trimmed); } const ELISION_LINE = /^\s*(?:(?:\/\/|#|--|\/\*|\*|))?\s*$/; @@ -604,9 +638,121 @@ function describeMissing( oldText: string, matching: WorkspaceEditMatching, lines: () => LineIndex, +): { reason: string; excerpt?: string } { + const needle = neededLines(oldText).lines; + const firstOffset = needle.findIndex((line) => line.trim().length > 0); + const nearest = nearestLine(needle[firstOffset], lines); + const reason = describeMissingReason(text, oldText, needle, matching, lines, nearest); + // The anchor is the first non-blank line, so the region starts that many lines earlier. + const anchor = nearest != null && (!nearest.exact || nearest.count === 1) + ? Math.max(0, nearest.index - firstOffset) + : undefined; + const start = closestRegion(needle, lines()) ?? anchor; + const excerpt = start == null ? undefined : currentTextExcerpt(needle, lines(), start); + return excerpt == null ? { reason } : { reason, excerpt }; +} + +type ExcerptMark = ' ' | '~' | '!'; + +/** + * Finds where a missing multi-line edit most likely belongs: the alignment on + * which the most of its distinctive lines, compared without surrounding + * whitespace, fall into place. One pass over the file with bounded votes. + */ +function closestRegion(needle: readonly string[], lines: LineIndex): number | undefined { + const offsets = new Map(); + let anchors = 0; + needle.forEach((line, offset) => { + const key = line.trim(); + if (!ANCHOR_LINE.test(key)) return; + anchors++; + const known = offsets.get(key); + if (known) known.push(offset); + else offsets.set(key, [offset]); + }); + if (anchors < 2) return undefined; + const votes = new Map(); + let budget = MAX_REGION_VOTES; + let best: number | undefined; + let bestVotes = 0; + for (let index = 0; index < lines.length && budget > 0; index++) { + const hits = offsets.get(lines.text(index).trim()); + if (!hits) continue; + // Offsets ascend, so every later one would start before the file too. + for (const offset of hits) { + if (budget-- <= 0) break; + const start = index - offset; + if (start < 0) break; + const count = (votes.get(start) ?? 0) + 1; + votes.set(start, count); + if (count > bestVotes || (count === bestVotes && best !== undefined && start < best)) { + best = start; + bestVotes = count; + } + } + } + // One shared line is too weak to claim a region; the first-line anchor covers that case. + return bestVotes >= 2 ? best : undefined; +} + +function excerptMark(current: string, expected: string | undefined): ExcerptMark { + if (expected === undefined) return '!'; + if (current === expected) return ' '; + return current.trim() === expected.trim() ? '~' : '!'; +} + +function shortenLine(line: string): string { + if (line.length <= MAX_EXCERPT_LINE_CHARS) return line; + let end = MAX_EXCERPT_LINE_CHARS; + if (/[\uD800-\uDBFF]/.test(line[end - 1]) && /[\uDC00-\uDFFF]/.test(line[end])) end--; + return `${line.slice(0, end)}…`; +} + +/** + * Quotes the current text where a missing edit most likely belongs, so the next + * attempt can copy it instead of re-reading the file. Each line is marked + * against the `old_text` line at the same position: a space when identical, + * `~` when only whitespace differs and `!` when the text differs. A region + * longer than the excerpt is shown from just before its first difference. + */ +function currentTextExcerpt(needle: readonly string[], lines: LineIndex, start: number): string | undefined { + const span = Math.max(1, Math.min(needle.length, lines.length - start)); + const marks = Array.from({ length: span }, (_, offset) => + excerptMark(lines.text(start + offset), needle[offset]), + ); + const shown = Math.min(span, MAX_EXCERPT_LINES); + const firstDifference = Math.max(0, marks.findIndex((mark) => mark !== ' ')); + const first = start + Math.min(Math.max(0, firstDifference - 1), span - shown); + const rows = marks + .slice(first - start, first - start + shown) + .map((mark, offset) => `${first + offset + 1}|${mark}${shortenLine(lines.text(first + offset))}`); + for (; rows.length > 0; rows.pop()) { + const where = rows.length === 1 ? `line ${first + 1}` : `lines ${first + 1}-${first + rows.length}`; + const excerpt = `the current text at ${where} (~ whitespace differs, ! text differs) is ${quote(rows.join('\n'))}`; + if (encodedBytes(excerpt) <= MAX_EXCERPT_BYTES) return excerpt; + } + return undefined; +} + +interface NearestLine { + exact: boolean; + count: number; + /** Line index of the first exact candidate, or of the best token match. */ + index: number; + starts: number[]; + text: string; +} + +function describeMissingReason( + text: string, + oldText: string, + needle: readonly string[], + matching: WorkspaceEditMatching, + lines: () => LineIndex, + nearest: NearestLine | undefined, ): string { const hints: string[] = []; - const nonBlank = neededLines(oldText).lines.filter((line) => line.trim().length > 0); + const nonBlank = needle.filter((line) => line.trim().length > 0); if (nonBlank.some((line) => ELISION_LINE.test(line))) { hints.push('it contains an elision placeholder ("..."); copy the exact lines instead of abbreviating'); } @@ -629,7 +775,6 @@ function describeMissing( hints.push('the file uses CRLF line endings'); } } - const nearest = nearestLine(nonBlank[0], lines); if (nearest != null && hints.length === 0) { hints.push( nearest.exact @@ -644,48 +789,106 @@ function describeMissing( function nearestLine( firstLine: string | undefined, getLines: () => LineIndex, -): { exact: boolean; count: number; starts: number[]; text: string } | undefined { +): NearestLine | undefined { const target = firstLine?.trim(); if (!target) return undefined; const lines = getLines(); const starts: number[] = []; let count = 0; - let firstMatch = ''; + let first = 0; for (let index = 0; index < lines.length; index++) { - const content = lines.text(index); - if (content.trim() !== target) continue; - if (count++ === 0) firstMatch = content; + if (lines.text(index).trim() !== target) continue; + if (count++ === 0) first = index; if (starts.length < MAX_REPORTED_LINES) starts.push(lines.start(index)); } if (count > 0) { - return { exact: true, count, starts, text: firstMatch }; + return { exact: true, count, index: first, starts, text: lines.text(first) }; } const tokens = new Set(target.split(/\W+/).filter((token) => token.length > 1)); if (tokens.size < 2) return undefined; - let best: Line | undefined; + let best: number | undefined; let bestScore = 0; for (let index = 0; index < lines.length; index++) { - const content = lines.text(index); let score = 0; - for (const token of new Set(content.split(/\W+/))) { + for (const token of new Set(lines.text(index).split(/\W+/))) { if (tokens.has(token)) score++; } if (score > bestScore) { - best = { start: lines.start(index), end: lines.end(index), next: lines.next(index), text: content }; + best = index; bestScore = score; } } return best != null && bestScore / tokens.size >= 0.5 - ? { exact: false, count: 1, starts: [best.start], text: best.text } + ? { exact: false, count: 1, index: best, starts: [lines.start(best)], text: lines.text(best) } : undefined; } +/** UTF-8 bytes `value` occupies once JSON-encoded as a string in an error body. */ +function encodedBytes(value: string): number { + return Buffer.byteLength(JSON.stringify(value)) - 2; +} + +/** A failure's reason, followed by its excerpt when the excerpt was granted room. */ +function failureText(failure: EditFailure, withExcerpt: boolean): string { + return withExcerpt && failure.excerpt ? `${failure.reason}; ${failure.excerpt}` : failure.reason; +} + +/** + * Grants excerpts in edit order while the message still fits. Excerpts are the + * most expendable detail, so granting one never shortens another reason. + */ +function grantExcerpts( + failures: readonly EditFailure[], + availableChars: number, + availableBytes: number, +): boolean[] { + let chars = availableChars - failures.reduce((total, failure) => total + failure.reason.length, 0); + let bytes = availableBytes - failures.reduce((total, failure) => total + encodedBytes(failure.reason), 0); + return failures.map((failure) => { + if (!failure.excerpt) return false; + const addition = `; ${failure.excerpt}`; + const byteCost = encodedBytes(addition); + if (addition.length > chars || byteCost > bytes) return false; + chars -= addition.length; + bytes -= byteCost; + return true; + }); +} + +/** + * The longest prefix of `reason` within both limits, ending in an ellipsis. It + * never splits a surrogate pair in a shortened source excerpt. + */ +function shortenReason(reason: string, charLimit: number, byteLimit: number): string { + if (reason.length <= charLimit && encodedBytes(reason) <= byteLimit) return reason; + const budget = byteLimit - encodedBytes('…'); + let end = 0; + let bytes = 0; + while (end < reason.length) { + const width = /[\uD800-\uDBFF]/.test(reason[end]) && /[\uDC00-\uDFFF]/.test(reason[end + 1] ?? '') ? 2 : 1; + const cost = encodedBytes(reason.slice(end, end + width)); + if (end + width > charLimit - 1 || bytes + cost > budget) break; + bytes += cost; + end += width; + } + return `${reason.slice(0, end)}…`; +} + +function fitsDiagnosticBounds(message: string): boolean { + return message.length <= EDIT_DIAGNOSTIC_MAX_CHARS && encodedBytes(message) <= EDIT_DIAGNOSTIC_MAX_BODY_BYTES; +} + function formatEditFailures(failures: readonly EditFailure[], editCount: number): string { if (editCount === 1) { - return `Workspace edit did not apply and nothing was written: ${failures[0]?.reason ?? 'no match'}.`.slice( - 0, - EDIT_DIAGNOSTIC_MAX_CHARS, + const prefix = 'Workspace edit did not apply and nothing was written: '; + const detailed = `${prefix}${failures[0] ? failureText(failures[0], true) : 'no match'}.`; + if (fitsDiagnosticBounds(detailed)) return detailed; + const reason = shortenReason( + failures[0]?.reason ?? 'no match', + EDIT_DIAGNOSTIC_MAX_CHARS - prefix.length - 1, + EDIT_DIAGNOSTIC_MAX_BODY_BYTES - encodedBytes(prefix) - 1, ); + return `${prefix}${reason}.`; } const header = `${failures.length} of ${editCount} workspace edits did not apply, so nothing was written. Every other edit matched.`; const footer = failures.some((failure) => failure.index > 0) @@ -696,17 +899,16 @@ function formatEditFailures(failures: readonly EditFailure[], editCount: number) // reasons or source excerpts. Never leave callers guessing which edits failed. const available = EDIT_DIAGNOSTIC_MAX_CHARS - header.length - footer.length - prefixes.reduce((total, prefix) => total + prefix.length + 1, 0); - const reasonLength = failures.reduce((total, failure) => total + failure.reason.length, 0); - const reasonLimit = reasonLength <= available ? Infinity : Math.floor(available / failures.length); - const details = failures.map((failure, index) => { - let reason = failure.reason; - if (reason.length > reasonLimit) { - let end = reasonLimit - 1; - // Do not split a surrogate pair in a shortened source excerpt. - if (end > 0 && /[\uD800-\uDBFF]/.test(reason[end - 1]) && /[\uDC00-\uDFFF]/.test(reason[end])) end--; - reason = `${reason.slice(0, end)}…`; - } - return `${prefixes[index]}${reason}.`; - }); + const frame = header + footer + prefixes.join('') + '.'.repeat(failures.length); + const availableBytes = EDIT_DIAGNOSTIC_MAX_BODY_BYTES - encodedBytes(frame); + const granted = grantExcerpts(failures, available, availableBytes); + const reasons = failures.map((failure, index) => failureText(failure, granted[index])); + const reasonLength = reasons.reduce((total, reason) => total + reason.length, 0); + const reasonBytes = reasons.reduce((total, reason) => total + encodedBytes(reason), 0); + const charLimit = reasonLength <= available ? Infinity : Math.floor(available / failures.length); + const byteLimit = reasonBytes <= availableBytes ? Infinity : Math.floor(availableBytes / failures.length); + const details = reasons.map((reason, index) => + `${prefixes[index]}${shortenReason(reason, charLimit, byteLimit)}.`, + ); return header + details.join('') + footer; } From 961777e3fd1347e9c0b571c02292371e4a6d8d86 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:33:34 -0400 Subject: [PATCH 64/75] feat: Persist Workspace Admission Across API Restarts (#302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧾 feat: Persist Workspace Admission Across API Restarts * 🧾 fix: Fence Durable Admission Recovery Across Replicas * 🧾 fix: Fence Durable Capacity Acquisition Before Dispatch * 🧾 fix: Retain Durable Quarantine Outcomes Across Restarts * 🧾 fix: Preserve Durable Admission and Read Cancellation --------- Co-authored-by: Lia --- .github/workflows/ci.yml | 2 +- docs/remote-bridge/README.md | 55 ++ service/openapi.yml | 105 +++ service/src/bridge/admission.ts | 55 ++ service/src/bridge/slots.ts | 22 +- service/src/bridge/store.ts | 235 +++++- service/src/workspace-tools/coordination.ts | 21 + service/src/workspace-tools/index.ts | 49 +- service/src/workspace-tools/outcome.ts | 4 +- .../workspace-tools/requests-router.test.ts | 96 +++ service/src/workspace-tools/requests.test.ts | 671 ++++++++++++++++++ service/src/workspace-tools/requests.ts | 251 +++++++ service/src/workspace-tools/router.ts | 320 +++++---- 13 files changed, 1718 insertions(+), 168 deletions(-) create mode 100644 service/src/workspace-tools/coordination.ts create mode 100644 service/src/workspace-tools/requests-router.test.ts create mode 100644 service/src/workspace-tools/requests.test.ts create mode 100644 service/src/workspace-tools/requests.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87aaa066..5ee14372 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,7 @@ jobs: redis_socket="$RUNNER_TEMP/byom-admission.sock" redis-server --port 0 --unixsocket "$redis_socket" --save '' --appendonly no --daemonize yes trap 'redis-cli -s "$redis_socket" shutdown nosave' EXIT - BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts + BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts src/workspace-tools/requests.test.ts - name: Build service run: bun run build diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index b2d33db7..dc7cb98b 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -384,3 +384,58 @@ capabilities. LibreChat's owner-scoped environment registry can issue these principal-bound pairings without changing the worker execution protocol or moving code tools into the Agents SDK. + +### Durable workspace requests + +New clients can probe authenticated `GET /v1/workspace-tools/capabilities` for +`durableWorkspaceRequests: 1`. Older servers return 404 or advertise 0. Keep the +legacy synchronous `/workspace-tools/execute` path for those servers. Do not +fall back to synchronous execution after submitting durable work. + +Submit the same workspace-tool body to `POST /v1/workspace-tools/requests`, with +`X-LibreChat-Workspace-Request-Id` set to a unique 16–128 character identifier +(letters, digits, `_`, `-`). The queue-wait header and command timeout retain their +existing limits. HTTP 202 acknowledges durable acceptance, not execution. +Disconnecting does not cancel accepted work. + +- Repeat an identical submission with the same ID to recover a lost response. + An ID reused for different work returns 409 `REQUEST_CONFLICT`. +- Poll `GET /v1/workspace-tools/requests/:requestId`. States are `queued`, + `admitted`, `completed`, `failed`, and `cancelled`. Lookup is tenant/user scoped; + unknown or another principal's IDs return 404. Results are non-consuming reads. +- Cancel with `DELETE /v1/workspace-tools/requests/:requestId`. Cancellation is a + request, not proof of termination. All operations notify the worker. A committed + result can win the race. Without a confirmed settlement, read-only calls retain + `cancelled`; mutations retain an unknown-outcome failure and their safety fence. + `ASSIGNMENT_EXPIRED` means execution may have occurred. Never replay it. +- Status includes worker/workspace/lane metadata, queue position when available, + measured queue wait, and the execution deadline after admission. Queue position + is advisory: negotiated independent roots may run concurrently. + +Requests and results are retained for 24 hours from acceptance. Idempotency is +bounded by that retention window. A client must not resubmit an expired ID after +404; its outcome is unknown. Automatic result claims and wake-ups remain the +client's responsibility. + +Bridge-enabled API replicas reconcile accepted work from Redis within a shared +bridge-policy scope. Disabled bridges and different scheduling policies never +claim that work. Keep backend/profile, bridge auth mode, worker selection, slot +ceiling and command-timeout policy identical on replicas behind one endpoint. +Drain durable requests before changing those settings. FIFO admission position +survives API restarts and temporary worker registration/readiness loss until the +queue deadline. Capacity acquisition and assignment enqueue atomically check the coordinator +claim, queued state, cancellation and admission deadline. Delayed callbacks +cannot reserve capacity after their claim ends. Assignment enqueue also checks +capacity ownership and atomically commits the durable `admitted` transition. +Recovery resumes only queued work and observes admitted assignments without +creating another command. Worker identity/incarnation changes fail pending work +rather than transferring it to another machine. Execution gets its full budget +after admission. Existing lease fencing, cancellation and workspace quarantine +continue to apply. + +This capability covers workspace tools, including `execute_command`, not +programmatic `/execute` jobs. It requires Redis state to survive the API restart; +Redis data loss is not recoverable. All API replicas behind an endpoint must +support the capability before a client enables it. Roll back clients to the legacy +path only for new work; drain durable requests before rolling back servers. +Worker protocol, worker concurrency and deployed rate limits are unchanged. diff --git a/service/openapi.yml b/service/openapi.yml index a257fd18..07d2d7e3 100644 --- a/service/openapi.yml +++ b/service/openapi.yml @@ -64,6 +64,40 @@ components: - $ref: '#/components/schemas/ExecutionProfileError' schemas: + WorkspaceToolRequest: + type: object + description: Workspace tool protocol v1 body; operation-specific fields follow the worker protocol. + required: [protocolVersion, operation, workspaceId] + properties: + protocolVersion: { type: integer, enum: [1] } + operation: + type: string + enum: [list_files, search_text, read_file, write_file, preview_edit, edit_file, execute_command] + workspaceId: { type: string } + additionalProperties: true + + WorkspaceRequestStatus: + type: object + required: [requestId, state, workerId, workspaceId, queueWaitMs, cancelRequested] + properties: + requestId: { type: string } + state: { type: string, enum: [queued, admitted, completed, failed, cancelled] } + workerId: { type: string } + workspaceId: { type: string } + workspaceInstanceId: { type: string } + worktree: { type: string } + queuePosition: { type: integer, minimum: 1 } + queueWaitMs: { type: integer, minimum: 0 } + executionDeadlineAt: { type: string, format: date-time } + cancelRequested: { type: boolean } + result: { type: object, additionalProperties: true } + error: + type: object + required: [code, message] + properties: + code: { type: string } + message: { type: string } + FileRef: type: object properties: @@ -312,6 +346,77 @@ components: type: string paths: + /workspace-tools/capabilities: + get: + summary: Discover opt-in durable workspace request support + responses: + '200': + description: All replicas must support version 1 before a client enables durable submission. + content: + application/json: + schema: + type: object + properties: + durableWorkspaceRequests: { type: integer, enum: [0, 1] } + + /workspace-tools/requests: + post: + summary: Submit an idempotent durable workspace request + description: >- + Disconnect does not cancel accepted work. IDs and results are retained for 24 hours + from acceptance. Never resubmit after that retention window or fall back to synchronous + execution after durable acceptance. The execution budget begins after admission. + parameters: + - $ref: '#/components/parameters/ExpectedExecutionProfile' + - in: header + name: X-LibreChat-Workspace-Request-Id + required: true + schema: { type: string, pattern: '^[A-Za-z0-9_-]{16,128}$' } + - in: header + name: X-LibreChat-Workspace-Queue-Wait-Ms + schema: { type: integer, minimum: 1, maximum: 300000, default: 30000 } + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceToolRequest' } + responses: + '202': + description: Accepted, or an identical request with this principal-scoped ID already exists. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '400': { $ref: '#/components/responses/BadRequest' } + '409': { $ref: '#/components/responses/Conflict' } + '429': { description: Admission capacity or execution rate limit reached } + '503': { description: Code environment unavailable } + + /workspace-tools/requests/{requestId}: + parameters: + - in: path + name: requestId + required: true + schema: { type: string, pattern: '^[A-Za-z0-9_-]{16,128}$' } + - $ref: '#/components/parameters/ExpectedExecutionProfile' + get: + summary: Read a retained workspace request and result without consuming it + responses: + '200': + description: Status scoped to the authenticated tenant and user. ASSIGNMENT_EXPIRED is not proof of non-execution. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '404': { description: Unknown, expired, or another principal's request } + delete: + summary: Request cancellation of the same logical call + responses: + '200': + description: Cancellation recorded, or terminal status returned. A committed result can win the race. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '404': { description: Unknown, expired, or another principal's request } + /hosted-apps: post: summary: Start or reassert a resident hosted app diff --git a/service/src/bridge/admission.ts b/service/src/bridge/admission.ts index 4002ed6d..18ab1f49 100644 --- a/service/src/bridge/admission.ts +++ b/service/src/bridge/admission.ts @@ -1,5 +1,22 @@ import type Redis from 'ioredis'; +/** Redis time and claim ownership gate every durable admission mutation. */ +export function durableAdmissionFence( + requestKey: string, + claimKey: string, + token: string, + rejected: number, +): string[] { + return [ + `if redis.call('GET', ${claimKey}) ~= ${token} then return ${rejected} end`, + `if redis.call('HGET', ${requestKey}, 'state') ~= 'queued' then return ${rejected} end`, + `if redis.call('HGET', ${requestKey}, 'cancelRequested') == '1' then return ${rejected} end`, + 'local admissionTime = redis.call(\'TIME\')', + 'local admissionNowMs = tonumber(admissionTime[1]) * 1000.0 + math.floor(tonumber(admissionTime[2]) / 1000)', + `if tonumber(redis.call('HGET', ${requestKey}, 'queueDeadlineAtMs')) <= admissionNowMs then return ${rejected} end`, + ]; +} + /** Bounded FIFO admission shared by API replicas. Entries expire after caller deadlines. */ export class BridgeAdmissionQueue { constructor( @@ -57,6 +74,44 @@ export class BridgeAdmissionQueue { ); } + async submit(args: { + workerId: string; id: string; deadlineAtMs: number; workspaceId?: string; + key: string; activeKey: string; fingerprint: string; record: string; retentionMs: number; + }): Promise<'accepted' | 'existing' | 'conflict' | 'full'> { + const result = Number(await this.redis.eval([ + 'local fingerprint = redis.call(\'HGET\', KEYS[5], \'fingerprint\')', + 'if fingerprint then', + ' if fingerprint == ARGV[6] then return 2 end', + ' return -1', + 'end', + 'local expired = redis.call(\'ZRANGEBYSCORE\', KEYS[2], \'-inf\', ARGV[2])', + 'for _, id in ipairs(expired) do', + ' redis.call(\'ZREM\', KEYS[1], id); redis.call(\'ZREM\', KEYS[2], id); redis.call(\'HDEL\', KEYS[4], id)', + 'end', + 'if redis.call(\'ZCARD\', KEYS[1]) >= tonumber(ARGV[4]) then return 0 end', + 'local sequence = redis.call(\'INCR\', KEYS[3])', + 'redis.call(\'ZADD\', KEYS[1], sequence, ARGV[1])', + 'redis.call(\'ZADD\', KEYS[2], ARGV[3], ARGV[1])', + 'if ARGV[5] ~= \'\' then redis.call(\'HSET\', KEYS[4], ARGV[1], ARGV[5]) end', + 'local latest = redis.call(\'ZREVRANGE\', KEYS[2], 0, 0, \'WITHSCORES\')', + 'for i = 1, 4 do redis.call(\'PEXPIREAT\', KEYS[i], tonumber(latest[2]) + 30000) end', + 'redis.call(\'HSET\', KEYS[5], \'record\', ARGV[7], \'fingerprint\', ARGV[6], \'state\', \'queued\', \'queueDeadlineAtMs\', ARGV[3])', + 'redis.call(\'PEXPIRE\', KEYS[5], ARGV[8])', + 'redis.call(\'ZADD\', KEYS[6], ARGV[2], KEYS[5])', + 'return 1', + ].join('\n'), 6, ...this.keys(args.workerId), args.key, args.activeKey, + args.id, Date.now(), args.deadlineAtMs, this.capacity, args.workspaceId ?? '', + args.fingerprint, args.record, args.retentionMs)); + if (result === 2) return 'existing'; + if (result === -1) return 'conflict'; + return result === 1 ? 'accepted' : 'full'; + } + + async position(workerId: string, id: string): Promise { + const rank = await this.redis.zrank(this.keys(workerId)[0], id); + return rank == null ? undefined : rank + 1; + } + async isHead(workerId: string, id: string): Promise { const [order, deadlines, , workspaces] = this.keys(workerId); return ( diff --git a/service/src/bridge/slots.ts b/service/src/bridge/slots.ts index 81f60ebe..f748054b 100644 --- a/service/src/bridge/slots.ts +++ b/service/src/bridge/slots.ts @@ -1,4 +1,5 @@ import type Redis from 'ioredis'; +import { durableAdmissionFence } from './admission'; /** Hard bound keeps every atomic scheduling scan constant-sized. */ export const MAX_WORKSPACE_LEASE_SLOTS = 8; @@ -37,6 +38,8 @@ export class BridgeWorkspaceSlots { workspaceId: string; capacity: number; expiresAtMs: number; + refreshOwned?: boolean; + guard?: { key: string; claimKey: string; token: string }; }): Promise { if ( !Number.isSafeInteger(args.capacity) || @@ -48,9 +51,12 @@ export class BridgeWorkspaceSlots { ) { throw new Error('Invalid workspace slot reservation'); } + const keys = this.keys(args.workerId); + if (args.guard != null) keys.push(args.guard.key, args.guard.claimKey); const result = Number( await this.redis.eval( [ + ...(args.guard == null ? [] : durableAdmissionFence('KEYS[9]', 'KEYS[10]', 'ARGV[8]', -3)), "if redis.call('GET', KEYS[4]) ~= ARGV[1] then return -2 end", "if (redis.call('GET', KEYS[8]) or '1') ~= ARGV[4] then return -2 end", // Parent of a linked-worktree lane key; nil for a checkout key. @@ -83,7 +89,15 @@ export class BridgeWorkspaceSlots { " redis.call('HDEL', KEYS[1], 'a:' .. slot, 'i:' .. slot, 'w:' .. slot, 'e:' .. slot)", ' occupied = false', ' else', - ' if entry[1] == ARGV[2] then return slot end', + ' if entry[1] == ARGV[2] then', + ' if ARGV[7] == \'1\' then', + ' redis.call(\'HSET\', KEYS[1], \'e:\' .. slot, ARGV[5])', + ' for _, key in ipairs({KEYS[1], KEYS[2], KEYS[3]}) do', + ' if redis.call(\'PTTL\', key) < tonumber(ARGV[5]) - tonumber(ARGV[6]) then redis.call(\'PEXPIREAT\', key, ARGV[5]) end', + ' end', + ' end', + ' return slot', + ' end', ' busy[entry[3]] = true', ' local busyParent = parentOf(entry[3])', ' if busyParent then busyChildren[busyParent] = true end', @@ -125,14 +139,16 @@ export class BridgeWorkspaceSlots { "redis.call('SET', KEYS[3], ARGV[1], 'PXAT', latest)", 'return free', ].join('\n'), - 8, - ...this.keys(args.workerId), + keys.length, + ...keys, args.incarnationId, args.assignmentId, args.workspaceId, args.capacity, args.expiresAtMs, Date.now(), + args.refreshOwned === true ? '1' : '0', + args.guard?.token ?? '', ), ); if (result === -2) diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index f64e748a..542c5903 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -21,8 +21,9 @@ import { workspaceIsolationParent, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; -import { BridgeAdmissionQueue } from './admission'; +import { BridgeAdmissionQueue, durableAdmissionFence } from './admission'; import { BridgeWorkspaceSlots } from './slots'; +import type { StoredWorkspaceRequest } from '../workspace-tools/requests'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; @@ -90,6 +91,22 @@ interface StoredAssignment extends CodeBridgeAssignment { leaseTokenHash: string; workerIdentityId?: string; workspaceFence?: string; + durableRequestKey?: string; +} + +export type DurableWorkspaceAssignment = StoredAssignment; + +export interface DurableWorkspaceGuard { + key: string; + claimKey: string; + token: string; +} + +export interface DurableWorkspaceOutcome { + state: 'completed' | 'failed' | 'cancelled'; + result?: WorkspaceToolResult; + error?: { code: string; message: string }; + settlement?: CodeBridgeWorkspaceSettlement; } type AssignmentOwnership = Pick< @@ -104,6 +121,7 @@ type AssignmentOwnership = Pick< | 'workerIdentityId' | 'expiresAt' | 'runtimeSessionId' + | 'durableRequestKey' >; const NATIVE_WORKSPACE_FENCE_PREFIX = 'native-workspace:'; @@ -148,6 +166,10 @@ export interface BridgeWorkerStatus { capabilities?: BridgeWorkerRegistration['capabilities']; } +function isWorkspaceMutation(request: WorkspaceToolRequest): boolean { + return request.operation === 'write_file' || request.operation === 'edit_file' || request.operation === 'execute_command'; +} + function supportsWorkspaceTool( registration: RegisteredBridgeWorker, request: WorkspaceToolRequest, @@ -381,6 +403,12 @@ function assignmentTtlSeconds(deadlineAtMs: number): number { return Math.max(1, Math.ceil((deadlineAtMs - Date.now()) / 1000) + 30); } +function assignmentOutcomeTtlSeconds(assignment: AssignmentOwnership): number { + return assignment.durableRequestKey == null + ? assignmentTtlSeconds(Date.parse(assignment.expiresAt)) + : 24 * 60 * 60; +} + async function delay(ms: number, signal?: AbortSignal): Promise { if (signal?.aborted === true) return; await new Promise((resolve) => { @@ -775,6 +803,127 @@ export class RedisBridgeStore { } } + async prepareDurableWorkspaceTool(args: { + workerId: string; tenantId: string; requireTenantBinding: boolean; + request: WorkspaceToolRequest; executionTimeoutMs: number; queueWaitMs: number; + }): Promise { + if (!isWorkspaceToolRequest(args.request) || + !Number.isSafeInteger(args.executionTimeoutMs) || args.executionTimeoutMs < 1 || args.executionTimeoutMs > 305_000 || + !Number.isSafeInteger(args.queueWaitMs) || args.queueWaitMs < 1 || args.queueWaitMs > 300_000) { + throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid durable workspace request'); + } + const current = await boundedCommand(this.dispatchableRegistration(args.workerId), this.redisCommandTimeoutMs, 'Durable worker registration'); + if (current == null) throw new BridgeStoreError('WORKER_OFFLINE', 'Code environment is offline'); + const registration = current.registration; + if ((args.requireTenantBinding && registration.binding == null) || + (registration.binding != null && registration.binding.tenantId !== args.tenantId)) { + throw new BridgeStoreError('WORKER_UNAUTHORIZED', 'Code environment is not authorized for this tenant'); + } + if ((registration.capabilities.workspaceLeaseSlots ?? 1) > this.maxWorkspaceLeaseSlots || + !supportsWorkspaceTool(registration, args.request)) { + throw new BridgeStoreError('WORKER_MISMATCH', 'Code environment does not support the requested workspace operation'); + } + return registration; + } + + async advanceDurableWorkspaceTool( + record: StoredWorkspaceRequest, + guard: DurableWorkspaceGuard, + ): Promise { + if (record.state === 'queued') { + if (record.cancelRequested === true) return { state: 'cancelled' }; + if (Date.now() >= record.queueDeadlineAtMs) return { + state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT', message: 'Workspace admission deadline expired. The operation was not started.' }, + }; + const registration = await this.prepareDurableWorkspaceTool({ + ...record, queueWaitMs: Math.max(1, record.queueDeadlineAtMs - Date.now()), + }); + if (registration.incarnationId !== record.registration.incarnationId || + registration.identityId !== record.registration.identityId || + registration.binding?.tenantId !== record.registration.binding?.tenantId) { + throw new BridgeStoreError('WORKER_FENCED', 'Code environment changed while the request was waiting'); + } + const admission = new BridgeAdmissionQueue(this.redis); + const workspace = workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree); + const capacity = registration.capabilities.workspaceLeaseSlots ?? 1; + if (capacity === 1 && !await admission.isHead(record.workerId, record.id)) return; + const expiresAtMs = Date.now() + record.executionTimeoutMs; + const ttlSeconds = assignmentTtlSeconds(expiresAtMs); + let slot: number | undefined; + if (capacity > 1) { + slot = await new BridgeWorkspaceSlots(this.redis).reserve({ + workerId: record.workerId, incarnationId: registration.incarnationId, + assignmentId: record.id, workspaceId: workspace, capacity, + expiresAtMs: Date.now() + ttlSeconds * 1000, refreshOwned: true, guard, + }); + if (slot === undefined) return; + } else if (!await this.acquireLock(record.workerId, record.id, registration.incarnationId, ttlSeconds, true, guard)) return; + if (Date.now() >= record.queueDeadlineAtMs) return { + state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT', message: 'Workspace admission deadline expired. The operation was not started.' }, + }; + const leaseToken = randomBytes(32).toString('base64url'); + const assignment: StoredAssignment = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + assignmentId: record.id, workerId: record.workerId, + incarnationId: registration.incarnationId, + generation: await this.redis.incr(generationKey(record.workerId)), + leaseToken, leaseTokenHash: tokenHash(leaseToken), + workerIdentityId: registration.identityId, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${workspace}`, + workspaceLeaseSlot: slot, executionKind: 'workspace_tool', durableRequestKey: record.key, + expiresAt: new Date(Date.now() + record.executionTimeoutMs).toISOString(), + request: record.request, + }; + const ready = await this.dispatchableRegistration(record.workerId); + if (ready == null) throw new BridgeStoreError('WORKER_OFFLINE', 'Code environment is temporarily unavailable'); + if (ready.registration.incarnationId !== registration.incarnationId || + ready.registration.identityId !== registration.identityId || + ready.registration.binding?.tenantId !== registration.binding?.tenantId || + !supportsWorkspaceTool(ready.registration, record.request)) { + throw new BridgeStoreError('WORKER_FENCED', 'Code environment changed before admission'); + } + await this.enqueueForActiveIncarnation(assignment, assignmentTtlSeconds(Date.parse(assignment.expiresAt)), ready.readyToken, guard); + return; + } + const assignment = record.assignment; + if (assignment == null) throw new Error('Durable admitted request has no assignment'); + let settlement: CodeBridgeWorkspaceSettlement; + try { + const raw = await boundedCommand(this.redis.get(settlementKey(record.id)), this.redisCommandTimeoutMs, 'Durable settlement lookup'); + if (raw == null && record.cancelRequested !== true && Date.now() < Date.parse(assignment.expiresAt)) return; + if (raw != null) settlement = JSON.parse(raw) as CodeBridgeWorkspaceSettlement; + else { + const controller = new AbortController(); + if (record.cancelRequested === true) controller.abort(); + settlement = await this.waitForSettlement(assignment, Date.parse(assignment.expiresAt), controller.signal) as unknown as CodeBridgeWorkspaceSettlement; + } + } catch (error) { + if (!(error instanceof BridgeStoreError) || error.code !== 'ASSIGNMENT_EXPIRED') throw error; + if (record.cancelRequested === true && !isWorkspaceMutation(record.request)) return { state: 'cancelled' }; + return { state: 'failed', error: { code: error.code, message: 'Assignment ended without a confirmed result. Do not replay this operation.' } }; + } + if (settlement.status === 'fulfilled') { + if (!isWorkspaceToolResult(record.request, settlement.result, record.registration.capabilities.workspaceTools)) { + return { state: 'failed', error: { code: 'RESULT_INVALID', message: 'Code environment returned an invalid workspace result. Do not replay this operation.' } }; + } + return { state: 'completed', result: settlement.result, settlement }; + } + return { + state: record.cancelRequested === true ? 'cancelled' : 'failed', + error: { code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', message: settlement.error }, settlement, + }; + } + + async finishDurableWorkspaceTool(record: StoredWorkspaceRequest): Promise { + if (record.assignment != null) { + if (record.settlement != null) await this.commitPendingWorkspace(record.assignment, record.settlement, record.key); + await this.cleanupDispatch(record.workerId, record.id, record.assignment); + } else { + await this.cleanupUnassignedSlot(record.workerId, record.registration.incarnationId, record.id); + } + await new BridgeAdmissionQueue(this.redis).leave(record.workerId, record.id); + } + async dispatchWorkspaceTool(args: { workerId: string; tenantId?: string; @@ -1414,6 +1563,7 @@ export class RedisBridgeStore { leaseTokenHash: _leaseTokenHash, workerIdentityId: _workerIdentityId, workspaceFence: _workspaceFence, + durableRequestKey: _durableRequestKey, ...wireAssignment } = assignment; return { @@ -1706,7 +1856,7 @@ export class RedisBridgeStore { 'Bridge assignment has expired', ); } - const ttlSeconds = assignmentTtlSeconds(Date.parse(assignment.expiresAt)); + const ttlSeconds = assignmentOutcomeTtlSeconds(assignment); const settlementKeys = [ assignmentKey(assignmentId), settlementKey(assignmentId), @@ -1985,7 +2135,7 @@ export class RedisBridgeStore { raw.epoch, assignmentId, JSON.stringify(settlement), - assignmentTtlSeconds(Date.parse(receipt.expiresAt)), + assignmentOutcomeTtlSeconds(receipt), quarantine ? '1' : '0', ), signal, @@ -2130,14 +2280,12 @@ export class RedisBridgeStore { const cancelledMutation = signal.aborted && (assignment.executionKind === 'workspace_programmatic' || - (workspaceRequest != null && - (workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file' || - workspaceRequest.operation === 'execute_command'))); - if (cancelledMutation) { + (workspaceRequest != null && isWorkspaceMutation(workspaceRequest))); + const cancelledDurableWorkspace = signal.aborted && assignment.durableRequestKey != null && workspaceRequest != null; + if (cancelledMutation || cancelledDurableWorkspace) { try { // Keep the acknowledged assignment available long enough for the - // worker to terminate its process tree and commit a clean rejection. + // worker to stop its operation and commit a clean rejection. // Closing it first makes that rejection impossible to acknowledge and // leaves the worker's durable mutation guard armed. await this.cancel(assignment.assignmentId, assignment); @@ -2237,17 +2385,28 @@ export class RedisBridgeStore { assignment: StoredAssignment, ttlSeconds: number, readyToken?: string, + guard?: DurableWorkspaceGuard, ): Promise { const script = [ + 'local keyCount = tonumber(ARGV[10])', + ...(guard == null ? [] : [ + ...durableAdmissionFence('KEYS[keyCount + 1]', 'KEYS[keyCount + 2]', 'ARGV[11]', -2), + ...(assignment.workspaceLeaseSlot === undefined + ? ['if redis.call(\'GET\', KEYS[keyCount + 3]) ~= ARGV[4] or redis.call(\'GET\', KEYS[4]) ~= ARGV[1] then return -2 end'] + : [ + `if redis.call('HGET', KEYS[keyCount + 3], 'a:${assignment.workspaceLeaseSlot}') ~= ARGV[4] then return -2 end`, + `if redis.call('HGET', KEYS[keyCount + 3], 'i:${assignment.workspaceLeaseSlot}') ~= ARGV[1] then return -2 end`, + ]), + ]), "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", 'if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[6]) ~= ARGV[7] then return 0 end', - "if #KEYS >= 7 and redis.call('EXISTS', KEYS[7]) == 1 then return -1 end", + 'if keyCount >= 7 and redis.call(\'EXISTS\', KEYS[7]) == 1 then return -1 end', // A lane refuses a quarantined checkout. A checkout reaches enqueue only once no // lane beneath it holds a slot, so any lane fence still indexed here is stuck. "if ARGV[9] == 'lane' and redis.call('EXISTS', KEYS[10]) == 1 then return -1 end", "if ARGV[9] == 'checkout' then", - " for i = 11, #KEYS do if redis.call('EXISTS', KEYS[i]) == 1 then return -1 end end", - " for i = 11, #KEYS do redis.call('SREM', KEYS[10], KEYS[i]) end", + ' for i = 11, keyCount do if redis.call(\'EXISTS\', KEYS[i]) == 1 then return -1 end end', + ' for i = 11, keyCount do redis.call(\'SREM\', KEYS[10], KEYS[i]) end', 'end', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"EX\", ARGV[3])', "redis.call('RPUSH', KEYS[3], ARGV[4])", @@ -2256,15 +2415,18 @@ export class RedisBridgeStore { ? ['redis.call(\'SET\', KEYS[4], ARGV[1], \"PX\", ARGV[5])'] : []), 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', - "if #KEYS >= 7 then redis.call('SET', KEYS[7], ARGV[4]) end", + 'if keyCount >= 7 then redis.call(\'SET\', KEYS[7], ARGV[4]) end', "if ARGV[9] == 'lane' then redis.call('SADD', KEYS[11], KEYS[7]) end", - 'if #KEYS >= 9 then', + 'if keyCount >= 9 then', " local epoch = redis.call('GET', KEYS[9])", - " if type(epoch) ~= 'string' then epoch = '0'; redis.call('SET', KEYS[9], epoch, 'EX', ARGV[3]) end", - " if redis.call('PTTL', KEYS[9]) < tonumber(ARGV[3]) * 1000 then redis.call('EXPIRE', KEYS[9], ARGV[3]) end", + ' if type(epoch) ~= \'string\' then epoch = \'0\'; redis.call(\'SET\', KEYS[9], epoch, \'EX\', ARGV[12]) end', + ' if redis.call(\'PTTL\', KEYS[9]) < tonumber(ARGV[12]) * 1000 then redis.call(\'EXPIRE\', KEYS[9], ARGV[12]) end', " redis.call('HSET', KEYS[8], 'metadata', ARGV[8], 'epoch', epoch)", - " redis.call('EXPIRE', KEYS[8], ARGV[3])", + ' redis.call(\'EXPIRE\', KEYS[8], ARGV[12])', 'end', + ...(guard == null ? [] : [ + 'redis.call(\'HSET\', KEYS[keyCount + 1], \'state\', \'admitted\', \'assignment\', ARGV[2], \'admittedAtMs\', admissionNowMs)', + ]), 'return 1', ].join('\n'); const keys = [ @@ -2297,6 +2459,7 @@ export class RedisBridgeStore { leaseTokenHash: assignment.leaseTokenHash, workerIdentityId: assignment.workerIdentityId, expiresAt: assignment.expiresAt, + durableRequestKey: assignment.durableRequestKey, }; let fenceScope: '' | 'lane' | 'checkout' = ''; if (assignment.workspaceLeaseSlot !== undefined) { @@ -2322,6 +2485,10 @@ export class RedisBridgeStore { fenceScope = 'checkout'; } } + const keyCount = keys.length; + if (guard != null) keys.push(guard.key, guard.claimKey, + assignment.workspaceLeaseSlot === undefined ? lockKey(assignment.workerId) + : `${workerKey(assignment.workerId)}:workspace-slots`); const result = await this.redis.eval( script, keys.length, @@ -2335,6 +2502,9 @@ export class RedisBridgeStore { readyToken ?? '', JSON.stringify(receipt), fenceScope, + keyCount, + guard?.token ?? '', + guard == null ? ttlSeconds : assignmentOutcomeTtlSeconds(assignment), ); if (Number(result) === -1) { throw new BridgeStoreError( @@ -2350,21 +2520,31 @@ export class RedisBridgeStore { assignmentId: string, incarnationId: string, ttlSeconds: number, + reuseOwned = false, + guard?: DurableWorkspaceGuard, ): Promise { const script = [ + ...(guard == null ? [] : [ + ...durableAdmissionFence('KEYS[3]', 'KEYS[4]', 'ARGV[5]', 0), + 'if redis.call(\'GET\', KEYS[5]) ~= ARGV[2] then return 0 end', + ]), + 'if ARGV[4] == \'1\' and redis.call(\'GET\', KEYS[1]) == ARGV[1] and redis.call(\'GET\', KEYS[2]) == ARGV[2] then redis.call(\'PEXPIRE\', KEYS[1], ARGV[3]); redis.call(\'PEXPIRE\', KEYS[2], ARGV[3]); return 1 end', 'if redis.call(\'EXISTS\', KEYS[1]) == 1 then return 0 end', 'redis.call(\'SET\', KEYS[1], ARGV[1], \"PX\", ARGV[3])', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"PX\", ARGV[3])', 'return 1', ].join('\n'); + const keys = [lockKey(workerId), lockIncarnationKey(workerId)]; + if (guard != null) keys.push(guard.key, guard.claimKey, workerIncarnationKey(workerId)); const result = await this.redis.eval( script, - 2, - lockKey(workerId), - lockIncarnationKey(workerId), + keys.length, + ...keys, assignmentId, incarnationId, String(ttlSeconds * 1000), + reuseOwned ? '1' : '0', + guard?.token ?? '', ); return Number(result) === 1; } @@ -2412,6 +2592,7 @@ export class RedisBridgeStore { private async commitPendingWorkspace( assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, + durableKey?: string, ): Promise { if (assignment.workspaceLeaseSlot !== undefined) { const committed = Number( @@ -2450,8 +2631,17 @@ export class RedisBridgeStore { } const runtimeSessionId = assignmentWorkspace(assignment)!; const script = [ + ...(durableKey == null ? [] : [ + 'if redis.call(\'EXISTS\', KEYS[2]) == 1 then return 1 end', + ]), "if redis.call('GET', KEYS[1]) == ARGV[1] then", - " return redis.call('DEL', KEYS[1])", + ...(durableKey == null + ? [' return redis.call(\'DEL\', KEYS[1])'] + : [ + ' redis.call(\'DEL\', KEYS[1])', + ' redis.call(\'SET\', KEYS[2], \'1\', \'PX\', 86400000)', + ' return 1', + ]), 'end', 'return 0', ].join('\n'); @@ -2459,8 +2649,9 @@ export class RedisBridgeStore { await boundedCommand( this.redis.eval( script, - 1, + durableKey == null ? 1 : 2, workspaceQuarantineKey(assignment.workerId, runtimeSessionId), + ...(durableKey == null ? [] : [`${durableKey}:committed:${assignment.assignmentId}`]), assignment.assignmentId, ), // Once settlement wins, caller cancellation must not prevent its diff --git a/service/src/workspace-tools/coordination.ts b/service/src/workspace-tools/coordination.ts new file mode 100644 index 00000000..0bcd5e91 --- /dev/null +++ b/service/src/workspace-tools/coordination.ts @@ -0,0 +1,21 @@ +import { createHash } from 'crypto'; + +export interface WorkspaceRequestCoordinationPolicy { + bridgeEnabled: boolean; + backend: string; + executionProfile: string; + authMode: string; + configuredWorkerId: string; + dynamicWorkers: boolean; + maxWorkspaceLeaseSlots: number; + maxCommandTimeoutMs: number; +} + +/** Only replicas with the same bridge policy may reconcile accepted work. */ +export function workspaceRequestCoordinationScope(policy: WorkspaceRequestCoordinationPolicy): string | undefined { + if (!policy.bridgeEnabled) return undefined; + return createHash('sha256').update(JSON.stringify([ + policy.backend, policy.executionProfile, policy.authMode, policy.configuredWorkerId, + policy.dynamicWorkers, policy.maxWorkspaceLeaseSlots, policy.maxCommandTimeoutMs, + ])).digest('hex'); +} diff --git a/service/src/workspace-tools/index.ts b/service/src/workspace-tools/index.ts index 6f105f16..3c7caee2 100644 --- a/service/src/workspace-tools/index.ts +++ b/service/src/workspace-tools/index.ts @@ -4,12 +4,59 @@ import { bridgeStore } from '../bridge'; import { env } from '../config'; import { executionLimiter } from '../middleware/limits'; import { createWorkspaceToolsRouter } from './router'; +import { RedisWorkspaceRequests } from './requests'; +import { connection } from '../queue'; +import logger from '../logger'; +import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; +import { withSpan } from '../telemetry'; +import { isBridgeEnabled } from '../bridge/enabled'; +import { workspaceRequestCoordinationScope } from './coordination'; const router = Router(); -router.use('/workspace-tools/execute', executionLimiter); +const coordinationScope = workspaceRequestCoordinationScope({ + bridgeEnabled: isBridgeEnabled(), backend: env.SANDBOX_BACKEND, + executionProfile: env.EXECUTION_PROFILE, authMode: env.BRIDGE_AUTH_MODE, + configuredWorkerId: env.BRIDGE_WORKER_ID, dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + maxWorkspaceLeaseSlots: env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, maxCommandTimeoutMs: env.JOB_TIMEOUT, +}); +const requests = coordinationScope === undefined ? undefined : new RedisWorkspaceRequests(connection, bridgeStore, record => { + const attributes = { + 'codeapi.request.id': record.id, + 'codeapi.admission.state': record.state, + 'codeapi.worker.id': record.workerId, + 'codeapi.workspace.id': record.request.workspaceId, + 'codeapi.workspace.instance_id': record.request.workspaceInstanceId ?? '', + 'codeapi.workspace.worktree': record.request.worktree ?? '', + 'codeapi.admission.wait_ms': Math.max(0, (record.admittedAtMs ?? Math.min(record.finishedAtMs ?? Date.now(), record.queueDeadlineAtMs)) - record.createdAtMs), + 'codeapi.admission.error_code': record.error?.code ?? '', + }; + logger.info('Durable workspace admission transition', attributes); + void withSpan('codeapi.workspace.admission.transition', attributes, () => undefined).catch(error => { + logger.warn('Admission transition telemetry failed', { error }); + }); +}, coordinationScope); +if (requests != null) { + const coordinator = requests; + let reconciling = false; + let unavailable = false; + const reconciliation = setInterval(() => { + if (reconciling || checkServiceStartUp() || checkServiceShutDown()) return; + reconciling = true; + void coordinator.reconcile().then(() => { unavailable = false; }).catch(error => { + if (!unavailable) logger.warn('Durable workspace reconciliation unavailable', { error }); + unavailable = true; + }).finally(() => { reconciling = false; }); + }, 250); + reconciliation.unref(); +} +router.use(['/workspace-tools/execute', '/workspace-tools/requests'], (req, res, next) => { + if (req.method === 'POST') executionLimiter(req, res, next); + else next(); +}); router.use( createWorkspaceToolsRouter({ store: bridgeStore, + requests, backend: env.SANDBOX_BACKEND, configuredWorkerId: env.BRIDGE_WORKER_ID, dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, diff --git a/service/src/workspace-tools/outcome.ts b/service/src/workspace-tools/outcome.ts index d965d515..e8dc100f 100644 --- a/service/src/workspace-tools/outcome.ts +++ b/service/src/workspace-tools/outcome.ts @@ -25,7 +25,7 @@ const earlyErrorCodes: Record = { 500: 'INTERNAL_ERROR', }; -export function getWorkspaceToolOutcome(res: Response): WorkspaceToolOutcome { +export function getWorkspaceToolOutcome(res: Response, route = '/workspace-tools/execute'): WorkspaceToolOutcome { const existing = outcomes.get(res); if (existing != null) return existing; const startedAt = performance.now(); @@ -39,7 +39,7 @@ export function getWorkspaceToolOutcome(res: Response): WorkspaceToolOutcome { outcomes.delete(res); const finished = res.writableFinished; logger.log(finished && res.statusCode < 400 ? 'info' : 'warn', 'Workspace tool request completed', { - route: '/workspace-tools/execute', + route, operation: outcome.operation, workerId: outcome.workerId, status: finished ? res.statusCode : undefined, diff --git a/service/src/workspace-tools/requests-router.test.ts b/service/src/workspace-tools/requests-router.test.ts new file mode 100644 index 00000000..856a7a38 --- /dev/null +++ b/service/src/workspace-tools/requests-router.test.ts @@ -0,0 +1,96 @@ +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import { afterEach, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { applyPrincipal } from '../auth/principal'; +import { RedisBridgeStore } from '../bridge/store'; +import { createWorkspaceToolsRouter } from './router'; +import { RedisWorkspaceRequests } from './requests'; + +let server: Server | undefined; +afterEach(() => { server?.close(); server = undefined; }); +const body = { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }; +const requestId = 'http-request-000001'; + +async function setup(durable = true, rejectSynchronous = false): Promise { + const redis = new RedisMock() as unknown as Redis; + const bridge = new RedisBridgeStore(redis); + await bridge.register({ protocolVersion: 1, workerId: 'worker', incarnationId: 'incarnation-00000001', capabilities: { + sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] }, + } }); + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + if (req.header('X-Test-User') !== 'unauthenticated') applyPrincipal(req, { + tenantId: 'tenant', userId: req.header('X-Test-User') ?? 'user', principalSource: 'local', codeWorkerId: 'worker', + }); + next(); + }); + app.use(createWorkspaceToolsRouter({ + store: rejectSynchronous ? { async dispatchWorkspaceTool(): Promise { throw new Error('Durable URL dispatched synchronously'); } } : bridge, requests: durable ? new RedisWorkspaceRequests(redis, bridge) : undefined, + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + return `http://127.0.0.1:${address.port}`; +} + +function post(url: string, id = requestId, value = body): Promise { + return fetch(`${url}/workspace-tools/requests`, { method: 'POST', headers: { + 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Request-Id': id, + }, body: JSON.stringify(value) }); +} + +test.each([false, true])('capability discovery explicitly distinguishes durable support (%s)', async durable => { + const url = await setup(durable); + expect(await (await fetch(`${url}/workspace-tools/capabilities`)).json()).toEqual({ durableWorkspaceRequests: durable ? 1 : 0 }); +}); + +test('submission, duplicate lookup and cancellation use the same principal-scoped handle without exposing leases', async () => { + const url = await setup(); + const accepted = await post(url); + expect(accepted.status).toBe(202); + expect(await accepted.json()).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + const duplicate = await post(url); + expect(duplicate.status).toBe(202); + const status = await (await fetch(`${url}/workspace-tools/requests/${requestId}`)).json(); + expect(status).toMatchObject({ requestId, state: 'queued' }); + expect(status).not.toHaveProperty('assignment'); + expect(status).not.toHaveProperty('leaseToken'); + expect((await fetch(`${url}/workspace-tools/requests/${requestId}`, { headers: { 'X-Test-User': 'another' } })).status).toBe(404); + const cancelled = await fetch(`${url}/workspace-tools/requests/${requestId}`, { method: 'DELETE' }); + expect(await cancelled.json()).toMatchObject({ requestId, cancelRequested: true }); +}); + +test('missing IDs and conflicting requests fail without dispatching', async () => { + const url = await setup(); + expect((await post(url, '')).status).toBe(400); + expect((await post(url)).status).toBe(202); + const conflict = await post(url, requestId, { ...body, path: 'different' }); + expect(conflict.status).toBe(409); + expect(await conflict.json()).toMatchObject({ code: 'REQUEST_CONFLICT' }); +}); + +test('unauthenticated request status and capability discovery are rejected', async () => { + const url = await setup(); + for (const path of ['capabilities', `requests/${requestId}`]) { + expect((await fetch(`${url}/workspace-tools/${path}`, { headers: { 'X-Test-User': 'unauthenticated' } })).status).toBe(401); + } +}); + +test.each(['/workspace-tools/requests/', '/workspace-tools/REQUESTS'])('durable route aliases remain idempotent (%s)', async path => { + const url = await setup(true, true); + for (let attempt = 0; attempt < 2; attempt++) { + const response = await fetch(`${url}${path}`, { method: 'POST', headers: { + 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Request-Id': requestId, + }, body: JSON.stringify(body) }); + expect(response.status).toBe(202); + expect(await response.json()).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + } + expect(await (await fetch(`${url}/workspace-tools/requests/${requestId}`)).json()).toMatchObject({ state: 'queued' }); +}); diff --git a/service/src/workspace-tools/requests.test.ts b/service/src/workspace-tools/requests.test.ts new file mode 100644 index 00000000..a0f7d99f --- /dev/null +++ b/service/src/workspace-tools/requests.test.ts @@ -0,0 +1,671 @@ +import { afterEach, expect, spyOn, test } from 'bun:test'; +import { createHash } from 'node:crypto'; +import RedisMock from 'ioredis-mock'; +import Redis from 'ioredis'; +import { RedisBridgeStore } from '../bridge/store'; +import { BridgeAdmissionQueue } from '../bridge/admission'; +import type { CodeBridgeAssignment } from '../bridge/store'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import { RedisWorkspaceRequests } from './requests'; +import { workspaceRequestCoordinationScope } from './coordination'; +import type { WorkspaceRequestCoordinationPolicy } from './coordination'; + +const testRedisUrl = process.env.BRIDGE_TEST_REDIS_URL; +const redis = testRedisUrl !== undefined && testRedisUrl.length > 0 + ? new Redis(testRedisUrl) + : new RedisMock() as unknown as Redis; +const bridge = new RedisBridgeStore(redis, 600, 1000, 2); +const owner = { tenantId: 'tenant', userId: 'user' }; +const workerId = 'durable-worker'; +const incarnationId = 'incarnation-00000001'; +const requestId = 'request-00000000001'; +const activeKey = 'codeapi:workspace-requests:v1:active'; +const queueKey = `codeapi:bridge:v1:worker:${workerId}:admission`; + +// Only a disposable test Redis may be supplied. +afterEach(async () => { await redis.flushall(); }); + +async function setup(slots = 1): Promise { + const generation = await bridge.register({ protocolVersion: 1, workerId, incarnationId, + capabilities: { + sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: ['bash'], + ...(slots > 1 ? { workspaceLeaseSlots: slots, requiresReadyConfirmation: true } : {}), + workspaceTools: { protocolVersion: 1, operations: ['read_file', 'execute_command', 'list_files', 'search_text', 'preview_edit'], + workspaces: [{ id: 'primary' }, { id: 'independent' }] }, + }, + }); + if (slots > 1) await bridge.confirmReady(workerId, incarnationId, generation); + return new RedisWorkspaceRequests(redis, bridge); +} + +function submit(requests: RedisWorkspaceRequests, id = requestId, workspaceId = 'primary'): ReturnType { + return requests.submit({ owner, requestId: id, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'read_file', workspaceId, path: 'README.md' }, + queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); +} + +async function tick(requests: RedisWorkspaceRequests): Promise { + const keys = await redis.zrange(activeKey, 0, -1); + for (const key of keys) await redis.zadd(activeKey, 0, key); + await requests.reconcile(); +} + +async function settle(assignment: CodeBridgeAssignment, fulfilled = true): Promise { + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, + assignment.generation, assignment.leaseToken); + await bridge.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, incarnationId, generation: assignment.generation, + leaseToken: assignment.leaseToken, + ...(fulfilled ? { status: 'fulfilled' as const, result: { protocolVersion: 1, operation: 'read_file', workspaceId: (assignment.request as { workspaceId: string }).workspaceId, path: 'README.md', content: 'hello', startLine: 1, endLine: 1, truncated: false } } + : { status: 'rejected' as const, error: 'stopped' }), + }); +} + +async function mutateRecord(update: (record: Record) => void): Promise { + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + update(record); + await redis.hset(key, 'record', JSON.stringify(record)); +} + +test('submission is idempotent across replicas, scoped by principal, and conflicts do not move FIFO position', async () => { + const requests = await setup(); + await submit(requests); + await submit(requests, 'request-00000000002'); + const replica = new RedisWorkspaceRequests(redis, bridge); + expect((await submit(replica)).queuePosition).toBe(1); + expect(await redis.zcard(queueKey)).toBe(2); + await expect(submit(replica, requestId, 'independent')).rejects.toThrow('different work'); + expect(await replica.get({ ...owner, userId: 'someone-else' }, requestId)).toBeUndefined(); + expect(await replica.cancel({ ...owner, tenantId: 'someone-else' }, requestId)).toBeUndefined(); +}); + +test('restart before dispatch preserves the queue and starts a full execution budget after sixty seconds waiting', async () => { + const requests = await setup(); + await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + await mutateRecord(record => { + record.createdAtMs = Number(record.createdAtMs) - 60_000; + record.queueDeadlineAtMs = Number(record.queueDeadlineAtMs) - 60_000; + }); + const record = JSON.parse((await redis.hget(key, 'record'))!); + await redis.hset(key, 'queueDeadlineAtMs', record.queueDeadlineAtMs); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis)); + await tick(restarted); + const assignment = await bridge.lease(workerId, incarnationId, 0); + expect(assignment).toBeDefined(); + expect(assignment!.remainingMs).toBeGreaterThan(29_000); + // Mock Redis TIME has subsecond clock skew. Execution still receives its full budget. + expect((await restarted.get(owner, requestId))?.queueWaitMs).toBeGreaterThan(59_000); + await settle(assignment!); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'completed', result: { content: 'hello' } }); +}); + +test.each([false, true])('restart around admission acknowledgement never enqueues another assignment (ack=%s)', async acknowledged => { + const requests = await setup(); + await submit(requests); + await tick(requests); + const assignment = await bridge.lease(workerId, incarnationId, 0); + if (acknowledged) await bridge.acknowledgeLease(workerId, incarnationId, assignment!.assignmentId, + assignment!.generation, assignment!.leaseToken); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis)); + await tick(restarted); + await submit(restarted); + expect((await bridge.lease(workerId, incarnationId, 0))?.assignmentId).toBe(assignment!.assignmentId); + expect(await redis.llen(`codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:assignments`)).toBe(0); + await settle(assignment!); + await tick(restarted); + expect((await restarted.get(owner, requestId))?.state).toBe('completed'); +}); + +test('a lost enqueue acknowledgement is reconciled rather than replayed', async () => { + const requests = await setup(); + await submit(requests); + const original = redis.eval.bind(redis); + const failure = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + const result = await original(...args); + if (String(args[0]).includes('\'state\', \'admitted\', \'assignment\'')) throw new Error('lost acknowledgement'); + return result; + }) as Redis['eval']); + try { await expect(tick(requests)).rejects.toThrow('lost acknowledgement'); } finally { failure.mockRestore(); } + const assignment = await bridge.lease(workerId, incarnationId, 0); + expect(assignment).toBeDefined(); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('admitted'); + await settle(assignment!); + await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); +}); + +test('cancel before admission is durable and does not dispatch', async () => { + const requests = await setup(); + await submit(requests); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); + expect(await redis.zcard(queueKey)).toBe(0); +}); + +test('cancel racing a persisted fulfillment returns the winning result without consuming it', async () => { + const requests = await setup(); + await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await settle(assignment); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + const statuses = await Promise.all([requests.get(owner, requestId), requests.get(owner, requestId), requests.cancel(owner, requestId)]); + for (const status of statuses) expect(status).toMatchObject({ state: 'completed', result: { content: 'hello' } }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('restart after result persistence repeats only idempotent cleanup', async () => { + const requests = await setup(); + await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await settle(assignment); + const cleanup = spyOn(bridge, 'finishDurableWorkspaceTool').mockRejectedValue(new Error('process stopped')); + try { await expect(tick(requests)).rejects.toThrow('process stopped'); } finally { cleanup.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('pending work rejects a replacement worker incarnation and never transfers to it', async () => { + const requests = await setup(); + await submit(requests); + await bridge.register({ protocolVersion: 1, workerId, incarnationId: 'replacement-00001', + capabilities: { sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + }); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKER_FENCED' } }); +}); + +test('durable requests preserve negotiated independent-root concurrency', async () => { + const requests = await setup(2); + await submit(requests); await submit(requests, 'request-00000000002', 'independent'); + await tick(requests); await tick(requests); + const first = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0); + const second = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1); + expect(first).toBeDefined(); expect(second).toBeDefined(); + await settle(first!); await settle(second!); await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +test.each([1, 2])('restart after capacity reservation refreshes its own reservation without duplicating work (slots=%s)', async slots => { + const requests = await setup(slots); + await submit(requests); + const generation = spyOn(redis, 'incr').mockRejectedValue(new Error('process stopped before enqueue')); + try { await expect(tick(requests)).rejects.toThrow('process stopped before enqueue'); } finally { generation.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + await redis.pexpire(`codeapi:bridge:v1:worker:${workerId}:lock`, 5000); + await tick(new RedisWorkspaceRequests(redis, bridge)); + const assignment = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0); + expect(assignment).toBeDefined(); + expect(await redis.pttl(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeGreaterThan(30_000); +}); + +test.each([1, 2])('a stale coordinator cannot acquire capacity or dispatch (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + await redis.set(`${key}:claim`, 'new-owner', 'PX', 10000); + await bridge.advanceDurableWorkspaceTool(record, { key, claimKey: `${key}:claim`, token: 'old-owner' }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); +}); + +test('a queued deadline expires as definitely unstarted work', async () => { + const requests = await setup(); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + await mutateRecord(record => { record.queueDeadlineAtMs = Date.now() - 1; }); + await redis.hset(key, 'queueDeadlineAtMs', Date.now() - 1); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT' } }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('an expired acknowledged command is unknown, remains fenced, and is not re-enqueued on restart', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + stored.expiresAt = new Date(Date.now() - 1).toISOString(); + await redis.hset(key, 'assignment', JSON.stringify(stored)); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'ASSIGNMENT_EXPIRED' } }); + await submit(requests); + expect(await redis.llen(`codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:assignments`)).toBe(0); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`)).toBe(assignment.assignmentId); +}); + +test('cancel during command execution waits for clean rejection and does not replay', async () => { + const requests = await setup(); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'sleep 30' }, + queueWaitMs: 300_000, executionTimeoutMs: 35_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + await requests.cancel(owner, requestId); + const cancelling = tick(requests); + const deadline = Date.now() + 1000; + while (!await bridge.cancelled(workerId, incarnationId, assignment.assignmentId)) { + if (Date.now() >= deadline) throw new Error('Worker never received cancellation'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await settle(assignment, false); + await cancelling; + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); +}); + +test('restart between workspace commit and cleanup preserves the result and releases the same assignment', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; await settle(assignment); + const original = redis.eval.bind(redis); + let failed = false; + const failure = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + if (!failed && String(args[0]).includes('local queued = redis.call(\'LREM\'')) { + failed = true; throw new Error('process stopped after commit'); + } + return original(...args); + }) as Redis['eval']); + try { await expect(tick(requests)).rejects.toThrow('process stopped after commit'); } finally { failure.mockRestore(); } + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + expect(await redis.zcard(activeKey)).toBe(0); +}); + +test('simultaneous submissions from replicas keep one request identity and one FIFO entry', async () => { + const requests = await setup(); + const replica = new RedisWorkspaceRequests(redis, bridge); + const statuses = await Promise.all([submit(requests), submit(replica)]); + for (const status of statuses) expect(status).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + expect(await redis.zcard(queueKey)).toBe(1); + expect(await redis.zcard(activeKey)).toBe(1); + await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeDefined(); +}); + +test.each([false, true])('temporary registration loss preserves FIFO until the original worker reconnects (replacement=%s)', async replacement => { + const requests = await setup(); + await submit(requests); await submit(requests, 'request-00000000002'); + // Registration/readiness TTL expiry, without losing accepted work. + const worker = `codeapi:bridge:v1:worker:${workerId}`; + await redis.del(worker, `${worker}:incarnation`, `${worker}:ready`); + const restarted = new RedisWorkspaceRequests(redis, bridge); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'queued', queuePosition: 1 }); + expect(await redis.zcard(activeKey)).toBe(2); + if (replacement) { + await bridge.register({ protocolVersion: 1, workerId, incarnationId: 'replacement-00001', + capabilities: { sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + }); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKER_FENCED' } }); + expect(await bridge.lease(workerId, 'replacement-00001', 0)).toBeUndefined(); + } else { + await setup(); await tick(restarted); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + expect(assignment).toBeDefined(); + await settle(assignment); await tick(restarted); + expect((await restarted.get(owner, requestId))?.state).toBe('completed'); + } +}); + +test('an offline worker cannot extend the queue deadline', async () => { + const requests = await setup(); await submit(requests); + const worker = `codeapi:bridge:v1:worker:${workerId}`; + await redis.del(worker, `${worker}:incarnation`, `${worker}:ready`); + await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + await mutateRecord(record => { record.queueDeadlineAtMs = Date.now() - 1; }); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT' } }); + expect(await redis.zcard(activeKey)).toBe(0); +}); + +test('durable reset epochs outlive their receipts and reject late quarantine after reset', async () => { + const requests = await setup(2); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await settle(assignment); await tick(requests); + const receipt = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:workspace-fence-owner`; + const fence = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`; + expect(await redis.pttl(`${fence}:epoch`)).toBeGreaterThanOrEqual(await redis.pttl(receipt) - 50); + expect(await redis.pttl(`${fence}:epoch`)).toBeGreaterThan(86_000_000); + // Simulate execution-ownership TTL expiry. The durable receipt and epoch survive. + await redis.del(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`, + `codeapi:bridge:v1:worker:${workerId}:lock`, `codeapi:bridge:v1:worker:${workerId}:lock:incarnation`); + await bridge.resetWorkspace(workerId, incarnationId, 'native-workspace:primary'); + await submit(requests, 'request-00000000002'); await tick(requests); + const next = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await expect(bridge.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, incarnationId, generation: assignment.generation, leaseToken: assignment.leaseToken, + status: 'rejected', error: 'delayed local cleanup failure', + }, undefined, undefined, true)).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + expect(await redis.get(fence)).toBe(next.assignmentId); + await settle(next); await tick(requests); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +const coordinationPolicy: WorkspaceRequestCoordinationPolicy = { + bridgeEnabled: true, backend: 'remote-bridge', executionProfile: 'default', authMode: 'paired', + configuredWorkerId: '', dynamicWorkers: true, maxWorkspaceLeaseSlots: 2, maxCommandTimeoutMs: 30_000, +}; + +test('disabled bridges do not participate and each scheduling policy has an isolated scope', () => { + expect(workspaceRequestCoordinationScope({ ...coordinationPolicy, bridgeEnabled: false })).toBeUndefined(); + const scope = workspaceRequestCoordinationScope(coordinationPolicy); + expect(workspaceRequestCoordinationScope({ ...coordinationPolicy })).toBe(scope); + const variants: Partial[] = [ + { maxWorkspaceLeaseSlots: 1 }, { backend: 'http' }, { executionProfile: 'stateful' }, + { authMode: 'static' }, { configuredWorkerId: 'another' }, { dynamicWorkers: false }, + { maxCommandTimeoutMs: 60_000 }, + ]; + for (const variant of variants) expect(workspaceRequestCoordinationScope({ ...coordinationPolicy, ...variant })).not.toBe(scope); +}); + +test('a different policy cannot claim or fail work accepted by a two-slot bridge API', async () => { + await setup(2); + const scope = workspaceRequestCoordinationScope(coordinationPolicy)!; + const requests = new RedisWorkspaceRequests(redis, bridge, undefined, scope); + await submit(requests); + const foreignScope = workspaceRequestCoordinationScope({ ...coordinationPolicy, maxWorkspaceLeaseSlots: 1 })!; + const foreign = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis), undefined, foreignScope); + await foreign.reconcile(); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await foreign.get(owner, requestId)).toBeUndefined(); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis, 600, 1000, 2), undefined, scope); + await restarted.reconcile(); + const assignment = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0); + expect(assignment).toBeDefined(); +}); + +test('a late FIFO response cannot relock a cancelled request after the coordinator timed out', async () => { + const requests = await setup(); await submit(requests); + let release!: () => void; + let entered!: () => void; + let finished!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const drained = new Promise(resolve => { finished = resolve; }); + const originalHead = BridgeAdmissionQueue.prototype.isHead; + const originalAdvance = bridge.advanceDurableWorkspaceTool.bind(bridge); + const head = spyOn(BridgeAdmissionQueue.prototype, 'isHead').mockImplementation(async (worker, id) => { + const result = await originalHead.call(new BridgeAdmissionQueue(redis), worker, id); + entered(); await gate; return result; + }); + const advance = spyOn(bridge, 'advanceDurableWorkspaceTool').mockImplementation(async (...args) => { + try { return await originalAdvance(...args); } finally { finished(); } + }); + const pending = tick(requests); + void pending.catch(() => undefined); + try { + await started; + await expect(pending).rejects.toThrow('Durable workspace transition timed out'); + head.mockRestore(); advance.mockRestore(); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const claimDeadline = Date.now() + 1000; + while (await redis.get(`${key}:claim`) != null) { + if (Date.now() >= claimDeadline) throw new Error('Timed-out coordinator did not release its claim'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + expect(await redis.zcard(activeKey)).toBe(0); + release(); await drained; + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + await submit(requests, 'request-00000000002'); await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeDefined(); + } finally { release(); head.mockRestore(); advance.mockRestore(); await drained; } +}, 15_000); + +test.each([1, 2])('Redis capacity writes delayed past cancellation cannot strand a reservation (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + let release!: () => void; + let entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const original = redis.eval.bind(redis); + let held = false; + const delay = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + const script = String(args[0]); + if (!held && script.includes('local admissionTime') && !script.includes('local keyCount')) { + held = true; entered(); await gate; + } + return original(...args); + }) as Redis['eval']); + const pending = tick(requests); + void pending.catch(() => undefined); + try { + await started; + const key = (await redis.zrange(activeKey, 0, -1))[0]; + // Expire this claim while its Redis command is still undelivered. + await redis.del(`${key}:claim`); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + release(); await pending; + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); + expect(await redis.zcard(activeKey)).toBe(0); + } finally { release(); delay.mockRestore(); await pending.catch(() => undefined); } +}); + +test.each([1, 2])('a durable enqueue refuses lost capacity ownership (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + const original = redis.eval.bind(redis); + const loss = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + if (String(args[0]).includes('local keyCount')) { + await redis.del(`codeapi:bridge:v1:worker:${workerId}:lock`, + `codeapi:bridge:v1:worker:${workerId}:workspace-slots`); + } + return original(...args); + }) as Redis['eval']); + try { await tick(requests); } finally { loss.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeDefined(); +}); + +test.each([1, 2])('capacity acquisition rejects terminal, cancelled, expired, and unclaimed requests (slots=%s)', async slots => { + for (const denial of ['terminal', 'cancelled', 'expired', 'unclaimed']) { + await redis.flushall(); + const requests = await setup(slots); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + const token = 'current-coordinator'; + await redis.set(`${key}:claim`, token, 'PX', 10000); + if (denial === 'terminal') await redis.hset(key, 'state', 'cancelled'); + if (denial === 'cancelled') await redis.hset(key, 'cancelRequested', '1'); + if (denial === 'expired') await redis.hset(key, 'queueDeadlineAtMs', Date.now() - 2000); + if (denial === 'unclaimed') await redis.del(`${key}:claim`); + await bridge.advanceDurableWorkspaceTool(record, { key, claimKey: `${key}:claim`, token }); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + } +}); + +test.each([false, true])('quarantine retains the winning durable outcome across two-minute recovery (settled=%s)', async settled => { + const requests = await setup(2); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, + queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + const firstError = 'Command stopped before local cleanup'; + const quarantineError = 'Workspace cleanup failed before settlement'; + const envelope = { protocolVersion: 1 as const, incarnationId, + generation: assignment.generation, leaseToken: assignment.leaseToken, + status: 'rejected' as const, + }; + if (settled) await bridge.settle(workerId, assignment.assignmentId, { ...envelope, error: firstError }); + await bridge.settle(workerId, assignment.assignmentId, { ...envelope, error: quarantineError }, undefined, undefined, true); + const resultKey = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:settlement`; + const receiptKey = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:workspace-fence-owner`; + const receipt = JSON.parse((await redis.hget(receiptKey, 'metadata'))!); + expect(receipt.durableRequestKey).toBe((await redis.zrange(activeKey, 0, -1))[0]); + const remaining = await redis.pttl(resultKey); + expect(remaining).toBeGreaterThan(86_000_000); + expect(await redis.get(`codeapi:bridge:v1:assignment:${assignment.assignmentId}`)).toBeNull(); + // Advance retained storage and the coordinator clock without a two-minute sleep. + await redis.pexpire(resultKey, Math.max(0, remaining - 120_000)); + const clock = spyOn(Date, 'now').mockReturnValue(Date.now() + 120_000); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis, 600, 1000, 2)); + try { + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ + state: 'failed', error: { code: 'WORKSPACE_TOOL_REJECTED', message: settled ? firstError : quarantineError }, + }); + expect(await redis.pttl(resultKey)).toBeGreaterThan(85_000_000); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(restarted, 'request-00000000002'); await tick(restarted); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0)).toBeUndefined(); + expect(await restarted.get(owner, 'request-00000000002')).toMatchObject({ + state: 'failed', error: { code: 'WORKSPACE_QUARANTINED' }, + }); + } finally { clock.mockRestore(); } +}); + +test('queued durable requests retain FIFO workspace metadata across same-incarnation slot changes', async () => { + const requests = await setup(); + await submit(requests); + await submit(requests, 'request-00000000002'); + await submit(requests, 'request-00000000003', 'independent'); + await submit(requests); + await setup(2); + await tick(requests); await tick(requests); + const first = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + const independent = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1))!; + expect(first).toBeDefined(); expect(independent).toBeDefined(); + expect(first.request).toMatchObject({ workspaceId: 'primary' }); + expect(independent.request).toMatchObject({ workspaceId: 'independent' }); + expect(await requests.get(owner, 'request-00000000002')).toMatchObject({ state: 'queued', queuePosition: 2 }); + await settle(first); await settle(independent); await tick(requests); + await bridge.confirmWorkspaceCleanup(workerId, first.assignmentId, { + protocolVersion: 1, incarnationId, generation: first.generation, leaseToken: first.leaseToken, + status: 'rejected', error: 'local cleanup confirmed', + }); + await tick(requests); + const next = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + expect(next).toBeDefined(); + await settle(next); await tick(requests); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +const readOnlyRequests: WorkspaceToolRequest[] = [ + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, + { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }, + { protocolVersion: 1, operation: 'search_text', workspaceId: 'primary', query: 'needle' }, + { protocolVersion: 1, operation: 'preview_edit', workspaceId: 'primary', path: 'README.md', oldText: 'old', newText: 'new' }, +]; +const readCancellationCases = readOnlyRequests.flatMap(request => [1, 2].flatMap(slots => + ['unleased', 'claimed', 'acknowledged'].map(phase => ({ request, slots, phase })), +)); + +test.each(readCancellationCases)('durable read cancellation notifies the worker before closing (%j)', async ({ request, slots, phase }) => { + const requests = await setup(slots); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request, queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); + await tick(requests); + const slot = slots === 1 ? undefined : 0; + let assignment = phase === 'unleased' ? undefined : await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slot); + if (phase === 'acknowledged') await bridge.acknowledgeLease(workerId, incarnationId, assignment!.assignmentId, + assignment!.generation, assignment!.leaseToken); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + await requests.cancel(owner, requestId); + const cancelling = tick(new RedisWorkspaceRequests(redis, bridge)); + const marker = `codeapi:bridge:v1:assignment:${stored.assignmentId}:cancelled`; + const limit = Date.now() + 1000; + try { + while (await redis.get(marker) !== '1') { + if (Date.now() >= limit) throw new Error('Read-only worker was never notified'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect((await requests.get(owner, requestId))?.state).toBe('admitted'); + assignment ??= await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slot); + expect(assignment).toBeDefined(); + expect(await bridge.cancelled(workerId, incarnationId, assignment!.assignmentId)).toBe(true); + await settle(assignment!, false); + await cancelling; + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'cancelled', cancelRequested: true }); + expect(await redis.zcard(activeKey)).toBe(0); + } finally { await cancelling; } +}); + +test.each([1, 2])('a durable read cancellation remains cancelled without worker settlement (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); await tick(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await redis.get(`codeapi:bridge:v1:assignment:${stored.assignmentId}:cancelled`)).toBe('1'); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'cancelled', cancelRequested: true }); + expect((await requests.get(owner, requestId))?.error).toBeUndefined(); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(requests, 'request-00000000002'); await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeDefined(); +}, 10_000); + +test('fulfillment racing durable read cancellation keeps the winning result', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await requests.cancel(owner, requestId); + const cancelling = tick(requests); + const marker = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:cancelled`; + const limit = Date.now() + 1000; + try { + while (await redis.get(marker) !== '1') { + if (Date.now() >= limit) throw new Error('Read-only worker was never notified'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await settle(assignment); + await cancelling; + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'completed', result: { content: 'hello' } }); + } finally { await cancelling; } +}); + +test('queued workspace metadata also permits a same-incarnation return to serial admission', async () => { + const requests = await setup(2); await submit(requests); await setup(); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + expect(assignment).toBeDefined(); + expect(assignment.workspaceLeaseSlot).toBeUndefined(); + await settle(assignment); await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); +}); + +test('unconfirmed durable mutation cancellation still reports an unknown outcome and retains its fence', async () => { + const requests = await setup(); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'sleep 30' }, + queueWaitMs: 300_000, executionTimeoutMs: 35_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + await requests.cancel(owner, requestId); await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ + state: 'failed', cancelRequested: true, error: { code: 'ASSIGNMENT_EXPIRED' }, + }); + const fence = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`; + expect(await redis.get(fence)).toBe(assignment.assignmentId); +}, 10_000); diff --git a/service/src/workspace-tools/requests.ts b/service/src/workspace-tools/requests.ts new file mode 100644 index 00000000..2988c7cc --- /dev/null +++ b/service/src/workspace-tools/requests.ts @@ -0,0 +1,251 @@ +import { createHash, randomBytes } from 'crypto'; +import type Redis from 'ioredis'; +import type { WorkspaceToolRequest, WorkspaceToolResult } from '../../../packages/code/src/protocol'; +import type { RedisBridgeStore, RegisteredBridgeWorker, DurableWorkspaceAssignment, CodeBridgeWorkspaceSettlement } from '../bridge/store'; +import { BridgeAdmissionQueue } from '../bridge/admission'; +import { BridgeStoreError, workspaceAdmissionId } from '../bridge/store'; + +const PREFIX = 'codeapi:workspace-requests:v1'; +const ACTIVE = `${PREFIX}:active`; +const RETENTION_MS = 24 * 60 * 60_000; +const CLAIM_MS = 10_000; +export const WORKSPACE_REQUEST_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; + +export class WorkspaceRequestConflict extends Error {} + +export interface WorkspaceRequestOwner { + tenantId: string; + userId: string; +} + +export interface StoredWorkspaceRequest { + id: string; + key: string; + fingerprint: string; + workerId: string; + tenantId: string; + requireTenantBinding: boolean; + request: WorkspaceToolRequest; + registration: RegisteredBridgeWorker; + createdAtMs: number; + queueDeadlineAtMs: number; + executionTimeoutMs: number; + assignment?: DurableWorkspaceAssignment; + settlement?: CodeBridgeWorkspaceSettlement; + admittedAtMs?: number; + finishedAtMs?: number; + state: 'queued' | 'admitted' | 'completed' | 'failed' | 'cancelled'; + cancelRequested?: boolean; + result?: WorkspaceToolResult; + error?: { code: string; message: string }; +} + +export interface WorkspaceRequestStatus { + requestId: string; + state: StoredWorkspaceRequest['state']; + workerId: string; + workspaceId: string; + workspaceInstanceId?: string; + worktree?: string; + queuePosition?: number; + queueWaitMs: number; + executionDeadlineAt?: string; + cancelRequested: boolean; + result?: WorkspaceToolResult; + error?: StoredWorkspaceRequest['error']; +} + +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value !== null && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).filter(key => record[key] !== undefined).sort() + .map(key => `${JSON.stringify(key)}:${canonical(record[key])}`).join(',')}}`; + } + return JSON.stringify(value); +} + +async function deadline(operation: Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([operation, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('Durable workspace transition timed out')), 8_000); + timer.unref(); + })]); + } finally { clearTimeout(timer); } +} + +function requestKey(owner: WorkspaceRequestOwner, requestId: string, scope: string): string { + return `${PREFIX}:${createHash('sha256').update(canonical([scope, owner.tenantId, owner.userId, requestId])).digest('hex')}`; +} + +/** Redis owns requests; API replicas only advance bounded, fenced transitions. */ +export class RedisWorkspaceRequests { + private readonly activeKey: string; + constructor( + private readonly redis: Redis, + private readonly bridge: RedisBridgeStore, + private readonly transition?: (record: StoredWorkspaceRequest) => void, + private readonly scope = '', + ) { + this.activeKey = scope.length === 0 ? ACTIVE : `${PREFIX}:${scope}:active`; + } + + async submit(args: { + owner: WorkspaceRequestOwner; + requestId: string; + workerId: string; + requireTenantBinding: boolean; + request: WorkspaceToolRequest; + queueWaitMs: number; + executionTimeoutMs: number; + }): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(args.requestId)) { + throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid durable workspace request ID'); + } + const key = requestKey(args.owner, args.requestId, this.scope); + const fingerprint = createHash('sha256').update(canonical({ + workerId: args.workerId, request: args.request, + queueWaitMs: args.queueWaitMs, executionTimeoutMs: args.executionTimeoutMs, + requireTenantBinding: args.requireTenantBinding, + })).digest('hex'); + const existing = await this.read(key); + if (existing != null) { + if (existing.fingerprint !== fingerprint) throw new WorkspaceRequestConflict('Request ID was already used for different work'); + return this.status(existing, args.requestId); + } + const registration = await this.bridge.prepareDurableWorkspaceTool({ + ...args, tenantId: args.owner.tenantId, + }); + const now = Date.now(); + const record: StoredWorkspaceRequest = { + id: `durable-${randomBytes(18).toString('base64url')}`, key, fingerprint, + workerId: args.workerId, tenantId: args.owner.tenantId, + requireTenantBinding: args.requireTenantBinding, + request: args.request, registration, createdAtMs: now, + queueDeadlineAtMs: now + args.queueWaitMs, + executionTimeoutMs: args.executionTimeoutMs, state: 'queued', + }; + const queue = new BridgeAdmissionQueue(this.redis); + const accepted = await queue.submit({ + workerId: record.workerId, id: record.id, deadlineAtMs: record.queueDeadlineAtMs, + workspaceId: workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree), + key, activeKey: this.activeKey, fingerprint, record: JSON.stringify(record), retentionMs: RETENTION_MS, + }); + if (accepted === 'conflict') throw new WorkspaceRequestConflict('Request ID was already used for different work'); + if (accepted === 'full') throw new BridgeStoreError('WORKER_QUEUE_FULL', 'Bridge worker pending request limit reached'); + const submitted = (await this.read(key))!; + if (accepted === 'accepted') this.transition?.(submitted); + return this.status(submitted, args.requestId); + } + + async get(owner: WorkspaceRequestOwner, requestId: string): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(requestId)) return undefined; + const record = await this.read(requestKey(owner, requestId, this.scope)); + return record == null ? undefined : this.status(record, requestId); + } + + async cancel(owner: WorkspaceRequestOwner, requestId: string): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(requestId)) return undefined; + const key = requestKey(owner, requestId, this.scope); + await this.redis.eval([ + 'local state = redis.call(\'HGET\', KEYS[1], \'state\')', + 'if state ~= \'queued\' and state ~= \'admitted\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'cancelRequested\', \'1\')', + 'redis.call(\'ZADD\', KEYS[2], ARGV[1], KEYS[1])', + 'return 1', + ].join('\n'), 2, key, this.activeKey, Date.now()); + return this.get(owner, requestId); + } + + private async read(key: string): Promise { + const fields: Partial> = await this.redis.hgetall(key); + if (fields.record == null) return undefined; + return { + ...JSON.parse(fields.record) as StoredWorkspaceRequest, + ...(fields.outcome == null ? {} : JSON.parse(fields.outcome)), + state: fields.state as StoredWorkspaceRequest['state'], + assignment: fields.assignment == null ? undefined : JSON.parse(fields.assignment), + admittedAtMs: fields.admittedAtMs == null ? undefined : Number(fields.admittedAtMs), + cancelRequested: fields.cancelRequested === '1', + }; + } + + private async status(record: StoredWorkspaceRequest, requestId: string): Promise { + const rank = record.state === 'queued' + ? await new BridgeAdmissionQueue(this.redis).position(record.workerId, record.id) + : undefined; + return { + requestId, state: record.state, workerId: record.workerId, + workspaceId: record.request.workspaceId, + workspaceInstanceId: record.request.workspaceInstanceId, worktree: record.request.worktree, + queuePosition: rank, queueWaitMs: Math.max(0, (record.admittedAtMs ?? Math.min(record.finishedAtMs ?? Date.now(), record.queueDeadlineAtMs)) - record.createdAtMs), + executionDeadlineAt: record.assignment?.expiresAt, + cancelRequested: record.cancelRequested === true, + result: record.result, error: record.error, + }; + } + + async reconcile(): Promise { + const keys = await deadline(this.redis.zrangebyscore(this.activeKey, '-inf', Date.now(), 'LIMIT', 0, 32)); + const results = await Promise.allSettled(keys.map(key => deadline(this.advance(key)))); + const failure = results.find(result => result.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; + } + + private async advance(key: string): Promise { + const claimKey = `${key}:claim`; + const token = randomBytes(18).toString('hex'); + const claimed = await this.redis.eval([ + 'if not redis.call(\'SET\', KEYS[1], ARGV[1], \'PX\', ARGV[2], \'NX\') then return 0 end', + 'redis.call(\'ZADD\', KEYS[2], ARGV[3], KEYS[3])', + 'return 1', + ].join('\n'), 3, claimKey, this.activeKey, key, token, CLAIM_MS, Date.now() + CLAIM_MS); + if (Number(claimed) !== 1) return; + try { + let record = await this.read(key); + if (record == null) { await this.redis.zrem(this.activeKey, key); return; } + if (record.state === 'queued' || record.state === 'admitted') { + const previousState = record.state; + try { + const outcome = await deadline(this.bridge.advanceDurableWorkspaceTool(record, { key, claimKey, token })); + if (outcome != null) { + await this.redis.eval([ + 'if redis.call(\'GET\', KEYS[2]) ~= ARGV[1] then return 0 end', + 'local state = redis.call(\'HGET\', KEYS[1], \'state\')', + 'if state ~= \'queued\' and state ~= \'admitted\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'state\', ARGV[3], \'outcome\', ARGV[2])', + 'return 1', + ].join('\n'), 2, key, claimKey, token, JSON.stringify({ ...outcome, finishedAtMs: Date.now() }), outcome.state); + } + } catch (error) { + // Infrastructure uncertainty is reconciled, never retried as a new command. + if (!(error instanceof BridgeStoreError)) throw error; + if (error.code === 'WORKER_OFFLINE') { + // The original worker may reconnect within the remaining admission budget. + await this.redis.zadd(this.activeKey, Date.now() + 250, key); + return; + } + const current = await this.read(key); + if (current?.state === 'admitted') throw error; + await this.redis.eval([ + 'if redis.call(\'GET\', KEYS[2]) ~= ARGV[1] then return 0 end', + 'if redis.call(\'HGET\', KEYS[1], \'state\') ~= \'queued\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'state\', \'failed\', \'outcome\', ARGV[2])', + 'return 1', + ].join('\n'), 2, key, claimKey, token, JSON.stringify({ state: 'failed', finishedAtMs: Date.now(), error: { code: error.code, message: error.message } })); + } + record = (await this.read(key))!; + if (record.state !== previousState) this.transition?.(record); + } + if (record.state !== 'queued' && record.state !== 'admitted') { + await deadline(this.bridge.finishDurableWorkspaceTool(record)); + await this.redis.zrem(this.activeKey, key); + } else { + await this.redis.zadd(this.activeKey, Date.now() + 250, key); + } + } finally { + await this.redis.eval('if redis.call(\'GET\', KEYS[1]) == ARGV[1] then return redis.call(\'DEL\', KEYS[1]) end return 0', 1, claimKey, token); + } + } +} diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 78dbd1e7..d4d65d14 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -19,6 +19,8 @@ import { resolveBridgeWorkerSelection, } from '../bridge/selection'; import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; +import { WorkspaceRequestConflict } from './requests'; +import type { RedisWorkspaceRequests } from './requests'; const MAX_WORKSPACE_QUEUE_WAIT_MS = 5 * 60_000; const DEFAULT_WORKSPACE_QUEUE_WAIT_MS = 30_000; @@ -26,6 +28,7 @@ const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms'; interface WorkspaceToolsRouterOptions { store: Pick; + requests?: RedisWorkspaceRequests; backend: 'http' | 'lambda-microvm' | 'remote-bridge'; configuredWorkerId: string; dynamicWorkers: boolean; @@ -67,161 +70,200 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) } const router = Router(); - router.post( - '/workspace-tools/execute', - asyncRoute(async (req, res) => { - const outcome = getWorkspaceToolOutcome(res); - const principal = getPrincipalOrReject(req, res); - if (!principal) { - outcome.errorCode = 'UNAUTHENTICATED'; - return; - } - if ((options.isShuttingDown ?? checkServiceShutDown)()) { - outcome.errorCode = 'SERVICE_SHUTTING_DOWN'; - res.status(503).json({ error: 'Service is shutting down' }); - return; - } - if (!isWorkspaceToolRequest(req.body)) { - outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; - res.status(400).json({ - error: 'Invalid workspace tool request', - }); - return; - } - const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); - if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { - outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; - res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + const execute = (durable: boolean): RequestHandler => asyncRoute(async (req, res) => { + const outcome = getWorkspaceToolOutcome(res, req.path); + const principal = getPrincipalOrReject(req, res); + if (!principal) { + outcome.errorCode = 'UNAUTHENTICATED'; + return; + } + if ((options.isShuttingDown ?? checkServiceShutDown)()) { + outcome.errorCode = 'SERVICE_SHUTTING_DOWN'; + res.status(503).json({ error: 'Service is shutting down' }); + return; + } + if (!isWorkspaceToolRequest(req.body)) { + outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; + res.status(400).json({ + error: 'Invalid workspace tool request', + }); + return; + } + const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); + if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const requestedQueueWaitMs = advertisedQueueWait === undefined + ? DEFAULT_WORKSPACE_QUEUE_WAIT_MS + : Number(advertisedQueueWait); + if (!Number.isSafeInteger(requestedQueueWaitMs) || requestedQueueWaitMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const queueBudgetMs = Math.min(requestedQueueWaitMs, queueCeilingMs); + outcome.operation = req.body.operation; + const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null + ? req.body + : { + ...req.body, + workspaceInstanceId: principalWorkspaceInstanceId({ + instanceId: req.body.workspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }), + }; + const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' + ? { ...principalRequest, timeoutMs: Math.min( + principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + options.timeoutMs ?? Number.MAX_SAFE_INTEGER, + ) } + : principalRequest; + const executionBudgetMs = request.operation === 'execute_command' + ? request.timeoutMs! + 5_000 + : Math.min(options.timeoutMs ?? 30_000, 30_000); + outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; + + let selection: { workerId: string; explicit: boolean } | undefined; + try { + selection = resolveBridgeWorkerSelection({ + backend: options.backend, + configuredWorkerId: options.configuredWorkerId, + dynamicWorkers: options.dynamicWorkers, + requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), + trustedWorkerId: principal.codeWorkerId, + }); + } catch (error) { + if (error instanceof BridgeWorkerSelectionError) { + outcome.errorCode = 'WORKER_SELECTION_REJECTED'; + res.status(error.status).json({ error: error.message }); return; } - const requestedQueueWaitMs = advertisedQueueWait === undefined - ? DEFAULT_WORKSPACE_QUEUE_WAIT_MS - : Number(advertisedQueueWait); - if (!Number.isSafeInteger(requestedQueueWaitMs) || requestedQueueWaitMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { - outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; - res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + throw error; + } + if (selection == null) { + outcome.errorCode = 'WORKSPACE_BACKEND_UNAVAILABLE'; + res.status(503).json({ + error: 'Workspace tools require the remote-bridge backend', + }); + return; + } + outcome.workerId = selection.workerId; + + if (durable) { + if (options.requests == null) { + res.status(404).json({ error: 'Durable workspace requests are unavailable' }); return; } - const queueBudgetMs = Math.min(requestedQueueWaitMs, queueCeilingMs); - outcome.operation = req.body.operation; - const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null - ? req.body - : { - ...req.body, - workspaceInstanceId: principalWorkspaceInstanceId({ - instanceId: req.body.workspaceInstanceId, - tenantId: principal.tenantId, - principalId: principal.userId, - }), - }; - const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' - ? { ...principalRequest, timeoutMs: Math.min( - principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, - options.timeoutMs ?? Number.MAX_SAFE_INTEGER, - ) } - : principalRequest; - const executionBudgetMs = request.operation === 'execute_command' - ? request.timeoutMs! + 5_000 - : Math.min(options.timeoutMs ?? 30_000, 30_000); - outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; - - let selection: { workerId: string; explicit: boolean } | undefined; try { - selection = resolveBridgeWorkerSelection({ - backend: options.backend, - configuredWorkerId: options.configuredWorkerId, - dynamicWorkers: options.dynamicWorkers, - requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), - trustedWorkerId: principal.codeWorkerId, + const status = await options.requests.submit({ + owner: principal, requestId: req.header('X-LibreChat-Workspace-Request-Id') ?? '', + workerId: selection.workerId, + requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), + request, queueWaitMs: queueBudgetMs, executionTimeoutMs: executionBudgetMs, }); + res.status(202).json(status); } catch (error) { - if (error instanceof BridgeWorkerSelectionError) { - outcome.errorCode = 'WORKER_SELECTION_REJECTED'; - res.status(error.status).json({ error: error.message }); - return; - } - throw error; - } - if (selection == null) { - outcome.errorCode = 'WORKSPACE_BACKEND_UNAVAILABLE'; - res.status(503).json({ - error: 'Workspace tools require the remote-bridge backend', - }); - return; + if (error instanceof WorkspaceRequestConflict) { + res.status(409).json({ error: error.message, code: 'REQUEST_CONFLICT' }); + } else if (error instanceof BridgeStoreError) { + res.status(bridgeStoreStatus(error)).json({ error: error.message, code: error.code }); + } else throw error; } - outcome.workerId = selection.workerId; + return; + } - const controller = new AbortController(); - const abort = (): void => controller.abort(); - req.once('aborted', abort); - const abortClosedResponse = (): void => { - if (!res.writableEnded) abort(); - }; - res.once('close', abortClosedResponse); - try { - outcome.dispatchPending = true; - const dispatchStartedAt = performance.now(); - const settlement = await options.store.dispatchWorkspaceTool({ - workerId: selection.workerId, - tenantId: principal.tenantId, - requireTenantBinding: + const controller = new AbortController(); + const abort = (): void => controller.abort(); + req.once('aborted', abort); + const abortClosedResponse = (): void => { + if (!res.writableEnded) abort(); + }; + res.once('close', abortClosedResponse); + try { + outcome.dispatchPending = true; + const dispatchStartedAt = performance.now(); + const settlement = await options.store.dispatchWorkspaceTool({ + workerId: selection.workerId, + tenantId: principal.tenantId, + requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), - request, - deadlineAtMs: Date.now() + queueBudgetMs, - executionTimeoutMs: executionBudgetMs, - signal: controller.signal, - }).finally(() => { - outcome.dispatchDurationMs = Math.round(performance.now() - dispatchStartedAt); - }); - if (settlement.status === 'rejected') { - outcome.errorCode = settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED'; - let status = 422; - if ( - settlement.errorCode === 'SEARCH_TIMEOUT' || + request, + deadlineAtMs: Date.now() + queueBudgetMs, + executionTimeoutMs: executionBudgetMs, + signal: controller.signal, + }).finally(() => { + outcome.dispatchDurationMs = Math.round(performance.now() - dispatchStartedAt); + }); + if (settlement.status === 'rejected') { + outcome.errorCode = settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED'; + let status = 422; + if ( + settlement.errorCode === 'SEARCH_TIMEOUT' || settlement.errorCode === 'LIST_TIMEOUT' || settlement.errorCode === 'COMMAND_TIMEOUT' - ) { - status = 504; - } - if ( - settlement.errorCode === 'SEARCH_UNAVAILABLE' || + ) { + status = 504; + } + if ( + settlement.errorCode === 'SEARCH_UNAVAILABLE' || settlement.errorCode === 'LIST_UNAVAILABLE' || settlement.errorCode === 'COMMAND_UNAVAILABLE' - ) { - status = 503; - } - if (settlement.errorCode === 'WRITE_DISABLED') status = 403; - if (settlement.errorCode === 'COMMAND_DISABLED') status = 403; - if (settlement.errorCode === 'WRITE_LIMIT_EXCEEDED') status = 413; - if (settlement.errorCode === 'WRITE_UNAVAILABLE') status = 503; - if (settlement.errorCode === 'EDIT_CONFLICT') status = 409; - res.status(status).json({ - error: settlement.error, - code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', - }); - return; - } - res.status(200).json(settlement.result); - } catch (error) { - if (error instanceof BridgeStoreError) { - outcome.errorCode = error.code; - if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); - res.status(bridgeStoreStatus(error)).json({ - error: error.message, - code: error.code, - }); - return; + ) { + status = 503; } - outcome.errorCode = 'INTERNAL_ERROR'; - throw error; - } finally { - outcome.dispatchPending = false; - outcome.flush(); - req.removeListener('aborted', abort); - res.removeListener('close', abortClosedResponse); + if (settlement.errorCode === 'WRITE_DISABLED') status = 403; + if (settlement.errorCode === 'COMMAND_DISABLED') status = 403; + if (settlement.errorCode === 'WRITE_LIMIT_EXCEEDED') status = 413; + if (settlement.errorCode === 'WRITE_UNAVAILABLE') status = 503; + if (settlement.errorCode === 'EDIT_CONFLICT') status = 409; + res.status(status).json({ + error: settlement.error, + code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', + }); + return; + } + res.status(200).json(settlement.result); + } catch (error) { + if (error instanceof BridgeStoreError) { + outcome.errorCode = error.code; + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); + res.status(bridgeStoreStatus(error)).json({ + error: error.message, + code: error.code, + }); + return; } - }), - ); + outcome.errorCode = 'INTERNAL_ERROR'; + throw error; + } finally { + outcome.dispatchPending = false; + outcome.flush(); + req.removeListener('aborted', abort); + res.removeListener('close', abortClosedResponse); + } + }); + router.post('/workspace-tools/execute', execute(false)); + router.post('/workspace-tools/requests', execute(true)); + + router.get('/workspace-tools/capabilities', asyncRoute(async (req, res) => { + if (!getPrincipalOrReject(req, res)) return; + res.json({ durableWorkspaceRequests: options.requests != null ? 1 : 0 }); + })); + + const requestStatus = (cancel: boolean): RequestHandler => asyncRoute(async (req, res) => { + const principal = getPrincipalOrReject(req, res); + if (!principal) return; + const status = cancel + ? await options.requests?.cancel(principal, req.params.requestId) + : await options.requests?.get(principal, req.params.requestId); + if (status == null) { res.status(404).json({ error: 'Workspace request not found' }); return; } + res.json(status); + }); + router.get('/workspace-tools/requests/:requestId', requestStatus(false)); + router.delete('/workspace-tools/requests/:requestId', requestStatus(true)); return router; } From 192f61cd36785a885980274d1792dd30efc296bb Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:33:52 -0400 Subject: [PATCH 65/75] =?UTF-8?q?=F0=9F=93=96=20feat:=20Stream=20Bounded?= =?UTF-8?q?=20Workspace=20File=20Reads=20(#303)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 feat: Stream Bounded Workspace File Reads * 🧹 fix: Settle Interrupted Reads Before Pending I/O Cleanup * ⏱️ fix: Track Workspace Read Interruption Through Cleanup * 🧪 test: Queue Descriptor I/O Consistently on Node 20 --------- Co-authored-by: Lia --- packages/code/README.md | 7 +- packages/code/WORKSPACE-READS.md | 32 ++ packages/code/src/root-access.ts | 10 +- packages/code/src/workspace-read-io.test.ts | 147 ++++++ packages/code/src/workspace-read.test.ts | 488 ++++++++++++++++++++ packages/code/src/workspace.ts | 295 ++++++++++-- 6 files changed, 928 insertions(+), 51 deletions(-) create mode 100644 packages/code/WORKSPACE-READS.md create mode 100644 packages/code/src/workspace-read-io.test.ts create mode 100644 packages/code/src/workspace-read.test.ts diff --git a/packages/code/README.md b/packages/code/README.md index 636cd1a8..d348fe88 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -729,9 +729,10 @@ native SRT command backend. This operator switch controls availability; LibreChat tool approval hooks remain the user-facing allow/deny boundary for each invocation. -Reads reject absolute paths, traversal, escaping symlinks, non-regular files, -and files larger than 1 MiB. The opened file is checked against its canonical -in-workspace inode before it is read. Text search uses `rg` only to enumerate a +Reads reject absolute paths, traversal, escaping symlinks, and non-regular files. +The opened file is checked against its canonical in-workspace inode before it is +read. Ordinary reads stream bounded line windows even from files larger than 1 MiB; +see the [read contract](WORKSPACE-READS.md). Text search uses `rg` only to enumerate a bounded set of ignored-aware candidates with configuration and symlink following disabled. It then opens and verifies each candidate through the same confined 1 MiB read boundary before matching locally. File listing invokes `rg` without diff --git a/packages/code/WORKSPACE-READS.md b/packages/code/WORKSPACE-READS.md new file mode 100644 index 00000000..f3edbe58 --- /dev/null +++ b/packages/code/WORKSPACE-READS.md @@ -0,0 +1,32 @@ +# Workspace file read contract + +Ordinary `read_file` streams complete LF-delimited line windows through the verified +file descriptor. It defaults to 200 lines, accepts at most 500, and returns at most +1 MiB of UTF-8 content, including inter-line separators, regardless of file size. +CR in CRLF is preserved; the final LF does not add a phantom line. An empty file +has one empty line. A start beyond EOF returns an empty, non-truncated window. + +Byte-limited windows stop before the first line that cannot fit and return +`nextStartLine = endLine + 1`. If the first requested line exceeds 1 MiB, the read +fails with `READ_LIMIT_EXCEEDED`, without a repeating continuation. Scanning and +collection check cancellation and share a 10-second deadline. Far-away starts +can fail with `READ_LIMIT_EXCEEDED`; use an earlier start or `search_text`. +Cancellation and timeout settle the request without waiting for queued I/O. +Node retains an interrupted read's descriptor and chunk until that I/O drains; +file and held-root cleanup are initiated immediately and finish asynchronously. +This releases the request lane, but does not cancel kernel I/O. + +Memory is bounded by 64 KiB chunks and the returned window. Reads stop at the +opened file's initial size, so concurrent growth cannot extend the scan. Truncation +ends the scan at observed EOF; pathname replacement never switches the open +handle. In-place writes can change observed content. Pagination is not a snapshot +across requests and assumes the file stays unchanged. + +UTF-8 and BOM-marked UTF-16LE/BE are decoded incrementally. One leading BOM is +removed; malformed sequences use replacement characters, as before. Binary bytes +are decoded as text, including NUL, not classified or rejected. Neither a successful +window nor continuation validates unread content. + +Request/result shapes, protocol version, and capabilities are unchanged. Search, +preview, edit, and write limits remain unchanged. Digest-checked repository +instruction snapshots retain their separate byte-bounded, newline-preserving path. diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts index 5bdddca9..2f049bcb 100644 --- a/packages/code/src/root-access.ts +++ b/packages/code/src/root-access.ts @@ -378,6 +378,12 @@ export class WorkspaceRootAccess { } const context = new AsyncLocalStorage(); +const deferredCleanup = new WeakSet(); +/** Interrupted I/O must not hold a request open while Node drains its handles. */ +export function deferWorkspaceRootCleanup(): void { + const access = context.getStore(); + if (access) deferredCleanup.add(access); +} /** Whether filesystem adapters in this call are anchored to a held root descriptor. */ export const holdsWorkspaceRoot = (): boolean => context.getStore() != null; export async function withWorkspaceRoot( @@ -390,7 +396,9 @@ export async function withWorkspaceRoot( try { return await context.run(access, action); } finally { - await access.close(); + const closing = access.close(); + if (deferredCleanup.delete(access)) void closing.catch(() => {}); + else await closing; } } diff --git a/packages/code/src/workspace-read-io.test.ts b/packages/code/src/workspace-read-io.test.ts new file mode 100644 index 00000000..2fdd2b90 --- /dev/null +++ b/packages/code/src/workspace-read-io.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +// Isolate libuv saturation from the test runner and sibling tests. +const queuedRead = ` +import assert from 'node:assert/strict'; +import * as fs from 'node:fs'; +import * as fsp from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { getEventListeners } from 'node:events'; +import { join } from 'node:path'; +import { mock } from 'node:test'; +import { LocalWorkspaceTools } from ${JSON.stringify( + new URL('./workspace.js', import.meta.url).href +)}; +import { captureWorkspaceRootIdentity } from ${JSON.stringify( + new URL('./root-identity.js', import.meta.url).href +)}; +import { WorkspaceRootAccess } from ${JSON.stringify( + new URL('./root-access.js', import.meta.url).href +)}; +const [root, cause, held] = process.argv.slice(1); +const tools = await LocalWorkspaceTools.create({ workspaces: [{ + id: 'primary', root, + ...(held === 'true' ? { identity: await captureWorkspaceRootIdentity(root) } : {}), +}] }); +const fifo = join(root, 'pool'); +execFileSync('mkfifo', [fifo]); +const pipe = fs.openSync(fifo, fs.constants.O_RDWR); +const probe = await fsp.open(join(root, 'file'), 'r'); +const prototype = Object.getPrototypeOf(probe); +await probe.close(); +const originalRead = prototype.read; +const controller = new AbortController(); +let entered, blocker, pendingIo, physicalFd, readSettled = false; +const started = new Promise(resolve => { entered = resolve; }); +const closing = []; +const descriptors = []; +let elapsed = 0; +const now = performance.now(); +mock.method(performance, 'now', () => now + elapsed); +function trackClose(handle) { + const originalClose = handle.close; + mock.method(handle, 'close', function (...args) { + descriptors.push(this.fd); + const promise = originalClose.apply(this, args); + closing.push(promise); + return promise; + }); +} +const originalOpen = WorkspaceRootAccess.open; +mock.method(WorkspaceRootAccess, 'open', async (...args) => { + const access = await originalOpen(...args); + trackClose(access.handle); + return access; +}); +mock.method(prototype, 'read', function (...args) { + physicalFd = this.fd; + trackClose(this); + // This actual pipe read occupies the sole libuv worker until explicitly drained. + blocker = new Promise((resolve, reject) => fs.read(pipe, Buffer.alloc(1), 0, 1, null, + error => error ? reject(error) : resolve())); + // Invoke Node's original FileHandle.read, including its active-I/O references. + pendingIo = originalRead.apply(this, args).finally(() => { readSettled = true; }); + if (cause === 'deadline') elapsed = 10_000; + if (cause === 'early-deadline') elapsed = 9_999.75; + entered(); + return pendingIo; +}); +let released = false; +function drain() { + if (!released) { released = true; fs.writeSync(pipe, Buffer.from('x')); } +} +const wallStart = Date.now(); +const pending = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'file' }, controller.signal); +const rejected = assert.rejects(pending, error => error.code === (cause === 'abort' ? 'EXECUTION_ABORTED' : 'READ_LIMIT_EXCEEDED')); +await started; +if (cause === 'abort') controller.abort(); +// Always drain, even if a regression prevents bounded settlement. +const fallback = setTimeout(drain, 2_000); +try { + await rejected; + assert.equal(released, false, 'request settled before pool was drained'); + if (cause === 'early-deadline') assert.ok(performance.now() < now + 10_000, 'timer settled before the monotonic deadline'); + assert.equal(readSettled, false, 'real descriptor read remained pending'); + assert.ok(fs.fstatSync(physicalFd).isFile(), 'Node still owns the physical descriptor'); + assert.equal(closing.length, held === 'true' ? 2 : 1, 'file and held-root closes initiated'); + assert.deepEqual(getEventListeners(controller.signal, 'abort'), []); + console.log(JSON.stringify({ cause, held: held === 'true', settledBeforeDrain: true, readSettled, elapsedMs: Date.now() - wallStart })); +} finally { + clearTimeout(fallback); + drain(); + await blocker; + await pendingIo; + await Promise.all(closing); + for (const fd of descriptors) assert.throws(() => fs.fstatSync(fd), { code: 'EBADF' }); + mock.restoreAll(); + fs.closeSync(pipe); +} +`; + +for (const cause of ['abort', 'deadline', 'early-deadline'] as const) { + for (const held of [false, true]) { + test(`${cause} settles before real queued I/O drains (${ + held ? 'held' : 'legacy' + } root)`, async t => { + if (!['linux', 'darwin'].includes(process.platform)) + return t.skip('requires POSIX FIFO and descriptor access'); + const root = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'workspace-read-io-')) + ); + t.after(() => fs.rm(root, { recursive: true, force: true })); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const { stdout } = await execFileAsync( + process.execPath, + [ + '--input-type=module', + '--eval', + queuedRead, + root, + cause, + String(held), + ], + { + // Node 20 can bypass the pool for regular files via io_uring. + env: { + ...process.env, + UV_THREADPOOL_SIZE: '1', + UV_USE_IO_URING: '0', + }, + timeout: 10_000, + } + ); + const observed = JSON.parse(stdout); + assert.equal(observed.settledBeforeDrain, true); + assert.equal(observed.readSettled, false); + t.diagnostic(stdout.trim()); + }); + } +} diff --git a/packages/code/src/workspace-read.test.ts b/packages/code/src/workspace-read.test.ts new file mode 100644 index 00000000..1c6a7f96 --- /dev/null +++ b/packages/code/src/workspace-read.test.ts @@ -0,0 +1,488 @@ +import assert from 'node:assert/strict'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; +import type { FileHandle } from 'node:fs/promises'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; +import { WorkspaceRootAccess } from './root-access.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { readRepositoryInstructions } from './instructions.js'; +import { + BRIDGE_WORKSPACE_READ_MAX_BYTES as MAX_BYTES, + isWorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceReadFileRequest } from './protocol.js'; + +async function fixture(t: TestContext, held = false) { + const parent = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'workspace-read-')) + ); + t.after(() => fs.rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + await fs.mkdir(root); + const tools = await LocalWorkspaceTools.create({ + workspaces: [ + { + id: 'primary', + root, + writable: true, + ...(held + ? { identity: await captureWorkspaceRootIdentity(root) } + : {}), + }, + ], + repositoryInstructions: true, + }); + const read = async ( + startLine?: number, + maxLines?: number, + signal?: AbortSignal, + path = 'file' + ) => { + const request: WorkspaceReadFileRequest = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path, + ...(startLine !== undefined ? { startLine } : {}), + ...(maxLines !== undefined ? { maxLines } : {}), + }; + const result = await tools.execute(request, signal); + assert.ok( + isWorkspaceToolResult(request, result), + 'current protocol accepts the window' + ); + assert.equal(result.operation, 'read_file'); + return result; + }; + const probe = await fs.open(join(root, 'file'), 'w+'); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + return { parent, root, tools, read, prototype }; +} + +function hasCode(code: string) { + return (error: unknown) => + error instanceof WorkspaceToolError && error.code === code; +} + +test('small windows from large files use bounded descriptor reads, not readFile', async t => { + const { root, read, prototype } = await fixture(t, true); + await fs.writeFile( + join(root, 'file'), + 'first\nsecond\n' + 'tail\n'.repeat(300_000) + ); + const originalRead = prototype.read; + let bytesRead = 0; + t.mock.method(prototype, 'readFile', () => assert.fail('whole-file read')); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + const result = await originalRead.apply(this, args); + bytesRead += result.bytesRead; + return result; + } + ); + const result = await read(1, 2); + assert.equal(result.content, 'first\nsecond'); + assert.equal(result.nextStartLine, 3); + assert.ok(bytesRead <= 64 * 1024 + 3); +}); + +test('sequential line and byte pagination reconstructs large content exactly', async t => { + const { root, read } = await fixture(t); + const lines = Array.from( + { length: 1_201 }, + (_, i) => `${i}: ${'é🦊'.repeat(300)}` + ); + await fs.writeFile(join(root, 'file'), '\ufeff' + lines.join('\n') + '\n'); + const collected: string[] = []; + let start = 1; + for (;;) { + const result = await read(start, 500); + assert.ok(Buffer.byteLength(result.content) <= MAX_BYTES); + collected.push(...result.content.split('\n')); + assert.equal(result.endLine, collected.length); + if (!result.truncated) break; + assert.ok(result.nextStartLine! > start); + start = result.nextStartLine!; + } + assert.deepEqual(collected, lines); + const defaults = await read(); + assert.equal(defaults.endLine, 200); + assert.equal(defaults.nextStartLine, 201); +}); + +test('byte ceilings include separators and never split or repeat an oversized line', async t => { + const { root, read } = await fixture(t); + const exact = 'a'.repeat(MAX_BYTES); + await fs.writeFile( + join(root, 'file'), + exact + '\nx\n' + 'z'.repeat(MAX_BYTES + 1) + ); + const first = await read(1, 500); + assert.equal(first.content, exact); + assert.equal(first.nextStartLine, 2); + const second = await read(2, 500); + assert.equal(second.content, 'x'); + assert.equal(second.nextStartLine, 3); + await assert.rejects(read(3), error => { + assert.ok(hasCode('READ_LIMIT_EXCEEDED')(error)); + assert.match( + (error as Error).message, + /line 3.*later startLine.*search_text/ + ); + assert.ok((error as Error).message.length < 300); + return true; + }); + await fs.writeFile( + join(root, 'file'), + 'a'.repeat(MAX_BYTES - 2) + '\nb\nc' + ); + const separators = await read(1, 500); + assert.equal(Buffer.byteLength(separators.content), MAX_BYTES); + assert.equal(separators.endLine, 2); + assert.equal(separators.nextStartLine, 3); +}); + +test('EOF, empty lines, CRLF, BOM-only files, and starts beyond EOF retain their contract', async t => { + const { root, read } = await fixture(t); + for (const [source, expected] of [ + ['', ['']], + ['\ufeff', ['']], + ['\n', ['']], + ['\n\n', ['', '']], + ['a\n', ['a']], + ['a\r\nb\r\n', ['a\r', 'b\r']], + ['a\nb', ['a', 'b']], + ['\ufeff\ufeffa', ['\ufeffa']], + ] as const) { + await fs.writeFile(join(root, 'file'), source); + for (let line = 1; line <= expected.length; line++) { + const result = await read(line, 1); + assert.equal(result.content, expected[line - 1]); + assert.equal(result.endLine, line); + assert.equal(result.truncated, line < expected.length); + } + const past = await read(expected.length + 1); + assert.equal(past.content, ''); + assert.equal(past.endLine, expected.length); + assert.equal(past.truncated, false); + } +}); + +test('streaming decoding handles short reads, split UTF-8, CRLF, and UTF-16 surrogates', async t => { + const { root, read, prototype } = await fixture(t); + const originalRead = prototype.read; + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + buffer: Buffer, + offset: number, + length: number, + position: number + ) { + return originalRead.call(this, { + buffer, + offset, + length: Math.min(length, 1), + position, + }); + } + ); + const text = 'é🦊\r\n\ufeffsecond\n'; + const little = Buffer.from(text, 'utf16le'); + const big = Buffer.from(little).swap16(); + for (const source of [ + Buffer.from('\ufeff' + text), + Buffer.concat([Buffer.from([0xff, 0xfe]), little]), + Buffer.concat([Buffer.from([0xfe, 0xff]), big]), + ]) { + await fs.writeFile(join(root, 'file'), source); + const result = await read(); + assert.equal(result.content, 'é🦊\r\n\ufeffsecond'); + assert.equal(result.endLine, 2); + assert.equal(result.truncated, false); + } +}); + +test('UTF-8 replacement decoding and binary bytes are bounded text, not file validation', async t => { + const { root, read } = await fixture(t); + await fs.writeFile( + join(root, 'file'), + Buffer.from([0x61, 0x0a, 0xff, 0x00, 0xe2, 0x82]) + ); + const first = await read(1, 1); + assert.equal(first.content, 'a'); + assert.equal(first.truncated, true); + const second = await read(2); + assert.equal(second.content, '\ufffd\0\ufffd'); + await fs.writeFile(join(root, 'file'), Buffer.alloc(MAX_BYTES / 2, 0xff)); + await assert.rejects(read(), hasCode('READ_LIMIT_EXCEEDED')); +}); + +test('scanning skips huge earlier lines without accumulating them', async t => { + const { root, read } = await fixture(t); + await fs.writeFile( + join(root, 'file'), + 'x'.repeat(4 * MAX_BYTES) + '\nrequested\n' + ); + assert.equal((await read(2, 1)).content, 'requested'); +}); + +test('cancellation during scanning and collection closes every read handle', async t => { + for (const startLine of [1, 300_000]) { + const { root, read, prototype } = await fixture(t); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const controller = new AbortController(); + const originalRead = prototype.read; + const handles = new Set(); + let calls = 0; + const mock = t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + handles.add(this); + const result = await originalRead.apply(this, args); + if (++calls === 2) controller.abort(); + return result; + } + ); + await assert.rejects( + read(startLine, 500, controller.signal), + hasCode('EXECUTION_ABORTED') + ); + assert.ok(handles.size > 0); + for (const handle of handles) assert.equal(handle.fd, -1); + mock.mock.restore(); + } +}); + +test('far-away starts hit the scan deadline and close the descriptor', async t => { + const { root, read, prototype } = await fixture(t); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const originalRead = prototype.read; + const now = performance.now(); + let elapsed = 0; + let handle: FileHandle | undefined; + t.mock.method(performance, 'now', () => now + elapsed); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + handle = this; + const result = await originalRead.apply(this, args); + elapsed += 6_000; + return result; + } + ); + await assert.rejects(read(300_000), hasCode('READ_LIMIT_EXCEEDED')); + assert.equal(handle?.fd, -1); +}); + +test('growth is capped at the opened extent and truncation produces a valid EOF window', async t => { + for (const change of ['grow', 'truncate'] as const) { + const { root, read, prototype } = await fixture(t); + const path = join(root, 'file'); + await fs.writeFile(path, 'first\nsecond\n'); + const originalRead = prototype.read; + let calls = 0; + const mock = t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + const result = await originalRead.apply(this, args); + if (++calls === 1) { + if (change === 'grow') + await fs.appendFile(path, 'appended\n'); + else await fs.truncate(path, 3); + } + return result; + } + ); + const result = await read(); + assert.equal( + result.content, + change === 'grow' ? 'first\nsecond' : 'fir' + ); + assert.equal(result.truncated, false); + mock.mock.restore(); + } +}); + +test('file and root replacement cannot redirect a held streaming read; descriptors close', async t => { + const { parent, root, read, prototype } = await fixture(t, true); + const path = join(root, 'file'); + await fs.writeFile(path, 'original\n' + 'inside\n'.repeat(200_000)); + await fs.writeFile(join(parent, 'secret'), 'outside secret'); + const originalRead = prototype.read; + const originalOpen = WorkspaceRootAccess.open; + let rootHandle: FileHandle | undefined; + let fileHandle: FileHandle | undefined; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const access = await originalOpen(...args); + rootHandle = access.handle; + return access; + } + ); + let calls = 0; + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + fileHandle = this; + const result = await originalRead.apply(this, args); + if (++calls === 1) { + await fs.rename(path, join(root, 'old-file')); + await fs.symlink(join(parent, 'secret'), path); + await fs.rename(root, join(parent, 'old-root')); + await fs.mkdir(root); + await fs.writeFile(path, 'replacement'); + } + return result; + } + ); + const result = await read(2, 500); + assert.equal(result.content, Array(500).fill('inside').join('\n')); + assert.equal(result.nextStartLine, 502); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); + await assert.rejects(read(), hasCode('REGISTRATION_INVALID')); +}); + +test('oversized lines and I/O failures close file and held-root descriptors', async t => { + const { root, read, prototype } = await fixture(t, true); + const originalRead = prototype.read; + const originalOpen = WorkspaceRootAccess.open; + let fileHandle: FileHandle | undefined; + let rootHandle: FileHandle | undefined; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const access = await originalOpen(...args); + rootHandle = access.handle; + return access; + } + ); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + fileHandle = this; + return originalRead.apply(this, args); + } + ); + await fs.writeFile(join(root, 'file'), 'x'.repeat(MAX_BYTES + 1)); + await assert.rejects(read(), hasCode('READ_LIMIT_EXCEEDED')); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); + t.mock.method(prototype, 'read', async function (this: FileHandle) { + fileHandle = this; + throw Object.assign(new Error('read failed'), { code: 'EIO' }); + }); + await assert.rejects(read(), hasCode('INVALID_PATH')); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); +}); + +test('search, preview, edit, and digest-checked instruction limits remain separate', async t => { + const { root, tools } = await fixture(t, true); + await fs.writeFile(join(root, 'file'), 'needle\n'.repeat(200_000)); + const search = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + path: 'file', + query: 'needle', + }); + assert.equal(search.operation, 'search_text'); + assert.deepEqual(search.matches, []); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation, + workspaceId: 'primary', + path: 'file', + oldText: 'needle', + newText: 'changed', + }), + hasCode( + operation === 'preview_edit' + ? 'READ_LIMIT_EXCEEDED' + : 'WRITE_LIMIT_EXCEEDED' + ) + ); + } + await fs.writeFile( + join(root, 'AGENTS.md'), + 'instructions\n'.repeat(200_000) + ); + const snapshot = await readRepositoryInstructions(root); + assert.ok(snapshot); + const request: WorkspaceReadFileRequest = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'AGENTS.md', + instructionSha256: snapshot.descriptor.sha256, + }; + const result = await tools.execute(request); + assert.ok(isWorkspaceToolResult(request, result)); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, snapshot.content); + assert.equal(result.truncated, true); + assert.equal(result.nextStartLine, undefined); + await fs.writeFile(join(root, 'AGENTS.md'), 'changed'); + await assert.rejects(tools.execute(request), hasCode('INVALID_PATH')); +}); + +test('byte-budget pagination reconstructs every complete line', async t => { + const { root, read } = await fixture(t); + const lines = Array.from( + { length: 1_201 }, + (_, i) => `${i}: ${'é🦊'.repeat(400)}` + ); + await fs.writeFile(join(root, 'file'), lines.join('\n') + '\n'); + const collected: string[] = []; + let startLine = 1; + for (;;) { + const result = await read(startLine, 500); + const selected = result.content.split('\n'); + assert.ok(Buffer.byteLength(result.content) <= MAX_BYTES); + if (result.truncated) + assert.ok(selected.length < 500, 'bytes forced continuation'); + collected.push(...selected); + if (!result.truncated) break; + assert.ok(result.nextStartLine! > startLine); + startLine = result.nextStartLine!; + } + assert.deepEqual(collected, lines); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index e48365c8..746c7249 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; +import { deferWorkspaceRootCleanup, holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; @@ -133,6 +133,8 @@ const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; const MAX_SEARCH_CANDIDATES = 20_000; const SEARCH_TIMEOUT_MS = 10_000; const LIST_TIMEOUT_MS = 10_000; +const READ_TIMEOUT_MS = 10_000; +const READ_CHUNK_BYTES = 64 * 1024; const READ_OPERATIONS = [ 'read_file', @@ -273,10 +275,18 @@ function resolveWorkspacePath(root: string, requestedPath: string): string { return candidate; } -async function readConfinedFileBuffer( +interface WorkspaceReadControl { + signal?: AbortSignal; + deadline: number; + interrupted: boolean; +} + +async function withConfinedFile( root: string, requestedPath: string, -): Promise { + read: (handle: FileHandle, size: number) => Promise, + control?: WorkspaceReadControl, +): Promise { const candidate = resolveWorkspacePath(root, requestedPath); let handle: FileHandle | undefined; try { @@ -297,7 +307,31 @@ async function readConfinedFileBuffer( ) { throw new Error('Invalid workspace path'); } - if (openedFile.size > BRIDGE_WORKSPACE_READ_MAX_BYTES) { + return await read(handle, openedFile.size); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, candidate); + } + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } finally { + if (handle) { + const closing = handle.close(); + if (control?.interrupted) { + // Node retains the descriptor and buffer until outstanding I/O drains. + deferWorkspaceRootCleanup(); + void closing.catch(() => {}); + } else await closing; + } + } +} + +async function readConfinedFileBuffer( + root: string, + requestedPath: string, +): Promise { + return withConfinedFile(root, requestedPath, async (handle, size) => { + if (size > BRIDGE_WORKSPACE_READ_MAX_BYTES) { throw new WorkspaceToolError( 'Workspace file exceeds read limit', 'READ_LIMIT_EXCEEDED', @@ -322,31 +356,222 @@ async function readConfinedFileBuffer( ); } return buffer.subarray(0, bytesRead); - } catch (error) { - if (error instanceof WorkspaceToolError) throw error; - if (isMissingEntry(error)) { - throw await classifyMissingWorkspacePath(root, candidate); - } - throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + }); +} + +function interruptRead( + control: WorkspaceReadControl, + message: string, + code: 'EXECUTION_ABORTED' | 'READ_LIMIT_EXCEEDED', +): WorkspaceToolError { + control.interrupted = true; + return new WorkspaceToolError(message, code); +} + +function checkReadDeadline(control: WorkspaceReadControl): void { + if (control.signal?.aborted) { + throw interruptRead( + control, + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (performance.now() >= control.deadline) { + throw interruptRead( + control, + 'Workspace read exceeded its scan time limit; request an earlier startLine or use search_text to locate content', + 'READ_LIMIT_EXCEEDED', + ); + } +} + +async function readWorkspaceChunk( + handle: FileHandle, + buffer: Buffer, + length: number, + position: number, + control: WorkspaceReadControl, +): Promise { + checkReadDeadline(control); + const { signal, deadline } = control; + let timer: ReturnType | undefined; + let abort: (() => void) | undefined; + try { + return await Promise.race([ + handle + .read(buffer, 0, length, position) + .then((result) => result.bytesRead), + new Promise((_, reject) => { + abort = () => + reject( + interruptRead( + control, + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ), + ); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + timer = setTimeout( + () => + reject( + interruptRead( + control, + 'Workspace read exceeded its scan time limit; retry with an earlier startLine', + 'READ_LIMIT_EXCEEDED', + ), + ), + Math.max(0, deadline - performance.now()), + ); + }), + ]); } finally { - await handle?.close(); + if (timer !== undefined) clearTimeout(timer); + if (abort) signal?.removeEventListener('abort', abort); } } async function readConfinedFile( root: string, - requestedPath: string, -): Promise { - const decoded = decodeWorkspaceText( - await readConfinedFileBuffer(root, requestedPath), - ); - if (Buffer.byteLength(decoded, 'utf8') > BRIDGE_WORKSPACE_READ_MAX_BYTES) { - throw new WorkspaceToolError( - 'Workspace file exceeds read limit', - 'READ_LIMIT_EXCEEDED', + request: WorkspaceReadFileRequest, + signal?: AbortSignal, +): Promise { + const startLine = request.startLine ?? 1; + const maxLines = request.maxLines ?? 200; + const control: WorkspaceReadControl = { + signal, + deadline: performance.now() + READ_TIMEOUT_MS, + interrupted: false, + }; + const read = async ( + handle: FileHandle, + size: number, + ): Promise => { + const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES); + // Fix the read extent at admission so an appending writer cannot extend the scan. + let position = 0; + const header = Buffer.alloc(3); + let headerBytes = 0; + while (headerBytes < header.length && position < size) { + const bytes = await readWorkspaceChunk( + handle, + buffer, + Math.min(header.length - headerBytes, size - position), + position, + control, + ); + checkReadDeadline(control); + if (bytes === 0) break; + buffer.copy(header, headerBytes, 0, bytes); + headerBytes += bytes; + position += bytes; + } + const utf16le = + headerBytes >= 2 && header[0] === 0xff && header[1] === 0xfe; + const utf16be = + headerBytes >= 2 && header[0] === 0xfe && header[1] === 0xff; + const bomBytes = + utf16le || utf16be + ? 2 + : headerBytes === 3 && + header[0] === 0xef && + header[1] === 0xbb && + header[2] === 0xbf + ? 3 + : 0; + // Preserve ordinary reads' replacement decoding and BOM-marked UTF-16 support. + const decoder = new TextDecoder( + utf16le ? 'utf-16le' : utf16be ? 'utf-16be' : 'utf-8', + { ignoreBOM: true }, ); - } - return decoded; + const lines: string[] = []; + let fragments: string[] = []; + let line = 1; + let lineBytes = 0; + let returnedBytes = 0; + let hasText = false; + let truncated = false; + const consume = (text: string): void => { + let offset = 0; + while (offset < text.length) { + checkReadDeadline(control); + const newline = text.indexOf('\n', offset); + const end = newline < 0 ? text.length : newline; + if (line >= startLine) { + if (lines.length === maxLines) { + truncated = true; + return; + } + const fragment = text.slice(offset, end); + const bytes = Buffer.byteLength(fragment, 'utf8'); + if ( + returnedBytes + (lines.length > 0 ? 1 : 0) + lineBytes + bytes > + BRIDGE_WORKSPACE_READ_MAX_BYTES + ) { + if (lines.length === 0) { + throw new WorkspaceToolError( + `Workspace file exceeds read limit: line ${line} cannot fit within ${BRIDGE_WORKSPACE_READ_MAX_BYTES} UTF-8 bytes; request a later startLine or use search_text`, + 'READ_LIMIT_EXCEEDED', + ); + } + truncated = true; + return; + } + if (fragment.length > 0) fragments.push(fragment); + lineBytes += bytes; + } + hasText ||= end > offset; + if (newline < 0) return; + if (line >= startLine) { + lines.push(fragments.join('')); + returnedBytes += (lines.length > 1 ? 1 : 0) + lineBytes; + fragments = []; + lineBytes = 0; + } + line += 1; + hasText = false; + offset = newline + 1; + } + }; + consume( + decoder.decode(header.subarray(bomBytes, headerBytes), { + stream: true, + }), + ); + while (!truncated && position < size) { + const bytes = await readWorkspaceChunk( + handle, + buffer, + Math.min(buffer.length, size - position), + position, + control, + ); + checkReadDeadline(control); + if (bytes === 0) break; + position += bytes; + consume(decoder.decode(buffer.subarray(0, bytes), { stream: true })); + } + if (!truncated) { + consume(decoder.decode()); + if (!truncated && line >= startLine && (hasText || line === 1)) { + lines.push(fragments.join('')); + } + } + checkReadDeadline(control); + const endLine = startLine + lines.length - 1; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: request.workspaceId, + path: request.path, + content: lines.join('\n'), + startLine, + endLine, + truncated, + ...(truncated ? { nextStartLine: endLine + 1 } : {}), + }; + }; + return withConfinedFile(root, request.path, read, control); } interface WorkspaceRoot { @@ -1831,31 +2056,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ) { throw new WorkspaceToolError('Invalid workspace read', 'INVALID_REQUEST'); } - const content = await readConfinedFile(root, request.path); - if (signal?.aborted) { - throw new WorkspaceToolError( - 'Workspace tool execution aborted', - 'EXECUTION_ABORTED', - ); - } - const lines = content.endsWith('\n') - ? content.slice(0, -1).split('\n') - : content.split('\n'); - const selected = lines.slice(startLine - 1, startLine - 1 + maxLines); - const endLine = startLine + selected.length - 1; - const truncated = endLine < lines.length; - - return { - protocolVersion: BRIDGE_PROTOCOL_VERSION, - operation: 'read_file', - workspaceId: request.workspaceId, - path: request.path, - content: selected.join('\n'), - startLine, - endLine, - truncated, - ...(truncated ? { nextStartLine: endLine + 1 } : {}), - }; + return readConfinedFile(root, request, signal); } } From e829f7554c05bfc16c51a698b4a8216b8c027e5c Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:41:29 -0400 Subject: [PATCH 66/75] =?UTF-8?q?=F0=9F=97=82=EF=B8=8F=20fix:=20Guard=20La?= =?UTF-8?q?ne=20Worktree=20Registration=20(#304)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🗂️ fix: Guard Lane Worktree Registration * test: Cover Linked Lane Worktree Registration Denials --------- Co-authored-by: Lia --- docs/adr/0010-worktree-metadata-admission.md | 101 ++++++++++++++++++ .../src/linked-worktree-git-guard.test.ts | 13 +++ .../code/src/linked-worktree-git-guard.ts | 6 ++ tests/worktree-metadata-phases.mjs | 23 ++++ 4 files changed, 143 insertions(+) create mode 100644 docs/adr/0010-worktree-metadata-admission.md create mode 100644 tests/worktree-metadata-phases.mjs diff --git a/docs/adr/0010-worktree-metadata-admission.md b/docs/adr/0010-worktree-metadata-admission.md new file mode 100644 index 00000000..4976adc3 --- /dev/null +++ b/docs/adr/0010-worktree-metadata-admission.md @@ -0,0 +1,101 @@ +# Target-Lane Worktree Lifecycle Admission + +Status: proposed. No scheduling or sandbox policy changes are enabled by this PR. + +## Failure + +A shell call to `git worktree add` currently uses the checkout isolation key. +`BridgeWorkspaceSlots.reserve` waits for every child lane; the worker's lane +family guard and native sandbox enforce the same exclusion. Repeated short +admission timeouts discard a waiting checkout request before lanes drain. +The reviewer in chat e2fedf18 (LibreChat PR #16722) could not create its lane. +Durable admission preserves FIFO, but does not remove the checkout-wide barrier. + +## Decision + +Add explicit `create_worktree` and `remove_worktree` workspace operations. +Never infer a weaker scope by parsing an arbitrary shell command. +Each lifecycle request reserves the target lane's ordinary isolation key, +including checkout exclusion and existing quarantine fencing. Sibling lanes +remain independent. Advertise support only when the service and worker agree. +Old clients keep using checkout-wide shell calls. + +Git metadata mutation gets a separate worker-owned mutex, scoped to the common +Git directory and held only while changing a worktree registration. The target +reservation covers the whole operation. It is not the metadata mutex. + +```text +create(name, frozenCommit) + target-lane reservation + validate root identity, target absence, commit and branch policy + metadata mutex → git worktree add --no-checkout --detach → unlock + target lane → git checkout frozenCommit → verify pointers and root identity + publish ready result, then release reservation + +remove(name) + target-lane reservation, require no mutation quarantine + verify pointers and identity, clean tracked/untracked state, no nested mounts + metadata mutex → rename lane to private tombstone + → git worktree remove original-path (now absent) → unlock + delete tombstone outside mutex; keep branch and reachable objects + persist outcome, then release reservation +``` + +A lifecycle journal outside untrusted roots records `reserved`, `registered`, +`ready`, `retiring`, `detached` and `removed`, with assignment owner and captured +identities. Worker recovery reconciles journal entries, never reruns uncertain +shell commands. Creation is not published until checkout completes. Failed +checkout retains the target fence until reconciled. Rename-before-detach must +roll back only while registration still points to the captured original target; +otherwise preserve the tombstone and require reconciliation. Never remove dirty +worktrees, force-remove, prune all registrations, or delete branches as recovery. + +## Boundaries + +- `service/src/bridge/slots.ts`: reserve the named target, not a new global slot + limit. A waiting root still prevents newer sibling admissions. A metadata + operation consumes an existing worker slot; fully occupied workers still wait. +- `packages/code/src/worker.ts`: treat lifecycle operations as mutations and + derive target-lane guards before execution. Keep assignment/identity fencing. +- `packages/code/src/linked-worktrees.ts`: target ownership includes absent + lanes during creation and excludes local retirement of that lane. +- `packages/code/src/linked-worktree-git-guard.ts`: ordinary lanes reject + registration-changing `git worktree` commands. Do not grant write access to + `.git/worktrees` or sibling checkouts to make shell operations work. +- Native sandbox: bulk checkout runs only in the target's admitted directory. + Run no repo hooks or credential helpers in the trusted metadata phase. + +## Alternatives + +1. Longer checkout waits: smallest fix and required for old servers, but still + drains all lanes and serializes bulk checkout/deletion. +2. Allow `git worktree` from arbitrary lane shells: rejected. A shell can remove + a busy sibling or repair foreign metadata; mutexing Git does not confine it. +3. Explicit lifecycle operations: recommended. More protocol/recovery surface, + but target ownership and metadata lifetime are independently testable. + +## Compatibility and rollout + +Deploy the worker journal/guard first, then service capability and matching +client tools. Never send target-scoped lifecycle work to an older worker. +Disable new submissions and drain lifecycle journals before rollback. Root +shell operations and destructive object maintenance remain checkout-exclusive. +Concurrent Git object/ref access still uses Git's own atomic locks; automatic +maintenance stays disabled in lanes. + +A read-only GitHub compare tool supplies merge-base and ancestry for public +repos without any worker call. This helps review orient while admission is busy, +but does not substitute for reading the frozen source or running tests. + +## Blocking verification + +- A long sibling lane remains running while another target is created/removed. +- Same-target/root exclusion and older-root FIFO barriers in Redis and worker. +- Short mutex lifetime verified while bulk checkout/deletion is deliberately gated. +- Cancellation/restart at each journal transition and lost acknowledgements. +- Dirty/untracked, symlink substitution, nested mount, quarantine and foreign-root denial. +- Lane Git guard rejects add/remove/move/repair/prune, including global options. +- Old/new service, worker and client combinations fail closed. + +`tests/worktree-metadata-phases.mjs` probes the local Git mechanism only. It does +not prove service scheduling, worker recovery, or live sandbox containment. diff --git a/packages/code/src/linked-worktree-git-guard.test.ts b/packages/code/src/linked-worktree-git-guard.test.ts index 96f16b4a..96386418 100644 --- a/packages/code/src/linked-worktree-git-guard.test.ts +++ b/packages/code/src/linked-worktree-git-guard.test.ts @@ -274,3 +274,16 @@ test('Git LFS fetch and pull cannot request pruning in a lane', async t => { assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); } }); + +test('lane guard rejects registration changes but permits read-only worktree listing', async t => { + const parent = await mkdtemp(join(tmpdir(), 'lane-registration-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + await mkdir(bin); await writeLinkedWorktreeGitGuard(bin); + await execFileAsync('/usr/bin/git', ['init', '-q', '-b', 'main'], { cwd: parent }); + const run = (args: string[]) => execFileAsync(join(bin, 'git'), args, { cwd: parent }); + assert.match((await run(['worktree', 'list', '--porcelain'])).stdout, /worktree/); + for (const operation of ['add', 'remove', 'move', 'repair', 'prune', 'lock', 'unlock']) { + await assert.rejects(run(['-C', parent, 'worktree', operation, '.worktrees/other']), /require checkout admission/); + } +}); diff --git a/packages/code/src/linked-worktree-git-guard.ts b/packages/code/src/linked-worktree-git-guard.ts index 227bfd3f..cee4a7f0 100644 --- a/packages/code/src/linked-worktree-git-guard.ts +++ b/packages/code/src/linked-worktree-git-guard.ts @@ -72,6 +72,12 @@ export async function writeLinkedWorktreeGitGuard(directory: string): Promise&2', + ' return 1', + ' fi ;;', + ' prune|gc|repack|prune-packed|maintenance|multi-pack-index|for-each-repo)', ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', ' return 1 ;;', diff --git a/tests/worktree-metadata-phases.mjs b/tests/worktree-metadata-phases.mjs new file mode 100644 index 00000000..089251b9 --- /dev/null +++ b/tests/worktree-metadata-phases.mjs @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { mkdtemp,writeFile,mkdir,rename,rm } from 'node:fs/promises'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +const exec=promisify(execFile),root=await mkdtemp(join(tmpdir(),'metadata-phases-')); +try { + const git=(args,cwd=root)=>exec('git',args,{cwd,env:{...process.env,GIT_CONFIG_NOSYSTEM:'1'}}); + await git(['init','-q','-b','main']); await writeFile(join(root,'file'),'tracked\n'); await git(['add','file']); + await git(['-c','user.name=test','-c','user.email=test@example.test','-c','commit.gpgsign=false','commit','-qm','seed']); + const commit=(await git(['rev-parse','HEAD'])).stdout.trim(); await mkdir(join(root,'.worktrees')); + const lane=join(root,'.worktrees','review'); + await git(['worktree','add','--no-checkout','--detach',lane,commit]); + await git(['-c','core.hooksPath=/dev/null','checkout','--detach',commit],lane); + assert.equal((await git(['status','--porcelain'],lane)).stdout,''); + const tombstone=join(root,'.retired-review'); await rename(lane,tombstone); + await git(['worktree','remove',lane]); + assert.doesNotMatch((await git(['worktree','list','--porcelain'])).stdout,/review/); + // Registration removal did not delete the detached bulk directory. + await rm(tombstone,{recursive:true}); + console.log('Split-phase registration/checkout and rename/detach/delete: passed'); +} finally { await rm(root,{recursive:true,force:true}); } From 829992a4d837da67737edcdc3c5b57eaaa5c3b3d Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:41:15 -0400 Subject: [PATCH 67/75] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20fix:=20Harden=20S?= =?UTF-8?q?andbox=20Startup=20and=20Internal=20Transports=20(#305)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lia --- api/src/nsjail.test.ts | 16 +++ api/src/nsjail.ts | 10 ++ docs/SECURITY-MAINTENANCE.md | 63 +++++++++++ launcher/src/main.rs | 46 ++++++-- service/bun.lock | 102 +++++++++++------- service/package.json | 10 +- service/src/egress-gateway-client.test.ts | 3 + service/src/egress-gateway-client.ts | 4 + service/src/egress-gateway.test.ts | 37 +++++++ service/src/egress-gateway.ts | 11 +- service/src/hosted-app/microvm-runtime.ts | 3 + service/src/internal-redirect-policy.test.ts | 56 ++++++++++ service/src/minio-compat.test.ts | 39 +++++++ service/src/runtime-session/checkpoint.ts | 4 + service/src/runtime-session/files.ts | 1 + service/src/sandbox-backend/http.test.ts | 24 ++++- service/src/sandbox-backend/http.ts | 1 + service/src/sandbox-backend/lambda-microvm.ts | 2 + service/src/service/programmatic-router.ts | 6 +- service/src/service/replay-state.ts | 2 + service/src/service/router.ts | 7 +- test-sandbox.sh | 24 ++++- 22 files changed, 400 insertions(+), 71 deletions(-) create mode 100644 docs/SECURITY-MAINTENANCE.md create mode 100644 service/src/internal-redirect-policy.test.ts create mode 100644 service/src/minio-compat.test.ts diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index ee6fbbc8..95de6251 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -374,6 +374,22 @@ describe('NsJail seccomp policy', () => { expect(errnoSocketRule).not.toContain('AF_VSOCK'); }); + test('blocks priority-inheritance futex operations without denying ordinary wait/wake', () => { + const policy = seccompPolicy(); + expect(policy).toContain('#define FUTEX_CMD_MASK 0x7f'); + const rule = policy.split('ERRNO(1)')[1]?.split('\n') + .find(line => line.includes('futex(uaddr, op)')); + expect(rule).toBeDefined(); + for (const [operation, command] of [ + ['LOCK_PI', 6], ['UNLOCK_PI', 7], ['TRYLOCK_PI', 8], + ['WAIT_REQUEUE_PI', 11], ['CMP_REQUEUE_PI', 12], ['LOCK_PI2', 13], + ] as const) { + expect(policy).toContain(`#define FUTEX_${operation} ${command}`); + expect(rule).toContain(`(op & FUTEX_CMD_MASK) == FUTEX_${operation}`); + } + expect(rule).not.toMatch(/== FUTEX_(WAIT|WAKE)(?:\s|$)/); + }); + test('rejects Copy Fail and Dirty Frag socket entry points in the sandbox', () => { const policy = seccompPolicy(); const errnoBlock = policy.split('ERRNO(1)')[1] ?? ''; diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index a9f02931..3a5de4ad 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -104,6 +104,13 @@ const SECCOMP_POLICY = [ '#define AF_VSOCK 40', '#define CLONE_NAMESPACE_FLAGS 0x7e020000', '#define KVM_IOCTL_MAGIC 0xAE00', + '#define FUTEX_CMD_MASK 0x7f', + '#define FUTEX_LOCK_PI 6', + '#define FUTEX_UNLOCK_PI 7', + '#define FUTEX_TRYLOCK_PI 8', + '#define FUTEX_WAIT_REQUEUE_PI 11', + '#define FUTEX_CMP_REQUEUE_PI 12', + '#define FUTEX_LOCK_PI2 13', 'POLICY sandbox {', ' KILL {', ' ptrace, memfd_create, personality, userfaultfd,', @@ -141,6 +148,9 @@ const SECCOMP_POLICY = [ ' },', ' ERRNO(1) {', ' io_uring_setup, io_uring_enter, io_uring_register, sched_setaffinity, vmsplice,', + /* GhostLock (CVE-2026-43499): deny PI operations, including private and + * realtime variants, without blocking ordinary runtime futex wait/wake. */ + ' futex(uaddr, op) { (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_UNLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_TRYLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_WAIT_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_CMP_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI2 },', ' clone(flags) { (flags & CLONE_NAMESPACE_FLAGS) != 0 },', /* Block signals to PID 1 of the sandbox PID namespace (the NsJail * monitor). With clone_newpid the user can't reach other tenants — but diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md new file mode 100644 index 00000000..24c1ee01 --- /dev/null +++ b/docs/SECURITY-MAINTENANCE.md @@ -0,0 +1,63 @@ +# Sandbox security maintenance + +The default boundary is NsJail inside a libkrun guest. Direct NsJail mode +shares the host kernel and is for trusted development, not hostile workloads. + +## Verify deployed patches + +Repository dependencies do not establish production kernel patch status. +Inventory each runner node and deployed runner image: + +- Host OS and running kernel release (`uname -r` on the node). +- Container runtime version and runner image digest. +- Installed `libkrun` and `libkrunfw` packages + (`rpm -q libkrun libkrunfw` inside the runner image). +- Bundled guest kernel release, from an operator-controlled guest inspection. +- Guest-visible nested virtualization features and exposed virtual devices. + +Match kernel packages against distro security advisories, including backports. +Patch both the host kernel and bundled guest kernel. Rebuild and replace runner +images after guest-kernel updates; reboot or use the vendor-supported livepatch +procedure for host fixes. A container rebuild does not update host KVM. + +Priority 2026 checks: + +| Boundary | Advisory | +|---|---| +| Guest kernel; host kernel in direct mode | Bad Epoll, CVE-2026-46242 | +| Guest and host kernels | GhostLock, CVE-2026-43499 | +| x86 host KVM | CVE-2026-46113, CVE-2026-53359, CVE-2026-64561 | +| ARM host KVM, when virtual ITS is exposed | CVE-2026-46316 | + +PI-futex filtering reduces the GhostLock surface but does not replace patches. +Do not disable ordinary futex or epoll operations: language runtimes use them. +Do not expose nested virtualization to untrusted guests. Confirm the installed +VMM's behavior rather than assuming current upstream defaults match its build. + +The October 2026 Vercel KVM report has no disclosed root cause or affected-version +matrix at the time of this change. These controls are not a verified fix for it. + +## Preserve host-side confinement + +- Run sandbox runners on isolated nodes, separate from credential-bearing + control-plane workloads, with least-privilege node IAM. +- Keep hardened startup, signed manifests, egress controls, network policies, + and disabled runner service-account token mounting. +- The launcher refuses to boot if its host-side seccomp filter cannot install. +- Internal authenticated HTTP requests reject redirects. Configure their final + endpoints directly; do not depend on HTTP-to-HTTPS redirects. + +## Dependency checks + +Run `bun audit --cwd api` and `bun audit --cwd service` against the lockfiles. +Distinguish development dependencies from production paths and establish the +advisory's prerequisites before claiming an application exploit. Do not force +incompatible transitive major upgrades solely to obtain a clean audit. + +Remaining service audit findings at this change: + +- `stream-json` (GHSA-528h-pc64-c93x): MinIO 8 uses its JSON-lines parser, + not the affected path filters. Keep MinIO's supported major dependency; + reassess if notifications or path filters become reachable. +- `braces` (GHSA-vfj7-8cjw-p6xm): development-only dependency with no published + fix. Do not feed untrusted patterns to build tooling; track an upstream fix. diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 27f08f92..84e48347 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -570,6 +570,16 @@ fn guest_cmdline_problem(env: &[(String, String)], args: &[String]) -> Option, + enter: impl FnOnce() -> i32, +) -> Result { + let allowed = filter_result?; + eprintln!("[launcher] VMM seccomp filter applied ({allowed} syscalls allowed, ioctl restricted to KVM+terminal)"); + eprintln!("[launcher] Starting microVM..."); + Ok(enter()) +} + fn main() { let vcpus: u8 = env::var("LAUNCHER_VCPUS") .ok() @@ -701,16 +711,13 @@ fn main() { ffi::check("set_rlimits", ffi::krun_set_rlimits(ctx, rlimit_ptrs.as_ptr())); eprintln!("[launcher] Guest RLIMIT_NOFILE target: {nofile_target}"); - match seccomp::apply_vmm_filter() { - Ok(n) => eprintln!("[launcher] VMM seccomp filter applied ({n} syscalls allowed, ioctl restricted to KVM+terminal)"), - Err(e) => { - eprintln!("[launcher] WARNING: Failed to apply VMM seccomp: {e}"); + let ret = match enter_microvm(seccomp::apply_vmm_filter(), || ffi::krun_start_enter(ctx)) { + Ok(ret) => ret, + Err(error) => { + eprintln!("[launcher] ERROR: refusing to start microVM without VMM seccomp: {error}"); + process::exit(1); } - } - - eprintln!("[launcher] Starting microVM..."); - - let ret = ffi::krun_start_enter(ctx); + }; eprintln!("[launcher] krun_start_enter returned {ret} (should not return on success)"); process::exit(1); } @@ -719,7 +726,7 @@ fn main() { #[cfg(test)] mod tests { use super::{ - desired_nofile_soft_limit, guest_cmdline_problem, guest_nofile_rlimit, + desired_nofile_soft_limit, enter_microvm, guest_cmdline_problem, guest_nofile_rlimit, is_allowed_guest_env_key, GUEST_CMDLINE_LIMIT, LIBKRUN_CMDLINE_RESERVE, }; @@ -738,6 +745,25 @@ mod tests { .collect() } + #[test] + fn microvm_does_not_start_when_seccomp_installation_fails() { + let result = enter_microvm(Err("filter installation denied".into()), || { + panic!("microVM entry must not run without seccomp"); + }); + assert_eq!(result, Err("filter installation denied".into())); + } + + #[test] + fn microvm_starts_after_seccomp_installation_succeeds() { + let mut entered = false; + let result = enter_microvm(Ok(80), || { + entered = true; + -1 + }); + assert!(entered); + assert_eq!(result, Ok(-1)); + } + #[test] fn guest_cmdline_accepts_single_line_printable_env() { let env = env(&[ diff --git a/service/bun.lock b/service/bun.lock index 0a0dea4a..9aacef7e 100644 --- a/service/bun.lock +++ b/service/bun.lock @@ -12,14 +12,14 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "axios": "^1.8.4", + "axios": "^1.20.0", "bullmq": "^5.13.1", "busboy": "^1.6.0", "dotenv": "^17.4.2", - "express": "^4.19.2", + "express": "^4.22.3", "express-rate-limit": "^7.4.1", "ioredis": "^5.4.1", - "minio": "^8.0.5", + "minio": "^8.0.7", "nanoid": "^5.1.16", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", @@ -52,6 +52,8 @@ }, "overrides": { "decode-uri-component": "0.5.0", + "qs": "6.16.0", + "uuid": "11.1.1", }, "packages": { "@aws-sdk/checksums": ["@aws-sdk/checksums@3.1000.19", "", { "dependencies": { "@aws-sdk/core": "^3.976.0", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.29.4", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-Hc4N100RdkuWshKBnhPzmpdftfi9mCLz+OHFELHM1QIgMH4QRUUWyWgfiebta/YX2Bd62wTcm3EqAP8TeXv0gA=="], @@ -142,6 +144,8 @@ "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@0.2.12", "", { "dependencies": { "@emnapi/core": "^1.4.3", "@emnapi/runtime": "^1.4.3", "@tybys/wasm-util": "^0.10.0" } }, "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ=="], + "@nodable/entities": ["@nodable/entities@3.1.0", "", {}, "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ=="], + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], @@ -312,8 +316,6 @@ "@unrs/resolver-binding-win32-x64-msvc": ["@unrs/resolver-binding-win32-x64-msvc@1.11.1", "", { "os": "win32", "cpu": "x64" }, "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g=="], - "@zxing/text-encoding": ["@zxing/text-encoding@0.9.0", "", {}, "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA=="], - "accepts": ["accepts@1.3.8", "", { "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" } }, "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw=="], "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], @@ -322,12 +324,16 @@ "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], - "ajv": ["ajv@6.12.6", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g=="], + "agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + + "ajv": ["ajv@6.14.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw=="], "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + "anynum": ["anynum@1.0.1", "", {}, "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A=="], + "arg": ["arg@4.1.3", "", {}, "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA=="], "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], @@ -358,7 +364,7 @@ "available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="], - "axios": ["axios@1.13.2", "", { "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA=="], + "axios": ["axios@1.20.0", "", { "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.6", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg=="], "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], @@ -366,11 +372,11 @@ "block-stream2": ["block-stream2@2.1.0", "", { "dependencies": { "readable-stream": "^3.4.0" } }, "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg=="], - "body-parser": ["body-parser@1.20.4", "", { "dependencies": { "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "~1.2.0", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "on-finished": "~2.4.1", "qs": "~6.14.0", "raw-body": "~2.5.3", "type-is": "~1.6.18", "unpipe": "~1.0.0" } }, "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA=="], + "body-parser": ["body-parser@1.20.6", "", { "dependencies": { "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "~1.2.0", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "on-finished": "~2.4.1", "qs": "~6.15.1", "raw-body": "~2.5.3", "type-is": "~1.6.18", "unpipe": "~1.0.0" } }, "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g=="], "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], - "brace-expansion": ["brace-expansion@1.1.12", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg=="], + "brace-expansion": ["brace-expansion@1.1.21", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], @@ -456,7 +462,7 @@ "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], - "diff": ["diff@4.0.2", "", {}, "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A=="], + "diff": ["diff@4.0.4", "", {}, "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ=="], "dir-glob": ["dir-glob@3.0.1", "", { "dependencies": { "path-type": "^4.0.0" } }, "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA=="], @@ -520,7 +526,7 @@ "eventemitter3": ["eventemitter3@5.0.1", "", {}, "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="], - "express": ["express@4.22.1", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.3", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g=="], + "express": ["express@4.22.3", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.13", "proxy-addr": "~2.0.7", "qs": "~6.16.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw=="], "express-rate-limit": ["express-rate-limit@7.5.1", "", { "peerDependencies": { "express": ">= 4.11" } }, "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw=="], @@ -532,7 +538,9 @@ "fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="], - "fast-xml-parser": ["fast-xml-parser@4.5.3", "", { "dependencies": { "strnum": "^1.1.1" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig=="], + "fast-xml-builder": ["fast-xml-builder@1.3.1", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug=="], + + "fast-xml-parser": ["fast-xml-parser@5.11.2", "", { "dependencies": { "@nodable/entities": "^3.0.1", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", "strnum": "^2.4.2", "xml-naming": "^0.3.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog=="], "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], @@ -556,15 +564,15 @@ "flat-cache": ["flat-cache@3.2.0", "", { "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.3", "rimraf": "^3.0.2" } }, "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw=="], - "flatted": ["flatted@3.3.3", "", {}, "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg=="], + "flatted": ["flatted@3.4.2", "", {}, "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA=="], "fn.name": ["fn.name@1.1.0", "", {}, "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="], - "follow-redirects": ["follow-redirects@1.15.11", "", {}, "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="], + "follow-redirects": ["follow-redirects@1.16.0", "", { "peerDependencies": { "debug": "*" }, "optionalPeers": ["debug"] }, "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw=="], "for-each": ["for-each@0.3.5", "", { "dependencies": { "is-callable": "^1.2.7" } }, "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg=="], - "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], + "form-data": ["form-data@4.0.6", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.4", "mime-types": "^2.1.35" } }, "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ=="], "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], @@ -624,6 +632,8 @@ "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], + "https-proxy-agent": ["https-proxy-agent@5.0.1", "", { "dependencies": { "agent-base": "6", "debug": "4" } }, "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA=="], + "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], "ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], @@ -644,8 +654,6 @@ "ipaddr.js": ["ipaddr.js@2.3.0", "", {}, "sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg=="], - "is-arguments": ["is-arguments@1.2.0", "", { "dependencies": { "call-bound": "^1.0.2", "has-tostringtag": "^1.0.2" } }, "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA=="], - "is-array-buffer": ["is-array-buffer@3.0.5", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "get-intrinsic": "^1.2.6" } }, "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A=="], "is-async-function": ["is-async-function@2.1.1", "", { "dependencies": { "async-function": "^1.0.0", "call-bound": "^1.0.3", "get-proto": "^1.0.1", "has-tostringtag": "^1.0.2", "safe-regex-test": "^1.1.0" } }, "sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ=="], @@ -702,6 +710,8 @@ "is-typed-array": ["is-typed-array@1.1.15", "", { "dependencies": { "which-typed-array": "^1.1.16" } }, "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ=="], + "is-unsafe": ["is-unsafe@2.0.2", "", {}, "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ=="], + "is-weakmap": ["is-weakmap@2.0.2", "", {}, "sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w=="], "is-weakref": ["is-weakref@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3" } }, "sha512-6i9mGWSlqzNMEqpCp93KwRS1uUOodk2OJ6b+sq7ZPDSy2WuI5NFIxp/254TytR8ftefexkWn5xNiHUNpPOfSew=="], @@ -712,7 +722,7 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + "js-yaml": ["js-yaml@4.3.2", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA=="], "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], @@ -732,7 +742,7 @@ "locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="], - "lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], + "lodash": ["lodash@4.18.0", "", {}, "sha512-l1mfj2atMqndAHI3ls7XqPxEjV2J9ZkcNyHpoZA3r2T1LLwDB69jgkMWh71YKwhBbK0G2f4WSn05ahmQXVxupA=="], "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], @@ -766,11 +776,11 @@ "mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], + "minimatch": ["minimatch@3.1.4", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-twmL+S8+7yIsE9wsqgzU3E8/LumN3M3QELrBZ20OdmQ9jB2JvW5oZtBEmft84k/Gs5CG9mqtWc6Y9vW+JEzGxw=="], "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], - "minio": ["minio@8.0.6", "", { "dependencies": { "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "fast-xml-parser": "^4.4.1", "ipaddr.js": "^2.0.1", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", "stream-json": "^1.8.0", "through2": "^4.0.2", "web-encoding": "^1.1.5", "xml2js": "^0.5.0 || ^0.6.2" } }, "sha512-sOeh2/b/XprRmEtYsnNRFtOqNRTPDvYtMWh+spWlfsuCV/+IdxNeKVUMKLqI7b5Dr07ZqCPuaRGU/rB9pZYVdQ=="], + "minio": ["minio@8.0.7", "", { "dependencies": { "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "fast-xml-parser": "^5.3.4", "ipaddr.js": "^2.0.1", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", "stream-json": "^1.8.0", "through2": "^4.0.2", "xml2js": "^0.5.0 || ^0.6.2" } }, "sha512-E737MgufW8CeQAsTAtnEMrxZ9scMSf29kkhZoXzDTKj/Jszzo2SfeZUH9wbDQH2Rsq6TCtl/yQL0+XdVKZansQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -822,17 +832,19 @@ "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], + "path-expression-matcher": ["path-expression-matcher@1.6.2", "", {}, "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ=="], + "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], "path-parse": ["path-parse@1.0.7", "", {}, "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="], - "path-to-regexp": ["path-to-regexp@0.1.12", "", {}, "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="], + "path-to-regexp": ["path-to-regexp@0.1.13", "", {}, "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA=="], "path-type": ["path-type@4.0.0", "", {}, "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw=="], - "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + "picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], "possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="], @@ -842,11 +854,11 @@ "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], - "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], + "proxy-from-env": ["proxy-from-env@2.1.0", "", {}, "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA=="], "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], - "qs": ["qs@6.14.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w=="], + "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], "query-string": ["query-string@7.1.3", "", { "dependencies": { "decode-uri-component": "^0.2.2", "filter-obj": "^1.1.0", "split-on-first": "^1.0.0", "strict-uri-encode": "^2.0.0" } }, "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg=="], @@ -880,7 +892,7 @@ "rimraf": ["rimraf@3.0.2", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "bin.js" } }, "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA=="], - "rollup": ["rollup@2.79.2", "", { "optionalDependencies": { "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ=="], + "rollup": ["rollup@2.80.0", "", { "optionalDependencies": { "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ=="], "rollup-plugin-copy": ["rollup-plugin-copy@3.5.0", "", { "dependencies": { "@types/fs-extra": "^8.0.1", "colorette": "^1.1.0", "fs-extra": "^8.1.0", "globby": "10.0.1", "is-plain-object": "^3.0.0" } }, "sha512-wI8D5dvYovRMx/YYKtUNt3Yxaw4ORC9xo6Gt9t22kveWz1enG9QrhVlagzwrxSC455xD1dHMKhIJkbsQ7d48BA=="], @@ -920,9 +932,9 @@ "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + "side-channel": ["side-channel@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4", "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ=="], - "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + "side-channel-list": ["side-channel-list@1.0.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4" } }, "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w=="], "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="], @@ -968,7 +980,7 @@ "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], - "strnum": ["strnum@1.1.2", "", {}, "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA=="], + "strnum": ["strnum@2.4.2", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw=="], "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], @@ -984,7 +996,7 @@ "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], - "tmp": ["tmp@0.2.5", "", {}, "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow=="], + "tmp": ["tmp@0.2.7", "", {}, "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], @@ -1028,20 +1040,16 @@ "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], - "util": ["util@0.12.5", "", { "dependencies": { "inherits": "^2.0.3", "is-arguments": "^1.0.4", "is-generator-function": "^1.0.7", "is-typed-array": "^1.1.3", "which-typed-array": "^1.1.2" } }, "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA=="], - "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], - "uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="], + "uuid": ["uuid@11.1.1", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ=="], "v8-compile-cache-lib": ["v8-compile-cache-lib@3.0.1", "", {}, "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="], "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], - "web-encoding": ["web-encoding@1.1.5", "", { "dependencies": { "util": "^0.12.3" }, "optionalDependencies": { "@zxing/text-encoding": "0.9.0" } }, "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "which-boxed-primitive": ["which-boxed-primitive@1.1.1", "", { "dependencies": { "is-bigint": "^1.1.0", "is-boolean-object": "^1.2.1", "is-number-object": "^1.1.1", "is-string": "^1.1.1", "is-symbol": "^1.1.1" } }, "sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA=="], @@ -1060,6 +1068,8 @@ "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="], + "xml2js": ["xml2js@0.6.2", "", { "dependencies": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" } }, "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA=="], "xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="], @@ -1088,11 +1098,11 @@ "@eslint/eslintrc/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], - "@types/minimatch/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + "@types/minimatch/minimatch": ["minimatch@9.0.7", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg=="], "@types/serve-static/@types/send": ["@types/send@0.17.6", "", { "dependencies": { "@types/mime": "^1", "@types/node": "*" } }, "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og=="], - "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.7", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg=="], "@typescript-eslint/typescript-estree/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], @@ -1122,13 +1132,17 @@ "finalhandler/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], + "form-data/hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], + "globby/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], + "internal-slot/side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + "ioredis-mock/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], "is-bun-module/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], - "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "micromatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "proxy-addr/ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -1146,9 +1160,9 @@ "@aws-sdk/middleware-sdk-s3/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.9", "", { "dependencies": { "@smithy/core": "^3.29.8", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-g5rnEii/mkT0mjVJmlsaOfyNBtHNTecD9Lo4NP8D5HzMUEnZNpz7/FbvBCjNcV4vteHFAxOGiLUYNxPkDZZAPw=="], - "@types/minimatch/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], + "@types/minimatch/minimatch/brace-expansion": ["brace-expansion@5.0.12", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ=="], - "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], + "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@5.0.12", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ=="], "body-parser/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], @@ -1158,12 +1172,18 @@ "finalhandler/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + "internal-slot/side-channel/side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + "rollup-plugin-sourcemaps/@rollup/pluginutils/@types/estree": ["@types/estree@0.0.39", "", {}, "sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw=="], "rollup-plugin-sourcemaps/@rollup/pluginutils/estree-walker": ["estree-walker@1.0.1", "", {}, "sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg=="], - "rollup-plugin-sourcemaps/@rollup/pluginutils/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "rollup-plugin-sourcemaps/@rollup/pluginutils/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "send/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + + "@types/minimatch/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + + "@typescript-eslint/typescript-estree/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], } } diff --git a/service/package.json b/service/package.json index c5a49a82..a3679c8f 100644 --- a/service/package.json +++ b/service/package.json @@ -33,14 +33,14 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "axios": "^1.8.4", + "axios": "^1.20.0", "bullmq": "^5.13.1", "busboy": "^1.6.0", "dotenv": "^17.4.2", - "express": "^4.19.2", + "express": "^4.22.3", "express-rate-limit": "^7.4.1", "ioredis": "^5.4.1", - "minio": "^8.0.5", + "minio": "^8.0.7", "nanoid": "^5.1.16", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", @@ -74,6 +74,8 @@ "typescript": "^5.5.4" }, "overrides": { - "decode-uri-component": "0.5.0" + "decode-uri-component": "0.5.0", + "uuid": "11.1.1", + "qs": "6.16.0" } } diff --git a/service/src/egress-gateway-client.test.ts b/service/src/egress-gateway-client.test.ts index c3d7ab9c..ef9e3fb7 100644 --- a/service/src/egress-gateway-client.test.ts +++ b/service/src/egress-gateway-client.test.ts @@ -21,6 +21,7 @@ type AxiosPostCall = { headers?: Record; signal?: AbortSignal; timeout?: number; + maxRedirects?: number; }; }; @@ -109,6 +110,7 @@ describe('egress gateway client', () => { allowedToolNames: ['query_clickhouse'], }, { signal: controller.signal }); + expect(calls.map(call => call.config.maxRedirects)).toEqual([0, 0, 0]); expect(calls.map(call => call.config.timeout)).toEqual([12_345, 12_345, 12_345]); expect(calls.map(call => call.config.signal)).toEqual([ controller.signal, @@ -147,6 +149,7 @@ describe('egress gateway client', () => { await revokeGatewayEgressGrant({ grantId: 'grant_123', reason: 'completed' }); await revokeGatewayEgressGrant({ grantId: 'grant_123', reason: 'failed', timeoutMs: 321 }); + expect(calls.map(call => call.config.maxRedirects)).toEqual([0, 0]); expect(calls.map(call => call.config.timeout)).toEqual([987, 321]); }); diff --git a/service/src/egress-gateway-client.ts b/service/src/egress-gateway-client.ts index ebcef4ad..67b7dca6 100644 --- a/service/src/egress-gateway-client.ts +++ b/service/src/egress-gateway-client.ts @@ -48,6 +48,7 @@ export async function createGatewayEgressGrant(args: { gatewayUrl('/internal/egress-grants'), body, { + maxRedirects: 0, headers: injectTraceHeaders(gatewayHeaders({ 'Content-Type': 'application/json' }, isSynthetic)), signal, timeout: env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS, @@ -73,6 +74,7 @@ export async function restoreGatewaySandboxResult { }); describe('egress gateway routes', () => { + test('rejects file-server redirects before forwarding internal credentials', async () => { + let redirected = 0; + let contacted = 0; + const target = Bun.serve({ + port: 0, + fetch() { + redirected++; + return new Response('unexpected'); + }, + }); + const upstream = Bun.serve({ + port: 0, + fetch(req) { + contacted++; + expect(req.headers.get(INTERNAL_SERVICE_TOKEN_HEADER)).toBe(INTERNAL_TOKEN); + return new Response(null, { + status: 307, + headers: { Location: `http://127.0.0.1:${target.port}/stolen` }, + }); + }, + }); + try { + globalThis.fetch = originalFetch; + env.EGRESS_GATEWAY_FILE_SERVER_URL = `http://127.0.0.1:${upstream.port}`; + const response = await gatewayFetch( + `/sessions/${sessionHandle({ dir: 'read', sessionId: 'sess_input' })}/objects/${objectHandle({})}`, + { headers: grantHeader() }, + ); + expect(response.status).toBe(500); + expect(contacted).toBe(1); + expect(redirected).toBe(0); + } finally { + upstream.stop(true); + target.stop(true); + } + }); + test('protects internal grant create, restore, and revoke routes', async () => { const createBody = JSON.stringify({ payload: payload(), claims: executionClaims() }); const unauthorized = await gatewayFetch('/internal/egress-grants', { diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 873db173..5a36ebc5 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -544,7 +544,7 @@ app.get('/sessions/:sessionHandle/objects', async (req, res) => { `/sessions/${encodeURIComponent(sessionId)}/objects`, '?detail=normalized', ), - { headers: injectTraceHeaders(internalServiceHeaders({ Accept: 'application/json' })) }, + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders({ Accept: 'application/json' })) }, ); if (!upstream.ok) { return pipeFetchResponse(upstream, res); @@ -638,7 +638,7 @@ app.post('/input-manifest', express.json({ limit: '4mb' }), async (req, res) => controller.signal.throwIfAborted(); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(objectId)}/metadata`), - { headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); if (!upstream.ok) { await upstream.body?.cancel(); // Recheck failures individually through the existing retry/classification path. @@ -668,7 +668,7 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle/metadata', async (req, r await recordEgressRead(grant); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}/metadata`), - { headers: injectTraceHeaders(internalServiceHeaders()) }); + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders()) }); if (!upstream.ok) return pipeFetchResponse(upstream, res); return res.json(inputMetadata(await upstream.json(), grant, sessionId, object.id)); } catch (error) { @@ -694,7 +694,7 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}`, ), - { headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, ); const headerOverrides = isOpaqueObjectContentDisposition( upstream.headers.get('content-disposition'), @@ -750,6 +750,7 @@ app.put('/sessions/:sessionHandle/objects/:fileId', async (req, res) => { ), { method: 'PUT', + redirect: 'error', headers, body: req as unknown as BodyInit, duplex: 'half', @@ -832,6 +833,7 @@ app.post('/tool-call', async (req, res) => { await recordEgressToolCall(callback.grant_id, executionId); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL, '/tool-call'), { method: 'POST', + redirect: 'error', headers: { ...injectTraceHeaders(), 'Content-Type': req.header('content-type') ?? 'application/json', @@ -847,6 +849,7 @@ app.post('/tool-call', async (req, res) => { await recordEgressToolCall(callback.grant_id, executionId); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL, '/tool-call'), { method: 'POST', + redirect: 'error', headers: { ...injectTraceHeaders(), 'Content-Type': req.header('content-type') ?? 'application/json', diff --git a/service/src/hosted-app/microvm-runtime.ts b/service/src/hosted-app/microvm-runtime.ts index 8b594f37..f0cb69e9 100644 --- a/service/src/hosted-app/microvm-runtime.ts +++ b/service/src/hosted-app/microvm-runtime.ts @@ -377,6 +377,7 @@ export class HostedAppMicrovmRuntime { token = await this.mintToken(vm.microvmId, this.config.controlPort, callerSignal); } const response = await this.deps.fetch(`${endpoint}/api/v2/health`, { + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), @@ -414,6 +415,7 @@ export class HostedAppMicrovmRuntime { `${normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? '')}/api/v2/hosted-app/start`, { method: 'POST', + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), @@ -454,6 +456,7 @@ export class HostedAppMicrovmRuntime { const response = await this.deps.fetch( `${normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? '')}/api/v2/hosted-app/status`, { + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), diff --git a/service/src/internal-redirect-policy.test.ts b/service/src/internal-redirect-policy.test.ts new file mode 100644 index 00000000..577b468c --- /dev/null +++ b/service/src/internal-redirect-policy.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as ts from 'typescript'; + +const transports: Record = { + 'egress-gateway-client.ts': 4, + 'egress-gateway.ts': 7, + 'hosted-app/microvm-runtime.ts': 3, + 'runtime-session/checkpoint.ts': 4, + 'runtime-session/files.ts': 1, + 'sandbox-backend/http.ts': 1, + 'sandbox-backend/lambda-microvm.ts': 2, + 'service/programmatic-router.ts': 3, + 'service/replay-state.ts': 2, + 'service/router.ts': 5, +}; + +// Every request to these internal endpoints must stay on its original origin. +describe('internal transport redirect policy', () => { + for (const [file, count] of Object.entries(transports)) { + test(`${file} rejects redirects on every internal request`, () => { + const source = ts.createSourceFile(file, + fs.readFileSync(path.join(__dirname, file), 'utf8'), ts.ScriptTarget.Latest, true); + let requests = 0; + function visit(node: ts.Node): void { + if (ts.isCallExpression(node)) { + const name = node.expression.getText(source); + const fetchCall = name === 'fetch' || name === 'this.deps.fetch'; + const axiosCall = name === 'axios' || /^axios\.(get|post|delete)$/.test(name); + if (fetchCall || axiosCall) { + requests++; + let index = 1; + if (name === 'axios') index = 0; + if (name === 'axios.post') index = 2; + let options = node.arguments[index]; + while (options && (ts.isAsExpression(options) || ts.isTypeAssertionExpression(options))) { + options = options.expression; + } + expect(options && ts.isObjectLiteralExpression(options)).toBe(true); + if (!options || !ts.isObjectLiteralExpression(options)) return; + const property = options.properties.find(prop => ts.isPropertyAssignment(prop) + && prop.name.getText(source) === (fetchCall ? 'redirect' : 'maxRedirects')); + expect(property && ts.isPropertyAssignment(property)).toBe(true); + if (property && ts.isPropertyAssignment(property)) { + expect(property.initializer.getText(source)).toBe(fetchCall ? '\'error\'' : '0'); + } + } + } + ts.forEachChild(node, visit); + } + visit(source); + expect(requests).toBe(count); + }); + } +}); diff --git a/service/src/minio-compat.test.ts b/service/src/minio-compat.test.ts new file mode 100644 index 00000000..255c6e00 --- /dev/null +++ b/service/src/minio-compat.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from 'bun:test'; +import { Client } from 'minio'; + +test('patched MinIO preserves S3 object-list XML decoding', async () => { + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + expect(url.pathname).toBe('/testbucket'); + expect(url.searchParams.get('list-type')).toBe('2'); + return new Response(` + + testbucketinputs/1 + 1000false + inputs/a&b.csv2026-01-01T00:00:00.000Z + "test-etag"12STANDARD + +`, { headers: { 'Content-Type': 'application/xml' } }); + }, + }); + try { + const client = new Client({ + endPoint: '127.0.0.1', + port: server.port, + useSSL: false, + region: 'us-east-1', + accessKey: 'test-access-key', + secretKey: 'test-secret-key', + }); + const objects = []; + for await (const object of client.listObjectsV2('testbucket', 'inputs/', true)) { + objects.push(object); + } + expect(objects).toHaveLength(1); + expect(objects[0]).toMatchObject({ name: 'inputs/a&b.csv', size: 12, etag: 'test-etag' }); + } finally { + server.stop(true); + } +}); diff --git a/service/src/runtime-session/checkpoint.ts b/service/src/runtime-session/checkpoint.ts index 3562cd17..e49d9290 100644 --- a/service/src/runtime-session/checkpoint.ts +++ b/service/src/runtime-session/checkpoint.ts @@ -94,6 +94,7 @@ export async function pullCheckpoint( ? AbortSignal.any([args.signal, transferDeadlineSignal]) : transferDeadlineSignal; const response = await axios.get(`${args.endpointBase}/api/v2/session/checkpoint`, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -125,6 +126,7 @@ export async function pushRestore( ): Promise { const token = await args.mintToken(); await axios.post(`${args.endpointBase}/api/v2/session/restore`, fs.createReadStream(data.path), { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -153,6 +155,7 @@ export async function probeInputs( `${args.endpointBase}/api/v2/session/inputs/probe`, { refs }, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -197,6 +200,7 @@ export async function pushInputs( * unlink the archive first and the stream then emits an unhandled ENOENT. */ const requestData = typeof data === 'function' ? data() : data; await axios.post(`${args.endpointBase}/api/v2/session/inputs`, requestData, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), diff --git a/service/src/runtime-session/files.ts b/service/src/runtime-session/files.ts index f17998dd..dd53508d 100644 --- a/service/src/runtime-session/files.ts +++ b/service/src/runtime-session/files.ts @@ -202,6 +202,7 @@ async function fetchFileObjectToPath( const url = `${baseUrl}/sessions/${encodeURIComponent(ref.storage_session_id)}/objects/${encodeURIComponent(ref.id)}`; try { const response = await axios.get(url, { + maxRedirects: 0, headers: internalServiceHeaders(), responseType: 'stream', timeout: opts.timeoutMs, diff --git a/service/src/sandbox-backend/http.test.ts b/service/src/sandbox-backend/http.test.ts index d3370646..403772be 100644 --- a/service/src/sandbox-backend/http.test.ts +++ b/service/src/sandbox-backend/http.test.ts @@ -14,7 +14,7 @@ type CapturedRequest = { let server: ReturnType; let captured: CapturedRequest[] = []; -let nextResponse: { status: number; body: unknown; delayMs?: number } = { status: 200, body: {} }; +let nextResponse: { status: number; body: unknown; delayMs?: number; location?: string } = { status: 200, body: {} }; const savedEndpoint = env.SANDBOX_ENDPOINT; @@ -33,7 +33,7 @@ beforeAll(() => { } return new Response(JSON.stringify(nextResponse.body), { status: nextResponse.status, - headers: { 'Content-Type': 'application/json' }, + headers: { 'Content-Type': 'application/json', ...(nextResponse.location ? { Location: nextResponse.location } : {}) }, }); }, }); @@ -104,6 +104,26 @@ describe('HttpSandboxBackend', () => { expect(result).toEqual(responseBody); }); + test('rejects redirects without sending the signed execution to another endpoint', async () => { + let redirected = 0; + const target = Bun.serve({ + port: 0, + fetch() { + redirected++; + return Response.json({ session_id: 'unexpected' }); + }, + }); + try { + for (const status of [302, 307, 308]) { + nextResponse = { status, body: {}, location: `http://127.0.0.1:${target.port}/stolen` }; + await expect(new HttpSandboxBackend().execute(request(), context())).rejects.toThrow(); + } + expect(redirected).toBe(0); + } finally { + target.stop(true); + } + }); + test('does not mutate the signed request body', async () => { const req = request(); const before = JSON.stringify(req.body); diff --git a/service/src/sandbox-backend/http.ts b/service/src/sandbox-backend/http.ts index ce5f5c0a..1ebdfcbe 100644 --- a/service/src/sandbox-backend/http.ts +++ b/service/src/sandbox-backend/http.ts @@ -20,6 +20,7 @@ export class HttpSandboxBackend implements SandboxBackend { `${env.SANDBOX_ENDPOINT}/${Jobs.execute}`, req.body, { + maxRedirects: 0, headers: injectTraceHeaders(req.headers), signal: ctx.signal, } diff --git a/service/src/sandbox-backend/lambda-microvm.ts b/service/src/sandbox-backend/lambda-microvm.ts index d947c76e..e4ccf9a5 100644 --- a/service/src/sandbox-backend/lambda-microvm.ts +++ b/service/src/sandbox-backend/lambda-microvm.ts @@ -1437,6 +1437,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { `${base}/api/v2/${Jobs.execute}`, req.body, { + maxRedirects: 0, headers: { ...injectTraceHeaders(req.headers), [token.headerName]: token.token, @@ -1644,6 +1645,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { private async assertHealthy(base: string, token: string, ctx: SandboxExecuteContext): Promise { try { const response = await axios.get(`${base}/api/v2/health`, { + maxRedirects: 0, headers: { 'X-aws-proxy-auth': token, ...microvmPortHeaders(this.config.port) }, timeout: this.config.healthTimeoutMs, signal: ctx.signal, diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 1e2a367c..86b1cdbb 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -319,7 +319,7 @@ function waitForExecutionState( () => axios.get( `${env.TOOL_CALL_SERVER_URL}/sessions/${id}/pending`, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Get pending tool calls', ); @@ -1598,7 +1598,7 @@ async function handleBlocking( error_message: r.error_message, })), }, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Submit tool results', ); @@ -1783,7 +1783,7 @@ async function handleBlocking( timeout, tools, }, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Create Tool Call Server session', ); diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 003ce6b5..9f33586c 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -800,6 +800,7 @@ export async function cleanupStaleExecutions(): Promise { if (state.mode === 'blocking') { await axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${state.execution_id}`, { + maxRedirects: 0, headers: internalServiceHeaders(), }) .catch(() => {}); @@ -835,6 +836,7 @@ export async function cleanupExecution(execution_id: string, mode: 'blocking' | if (mode === 'blocking') { ops.push( axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}`, { + maxRedirects: 0, headers: internalServiceHeaders(), }).catch(() => {}), ); diff --git a/service/src/service/router.ts b/service/src/service/router.ts index 43d0ff4b..a977182e 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -71,6 +71,7 @@ async function putFileToFileServer( ): Promise { const response = await fetch(url, { method: 'PUT', + redirect: 'error', headers, body: Readable.toWeb(file) as unknown as ReadableStream, signal, @@ -416,6 +417,7 @@ router.get('/download/:session_id/:fileId', downloadLimiter, sessionAuth, async const response = await axios({ method: 'get', url: `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, + maxRedirects: 0, headers: internalServiceHeaders(), responseType: 'stream' }); @@ -909,6 +911,7 @@ router.get('/files/:session_id', fetchLimiter, sessionAuth, async (req: t.Authen try { const response = await axios.get(`${env.FILE_SERVER_URL}/sessions/${session_id}/objects`, { params: { detail }, + maxRedirects: 0, headers: internalServiceHeaders({ 'Accept': 'application/json' }) }); @@ -941,7 +944,7 @@ router.get('/sessions/:session_id/objects/:fileId', fetchLimiter, sessionAuth, a try { const response = await axios.get( `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}/metadata`, - { headers: internalServiceHeaders({ Accept: 'application/json' }) }, + { maxRedirects: 0, headers: internalServiceHeaders({ Accept: 'application/json' }) }, ); return res.status(200).json(response.data); @@ -972,7 +975,7 @@ const deleteSessionObject = async (req: t.AuthenticatedRequest, res: Response) = try { const response = await axios.delete( `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, - { headers: internalServiceHeaders() } + { maxRedirects: 0, headers: internalServiceHeaders() } ); await connection.del(`upload:${req.sessionKey}${session_id}${fileId}`); diff --git a/test-sandbox.sh b/test-sandbox.sh index 27c8a877..cf6db2cd 100755 --- a/test-sandbox.sh +++ b/test-sandbox.sh @@ -286,15 +286,28 @@ expect_socket_blocked("AF_RXRPC", getattr(socket, "AF_RXRPC", 33), socket.SOCK_D expect_socket_blocked("AF_ALG", getattr(socket, "AF_ALG", 38), socket.SOCK_SEQPACKET, 0) syscalls = { - "x86_64": {"clone": 56, "clone3": 435, "vmsplice": 278}, - "amd64": {"clone": 56, "clone3": 435, "vmsplice": 278}, - "aarch64": {"clone": 220, "clone3": 435, "vmsplice": 75}, - "arm64": {"clone": 220, "clone3": 435, "vmsplice": 75}, + "x86_64": {"clone": 56, "clone3": 435, "vmsplice": 278, "futex": 202}, + "amd64": {"clone": 56, "clone3": 435, "vmsplice": 278, "futex": 202}, + "aarch64": {"clone": 220, "clone3": 435, "vmsplice": 75, "futex": 98}, + "arm64": {"clone": 220, "clone3": 435, "vmsplice": 75, "futex": 98}, } arch = platform.machine().lower() if arch not in syscalls: raise SystemExit(f"unsupported arch for syscall smoke test: {arch}") +# PI operations must be denied before the kernel sees even invalid addresses. +for command in (6, 7, 8, 11, 12, 13): + for flags in (0, 128, 256, 384): + ctypes.set_errno(0) + rc = libc.syscall(syscalls[arch]["futex"], 0, command | flags, 0, 0, 0, 0) + if rc != -1 or ctypes.get_errno() != errno.EPERM: + raise SystemExit(f"PI futex operation was not denied: {command | flags}") +word = ctypes.c_int(0) +rc = libc.syscall(syscalls[arch]["futex"], ctypes.byref(word), 1 | 128, 1, 0, 0, 0) +if rc != 0: + raise SystemExit("ordinary FUTEX_WAKE_PRIVATE was blocked") +print("futex_pi=blocked:ordinary_wake=ok") + CLONE_NEWUSER = 0x10000000 CLONE_NEWNET = 0x40000000 @@ -359,7 +372,8 @@ PY && [[ "$stdout" == *"AF_ALG=errno:1"* ]] \ && [[ "$stdout" == *"clone_namespace=rc:-1:errno:1"* ]] \ && [[ "$stdout" == *"clone3=rc:-1:errno:38"* ]] \ - && [[ "$stdout" == *"vmsplice=rc:-1:errno:1"* ]]; then + && [[ "$stdout" == *"vmsplice=rc:-1:errno:1"* ]] \ + && [[ "$stdout" == *"futex_pi=blocked:ordinary_wake=ok"* ]]; then log_success "Kernel attack-surface syscalls blocked" return 0 else From e68fae8f73ef98bdc507a1882c52f7a1345ebe10 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 4 Oct 2026 21:23:44 -0400 Subject: [PATCH 68/75] =?UTF-8?q?=F0=9F=90=B3=20ci:=20Publish=20Code=20API?= =?UTF-8?q?=20Images=20to=20GHCR=20(#307)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐳 ci: Publish Code API Images to GHCR Build one linux/amd64 image per Compose build on pushes to main that touch an image input, tag it sha-, and move the main tags together once every image has built. Pull requests build without pushing. Each image keeps a registry build cache so the KVM sandbox runner does not recompile its runtime packages on every push. README documents the tags and a Compose override for pulling instead of building. * 🏷️ fix: Build Every Main Commit and Promote the Newest Complete One Give each main commit its own concurrency group so a queued run can no longer be replaced and lose its sha tags. The promote job now serializes, resolves the newest main commit that has every image, retries each retag, and finishes a partial move on the next run. Report each pushed digest in the job summary and document per-image digest pinning. Drop the disk cleanup step: a cold KVM runner build peaked at about 17 GB of builder state with over 95 GB free. * 🔒 fix: Write Commit Tags Once and Never Move main Backwards Skip building any image whose sha tag for the commit already exists, so a re-run that fails part-way cannot mix two builds under one commit. Promotion retries tag inspections and refuses to move an image's main to an older commit than it names, so a transient registry error can delay a promotion but not reverse one. Note that the macOS override keeps building its own sandbox target. * 🧯 fix: Fail Closed When the Registry Cannot Answer Read image revisions through one script that tells a missing tag (or one this token may not read, and so may not write) apart from a registry that does not answer. The write-once check and the promotion now fail on the latter instead of treating it as a missing tag, so neither can rebuild over an existing tag or move main backwards. Restore the disk cleanup on the KVM runner leg, since the hosted runner only guarantees 14 GB free. --- .github/scripts/image-revision.sh | 35 +++ .github/workflows/images.yml | 350 ++++++++++++++++++++++++++++++ README.md | 84 +++++++ 3 files changed, 469 insertions(+) create mode 100755 .github/scripts/image-revision.sh create mode 100644 .github/workflows/images.yml diff --git a/.github/scripts/image-revision.sh b/.github/scripts/image-revision.sh new file mode 100755 index 00000000..ae4c88b4 --- /dev/null +++ b/.github/scripts/image-revision.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Prints the org.opencontainers.image.revision label of an image reference. +# Used by .github/workflows/images.yml, which must tell a missing tag apart +# from a registry that cannot answer. +# +# Exit 0 the revision was printed. +# Exit 3 the tag does not exist, or this token may not read it (a token that +# may not read a repository may not write to it either). +# Exit 1 the registry did not answer after retries, or the image has no +# revision label. +set -uo pipefail + +ref=${1:?usage: image-revision.sh REF} +attempts=${IMAGE_REVISION_ATTEMPTS:-3} +err=$(mktemp) +trap 'rm -f "$err"' EXIT + +for attempt in $(seq 1 "$attempts"); do + if json=$(docker buildx imagetools inspect "$ref" --format '{{ json .Image }}' 2>"$err"); then + if jq -er 'if has("config") then . else .["linux/amd64"] end + | .config.Labels["org.opencontainers.image.revision"]' <<<"$json"; then + exit 0 + fi + echo "$ref has no revision label" >&2 + exit 1 + fi + if grep -qiE ': not found$|denied|\b40[134]\b' "$err"; then + exit 3 + fi + cat "$err" >&2 + if [ "$attempt" -lt "$attempts" ]; then + sleep $(( attempt * 5 )) + fi +done +exit 1 diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml new file mode 100644 index 00000000..7c626767 --- /dev/null +++ b/.github/workflows/images.yml @@ -0,0 +1,350 @@ +# Builds the Code API container images and publishes them to GHCR, so a host +# can pull a tested image instead of compiling one in place. Like ci.yml this +# file is inert inside the monorepo — GitHub only runs workflows from the repo +# root — and becomes a root workflow in the published repo. +# +# Entry points: +# +# * push to main — build every image, push `sha-` tags, and refresh +# each image's registry build cache. The promote job then points the +# `main` tags at the newest main commit that has all images. +# * pull_request — build every image without pushing, reading the cache that +# main wrote, so Dockerfile breakage surfaces before merge. +# * workflow_dispatch — on main, the same as a push; on any other branch, a +# build-only run. +# +# The image list lives in the plan job and feeds both the build matrix and the +# promotion, so the two cannot drift apart. README.md ("Prebuilt images") +# documents the tags and how a Compose host switches to them. +name: Images + +on: + push: + branches: [main] + # Everything the Dockerfiles below COPY from the build context, plus this + # workflow. Every run builds every image, so a commit that has tags has + # them for all images; unchanged images are cache hits. + paths: &image-inputs + - .dockerignore + - .github/scripts/image-revision.sh + - .github/workflows/images.yml + - api/** + - docker/** + - javascript-packages.txt + - launcher/** + - packages/code/src/** + - service/** + - shared/** + pull_request: + paths: *image-inputs + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Each main commit gets its own group and is never cancelled, so no commit's + # build can be dropped by a later queued run (GitHub keeps only one pending + # run per group). Other refs share one group per ref and cancel superseded + # runs. + group: images-${{ github.ref == 'refs/heads/main' && github.sha || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + NAMESPACE: ghcr.io/librechat-ai + +jobs: + plan: + name: Plan + runs-on: ubuntu-24.04 + outputs: + images: ${{ steps.images.outputs.images }} + publish: ${{ github.repository == 'LibreChat-AI/code-interpreter' && github.ref == 'refs/heads/main' }} + steps: + - name: List images + id: images + # One image per distinct Compose build in docker-compose.yaml. The + # sandbox runner's two Compose targets (KVM_ENABLED=true/false) are + # published as two images. The Lambda MicroVM targets are not built + # here; they are not used by Compose hosts. + run: | + images=$(jq -c . <<'JSON' + [ + {"image": "code-interpreter-api", "dockerfile": "service/Dockerfile", "target": "api", + "description": "Code API HTTP server (Compose service api)", "timeout": 30}, + {"image": "code-interpreter-worker", "dockerfile": "service/Dockerfile", "target": "worker", + "description": "Code API job worker (Compose service service-worker)", "timeout": 30}, + {"image": "code-interpreter-file-server", "dockerfile": "service/Dockerfile", "target": "production", + "description": "Code API file server (Compose service file_server)", "timeout": 30}, + {"image": "code-interpreter-egress-gateway", "dockerfile": "service/Dockerfile.egress-gateway", "target": "production", + "description": "Sandbox egress gateway (Compose service egress_gateway)", "timeout": 30}, + {"image": "code-interpreter-tool-call-server", "dockerfile": "service/Dockerfile.tool-call-server", "target": "production", + "description": "Tool call server (Compose service tool_call_server)", "timeout": 30}, + {"image": "code-interpreter-sandbox-runner", "dockerfile": "api/Dockerfile", "target": "sandbox-runner-true", + "description": "libkrun microVM sandbox runner with the guest rootfs and runtime packages baked in (Compose service sandbox-runner, KVM_ENABLED=true)", + "timeout": 60, "free_disk": true}, + {"image": "code-interpreter-sandbox-runner-direct", "dockerfile": "api/Dockerfile", "target": "sandbox-runner-false", + "description": "Directory-root sandbox runner; runtime packages are mounted from the host (Compose service sandbox-runner, KVM_ENABLED=false)", + "timeout": 30} + ] + JSON + ) + echo "images=$images" >> "$GITHUB_OUTPUT" + + build: + name: ${{ matrix.image }} + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: ${{ matrix.timeout }} + permissions: + contents: read + # Pushes happen only when needs.plan.outputs.publish is true (main). + # Pull request runs read the build cache and never push; GitHub caps a + # fork's token at read access regardless. + packages: write + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.images) }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Free runner disk + # A cold KVM runner build peaks at roughly 17 GB of builder state. The + # hosted runner had about 87 GB free when measured, but GitHub only + # guarantees 14 GB, so this leg removes unused preinstalled SDKs + # (about 28 GB, 1.5 minutes) rather than depend on the surplus. + if: matrix.free_disk + run: | + sudo rm -rf \ + /opt/ghc \ + /opt/hostedtoolcache/CodeQL \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift + docker image prune --all --force >/dev/null + df -h / + + - name: Sample free disk + run: | + nohup bash -c 'while :; do df -BM --output=avail / | tail -n1 | tr -dc 0-9; echo; sleep 10; done' \ + >"$RUNNER_TEMP/disk-avail-mib.log" 2>/dev/null & + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Log in to GHCR + # Pull request runs log in only to read the cache, and only from this + # repository; fork runs read public packages anonymously. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Check for an existing tag + # `sha-` tags are written once. A re-run (or a dispatch on the same + # commit) builds only the images whose tag is missing, so a failed + # re-run cannot leave the commit's set mixed across two builds. + id: existing + if: needs.plan.outputs.publish == 'true' + env: + REF: ${{ env.NAMESPACE }}/${{ matrix.image }}:sha-${{ github.sha }} + # A registry that cannot answer fails the job rather than risk + # rebuilding over a tag that may exist. + run: | + status=0 + revision=$(.github/scripts/image-revision.sh "$REF") || status=$? + case $status in + 0) + if [ "$revision" != "$GITHUB_SHA" ]; then + echo "::error::$REF records revision $revision" + exit 1 + fi + echo "$REF already exists; not rebuilding it." + echo "exists=true" >> "$GITHUB_OUTPUT" + ;; + 3) + echo "exists=false" >> "$GITHUB_OUTPUT" + ;; + *) + echo "::error::Could not read $REF from the registry." + exit 1 + ;; + esac + + - name: Image metadata + id: meta + if: steps.existing.outputs.exists != 'true' + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + with: + images: ${{ env.NAMESPACE }}/${{ matrix.image }} + tags: type=sha,format=long + labels: | + org.opencontainers.image.title=${{ matrix.image }} + org.opencontainers.image.description=${{ matrix.description }} + annotations: | + org.opencontainers.image.title=${{ matrix.image }} + org.opencontainers.image.description=${{ matrix.description }} + + - name: Build${{ needs.plan.outputs.publish == 'true' && ' and push' || '' }} + id: build + if: steps.existing.outputs.exists != 'true' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: ${{ matrix.dockerfile }} + target: ${{ matrix.target }} + platforms: linux/amd64 + push: ${{ needs.plan.outputs.publish == 'true' }} + # Without a push, keep the result in the builder only: nothing is + # loaded into the runner's Docker engine, which would need a second + # copy of the multi-gigabyte sandbox image on disk. + outputs: ${{ needs.plan.outputs.publish != 'true' && 'type=cacheonly' || '' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + # A registry cache next to each image, written only from main. + # mode=max keeps the intermediate stages (the KVM runner's package + # stage above all), which the GitHub Actions cache could not hold: + # its 10 GB repository limit would evict them between pushes. + cache-from: type=registry,ref=${{ env.NAMESPACE }}/${{ matrix.image }}:buildcache + cache-to: ${{ needs.plan.outputs.publish == 'true' && format('type=registry,ref={0}/{1}:buildcache,mode=max,ignore-error=true', env.NAMESPACE, matrix.image) || '' }} + + - name: Summarize + if: always() + env: + IMAGE_NAME: ${{ matrix.image }} + PUSHED: ${{ needs.plan.outputs.publish == 'true' && steps.build.outcome == 'success' }} + EXISTED: ${{ steps.existing.outputs.exists == 'true' }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + lowest=$(sort -n "$RUNNER_TEMP/disk-avail-mib.log" 2>/dev/null | head -n1) + { + echo "### $IMAGE_NAME" + if [ "$EXISTED" = true ]; then + echo "\`$NAMESPACE/$IMAGE_NAME:sha-$GITHUB_SHA\` was already published; left unchanged." + echo + fi + if [ "$PUSHED" = true ]; then + echo "Pushed \`$NAMESPACE/$IMAGE_NAME:sha-$GITHUB_SHA\`, pinnable as \`$NAMESPACE/$IMAGE_NAME@$DIGEST\`." + echo + fi + echo "Lowest free space on / while building: ${lowest:-unknown} MiB" + echo '```' + df -h / + docker buildx du 2>/dev/null | tail -n1 || true + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + promote: + name: Move main tags + needs: [plan, build] + if: needs.plan.outputs.publish == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + # Promotions run one at a time. Each resolves its target when it runs + # instead of trusting its own commit, so a stale re-run, or a queued + # promotion that GitHub replaces, cannot leave `main` behind or move it + # backwards. + concurrency: + group: images-promote + cancel-in-progress: false + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + sparse-checkout: .github/scripts + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Point main at the newest complete commit + # Target: the newest of the last 100 commits on main for which every + # image has a `sha-` tag. No image's `main` moves to an older commit + # than the one it names, and a registry that cannot answer fails the + # promotion instead of being read as a missing tag, so an error can + # delay a promotion but never reverse one. A registry cannot move + # several repositories' tags in one transaction, so a failure part-way + # leaves some `main` tags behind; the next promotion, or a re-run of + # this job, finishes the move. Deployments should pin `sha-` tags + # rather than `main`. + env: + GH_TOKEN: ${{ github.token }} + IMAGES: ${{ needs.plan.outputs.images }} + run: | + set -euo pipefail + + # Prints the revision a tag records, or nothing if the tag does not + # exist; exits the step if the registry cannot answer. + revision_of() { + local revision status=0 + revision=$(.github/scripts/image-revision.sh "$1") || status=$? + case $status in + 0) echo "$revision" ;; + 3) ;; + *) echo "::error::Could not read $1 from the registry." >&2; exit 1 ;; + esac + } + + complete() { + local image revision + for image in "${images[@]}"; do + revision=$(revision_of "$NAMESPACE/$image:sha-$1") || exit 1 + [ "$revision" = "$1" ] || return 1 + done + } + + retag() { + local attempt + for attempt in 1 2 3; do + docker buildx imagetools create --tag "$1:main" "$1:sha-$2" && return 0 + sleep $(( attempt * 10 )) + done + return 1 + } + + mapfile -t images < <(jq -r '.[].image' <<<"$IMAGES") + mapfile -t commits < <(gh api "repos/$GITHUB_REPOSITORY/commits?sha=main&per_page=100" --jq '.[].sha') + declare -A position + for i in "${!commits[@]}"; do position[${commits[$i]}]=$i; done + + target='' + for sha in "${commits[@]}"; do + if complete "$sha"; then + target=$sha + break + fi + done + if [ -z "$target" ]; then + echo "::error::None of the last ${#commits[@]} commits on main has all images." + exit 1 + fi + + for image in "${images[@]}"; do + ref="$NAMESPACE/$image" + current=$(revision_of "$ref:main") || exit 1 + if [ "$current" = "$target" ]; then + echo "$ref:main already at sha-$target" | tee -a "$GITHUB_STEP_SUMMARY" + continue + fi + if [ -n "$current" ] && [ -n "${position[$current]:-}" ] && + [ "${position[$current]}" -lt "${position[$target]}" ]; then + echo "::warning::$ref:main names $current, newer than $target; leaving it." + continue + fi + retag "$ref" "$target" + echo "$ref:main -> sha-$target" | tee -a "$GITHUB_STEP_SUMMARY" + done diff --git a/README.md b/README.md index b746e3c0..db4038c4 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,90 @@ Versions are `vMAJOR.MINOR.PATCH`, with `-rcN` release candidates published as pre-releases. See [docs/RELEASING.md](docs/RELEASING.md) for how releases are cut. +## Prebuilt images + +The [Images workflow](.github/workflows/images.yml) builds one `linux/amd64` +image per Compose build and publishes it to GHCR whenever a push to `main` +changes an image input: + +| Image (`ghcr.io/librechat-ai/…`) | Dockerfile (target) | Compose service | +|---|---|---| +| `code-interpreter-api` | `service/Dockerfile` (`api`) | `api` | +| `code-interpreter-worker` | `service/Dockerfile` (`worker`) | `service-worker` | +| `code-interpreter-file-server` | `service/Dockerfile` (`production`) | `file_server` | +| `code-interpreter-egress-gateway` | `service/Dockerfile.egress-gateway` (`production`) | `egress_gateway` | +| `code-interpreter-tool-call-server` | `service/Dockerfile.tool-call-server` (`production`) | `tool_call_server` | +| `code-interpreter-sandbox-runner` | `api/Dockerfile` (`sandbox-runner-true`) | `sandbox-runner`, `KVM_ENABLED=true` (default) | +| `code-interpreter-sandbox-runner-direct` | `api/Dockerfile` (`sandbox-runner-false`) | `sandbox-runner`, `KVM_ENABLED=false` | + +Tags: + +- `sha-` for every commit on `main` that changes an image + input. Commits that change none (docs, Helm, tests) get no tags, so take the + SHA from a successful Images run or the package page. +- `main`, pointed at the newest commit on `main` that has all seven images. + The seven tags move one after another, so pin `sha-` tags for deployments + and treat `main` as a convenience. + +There is no `latest`. `sha-` tags are written once: re-running the workflow +for a commit builds only the images whose tag is missing. For exact bytes, +replace a service's +`image:` below with `ghcr.io/librechat-ai/@sha256:`; each image +has its own digest, listed in the summary of the run that pushed it. + +Pulling a public package needs no login. A private package needs +`docker login ghcr.io` with a token that has `read:packages`; GHCR creates new +packages as private until an organization owner changes their visibility. + +To switch a Compose host from building to pulling, save this override as +`docker-compose.images.yml` (`!reset` needs Docker Compose 2.24.4 or later): + +```yaml +services: + api: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-api:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + service-worker: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-worker:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + file_server: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-file-server:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + egress_gateway: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-egress-gateway:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + tool_call_server: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-tool-call-server:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + sandbox-runner: + build: !reset null + # With KVM_ENABLED=false use code-interpreter-sandbox-runner-direct, which + # still reads runtime packages from SANDBOX_PACKAGES_PATH. + image: ghcr.io/librechat-ai/code-interpreter-sandbox-runner:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing +``` + +Then pin a built commit and start the stack, listing any host override after +it. Keep the checkout at the same commit as the images, so the Compose +configuration matches what the images expect. `docker-compose.mac.yml` runs +the unpublished `sandbox-build` target with its own entrypoint, so on macOS +leave `sandbox-runner` out of this override and keep building it: + +```bash +export CODEAPI_IMAGE_TAG=sha- +docker compose -f docker-compose.yaml -f docker-compose.images.yml pull +docker compose -f docker-compose.yaml -f docker-compose.images.yml up -d +``` + +`pull_policy: missing` never re-pulls a tag the host already has, which suits +an immutable `sha-` tag. To track `main` instead, set `CODEAPI_IMAGE_TAG=main` +and run `pull` before every `up`. + ## Local Development Copy `.env.example` to `.env` and set `CODEAPI_BRIDGE_TOKEN` to a private value From 4b2b8e159225df3cb894781a003d843069503e12 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 09:14:48 -0400 Subject: [PATCH 69/75] fix: block sandbox descriptor passing into unix socket gc (#309) --- .github/workflows/ci.yml | 28 ++++++ api/src/nsjail.test.ts | 14 +++ api/src/nsjail.ts | 7 ++ docs/SECURITY-MAINTENANCE.md | 46 +++++++++ tests/nsjail_unix_gc.py | 176 +++++++++++++++++++++++++++++++++++ 5 files changed, 271 insertions(+) create mode 100644 tests/nsjail_unix_gc.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee14372..37f0fe2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,6 +138,34 @@ jobs: oven/bun:1.3.14-debian \ bun test src/cleanup.integration.test.ts + nsjail-ipc-tests: + name: NsJail IPC Filter (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Render native sandbox policy + working-directory: api + run: | + bun ci + bun scripts/dump-seccomp-policy.ts > "$RUNNER_TEMP/nsjail.policy" + - name: Build pinned NsJail and Kafel + run: docker build --target nsjail-builder -t nsjail-ipc-test -f api/Dockerfile . + - name: Verify descriptor denial and runtime IPC with the installed filter + run: | + docker run --rm --security-opt seccomp=unconfined \ + --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test \ + python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so + service-unit-tests: name: Service Unit Tests runs-on: ubuntu-latest diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index 95de6251..eb7100f8 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -390,6 +390,20 @@ describe('NsJail seccomp policy', () => { expect(rule).not.toMatch(/== FUTEX_(WAIT|WAKE)(?:\s|$)/); }); + test('blocks descriptor-passing sends while preserving ordinary Unix socket IPC', () => { + const policy = seccompPolicy(); + const errnoBlock = policy.split('ERRNO(1) {')[1]?.split(' }')[0] ?? ''; + for (const name of ['sendmsg', 'sendmmsg', 'io_uring_setup', 'io_uring_enter', 'io_uring_register']) { + expect(errnoBlock).toMatch(new RegExp(`\\b${name}\\b[,\\s]`)); + } + // AF_UNIX/socketpair are needed by the tool-call channel and runtime IPC. + expect(policy).not.toContain('AF_UNIX'); + for (const name of ['socketpair', 'read', 'write', 'sendto', 'recvfrom']) { + expect(policy).not.toMatch(new RegExp(`\\b${name}\\b`)); + } + expect(policy).toContain('USE sandbox DEFAULT ALLOW'); + }); + test('rejects Copy Fail and Dirty Frag socket entry points in the sandbox', () => { const policy = seccompPolicy(); const errnoBlock = policy.split('ERRNO(1)')[1] ?? ''; diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 3a5de4ad..ce1353d7 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -148,6 +148,13 @@ const SECCOMP_POLICY = [ ' },', ' ERRNO(1) {', ' io_uring_setup, io_uring_enter, io_uring_register, sched_setaffinity, vmsplice,', + /* CVE-2026-80521: AF_UNIX GC is reachable through SCM_RIGHTS descriptor + * passing. Seccomp cannot inspect the msghdr/mmsghdr control data, so + * refuse both send APIs outright, including on inherited sockets. + * Keep AF_UNIX/socketpair and read/write/sendto for /tmp/tcs.sock and + * ordinary multiprocessing IPC. io_uring above must remain blocked: + * its sendmsg operations bypass syscall-level filtering. */ + ' sendmsg, sendmmsg,', /* GhostLock (CVE-2026-43499): deny PI operations, including private and * realtime variants, without blocking ordinary runtime futex wait/wake. */ ' futex(uaddr, op) { (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_UNLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_TRYLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_WAIT_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_CMP_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI2 },', diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md index 24c1ee01..c8494a90 100644 --- a/docs/SECURITY-MAINTENANCE.md +++ b/docs/SECURITY-MAINTENANCE.md @@ -26,6 +26,7 @@ Priority 2026 checks: |---|---| | Guest kernel; host kernel in direct mode | Bad Epoll, CVE-2026-46242 | | Guest and host kernels | GhostLock, CVE-2026-43499 | +| Guest kernel; host kernel in direct mode | AF_UNIX garbage collection, CVE-2026-80521 | | x86 host KVM | CVE-2026-46113, CVE-2026-53359, CVE-2026-64561 | | ARM host KVM, when virtual ITS is exposed | CVE-2026-46316 | @@ -37,6 +38,51 @@ VMM's behavior rather than assuming current upstream defaults match its build. The October 2026 Vercel KVM report has no disclosed root cause or affected-version matrix at the time of this change. These controls are not a verified fix for it. +## Unix-socket descriptor passing + +The NsJail policy returns EPERM for `sendmsg` and `sendmmsg` to prevent +`SCM_RIGHTS` descriptor passing into the AF_UNIX garbage collector affected by +[CVE-2026-80521](https://www.cve.org/CVERecord?id=CVE-2026-80521). +Seccomp cannot inspect ancillary data behind a userspace pointer, so both calls +are denied even for messages without descriptors and for inherited sockets. +Keep `io_uring_setup`, `io_uring_enter`, and `io_uring_register` denied: ring +operations can submit sends without passing through these syscall filters. + +Unlike the UDF sandbox's socket-family restriction, this sandbox retains +AF_UNIX and `socketpair` for `/tmp/tcs.sock` and ordinary IPC using reads, +writes, and `sendto`. Python multiprocessing pipes, queues, and pools remain +supported; transferring socket/file handles through `multiprocessing.reduction` +or similar descriptor-passing APIs returns EPERM. Do not restore `sendmsg` or +`sendmmsg` to make those APIs work without reassessing the kernel boundary. + +This mitigation is separate from PI-futex filtering. Verify vendor patches for +the deployed guest and node kernels; a policy update does not establish patch +status. Rebuild and replace sandbox runner images to deploy the policy change. + +The native `NsJail IPC Filter` CI jobs compile the rendered policy with the +image's pinned Kafel on amd64 and arm64, install it in a Linux process, and +check descriptor sends, inherited sockets, Unix HTTP, and Python fork/spawn +queues and pools. This validates the filter and IPC compatibility, not a +production kernel's patch status or the full microVM deployment. + +### Planned transport hardening + +Replace the job-visible Unix HTTP socket with two dedicated per-job pipes to +a trusted broker. Keep them separate from stdin/stdout/stderr, which carry user +input and execution output. Limit message sizes, outstanding requests, and +timeouts; retain the existing job authorization and tool-call budgets at the +broker. Update each language's tool-call client to use framed messages, and +explicitly preserve only these pipe FDs through NsJail and `spec-guard`. +The current guard closes all FDs above 2; a redesign must preserve its protection +against unrelated inherited descriptors. + +Then deny `socket` and `socketpair` in the job policy and retain the send and +io_uring denials. Validate language/runtime compatibility, including Python's +multiprocessing resource tracker and descriptor-sharing features, before making +that stricter policy the default. The broker retains network access outside the +job filter; it must not expose raw socket handles to the job. This is follow-up +work, not part of the immediate descriptor-passing mitigation. + ## Preserve host-side confinement - Run sandbox runners on isolated nodes, separate from credential-bearing diff --git a/tests/nsjail_unix_gc.py b/tests/nsjail_unix_gc.py new file mode 100644 index 00000000..84cec7b5 --- /dev/null +++ b/tests/nsjail_unix_gc.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Exercise the rendered policy with the pinned Kafel on Linux. + +Usage: python3 tests/nsjail_unix_gc.py POLICY /nsjail/kafel/libkafel.so +Run in the nsjail-builder image. No exploit or cyclic descriptor graph is used. +The filter stays installed until this test process and its children exit. +""" +import array +import ctypes as c +import errno +import http.client +import multiprocessing as mp +import os +import socket +import struct +import sys +import tempfile +import threading + + +class SockFilter(c.Structure): + _fields_ = [("code", c.c_ushort), ("jt", c.c_ubyte), + ("jf", c.c_ubyte), ("k", c.c_uint)] + + +class SockFprog(c.Structure): + _fields_ = [("len", c.c_ushort), ("filter", c.POINTER(SockFilter))] + + +class Iovec(c.Structure): + _fields_ = [("base", c.c_void_p), ("length", c.c_size_t)] + + +class Msghdr(c.Structure): + _fields_ = [("name", c.c_void_p), ("namelen", c.c_uint), + ("iov", c.POINTER(Iovec)), ("iovlen", c.c_size_t), + ("control", c.c_void_p), ("controllen", c.c_size_t), + ("flags", c.c_int)] + + +class Mmsghdr(c.Structure): + _fields_ = [("header", Msghdr), ("length", c.c_uint)] + + +def expect_eperm(call): + try: + call() + except OSError as error: + assert error.errno == errno.EPERM, error + else: + raise AssertionError("descriptor-passing send was allowed") + + +def square(value): + return value * value + + +def queue_child(queue): + queue.put("queue works") + + +def test_sends(libc, pair): + left, right = pair + # A regular-file descriptor avoids creating any socket reference cycle, + # even if a regressed policy allows the send and the assertion fails. + descriptor = os.open("/dev/null", os.O_RDONLY) + rights = [(socket.SOL_SOCKET, socket.SCM_RIGHTS, + array.array("i", [descriptor]))] + expect_eperm(lambda: left.sendmsg([b"x"], rights)) + expect_eperm(lambda: left.sendmsg([b"x"])) + # Valid batch: seccomp must deny even messages without ancillary data. + data = c.create_string_buffer(b"x") + iov = Iovec(c.cast(data, c.c_void_p), 1) + message = Mmsghdr() + message.header.iov = c.pointer(iov) + message.header.iovlen = 1 + c.set_errno(0) + assert libc.sendmmsg(left.fileno(), c.byref(message), 1, 0) == -1 + assert c.get_errno() == errno.EPERM + control = c.create_string_buffer(socket.CMSG_SPACE(c.sizeof(c.c_int))) + header = struct.pack("@Nii", socket.CMSG_LEN(c.sizeof(c.c_int)), + socket.SOL_SOCKET, socket.SCM_RIGHTS) + control[:len(header)] = header + offset = socket.CMSG_LEN(0) + control[offset:offset + c.sizeof(c.c_int)] = struct.pack("@i", descriptor) + message.header.control = c.cast(control, c.c_void_p) + message.header.controllen = socket.CMSG_SPACE(c.sizeof(c.c_int)) + assert libc.sendmmsg(left.fileno(), c.byref(message), 1, 0) == -1 + assert c.get_errno() == errno.EPERM + os.close(descriptor) + left.sendall(b"ordinary IPC") + assert right.recv(12) == b"ordinary IPC" + left.close() + right.close() + + +def test_unix_http(): + with tempfile.TemporaryDirectory() as directory: + pathname = os.path.join(directory, "tcs.sock") + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: + listener.bind(pathname) + listener.listen(1) + listener.settimeout(5) + errors = [] + + def serve(): + try: + with listener.accept()[0] as peer: + peer.settimeout(5) + request = b"" + while b"\r\n\r\n" not in request: + chunk = peer.recv(4096) + assert chunk, "client disconnected before HTTP headers" + request += chunk + assert request.startswith(b"POST /tool-call HTTP/1.1") + peer.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\n" + b"Connection: close\r\n\r\n{\"ok\":true}") + except Exception as error: + errors.append(error) + + thread = threading.Thread(target=serve) + thread.start() + connection = http.client.HTTPConnection("localhost", timeout=5) + connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.sock.settimeout(5) + connection.sock.connect(pathname) + try: + connection.request("POST", "/tool-call", body=b"{}") + response = connection.getresponse() + assert response.status == 200 + assert response.read() == b'{"ok":true}' + finally: + connection.close() + thread.join(5) + assert not thread.is_alive() + assert not errors, errors + + +def main(): + # Open before installation to verify the rule also covers inherited FDs. + inherited = socket.socketpair() + libc = c.CDLL(None, use_errno=True) + libc.sendmmsg.argtypes = [c.c_int, c.POINTER(Mmsghdr), c.c_uint, c.c_int] + kafel = c.CDLL(sys.argv[2]) + kafel.kafel_compile_string.argtypes = [c.c_char_p, c.POINTER(SockFprog)] + program = SockFprog() + with open(sys.argv[1], "rb") as source: + assert kafel.kafel_compile_string(source.read(), c.byref(program)) == 0 + # PR_SET_NO_NEW_PRIVS, followed by PR_SET_SECCOMP / SECCOMP_MODE_FILTER. + assert libc.prctl(38, 1, 0, 0, 0) == 0 + assert libc.prctl(22, 2, c.byref(program), 0, 0) == 0, c.get_errno() + # These numbers are shared on x86_64 and arm64. No ring may bypass sends. + for syscall in (425, 426, 427): + assert libc.syscall(syscall, 0, 0, 0, 0, 0, 0) == -1 + assert c.get_errno() == errno.EPERM + test_sends(libc, inherited) + for kind in (socket.SOCK_STREAM, socket.SOCK_DGRAM): + test_sends(libc, socket.socketpair(socket.AF_UNIX, kind)) + test_unix_http() + for method in ("fork", "spawn"): + context = mp.get_context(method) + queue = context.Queue() + process = context.Process(target=queue_child, args=(queue,)) + process.start() + assert queue.get(timeout=10) == "queue works" + process.join(10) + assert process.exitcode == 0 + queue.close() + queue.join_thread() + with context.Pool(2) as pool: + assert pool.map(square, [2, 3, 4]) == [4, 9, 16] + print("PASS: descriptor sends denied; Unix IPC, HTTP, queues and pools work") + + +if __name__ == "__main__": + main() From 2318ba0d27517310586dcc4692dc53fc9d375e5c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 11:40:19 -0400 Subject: [PATCH 70/75] feat: replace tool-call sockets with anonymous pipes (#310) * feat: replace job-visible tool-call sockets with anonymous pipes * test: Observe Broker Cancellation Through TCP Peer * fix: Align Bun 1.4.2 and Verify Packaged Pipe Runtime * chore: Drop Node 20 Support * fix: Bound Pipe Failure Paths and Package Launcher Runtime --- .github/workflows/ci.yml | 46 +- api/Dockerfile | 22 +- api/package.json | 2 +- api/src/api/v2.ts | 4 - api/src/entrypoint.sh | 10 +- api/src/index.ts | 4 - api/src/job.ts | 6 +- api/src/nsjail.test.ts | 25 +- api/src/nsjail.ts | 55 +- api/src/session-inputs.test.ts | 1 + api/src/spec-guard.c | 28 +- api/src/tool-call-pipe-bridge.c | 123 +++ api/src/tool-call-pipe-broker.test.ts | 158 +++ api/src/tool-call-pipe-broker.ts | 165 ++++ api/src/tool-call-socket-process.test.ts | 119 --- api/src/tool-call-socket-process.ts | 191 ---- .../tool-call-socket-proxy-runtime.test.ts | 905 ------------------ api/src/tool-call-socket-proxy.test.ts | 448 --------- api/src/tool-call-socket-proxy.ts | 669 ------------- build-packages.sh | 4 +- docker/Dockerfile.worker-sandbox | 17 +- docker/package-init.sh | 2 +- docs/SECURITY-MAINTENANCE.md | 125 ++- docs/adr/0011-tool-call-pipes.md | 61 ++ docs/lambda-microvm/README.md | 2 +- docs/openshell/README.md | 2 +- docs/remote-bridge/worker-runbook.md | 2 +- helm/codeapi/values.yaml | 2 +- launcher/Dockerfile | 15 +- packages/code/README.md | 3 + packages/code/package-lock.json | 2 +- packages/code/package.json | 2 +- service/scripts/dump-pipe-preamble.ts | 6 + service/src/config.spec.ts | 6 +- service/src/config.ts | 4 +- service/src/egress-gateway.ts | 4 +- service/src/preamble.test.ts | 79 +- service/src/preamble.ts | 188 ++-- tests/nsjail_unix_gc.py | 57 +- tests/pipe_runtime_image.cjs | 61 ++ tests/specguard_fd_isolation.py | 289 ------ tests/specguard_tool_call_pipes.py | 45 + tests/tool_call_pipes.py | 171 ++++ tests/tool_call_runner.cjs | 127 +++ 44 files changed, 1321 insertions(+), 2936 deletions(-) create mode 100644 api/src/tool-call-pipe-bridge.c create mode 100644 api/src/tool-call-pipe-broker.test.ts create mode 100644 api/src/tool-call-pipe-broker.ts delete mode 100644 api/src/tool-call-socket-process.test.ts delete mode 100644 api/src/tool-call-socket-process.ts delete mode 100644 api/src/tool-call-socket-proxy-runtime.test.ts delete mode 100644 api/src/tool-call-socket-proxy.test.ts delete mode 100644 api/src/tool-call-socket-proxy.ts create mode 100644 docs/adr/0011-tool-call-pipes.md create mode 100644 service/scripts/dump-pipe-preamble.ts create mode 100644 tests/pipe_runtime_image.cjs delete mode 100644 tests/specguard_fd_isolation.py create mode 100644 tests/specguard_tool_call_pipes.py create mode 100644 tests/tool_call_pipes.py create mode 100644 tests/tool_call_runner.cjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f0fe2c..58f68b7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: run: | docker buildx build --check -f api/Dockerfile . docker buildx build --check -f docker/Dockerfile.worker-sandbox . + docker buildx build --check -f launcher/Dockerfile . launcher-unit-tests: name: Launcher Unit Tests @@ -89,15 +90,15 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Cache Bun packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.bun/install/cache - key: ${{ runner.os }}-codeapi-api-bun-1.3.14-${{ hashFiles('api/bun.lock') }} + key: ${{ runner.os }}-codeapi-api-bun-1.4.2-${{ hashFiles('api/bun.lock') }} restore-keys: | - ${{ runner.os }}-codeapi-api-bun-1.3.14- + ${{ runner.os }}-codeapi-api-bun-1.4.2- - name: Install dependencies run: bun ci @@ -135,7 +136,7 @@ jobs: --workdir /work/api \ --env SANDBOX_CLEANUP_TMPFS_TEST=1 \ --env SANDBOX_LOG_LEVEL=error \ - oven/bun:1.3.14-debian \ + oven/bun:1.4.2-debian \ bun test src/cleanup.integration.test.ts nsjail-ipc-tests: @@ -150,21 +151,40 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Render native sandbox policy working-directory: api run: | bun ci bun scripts/dump-seccomp-policy.ts > "$RUNNER_TEMP/nsjail.policy" + - name: Build pipe broker and generate Python client + run: | + bun run --cwd api build + bun ci --cwd service + bun service/scripts/dump-pipe-preamble.ts > "$RUNNER_TEMP/pipe-client.py" - name: Build pinned NsJail and Kafel - run: docker build --target nsjail-builder -t nsjail-ipc-test -f api/Dockerfile . + run: docker build --target nsjail-ipc-test -t nsjail-ipc-test -f api/Dockerfile . + - name: Verify standalone launcher guest pipe runtime + run: | + docker build --target sandbox-build -t launcher-guest-test -f launcher/Dockerfile . + docker run --rm --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/node \ + launcher-guest-test /tests/pipe_runtime_image.cjs - name: Verify descriptor denial and runtime IPC with the installed filter run: | docker run --rm --security-opt seccomp=unconfined \ --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ nsjail-ipc-test \ - python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so + bash -c 'python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so && python3 /tests/specguard_tool_call_pipes.py && gcc -O2 -static -Wall -Wextra -Werror -DSPEC_GUARD_TEST_DISABLE_CLOSE_RANGE -o /tmp/guard-fallback /tmp/spec-guard.c && SPEC_GUARD_BINARY=/tmp/guard-fallback python3 /tests/specguard_tool_call_pipes.py' + docker run --rm --security-opt seccomp=unconfined \ + --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ + --mount "type=bind,source=$RUNNER_TEMP/pipe-client.py,target=/client.py,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/api/.build/tool-call-pipe-broker.cjs,target=/broker.cjs,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test \ + python3 /tests/tool_call_pipes.py /policy /broker.cjs /client.py service-unit-tests: name: Service Unit Tests @@ -177,15 +197,15 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Cache Bun packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.bun/install/cache - key: ${{ runner.os }}-codeapi-service-bun-1.3.14-${{ hashFiles('service/bun.lock') }} + key: ${{ runner.os }}-codeapi-service-bun-1.4.2-${{ hashFiles('service/bun.lock') }} restore-keys: | - ${{ runner.os }}-codeapi-service-bun-1.3.14- + ${{ runner.os }}-codeapi-service-bun-1.4.2- - name: Install dependencies run: bun ci @@ -212,7 +232,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: ['20.11.0', '22.21.0', '24.16.0'] + node-version: ['22.21.0', '24.16.0'] defaults: run: working-directory: packages/code @@ -301,7 +321,7 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Install service dependencies working-directory: service @@ -373,7 +393,7 @@ jobs: docker buildx inspect --bootstrap - name: Build Lambda MicroVM runner image - # The target pins Bun 1.3.14 and installs api/bun.lock frozen. Building + # The target pins Bun 1.4.2 and installs api/bun.lock frozen. Building # natively catches architecture-specific package and Docker layer # failures without retaining a second multi-gigabyte image copy. run: | diff --git a/api/Dockerfile b/api/Dockerfile index 66db9e3d..0ee343a7 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -29,11 +29,18 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup +# Native policy/transport verification without building the full runner. +FROM nsjail-builder AS nsjail-ipc-test +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=oven/bun:1.4.2-debian /usr/local/bin/bun /usr/local/bin/bun + # ============================================================================ # Stage 1b: Build language runtime packages (only consumed by sandbox-runner-baked) # @@ -46,7 +53,7 @@ FROM buildpack-deps:bookworm AS package-builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHON_VERSION=3.14.4 -ENV BUN_VERSION=1.3.14 +ENV BUN_VERSION=1.4.2 COPY docker/apt-install.sh /usr/local/bin/apt-install RUN chmod +x /usr/local/bin/apt-install @@ -68,7 +75,7 @@ RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh # ============================================================================ # Stage 2: Build sandbox rootfs # ============================================================================ -FROM oven/bun:1.3.14-debian AS sandbox-build +FROM oven/bun:1.4.2-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -110,14 +117,11 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge RUN chmod +x /usr/sbin/nsjail -# The tool-call socket proxy must run under Node, not Bun. Bun's node:http -# compat layer never fires `'connection'` events and Bun.serve's idleTimeout -# does not close silent unix-socket connections — so our DoS protections -# (maxConnections, idleSocketTimeoutMs, accept-time tracking) silently no-op -# under Bun and a `for i in range(N): socket.connect` storm pins FDs in the -# proxy until the OS limit. Pulling just the Node binary from a slim image +# The per-invocation broker starts Node after restore, so its OpenSSL process +# state is not captured in the MicroVM snapshot. Pulling the Node binary # is ~30 MB and avoids inheriting npm/node_modules from a full apt install. COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node diff --git a/api/package.json b/api/package.json index de62fb55..a27fd780 100644 --- a/api/package.json +++ b/api/package.json @@ -5,7 +5,7 @@ "main": "src/index.ts", "scripts": { "dev": "bun run --watch src/index.ts", - "build": "bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' && bun build ./src/tool-call-socket-proxy.ts --target=node --format=cjs --outfile=.build/tool-call-socket-proxy.cjs", + "build": "bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' && bun build ./src/tool-call-pipe-broker.ts --target=node --format=cjs --outfile=.build/tool-call-pipe-broker.cjs", "start": "bun run .build/index.js", "test": "bun test" }, diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index 3ace08cf..cba8e80a 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -26,7 +26,6 @@ import { parseSessionBindingFromHeader, } from '../session-workspace'; import { streamSessionCheckpoint, restoreSessionCheckpoint } from '../session-checkpoint'; -import { ensureToolCallSocketProxyReady } from '../tool-call-socket-process'; import { SESSION_INPUT_CACHE_MAX_OBJECTS, hasCachedInput, @@ -579,9 +578,6 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn } try { - if (toolCallSocketEnabled) { - await ensureToolCallSocketProxyReady(); - } await withSpan('codeapi.sandbox.prime', { 'codeapi.language': job.runtime.language, }, () => job!.prime()); diff --git a/api/src/entrypoint.sh b/api/src/entrypoint.sh index 4fce5ba4..d670a359 100755 --- a/api/src/entrypoint.sh +++ b/api/src/entrypoint.sh @@ -123,8 +123,8 @@ fi ALLOWED_PORT="${SANDBOX_ALLOWED_LOCAL_NETWORK_PORT:-0}" # NsJail runs sandboxed processes as inside UID 65534 (nobody), mapped to a -# per-job outside UID by the sandbox API. The proxy socket remains mounted -# into each jail, so it does not need per-UID ownership. +# per-job outside UID by the sandbox API. Authorized tool-call invocations +# receive two anonymous pipe ends; no socket path is mounted into a jail. SANDBOX_UID=65534 # Extract API port from bind address (default 2000) @@ -136,11 +136,11 @@ if [ "$ALLOWED_PORT" -gt 0 ] 2>/dev/null; then exit 1 fi - echo "Tool-call socket forwarding enabled for UID $SANDBOX_UID (port $ALLOWED_PORT)" + echo "Tool-call pipe forwarding enabled for UID $SANDBOX_UID (port $ALLOWED_PORT)" # Do not start Node here. Lambda MicroVM image creation snapshots this # already-running container, and the official Node binary embeds OpenSSL. - # The API starts and awaits the narrow Unix-socket proxy only after a - # post-restore /execute is authorized for tool calls. + # The API starts a per-invocation pipe broker only after a post-restore + # /execute is authorized for tool calls. fi # Package permissions are finalized by package-init when the PVC is populated. diff --git a/api/src/index.ts b/api/src/index.ts index cfbc936c..00d10e9d 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -7,7 +7,6 @@ import { initializeSandboxWorkspaceIsolation, startWorkspaceReaper } from './wor import { httpMetricsMiddleware, metricsHandler } from './metrics'; import { positiveInt, shutdownTelemetry, traceHttpRequest } from './telemetry'; import { startWarmupCommand } from './warmup'; -import { stopToolCallSocketProxy } from './tool-call-socket-process'; import { hostedAppSupervisor, validateHostedAppStartup } from './hosted-app'; import v2Router from './api/v2'; import lifecycleRouter, { LIFECYCLE_HOOK_BASE_PATH } from './api/lifecycle'; @@ -116,9 +115,6 @@ async function main(): Promise { await hostedAppSupervisor.shutdown().catch((err) => { logger.warn({ err }, 'Hosted-app process shutdown failed'); }); - await stopToolCallSocketProxy().catch((err) => { - logger.warn({ err }, 'Tool-call socket proxy shutdown failed'); - }); try { await shutdownTelemetry(); } catch (err) { diff --git a/api/src/job.ts b/api/src/job.ts index 5c2fddef..bfa99b63 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -701,7 +701,7 @@ export class Job { memory_limits: { run: number; compile: number }; extra_env_vars?: Record; egressGrantToken?: string; - toolCallSocketEnabled: boolean; + toolCallPipesEnabled: boolean; isSynthetic: boolean; outputSessionId: string; @@ -786,7 +786,7 @@ export class Job { this.memory_limits = opts.memory_limits; this.extra_env_vars = opts.extra_env_vars; this.egressGrantToken = opts.egress_grant; - this.toolCallSocketEnabled = opts.tool_call_socket_enabled === true; + this.toolCallPipesEnabled = opts.tool_call_socket_enabled === true; this.isSynthetic = opts.is_synthetic === true; } @@ -1644,7 +1644,7 @@ export class Job { stdin, extraPkgdirs, identity: this.sandboxIdentity(), - enableToolCallSocket: this.toolCallSocketEnabled && script === 'run', + enableToolCallPipes: this.toolCallPipesEnabled && script === 'run', suppressSuccessLogs: this.isSynthetic, }); } diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index eb7100f8..73dc3863 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -91,7 +91,7 @@ describe('NsJail args', () => { expect(valueAfter(args, '--config')).toBe('/tmp/nsjail-job-xyz.cfg'); }); - test('does not export TOOL_CALL_SOCKET into the jail (preamble references the literal path)', () => { + test('does not export a legacy tool-call socket path into the jail', () => { const args = buildArgs({ logPath: '/tmp/nsjail-test.log', pkgdir: '/pkgs/python/3.14.4', @@ -109,7 +109,7 @@ describe('NsJail args', () => { } }); - test('binds the tool-call socket only for jobs that explicitly request it', () => { + test('passes tool-call pipes only for jobs that explicitly request the capability', () => { const originalAllowedPort = config.allowed_local_network_port; config.allowed_local_network_port = 3190; try { @@ -126,8 +126,12 @@ describe('NsJail args', () => { const withoutSocket = buildArgs(baseOpts); expect(hasArgPair(withoutSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(false); - const withSocket = buildArgs({ ...baseOpts, enableToolCallSocket: true }); - expect(hasArgPair(withSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(true); + const withSocket = buildArgs({ ...baseOpts, enableToolCallPipes: true }); + expect(hasArgPair(withSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(false); + expect(hasArgPair(withSocket, '--pass_fd', '3')).toBe(true); + expect(hasArgPair(withSocket, '--pass_fd', '4')).toBe(true); + expect(withSocket).toContain('--tool-call-pipes'); + expect(withoutSocket).not.toContain('--pass_fd'); } finally { config.allowed_local_network_port = originalAllowedPort; } @@ -371,7 +375,7 @@ describe('NsJail seccomp policy', () => { const errnoSocketRule = errnoBlock.split('\n').find(line => line.includes('socket(domain)')); expect(errnoSocketRule).toBeDefined(); - expect(errnoSocketRule).not.toContain('AF_VSOCK'); + expect(errnoSocketRule).toContain('domain != AF_VSOCK'); }); test('blocks priority-inheritance futex operations without denying ordinary wait/wake', () => { @@ -396,9 +400,9 @@ describe('NsJail seccomp policy', () => { for (const name of ['sendmsg', 'sendmmsg', 'io_uring_setup', 'io_uring_enter', 'io_uring_register']) { expect(errnoBlock).toMatch(new RegExp(`\\b${name}\\b[,\\s]`)); } - // AF_UNIX/socketpair are needed by the tool-call channel and runtime IPC. - expect(policy).not.toContain('AF_UNIX'); - for (const name of ['socketpair', 'read', 'write', 'sendto', 'recvfrom']) { + // Restrict socketpairs to runtime stream IPC; tool calls use real pipes. + expect(policy).toContain('socketpair(domain, type, protocol) { domain != AF_UNIX || (type & 0xf) != SOCK_STREAM || protocol != 0 }'); + for (const name of ['read', 'write', 'sendto', 'recvfrom']) { expect(policy).not.toMatch(new RegExp(`\\b${name}\\b`)); } expect(policy).toContain('USE sandbox DEFAULT ALLOW'); @@ -409,10 +413,7 @@ describe('NsJail seccomp policy', () => { const errnoBlock = policy.split('ERRNO(1)')[1] ?? ''; const socketRule = errnoBlock.split('\n').find(line => line.includes('socket(domain)')); expect(socketRule).toBeDefined(); - for (const family of ['AF_ALG', 'AF_RXRPC', 'AF_INET', 'AF_INET6']) { - expect(socketRule).toContain(family); - } - expect(socketRule).not.toContain('AF_VSOCK'); + expect(socketRule).toContain('domain != AF_VSOCK'); }); test('KILLs the defense-in-depth batch from the audit', () => { diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index ce1353d7..51b8700e 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -95,6 +95,8 @@ const archSpecificLowPrioritySyscalls = process.arch === 'arm64' const SECCOMP_POLICY = [ ...syscallDefines, + '#define AF_UNIX 1', + '#define SOCK_STREAM 1', '#define AF_INET 2', '#define AF_INET6 10', '#define AF_NETLINK 16', @@ -151,8 +153,9 @@ const SECCOMP_POLICY = [ /* CVE-2026-80521: AF_UNIX GC is reachable through SCM_RIGHTS descriptor * passing. Seccomp cannot inspect the msghdr/mmsghdr control data, so * refuse both send APIs outright, including on inherited sockets. - * Keep AF_UNIX/socketpair and read/write/sendto for /tmp/tcs.sock and - * ordinary multiprocessing IPC. io_uring above must remain blocked: + * The job receives only anonymous tool-call pipes. Restricted Unix + * stream socketpairs remain for asyncio and subprocess/runtime IPC. + * io_uring above must remain blocked: * its sendmsg operations bypass syscall-level filtering. */ ' sendmsg, sendmmsg,', /* GhostLock (CVE-2026-43499): deny PI operations, including private and @@ -181,10 +184,11 @@ const SECCOMP_POLICY = [ * runtimes the sandbox supports. */ ' pidfd_open(pid) { pid == 1 },', ' pidfd_send_signal,', - /* Block direct network and kernel-control socket domains from sandboxed - * code. AF_ALG covers the Linux kernel crypto API used by Copy Fail, and - * AF_RXRPC covers the RxRPC family used by Dirty Frag. */ - ' socket(domain) { domain == AF_INET || domain == AF_INET6 || domain == AF_NETLINK || domain == AF_KEY || domain == AF_RXRPC || domain == AF_ALG }', + /* Deny every socket() family. VSOCK keeps its stronger KILL action above. + * Only anonymous Unix stream socketpairs are allowed for runtime IPC; + * sendmsg/sendmmsg remain denied, so no SCM_RIGHTS graph can be built. */ + ' socket(domain) { domain != AF_VSOCK },', + ' socketpair(domain, type, protocol) { domain != AF_UNIX || (type & 0xf) != SOCK_STREAM || protocol != 0 }', ' }', '}', 'USE sandbox DEFAULT ALLOW', @@ -242,7 +246,7 @@ interface ExecuteOptions { stdin?: string; extraPkgdirs?: string[]; identity: SandboxJobIdentity; - enableToolCallSocket?: boolean; + enableToolCallPipes?: boolean; suppressSuccessLogs?: boolean; } @@ -258,7 +262,7 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = stdin, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, suppressSuccessLogs, } = opts; const logId = nanoid(); @@ -277,7 +281,7 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = command, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, }); const startTime = Date.now(); @@ -399,11 +403,20 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = const childDiedSignal = new AbortController(); try { ({ value: proc, markerSeen, pollError } = await setupGate.runSetup(logPath, () => { - const child = spawn(config.nsjail_path, nsjailArgs, { + const pipeMode = enableToolCallPipes === true; + if (pipeMode && (config.allowed_local_network_port <= 0 || !process.env.SANDBOX_FORWARD_TARGET?.trim())) { + throw new Error('tool-call pipe broker is not configured'); + } + const child = spawn(pipeMode ? (process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge') : config.nsjail_path, pipeMode ? [ + '--broker', process.env.TCS_NODE_BINARY || 'node', + process.env.TCS_PIPE_BROKER_BUNDLE ?? path.resolve(__dirname, '..', '.build', 'tool-call-pipe-broker.cjs'), + config.nsjail_path, ...nsjailArgs, + ] : nsjailArgs, { /* stdin always 'pipe' so we have a writable handle to .end() even * when no input is supplied. The child sees EOF on stdin * immediately, equivalent to the previous Bun.spawn 'ignore'. */ stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, TCS_REQUEST_TIMEOUT_MS: String(timeout) }, }) as ChildProcessWithoutNullStreams; /* Attach the 'error' listener SYNCHRONOUSLY before any await. The * setup gate then polls the log file for ~tens of ms; during that @@ -654,7 +667,7 @@ interface BuildArgsOptions { command: string[]; extraPkgdirs?: string[]; identity: SandboxJobIdentity; - enableToolCallSocket?: boolean; + enableToolCallPipes?: boolean; } export function buildArgs(opts: BuildArgsOptions): string[] { @@ -668,7 +681,7 @@ export function buildArgs(opts: BuildArgsOptions): string[] { command, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, } = opts; const timeoutSecs = Math.max(1, Math.ceil(timeout / 1000)); @@ -721,25 +734,17 @@ export function buildArgs(opts: BuildArgsOptions): string[] { args.push('--cgroup_mem_max', String(memoryLimit)); } - if (config.allowed_local_network_port > 0 && enableToolCallSocket === true) { - const socketPath = '/tmp/tcs.sock'; - args.push('-B', `${socketPath}:${socketPath}`); + if (enableToolCallPipes === true) { + args.push('--pass_fd', '3', '--pass_fd', '4'); } for (const [key, value] of Object.entries(envVars)) { args.push('-E', `${key}=${value}`); } - /* TOOL_CALL_SOCKET is intentionally NOT exported: the path is fixed at - * /tmp/tcs.sock and the blocking-mode runtime preamble references it - * as a literal. - * Skipping the env entry shrinks the surface user code can introspect - * (`os.environ`). The per-job /tmp/tcs.sock bind-mount above is what - * actually makes the path connectable, and ordinary execute/replay jobs - * do not receive that mount. */ - - args.push('--'); - args.push('/usr/local/bin/spec-guard', ...command); + args.push('--', '/usr/local/bin/spec-guard'); + if (enableToolCallPipes === true) args.push('--tool-call-pipes'); + args.push(...command); return args; } diff --git a/api/src/session-inputs.test.ts b/api/src/session-inputs.test.ts index 639ccadd..6efdb610 100644 --- a/api/src/session-inputs.test.ts +++ b/api/src/session-inputs.test.ts @@ -151,6 +151,7 @@ describe('pushed input cache', () => { const readOnly = cachedInputResponse((await openCachedInput('s1', 'ro'))!); expect(readOnly.headers.get('x-read-only')).toBe('true'); + await readOnly.text(); // Consume the response so its FileHandle is closed. }); test('an opened cache hit remains readable when concurrent pruning unlinks its path', async () => { diff --git a/api/src/spec-guard.c b/api/src/spec-guard.c index 31f153f9..4ce56de4 100644 --- a/api/src/spec-guard.c +++ b/api/src/spec-guard.c @@ -34,8 +34,10 @@ # endif #endif +static int first_untrusted_fd = 3; + static void close_fd_loop(long start, long end) { - if (start < 3) start = 3; + if (start < first_untrusted_fd) start = first_untrusted_fd; if (end > INT_MAX) end = INT_MAX; if (start > end) return; for (long fd = start; fd <= end; fd++) { @@ -54,7 +56,7 @@ static bool close_proc_fds(void) { char *end = NULL; long fd = strtol(entry->d_name, &end, 10); if (end == entry->d_name || *end != '\0') continue; - if (fd < 3 || fd > (long)INT_MAX) continue; + if (fd < first_untrusted_fd || fd > (long)INT_MAX) continue; if ((int)fd == dir_fd) continue; /* don't close our iterator */ close((int)fd); } @@ -62,7 +64,8 @@ static bool close_proc_fds(void) { return true; } -/* Close every file descriptor >= 3 (preserve stdin/stdout/stderr) before +/* Close every unrelated descriptor (preserve stdin/stdout/stderr and, + * only with --tool-call-pipes, the validated pipe endpoints 3/4) before * the user command runs. The bug this prevents: * * (1) sandbox API / proxy / NsJail allocate FDs in the runner; @@ -86,7 +89,7 @@ static bool close_proc_fds(void) { */ static void close_inherited_fds(void) { #if defined(__NR_close_range) && !defined(SPEC_GUARD_TEST_DISABLE_CLOSE_RANGE) - long rc = syscall(__NR_close_range, (unsigned int)3, ~0U, 0u); + long rc = syscall(__NR_close_range, (unsigned int)first_untrusted_fd, ~0U, 0u); if (rc == 0) return; /* ENOSYS on pre-5.9 kernels, EINVAL on some odd builds. Fall through. */ #endif @@ -123,9 +126,24 @@ int main(int argc, char *argv[]) { umask(0077); + int command = 1; + if (strcmp(argv[1], "--tool-call-pipes") == 0) { + struct stat writer, reader; + int write_flags = fcntl(3, F_GETFL), read_flags = fcntl(4, F_GETFL); + if (argc < 3 || fstat(3, &writer) != 0 || fstat(4, &reader) != 0 + || !S_ISFIFO(writer.st_mode) || !S_ISFIFO(reader.st_mode) + || write_flags < 0 || read_flags < 0 + || (write_flags & O_ACCMODE) != O_WRONLY + || (read_flags & O_ACCMODE) != O_RDONLY) { + fprintf(stderr, "invalid tool-call pipes\n"); + return 1; + } + first_untrusted_fd = 5; + command = 2; + } close_inherited_fds(); - execvp(argv[1], &argv[1]); + execvp(argv[command], &argv[command]); perror("exec"); return 1; } diff --git a/api/src/tool-call-pipe-bridge.c b/api/src/tool-call-pipe-bridge.c new file mode 100644 index 00000000..a3e9f084 --- /dev/null +++ b/api/src/tool-call-pipe-bridge.c @@ -0,0 +1,123 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif + +/* Node/Bun extra stdio uses socketpairs. Keep that trusted-side socket in + * this relay; only two anonymous pipe ends ever reach NsJail/the job. + * FD 3 in the job writes requests; FD 4 reads responses. */ +struct flow { + int source, destination; + unsigned char bytes[65536]; + size_t start, length; +}; + +static int nonblock(int fd) { + int flags = fcntl(fd, F_GETFL); + return flags < 0 ? -1 : fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +static int transfer(struct flow *flow, short readable, short writable) { + if (writable && flow->length) { + ssize_t n = write(flow->destination, flow->bytes + flow->start, flow->length); + if (n > 0) { + flow->start += (size_t)n; + flow->length -= (size_t)n; + if (!flow->length) flow->start = 0; + } else if (n < 0 && errno != EINTR && errno != EAGAIN) return -1; + } + if (flow->source >= 0 && readable && !flow->length) { + ssize_t n = read(flow->source, flow->bytes, sizeof(flow->bytes)); + if (n > 0) flow->length = (size_t)n; + else if (n == 0) flow->source = -1; + else if (errno != EINTR && errno != EAGAIN) return -1; + } + return 0; +} + +static int parent_death(pid_t parent) { +#ifdef __linux__ + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) return -1; +#endif + return getppid() == parent ? 0 : -1; +} + +int main(int argc, char **argv) { + if (argc < 2) return 125; + pid_t broker = getppid(); + if (parent_death(broker) != 0) return 125; + /* The API also enters here to exec Node with a parent-death signal. + * Without this, an API crash could orphan a still-running broker. */ + if (strcmp(argv[1], "--broker") == 0) { + if (argc < 3) return 125; + execvp(argv[2], &argv[2]); + perror("tool-call broker exec"); + return 125; + } + int ipc = fcntl(3, F_DUPFD_CLOEXEC, 5); + int requests[2], responses[2]; + if (ipc < 0 || pipe(requests) != 0 || pipe(responses) != 0) return 125; + pid_t relay = getpid(); + pid_t child = fork(); + if (child < 0) return 125; + if (child == 0) { + if (parent_death(relay) != 0 || dup2(requests[1], 3) < 0 + || dup2(responses[0], 4) < 0) _exit(125); + int fds[] = {ipc, requests[0], requests[1], responses[0], responses[1]}; + for (unsigned i = 0; i < sizeof(fds) / sizeof(fds[0]); i++) { + if (fds[i] > 4) close(fds[i]); + } + execvp(argv[1], &argv[1]); + perror("tool-call pipe exec"); + _exit(125); + } + close(3); + close(requests[1]); + close(responses[0]); + signal(SIGPIPE, SIG_IGN); + if (nonblock(ipc) < 0 || nonblock(requests[0]) < 0 || nonblock(responses[1]) < 0) { + kill(child, SIGKILL); + waitpid(child, NULL, 0); + return 125; + } + struct flow request = {.source = requests[0], .destination = ipc}; + struct flow response = {.source = ipc, .destination = responses[1]}; + int status = 125 << 8; + for (;;) { + pid_t done = waitpid(child, &status, WNOHANG); + if (done == child) break; + if (done < 0 && errno != EINTR) return 125; + struct pollfd pollfds[] = { + {.fd = ipc, .events = (response.length || response.source < 0 ? 0 : POLLIN) | (request.length ? POLLOUT : 0)}, + {.fd = request.source, .events = request.length ? 0 : POLLIN}, + {.fd = responses[1], .events = response.length ? POLLOUT : 0}, + }; + int result = poll(pollfds, 3, 100); + if (result < 0 && errno == EINTR) continue; + if (result < 0 || (pollfds[0].revents & (POLLERR | POLLNVAL)) + || (pollfds[1].revents & (POLLERR | POLLNVAL)) + || (pollfds[2].revents & (POLLERR | POLLHUP | POLLNVAL)) + || transfer(&request, pollfds[1].revents & (POLLIN | POLLHUP), pollfds[0].revents & POLLOUT) < 0 + || transfer(&response, pollfds[0].revents & (POLLIN | POLLHUP), pollfds[2].revents & POLLOUT) < 0 || response.source < 0) { + /* EOF/error terminates the entire invocation, including an + * abandoned tool call. Never leave an orphaned NsJail monitor. */ + if (waitpid(child, &status, WNOHANG) == child) break; + kill(child, SIGKILL); + while (waitpid(child, &status, 0) < 0 && errno == EINTR) {} + break; + } + } + close(ipc); + close(requests[0]); + close(responses[1]); + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); +} diff --git a/api/src/tool-call-pipe-broker.test.ts b/api/src/tool-call-pipe-broker.test.ts new file mode 100644 index 00000000..a55c6c33 --- /dev/null +++ b/api/src/tool-call-pipe-broker.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, test, spyOn } from 'bun:test'; +import * as http from 'node:http'; +import * as net from 'node:net'; +import { once } from 'node:events'; +import { MAX_FRAME_BYTES, serveToolCallPipe } from './tool-call-pipe-broker'; + +const cleanups: (() => void)[] = []; +afterEach(() => { for (const close of cleanups.splice(0).reverse()) close(); }); + +async function setup(handler: http.RequestListener, options = {}, target?: string): Promise<{ + client: net.Socket; replies: () => Promise; send: (value: unknown) => void; +}> { + const upstream = http.createServer(handler); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + cleanups.push(() => { upstream.closeAllConnections(); upstream.close(); }); + const server = net.createServer(channel => { + cleanups.push(serveToolCallPipe(channel, target ?? `http://127.0.0.1:${(upstream.address() as net.AddressInfo).port}`, options)); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + cleanups.push(() => server.close()); + const client = net.createConnection((server.address() as net.AddressInfo).port, '127.0.0.1'); + await once(client, 'connect'); + cleanups.push(() => client.destroy()); + let buffered = Buffer.alloc(0); + const queued: any[] = []; + const waiters: ((response: any) => void)[] = []; + client.on('data', bytes => { + buffered = Buffer.concat([buffered, typeof bytes === 'string' ? Buffer.from(bytes) : bytes]); + while (buffered.length >= 4 && buffered.length >= 4 + buffered.readUInt32BE()) { + const size = buffered.readUInt32BE(); + const value = JSON.parse(buffered.subarray(4, 4 + size).toString()); + buffered = buffered.subarray(4 + size); + const waiter = waiters.shift(); + if (waiter) waiter(value); else queued.push(value); + } + }); + return { client, replies: () => queued.length ? Promise.resolve(queued.shift()) : new Promise(resolve => waiters.push(resolve)), + send: value => { const frame = Buffer.from(JSON.stringify(value)); const prefix = Buffer.alloc(4); prefix.writeUInt32BE(frame.length); client.write(Buffer.concat([prefix, frame])); } }; +} + +const claims = { 'x-execution-id': 'exec', 'x-callback-token': 'token', 'x-tool-call-id': 'call' }; +const frame = (id: number) => ({ id, headers: claims, body: '{"tool_name":"echo","input":{}}' }); + +describe('per-invocation tool-call pipe broker', () => { + test('forwards only its fixed route and whitelisted claims, preserving JSON errors', async () => { + let observed = false; + const harness = await setup((req, res) => { + observed = true; + expect(req.url).toBe('/tool-call'); + expect(req.method).toBe('POST'); + expect(req.headers['x-callback-token']).toBe('token'); + expect(req.headers['x-forged']).toBeUndefined(); + req.resume(); + res.writeHead(404).end('{"success":false,"error":"Session not found"}'); + }); + harness.send({ ...frame(7), path: '/admin', headers: { ...claims, 'x-forged': 'secret' } }); + expect(await harness.replies()).toEqual({ id: 7, status: 404, body: '{"success":false,"error":"Session not found"}' }); + expect(observed).toBe(true); + }); + + test('isolates identical frame IDs across invocation channels', async () => { + const first = await setup((_req, res) => res.end('first')); + const second = await setup((_req, res) => res.end('second')); + first.send(frame(1)); + second.send(frame(1)); + const replies = await Promise.all([first.replies(), second.replies()]); + expect(replies.map(reply => reply.body)).toEqual(['first', 'second']); + }); + + test('rejects missing claims before opening an upstream connection', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.end(); }); + harness.send({ ...frame(1), headers: { ...claims, 'x-callback-token': ', , ' } }); + expect((await harness.replies()).status).toBe(400); + expect(count).toBe(0); + }); + + test('returns redirects without following them', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.writeHead(302, { Location: '/credential-sink' }).end('redirect'); }); + harness.send(frame(1)); + expect((await harness.replies()).status).toBe(302); + expect(count).toBe(1); + }); + + test('bounds outstanding upstream calls and aborts them when the channel closes', async () => { + let upstreamClosed = false; + let received: () => void; + const started = new Promise(resolve => { received = resolve; }); + // Observe the actual TCP peer: Bun 1.3.14's HTTP IncomingMessage socket + // does not emit close when an unfinished response loses its connection. + const upstream = net.createServer(socket => { + socket.once('close', () => { upstreamClosed = true; }); + socket.once('data', () => received()); + socket.resume(); + }); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + cleanups.push(() => upstream.close()); + const harness = await setup(() => {}, { maxActiveRequests: 1 }, + `http://127.0.0.1:${(upstream.address() as net.AddressInfo).port}`); + harness.send(frame(1)); + await started; + harness.send(frame(2)); + expect((await harness.replies()).status).toBe(429); + harness.client.destroy(); + for (let n = 0; n < 100 && !upstreamClosed; n++) await Bun.sleep(5); + expect(upstreamClosed).toBe(true); + }); + + test.each(['invalid claims', 'invalid body', 'active limit'])('closes a drained channel when %s exhausts the frame budget', async kind => { + const clock = spyOn(Date, 'now').mockReturnValue(1_000); + try { + let count = 0; + const harness = await setup(req => { count++; req.resume(); }, { maxActiveRequests: 1 }); + const ended = once(harness.client, 'close'); + for (let id = 1; id <= 65; id++) { + harness.send(kind === 'invalid claims' ? { ...frame(id), headers: {} } + : kind === 'invalid body' ? { ...frame(id), body: null } : frame(id)); + } + await ended; // setup drains replies continuously, so backpressure cannot hide the bypass. + expect(count).toBeLessThanOrEqual(kind === 'active limit' ? 1 : 0); + } finally { clock.mockRestore(); } + }); + + test('enforces an absolute deadline on a stalled upstream', async () => { + const harness = await setup(req => req.resume(), { requestTimeoutMs: 30 }); + harness.send(frame(1)); + expect((await harness.replies()).status).toBe(504); + }); + + test('rejects oversized upstream responses without buffering an unbounded body', async () => { + const harness = await setup((_req, res) => res.end(Buffer.alloc(MAX_FRAME_BYTES + 1, 'x'))); + harness.send(frame(1)); + const response = await harness.replies(); + expect(response.status).toBe(502); + expect(response.body).toContain('too large'); + }); + + test('closes an oversized frame before any upstream request', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.end(); }); + const ended = once(harness.client, 'close'); + const prefix = Buffer.alloc(4); prefix.writeUInt32BE(MAX_FRAME_BYTES + 1); + harness.client.write(prefix); + await ended; + expect(count).toBe(0); + }); + + test('bounds a partial-header slow writer', async () => { + const harness = await setup((_req, res) => res.end(), { frameTimeoutMs: 30 }); + const ended = once(harness.client, 'close'); + harness.client.write(Buffer.from([0])); + await ended; + }); +}); diff --git a/api/src/tool-call-pipe-broker.ts b/api/src/tool-call-pipe-broker.ts new file mode 100644 index 00000000..4fb923cc --- /dev/null +++ b/api/src/tool-call-pipe-broker.ts @@ -0,0 +1,165 @@ +import { spawn } from 'node:child_process'; +import type { Duplex } from 'node:stream'; +import * as http from 'node:http'; +import * as https from 'node:https'; + +export const MAX_FRAME_BYTES = 1024 * 1024; +const HEADER_NAMES = ['x-execution-id', 'x-callback-token', 'x-tool-call-id'] as const; + +/** One broker/channel per authorized invocation. The target and route come + * from trusted runner configuration; the job supplies only PTC claims/body. + * The upstream still authenticates those claims and enforces tool budgets. */ +export function serveToolCallPipe(channel: Duplex, rawTarget: string, options: { + requestTimeoutMs?: number; frameTimeoutMs?: number; maxActiveRequests?: number; + onFailure?: () => void; +} = {}): () => void { + const target = new URL(rawTarget.includes('://') ? rawTarget : `http://${rawTarget}`); + if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) { + throw new Error('invalid tool-call broker target'); + } + const transport = target.protocol === 'https:' ? https : http; + const requests = new Map(); + const timeout = options.requestTimeoutMs ?? 300_000; + const frameTimeout = options.frameTimeoutMs ?? 5_000; + const maxActive = options.maxActiveRequests ?? 16; + let closed = false; + let frameTimer: ReturnType | undefined; + let drainTimer: ReturnType | undefined; + let header = Buffer.alloc(4); + let offset = 0; + let frame: Buffer | undefined; + let tokens = 64; + let lastRefill = Date.now(); + const close = (): void => { + if (closed) return; + closed = true; + clearTimeout(frameTimer); + clearTimeout(drainTimer); + for (const request of requests.values()) request.destroy(); + requests.clear(); + channel.destroy(); + }; + const fail = (): void => { close(); options.onFailure?.(); }; + const reply = (id: number, status: number, body: string): void => { + if (closed) return; + let bytes = Buffer.from(JSON.stringify({ id, status, body })); + if (bytes.length > MAX_FRAME_BYTES) { + bytes = Buffer.from(JSON.stringify({ id, status: 502, body: JSON.stringify({ success: false, error: 'tool-call response too large' }) })); + } + const prefix = Buffer.alloc(4); + prefix.writeUInt32BE(bytes.length); + if (!channel.write(Buffer.concat([prefix, bytes]))) { + channel.pause(); + drainTimer ??= setTimeout(fail, frameTimeout); + } + }; + const reject = (id: number, status: number, error: string): void => reply(id, status, JSON.stringify({ success: false, error })); + const accept = (bytes: Buffer): void => { + // Charge every complete frame before JSON/claim validation, including + // rejected concurrency requests. Exhaustion closes the capability rather + // than generating an unlimited stream of cheap 429/400 replies. + const now = Date.now(); + tokens = Math.min(64, tokens + Math.max(0, now - lastRefill) * 0.02); + lastRefill = now; + if (tokens < 1) { fail(); return; } + tokens -= 1; + let parsed: { id?: unknown; headers?: unknown; body?: unknown }; + try { parsed = JSON.parse(bytes.toString('utf8')); } catch { fail(); return; } + if (!parsed || typeof parsed !== 'object' || !Number.isSafeInteger(parsed.id) + || (parsed.id as number) < 1 || requests.has(parsed.id as number)) { fail(); return; } + const id = parsed.id as number; + if (!parsed.headers || typeof parsed.headers !== 'object' || Array.isArray(parsed.headers) || typeof parsed.body !== 'string') { + reject(id, 400, 'invalid tool-call frame'); return; + } + const headers: http.OutgoingHttpHeaders = { 'content-type': 'application/json', connection: 'close' }; + for (const name of HEADER_NAMES) { + const value = (parsed.headers as Record)[name]; + if (typeof value !== 'string' || !value.replace(/[,\s]/g, '') || value.length > 8192 || /[^\x20-\x7e]/.test(value)) { + reject(id, 400, 'missing or invalid tool-call claims'); return; + } + headers[name] = value; + } + if (requests.size >= maxActive) { reject(id, 429, 'tool-call request budget exceeded'); return; } + const body = Buffer.from(parsed.body); + headers['content-length'] = body.length; + let deadline: ReturnType; + let finished = false; + const finish = (status: number, response: string): void => { + if (finished) return; + finished = true; + clearTimeout(deadline); + requests.delete(id); + reply(id, status, response); + }; + const upstream = transport.request({ protocol: target.protocol, hostname: target.hostname, + port: target.port || undefined, method: 'POST', path: '/tool-call', headers, agent: false }, response => { + const chunks: Buffer[] = []; + let size = 0; + response.on('data', (chunk: Buffer) => { + size += chunk.length; + if (size > MAX_FRAME_BYTES) { + finish(502, JSON.stringify({ success: false, error: 'tool-call response too large' })); + upstream.destroy(); + } else chunks.push(chunk); + }); + response.on('end', () => finish(response.statusCode ?? 502, Buffer.concat(chunks).toString('utf8'))); + response.on('error', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream disconnected' }))); + response.on('aborted', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream disconnected' }))); + }); + requests.set(id, upstream); + deadline = setTimeout(() => { + finish(504, JSON.stringify({ success: false, error: 'tool-call upstream timeout' })); + upstream.destroy(); + }, timeout); + upstream.on('error', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream unavailable' }))); + upstream.end(body); + }; + channel.on('data', (chunk: Buffer) => { + let cursor = 0; + while (cursor < chunk.length && !closed) { + frameTimer ??= setTimeout(fail, frameTimeout); + const buffer = frame ?? header; + const count = Math.min(buffer.length - offset, chunk.length - cursor); + chunk.copy(buffer, offset, cursor, cursor + count); + offset += count; + cursor += count; + if (offset !== buffer.length) continue; + offset = 0; + if (!frame) { + const length = header.readUInt32BE(); + if (length < 1 || length > MAX_FRAME_BYTES) { fail(); break; } + frame = Buffer.allocUnsafe(length); + } else { + const complete = frame; + frame = undefined; + clearTimeout(frameTimer); + frameTimer = undefined; + accept(complete); + } + } + }); + channel.on('drain', () => { clearTimeout(drainTimer); drainTimer = undefined; channel.resume(); }); + channel.on('error', fail); + channel.on('end', fail); + channel.on('close', close); + return close; +} + +// Node starts only after restore, on an authenticated job's execution path. +// Never create a persistent listener or snapshot Node's TLS process state. +if (require.main === module) { + const target = process.env.SANDBOX_FORWARD_TARGET?.trim(); + if (!target || !process.argv[2]) throw new Error('tool-call pipe broker is not configured'); + const child = spawn(process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge', process.argv.slice(2), { + stdio: [0, 1, 2, 'pipe'], + }); + const channel = child.stdio[3] as Duplex; + const stop = serveToolCallPipe(channel, target, { + requestTimeoutMs: Math.min(600_000, Math.max(5_000, Number(process.env.TCS_REQUEST_TIMEOUT_MS) || 300_000)), + onFailure: () => child.kill('SIGKILL'), + }); + child.on('error', () => { stop(); process.exitCode = 125; }); + child.on('exit', (code, signal) => { stop(); process.exitCode = code ?? (signal === 'SIGKILL' ? 137 : 125); }); + process.once('SIGTERM', () => { stop(); child.kill('SIGKILL'); }); + process.once('SIGINT', () => { stop(); child.kill('SIGKILL'); }); +} diff --git a/api/src/tool-call-socket-process.test.ts b/api/src/tool-call-socket-process.test.ts deleted file mode 100644 index 3efe2a4b..00000000 --- a/api/src/tool-call-socket-process.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import * as fsp from 'fs/promises'; -import * as os from 'os'; -import * as path from 'path'; -import { config } from './config'; -import { - ensureToolCallSocketProxyReady, - stopToolCallSocketProxy, -} from './tool-call-socket-process'; - -const originalPort = config.allowed_local_network_port; -const originalEnv = { - target: process.env.SANDBOX_FORWARD_TARGET, - socket: process.env.TCS_SOCKET, - bundle: process.env.TCS_PROXY_BUNDLE, - node: process.env.TCS_NODE_BINARY, - marker: process.env.TCS_TEST_MARKER, -}; -let tempDir: string | undefined; - -function restoreEnv(name: string, value: string | undefined): void { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; -} - -afterEach(async () => { - await stopToolCallSocketProxy(); - config.allowed_local_network_port = originalPort; - restoreEnv('SANDBOX_FORWARD_TARGET', originalEnv.target); - restoreEnv('TCS_SOCKET', originalEnv.socket); - restoreEnv('TCS_PROXY_BUNDLE', originalEnv.bundle); - restoreEnv('TCS_NODE_BINARY', originalEnv.node); - restoreEnv('TCS_TEST_MARKER', originalEnv.marker); - if (tempDir) await fsp.rm(tempDir, { recursive: true, force: true }); - tempDir = undefined; -}); - -describe('lazy tool-call socket process', () => { - test('starts once on demand, awaits readiness, and stops cleanly', async () => { - tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'lazy-tcs-')); - const socketPath = path.join(tempDir, 'proxy.sock'); - const markerPath = path.join(tempDir, 'launches.txt'); - const bundlePath = path.join(tempDir, 'fake-proxy.cjs'); - await fsp.writeFile(bundlePath, ` - const fs = require('fs'); - const net = require('net'); - try { fs.unlinkSync(process.env.TCS_SOCKET); } catch {} - fs.appendFileSync(process.env.TCS_TEST_MARKER, 'launch\\n'); - const server = net.createServer(socket => socket.end()); - server.listen(process.env.TCS_SOCKET); - const stop = () => server.close(() => process.exit(0)); - process.on('SIGTERM', stop); - process.on('SIGINT', stop); - `); - - config.allowed_local_network_port = 443; - process.env.SANDBOX_FORWARD_TARGET = 'https://gateway.example'; - process.env.TCS_SOCKET = socketPath; - process.env.TCS_PROXY_BUNDLE = bundlePath; - process.env.TCS_TEST_MARKER = markerPath; - - await Promise.all([ - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ]); - - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - expect((await fsp.readFile(markerPath, 'utf8')).trim().split('\n')).toHaveLength(1); - - /* A live child can lose its socket pathname (for example after an external - * cleanup). A previously-resolved launch promise must not mask that loss. */ - await fsp.unlink(socketPath); - await Promise.all([ - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ]); - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - expect((await fsp.readFile(markerPath, 'utf8')).trim().split('\n')).toHaveLength(2); - - await stopToolCallSocketProxy(); - expect(await fsp.lstat(socketPath).catch(() => null)).toBeNull(); - }); - - test('fails closed when forwarding is not configured', async () => { - config.allowed_local_network_port = 0; - process.env.SANDBOX_FORWARD_TARGET = ''; - await expect(ensureToolCallSocketProxyReady()).rejects.toThrow('not configured'); - }); - - test('reports a spawn failure without crashing and permits a later retry', async () => { - tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'lazy-tcs-spawn-')); - const socketPath = path.join(tempDir, 'proxy.sock'); - const bundlePath = path.join(tempDir, 'fake-proxy.cjs'); - await fsp.writeFile(bundlePath, ` - const fs = require('fs'); - const net = require('net'); - try { fs.unlinkSync(process.env.TCS_SOCKET); } catch {} - const server = net.createServer(socket => socket.end()); - server.listen(process.env.TCS_SOCKET); - process.on('SIGTERM', () => server.close(() => process.exit(0))); - `); - - config.allowed_local_network_port = 443; - process.env.SANDBOX_FORWARD_TARGET = 'https://gateway.example'; - process.env.TCS_SOCKET = socketPath; - process.env.TCS_PROXY_BUNDLE = bundlePath; - process.env.TCS_NODE_BINARY = path.join(tempDir, 'missing-node'); - - await expect(ensureToolCallSocketProxyReady()).rejects.toThrow( - 'tool-call socket proxy failed to spawn', - ); - - process.env.TCS_NODE_BINARY = process.execPath; - await ensureToolCallSocketProxyReady(); - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - }); -}); diff --git a/api/src/tool-call-socket-process.ts b/api/src/tool-call-socket-process.ts deleted file mode 100644 index 4694c07f..00000000 --- a/api/src/tool-call-socket-process.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { spawn, type ChildProcess } from 'child_process'; -import * as net from 'net'; -import * as path from 'path'; -import { logger } from './logger'; -import { config } from './config'; - -const START_TIMEOUT_MS = 10_000; -const STOP_TIMEOUT_MS = 3_000; - -let child: ChildProcess | undefined; -let starting: Promise | undefined; - -function proxyBundlePath(): string { - return process.env.TCS_PROXY_BUNDLE - ?? path.resolve(__dirname, '..', '.build', 'tool-call-socket-proxy.cjs'); -} - -function proxySocketPath(): string { - return process.env.TCS_SOCKET || '/tmp/tcs.sock'; -} - -function proxyOwner(): { uid: string; gid: string } { - const runningAsRoot = typeof process.getuid === 'function' && process.getuid() === 0; - return { - uid: String(runningAsRoot ? 65534 : (process.getuid?.() ?? 65534)), - gid: String(runningAsRoot ? 65534 : (process.getgid?.() ?? 65534)), - }; -} - -function socketAcceptsConnections(socketPath: string): Promise { - return new Promise((resolve) => { - const socket = net.createConnection(socketPath); - let settled = false; - const finish = (ready: boolean): void => { - if (settled) return; - settled = true; - socket.destroy(); - resolve(ready); - }; - socket.once('connect', () => finish(true)); - socket.once('error', () => finish(false)); - socket.setTimeout(100, () => finish(false)); - }); -} - -async function waitUntilReady( - started: ChildProcess, - socketPath: string, - timeoutMs: number, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (started.exitCode !== null || started.signalCode !== null) { - throw new Error( - `tool-call socket proxy exited before readiness` - + ` (code=${String(started.exitCode)}, signal=${String(started.signalCode)})`, - ); - } - if (await socketAcceptsConnections(socketPath)) return; - await new Promise(resolve => setTimeout(resolve, 25)); - } - throw new Error(`tool-call socket proxy did not become ready within ${timeoutMs}ms`); -} - -async function launch(): Promise { - const rawTarget = process.env.SANDBOX_FORWARD_TARGET?.trim() ?? ''; - if (config.allowed_local_network_port <= 0 || !rawTarget) { - throw new Error('tool-call socket proxy is not configured'); - } - - const socketPath = proxySocketPath(); - const owner = proxyOwner(); - const started = spawn(process.env.TCS_NODE_BINARY || 'node', [proxyBundlePath()], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - TCS_SOCKET_UID: owner.uid, - TCS_SOCKET_GID: owner.gid, - SANDBOX_FORWARD_TARGET: rawTarget, - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - child = started; - let rejectSpawnFailure: (error: Error) => void; - const spawnFailure = new Promise((_resolve, reject) => { - rejectSpawnFailure = reject; - }); - /* A failed spawn emits `error`; it does not reliably set exitCode or - * signalCode. Without an immediate listener Node treats ENOENT/EACCES as an - * uncaught event and crashes the runner instead of failing this execution. */ - started.on('error', (error) => { - if (child === started) child = undefined; - const launchError = new Error( - `tool-call socket proxy failed to spawn: ${error.message}`, - { cause: error }, - ); - logger.error({ error }, 'tool-call socket proxy spawn failed'); - rejectSpawnFailure(launchError); - }); - started.stdout?.on('data', (chunk: Buffer) => { - logger.debug({ proxy: chunk.toString().trim() }, 'tool-call socket proxy'); - }); - started.stderr?.on('data', (chunk: Buffer) => { - logger.warn({ proxy: chunk.toString().trim() }, 'tool-call socket proxy stderr'); - }); - started.once('exit', (code, signal) => { - if (child === started) { - child = undefined; - } - if (code !== 0 && signal !== 'SIGTERM') { - logger.error({ code, signal }, 'tool-call socket proxy exited unexpectedly'); - } - }); - - try { - await Promise.race([ - waitUntilReady(started, socketPath, START_TIMEOUT_MS), - spawnFailure, - ]); - logger.info( - { socketPath, target: rawTarget }, - 'Tool-call socket proxy started after MicroVM restore', - ); - } catch (error) { - started.kill('SIGKILL'); - if (child === started) child = undefined; - throw error; - } -} - -/** - * Starts Node only when an authenticated execute actually receives the - * tool-call socket capability. Lambda MicroVM image creation snapshots the - * already-running container; starting Node in entrypoint would therefore clone - * its embedded OpenSSL process state into every VM. Lazy initialization keeps - * Node entirely on the post-restore side of that boundary. - */ -async function ensureToolCallSocketProxyReadyOnce(): Promise { - const socketPath = proxySocketPath(); - const running = child; - if (running && running.exitCode === null && running.signalCode === null) { - if (await socketAcceptsConnections(socketPath)) return; - running.kill('SIGKILL'); - if (child === running) child = undefined; - } - await launch(); -} - -export async function ensureToolCallSocketProxyReady(): Promise { - /* Serialize the whole probe/relaunch transaction, not only spawn readiness. - * Otherwise concurrent recovery callers can both await a failed socket probe, - * then each kill/relaunch through the shared `child` slot. Publishing this - * promise before the first probe await makes every follower join one attempt. */ - if (starting) { - await starting; - return; - } - const attempt = ensureToolCallSocketProxyReadyOnce(); - starting = attempt; - try { - await attempt; - } finally { - /* A successful launch must not leave a resolved promise cached forever: - * if the socket later disappears while the process is still alive, the - * next caller has to kill/relaunch rather than return false readiness. */ - if (starting === attempt) starting = undefined; - } -} - -export async function stopToolCallSocketProxy(): Promise { - const running = child; - child = undefined; - starting = undefined; - if (!running || running.exitCode !== null || running.signalCode !== null) return; - - await new Promise((resolve) => { - let settled = false; - const finish = (): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(); - }; - const timer = setTimeout(() => { - running.kill('SIGKILL'); - finish(); - }, STOP_TIMEOUT_MS); - running.once('exit', finish); - running.kill('SIGTERM'); - }); -} diff --git a/api/src/tool-call-socket-proxy-runtime.test.ts b/api/src/tool-call-socket-proxy-runtime.test.ts deleted file mode 100644 index 496f017b..00000000 --- a/api/src/tool-call-socket-proxy-runtime.test.ts +++ /dev/null @@ -1,905 +0,0 @@ -/** - * Runtime regression tests for the tool-call socket proxy. - * - * The tests in `tool-call-socket-proxy.test.ts` instantiate the proxy - * in-process via `startToolCallSocketProxy()` under bun:test. That is fine - * for verifying request-handler logic, but it CANNOT catch the audit DoS - * because Bun's node:http compat layer never fires `'connection'` events - * and Bun.serve's idleTimeout doesn't close silent unix-socket connects — - * so `activeConnections()`, `maxConnections`, and `idleSocketTimeoutMs` - * silently no-op when the proxy is hosted by Bun. The in-process tests - * pass vacuously (counters stay 0) while the production runtime is - * unprotected. - * - * In production the proxy runs under Node (see api/Dockerfile + the - * entrypoint.sh edit). These tests spawn the BUILT proxy bundle as a - * Node subprocess so the runtime under test matches production exactly. - * - * Skipped automatically when: - * - `node` isn't on PATH (CI without Node) - * - `.build/tool-call-socket-proxy.cjs` doesn't exist (test invoked - * before `bun run build`) - * - we're on Windows (unix sockets in temp dirs hit AppData EACCES - * quirks; production is Linux) - */ -import { afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; -import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; -import * as fs from 'node:fs'; -import * as fsp from 'node:fs/promises'; -import * as http from 'node:http'; -import * as net from 'node:net'; -import * as os from 'node:os'; -import * as path from 'node:path'; - -const PROXY_BUNDLE = path.resolve(__dirname, '..', '.build', 'tool-call-socket-proxy.cjs'); -const NODE_AVAILABLE = (() => { - try { return spawnSync('node', ['--version'], { stdio: 'ignore' }).status === 0; } - catch { return false; } -})(); -const SHOULD_RUN = NODE_AVAILABLE && fs.existsSync(PROXY_BUNDLE) && process.platform !== 'win32'; - -const describeIfRuntime = SHOULD_RUN ? describe : describe.skip; - -interface UpstreamHandle { - server: http.Server; - url: string; - calls: () => number; - close: () => Promise; -} - -interface ProxyHandle { - socketPath: string; - proc: ChildProcess; - stop: () => Promise; -} - -let workdir: string; - -beforeAll(() => { - if (!NODE_AVAILABLE) { - console.warn('[tcs-proxy-runtime] skipping: `node` not on PATH'); - } else if (!fs.existsSync(PROXY_BUNDLE)) { - console.warn(`[tcs-proxy-runtime] skipping: ${PROXY_BUNDLE} missing — run \`bun run build\` first`); - } else if (process.platform === 'win32') { - console.warn('[tcs-proxy-runtime] skipping: Windows (unix-socket EACCES quirk); production is Linux'); - } -}); - -beforeEach(async () => { - workdir = await fsp.mkdtemp(path.join(os.tmpdir(), 'tcs-proxy-runtime-')); -}); - -afterEach(async () => { - if (workdir) await fsp.rm(workdir, { recursive: true, force: true }); -}); - -async function startUpstream(handler?: http.RequestListener): Promise { - let calls = 0; - const server = http.createServer((req, res) => { - calls++; - if (handler) return handler(req, res); - let total = 0; - req.on('data', chunk => { total += chunk.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ ok: true, received: total })); - }); - }); - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(0, '127.0.0.1', resolve); - }); - const addr = server.address(); - if (!addr || typeof addr === 'string') throw new Error('upstream listen failed'); - return { - server, - url: `http://127.0.0.1:${addr.port}`, - calls: () => calls, - close: () => new Promise(r => server.close(() => r())), - }; -} - -async function spawnProxy(opts: { upstreamUrl: string; env?: Record }): Promise { - const socketPath = path.join(workdir, `tcs-${Math.random().toString(36).slice(2)}.sock`); - const proc = spawn('node', [PROXY_BUNDLE], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - SANDBOX_FORWARD_TARGET: opts.upstreamUrl, - ...(opts.env || {}), - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - /* Wait for the proxy to be connectable. fs.existsSync on a unix socket - * is unreliable on some platforms; try-connect-then-disconnect is the - * authoritative readiness probe. */ - let ready = false; - for (let i = 0; i < 50; i++) { - try { - const probe = net.createConnection(socketPath); - await new Promise((res, rej) => { - probe.once('connect', () => { probe.destroy(); res(); }); - probe.once('error', rej); - }); - ready = true; - break; - } catch { /* not yet */ } - await new Promise(r => setTimeout(r, 100)); - } - if (!ready) { - proc.kill('SIGKILL'); - throw new Error('proxy never became connectable'); - } - - return { - socketPath, - proc, - stop: async () => { - proc.kill('SIGTERM'); - await new Promise(r => { - const killTimer = setTimeout(() => { proc.kill('SIGKILL'); r(); }, 1000); - proc.once('exit', () => { - clearTimeout(killTimer); - r(); - }); - }); - }, - }; -} - -function rawConnect(socketPath: string): Promise { - return new Promise((resolve, reject) => { - const sock = net.createConnection(socketPath); - sock.once('connect', () => resolve(sock)); - sock.once('error', reject); - }); -} - -function slowHeaderConnect( - socketPath: string, - dripEveryMs: number, -): Promise<{ socket: net.Socket; close: () => void }> { - return new Promise((resolve, reject) => { - const sock = net.createConnection(socketPath); - let drip: ReturnType | undefined; - const clearDrip = (): void => { - if (drip) clearInterval(drip); - drip = undefined; - }; - const onInitialError = (error: Error): void => { - clearDrip(); - reject(error); - }; - - sock.once('error', onInitialError); - sock.once('connect', () => { - sock.off('error', onInitialError); - sock.on('error', () => { /* connection was destroyed by the proxy */ }); - sock.once('close', clearDrip); - sock.write('POST /tool-call HTTP/1.1\r\nHost: x\r\nX-Drip: '); - drip = setInterval(() => { - if (sock.destroyed) { - clearDrip(); - return; - } - try { sock.write('x'); } catch { clearDrip(); } - }, dripEveryMs); - resolve({ - socket: sock, - close: () => { - clearDrip(); - sock.destroy(); - }, - }); - }); - }); -} - -/* The proxy's PTC-presence filter rejects /tool-call requests that lack - * the SDK-supplied headers (canonical 404 — see proxy source for why). - * Tests that want to exercise the forwarding path supply matching shape - * here. Cryptographic validation is the upstream's job; presence is the - * proxy's. */ -const PTC_HEADERS = { - 'X-Execution-ID': 'test-exec-id', - 'X-Tool-Call-ID': 'test-call-001', - 'X-Callback-Token': 'test-callback-token', -} as const; - -function postToolCall( - socketPath: string, - body: string, - extraHeaders: Record = {}, -): Promise<{ status: number; body: string }> { - return new Promise((resolve, reject) => { - const req = http.request( - { - socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body), - ...PTC_HEADERS, - ...extraHeaders, - }, - }, - res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() })); - }, - ); - req.on('error', reject); - req.write(body); - req.end(); - }); -} - -describeIfRuntime('tool-call-socket-proxy under Node runtime (production parity)', () => { - test('forwards a normal POST /tool-call', async () => { - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 7 }); - expect(upstream.calls()).toBe(1); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('SILENT-CONNECTION DoS: idle accept-then-do-nothing connections close before they pin FDs', async () => { - /* This is the audit's actual attack: - * for i in range(N): socket.connect('/tmp/tcs.sock') - * Open many connections without writing anything. Without the Node - * runtime + working `'connection'` events + working socket.setTimeout, - * each accepted FD sits in the proxy until the OS limit. With them - * all working, every FD must be reclaimed within a small multiple of - * idleSocketTimeoutMs. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { /* defaults: idle 2s, max 64. Test against the production defaults. */ }, - }); - try { - const sockets: net.Socket[] = []; - for (let i = 0; i < 32; i++) { - try { sockets.push(await rawConnect(proxy.socketPath)); } - catch { /* over-cap rejection is fine — also "didn't pin a FD" */ } - } - /* Wait for idle timer (default 2000ms) + slack. */ - const start = Date.now(); - const closeTimes = await Promise.all(sockets.map(s => new Promise(resolve => { - if (s.destroyed) { resolve(0); return; } - s.once('close', () => resolve(Date.now() - start)); - setTimeout(() => resolve(-1), 8000); - }))); - const closed = closeTimes.filter(t => t >= 0).length; - const stuck = closeTimes.filter(t => t < 0).length; - expect(stuck).toBe(0); - /* All idle FDs must be reclaimed within a small multiple of the - * default idleSocketTimeoutMs (2s). Slack is generous. */ - const maxClose = Math.max(...closeTimes); - expect(maxClose).toBeLessThan(7000); - expect(closed).toBeGreaterThan(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('legitimate POST still succeeds after a silent-connection storm clears', async () => { - /* Regression guard for the actual user-facing failure: while - * malicious code spams idle connections, real tool calls from - * sibling sandboxes must still succeed. With the default cap (64) - * and idleSocketTimeoutMs=2s, we open the storm, wait ~2.5s for - * the proxy to reclaim those FDs, then verify a real POST works. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - const stormSockets: net.Socket[] = []; - try { - for (let i = 0; i < 32; i++) { - try { stormSockets.push(await rawConnect(proxy.socketPath)); } - catch { /* over-cap rejection is fine */ } - } - /* Let the proxy's idleSocketTimeoutMs (default 2s) reclaim FDs. */ - await new Promise(r => setTimeout(r, 2500)); - const res = await postToolCall(proxy.socketPath, '{"during":"storm"}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 18 }); - } finally { - stormSockets.forEach(s => { try { s.destroy(); } catch { /* ignore */ } }); - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: slow incomplete headers cannot starve all connection slots', async () => { - /* Header-drip clients are more subtle than silent sockets: they send - * bytes under the idle timeout, so socket.setTimeout never fires. The - * proxy must still reclaim each accepted socket on an absolute - * accept-to-headers deadline before all maxConnections slots can be held. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - TCS_MAX_CONNECTIONS: '4', - TCS_HEADER_TIMEOUT_MS: '500', - TCS_CONNECTION_RATE_BURST: '32', - TCS_CONNECTION_RATE_REFILL_PER_SEC: '200', - }, - }); - const drips: Array<{ socket: net.Socket; close: () => void }> = []; - - try { - for (let i = 0; i < 4; i++) { - drips.push(await slowHeaderConnect(proxy.socketPath, 100)); - } - - await new Promise(r => setTimeout(r, 900)); - - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 7 }); - expect(upstream.calls()).toBe(1); - } finally { - drips.forEach(drip => drip.close()); - await proxy.stop(); - await upstream.close(); - } - }, { timeout: 10_000 }); - - test('CIRCUMVENTION: HTTP request smuggling (Content-Length + Transfer-Encoding) does not reach upstream', async () => { - /* The classic CL.TE / TE.CL smuggling primitive: send both headers - * with conflicting interpretations. If the proxy and upstream - * disagree on body boundary, the attacker can prepend bytes to a - * synthesized "second request" that bypasses the path filter and - * could reach a different upstream route. The defense per RFC 7230 - * §3.3.3 is to reject the request rather than try to reconcile. - * Whether node:http rejects in its own parser or our isSmugglingShaped - * gate fires first, the security property is the same: upstream - * must never see the smuggled bytes. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - const smuggled = - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Length: 13\r\n' + - 'Transfer-Encoding: chunked\r\n' + - 'Connection: close\r\n' + - '\r\n' + - '0\r\n\r\n' + - 'POST /admin HTTP/1.1\r\nHost: x\r\nContent-Length: 0\r\n\r\n'; - sock.write(smuggled); - await new Promise(resolve => { - sock.once('close', () => resolve()); - setTimeout(resolve, 3000); - }); - /* Critical security property: upstream MUST NOT have been called. */ - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: hop-by-hop headers (Connection, Upgrade, TE, Trailer, Transfer-Encoding) do not reach upstream', async () => { - /* RFC 7230 §6.1: a proxy MUST NOT forward hop-by-hop headers. If we - * leak them through, an upstream that treats Upgrade specially could - * be coerced into an unexpected protocol switch, and Transfer- - * Encoding pollution feeds smuggling chains at the next hop. */ - let receivedHeaders: http.IncomingHttpHeaders = {}; - const upstream = await startUpstream((req, res) => { - receivedHeaders = req.headers; - let total = 0; - req.on('data', c => { total += c.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json', 'Connection': 'keep-alive' }); - res.end(JSON.stringify({ ok: true, received: total })); - }); - }); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const body = '{"x":1}'; - const reply = await new Promise<{ status: number; body: string; headers: http.IncomingHttpHeaders }>((resolve, reject) => { - const req = http.request({ - socketPath: proxy.socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body), - /* PTC presence required to forward (proxy filter). */ - ...PTC_HEADERS, - /* Hop-by-hop headers we expect the proxy to strip: */ - 'Connection': 'keep-alive, Upgrade-Insecure-Requests', - 'Keep-Alive': 'timeout=300, max=1000', - 'Upgrade': 'websocket', - 'TE': 'trailers', - 'Trailer': 'X-Foo', - 'Proxy-Authorization': 'Basic c3RlYWw=', - 'Proxy-Connection': 'close', - 'Host': 'attacker-controlled.example', - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - headers: res.headers, - })); - }); - req.on('error', reject); - req.write(body); - req.end(); - }); - expect(reply.status).toBe(200); - /* Upstream must not have seen the attacker's hop-by-hop values. - * The Connection header is special: node:http always sets one - * automatically (we tell it `connection: close`), so what we - * verify there is "the proxy's choice, not the attacker's". */ - expect(receivedHeaders.connection).toBe('close'); - expect(receivedHeaders.connection).not.toContain('Upgrade'); - for (const stripped of [ - 'keep-alive', 'upgrade', 'te', 'trailer', - 'transfer-encoding', 'proxy-authorization', 'proxy-connection', - ]) { - expect(receivedHeaders[stripped]).toBeUndefined(); - } - /* Host header must be the upstream's, not the attacker's value. */ - expect(receivedHeaders.host).not.toBe('attacker-controlled.example'); - /* And the upstream's hop-by-hop response headers must not reach - * the sandbox client (Connection: keep-alive from upstream gets - * stripped, our Connection: close wins). */ - expect(reply.headers.connection).toBe('close'); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: slow-loris body upload is bounded by requestBodyTimeoutMs (~5s default)', async () => { - /* Drip body bytes one at a time, each within the per-byte idle - * window. Without an absolute server.requestTimeout, this would - * stretch up to activeRequestTimeoutMs (default 30s) and let an - * attacker monopolize a slot. With server.requestTimeout = - * requestBodyTimeoutMs, the proxy must close us within ~5s. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - sock.write( - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Type: application/json\r\n' + - 'Content-Length: 100\r\n' + - 'Connection: close\r\n' + - /* PTC headers required so the proxy forwards (presence-only check; - * lets us reach the bodyUploadDeadline path the test exercises). */ - 'X-Execution-ID: test-exec-id\r\n' + - 'X-Tool-Call-ID: test-call-001\r\n' + - 'X-Callback-Token: test-callback-token\r\n' + - '\r\n', - ); - /* Drip 1 byte every 800ms. After 5s we've sent ~6 bytes — far short - * of the declared 100 — and the proxy must have closed us. */ - const start = Date.now(); - const dripInterval = setInterval(() => { - if (sock.destroyed) { clearInterval(dripInterval); return; } - try { sock.write('x'); } catch { clearInterval(dripInterval); } - }, 800); - const closedAt = await new Promise(resolve => { - sock.once('close', () => resolve(Date.now() - start)); - setTimeout(() => resolve(-1), 15000); - }); - clearInterval(dripInterval); - expect(closedAt).toBeGreaterThanOrEqual(0); - /* Bounded close — proves slow-loris is mitigated. Generous upper - * bound to absorb scheduler jitter; the point is "<<30s". */ - expect(closedAt).toBeLessThan(12000); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }, { timeout: 15_000 }); - - test('CIRCUMVENTION: HTTP/2 binary framing on a /tmp/tcs.sock connection fails clean (no crash)', async () => { - /* If the proxy crashed on unexpected protocol bytes, an attacker - * could brick it. Send raw HTTP/2 connection-preface bytes; the - * node:http parser must close the socket with clientError without - * affecting subsequent legitimate clients. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - /* HTTP/2 connection preface (24 bytes) per RFC 7540. */ - const h2Preface = Buffer.from('PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'); - for (let i = 0; i < 5; i++) { - const sock = await rawConnect(proxy.socketPath); - sock.write(h2Preface); - await new Promise(r => sock.once('close', () => r())); - } - /* Proxy must still serve a normal POST. */ - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: oversize request body is rejected with 413 before reaching upstream', async () => { - /* maxBodyBytes (default 1 MiB) bounds memory pressure. The - * Content-Length precheck closes the loophole where streaming the - * body to upstream would let the first chunks of an honestly- - * declared oversize request leak through. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { /* default maxBodyBytes */ }, - }); - try { - const big = 'A'.repeat(2 * 1024 * 1024); // 2 MiB body - const reply = await new Promise<{ status: number; body: string }>(resolve => { - const req = http.request({ - socketPath: proxy.socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Length': Buffer.byteLength(big), - 'Connection': 'close', - /* PTC presence so the proxy actually attempts to forward, - * letting the maxBodyBytes / Content-Length precheck fire. */ - ...PTC_HEADERS, - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - })); - }); - req.on('error', err => resolve({ status: 0, body: err.message })); - req.write(big); - req.end(); - }); - /* Either 413 (CL precheck or runtime byte counter) or transport - * error (proxy closed mid-stream). The security property is the - * same: NO bytes of the oversize body reach the upstream. */ - expect([413, 0]).toContain(reply.status); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('PTC presence filter: duplicate empty header lines do not bypass the gate', async () => { - /* Codex caught this: Node joins duplicate request headers with - * `, ` (per RFC 7230 §3.2.2). A naive `!headers[name]` check on - * `X-Execution-ID:` sent twice with empty values produces the - * truthy string `", "`, slips past the gate, and lets the - * attacker reach the upstream — whose structured auth-error - * response defeats the route-opacity goal. The proxy must - * normalise (strip commas + whitespace) before checking. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - /* Each required PTC header sent TWICE with empty values. Without - * the fix, Node's join produces `, ` for each name and the - * proxy forwards. With the fix, the values normalise to '' and - * the proxy returns its canonical 404. */ - sock.write( - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Length: 0\r\n' + - 'Connection: close\r\n' + - 'X-Execution-ID:\r\n' + - 'X-Execution-ID:\r\n' + - 'X-Tool-Call-ID:\r\n' + - 'X-Tool-Call-ID:\r\n' + - 'X-Callback-Token:\r\n' + - 'X-Callback-Token:\r\n' + - '\r\n', - ); - const reply = await new Promise(resolve => { - const chunks: Buffer[] = []; - sock.on('data', c => chunks.push(c)); - sock.on('close', () => resolve(Buffer.concat(chunks).toString())); - setTimeout(() => resolve(Buffer.concat(chunks).toString()), 3000); - }); - /* Critical: upstream MUST NOT have been called. Status MUST be - * 404. Body MUST be the canonical "not found". */ - expect(reply).toContain('404'); - expect(reply).toContain('not found'); - expect(reply).not.toMatch(/X-Request-ID|ETag/i); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('PTC presence filter: missing headers get canonical 404, never reach upstream', async () => { - /* The proxy filters on PTC-header presence before forwarding so a - * sandbox attacker probing /tool-call without SDK headers cannot - * fingerprint the route via the upstream's structured "missing PTC - * headers" response. The `bogus_headers` case proves cryptographic - * validation is still upstream's job — bogus values DO forward and - * upstream's structured error is preserved (so the SDK can parse it). */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - // No PTC headers → proxy 404, upstream NOT called - const noHeaders = await new Promise<{ status: number; body: string; ct?: string }>(resolve => { - const r = http.request({ socketPath: proxy.socketPath, method: 'POST', path: '/tool-call' }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - ct: res.headers['content-type'] as string | undefined, - })); - }); - r.on('error', err => resolve({ status: 0, body: err.message })); - r.end(); - }); - expect(noHeaders.status).toBe(404); - expect(noHeaders.body).toBe('not found'); - expect(noHeaders.ct).toContain('text/plain'); - const callsAfterNoHeaders = upstream.calls(); - expect(callsAfterNoHeaders).toBe(0); - - // With PTC headers (any values) → proxy forwards - const withHeaders = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(withHeaders.status).toBe(200); - expect(upstream.calls()).toBe(1); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: pipelined requests on a single socket only target /tool-call and respect active-request budget', async () => { - /* HTTP/1.1 pipelining lets a client batch requests on one socket. - * Both pipelined requests still go through the proxy's path filter - * and active-request budget — they cannot reach a different upstream - * route. node:http may parse and dispatch the second request - * before the first response's Connection: close fires; that is fine - * as long as the second is ALSO subject to all the proxy's defenses. */ - let receivedPaths: string[] = []; - const upstream = await startUpstream((req, res) => { - receivedPaths.push(req.url ?? ''); - let n = 0; req.on('data', c => { n += c.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ ok: true, received: n })); - }); - }); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - /* Second request tries to hit /admin (a path the proxy's filter - * MUST reject). If pipelining could bypass the filter, we'd see - * /admin in receivedPaths. */ - /* PTC headers on the first request so it actually forwards; the - * second request deliberately omits them AND targets /admin to - * verify both the path filter AND the presence filter hold even - * across pipelined dispatch. */ - const pipelined = - 'POST /tool-call HTTP/1.1\r\nHost: x\r\nContent-Length: 7\r\n' + - 'X-Execution-ID: t\r\nX-Tool-Call-ID: t\r\nX-Callback-Token: t\r\n\r\n{"x":1}' + - 'POST /admin HTTP/1.1\r\nHost: x\r\nContent-Length: 7\r\n\r\n{"y":2}'; - sock.write(pipelined); - await new Promise(r => sock.once('close', () => r())); - /* Whatever node:http chose to dispatch, the path filter must hold: - * upstream must NEVER see /admin. */ - expect(receivedPaths.every(p => p === '/tool-call')).toBe(true); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('ACTIVE-FLOOD DoS: excess concurrent /tool-call requests get 429 and do not reach upstream', async () => { - /* Hold the upstream so requests pile up at the proxy's active cap. - * With maxActiveRequests=2, only 2 requests should reach upstream; - * the others should get 429 + Retry-After. */ - let pendingUpstreams = 0; - let release: (() => void) | undefined; - const releaseAll = new Promise(r => { release = r; }); - const upstream = await startUpstream(async (req, res) => { - pendingUpstreams++; - req.resume(); - await new Promise(r => req.on('end', () => r())); - await releaseAll; - res.writeHead(200).end('{"ok":true}'); - }); - - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - SANDBOX_MAX_CONCURRENT_JOBS: '2', - }, - }); - - try { - const inflight = Array.from({ length: 32 }, () => - postToolCall(proxy.socketPath, '{"x":1}').catch(err => ({ status: -1, body: String(err) })), - ); - /* Give the proxy time to admit the first batch and 429 the rest. */ - await new Promise(r => setTimeout(r, 500)); - release!(); - const results = await Promise.all(inflight); - const ok = results.filter(r => r.status === 200).length; - const tooMany = results.filter(r => r.status === 429).length; - /* Active cap default-derives from SANDBOX_MAX_CONCURRENT_JOBS but is - * floored at 16. Launch above the floor so this test actually exercises - * the 429 branch instead of merely proving that 16 requests succeed. */ - expect(ok).toBeGreaterThan(0); - expect(ok).toBeLessThanOrEqual(16); - expect(tooMany).toBeGreaterThan(0); - expect(upstream.calls()).toBeLessThanOrEqual(16); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: SIGTERM during proxy init exits cleanly (no TypeError on undefined handle)', async () => { - /* Codex flagged that `shutdown()` could fall through to `handle.close()` - * when a signal arrives before `startToolCallSocketProxy()` resolves — - * which would crash with `TypeError: Cannot read properties of undefined`. - * Hold the socket path with a sentinel server so the real proxy's - * listen() blocks indefinitely (EADDRINUSE on a unix socket retries - * via Node's listen-with-existing-socket dance, but with another - * server actively listening the bind never resolves), then signal it. - * The proxy must exit code 0 (signal handler ran cleanly) — never - * exit on the SIGTERM signal directly (which would mean the handler - * never ran) and never exit non-zero (which would mean the handler - * ran but threw). */ - const socketPath = path.join(workdir, 'sigterm-init-race.sock'); - /* Pre-bind the socket path so the proxy's listen() hangs in EADDRINUSE - * retry loops, keeping `handle` undefined long enough to test the race. */ - const blocker = http.createServer(() => { /* noop */ }); - await new Promise(r => blocker.listen(socketPath, () => r())); - - const proc = spawn('node', [PROXY_BUNDLE], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - SANDBOX_FORWARD_TARGET: 'http://127.0.0.1:1', - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - /* Wait long enough for Node to install the SIGTERM handler but - * NOT long enough for startToolCallSocketProxy to have resolved - * (it can't — the socket is bound by `blocker`). 100ms is well - * above the ~10ms handler-install window observed in probes. */ - await new Promise(r => setTimeout(r, 100)); - proc.kill('SIGTERM'); - - const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(resolve => { - const t = setTimeout(() => { - proc.kill('SIGKILL'); - resolve({ code: -1, signal: null }); - }, 3000); - proc.once('exit', (code, signal) => { - clearTimeout(t); - resolve({ code, signal }); - }); - }); - - /* Critical assertion: handler ran AND exited cleanly. Exit code 0 is - * the only acceptable outcome — anything else (signal:'SIGTERM' meaning - * default behavior, code 1 meaning thrown error, code -1 meaning hung - * past 3s) indicates the race window is broken. */ - expect(exit.code).toBe(0); - expect(exit.signal).toBeNull(); - - await new Promise(r => blocker.close(() => r())); - }); - - test('CONNECT-FLOOD DoS: connection-rate limiter drops fast-burst accepts that would pressure kernel slab', async () => { - /* This is the SPECIFIC audit scenario PR #1648 follow-up was opened - * for: 500 AF_UNIX connect()s in <50 ms. Without the rate limiter, - * every accept allocates kernel unix_sock + dentry + skb queue slab - * entries; the proxy's existing maxConnections only bounds the - * application-level activeSockets set, not the allocation rate. - * Observation from the audit: in-VM kernel slab pressure surfaces - * as EMFILE in subsequent loader open() calls even though the fd - * table itself sits at 0.1% utilization. - * - * Test verifies the wire-level effect: with burst=8, refill=0 (frozen - * for determinism), only the first 8 of 100 concurrent /tool-call - * requests reach the upstream. The remaining 92 hit the rate limiter - * and are destroyed before the HTTP request layer sees them — they - * surface as either ECONNRESET / 'socket hang up' or EPIPE on the - * client side. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - /* burst=8 — much smaller than maxConnections so the RATE limit - * bites first. refill=1/sec is effectively frozen over the - * test's ~100ms window. */ - TCS_CONNECTION_RATE_BURST: '8', - TCS_CONNECTION_RATE_REFILL_PER_SEC: '1', - }, - }); - - try { - const results = await Promise.allSettled( - Array.from({ length: 100 }, () => - postToolCall(proxy.socketPath, '{"x":1}'), - ), - ); - - const ok = results.filter(r => r.status === 'fulfilled' && r.value.status === 200).length; - const dropped = results.length - ok; - - /* Burst=8 with a near-frozen refill — somewhere in [8, 12] should - * succeed (burst + occasional refill during test window). The - * key assertion is that the count is bounded WELL below 100. */ - expect(ok).toBeGreaterThanOrEqual(1); - expect(ok).toBeLessThanOrEqual(20); - expect(dropped).toBeGreaterThanOrEqual(80); - /* And critically: upstream sees only the rate-limit-allowed ones, - * NOT 100. This bounds kernel slab pressure: the proxy never - * creates the in-VM unix_sock entries for the dropped batch. */ - expect(upstream.calls()).toBeLessThanOrEqual(ok); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CONNECT-FLOOD recovery: bucket refills so legitimate traffic resumes after a flood', async () => { - /* Follow-up to the test above: confirm the limiter doesn't permanently - * starve a connection. After draining the burst bucket, wait long - * enough for the refill to top it back up, then verify a new - * /tool-call succeeds. This guards against a bug where dropped - * connections might leak the rate-limit credit (e.g. if the bucket - * was decremented BEFORE the maxConnections check and never - * compensated when both gates were hit by the same connection). */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - TCS_CONNECTION_RATE_BURST: '4', - /* 20/sec refill — burst recovers in ~250ms. */ - TCS_CONNECTION_RATE_REFILL_PER_SEC: '20', - }, - }); - - try { - /* Drain the burst with a parallel flood. */ - const drain = await Promise.allSettled( - Array.from({ length: 30 }, () => postToolCall(proxy.socketPath, '{"x":1}')), - ); - const okDuringDrain = drain.filter(r => r.status === 'fulfilled' && r.value.status === 200).length; - expect(okDuringDrain).toBeGreaterThanOrEqual(1); - expect(okDuringDrain).toBeLessThanOrEqual(15); - - /* Wait for the bucket to refill past 1 token (50ms refills 1 token - * at 20/sec; 500ms gives us plenty of margin). */ - await new Promise(r => setTimeout(r, 500)); - - /* A single legitimate request after the wait must succeed. */ - const post = await postToolCall(proxy.socketPath, '{"x":2}'); - expect(post.status).toBe(200); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); -}); diff --git a/api/src/tool-call-socket-proxy.test.ts b/api/src/tool-call-socket-proxy.test.ts deleted file mode 100644 index c15cde60..00000000 --- a/api/src/tool-call-socket-proxy.test.ts +++ /dev/null @@ -1,448 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import fs from 'node:fs'; -import http from 'node:http'; -import net from 'node:net'; -import os from 'node:os'; -import path from 'node:path'; -import { - createTokenBucket, - defaultConnectionRateBurst, - defaultConnectionRateRefillPerSec, - startToolCallSocketProxy, - type ToolCallSocketProxyHandle, -} from './tool-call-socket-proxy'; - -/* PTC-presence headers required by the proxy's filter so legitimate - * /tool-call requests forward instead of getting the canonical 404. The - * proxy only checks PRESENCE; cryptographic validation happens upstream. */ -const PTC_HEADERS = { - 'X-Execution-ID': 'test-exec-id', - 'X-Tool-Call-ID': 'test-call-001', - 'X-Callback-Token': 'test-callback-token', -} as const; - -const handles: ToolCallSocketProxyHandle[] = []; -const tempDirs: string[] = []; - -function makeSocketPath(): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tcs-proxy-test-')); - tempDirs.push(dir); - return path.join(dir, 'tcs.sock'); -} - -function wait(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -function createRawSocket(socketPath: string): Promise { - return new Promise((resolve, reject) => { - const socket = net.createConnection(socketPath); - socket.once('connect', () => resolve(socket)); - socket.once('error', reject); - }); -} - -async function startUpstream(): Promise<{ url: string; close: () => Promise }> { - const server = http.createServer((req, res) => { - if (req.method !== 'POST' || req.url !== '/tool-call') { - res.writeHead(404).end(); - return; - } - const chunks: Buffer[] = []; - req.on('data', chunk => chunks.push(Buffer.from(chunk))); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ - success: true, - body: Buffer.concat(chunks).toString('utf8'), - })); - }); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') { - throw new Error('unexpected upstream address'); - } - return { - url: `http://127.0.0.1:${address.port}`, - close: () => new Promise(resolve => server.close(() => resolve())), - }; -} - -afterEach(async () => { - while (handles.length > 0) { - await handles.pop()!.close(); - } - for (const dir of tempDirs.splice(0)) { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -describe('tool-call socket proxy', () => { - test('forwards valid tool-call requests with Connection: close', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - idleSocketTimeoutMs: 100, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const response = await new Promise<{ status: number; body: string; closeHeader: unknown }>((resolve, reject) => { - const req = http.request({ - socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - Connection: 'keep-alive', - ...PTC_HEADERS, - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - closeHeader: res.headers.connection, - })); - }); - req.on('error', reject); - req.end('{"tool_name":"safe","input":{}}'); - }); - - expect(response.status).toBe(200); - expect(response.closeHeader).toBe('close'); - expect(JSON.parse(response.body).body).toBe('{"tool_name":"safe","input":{}}'); - await upstream.close(); - }); - - test('destroys idle raw connections so socket probes cannot pin FDs', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - idleSocketTimeoutMs: 50, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const sockets = await Promise.all( - Array.from({ length: 8 }, () => createRawSocket(socketPath)), - ); - - await wait(250); - - expect(proxy.activeConnections()).toBe(0); - for (const socket of sockets) { - socket.destroy(); - } - await upstream.close(); - }); - - test('keeps socket connectable for per-job UID sandboxes', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - expect(fs.statSync(socketPath).mode & 0o777).toBe(0o666); - await upstream.close(); - }); - - test('caps active connections and drops excess clients', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - maxConnections: 2, - idleSocketTimeoutMs: 500, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const sockets = await Promise.allSettled( - Array.from({ length: 6 }, () => createRawSocket(socketPath)), - ); - - await wait(50); - - expect(sockets.filter(result => result.status === 'fulfilled').length).toBeGreaterThanOrEqual(2); - expect(proxy.activeConnections()).toBeLessThanOrEqual(2); - for (const result of sockets) { - if (result.status === 'fulfilled') result.value.destroy(); - } - await upstream.close(); - }); - - test('caps active tool-call requests separately from raw socket connections', async () => { - let releaseUpstream!: () => void; - let upstreamCalls = 0; - const upstreamReleased = new Promise(resolve => { - releaseUpstream = resolve; - }); - const server = http.createServer(async (req, res) => { - upstreamCalls += 1; - req.resume(); - await new Promise(resolve => req.on('end', resolve)); - await upstreamReleased; - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ success: true })); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxConnections: 8, - maxActiveRequests: 1, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const first = new Promise((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - res.on('end', () => resolve(res.statusCode ?? 0)); - res.resume(); - }); - req.on('error', reject); - req.end('{"tool_name":"slow","input":{}}'); - }); - - while (proxy.activeRequests() < 1) { - await wait(5); - } - - const second = await new Promise<{ status: number; body: string; retryAfter: unknown }>((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - retryAfter: res.headers['retry-after'], - })); - }); - req.on('error', reject); - req.end('{"tool_name":"second","input":{}}'); - }); - - expect(second.status).toBe(429); - expect(second.retryAfter).toBe('1'); - expect(JSON.parse(second.body)).toEqual({ - success: false, - error: 'Too many concurrent tool-call requests', - }); - expect(upstreamCalls).toBe(1); - - releaseUpstream(); - expect(await first).toBe(200); - await new Promise(resolve => server.close(() => resolve())); - }); - - test('expires active tool-call requests instead of holding proxy FDs indefinitely', async () => { - let upstreamCalls = 0; - let resolveUpstreamSeen!: () => void; - const upstreamSeen = new Promise(resolve => { - resolveUpstreamSeen = resolve; - }); - const server = http.createServer((req, res) => { - upstreamCalls += 1; - req.resume(); - req.on('end', resolveUpstreamSeen); - // Intentionally never respond; the proxy must time out and release its - // active request/upstream budgets. - void res; - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxActiveRequests: 1, - activeRequestTimeoutMs: 50, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const client = await createRawSocket(socketPath); - const body = '{"tool_name":"slow","input":{}}'; - client.end([ - 'POST /tool-call HTTP/1.1', - 'Host: localhost', - 'Content-Type: application/json', - `Content-Length: ${Buffer.byteLength(body)}`, - 'Connection: close', - 'X-Execution-ID: t', - 'X-Tool-Call-ID: t', - 'X-Callback-Token: t', - '', - body, - ].join('\r\n')); - - await upstreamSeen; - expect(upstreamCalls).toBe(1); - expect(proxy.activeRequests()).toBe(1); - expect(proxy.activeUpstreams()).toBe(1); - - const deadline = Date.now() + 500; - while (proxy.activeUpstreams() > 0 && Date.now() < deadline) { - await wait(5); - } - - expect(proxy.activeRequests()).toBe(0); - expect(proxy.activeUpstreams()).toBe(0); - client.destroy(); - await new Promise(resolve => server.close(() => resolve())); - }); - - test('rejects oversized request bodies before proxying', async () => { - let upstreamCalls = 0; - const server = http.createServer((_req, res) => { - upstreamCalls += 1; - res.writeHead(200).end('unexpected'); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxBodyBytes: 4, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const response = await new Promise<{ status: number; body: string }>((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - })); - }); - req.on('error', reject); - req.end('too large'); - }); - - expect(response.status).toBe(413); - expect(response.body).toBe('request body too large'); - expect(upstreamCalls).toBe(0); - await new Promise(resolve => server.close(() => resolve())); - }); - - /* Integration tests for the connection-rate limiter live in - * tool-call-socket-proxy-runtime.test.ts. Bun's node:http compat layer - * does not fire `'connection'` events, so the in-process proxy here - * never invokes the rate-limiter code path under `bun test` — the - * checks would pass vacuously. The runtime tests spawn the proxy as a - * real Node subprocess, matching the production runtime. The pure - * createTokenBucket logic is fully unit-tested below and runs fine - * under either runtime. */ -}); - -describe('createTokenBucket', () => { - test('starts full at burst capacity', () => { - const b = createTokenBucket({ burst: 10, refillPerSec: 1 }); - expect(b.tokens()).toBeCloseTo(10); - }); - - test('returns true until the bucket is drained, then false', () => { - const b = createTokenBucket({ burst: 3, refillPerSec: 0, now: () => 0 }); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - expect(b.tryConsume()).toBe(false); - }); - - test('refills at refillPerSec without exceeding the burst cap', () => { - let t = 1000; - const b = createTokenBucket({ burst: 10, refillPerSec: 100, now: () => t }); - /* Drain. */ - for (let i = 0; i < 10; i++) expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - /* Advance 50ms — 100/sec * 0.050 = 5 tokens. */ - t += 50; - for (let i = 0; i < 5; i++) expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - /* Advance way more than burst's worth — capacity caps at 10, not - * unbounded growth. */ - t += 10_000; - expect(b.tokens()).toBeCloseTo(10); - }); - - test('refillPerSec=0 freezes the bucket once drained (test-determinism guarantee)', () => { - const b = createTokenBucket({ burst: 2, refillPerSec: 0, now: () => 0 }); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - /* No matter how much time we'd advance with a real clock, refill of - * 0/sec adds zero tokens. */ - expect(b.tryConsume()).toBe(false); - expect(b.tokens()).toBe(0); - }); -}); - -describe('connection-rate defaults scale with SANDBOX_MAX_CONCURRENT_JOBS', () => { - /* Codex P2 follow-up on PR #1652: the previous fixed 20-accept/sec - * refill would throttle legitimate sustained traffic on deployments - * where maxConcurrentJobs is scaled up (e.g. 64), because the proxy's - * other defaults (maxConnections, maxActiveRequests) already scale - * from SANDBOX_MAX_CONCURRENT_JOBS but the rate limiter did not. */ - const originalEnv = process.env.SANDBOX_MAX_CONCURRENT_JOBS; - - afterEach(() => { - if (originalEnv === undefined) { - delete process.env.SANDBOX_MAX_CONCURRENT_JOBS; - } else { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = originalEnv; - } - }); - - test('unset env keeps the fixed 64/20 defaults', () => { - delete process.env.SANDBOX_MAX_CONCURRENT_JOBS; - expect(defaultConnectionRateBurst()).toBe(64); - expect(defaultConnectionRateRefillPerSec()).toBe(20); - }); - - test('small concurrency floors at the baseline (never tighter than the fixed defaults)', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '8'; - /* 8 * 4 = 32, but floor at 64 baseline. */ - expect(defaultConnectionRateBurst()).toBe(64); - /* 8 * 2 = 16, but floor at 20 baseline. */ - expect(defaultConnectionRateRefillPerSec()).toBe(20); - }); - - test('mid concurrency scales burst proportionally', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '32'; - /* 32 * 4 = 128 (above 64 floor, below 256 ceiling). */ - expect(defaultConnectionRateBurst()).toBe(128); - /* 32 * 2 = 64 (above 20 floor, below 200 ceiling). */ - expect(defaultConnectionRateRefillPerSec()).toBe(64); - }); - - test('high concurrency caps at the ceiling — bounded resource use even with extreme config', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '256'; - /* 256 * 4 = 1024, capped at 256 ceiling. */ - expect(defaultConnectionRateBurst()).toBe(256); - /* 256 * 2 = 512, capped at 200 ceiling. */ - expect(defaultConnectionRateRefillPerSec()).toBe(200); - }); -}); diff --git a/api/src/tool-call-socket-proxy.ts b/api/src/tool-call-socket-proxy.ts deleted file mode 100644 index 78e689b7..00000000 --- a/api/src/tool-call-socket-proxy.ts +++ /dev/null @@ -1,669 +0,0 @@ -import fs from 'node:fs'; -import http from 'node:http'; -import https from 'node:https'; -import type net from 'node:net'; - -export interface ToolCallSocketProxyOptions { - socketPath: string; - rawTarget: string; - maxConnections?: number; - maxActiveRequests?: number; - idleSocketTimeoutMs?: number; - headerTimeoutMs?: number; - requestBodyTimeoutMs?: number; - activeRequestTimeoutMs?: number; - maxBodyBytes?: number; - listenBacklog?: number; - /** Token-bucket capacity for the connection-rate limiter. The bucket - * starts full and refills at `connectionRateRefillPerSec`. Burst of - * legitimate concurrent tool calls (up to maxActiveRequests=16 by - * default) easily fits; a 500-connect()-in-50ms flood drains the - * bucket and subsequent connections are dropped at the application - * layer instead of allocating kernel unix_sock slab entries. */ - connectionRateBurst?: number; - /** Sustained connection-acceptance rate in connections/second after the - * burst bucket is drained. The SDK opens one connection per tool call; - * even an aggressive agent rarely exceeds a few /sec. */ - connectionRateRefillPerSec?: number; - socketUid?: number; - socketGid?: number; - socketMode?: number; - log?: Pick; -} - -export interface ToolCallSocketProxyHandle { - server: http.Server; - close: () => Promise; - activeConnections: () => number; - activeRequests: () => number; - activeUpstreams: () => number; - /** For tests: current available tokens in the connection-rate bucket. */ - connectionRateTokens: () => number; - /** For tests + observability: cumulative count of connections dropped - * by the connection-rate limiter since proxy start. */ - connectionRateDropped: () => number; -} - -const DEFAULT_MAX_CONNECTIONS = 64; -const DEFAULT_MAX_ACTIVE_REQUESTS = 16; -const DEFAULT_IDLE_SOCKET_TIMEOUT_MS = 2_000; -const DEFAULT_REQUEST_BODY_TIMEOUT_MS = 5_000; -const DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS = 30_000; -const DEFAULT_MAX_BODY_BYTES = 1_048_576; -const DEFAULT_LISTEN_BACKLOG = 16; -const MAX_DEFAULT_CONNECTIONS = 256; -const MAX_DEFAULT_ACTIVE_REQUESTS = 64; -const ACTIVE_REQUEST_TIMEOUT_GRACE_MS = 5_000; -const MAX_DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS = 60_000; -const DEFAULT_CONNECTION_RATE_BURST = 64; -const DEFAULT_CONNECTION_RATE_REFILL_PER_SEC = 20; -const MAX_DEFAULT_CONNECTION_RATE_BURST = 256; -const MAX_DEFAULT_CONNECTION_RATE_REFILL_PER_SEC = 200; - -/** Monotonic-ish token bucket. Tokens refill continuously at a configured - * rate up to a burst cap. `tryConsume()` returns false instead of waiting - * when empty — callers drop the connection synchronously, which is the - * right behavior for a SOCK_STREAM accept handler (queuing in JS would - * still hold the kernel socket struct alive, defeating the point). */ -export interface TokenBucket { - tryConsume(): boolean; - tokens(): number; -} - -export function createTokenBucket(opts: { - burst: number; - refillPerSec: number; - now?: () => number; -}): TokenBucket { - const now = opts.now ?? Date.now; - const refillPerMs = opts.refillPerSec / 1000; - let tokens = opts.burst; - let last = now(); - - function refill(): void { - const t = now(); - const elapsed = t - last; - if (elapsed <= 0) return; - tokens = Math.min(opts.burst, tokens + elapsed * refillPerMs); - last = t; - } - - return { - tryConsume(): boolean { - refill(); - if (tokens >= 1) { - tokens -= 1; - return true; - } - return false; - }, - tokens(): number { - refill(); - return tokens; - }, - }; -} - -function normalizeTarget(rawTarget: string): URL { - if (!rawTarget) { - throw new Error('SANDBOX_FORWARD_TARGET is required'); - } - return new URL(rawTarget.includes('://') ? rawTarget : `http://${rawTarget}`); -} - -function destroySocket(socket: net.Socket): void { - socket.destroy(); -} - -/* RFC 7230 §6.1 hop-by-hop headers. A proxy MUST NOT forward these to the - * upstream — they describe the proxy<->client connection, not the request. - * `host` is rewritten separately to point at the upstream. Forwarding the - * client's `host` would let a malicious sandbox steer the upstream's - * routing if it ever virtual-hosts. */ -const HOP_BY_HOP_HEADERS: ReadonlySet = new Set([ - 'connection', - 'keep-alive', - 'proxy-authenticate', - 'proxy-authorization', - 'proxy-connection', - 'te', - 'trailer', - 'transfer-encoding', - 'upgrade', - 'host', -]); - -function buildForwardedHeaders( - reqHeaders: http.IncomingHttpHeaders, - upstreamHost: string, -): http.OutgoingHttpHeaders { - const out: http.OutgoingHttpHeaders = {}; - for (const [k, v] of Object.entries(reqHeaders)) { - if (HOP_BY_HOP_HEADERS.has(k.toLowerCase())) continue; - if (v == null) continue; - out[k] = v; - } - /* Set our own connection-management headers; the proxy never keep-alives - * to upstream because each request is one-shot. */ - out.host = upstreamHost; - out.connection = 'close'; - return out; -} - -/* True only when the header has at least one non-empty, non-whitespace - * character. Naive `!headers[name]` is bypassable: Node joins duplicate - * request headers with `, ` (per RFC 7230 §3.2.2), so two empty - * `X-Foo:` lines arrive as the truthy string `", "`. An attacker who - * sends each PTC header twice with empty values would slip past the - * presence filter and reach the upstream — defeating the route-opacity - * goal. Stripping commas + whitespace before the empty check closes - * that gap. */ -function hasPtcHeaderValue(value: string | string[] | undefined): boolean { - if (value == null) return false; - const joined = Array.isArray(value) ? value.join(',') : value; - /* Strip every comma (Node's separator for joined duplicates) and every - * whitespace character. If anything is left, the header carries real - * content. */ - return joined.replace(/[,\s]/g, '') !== ''; -} - -/* Request smuggling defense (CVE class). When a request carries BOTH - * Transfer-Encoding and Content-Length, an upstream may pick the - * different one than the proxy did, letting an attacker prepend bytes - * to a synthesized "second request" that bypasses the proxy's path - * filter. The fix per RFC 7230 §3.3.3 is to reject the request rather - * than try to reconcile. */ -function isSmugglingShaped(headers: http.IncomingHttpHeaders): boolean { - const hasCL = headers['content-length'] != null; - const hasTE = headers['transfer-encoding'] != null; - return hasCL && hasTE; -} - -function parsePositiveInt(raw: string | undefined): number | undefined { - if (raw == null) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n < 1) return undefined; - return Math.floor(n); -} - -function defaultMaxConnections(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_MAX_CONNECTIONS; - return Math.min(Math.max(maxConcurrentJobs * 4, DEFAULT_MAX_CONNECTIONS), MAX_DEFAULT_CONNECTIONS); -} - -function defaultMaxActiveRequests(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_MAX_ACTIVE_REQUESTS; - return Math.min(Math.max(maxConcurrentJobs, DEFAULT_MAX_ACTIVE_REQUESTS), MAX_DEFAULT_ACTIVE_REQUESTS); -} - -function defaultActiveRequestTimeoutMs(): number { - const runTimeoutMs = parsePositiveInt(process.env.SANDBOX_RUN_TIMEOUT); - if (runTimeoutMs == null) return DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS; - return Math.min( - Math.max(runTimeoutMs + ACTIVE_REQUEST_TIMEOUT_GRACE_MS, DEFAULT_REQUEST_BODY_TIMEOUT_MS), - MAX_DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS, - ); -} - -/* Rate-limit defaults scale with SANDBOX_MAX_CONCURRENT_JOBS in the same - * shape as defaultMaxConnections / defaultMaxActiveRequests above. A - * deployment with maxConcurrentJobs=64 is sized to run ~64 tool calls - * in parallel; capping the connection-acceptance rate at the fixed 20/sec - * baseline would throttle legitimate sustained traffic from that - * workload — connections that would have been 429'd by maxActiveRequests - * instead get destroyed at the rate limiter, which is the wrong shape - * (clients see ECONNRESET instead of a structured Retry-After response). */ -export function defaultConnectionRateBurst(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_CONNECTION_RATE_BURST; - return Math.min( - Math.max(maxConcurrentJobs * 4, DEFAULT_CONNECTION_RATE_BURST), - MAX_DEFAULT_CONNECTION_RATE_BURST, - ); -} - -export function defaultConnectionRateRefillPerSec(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_CONNECTION_RATE_REFILL_PER_SEC; - /* 2x maxConcurrentJobs gives headroom over the steady-state rate - * (~1 connection per active job per second for typical tool-call - * workloads). Floor at 20/sec so small deployments don't get a - * tighter limit than the fixed baseline. */ - return Math.min( - Math.max(maxConcurrentJobs * 2, DEFAULT_CONNECTION_RATE_REFILL_PER_SEC), - MAX_DEFAULT_CONNECTION_RATE_REFILL_PER_SEC, - ); -} - -export async function startToolCallSocketProxy( - opts: ToolCallSocketProxyOptions, -): Promise { - const log = opts.log ?? console; - const socketPath = opts.socketPath; - const target = normalizeTarget(opts.rawTarget); - const transport = target.protocol === 'https:' ? https : http; - const maxConnections = opts.maxConnections ?? defaultMaxConnections(); - const maxActiveRequests = opts.maxActiveRequests ?? defaultMaxActiveRequests(); - const idleSocketTimeoutMs = opts.idleSocketTimeoutMs ?? DEFAULT_IDLE_SOCKET_TIMEOUT_MS; - const headerTimeoutMs = opts.headerTimeoutMs ?? idleSocketTimeoutMs; - const requestBodyTimeoutMs = opts.requestBodyTimeoutMs ?? DEFAULT_REQUEST_BODY_TIMEOUT_MS; - const activeRequestTimeoutMs = opts.activeRequestTimeoutMs ?? defaultActiveRequestTimeoutMs(); - const maxBodyBytes = opts.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES; - const listenBacklog = opts.listenBacklog ?? DEFAULT_LISTEN_BACKLOG; - const connectionRateBurst = opts.connectionRateBurst ?? defaultConnectionRateBurst(); - const connectionRateRefillPerSec = opts.connectionRateRefillPerSec ?? defaultConnectionRateRefillPerSec(); - // The socket is intentionally connectable by sandbox jobs even when - // SANDBOX_PER_JOB_UIDS maps each job to a distinct outside UID. Abuse is - // bounded by the proxy's connection caps/timeouts, not by inode ownership. - const socketMode = opts.socketMode ?? 0o666; - - const activeSockets = new Set(); - /* Absolute accept-to-headers deadlines cover slow-header clients. Node's - * headersTimeout and socket idle timers are not a short per-socket deadline - * for AF_UNIX header drip attacks, so track the accepted socket ourselves - * until node:http has parsed enough bytes to dispatch a request. */ - const headerReceiveDeadlines = new WeakMap>(); - let activeRequests = 0; - let activeUpstreams = 0; - - const clearHeaderReceiveDeadline = (socket: net.Socket): void => { - const deadline = headerReceiveDeadlines.get(socket); - if (deadline) clearTimeout(deadline); - headerReceiveDeadlines.delete(socket); - }; - - /* Connection-rate limiter — counters the audit pattern of 500 AF_UNIX - * connect()s in <50 ms. Without this, even though the proxy itself - * stays bounded (maxConnections destroys excess accepts), each accept - * still allocates a unix_sock + dentry + skb queue in the in-VM - * kernel slab. On a memory-tight microVM, 500+ allocations in a - * single tick pressures the slab allocator hard enough that - * subsequent legitimate openat() / mmap() calls (e.g. ld.so loading - * shared libraries) sporadically fail with EMFILE — even though the - * fd table itself is at <1% utilization. Capping the *rate* of - * accepts at the application layer bounds how fast new slab entries - * can be created, giving the kernel time to reclaim. */ - const connectionRateLimiter = createTokenBucket({ - burst: connectionRateBurst, - refillPerSec: connectionRateRefillPerSec, - }); - let connectionRateDroppedCount = 0; - - const server = http.createServer((req, res) => { - const socket = req.socket; - clearHeaderReceiveDeadline(socket); - socket.setTimeout(requestBodyTimeoutMs, () => destroySocket(socket)); - req.setTimeout(requestBodyTimeoutMs, () => destroySocket(socket)); - res.setHeader('Connection', 'close'); - - res.on('finish', () => { - socket.end(); - }); - - if (req.method !== 'POST' || req.url !== '/tool-call') { - req.resume(); - res.writeHead(404, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('not found'); - return; - } - - /* PTC-header presence filter at the proxy boundary. Three reasons: - * - * (1) Route opacity. If a sandbox attacker probes /tool-call without - * the SDK-supplied PTC headers, the upstream's 404 leaks Express's - * framing (ETag/X-Request-ID/charset/content-length) which is - * visibly different from the proxy's own 404 for unknown paths. - * Filtering at the proxy means missing-header probes never reach - * the upstream and the response is byte-identical to /any/unknown. - * - * (2) Cheap rejection. Don't burn an active-request slot or open an - * upstream connection for a request that's going to be rejected. - * - * (3) Preserves legitimate 404s. The previous shape blanket-masked - * ALL upstream 404 responses, which broke "Session not found" and - * other structured upstream errors that the SDK's preamble parses - * as JSON (json.loads("not found") raises). With the filter - * here, only missing-header probes are masked; a genuine 404 with - * a JSON body for an authenticated request still reaches the - * SDK intact. - * - * The proxy only checks PRESENCE — actual cryptographic validation - * happens at the upstream. An attacker supplying garbage values for - * these headers gets forwarded and the upstream rejects with its - * structured response, but they've already revealed sandbox-side - * activity by sending non-empty headers, so route opacity isn't the - * goal there. */ - if ( - !hasPtcHeaderValue(req.headers['x-execution-id']) - || !hasPtcHeaderValue(req.headers['x-tool-call-id']) - || !hasPtcHeaderValue(req.headers['x-callback-token']) - ) { - req.resume(); - res.writeHead(404, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('not found'); - return; - } - - /* Smuggling defense — block before we do anything else with the - * request so neither our active-request budget nor any upstream - * connection is consumed. */ - if (isSmugglingShaped(req.headers)) { - req.resume(); - res.writeHead(400, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('ambiguous Content-Length and Transfer-Encoding'); - return; - } - - /* Content-Length precheck — if the client honestly declares a body - * larger than maxBodyBytes, reject BEFORE opening an upstream and - * BEFORE leaking any of those bytes through req.pipe(upstream). The - * runtime byte counter below is the second line of defense for - * chunked uploads and dishonest CL values, but this gate handles - * the common case cleanly. */ - const declaredCL = Number(req.headers['content-length'] ?? '0'); - if (Number.isFinite(declaredCL) && declaredCL > maxBodyBytes) { - req.resume(); - res.writeHead(413, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('request body too large'); - return; - } - - if (activeRequests >= maxActiveRequests) { - req.resume(); - res.writeHead(429, { - 'Content-Type': 'application/json', - Connection: 'close', - 'Retry-After': '1', - }); - res.end(JSON.stringify({ - success: false, - error: 'Too many concurrent tool-call requests', - })); - return; - } - - activeRequests += 1; - let releasedActiveRequest = false; - const releaseActiveRequest = (): void => { - if (releasedActiveRequest) return; - releasedActiveRequest = true; - activeRequests = Math.max(0, activeRequests - 1); - }; - res.on('finish', releaseActiveRequest); - res.on('close', releaseActiveRequest); - - let bodyBytes = 0; - let rejected = false; - let upstreamClosed = false; - - /* Absolute body-upload deadline. socket.setTimeout / req.setTimeout - * are idle timers and reset on every byte, so a malicious client - * dripping bytes just under the idle threshold (slow-loris) bypasses - * them. Node's `server.requestTimeout` was meant to bound this but - * empirically does NOT fire mid-body for unix-socket clients in - * Node 22 (probed; drip kept flowing past 3x the configured value). - * This explicit setTimeout fires unconditionally `requestBodyTimeoutMs` - * after the request handler runs and is cleared once the body is - * fully received. */ - const bodyUploadDeadline = setTimeout(() => { - if (req.complete || rejected) return; - destroySocket(socket); - }, requestBodyTimeoutMs); - const clearBodyUploadDeadline = (): void => clearTimeout(bodyUploadDeadline); - req.on('end', clearBodyUploadDeadline); - req.on('aborted', clearBodyUploadDeadline); - res.on('finish', clearBodyUploadDeadline); - res.on('close', clearBodyUploadDeadline); - - req.on('end', () => { - socket.setTimeout(activeRequestTimeoutMs, () => destroySocket(socket)); - }); - - /* BUFFER then forward — do NOT open upstream or pipe bytes until the - * full body is received. Streaming would let a slow-loris that drips - * body bytes pin an UPSTREAM connection slot for the entire body- - * upload window, even though the proxy itself bounds its own socket - * lifetime. Tool-call payloads are small JSON (capped at maxBodyBytes, - * default 1 MiB), so buffering is cheap and the security property - * "upstream never sees a partial request" is much stronger. */ - const bodyChunks: Buffer[] = []; - let upstream: http.ClientRequest | undefined; - const abortUpstream = (): void => { - if (!upstream || upstreamClosed || res.writableEnded) return; - upstream.destroy(new Error('tool-call client disconnected')); - }; - req.on('aborted', abortUpstream); - res.on('close', abortUpstream); - socket.on('close', abortUpstream); - - req.on('data', chunk => { - if (rejected) return; - bodyBytes += chunk.length; - if (bodyBytes > maxBodyBytes) { - rejected = true; - res.writeHead(413, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('request body too large'); - destroySocket(socket); - return; - } - bodyChunks.push(Buffer.from(chunk)); - }); - - req.on('end', () => { - if (rejected) return; - const body = bodyChunks.length === 1 ? bodyChunks[0] : Buffer.concat(bodyChunks); - - /* Strip ALL hop-by-hop headers before forwarding (RFC 7230 §6.1). - * Previously we only stripped `proxy-connection`, which left - * Upgrade, TE, Trailer, and friends as a vector for protocol - * confusion at the upstream. */ - const headers = buildForwardedHeaders(req.headers, target.host); - /* Override declared content-length with the actual buffered size - * — if the client lied, what we send is what we have. */ - headers['content-length'] = body.length; - - upstream = transport.request({ - protocol: target.protocol, - hostname: target.hostname, - port: target.port || undefined, - method: 'POST', - path: '/tool-call', - headers, - }, upstreamRes => { - if (rejected) { - upstreamRes.resume(); - return; - } - /* Strip hop-by-hop on the response path. Upstream's Connection / - * Keep-Alive / Upgrade headers describe the proxy <-> upstream - * link, not what we should tell the sandbox client. */ - const respHeaders: http.OutgoingHttpHeaders = {}; - for (const [k, v] of Object.entries(upstreamRes.headers)) { - if (HOP_BY_HOP_HEADERS.has(k.toLowerCase())) continue; - if (v == null) continue; - respHeaders[k] = v; - } - respHeaders.Connection = 'close'; - res.writeHead(upstreamRes.statusCode || 502, respHeaders); - upstreamRes.pipe(res); - }); - activeUpstreams += 1; - - upstream.on('close', () => { - upstreamClosed = true; - activeUpstreams = Math.max(0, activeUpstreams - 1); - releaseActiveRequest(); - }); - - upstream.setTimeout(activeRequestTimeoutMs, () => { - upstream?.destroy(new Error('tool-call upstream timeout')); - }); - - upstream.on('error', error => { - if (rejected) return; - log.error('tool-call socket proxy upstream error', error); - if (!res.headersSent) { - res.writeHead(502, { 'Content-Type': 'text/plain', Connection: 'close' }); - } - res.end('bad gateway'); - }); - - upstream.end(body); - }); - }); - - server.maxConnections = maxConnections; - server.headersTimeout = headerTimeoutMs; - server.keepAliveTimeout = 1; - /* Bound the entire request reception (first byte -> last body byte) to - * the body-upload window. Prevents slow-loris drip attacks: socket-idle - * timers reset on every byte, so a malicious client could otherwise - * stretch body upload to activeRequestTimeoutMs (default 30s) by - * sending one byte just under the idle threshold. */ - server.requestTimeout = requestBodyTimeoutMs; - server.timeout = activeRequestTimeoutMs; - - server.on('connection', socket => { - /* Rate-limit BEFORE the concurrency check. The concurrency check - * (`activeSockets.size`) tells us how many connections are alive - * RIGHT NOW; the rate limiter tells us how fast new ones are being - * created. The audit's flood passes the concurrency check trivially - * (most connections accepted, processed, and destroyed faster than - * the count grows) but creates thousands of socket structs in the - * process. Rate-limiting at connect-time is what bounds the kernel - * slab pressure that surfaces as EMFILE in subsequent loader open() - * calls — see createTokenBucket comment for the full mechanism. */ - if (!connectionRateLimiter.tryConsume()) { - connectionRateDroppedCount += 1; - destroySocket(socket); - return; - } - if (activeSockets.size >= maxConnections) { - log.warn('tool-call socket proxy connection limit reached; dropping connection'); - destroySocket(socket); - return; - } - - activeSockets.add(socket); - const headerReceiveDeadline = setTimeout(() => { - headerReceiveDeadlines.delete(socket); - destroySocket(socket); - }, headerTimeoutMs); - headerReceiveDeadlines.set(socket, headerReceiveDeadline); - - socket.setTimeout(idleSocketTimeoutMs, () => destroySocket(socket)); - socket.on('close', () => { - clearHeaderReceiveDeadline(socket); - activeSockets.delete(socket); - }); - socket.on('error', () => { - clearHeaderReceiveDeadline(socket); - activeSockets.delete(socket); - }); - }); - - server.on('clientError', (_error, socket) => { - destroySocket(socket as net.Socket); - }); - - try { - fs.unlinkSync(socketPath); - } catch { - // Missing stale socket is fine. - } - - await new Promise((resolve, reject) => { - const onError = (error: Error): void => { - server.off('listening', onListening); - reject(error); - }; - const onListening = (): void => { - server.off('error', onError); - if (opts.socketUid != null && opts.socketGid != null) { - fs.chownSync(socketPath, opts.socketUid, opts.socketGid); - } - fs.chmodSync(socketPath, socketMode); - log.log(`tool-call socket proxy listening on ${socketPath}`); - resolve(); - }; - server.once('error', onError); - server.once('listening', onListening); - server.listen(socketPath, listenBacklog); - }); - - return { - server, - close: async () => { - for (const socket of activeSockets) { - destroySocket(socket); - } - await new Promise((resolve) => { - server.close(() => { - try { - fs.unlinkSync(socketPath); - } catch { - // Socket was already removed. - } - resolve(); - }); - }); - }, - activeConnections: () => activeSockets.size, - activeRequests: () => activeRequests, - activeUpstreams: () => activeUpstreams, - connectionRateTokens: () => connectionRateLimiter.tokens(), - connectionRateDropped: () => connectionRateDroppedCount, - }; -} - -if (require.main === module) { - const socketPath = process.env.TCS_SOCKET || '/tmp/tcs.sock'; - const rawTarget = process.env.SANDBOX_FORWARD_TARGET || ''; - const socketUid = process.env.TCS_SOCKET_UID ? Number(process.env.TCS_SOCKET_UID) : undefined; - const socketGid = process.env.TCS_SOCKET_GID ? Number(process.env.TCS_SOCKET_GID) : undefined; - const maxConnections = parsePositiveInt(process.env.TCS_MAX_CONNECTIONS); - const headerTimeoutMs = parsePositiveInt(process.env.TCS_HEADER_TIMEOUT_MS); - /* Operator overrides for the connection-rate limiter — see - * createTokenBucket comment for the kernel-slab-pressure motivation. - * Defaults (burst=64, refill=20/sec) are sized for the SDK's tool-call - * pattern (one connection per call) plus generous headroom; raise only - * if a legitimate workload trips the dropped-connection counter. */ - const rateBurst = parsePositiveInt(process.env.TCS_CONNECTION_RATE_BURST); - const rateRefillPerSec = parsePositiveInt(process.env.TCS_CONNECTION_RATE_REFILL_PER_SEC); - let handle: ToolCallSocketProxyHandle | undefined; - let shuttingDown = false; - - const shutdown = (): void => { - if (shuttingDown) return; - shuttingDown = true; - if (!handle) { - /* Signal arrived before startToolCallSocketProxy() resolved. - * `process.exit` is terminal in Node, but the explicit `return` makes - * the contract local: no `handle.close()` on `undefined` even if a - * future runtime ever defers exit (atexit hook, async-cleanup mode). */ - process.exit(0); - return; - } - void handle.close().finally(() => process.exit(0)); - }; - - process.on('SIGTERM', shutdown); - process.on('SIGINT', shutdown); - - startToolCallSocketProxy({ - socketPath, rawTarget, socketUid, socketGid, - maxConnections, - headerTimeoutMs, - connectionRateBurst: rateBurst, - connectionRateRefillPerSec: rateRefillPerSec, - }) - .then(started => { - handle = started; - }) - .catch(error => { - console.error('tool-call socket proxy failed to start', error); - process.exit(1); - }); -} diff --git a/build-packages.sh b/build-packages.sh index 2ae04d02..67912ee5 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -10,7 +10,7 @@ # Environment Variables: # PYTHON_VERSION=3.14.4 # Python version to install # NODE_VERSION=24.15.0 # Node.js version to install -# BUN_VERSION=1.3.14 # Bun version to install +# BUN_VERSION=1.4.2 # Bun version to install # BASH_PACKAGE_VERSION=5.2.0 # Bash package registration version (semver-like x.y.z) # SKIP_PYTHON_PACKAGES=1 # Skip Python pip packages # SKIP_JS_PACKAGES=1 # Skip JavaScript npm packages for Node/Bun @@ -28,7 +28,7 @@ cd "$SCRIPT_DIR" PYTHON_VERSION="${PYTHON_VERSION:-3.14.4}" NODE_VERSION="${NODE_VERSION:-24.15.0}" -BUN_VERSION="${BUN_VERSION:-1.3.14}" +BUN_VERSION="${BUN_VERSION:-1.4.2}" PACKAGES_DIR="./data/pkgs" JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 7156719c..5e66baed 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -42,8 +42,10 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup @@ -58,7 +60,7 @@ FROM buildpack-deps:bookworm AS package-builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHON_VERSION=3.14.4 -ENV BUN_VERSION=1.3.14 +ENV BUN_VERSION=1.4.2 COPY docker/apt-install.sh /usr/local/bin/apt-install RUN chmod +x /usr/local/bin/apt-install @@ -80,7 +82,7 @@ RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh # ============================================================================ # Stage 2: Build worker # ============================================================================ -FROM oven/bun:1.3.14 AS worker-builder +FROM oven/bun:1.4.2 AS worker-builder WORKDIR /app COPY service/package.json service/bun.lock ./ RUN bun install --frozen-lockfile @@ -93,19 +95,20 @@ RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --ext # ============================================================================ # Stage 3: Build sandbox API # ============================================================================ -FROM oven/bun:1.3.14 AS sandbox-builder +FROM oven/bun:1.4.2 AS sandbox-builder WORKDIR /app COPY api/package.json ./ RUN bun install --frozen-lockfile COPY api/src ./src COPY shared /shared COPY api/tsconfig.json ./ -RUN bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' +RUN bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' \ + && bun build ./src/tool-call-pipe-broker.ts --target=node --format=cjs --outfile=.build/tool-call-pipe-broker.cjs # ============================================================================ # Stage 4: Build sandbox rootfs (full OS layer for the microVM guest) # ============================================================================ -FROM oven/bun:1.3.14-debian AS sandbox-rootfs +FROM oven/bun:1.4.2-debian AS sandbox-rootfs ENV DEBIAN_FRONTEND=noninteractive @@ -143,7 +146,9 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge RUN chmod +x /usr/sbin/nsjail +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node WORKDIR /sandbox_api diff --git a/docker/package-init.sh b/docker/package-init.sh index 60855b95..1ffaaf3a 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -21,7 +21,7 @@ PYTHON_ALIAS="python${PYTHON_SITE_VERSION}" PYTHON_PACKAGE_INSTALLER="${PYTHON_PACKAGE_INSTALLER:-uv}" UV_VERSION="${UV_VERSION:-0.11.26}" NODE_VERSION="${NODE_VERSION:-24.15.0}" -BUN_VERSION="${BUN_VERSION:-1.3.14}" +BUN_VERSION="${BUN_VERSION:-1.4.2}" BASH_PACKAGE_VERSION="${BASH_PACKAGE_VERSION:-5.2.0}" INSTALL_FAILED=false JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md index c8494a90..b8026383 100644 --- a/docs/SECURITY-MAINTENANCE.md +++ b/docs/SECURITY-MAINTENANCE.md @@ -48,40 +48,107 @@ are denied even for messages without descriptors and for inherited sockets. Keep `io_uring_setup`, `io_uring_enter`, and `io_uring_register` denied: ring operations can submit sends without passing through these syscall filters. -Unlike the UDF sandbox's socket-family restriction, this sandbox retains -AF_UNIX and `socketpair` for `/tmp/tcs.sock` and ordinary IPC using reads, -writes, and `sendto`. Python multiprocessing pipes, queues, and pools remain -supported; transferring socket/file handles through `multiprocessing.reduction` -or similar descriptor-passing APIs returns EPERM. Do not restore `sendmsg` or -`sendmmsg` to make those APIs work without reassessing the kernel boundary. +The job policy denies every `socket()` family (VSOCK retains its KILL action). +It allows only AF_UNIX SOCK_STREAM `socketpair` with protocol 0, including +CLOEXEC/NONBLOCK flags. Python asyncio and duplex multiprocessing pipes need +these anonymous socketpairs. Ordinary fork/spawn queues and pools are covered +by native tests. Named Unix listeners, `multiprocessing.Manager`, datagram or +seqpacket pairs, and descriptor-sharing APIs are unsupported. The send and +io_uring denials still prevent SCM_RIGHTS descriptor graphs on allowed pairs. This mitigation is separate from PI-futex filtering. Verify vendor patches for the deployed guest and node kernels; a policy update does not establish patch status. Rebuild and replace sandbox runner images to deploy the policy change. -The native `NsJail IPC Filter` CI jobs compile the rendered policy with the -image's pinned Kafel on amd64 and arm64, install it in a Linux process, and -check descriptor sends, inherited sockets, Unix HTTP, and Python fork/spawn -queues and pools. This validates the filter and IPC compatibility, not a -production kernel's patch status or the full microVM deployment. - -### Planned transport hardening - -Replace the job-visible Unix HTTP socket with two dedicated per-job pipes to -a trusted broker. Keep them separate from stdin/stdout/stderr, which carry user -input and execution output. Limit message sizes, outstanding requests, and -timeouts; retain the existing job authorization and tool-call budgets at the -broker. Update each language's tool-call client to use framed messages, and -explicitly preserve only these pipe FDs through NsJail and `spec-guard`. -The current guard closes all FDs above 2; a redesign must preserve its protection -against unrelated inherited descriptors. - -Then deny `socket` and `socketpair` in the job policy and retain the send and -io_uring denials. Validate language/runtime compatibility, including Python's -multiprocessing resource tracker and descriptor-sharing features, before making -that stricter policy the default. The broker retains network access outside the -job filter; it must not expose raw socket handles to the job. This is follow-up -work, not part of the immediate descriptor-passing mitigation. +## Tool-call pipe transport + +Authorized blocking Python invocations receive two anonymous pipe endpoints: +FD 3 writes requests and FD 4 reads responses. No tool-call socket path is +mounted in the jail. The signed, body-bound `tool_call_socket` boolean remains +the wire capability name for compatibility with manifest verification; it now +grants pipes. Ordinary executions, compile steps, and replay/Bash tool calls +receive no endpoints. The Python client makes both ends non-inheritable before +user code runs and refuses tool calls from forked children. + +A Node broker starts per invocation after MicroVM restore and forwards only +POST `/tool-call` to the configured `SANDBOX_FORWARD_TARGET`. It forwards only +the three PTC claim headers; upstream session/token/tool authorization remains +in force. It never follows redirects. Each channel bounds frames and upstream +responses to 1 MiB, active upstream requests to 16, admission to a 64-request +burst refilling at 20 requests/second, partial-frame and response-drain stalls +to 5 seconds, and upstream requests to the job timeout (5–600 seconds). +Malformed framing closes the invocation; disconnect/exit aborts upstream work. +The Python client correlates concurrent responses by frame ID. + +Node/Bun extra stdio descriptors are socketpairs. A small trusted C relay +converts the broker's channel into real anonymous pipes before execing NsJail. +NsJail explicitly passes only FDs 3/4; `spec-guard --tool-call-pipes` checks their +FIFO type and access direction, retains those two ends, and closes unrelated +FDs. Its ordinary mode still closes every descriptor above 2. API/broker/relay +parent-death handling kills the invocation if either supervisor disappears. +Each invocation adds a Node broker and C relay outside the job cgroup; size +runner CPU/memory and PID limits for the configured simultaneous-job count. + +The amd64/arm64 `NsJail IPC Filter` CI jobs compile the policy with pinned Kafel, +exercise socket/send denials plus asyncio/multiprocessing, check guard FD +cleanup, and run the actual broker/relay/NsJail/guard/generated Python client +with concurrent replies. These checks do not boot libkrun or establish deployed +kernel patch status. + +Every complete frame consumes the admission budget before JSON or claim +validation, including frames rejected by the active-request limit. Exhausting +that budget closes the channel and terminates the invocation instead of +producing an unlimited stream of rejection replies. Losing the response pipe's +reader also terminates the invocation, even when no reply is buffered. + +The standalone launcher guest, API runner and combined worker runner all +package Node, the broker bundle and the compiled pipe bridge. Native CI builds +the standalone launcher guest on both architectures and exercises a round trip +using that image's own binaries. + +### Local Mac/Docker canary + +`tests/tool_call_runner.cjs` exercises signed API authorization, the normal +NsJail mount/user/PID/network/IPC/UTS/cgroup namespaces, the generated Python +client, concurrent tool calls, descriptor cleanup, spawn/fork pipe IPC, ordinary +Python jobs and basic Bun JavaScript/TypeScript execution. Spawned workers can +import the preamble but cannot issue tool calls. Python 3.14's default forkserver +requires a named Unix listener and is denied; select an explicit spawn or fork +context. Semaphore-based pools/queues require `/dev/shm`, which the default jail +does not mount. The namespace-disabled native policy tests cover those APIs +when shared memory exists, not their availability in the default jail. + +With runtime packages built by `build-packages.sh` (or a compatible `/pkgs` +tree containing Python 3.14.4 and Bun 1.4.2), run from the repository root: + +```sh +bun service/scripts/dump-pipe-preamble.ts > /tmp/codeapi-pipe-client.py +docker build -f api/Dockerfile --target sandbox-build -t codeapi-pipe-canary . +docker run --rm --init \ + --cap-add SYS_ADMIN --cap-add SYS_CHROOT --cap-add SETUID \ + --cap-add SETGID --cap-add NET_ADMIN \ + --security-opt seccomp=./seccomp/nsjail.json \ + --mount "type=bind,source=$PWD/data/pkgs,target=/pkgs,readonly" \ + --mount "type=bind,source=/tmp/codeapi-pipe-client.py,target=/client.py,readonly" \ + --mount "type=bind,source=$PWD/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/node \ + codeapi-pipe-canary /tests/tool_call_runner.cjs /client.py +``` + +The canary disables resource cgroup enforcement as the Mac compose override +does, while retaining the cgroup namespace and all remaining NsJail isolation. +It exercises Docker's Linux VM, not a libkrun guest. It does not establish +production kernel patch status or third-party language-package compatibility. + +### Coordinated rollout + +Deploy the service's new blocking preamble together with rebuilt runner images +containing the broker, relay and updated guard/policy. Drain old blocking jobs +and route new blocking requests only to matching images during rollout. Old +socket preambles fail on pipe-only runners; new pipe preambles fail immediately +on old runners. There is no TCP or Unix-socket fallback. Replay mode is unchanged. +Rollback must restore both service and runner versions; retain #309's send and +io_uring denials. See [ADR 0011](adr/0011-tool-call-pipes.md) for the design. ## Preserve host-side confinement diff --git a/docs/adr/0011-tool-call-pipes.md b/docs/adr/0011-tool-call-pipes.md new file mode 100644 index 00000000..69e0b0ee --- /dev/null +++ b/docs/adr/0011-tool-call-pipes.md @@ -0,0 +1,61 @@ +# 0011: Per-invocation anonymous pipes for blocking tool calls + +Status: Accepted + +## Context + +Blocking Python tool calls used an HTTP listener at `/tmp/tcs.sock` mounted into +selected NsJail jobs. CVE-2026-80521 motivated descriptor-passing denials in +#309. Keeping socket creation enabled nevertheless exposes named endpoints +and socket families to untrusted code. The runner is snapshotted before work: +Node and its TLS state must continue to start after MicroVM restore. + +The current stdio spawn APIs create socketpairs, not anonymous pipe endpoints. +The existing guard also intentionally removes unrelated inherited descriptors. +Python asyncio and duplex multiprocessing pipes use Unix stream socketpairs; +a blanket socketpair denial would break these ordinary workloads. + +## Decision + +Use one post-restore Node broker per authorized blocking invocation. Keep the +existing signed/body-bound `tool_call_socket` capability name on the wire, but +grant two pipe endpoints instead of a socket mount. Use length-prefixed JSON +frames with response IDs, a fixed upstream route, whitelisted claim headers, +and explicit size, rate, concurrency and deadline limits. Keep upstream +cryptographic/session/tool authorization authoritative. + +A trusted C relay holds the spawn API's socket on its own side and passes real +anonymous pipe ends at FDs 3/4 into NsJail. The guard preserves them only under +an explicit option after validating FIFO type and write/read access; all +unrelated inherited descriptors are closed. API/broker parent death, channel failure and +job exit terminate the relay and its invocation and abort outstanding upstream +requests. The Python client clears descriptor inheritance and refuses calls +from forked or spawned children. Spawned workers may re-import tool definitions +without receiving or using the pipe capability. stdout/stderr and user stdin remain separate. + +Deny all socket creation. Permit only protocol-0 Unix stream socketpairs for +runtime IPC, retaining the sendmsg/sendmmsg/io_uring denials. This preserves +asyncio and pipe-based multiprocessing using explicit spawn/fork contexts without permitting named socket access +or descriptor passing. No legacy TCP/socket fallback is provided. + +## Consequences + +New service preambles and runner images require coordinated deployment; +blocking jobs must be drained or version-routed during upgrades and rollbacks. +Bash/replay mode has no pipe grant. Unix listeners/managers, non-stream pairs, +and descriptor-sharing APIs are unsupported. Python 3.14 defaults to forkserver, +which requires a named Unix listener and is denied; use an explicit spawn or +fork context. Semaphore-based queues/pools also require `/dev/shm`, which the +normal jail currently does not mount. Each active invocation adds two +trusted processes outside its job cgroup; runner sizing must account for them. + +A shared post-restore broker could reduce process overhead, but would require +an additional channel admission/multiplexing mechanism. Separate brokers keep +request state, failure and lifetime tied directly to one invocation. Named +FIFOs avoid inherited-FD changes but add filesystem names and open/unlink races; +anonymous pipes require neither path discovery nor shared listener state. + +Native CI verifies both CPU architectures, actual FIFO descriptors, guard +cleanup, socket/send denials, runtime compatibility and concurrent generated +client calls through the compiled policy. Production guest/node patches and +full libkrun rollout remain operator checks. diff --git a/docs/lambda-microvm/README.md b/docs/lambda-microvm/README.md index 80e5e468..b8fbbcc5 100644 --- a/docs/lambda-microvm/README.md +++ b/docs/lambda-microvm/README.md @@ -511,7 +511,7 @@ appear in `api/src/config.ts`. | `EGRESS_GATEWAY_FILE_SERVER_URL` / `_TOOL_CALL_SERVER_URL` | Gateway only | Private upstream origins. Do not expose them to the runner. | | `FILE_SERVER_URL` | Worker only | Direct source used to fetch authorized by-reference inputs before pushing them into the runner cache. Do not bake it into the runner. | | `CODEAPI_EGRESS_LEDGER_REQUIRED=true` / `REDIS_*` | Gateway | Makes grant replay/revocation state fail closed in Redis. | -| `SANDBOX_ALLOWED_LOCAL_NETWORK_PORT` / `SANDBOX_FORWARD_TARGET` | Runner image | Configures the narrow tool-call socket proxy; it does not start or expose the socket by itself. The target host/port must match `EGRESS_GATEWAY_URL`. | +| `SANDBOX_ALLOWED_LOCAL_NETWORK_PORT` / `SANDBOX_FORWARD_TARGET` | Runner image | Configures the per-invocation tool-call pipe broker; Node starts only on an authorized post-restore execution. The target host/port must match `EGRESS_GATEWAY_URL`. | | `CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY` | Worker only | Signs the manifest after the gateway returns the scoped grant. | | `SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY` / `SANDBOX_REQUIRE_EGRESS_MANIFEST=true` | Runner image | Verifies and requires the signed per-execution manifest. | diff --git a/docs/openshell/README.md b/docs/openshell/README.md index a139ab7c..880225b0 100644 --- a/docs/openshell/README.md +++ b/docs/openshell/README.md @@ -143,7 +143,7 @@ full authorization and delivery contract before implementation. ## Run the lifecycle probe -Requirements: Node 20+, an OpenShell `0.1.2` CLI, a registered named evaluation +Requirements: Node 22.21+, an OpenShell `0.1.2` CLI, a registered named evaluation gateway using the matching release, and a digest-pinned shell/coreutils image pullable by that gateway. Install and configure these separately on dedicated infrastructure. This repository does not install or start a gateway. diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 4026a755..99f19992 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -119,7 +119,7 @@ a separate machine. Install: -- Node.js 20.11 or newer (Node.js 24 is supported); +- Node.js 22.21 or newer (Node.js 24 is supported); - Git; - `bubblewrap`, `socat`, and `ripgrep` on Linux; - Bash 5.2 or newer and `jq` when Bash Programmatic Tool Calling is enabled; diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 0a242639..3ce4cd09 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -310,7 +310,7 @@ workerSandbox: pullPolicy: IfNotPresent pythonVersion: "3.14.4" nodeVersion: "24.15.0" - bunVersion: "1.3.14" + bunVersion: "1.4.2" bashPackageVersion: "5.2.0" backoffLimit: 2 ttlSecondsAfterFinished: 3600 diff --git a/launcher/Dockerfile b/launcher/Dockerfile index b2b02386..4acd2094 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -28,17 +28,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ - && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800 + && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ + && git submodule update --init --recursive RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup -FROM oven/bun:1.3.14-debian AS sandbox-build +FROM oven/bun:1.4.2-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -64,12 +67,14 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node RUN chmod +x /usr/sbin/nsjail WORKDIR /sandbox_api -COPY api/package.json ./ -RUN bun install +COPY api/package.json api/bun.lock ./ +RUN bun ci COPY api/tsconfig.json ./ COPY api/src ./src diff --git a/packages/code/README.md b/packages/code/README.md index d348fe88..0437ecfb 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -3,6 +3,9 @@ Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed code environment to LibreChat Code API. +Requires Node.js 22.21 or newer. CI tests Node.js 22 and 24; Node.js 20 is +unsupported. + For a complete machine setup and operations guide, see the [self-hosted worker runbook](../../docs/remote-bridge/worker-runbook.md). diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json index 426b950f..a27188a6 100644 --- a/packages/code/package-lock.json +++ b/packages/code/package-lock.json @@ -21,7 +21,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": ">=20.11" + "node": ">=22.21" } }, "node_modules/@anthropic-ai/sandbox-runtime": { diff --git a/packages/code/package.json b/packages/code/package.json index d3cf714b..95eea841 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -65,7 +65,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": ">=20.11" + "node": ">=22.21" }, "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", diff --git a/service/scripts/dump-pipe-preamble.ts b/service/scripts/dump-pipe-preamble.ts new file mode 100644 index 00000000..86ec11fa --- /dev/null +++ b/service/scripts/dump-pipe-preamble.ts @@ -0,0 +1,6 @@ +process.env.DOTENV_CONFIG_QUIET = 'true'; +const { generatePreamble } = await import('../src/preamble'); +process.stdout.write(generatePreamble({ + callbackUrl: 'http://unused.invalid', callbackToken: 'pipe-test-token', + executionId: 'pipe-test-execution', tools: [], +})); diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 69b416d8..3c2ca9d2 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -67,15 +67,15 @@ describe('runtime version configuration', () => { }); }); - it('maps Bun JavaScript and TypeScript requests to Bun 1.3.14', () => { + it('maps Bun JavaScript and TypeScript requests to Bun 1.4.2', () => { expect(languageConfig[Languages.js]).toMatchObject({ language: 'bun-js', - version: '1.3.14', + version: '1.4.2', fileName: 'index.js', }); expect(languageConfig[Languages.ts]).toMatchObject({ language: 'bun-ts', - version: '1.3.14', + version: '1.4.2', fileName: 'main.ts', }); }); diff --git a/service/src/config.ts b/service/src/config.ts index 29ae854d..7ba8b9b4 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -10,10 +10,10 @@ import { export const languageConfig: Record = { [Languages.bash]: { language: 'bash', version: '5.2.0', fileName: 'script.sh' }, - [Languages.js]: { language: 'bun-js', version: '1.3.14', fileName: 'index.js' }, + [Languages.js]: { language: 'bun-js', version: '1.4.2', fileName: 'index.js' }, [Languages.node]: { language: 'node', version: '24.15.0', fileName: 'index.js' }, [Languages.py]: { language: 'python', version: '3.14.4', fileName: 'main.py' }, - [Languages.ts]: { language: 'bun-ts', version: '1.3.14', fileName: 'main.ts' }, + [Languages.ts]: { language: 'bun-ts', version: '1.4.2', fileName: 'main.ts' }, }; const languageAliases: Record = { diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 5a36ebc5..2af379c1 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -785,8 +785,8 @@ app.post('/tool-call', async (req, res) => { if (!executionId || !callId || !opaqueCallbackToken) { /* Generic 404 so a sandbox attacker probing paths cannot tell that * `/tool-call` is the live route by reading the error body. The - * body and headers must MATCH the tool-call socket proxy's own 404 - * for unknown paths (see api/src/tool-call-socket-proxy.ts) byte- + * body and headers must match the generic unknown-route response + * for unknown paths byte- * for-byte — case included — otherwise the sandbox can still * fingerprint the real route by comparing response shapes. */ logger.warn( diff --git a/service/src/preamble.test.ts b/service/src/preamble.test.ts index 803e1839..b4293fa5 100644 --- a/service/src/preamble.test.ts +++ b/service/src/preamble.test.ts @@ -15,71 +15,16 @@ const baseConfig = { tools: [], }; -describe('generatePreamble — Unix-vs-TCP transport gate', () => { - /* Codex audit follow-up: a path-existence check (`os.path.exists`) on - * /tmp/tcs.sock was user-spoofable inside the sandbox — in - * legacy/no-proxy mode, user code could `open('/tmp/tcs.sock', 'w')` - * and force the next tool call into the unix-socket branch (where - * connect() then fails with ENOTSOCK) instead of falling back to TCP. - * The fix probes a real AF_UNIX connect() at preamble import time - * (before user code runs) and caches the verdict. These tests assert - * the rendered preamble keeps that property. */ - - test('renders a probe that performs an AF_UNIX connect, not a path-existence check', () => { - const preamble = generatePreamble(baseConfig); - expect(preamble).toContain('def _probe_tool_call_socket():'); - /* The probe must construct an AF_UNIX socket and call connect(). A - * file-existence check would not detect a regular file or stale - * inode planted by the user. */ - expect(preamble).toMatch(/AF_UNIX/); - expect(preamble).toMatch(/\.connect\(_TOOL_CALL_SOCKET\)/); - /* Regression guard against reintroducing the user-spoofable check. */ - expect(preamble).not.toMatch( - /if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/, - ); - }); - - test('caches the probe verdict at module load (before user code can plant a spoof)', () => { +describe('generatePreamble — anonymous pipe transport', () => { + test('uses fixed FIFO descriptors and fails closed without a pipe-enabled runner', () => { const preamble = generatePreamble(baseConfig); - /* The probe call must appear at top level of the preamble, NOT - * inside _do_request. Otherwise a user could plant a regular file - * at the path between calls and flip the gate per-request. */ - const probeCallIdx = preamble.indexOf( - '_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()', - ); - expect(probeCallIdx).toBeGreaterThan(-1); - /* _do_request must consult the cached verdict, not re-probe. */ - const doReqMatch = preamble.match( - /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, - ); - expect(doReqMatch).not.toBeNull(); - expect(doReqMatch![0]).toContain('_USE_TOOL_CALL_SOCKET'); - expect(doReqMatch![0]).not.toContain('_probe_tool_call_socket('); - }); - - test('keeps the TCP fallback path for legacy / no-proxy deployments', () => { - const preamble = generatePreamble(baseConfig); - /* The fallback must still construct the URL from _CALLBACK_URL and - * delegate to _tcp_request. Without this, runners without the - * proxy bind-mount would have no way to reach the orchestrator. */ - const doReqMatch = preamble.match( - /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, - ); - expect(doReqMatch).not.toBeNull(); - expect(doReqMatch![0]).toContain('_CALLBACK_URL + path'); - expect(doReqMatch![0]).toContain('_tcp_request('); - }); - - test('the literal socket path stays inside the preamble, not in os.environ', () => { - const preamble = generatePreamble(baseConfig); - /* Hardening item from PR #1648: TOOL_CALL_SOCKET is no longer - * exported. The path must remain hardcoded so the preamble does - * not depend on env-var injection. */ - expect(preamble).toContain('_TOOL_CALL_SOCKET = "/tmp/tcs.sock"'); - expect(preamble).not.toMatch( - /os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/, - ); - expect(preamble).not.toMatch(/os\.environ\[['"]TOOL_CALL_SOCKET['"]\]/); + expect(preamble).toContain('_PIPE_WRITE_FD = 3'); + expect(preamble).toContain('_PIPE_READ_FD = 4'); + expect(preamble).toContain('_stat.S_ISFIFO(os.fstat(_fd).st_mode)'); + expect(preamble).toContain('blocking tool calls require a pipe-enabled runner'); + expect(preamble).not.toContain('_tcp_request'); + expect(preamble).not.toContain('/tmp/tcs.sock'); + expect(preamble).not.toContain('import socket'); }); }); @@ -131,14 +76,14 @@ describe('extractPendingFromStdout — input hash metadata', () => { }); }); -describe('createProgrammaticPayload — tool-call socket opt-in', () => { +describe('createProgrammaticPayload — tool-call pipe capability', () => { const req = { body: { code: 'print("ok")', }, } as Parameters[0]['req']; - test('requests the sandbox socket for blocking PTC only', () => { + test('requests the sandbox pipe capability for blocking PTC only', () => { const payload = createProgrammaticPayload({ req, session_id: 'session-blocking', @@ -152,7 +97,7 @@ describe('createProgrammaticPayload — tool-call socket opt-in', () => { expect(payload.tool_call_socket).toBe(true); }); - test('does not request the sandbox socket for replay PTC', () => { + test('does not request the sandbox pipe capability for replay PTC', () => { const payload = createProgrammaticPayload({ req, session_id: 'session-replay', diff --git a/service/src/preamble.ts b/service/src/preamble.ts index 94685c6c..4e8003c2 100644 --- a/service/src/preamble.ts +++ b/service/src/preamble.ts @@ -324,7 +324,7 @@ ${nameComment} _input = {${inputDict}} * Generates dual-mode stubs that work with or without await */ export function generatePreamble(config: PreambleConfig): string { - const { callbackUrl, callbackToken, executionId, tools } = config; + const { callbackToken, executionId, tools } = config; // Header with HTTP callback infrastructure (dual-mode version) let preamble = ` @@ -337,73 +337,137 @@ import json import sys import os import asyncio -import socket as _socket -import http.client as _http_client from typing import Any, Dict, List, Optional, Union -from urllib import request, error as urllib_error -_CALLBACK_URL = "${callbackUrl}" -_CALLBACK_TOKEN = "${callbackToken}" -_EXECUTION_ID = "${executionId}" +_CALLBACK_TOKEN = ${JSON.stringify(callbackToken)} +_EXECUTION_ID = ${JSON.stringify(executionId)} _TOOL_CALL_COUNTER = 0 _TOOL_CALL_LOCK = asyncio.Lock() -# The proxy bind-mounts this fixed path into every sandbox; the env var -# version was dropped to keep the path off os.environ introspection. -_TOOL_CALL_SOCKET = "/tmp/tcs.sock" - -class _UnixHTTPConnection(_http_client.HTTPConnection): - """HTTPConnection that tunnels through a Unix domain socket""" - def __init__(self, socket_path): - super().__init__("localhost") - self._socket_path = socket_path - def connect(self): - self.sock = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) - self.sock.settimeout(self.timeout) - self.sock.connect(self._socket_path) - -def _unix_request(method, path, body=None, headers=None, timeout=300): - conn = _UnixHTTPConnection(_TOOL_CALL_SOCKET) - conn.timeout = timeout - conn.connect() - conn.request(method, path, body=body, headers=headers or {}) - resp = conn.getresponse() - data = resp.read() - conn.close() - return resp.status, data - -def _tcp_request(method, url, body=None, headers=None, timeout=300): - req = request.Request(url, data=body, headers=headers or {}, method=method) - with request.urlopen(req, timeout=timeout) as resp: - return resp.status, resp.read() - -def _probe_tool_call_socket(): - # One-shot probe: is the path a live AF_UNIX listener? A real - # connect() succeeds against the proxy; a regular file or stale - # node returns ENOTSOCK / ECONNREFUSED. Done at preamble import - # time, *before* user code runs, so a malicious user cannot spoof - # /tmp/tcs.sock to flip the gate (a previous os.path.exists check - # was user-spoofable in legacy/no-proxy mode). Each NsJail - # invocation gets a fresh /tmp tmpfs so the probe result cannot - # bleed across jobs. +import stat as _stat +import struct as _struct +import threading as _threading +import select as _select +import time as _time +from concurrent.futures import Future as _Future + +# These two anonymous pipe ends are granted only to an authorized invocation. +_PIPE_WRITE_FD = 3 +_PIPE_READ_FD = 4 +_PIPE_MAX_FRAME = 1024 * 1024 +_PIPE_OWNER_PID = os.getpid() +_PIPE_LOCK = _threading.Lock() +_PIPE_WRITE_LOCK = _threading.Lock() +_PIPE_PENDING = {} +_PIPE_SEQUENCE = 0 +_PIPE_READER = None +_PIPE_FAILURE = None + +# Spawned multiprocessing workers re-import this module without the granted +# descriptors. They may import tool definitions, but cannot use the capability. +if __name__ == "__mp_main__": + _PIPE_OWNER_PID = None +else: + for _fd in (_PIPE_WRITE_FD, _PIPE_READ_FD): + try: + if not _stat.S_ISFIFO(os.fstat(_fd).st_mode): + raise OSError("tool-call descriptor is not an anonymous pipe") + os.set_inheritable(_fd, False) + except OSError as _error: + raise RuntimeError("blocking tool calls require a pipe-enabled runner") from _error + os.set_blocking(_PIPE_WRITE_FD, False) + +def _pipe_fail(error): + global _PIPE_FAILURE + with _PIPE_LOCK: + _PIPE_FAILURE = error + pending = list(_PIPE_PENDING.values()) + _PIPE_PENDING.clear() + for future in pending: + if not future.done(): + future.set_exception(error) + +def _pipe_read_exact(size): + chunks = [] + remaining = size + while remaining: + chunk = os.read(_PIPE_READ_FD, remaining) + if not chunk: + raise EOFError("tool-call broker disconnected") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + +def _pipe_read_responses(): try: - s = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) - s.settimeout(0.5) - s.connect(_TOOL_CALL_SOCKET) - s.close() - return True - except OSError: - return False - -_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket() + while True: + size = _struct.unpack("!I", _pipe_read_exact(4))[0] + if not 0 < size <= _PIPE_MAX_FRAME: + raise ValueError("invalid tool-call response size") + response = json.loads(_pipe_read_exact(size).decode("utf-8")) + if not isinstance(response, dict) or not isinstance(response.get("status"), int) or not isinstance(response.get("body"), str): + raise ValueError("invalid tool-call response") + with _PIPE_LOCK: + future = _PIPE_PENDING.pop(response.get("id"), None) + # A late response after a caller timeout has no pending waiter. + if future is not None: + future.set_result((response["status"], response["body"].encode("utf-8"))) + except Exception as error: + _pipe_fail(error) def _do_request(method, path, body=None, headers=None, timeout=300): - # Cached at import time (see _probe_tool_call_socket). User code - # cannot influence this decision because it has not yet executed. - if _USE_TOOL_CALL_SOCKET: - return _unix_request(method, path, body, headers, timeout) - url = _CALLBACK_URL + path - return _tcp_request(method, url, body, headers, timeout) + global _PIPE_READER, _PIPE_SEQUENCE + if os.getpid() != _PIPE_OWNER_PID: + raise RuntimeError("tool calls must run in the invocation's primary process") + if method != "POST" or path != "/tool-call": + raise ValueError("unsupported tool-call route") + deadline = _time.monotonic() + timeout + future = _Future() + with _PIPE_LOCK: + if _PIPE_FAILURE is not None: + raise _PIPE_FAILURE + if len(_PIPE_PENDING) >= 16: + raise RuntimeError("too many concurrent tool calls") + if _PIPE_READER is None: + _PIPE_READER = _threading.Thread(target=_pipe_read_responses, daemon=True) + _PIPE_READER.start() + _PIPE_SEQUENCE += 1 + sequence = _PIPE_SEQUENCE + _PIPE_PENDING[sequence] = future + try: + payload = json.dumps({ + "id": sequence, + "headers": {key.lower(): value for key, value in (headers or {}).items()}, + "body": (body or b"").decode("utf-8") + }).encode("utf-8") + if not 0 < len(payload) <= _PIPE_MAX_FRAME: + raise ValueError("tool-call request too large") + framed = _struct.pack("!I", len(payload)) + payload + # Serialize writes, but allow concurrent upstream requests and responses. + if not _PIPE_WRITE_LOCK.acquire(timeout=max(0, deadline - _time.monotonic())): + raise TimeoutError("tool-call write timeout") + try: + remaining = memoryview(framed) + while remaining: + wait = deadline - _time.monotonic() + if wait <= 0 or not _select.select([], [_PIPE_WRITE_FD], [], wait)[1]: + raise TimeoutError("tool-call write timeout") + try: + written = os.write(_PIPE_WRITE_FD, remaining[:4096]) + except BlockingIOError: + continue + remaining = remaining[written:] + except Exception as error: + # A partial frame cannot be reused for the next request. + _pipe_fail(error) + os.close(_PIPE_WRITE_FD) + raise + finally: + _PIPE_WRITE_LOCK.release() + return future.result(timeout=max(0, deadline - _time.monotonic())) + finally: + with _PIPE_LOCK: + _PIPE_PENDING.pop(sequence, None) class ToolExecutionError(Exception): """Raised when a tool call fails""" @@ -500,7 +564,7 @@ async def _execute_tool_internal_async(tool_name: str, tool_input: Dict[str, Any * History is delivered as a file (not an env var) so very large histories * are not bounded by the Linux ARG_MAX / MAX_ARG_STRLEN ceiling. * - * No socket bind mount, no Tool Call Server callback, no long-poll. + * No pipe capability, no Tool Call Server callback, no long-poll. */ export function generateReplayPreamble(config: ReplayPreambleConfig): string { const { executionId, tools } = config; @@ -519,7 +583,7 @@ import os import asyncio from typing import Any, Dict, List, Optional, Union -_EXECUTION_ID = "${executionId}" +_EXECUTION_ID = ${JSON.stringify(executionId)} _PTC_SENTINEL_START = "${scopedStart}" _PTC_SENTINEL_END = "${scopedEnd}" _PTC_HISTORY_PATH = os.environ.get("PTC_HISTORY_PATH") or "${PTC_HISTORY_SANDBOX_PATH}" diff --git a/tests/nsjail_unix_gc.py b/tests/nsjail_unix_gc.py index 84cec7b5..a576fcdb 100644 --- a/tests/nsjail_unix_gc.py +++ b/tests/nsjail_unix_gc.py @@ -8,14 +8,11 @@ import array import ctypes as c import errno -import http.client import multiprocessing as mp import os import socket import struct import sys -import tempfile -import threading class SockFilter(c.Structure): @@ -94,48 +91,6 @@ def test_sends(libc, pair): right.close() -def test_unix_http(): - with tempfile.TemporaryDirectory() as directory: - pathname = os.path.join(directory, "tcs.sock") - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: - listener.bind(pathname) - listener.listen(1) - listener.settimeout(5) - errors = [] - - def serve(): - try: - with listener.accept()[0] as peer: - peer.settimeout(5) - request = b"" - while b"\r\n\r\n" not in request: - chunk = peer.recv(4096) - assert chunk, "client disconnected before HTTP headers" - request += chunk - assert request.startswith(b"POST /tool-call HTTP/1.1") - peer.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\n" - b"Connection: close\r\n\r\n{\"ok\":true}") - except Exception as error: - errors.append(error) - - thread = threading.Thread(target=serve) - thread.start() - connection = http.client.HTTPConnection("localhost", timeout=5) - connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - connection.sock.settimeout(5) - connection.sock.connect(pathname) - try: - connection.request("POST", "/tool-call", body=b"{}") - response = connection.getresponse() - assert response.status == 200 - assert response.read() == b'{"ok":true}' - finally: - connection.close() - thread.join(5) - assert not thread.is_alive() - assert not errors, errors - - def main(): # Open before installation to verify the rule also covers inherited FDs. inherited = socket.socketpair() @@ -154,9 +109,13 @@ def main(): assert libc.syscall(syscall, 0, 0, 0, 0, 0, 0) == -1 assert c.get_errno() == errno.EPERM test_sends(libc, inherited) - for kind in (socket.SOCK_STREAM, socket.SOCK_DGRAM): - test_sends(libc, socket.socketpair(socket.AF_UNIX, kind)) - test_unix_http() + test_sends(libc, socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)) + for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6, socket.AF_NETLINK): + expect_eperm(lambda: socket.socket(domain, socket.SOCK_STREAM)) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM)) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 6)) + import asyncio + asyncio.run(asyncio.sleep(0)) for method in ("fork", "spawn"): context = mp.get_context(method) queue = context.Queue() @@ -169,7 +128,7 @@ def main(): queue.join_thread() with context.Pool(2) as pool: assert pool.map(square, [2, 3, 4]) == [4, 9, 16] - print("PASS: descriptor sends denied; Unix IPC, HTTP, queues and pools work") + print("PASS: descriptor sends denied; socket creation denied; stream IPC, asyncio, queues and pools work") if __name__ == "__main__": diff --git a/tests/pipe_runtime_image.cjs b/tests/pipe_runtime_image.cjs new file mode 100644 index 00000000..8b2d1008 --- /dev/null +++ b/tests/pipe_runtime_image.cjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +// Packaging regression: run in each guest rootfs. Uses the image's own broker, +// Node, bridge, NsJail and guard, with namespaces disabled for this smoke test. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); + +if (process.argv[2] === '--client') { + assert(fs.fstatSync(3).isFIFO()); + assert(fs.fstatSync(4).isFIFO()); + const frame = Buffer.from(JSON.stringify({ id: 1, headers: { + 'x-execution-id': 'image-smoke', 'x-callback-token': 'image-token', 'x-tool-call-id': 'image-call', + }, body: '{"tool_name":"echo","input":{}}' })); + const prefix = Buffer.alloc(4); prefix.writeUInt32BE(frame.length); + const bytes = Buffer.concat([prefix, frame]); + for (let offset = 0; offset < bytes.length;) offset += fs.writeSync(3, bytes, offset, bytes.length - offset); + function readExact(size) { + const bytes = Buffer.alloc(size); + for (let offset = 0; offset < size;) { + const count = fs.readSync(4, bytes, offset, size - offset); + assert(count > 0, 'broker response ended prematurely'); offset += count; + } + return bytes; + } + const reply = JSON.parse(readExact(readExact(4).readUInt32BE()).toString()); + assert.deepEqual(reply, { id: 1, status: 200, body: 'packaged runtime works' }); + console.log('PASS: image Node/broker/bridge/NsJail/guard pipe round trip'); +} else { + (async () => { + const upstream = http.createServer(async (req, res) => { + assert.equal(req.url, '/tool-call'); + assert.equal(req.headers['x-callback-token'], 'image-token'); + for await (const bytes of req) {} + res.end('packaged runtime works'); + }); + upstream.listen(0, '127.0.0.1'); await once(upstream, 'listening'); + let child; + let timer; + try { + child = spawn('/usr/local/bin/tool-call-pipe-bridge', [ + '--broker', '/usr/local/bin/node', '/sandbox_api/.build/tool-call-pipe-broker.cjs', + '/usr/sbin/nsjail', '--mode', 'e', '--disable_clone_newnet', '--disable_clone_newuser', + '--disable_clone_newns', '--disable_clone_newpid', '--disable_clone_newipc', + '--disable_clone_newuts', '--disable_clone_newcgroup', '--disable_proc', '--disable_rlimits', + '--pass_fd', '3', '--pass_fd', '4', '--', '/usr/local/bin/spec-guard', + '--tool-call-pipes', '/usr/local/bin/node', '/tests/pipe_runtime_image.cjs', '--client', + ], { stdio: ['ignore', 'inherit', 'inherit'], env: { + ...process.env, SANDBOX_FORWARD_TARGET: `http://127.0.0.1:${upstream.address().port}`, + }}); + timer = setTimeout(() => child.kill('SIGKILL'), 5000); + const [code, signal] = await once(child, 'exit'); + assert.equal(code, 0, `image pipe runtime failed (${signal})`); + } finally { + clearTimeout(timer); + if (child && child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + upstream.closeAllConnections(); upstream.close(); + } + })().catch(error => { console.error(error); process.exitCode = 1; }); +} diff --git a/tests/specguard_fd_isolation.py b/tests/specguard_fd_isolation.py deleted file mode 100644 index d7aa79cf..00000000 --- a/tests/specguard_fd_isolation.py +++ /dev/null @@ -1,289 +0,0 @@ -""" -spec-guard FD isolation regression test. - -Reproduces the audit's "permanent sandbox brick" pattern: - - 1. A first /api/v2/execute opens N socket FDs against /tmp/tcs.sock - and never closes them (simulates malicious sandbox code). - 2. After the first job exits, the runner's per-job FD slot bookkeeping - is supposed to be back to baseline. - 3. A SECOND /api/v2/execute runs trivial Python and reports its - inherited FD count via /proc/self/fd. - -Without `spec-guard` closing inherited descriptors before `execvp()`, the -second job's user process starts with whatever FDs leaked through the -runner -> NsJail -> child chain. With the fix, the second job's Python -process sees only stdin/stdout/stderr (plus whatever it itself opens). - -The test asserts BOTH: - - The simulated leak in job 1 actually opens many FDs. - - The follow-up job 2 starts with a clean FD table (<= a small bound). - -Usage (local docker compose stack): - - SANDBOX_REQUIRE_EGRESS_MANIFEST=false CODEAPI_HARDENED_SANDBOX_MODE=false \\ - docker compose up -d - python3 services/codeapi/tests/specguard_fd_isolation.py -""" -from __future__ import annotations - -import json -import os -import sys -import urllib.error -import urllib.request - -SANDBOX_URL = os.environ.get("SANDBOX_URL", "http://127.0.0.1:2000") -REQUEST_TIMEOUT_S = float(os.environ.get("SPEC_GUARD_TEST_TIMEOUT_S", "30")) - -# After spec-guard runs and the user's Python interpreter starts, a tiny -# number of FDs are legitimately open: 0/1/2 plus whatever Python opened -# during startup (typically the dirfd it iterates /proc/self/fd through). -# Anything above this bound is leaked descriptors. -MAX_LEGITIMATE_INHERITED_FDS = 10 - - -def execute(language: str, code: str, timeout_ms: int = 15_000) -> dict: - body = { - "language": language, - "version": "*", - "files": [{"name": "main.py" if language == "python" else "main.sh", - "content": code}], - "stdin": "", - "args": [], - "run_timeout": timeout_ms, - } - data = json.dumps(body).encode("utf-8") - req = urllib.request.Request( - f"{SANDBOX_URL}/api/v2/execute", - data=data, - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_S) as resp: - return json.loads(resp.read().decode("utf-8")) - - -STORM_TARGET = 500 -"""Open many more connections than the proxy's maxConnections cap (64 by -default). The kernel queues some (listen backlog) and accepts the rest as -slots free up via the proxy's idle timer. This matches the audit's -literal red-team script — `for i in range(500): connect(); leak`. Without -this volume, the test wouldn't exercise the steady-state-pressure path -the audit broke on. The storm job exits without closing any of them; the -proxy must reclaim accept-side FDs and the runner must remain serviceable.""" - -MIN_STORM_OPENED = 200 -"""Lower bound on what the storm job must successfully open before we -trust the test was meaningful. Without this assertion the test would -silently degrade — if the proxy ever started refusing all connections -at the kernel layer, the storm would open 0 sockets, the FD-poisoning -preconditions would never be exercised, and the regression would still -report PASS. 200 is comfortably above the production cap of 64 so the -kernel accept queue + proxy reaping cycle is exercised.""" - - -def fd_storm_then_report() -> None: - """First job: open many connections to /tmp/tcs.sock and never close them. - The connections live until the user process exits — but the proxy and - runner-side accounting may retain state. Spec-guard must not let any of - that leak into the NEXT job's process.""" - code = ( - "import os, socket, json\n" - # The runner bind-mounts the proxy socket at this fixed path; the - # TOOL_CALL_SOCKET env var was dropped to keep the path off - # `os.environ`, so the test references the literal path instead. - "s = '/tmp/tcs.sock'\n" - "leaked = []\n" - "errors = []\n" - # Append every socket to a list so CPython's refcount keeps the FD - # alive until process exit. Without this, each loop iteration - # rebinds `sk` and the previous socket is GC'd before the next - # connect() runs — the loop reports `opened == STORM_TARGET` but - # only ONE FD is ever actually held, so the FD-poisoning - # preconditions the test exists to exercise are never created. - f"for _ in range({STORM_TARGET}):\n" - " try:\n" - " sk = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" - " sk.connect(s); leaked.append(sk)\n" - " except OSError as e:\n" - " errors.append(str(e)); break\n" - # Read /proc/self/fd to PROVE we're actually holding `opened` FDs - # concurrently, so the assertion downstream isn't satisfied by - # the loop counter alone. - "concurrent_fds = len([x for x in os.listdir('/proc/self/fd') if x.isdigit()])\n" - "print(json.dumps({'opened': len(leaked), 'concurrent_fds': concurrent_fds, " - "'target': " - + str(STORM_TARGET) + ", 'errors': errors[:3]}))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: storm job exited non-zero: {run!r}", file=sys.stderr) - sys.exit(1) - stdout = (run.get("stdout") or "").strip() - print(f"[storm-job] {stdout}") - last_line = stdout.splitlines()[-1] if stdout else "" - try: - payload = json.loads(last_line) - except json.JSONDecodeError: - print( - f"FAIL: storm job did not produce JSON status line; got {last_line!r}", - file=sys.stderr, - ) - sys.exit(1) - opened = int(payload.get("opened", 0)) - concurrent = int(payload.get("concurrent_fds", 0)) - if opened < MIN_STORM_OPENED: - print( - f"FAIL: storm job opened only {opened} sockets (required >= " - f"{MIN_STORM_OPENED}); the FD-poisoning preconditions were not " - f"exercised, so a downstream PASS would be vacuous. errors: " - f"{payload.get('errors')}", - file=sys.stderr, - ) - sys.exit(1) - # Defend against the "vacuous pass via socket reuse" mode — assert - # the storm process is actually HOLDING the FDs concurrently in - # /proc/self/fd, not just opening-then-closing in a loop. - if concurrent < MIN_STORM_OPENED: - print( - f"FAIL: storm job reported {opened} opens but /proc/self/fd " - f"shows only {concurrent} concurrent FDs — sockets are being " - f"GC'd between iterations and the test is vacuous.", - file=sys.stderr, - ) - sys.exit(1) - - -def report_inherited_fd_count() -> int: - """Second job: introspect /proc/self/fd. Spec-guard should have closed - everything but stdin/out/err before this Python invocation began.""" - code = ( - "import os, json\n" - "fds = sorted(int(x) for x in os.listdir('/proc/self/fd') if x.isdigit())\n" - "print(json.dumps({'fd_count': len(fds), 'fds': fds[:20]}))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: report job exited non-zero: {run!r}", file=sys.stderr) - sys.exit(1) - stdout = (run.get("stdout") or "").strip() - print(f"[report-job] {stdout}") - payload = json.loads(stdout.split("\n")[-1]) - return int(payload["fd_count"]) - - -def post_storm_recovery() -> int: - """The audit's specific failure: after a socket storm, subsequent - dynamic-binary executions (python/node/bash/bun) failed during - library-load with `Error 24` (EMFILE). Run a real exec in each - language and require code 0 — proves the dynamic loader is no - longer poisoned by inherited descriptors.""" - cases = [ - ("python", "import sys; print('python', sys.version_info[:3])"), - ("node", "console.log('node', process.version)"), - ("bash", "echo bash $(/bin/bash --version | head -1)"), - ] - failed = 0 - for lang, code in cases: - result = execute(lang, code) - run = result.get("run") or {} - out = (run.get("stdout") or "").strip() - err = (run.get("stderr") or "").strip() - ok = run.get("code") == 0 and out and not err - marker = "ok " if ok else "FAIL" - print(f" [{marker}] {lang:6} code={run.get('code')!r:>4} stdout={out!r:<50} stderr={err[:80]!r}") - if not ok: - failed += 1 - # Specifically guard against the audit's EMFILE-on-loader pattern - if "Error 24" in err or "Too many open files" in err: - print(f" ^ audit-mode regression: dynamic loader hit EMFILE") - failed += 1 - return failed - - -def ptc_route_opacity_check() -> bool: - """Probe the proxy at /tool-call (with no PTC headers) and at unknown - paths from inside the sandbox; both must produce BYTE-IDENTICAL wire - responses, otherwise an attacker can still fingerprint the real route - by reading the response framing (ETag, X-Request-ID, charset, etc.). - The original audit broke on this exact gap — the response BODY matched - but the upstream's Express-added headers leaked through the proxy.""" - code = ( - "import os, socket, json\n" - # Same fixed-path reasoning as fd_storm_then_report(). - "s = '/tmp/tcs.sock'\n" - "responses = {}\n" - "for path in ['/tool-call', '/randomroute']:\n" - " sk = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" - " sk.connect(s)\n" - " sk.sendall(('POST ' + path + ' HTTP/1.1\\r\\nHost: x\\r\\n" - "Content-Length: 0\\r\\nConnection: close\\r\\n\\r\\n').encode())\n" - " buf = b''\n" - " while True:\n" - " chunk = sk.recv(4096)\n" - " if not chunk: break\n" - " buf += chunk\n" - " sk.close()\n" - " # Strip the volatile Date header before comparing\n" - " lines = buf.split(b'\\r\\n')\n" - " filtered = b'\\r\\n'.join(l for l in lines if not l.lower().startswith(b'date:'))\n" - " responses[path] = filtered.decode('utf-8', 'replace')\n" - "print(json.dumps(responses))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: opacity probe exited non-zero: {run!r}", file=sys.stderr) - return False - payload = json.loads((run.get("stdout") or "").splitlines()[-1]) - tc, rr = payload["/tool-call"], payload["/randomroute"] - if tc == rr: - print(f" [ok ] /tool-call response is byte-identical to /randomroute " - f"({len(tc)} bytes)") - return True - print(f" [FAIL] wire responses differ — sandbox can fingerprint the route:") - print(f" /tool-call: {tc!r}") - print(f" /randomroute: {rr!r}") - return False - - -def main() -> int: - print(f"=== spec-guard FD isolation regression — runner: {SANDBOX_URL} ===") - print(f"step 1: open {STORM_TARGET} socket FDs in a first job (audit's exact red-team script)") - fd_storm_then_report() - print("step 2: launch a second job and report its inherited FD count") - fd_count = report_inherited_fd_count() - print(f" -> second job sees {fd_count} FDs (bound: {MAX_LEGITIMATE_INHERITED_FDS})") - if fd_count > MAX_LEGITIMATE_INHERITED_FDS: - print( - f"FAIL: second job inherited {fd_count} FDs — " - f"spec-guard did NOT close inherited descriptors before execvp", - file=sys.stderr, - ) - return 1 - print("step 3: post-storm recovery for python / node / bash") - failed = post_storm_recovery() - if failed: - print(f"FAIL: {failed} language(s) regressed", file=sys.stderr) - return 1 - print("step 4: PTC route opacity (byte-identical 404 wire responses)") - if not ptc_route_opacity_check(): - return 1 - print("PASS: spec-guard isolated the second job's FD table, " - "all runtimes recovered post-storm, and the /tool-call route " - "is wire-indistinguishable from any unknown path") - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except urllib.error.HTTPError as exc: - print(f"HTTPError {exc.code}: {exc.read()[:300]!r}", file=sys.stderr) - sys.exit(2) - except urllib.error.URLError as exc: - print(f"URLError: {exc}", file=sys.stderr) - sys.exit(2) diff --git a/tests/specguard_tool_call_pipes.py b/tests/specguard_tool_call_pipes.py new file mode 100644 index 00000000..549b1fa9 --- /dev/null +++ b/tests/specguard_tool_call_pipes.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Verify the native guard preserves only validated pipe capabilities.""" +import fcntl +import os + + +def run_guard(pipes, preserve, reversed_ends=False): + request = os.pipe() + response = os.pipe() + ordinary = os.open('/dev/null', os.O_RDWR) + original = [request[0 if reversed_ends else 1] if pipes else ordinary, + response[1 if reversed_ends else 0] if pipes else ordinary, ordinary] + sources = [fcntl.fcntl(fd, fcntl.F_DUPFD_CLOEXEC, 10) for fd in original] + child = os.fork() + if child == 0: + for source, destination in zip(sources, [3, 4, 9]): + os.dup2(source, destination, inheritable=True) + code = ''' +import os, stat +for fd in ([9] if PRESERVE else [3,4,9]): + try: + os.fstat(fd) + except OSError: + pass + else: + raise AssertionError("unrelated FD survived guard: " + str(fd)) +if PRESERVE: + assert stat.S_ISFIFO(os.fstat(3).st_mode) + assert stat.S_ISFIFO(os.fstat(4).st_mode) +'''.replace('PRESERVE', repr(preserve)) + args = [os.environ.get('SPEC_GUARD_BINARY', '/usr/local/bin/spec-guard')] + if preserve: + args.append('--tool-call-pipes') + os.execv(args[0], args + ['/usr/bin/python3', '-c', code]) + for fd in sources + list(request) + list(response) + [ordinary]: + os.close(fd) + _, status = os.waitpid(child, 0) + return os.waitstatus_to_exitcode(status) + + +assert run_guard(True, True) == 0 +assert run_guard(True, False) == 0 +assert run_guard(False, True) == 1 +assert run_guard(True, True, reversed_ends=True) == 1 +print('PASS: guard preserves only explicit FIFO endpoints and rejects regular files') diff --git a/tests/tool_call_pipes.py b/tests/tool_call_pipes.py new file mode 100644 index 00000000..c9388530 --- /dev/null +++ b/tests/tool_call_pipes.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""End-to-end broker -> anonymous pipes -> NsJail -> guard -> Python client. + +Runs in the native nsjail-ipc-test image with Docker's outer filter disabled. +Arguments: rendered policy, built Node broker, generated Python preamble. +""" +import ctypes +import http.server +import json +import os +import subprocess +import sys +import tempfile +import threading +import time + + +class Upstream(http.server.BaseHTTPRequestHandler): + calls = [] + + def log_message(self, *args): + pass + + def do_POST(self): + assert self.path == '/tool-call' + assert self.headers['X-Execution-ID'] == 'pipe-test-execution' + assert self.headers['X-Callback-Token'] == 'pipe-test-token' + data = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + value = data['input']['n'] + self.calls.append(value) + time.sleep((8 - value) * 0.01) # Responses arrive out of order. + body = json.dumps({'success': True, 'result': value}).encode() + self.send_response(200) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def main(): + policy, broker, preamble = sys.argv[1:] + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Upstream) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + with tempfile.TemporaryDirectory() as directory: + program = os.path.join(directory, 'client.py') + with open(preamble) as source: + client = source.read() + client += ''' +import errno +import socket +import subprocess +assert _stat.S_ISFIFO(os.fstat(3).st_mode) +assert _stat.S_ISFIFO(os.fstat(4).st_mode) +try: + os.fstat(9) +except OSError: + pass +else: + raise AssertionError("unrelated inherited descriptor survived spec-guard") +for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6): + try: + socket.socket(domain, socket.SOCK_STREAM) + except OSError as error: + assert error.errno == errno.EPERM + else: + raise AssertionError("socket creation was allowed") +async def check(): + results = await asyncio.gather(*[_execute_tool_internal_async("echo", {"n": n}) for n in range(8)]) + assert results == list(range(8)), results +asyncio.run(check()) +# The capability must not leak through ordinary subprocess exec. +assert subprocess.check_output([sys.executable, "-c", "import os; print(os.path.exists('/proc/self/fd/3'))"]).strip() == b'False' +print("PASS: real pipes, restricted filter, guard cleanup and concurrent Python tool calls") +''' + with open(program, 'w') as output: + output.write(client) + env = {**os.environ, 'SANDBOX_FORWARD_TARGET': f'http://127.0.0.1:{server.server_port}', + 'TCS_PIPE_BRIDGE': '/usr/local/bin/tool-call-pipe-bridge'} + command = ['/usr/local/bin/tool-call-pipe-bridge', '--broker', '/usr/local/bin/node', broker, '/nsjail/nsjail', '--mode', 'e', + '--disable_clone_newnet', '--disable_clone_newuser', '--disable_clone_newns', + '--disable_clone_newpid', '--disable_clone_newipc', '--disable_clone_newuts', + '--disable_clone_newcgroup', '--disable_proc', '--disable_rlimits', + '--pass_fd', '3', '--pass_fd', '4', + '--seccomp_policy', policy, '--', '/usr/local/bin/spec-guard', + '--tool-call-pipes', '/usr/bin/python3', program] + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + assert sorted(Upstream.calls) == list(range(8)), Upstream.calls + print(result.stdout.strip()) + # Losing the response reader must terminate even with no buffered reply. + with tempfile.TemporaryDirectory() as directory: + closer = os.path.join(directory, 'close-response.py') + with open(closer, 'w') as output: + output.write("import os,time; os.close(4); time.sleep(30)") + interrupted = subprocess.run(command[:-1] + [closer], env=env, + capture_output=True, text=True, timeout=2) + assert interrupted.returncode == 137, (interrupted.returncode, interrupted.stdout, interrupted.stderr) + print('PASS: closing the response reader promptly terminates the invocation') + # A guest that drains rejection replies must still lose the capability + # once malformed-claim frames exhaust the broker's admission budget. + with tempfile.TemporaryDirectory() as directory: + flooder = os.path.join(directory, 'rejected-frames.py') + with open(flooder, 'w') as output: + output.write("""import json,os,struct,threading,time +threading.Thread(target=lambda: [os.read(4, 4096) for _ in range(1000)], daemon=True).start() +frames = [] +for id in range(1, 258): + frame = json.dumps({'id': id, 'headers': {}, 'body': ''}).encode() + frames.append(struct.pack('!I', len(frame)) + frame) +data = b''.join(frames) +while data: + data = data[os.write(3, data):] +time.sleep(30) +""") + interrupted = subprocess.run(command[:-1] + [flooder], env=env, + capture_output=True, text=True, timeout=2) + assert interrupted.returncode == 137, (interrupted.returncode, interrupted.stdout, interrupted.stderr) + print('PASS: drained rejection-frame flood promptly terminates the invocation') + # A killed API/controller must not orphan its broker, relay or job. Become a + # subreaper so this test can reap both adopted descendants in Docker. + assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0 + with tempfile.TemporaryDirectory() as directory: + marker = os.path.join(directory, 'job.pid') + sleeper = os.path.join(directory, 'sleep.py') + with open(sleeper, 'w') as output: + output.write(f"import os,time; open({marker!r},'w').write(str(os.getpid())); time.sleep(300)") + command[-1] = sleeper + controller = os.path.join(directory, 'controller.js') + broker_marker = os.path.join(directory, 'broker.pid') + with open(controller, 'w') as output: + output.write("import {spawn} from 'node:child_process'; import fs from 'node:fs';" + + f"const args={json.dumps(command)};" + + "const child=spawn(args[0], args.slice(1), {stdio:['pipe','inherit','inherit']}); child.stdin.end();" + + f"fs.writeFileSync({json.dumps(broker_marker)},String(child.pid));" + + "setInterval(()=>{},1000);") + running = subprocess.Popen(['/usr/local/bin/bun', controller], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 5 + while not os.path.exists(marker) and time.monotonic() < deadline: + time.sleep(0.01) + assert os.path.exists(marker), 'job did not start' + with open(marker) as source: + job_pid = int(source.read()) + with open(f'/proc/{job_pid}/status') as source: + relay_pid = next(int(line.split()[1]) for line in source if line.startswith('PPid:')) + with open(broker_marker) as source: + broker_pid = int(source.read()) + running.kill() + running.communicate(timeout=5) + for pid in [job_pid, relay_pid, broker_pid]: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + reaped, status = os.waitpid(pid, os.WNOHANG) + if reaped: + assert os.waitstatus_to_exitcode(status) == -9 + break + time.sleep(0.01) + else: + os.kill(pid, 9) + raise AssertionError('orphan survived broker death') + finally: + if running.poll() is None: + running.kill() + running.communicate(timeout=5) + print('PASS: API death kills and reaps broker, relay and invocation') + server.shutdown() + server.server_close() + + +if __name__ == '__main__': + main() diff --git a/tests/tool_call_runner.cjs b/tests/tool_call_runner.cjs new file mode 100644 index 00000000..d40e0721 --- /dev/null +++ b/tests/tool_call_runner.cjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node +// Run inside the sandbox-build image with its Mac/Docker capabilities and a +// read-only /pkgs mount. Arguments: generated pipe preamble, Python version. +// Exercises the signed API route and the normal NsJail namespace/mount setup. +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const http = require('node:http'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +function canonical(value) { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; + return JSON.stringify(value); +} +async function main() { + const [preamblePath, version = '3.14.4'] = process.argv.slice(2); + const calls = []; + const upstream = http.createServer(async (req, res) => { + try { + assert.equal(req.url, '/tool-call'); + assert.equal(req.headers['x-execution-id'], 'pipe-test-execution'); + assert.equal(req.headers['x-callback-token'], 'pipe-test-token'); + let bytes = ''; + for await (const part of req) bytes += part; + const { n } = JSON.parse(bytes).input; + calls.push(n); + await sleep((8 - n) * 10); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ success: true, result: n })); + } catch (error) { res.writeHead(500).end(JSON.stringify({ success: false, error: String(error) })); } + }); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + const port = upstream.address().port; + const { privateKey, publicKey } = crypto.generateKeyPairSync('ed25519'); + const api = spawn('/sandbox_api/entrypoint.sh', [], { cwd: '/sandbox_api', stdio: ['ignore', 'inherit', 'inherit'], env: { + ...process.env, PORT: '2000', SANDBOX_USE_CGROUPV2: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: publicKey.export({ type: 'spki', format: 'pem' }), + SANDBOX_ALLOWED_LOCAL_NETWORK_PORT: String(port), SANDBOX_FORWARD_TARGET: `http://127.0.0.1:${port}`, + SANDBOX_OUTPUT_MAX_SIZE: '65536', SANDBOX_LOG_LEVEL: 'info', + }}); + try { + for (let n = 0; n < 200; n++) { + if (api.exitCode !== null) throw new Error(`API exited: ${api.exitCode}`); + try { if ((await fetch('http://127.0.0.1:2000/')).ok) break; } catch {} + if (n === 199) throw new Error('API did not start'); + await sleep(50); + } + const runtimes = await (await fetch('http://127.0.0.1:2000/api/v2/runtimes')).json(); + assert(runtimes.some(runtime => runtime.language === 'python' && runtime.version === version), JSON.stringify(runtimes)); + const outerNamespaces = Object.fromEntries(['net', 'pid', 'mnt', 'user', 'ipc', 'uts', 'cgroup'].map(name => [name, fs.readlinkSync(`/proc/self/ns/${name}`)])); + const client = fs.readFileSync(preamblePath, 'utf8') + ` +import errno, socket, subprocess, sys, multiprocessing as mp +def _mp_echo(sender): + try: + _do_request('POST', '/tool-call', '{}') + except RuntimeError as error: + assert 'primary process' in str(error) + else: + raise AssertionError('child acquired tool-call capability') + sender.send('child IPC works') + sender.close() +if __name__ == '__main__': + assert sys.version.split()[0] == ${JSON.stringify(version)} + assert os.getuid() == 65534 + assert _stat.S_ISFIFO(os.fstat(3).st_mode) and _stat.S_ISFIFO(os.fstat(4).st_mode) + for _namespace, _outside in ${JSON.stringify(outerNamespaces)}.items(): + assert os.readlink('/proc/self/ns/' + _namespace) != _outside, _namespace + for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6): + try: socket.socket(domain, socket.SOCK_STREAM) + except OSError as error: assert error.errno == errno.EPERM + else: raise AssertionError('socket creation allowed') + assert subprocess.check_output([sys.executable, '-c', "import os; print(os.path.exists('/proc/self/fd/3'))"]).strip() == b'False' + async def check(): + results = await asyncio.gather(*[_execute_tool_internal_async('echo', {'n': n}) for n in range(8)]) + assert results == list(range(8)), results + asyncio.run(check()) + # Pools need /dev/shm and forkserver needs a named AF_UNIX listener. + # Test the supported pipe-only spawn/fork multiprocessing paths. + for method in ('spawn', 'fork'): + ctx = mp.get_context(method) + receiver, sender = ctx.Pipe(duplex=False) + child = ctx.Process(target=_mp_echo, args=(sender,)) + child.start() + sender.close() + assert receiver.poll(5), method + assert receiver.recv() == 'child IPC works' + receiver.close() + child.join(5) + assert child.exitcode == 0, (method, child.exitcode) + print('PASS: packaged Python, signed API, NsJail namespaces, pipes, concurrent tools, subprocess and multiprocessing') +`; + async function execute(content, pipes, language = 'python', runtimeVersion = version) { + const body = { language, version: runtimeVersion, session_id: `pipe-canary-${crypto.randomUUID()}`, files: [{ name: language === 'bun-js' ? 'main.js' : language === 'bun-ts' ? 'main.ts' : 'main.py', content }], tool_call_socket: pipes, run_timeout: 30000 }; + const now = Math.floor(Date.now() / 1000); + const claims = { v: 1, exec_id: 'pipe-test-execution', tenant_id: 'canary', user_id: 'canary', session_key: 'canary:user:canary', input_files: [], read_sessions: [], output_session_id: body.session_id, max_upload_bytes: 52428800, max_output_files: 50, max_requests: 1000, iat: now, exp: now + 300, tool_call_socket: pipes, execute_body_sha256: crypto.createHash('sha256').update(canonical(body)).digest('base64url') }; + const payload = canonical(claims); + body.execution_manifest = `${Buffer.from(payload).toString('base64url')}.${crypto.sign(null, Buffer.from(payload), privateKey).toString('base64url')}`; + const response = await fetch('http://127.0.0.1:2000/api/v2/execute', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const result = await response.json(); + assert.equal(response.status, 200, JSON.stringify(result)); + assert.equal(result.run.code, 0, JSON.stringify(result)); + return result; + } + const result = await execute(client, true); + assert.equal(calls.length, 8); + assert.deepEqual([...calls].sort((a, b) => a - b), [0,1,2,3,4,5,6,7]); + assert(result.run.stdout.includes('PASS:'), JSON.stringify(result)); + console.log(result.run.stdout.trim()); + // Ordinary jobs still work and do not receive the tool-call descriptors. + const plain = await execute("import os; assert not os.path.exists('/proc/self/fd/3'); print('PASS: ordinary execution without pipe capability')", false); + console.log(plain.run.stdout.trim()); + for (const language of ['bun-js', 'bun-ts']) { + const js = await execute("console.log('PASS: Bun ' + Bun.version)", false, language, '1.4.2'); + assert(js.run.stdout.includes('PASS: Bun 1.4.2'), JSON.stringify(js)); + console.log(language + ': ' + js.run.stdout.trim()); + } + } finally { + api.kill('SIGTERM'); + await once(api, 'exit'); + upstream.closeAllConnections(); upstream.close(); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; }); From 4401420df463cb195230a201404056cca1e10f81 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 13:52:11 -0400 Subject: [PATCH 71/75] =?UTF-8?q?=F0=9F=A7=B5=20fix:=20Replace=20Runtime?= =?UTF-8?q?=20Socketpairs=20With=20Anonymous=20Pipes=20(#312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: deny socketpairs with pipe-only Python runtime IPC * test: document and verify Bun socketpair spawn denial * fix: refresh mutable package-init images during runtime rollout * test: render runtime rollout chart without optional dependencies * fix: preserve offline package-init image policy --- .github/workflows/ci.yml | 7 ++- api/Dockerfile | 3 + api/src/nsjail.test.ts | 6 +- api/src/nsjail.ts | 12 ++-- build-packages.sh | 3 + docker/Dockerfile.package-init | 1 + docker/Dockerfile.worker-sandbox | 1 + docker/package-init.sh | 2 + .../_sandbox_multiprocessing.py | 47 +++++++++++++++ docker/python-runtime/_sandbox_pipe.py | 32 ++++++++++ docker/python-runtime/install.py | 41 +++++++++++++ docs/SECURITY-MAINTENANCE.md | 32 +++++++--- docs/adr/0011-tool-call-pipes.md | 11 ++-- docs/adr/0012-pipe-only-python-runtime.md | 58 +++++++++++++++++++ helm/codeapi/templates/package-init-job.yaml | 3 +- helm/codeapi/values.yaml | 3 +- tests/nsjail_unix_gc.py | 23 +++++++- tests/package_init_runtime_rollout.sh | 31 ++++++++++ tests/python_pipe_runtime.py | 47 +++++++++++++++ tests/tool_call_runner.cjs | 10 +++- 20 files changed, 340 insertions(+), 33 deletions(-) create mode 100644 docker/python-runtime/_sandbox_multiprocessing.py create mode 100644 docker/python-runtime/_sandbox_pipe.py create mode 100644 docker/python-runtime/install.py create mode 100644 docs/adr/0012-pipe-only-python-runtime.md create mode 100755 tests/package_init_runtime_rollout.sh create mode 100644 tests/python_pipe_runtime.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58f68b7b..98ddb2ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: - name: Block-root package delivery run: tests/block_root_package_delivery.sh + - name: Python runtime package-init rollout + run: tests/package_init_runtime_rollout.sh + - name: KVM guest resolver handoff run: tests/kvm_guest_dns.sh @@ -173,12 +176,12 @@ jobs: launcher-guest-test /tests/pipe_runtime_image.cjs - name: Verify descriptor denial and runtime IPC with the installed filter run: | - docker run --rm --security-opt seccomp=unconfined \ + docker run --rm --init --security-opt seccomp=unconfined \ --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ nsjail-ipc-test \ bash -c 'python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so && python3 /tests/specguard_tool_call_pipes.py && gcc -O2 -static -Wall -Wextra -Werror -DSPEC_GUARD_TEST_DISABLE_CLOSE_RANGE -o /tmp/guard-fallback /tmp/spec-guard.c && SPEC_GUARD_BINARY=/tmp/guard-fallback python3 /tests/specguard_tool_call_pipes.py' - docker run --rm --security-opt seccomp=unconfined \ + docker run --rm --init --security-opt seccomp=unconfined \ --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ --mount "type=bind,source=$RUNNER_TEMP/pipe-client.py,target=/client.py,readonly" \ --mount "type=bind,source=$GITHUB_WORKSPACE/api/.build/tool-call-pipe-broker.cjs,target=/broker.cjs,readonly" \ diff --git a/api/Dockerfile b/api/Dockerfile index 0ee343a7..f953f537 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -38,6 +38,8 @@ RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridg # Native policy/transport verification without building the full runner. FROM nsjail-builder AS nsjail-ipc-test +COPY docker/python-runtime /python-runtime +RUN python3 /python-runtime/install.py COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node COPY --from=oven/bun:1.4.2-debian /usr/local/bin/bun /usr/local/bin/bun @@ -68,6 +70,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index 73dc3863..ab046b7f 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -394,14 +394,14 @@ describe('NsJail seccomp policy', () => { expect(rule).not.toMatch(/== FUTEX_(WAIT|WAKE)(?:\s|$)/); }); - test('blocks descriptor-passing sends while preserving ordinary Unix socket IPC', () => { + test('blocks descriptor-passing sends and all socketpair creation', () => { const policy = seccompPolicy(); const errnoBlock = policy.split('ERRNO(1) {')[1]?.split(' }')[0] ?? ''; for (const name of ['sendmsg', 'sendmmsg', 'io_uring_setup', 'io_uring_enter', 'io_uring_register']) { expect(errnoBlock).toMatch(new RegExp(`\\b${name}\\b[,\\s]`)); } - // Restrict socketpairs to runtime stream IPC; tool calls use real pipes. - expect(policy).toContain('socketpair(domain, type, protocol) { domain != AF_UNIX || (type & 0xf) != SOCK_STREAM || protocol != 0 }'); + // The patched runtime and tool calls use anonymous pipes. + expect(policy).toContain(' socketpair'); for (const name of ['read', 'write', 'sendto', 'recvfrom']) { expect(policy).not.toMatch(new RegExp(`\\b${name}\\b`)); } diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 51b8700e..5c39884f 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -95,8 +95,6 @@ const archSpecificLowPrioritySyscalls = process.arch === 'arm64' const SECCOMP_POLICY = [ ...syscallDefines, - '#define AF_UNIX 1', - '#define SOCK_STREAM 1', '#define AF_INET 2', '#define AF_INET6 10', '#define AF_NETLINK 16', @@ -153,8 +151,8 @@ const SECCOMP_POLICY = [ /* CVE-2026-80521: AF_UNIX GC is reachable through SCM_RIGHTS descriptor * passing. Seccomp cannot inspect the msghdr/mmsghdr control data, so * refuse both send APIs outright, including on inherited sockets. - * The job receives only anonymous tool-call pipes. Restricted Unix - * stream socketpairs remain for asyncio and subprocess/runtime IPC. + * The job receives only anonymous tool-call pipes. Anonymous + * pipes replace socketpairs in the packaged Python runtime. * io_uring above must remain blocked: * its sendmsg operations bypass syscall-level filtering. */ ' sendmsg, sendmmsg,', @@ -185,10 +183,10 @@ const SECCOMP_POLICY = [ ' pidfd_open(pid) { pid == 1 },', ' pidfd_send_signal,', /* Deny every socket() family. VSOCK keeps its stronger KILL action above. - * Only anonymous Unix stream socketpairs are allowed for runtime IPC; - * sendmsg/sendmmsg remain denied, so no SCM_RIGHTS graph can be built. */ + * Deny socketpairs as well; packaged runtime IPC uses anonymous pipes. + * Keep sendmsg/sendmmsg denied for inherited descriptors too. */ ' socket(domain) { domain != AF_VSOCK },', - ' socketpair(domain, type, protocol) { domain != AF_UNIX || (type & 0xf) != SOCK_STREAM || protocol != 0 }', + ' socketpair', ' }', '}', 'USE sandbox DEFAULT ALLOW', diff --git a/build-packages.sh b/build-packages.sh index 67912ee5..1d635297 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -118,6 +118,9 @@ install_python() { rm -rf /tmp/Python-${PYTHON_VERSION}* " + docker cp "$SCRIPT_DIR/docker/python-runtime" "$CONTAINER_NAME:/tmp/python-runtime" + docker exec "$CONTAINER_NAME" "${pkg_dest}/bin/python3" /tmp/python-runtime/install.py + docker exec "$CONTAINER_NAME" bash -c "cat > ${pkg_dest}/pkg-info.json << 'EOF' { \"language\": \"python\", diff --git a/docker/Dockerfile.package-init b/docker/Dockerfile.package-init index c94cb8af..745346f8 100644 --- a/docker/Dockerfile.package-init +++ b/docker/Dockerfile.package-init @@ -23,6 +23,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 5e66baed..0117132d 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -75,6 +75,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh diff --git a/docker/package-init.sh b/docker/package-init.sh index 1ffaaf3a..4b96305f 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -87,6 +87,7 @@ echo "==============================================" echo "" packages_ready() { + [ -f "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/.sandbox-pipe-runtime-v1" ] && [ -f "/pkgs/python/${PYTHON_VERSION}/.package-installed" ] && [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/PIL" ] && [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/markitdown" ] && @@ -138,6 +139,7 @@ cd "Python-${PYTHON_VERSION}" ./configure --prefix="$PKG_DEST" --enable-optimizations 2>/dev/null make -j$(nproc) make install +"${PKG_DEST}/bin/python3" "${SCRIPT_DIR}/python-runtime/install.py" cd /tmp rm -rf /tmp/Python-${PYTHON_VERSION}* diff --git a/docker/python-runtime/_sandbox_multiprocessing.py b/docker/python-runtime/_sandbox_multiprocessing.py new file mode 100644 index 00000000..62fc2362 --- /dev/null +++ b/docker/python-runtime/_sandbox_multiprocessing.py @@ -0,0 +1,47 @@ +"""Duplex multiprocessing connections backed by two anonymous pipes.""" +import os +from multiprocessing.connection import Connection +from multiprocessing.context import reduction + + +class DuplexConnection(Connection): + def __init__(self, reader, writer): + super().__init__(reader) + self._writer = writer + + def _send(self, data): + remaining = memoryview(data) + while remaining: + written = os.write(self._writer, remaining) + remaining = remaining[written:] + + def _close(self): + try: + os.close(self._handle) + finally: + os.close(self._writer) + + +def rebuild(reader, writer): + return DuplexConnection(reader.detach(), writer.detach()) + + +def reduce_connection(connection): + connection._check_closed() + return rebuild, (reduction.DupFd(connection.fileno()), + reduction.DupFd(connection._writer)) + + +reduction.register(DuplexConnection, reduce_connection) + + +def duplex_pipe(): + first_reader, first_writer = os.pipe() + try: + second_reader, second_writer = os.pipe() + except BaseException: + os.close(first_reader) + os.close(first_writer) + raise + return (DuplexConnection(first_reader, second_writer), + DuplexConnection(second_reader, first_writer)) diff --git a/docker/python-runtime/_sandbox_pipe.py b/docker/python-runtime/_sandbox_pipe.py new file mode 100644 index 00000000..645dbc2a --- /dev/null +++ b/docker/python-runtime/_sandbox_pipe.py @@ -0,0 +1,32 @@ +"""Pipe-only Linux runtime IPC; seccomp still enforces socket denial.""" +import os + + +class WakeupPipe: + def __init__(self, fd): + self._fd = fd + + def __del__(self): + self.close() + + def fileno(self): + return self._fd + + def setblocking(self, blocking): + os.set_blocking(self._fd, blocking) + + def recv(self, size): + return os.read(self._fd, size) + + def send(self, data): + return os.write(self._fd, data) + + def close(self): + fd, self._fd = self._fd, -1 + if fd >= 0: + os.close(fd) + + +def wakeup_pipe(): + reader, writer = os.pipe2(os.O_CLOEXEC | os.O_NONBLOCK) + return WakeupPipe(reader), WakeupPipe(writer) diff --git a/docker/python-runtime/install.py b/docker/python-runtime/install.py new file mode 100644 index 00000000..61194cbb --- /dev/null +++ b/docker/python-runtime/install.py @@ -0,0 +1,41 @@ +"""Apply checked stdlib edits to the interpreter running this installer.""" +from pathlib import Path +import shutil +import sysconfig + +root = Path(sysconfig.get_path('stdlib')) +source = Path(__file__).resolve().parent +edits = [ + ('asyncio/selector_events.py', + ' self._ssock, self._csock = socket.socketpair()', + ' from _sandbox_pipe import wakeup_pipe\n self._ssock, self._csock = wakeup_pipe()'), + ('multiprocessing/connection.py', + ''' if duplex: + s1, s2 = socket.socketpair() + s1.setblocking(True) + s2.setblocking(True) + c1 = Connection(s1.detach()) + c2 = Connection(s2.detach())''', + ''' if duplex: + from _sandbox_multiprocessing import duplex_pipe + return duplex_pipe()'''), +] +prepared = [] +for name, old, new in edits: + path = root / name + text = path.read_text() + if text.count(new) == 1: + continue + if text.count(old) != 1: + raise RuntimeError(f'unsupported Python runtime layout: {path}') + prepared.append((path, text.replace(old, new))) +for name in ('_sandbox_pipe.py', '_sandbox_multiprocessing.py'): + shutil.copyfile(source / name, root / name) + for cached in (root / '__pycache__').glob(Path(name).stem + '.*.pyc'): + cached.unlink() +for path, text in prepared: + path.write_text(text) + # Avoid reusing timestamp-based bytecode after editing installed sources. + for cached in (path.parent / '__pycache__').glob(path.stem + '.*.pyc'): + cached.unlink() +(root / '.sandbox-pipe-runtime-v1').write_text('1\n') diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md index b8026383..f33e509f 100644 --- a/docs/SECURITY-MAINTENANCE.md +++ b/docs/SECURITY-MAINTENANCE.md @@ -49,12 +49,13 @@ Keep `io_uring_setup`, `io_uring_enter`, and `io_uring_register` denied: ring operations can submit sends without passing through these syscall filters. The job policy denies every `socket()` family (VSOCK retains its KILL action). -It allows only AF_UNIX SOCK_STREAM `socketpair` with protocol 0, including -CLOEXEC/NONBLOCK flags. Python asyncio and duplex multiprocessing pipes need -these anonymous socketpairs. Ordinary fork/spawn queues and pools are covered -by native tests. Named Unix listeners, `multiprocessing.Manager`, datagram or -seqpacket pairs, and descriptor-sharing APIs are unsupported. The send and -io_uring denials still prevent SCM_RIGHTS descriptor graphs on allowed pairs. +It also denies every `socketpair()`. The packaged Python runtime replaces +asyncio's wakeup socketpair with a nonblocking anonymous pipe and duplex +multiprocessing socketpairs with two anonymous pipes. Native tests cover +thread wakeups, signals, cancellation, subprocesses, duplex spawn/fork IPC, +queues and pools. Named Unix listeners, `multiprocessing.Manager`, forkserver, +and descriptor-sharing APIs remain unsupported. Keep the send and io_uring +denials for inherited descriptors and defense in depth. This mitigation is separate from PI-futex filtering. Verify vendor patches for the deployed guest and node kernels; a policy update does not establish patch @@ -143,7 +144,11 @@ production kernel patch status or third-party language-package compatibility. ### Coordinated rollout Deploy the service's new blocking preamble together with rebuilt runner images -containing the broker, relay and updated guard/policy. Drain old blocking jobs +containing the broker, relay and updated guard/policy. Rebuild Python packages +with the pipe-runtime patch before enabling socketpair denial. Package init +checks the `.sandbox-pipe-runtime-v1` marker and rebuilds unpatched packages. +Drain or version-route execution while replacing packages; rollback must pair +the policy with its compatible runtime packages. Drain old blocking jobs and route new blocking requests only to matching images during rollout. Old socket preambles fail on pipe-only runners; new pipe preambles fail immediately on old runners. There is no TCP or Unix-socket fallback. Replay mode is unchanged. @@ -174,3 +179,16 @@ Remaining service audit findings at this change: reassess if notifications or path filters become reachable. - `braces` (GHSA-vfj7-8cjw-p6xm): development-only dependency with no published fix. Do not feed untrusted patterns to build tooling; track an upstream fix. + +Bun 1.4.2 subprocess spawning with piped stdio also uses socketpairs and is +denied by this policy. Basic JS/TS execution remains supported. Replacing Bun +subprocess IPC requires a separate runtime change; do not re-enable socketpairs +to provide that compatibility. + +For PVC-backed Helm deployments, pin the package-init image to the same immutable +release as the runner. The chart refreshes the mutable `latest` tag with `Always`, +including upgrades that reuse an old `IfNotPresent` value, so a cached old init +image cannot skip the runtime-marker check. Publish the patched init image +before upgrading the runner; refresh does not make an unpublished image current. +An explicit `Never` policy is preserved for locally loaded kind/minikube images; +rebuild and reload the patched package-init image before a local upgrade. diff --git a/docs/adr/0011-tool-call-pipes.md b/docs/adr/0011-tool-call-pipes.md index 69e0b0ee..c4b53f96 100644 --- a/docs/adr/0011-tool-call-pipes.md +++ b/docs/adr/0011-tool-call-pipes.md @@ -12,8 +12,8 @@ Node and its TLS state must continue to start after MicroVM restore. The current stdio spawn APIs create socketpairs, not anonymous pipe endpoints. The existing guard also intentionally removes unrelated inherited descriptors. -Python asyncio and duplex multiprocessing pipes use Unix stream socketpairs; -a blanket socketpair denial would break these ordinary workloads. +Unmodified Python asyncio and duplex multiprocessing pipes use Unix stream +socketpairs. ADR 0012 replaces those runtime internals with anonymous pipes. ## Decision @@ -33,10 +33,9 @@ requests. The Python client clears descriptor inheritance and refuses calls from forked or spawned children. Spawned workers may re-import tool definitions without receiving or using the pipe capability. stdout/stderr and user stdin remain separate. -Deny all socket creation. Permit only protocol-0 Unix stream socketpairs for -runtime IPC, retaining the sendmsg/sendmmsg/io_uring denials. This preserves -asyncio and pipe-based multiprocessing using explicit spawn/fork contexts without permitting named socket access -or descriptor passing. No legacy TCP/socket fallback is provided. +Deny all socket and socketpair creation, retaining sendmsg/sendmmsg/io_uring +denials. The patched Python runtime preserves asyncio and duplex multiprocessing +using explicit spawn/fork contexts. No TCP/socket fallback is provided. ## Consequences diff --git a/docs/adr/0012-pipe-only-python-runtime.md b/docs/adr/0012-pipe-only-python-runtime.md new file mode 100644 index 00000000..ac1b844b --- /dev/null +++ b/docs/adr/0012-pipe-only-python-runtime.md @@ -0,0 +1,58 @@ +# 0012: Deny socketpairs with pipe-only Python runtime IPC + +Status: Proposed + +## Context + +ADR 0011 removed the named tool-call socket but retained Unix stream socketpairs +for Python runtime compatibility. Removing that exception reduces the kernel +surface available to untrusted code. A seccomp-only change breaks standard +asyncio initialization and duplex multiprocessing connections. + +## Decision + +Deny every socketpair in the job filter. Patch the packaged Linux Python stdlib +at build time, including the package-init and local package-builder paths: + +- Replace asyncio's self-socket with nonblocking CLOEXEC anonymous pipe endpoints. + Preserve thread wakeups, signal wakeup FDs, read callbacks and loop cleanup. +- Replace duplex multiprocessing connections with two anonymous pipes. Preserve + Connection framing, readable fileno/poll, close/EOF, and both descriptor ends + through the existing spawn reduction mechanism. +- Validate each stdlib edit before writing any edits. Unknown source layouts fail + the package build. Mark successful installation so package init cannot reuse an + unpatched package solely because its Python version matches. + +The trusted broker/relay may continue using sockets outside the jail. Untrusted +code receives only anonymous pipe capabilities. No socketpair emulation is +installed globally: direct socket APIs remain denied by seccomp. + +## Consequences + +Rebuild Python packages and deploy them with the new runner policy. Existing +unpatched packages are incompatible with the policy; drain or version-route +jobs during replacement. Snapshot/baked runners need rebuilt runtime contents. +The marker is build bookkeeping, not an authorization boundary. + +Other runtimes or third-party event loops that require socketpairs remain denied; +the patch only covers the standard Python runtime. Named listeners, forkserver, +managers and post-start descriptor sharing remain unsupported. Queues/pools +still require the jail to mount /dev/shm, independent of this change. + +Native regression tests exercise the compiled policy on both CPU architectures, +thread completion, signals, cancellation, async subprocesses, duplex spawn/fork +IPC and EOF, queues/pools, and generated concurrent tool calls. A local signed +runner canary additionally checks packaged Python and ordinary asyncio jobs. + +Bun 1.4.2 subprocess spawning with piped stdio also uses socketpairs and is +denied by this policy. Basic JS/TS execution remains supported. Replacing Bun +subprocess IPC requires a separate runtime change; do not re-enable socketpairs +to provide that compatibility. + +For PVC-backed Helm deployments, pin the package-init image to the same immutable +release as the runner. The chart refreshes the mutable `latest` tag with `Always`, +including upgrades that reuse an old `IfNotPresent` value, so a cached old init +image cannot skip the runtime-marker check. Publish the patched init image +before upgrading the runner; refresh does not make an unpublished image current. +An explicit `Never` policy is preserved for locally loaded kind/minikube images; +rebuild and reload the patched package-init image before a local upgrade. diff --git a/helm/codeapi/templates/package-init-job.yaml b/helm/codeapi/templates/package-init-job.yaml index 8d927503..baf81b17 100644 --- a/helm/codeapi/templates/package-init-job.yaml +++ b/helm/codeapi/templates/package-init-job.yaml @@ -40,7 +40,8 @@ spec: containers: - name: package-init image: "{{ .Values.workerSandbox.packages.initJob.image.repository }}:{{ .Values.workerSandbox.packages.initJob.image.tag }}" - imagePullPolicy: {{ .Values.workerSandbox.packages.initJob.image.pullPolicy }} + {{/* Refresh cached latest images; preserve explicit offline Never. */}} + imagePullPolicy: {{ if and (eq (.Values.workerSandbox.packages.initJob.image.tag | toString) "latest") (ne .Values.workerSandbox.packages.initJob.image.pullPolicy "Never") }}Always{{ else }}{{ .Values.workerSandbox.packages.initJob.image.pullPolicy }}{{ end }} env: - name: PYTHON_VERSION value: {{ .Values.workerSandbox.packages.initJob.pythonVersion | quote }} diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 3ce4cd09..4858fd71 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -306,8 +306,9 @@ workerSandbox: forceRebuild: false image: repository: codeapi-package-init + # Pin this tag to the same immutable release as the runner in production. tag: latest - pullPolicy: IfNotPresent + pullPolicy: Always pythonVersion: "3.14.4" nodeVersion: "24.15.0" bunVersion: "1.4.2" diff --git a/tests/nsjail_unix_gc.py b/tests/nsjail_unix_gc.py index a576fcdb..08c7cf8d 100644 --- a/tests/nsjail_unix_gc.py +++ b/tests/nsjail_unix_gc.py @@ -109,15 +109,32 @@ def main(): assert libc.syscall(syscall, 0, 0, 0, 0, 0, 0) == -1 assert c.get_errno() == errno.EPERM test_sends(libc, inherited) - test_sends(libc, socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)) for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6, socket.AF_NETLINK): expect_eperm(lambda: socket.socket(domain, socket.SOCK_STREAM)) expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM)) expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 6)) import asyncio - asyncio.run(asyncio.sleep(0)) + from python_pipe_runtime import check_asyncio, duplex_child + asyncio.run(check_asyncio()) for method in ("fork", "spawn"): context = mp.get_context(method) + left, right = context.Pipe() + process = context.Process(target=duplex_child, args=(right,)) + process.start() + right.close() + left.send_bytes(b'x' * 100_000) + assert left.poll(10) + assert left.recv_bytes() == b'x' * 100_000 + process.join(10) + assert process.exitcode == 0 + try: + left.recv_bytes() + except EOFError: + pass + else: + raise AssertionError('duplex peer EOF missing') + left.close() queue = context.Queue() process = context.Process(target=queue_child, args=(queue,)) process.start() @@ -128,7 +145,7 @@ def main(): queue.join_thread() with context.Pool(2) as pool: assert pool.map(square, [2, 3, 4]) == [4, 9, 16] - print("PASS: descriptor sends denied; socket creation denied; stream IPC, asyncio, queues and pools work") + print("PASS: descriptor sends denied; socket creation denied; socketpairs denied; pipe-based asyncio, duplex IPC, queues and pools work") if __name__ == "__main__": diff --git a/tests/package_init_runtime_rollout.sh b/tests/package_init_runtime_rollout.sh new file mode 100755 index 00000000..3bb8cf6c --- /dev/null +++ b/tests/package_init_runtime_rollout.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -r "$TMP_DIR"' EXIT +mkdir "$TMP_DIR/chart" +cp "$ROOT/helm/codeapi/values.yaml" "$TMP_DIR/chart/values.yaml" +cp -R "$ROOT/helm/codeapi/templates" "$TMP_DIR/chart/templates" +awk '/^dependencies:/{exit} {print}' "$ROOT/helm/codeapi/Chart.yaml" > "$TMP_DIR/chart/Chart.yaml" +render() { + helm template runtime-rollout "$TMP_DIR/chart" \ + --show-only templates/package-init-job.yaml \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.enabled=true \ + --set workerSandbox.packages.source=pvc "$@" +} +# Default mutable image must refresh on every pre-install/pre-upgrade hook. +render | grep -q 'imagePullPolicy: Always' +# Reused historical values must not bypass the runtime migration. +render --set workerSandbox.packages.initJob.image.pullPolicy=IfNotPresent \ + | grep -q 'imagePullPolicy: Always' +# Immutable release images retain their explicitly configured cache policy. +render --set workerSandbox.packages.initJob.image.tag=release-123 \ + --set workerSandbox.packages.initJob.image.pullPolicy=IfNotPresent \ + | grep -q 'imagePullPolicy: IfNotPresent' +# Local kind/minikube builds are deliberately loaded without a registry. +render -f "$ROOT/helm/codeapi/values-local.yaml" | grep -q 'imagePullPolicy: Never' +render --set workerSandbox.packages.initJob.image.pullPolicy=Never \ + | grep -q 'imagePullPolicy: Never' +echo 'PASS: package-init refreshes cached latest and preserves offline images' diff --git a/tests/python_pipe_runtime.py b/tests/python_pipe_runtime.py new file mode 100644 index 00000000..52a48095 --- /dev/null +++ b/tests/python_pipe_runtime.py @@ -0,0 +1,47 @@ +"""Runtime transition checks also run under the installed seccomp filter.""" +import asyncio +import os +import signal +import stat +import sys + + +def duplex_child(connection): + try: + assert stat.S_ISFIFO(os.fstat(connection.fileno()).st_mode) + connection.send_bytes(connection.recv_bytes()) + finally: + connection.close() + + +async def check_asyncio(): + loop = asyncio.get_running_loop() + assert stat.S_ISFIFO(os.fstat(loop._ssock.fileno()).st_mode) + assert stat.S_ISFIFO(os.fstat(loop._csock.fileno()).st_mode) + assert not os.get_inheritable(loop._ssock.fileno()) + assert not os.get_inheritable(loop._csock.fileno()) + # Executor completion crosses from a worker thread into the event loop. + values = await asyncio.wait_for(asyncio.gather(*[ + asyncio.to_thread(lambda n=n: n * n) for n in range(64) + ]), 10) + assert values == [n * n for n in range(64)] + awakened = asyncio.Event() + loop.add_signal_handler(signal.SIGUSR1, awakened.set) + try: + os.kill(os.getpid(), signal.SIGUSR1) + await asyncio.wait_for(awakened.wait(), 2) + finally: + loop.remove_signal_handler(signal.SIGUSR1) + process = await asyncio.create_subprocess_exec( + sys.executable, '-c', 'print("child works")', + stdout=asyncio.subprocess.PIPE) + output, _ = await asyncio.wait_for(process.communicate(), 10) + assert output == b'child works\n' and process.returncode == 0 + pending = asyncio.create_task(asyncio.sleep(30)) + pending.cancel() + try: + await pending + except asyncio.CancelledError: + pass + else: + raise AssertionError('cancellation lost') diff --git a/tests/tool_call_runner.cjs b/tests/tool_call_runner.cjs index d40e0721..4aab25e4 100644 --- a/tests/tool_call_runner.cjs +++ b/tests/tool_call_runner.cjs @@ -73,6 +73,9 @@ if __name__ == '__main__': try: socket.socket(domain, socket.SOCK_STREAM) except OSError as error: assert error.errno == errno.EPERM else: raise AssertionError('socket creation allowed') + try: socket.socketpair() + except OSError as error: assert error.errno == errno.EPERM + else: raise AssertionError('socketpair allowed') assert subprocess.check_output([sys.executable, '-c', "import os; print(os.path.exists('/proc/self/fd/3'))"]).strip() == b'False' async def check(): results = await asyncio.gather(*[_execute_tool_internal_async('echo', {'n': n}) for n in range(8)]) @@ -82,7 +85,8 @@ if __name__ == '__main__': # Test the supported pipe-only spawn/fork multiprocessing paths. for method in ('spawn', 'fork'): ctx = mp.get_context(method) - receiver, sender = ctx.Pipe(duplex=False) + receiver, sender = ctx.Pipe() + assert _stat.S_ISFIFO(os.fstat(receiver.fileno()).st_mode) child = ctx.Process(target=_mp_echo, args=(sender,)) child.start() sender.close() @@ -111,10 +115,10 @@ if __name__ == '__main__': assert(result.run.stdout.includes('PASS:'), JSON.stringify(result)); console.log(result.run.stdout.trim()); // Ordinary jobs still work and do not receive the tool-call descriptors. - const plain = await execute("import os; assert not os.path.exists('/proc/self/fd/3'); print('PASS: ordinary execution without pipe capability')", false); + const plain = await execute("import os, asyncio; assert not os.path.exists('/proc/self/fd/3'); asyncio.run(asyncio.to_thread(lambda: 1)); print('PASS: ordinary asyncio execution without pipe capability')", false); console.log(plain.run.stdout.trim()); for (const language of ['bun-js', 'bun-ts']) { - const js = await execute("console.log('PASS: Bun ' + Bun.version)", false, language, '1.4.2'); + const js = await execute("let denied = false; try { Bun.spawn([process.execPath, '-e', '0'], {stdout:'pipe', stderr:'pipe'}); } catch (error) { if (error.code !== 'EPERM' || error.syscall !== 'socketpair') throw error; denied = true; } if (!denied) throw new Error('socketpair spawn allowed'); console.log('PASS: Bun ' + Bun.version + ', socketpair-based spawn denied')", false, language, '1.4.2'); assert(js.run.stdout.includes('PASS: Bun 1.4.2'), JSON.stringify(js)); console.log(language + ': ' + js.run.stdout.trim()); } From 26d1ab8db0f040d714b9f7abbfa902f59474e107 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:47:30 +0200 Subject: [PATCH 72/75] =?UTF-8?q?=F0=9F=8C=BF=20feat:=20Report=20Lane=20Gi?= =?UTF-8?q?t=20Branch=20And=20Head=20On=20Command=20Results=20(#311)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Report Lane Git Branch And Head On Command Results The worker reads the checked-out branch and HEAD of the lane a command ran in and attaches them to execute_command results as laneGit. Code API validates the value and forwards it only when CODEAPI_BRIDGE_LANE_GIT is enabled and the worker advertised lane_git. * fix: Address Review Findings on Lane Git Reporting Compare a retried settlement with the stored one before replica-local lane Git sanitization, and skip the Git probe until lane_git is negotiated at registration. * fix: Bound Lane Git Probe and Pass Flag Through Compose Cap the post-command Git probe to the budget the command left, reserving settlement time, and pass CODEAPI_BRIDGE_LANE_GIT to the api and service-worker compose services. * fix: Harden Lane Git Probe Bounds and Reported Values Bound root resolution with the probe limit, enable the probe from the worker's active registration, require HEAD to peel to a commit, and reject Unicode control and format characters in branch names. * fix: Trust Only Probed Lane Git and Use the Worker Deadline Assign the lane Git worker on the normal startup path, budget the probe from the worker's assignment deadline, and drop any laneGit supplied by the command executor. * fix: Reuse Settlement Reads and Preserve Branch Whitespace Fold lane Git sanitization into the settle path's existing reads instead of a separate pre-pass, and strip only Git's trailing newline from probe output. * fix: Report Null for Non-UTF-8 Branch Names Read the lane branch as bytes and decode it strictly, so a ref name that is not valid UTF-8 is reported as null instead of a lossy copy. * fix: Revalidate Source Git Metadata Before Probing Probe a source or conversation checkout only while its .git is its own plain directory, so a command that rewrites .git to point at another repository cannot make the host-side probe report that repository's branch and head. * fix: Reject Linked Refs Before Probing Source Git Walk refs/ and require every entry to be a plain file or directory before the host-side probe runs, so a ref symlinked to another checkout cannot be followed. * fix: Guard Linked Lane Refs and Report Full Branch Names Check a linked lane's metadata HEAD and the shared refs before probing, read the branch from the full symbolic ref instead of --short, and avoid Dirent.parentPath so the refs walk works on Node 20.11. * fix: Stop the Git Refs Walk When the Probe Aborts Pass the probe signal into the source and linked worktree metadata checks and check it between directory operations, so an abandoned walk does not keep running after the command result is returned. * fix: Report Lane Git Only From a Stable Branch Read Read the branch before and after the head and omit laneGit when it changed in between, so a concurrent checkout cannot pair one branch with another branch's commit. * fix: Own Object Storage and Pin the Head Before Probing Require a link-free refs tree and object store with no alternates or reftable before the host-side probe, ignore replacement refs, and read the head from the branch ref the first read named. * fix: Disable Transports and Lazy Fetch in the Lane Git Probe Run the probe with no allowed protocols and lazy fetch disabled, so a rewritten .git/config cannot make a missing object trigger a fetch or a transport command on the host. --- docker-compose.yaml | 2 + packages/code/src/cli.ts | 26 +- packages/code/src/environment.ts | 4 +- packages/code/src/lane-git.test.ts | 830 +++++++++++++++++++++++ packages/code/src/lane-git.ts | 297 ++++++++ packages/code/src/projects.ts | 14 +- packages/code/src/protocol.ts | 61 ++ packages/code/src/worker.ts | 28 + packages/code/src/workspace.ts | 2 + packages/code/src/worktree-retirement.ts | 19 +- service/src/bridge/index.ts | 1 + service/src/bridge/lane-git.test.ts | 251 +++++++ service/src/bridge/lane-git.ts | 36 + service/src/bridge/router.test.ts | 49 ++ service/src/bridge/router.ts | 3 + service/src/bridge/store.ts | 64 +- service/src/config.ts | 5 + tests/compose-bridge-config.cjs | 3 + 18 files changed, 1681 insertions(+), 14 deletions(-) create mode 100644 packages/code/src/lane-git.test.ts create mode 100644 packages/code/src/lane-git.ts create mode 100644 service/src/bridge/lane-git.test.ts create mode 100644 service/src/bridge/lane-git.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index 1d7d28cf..db517b4c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,6 +14,7 @@ services: - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} + - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} @@ -69,6 +70,7 @@ services: - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} + - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 6b450b64..85ce947e 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -41,7 +41,8 @@ import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; -import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; +import { LaneGitWorkspaceTools, ownsGitMetadata, ownsLinkedWorktreeMetadata } from './lane-git.js'; +import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools, verifyLinkedWorktree } from './linked-worktrees.js'; import { GitWorktreeManager } from './worktrees.js'; import { WorktreeRetirementScheduler, @@ -1312,6 +1313,28 @@ async function run( }); workspaceTools = linkedWorktreeTools; } + // Probe Git only while the active registration advertises lane_git, which the worker decides. + let laneGitWorker: BridgeWorker | undefined; + if (workspaceTools?.capabilities.operations.includes('execute_command')) { + workspaceTools = new LaneGitWorkspaceTools({ + delegate: workspaceTools, + isEnabled: () => laneGitWorker?.commandResultFeatureActive('lane_git') === true, + async resolveRoot(request, signal) { + const source = roots.find((root) => root.id === request.workspaceId); + if (!source) return undefined; + if (request.workspaceInstanceId != null) { + const root = await conversationWorktrees?.plannedRoot(request.workspaceId, request.workspaceInstanceId); + return root != null && (await ownsGitMetadata(root, signal)) ? root : undefined; + } + if (request.worktree != null) { + signal?.throwIfAborted(); + const lane = await verifyLinkedWorktree(source.root, request.worktree, source.identity); + return (await ownsLinkedWorktreeMetadata(lane.commonGitDir, request.worktree, signal)) ? lane.root : undefined; + } + return (await ownsGitMetadata(source.root, signal)) ? source.root : undefined; + }, + }); + } const retirementSources = roots.filter((root) => root.writable); const debugLogs = process.env.LIBRECHAT_CODE_LOG_LEVEL?.trim().toLowerCase() === 'debug'; @@ -1633,6 +1656,7 @@ async function run( ); }, }); + laneGitWorker = worker; if (runtimeSessionId !== undefined) { await worker.refreshCredential(controller.signal); await worker.register(controller.signal); diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 576a149f..c527729f 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -237,12 +237,13 @@ export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { async execute( request: WorkspaceToolRequest, signal?: AbortSignal, + context?: { deadlineAtMs?: number }, ): Promise { if ( request.operation !== 'execute_command' || !request.environmentAction ) { - return this.delegate.execute(request, signal); + return this.delegate.execute(request, signal, context); } const environment = this.environments.get(request.workspaceId); const action = environment?.definition.actions?.find( @@ -271,6 +272,7 @@ export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { cwd: '.', }, signal, + context, ); } } diff --git a/packages/code/src/lane-git.test.ts b/packages/code/src/lane-git.test.ts new file mode 100644 index 00000000..27b400b5 --- /dev/null +++ b/packages/code/src/lane-git.test.ts @@ -0,0 +1,830 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rename, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +import { LaneGitWorkspaceTools, ownsGitMetadata, ownsLinkedWorktreeMetadata, readLaneGit, snapshotLaneGit } from './lane-git.js'; +import { isValidBridgeWorkspaceToolCapabilities, isWorkspaceLaneGit, isWorkspaceToolResult } from './protocol.js'; +import { BridgeWorker } from './worker.js'; + +import type { TestContext } from 'node:test'; +import type { WorkspaceExecuteCommandRequest, WorkspaceToolRequest } from './protocol.js'; + +const exec = promisify(execFile); +const incarnationId = 'incarnation-00000001'; +const identity = ['-c', 'user.name=Test', '-c', 'user.email=test@example.com']; + +async function scratch(t: TestContext): Promise { + const root = await realpath(await mkdtemp(join(tmpdir(), 'lane-git-'))); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} + +async function repo(root: string, name: string, commit = true): Promise { + const directory = join(root, name); + await mkdir(directory, { recursive: true }); + await exec('git', ['init', '--initial-branch=main', directory]); + if (commit) await exec('git', ['-C', directory, ...identity, 'commit', '--allow-empty', '-m', 'initial']); + return directory; +} + +async function sha(directory: string): Promise { + return (await exec('git', ['-C', directory, 'rev-parse', 'HEAD'])).stdout.trim(); +} + +test('isolated worktree lane reports its own branch and head', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'librechat/conversation-abcd1234-0123', lane]); + assert.deepEqual(await readLaneGit(lane), { branch: 'librechat/conversation-abcd1234-0123', head: await sha(lane) }); + assert.equal((await readLaneGit(source))?.branch, 'main'); +}); + +test('source checkout reports whatever is checked out', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/pr-header']); + assert.deepEqual(await readLaneGit(source), { branch: 'feature/pr-header', head: await sha(source) }); +}); + +test('a branch renamed by the agent shows up on the next read', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + assert.equal((await readLaneGit(source))?.branch, 'main'); + await exec('git', ['-C', source, 'branch', '-m', 'renamed-by-agent']); + assert.equal((await readLaneGit(source))?.branch, 'renamed-by-agent'); +}); + +test('detached HEAD reports a null branch and the commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '--detach']); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); +}); + +test('a repository without commits reports the branch and a null head', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source', false); + assert.deepEqual(await readLaneGit(source), { branch: 'main', head: null }); +}); + +test('a branch name beyond the bound reports null and never forwards it', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const long = ['a'.repeat(100), 'b'.repeat(100), 'c'.repeat(100)].join('/'); + await exec('git', ['-C', source, 'checkout', '-b', long]); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); +}); + +test('a directory that is not a repository reports nothing', async (t) => { + const root = await scratch(t); + assert.equal(await readLaneGit(root), undefined); +}); + +test('lane Git state never carries paths, remotes or credentials', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'remote', 'add', 'origin', 'https://user:secret@github.com/example/app.git']); + const state = await readLaneGit(source); + assert.deepEqual(Object.keys(state ?? {}).sort(), ['branch', 'head']); + const wire = JSON.stringify(state); + assert.equal(wire.includes(root), false); + assert.equal(wire.includes('secret'), false); + assert.equal(wire.includes('github.com'), false); +}); + +test('wire validator accepts only bounded branch and head', () => { + const head = 'a'.repeat(40); + assert.equal(isWorkspaceLaneGit({ branch: 'main', head }), true); + assert.equal(isWorkspaceLaneGit({ branch: null, head: null }), true); + assert.equal(isWorkspaceLaneGit({ branch: 'main', head: 'b'.repeat(64) }), true); + for (const bad of [ + { branch: 'x'.repeat(257), head }, + { branch: 'bad\nname', head }, + { branch: '', head }, + { branch: 'main', head: 'A'.repeat(40) }, + { branch: 'main', head: 'a'.repeat(41) }, + { branch: 'main' }, + { branch: 'main', head, path: '/srv/repo' }, + { branch: undefined, head }, + null, + [], + ]) { + assert.equal(isWorkspaceLaneGit(bad), false, JSON.stringify(bad)); + } +}); + +const commandRequest: WorkspaceExecuteCommandRequest = { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'git checkout -b agent-branch', +}; +const commandResult = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, +}; + +function delegate(onRun?: () => Promise) { + return { + capabilities: { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary' }], + }, + async execute(request: WorkspaceToolRequest) { + await onRun?.(); + return request.operation === 'execute_command' + ? commandResult + : { protocolVersion: 1 as const, operation: 'read_file' as const, workspaceId: 'primary', path: 'a', content: '', startLine: 1, endLine: 1, truncated: false }; + }, + }; +} + +test('a finished command refreshes the lane state, including a branch the command created', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(async () => { + await exec('git', ['-C', source, 'checkout', '-b', 'agent-branch']); + }), + resolveRoot: async () => source, + }); + const result = await tools.execute(commandRequest); + assert.deepEqual('laneGit' in result && result.laneGit, { branch: 'agent-branch', head: await sha(source) }); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + +test('file operations do not read Git', async () => { + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => { + resolved++; + return undefined; + }, + }); + const result = await tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'a' }); + assert.equal('laneGit' in result, false); + assert.equal(resolved, 0); +}); + +test('an unreadable lane leaves the command result untouched', async (t) => { + const root = await scratch(t); + for (const resolveRoot of [async () => root, async () => undefined, async () => Promise.reject(new Error('gone'))]) { + const result = await new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot }).execute(commandRequest); + assert.deepEqual(result, commandResult); + } +}); + +test('the feature is advertised only when commands are', () => { + const withCommands = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => undefined }); + assert.deepEqual(withCommands.capabilities.commandResultFeatures, ['lane_git']); + assert.equal(isValidBridgeWorkspaceToolCapabilities(withCommands.capabilities), true); + const readOnly = new LaneGitWorkspaceTools({ + delegate: { ...delegate(), capabilities: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + resolveRoot: async () => undefined, + }); + assert.equal(readOnly.capabilities.commandResultFeatures, undefined); + assert.equal( + isValidBridgeWorkspaceToolCapabilities({ ...withCommands.capabilities, commandResultFeatures: ['other'] }), + false, + ); +}); + +test('the probe is skipped until lane_git is negotiated, then runs', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + let negotiated = false; + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + isEnabled: () => negotiated, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + const before = await tools.execute(commandRequest); + assert.equal('laneGit' in before, false); + assert.equal(resolved, 0); + negotiated = true; + const after = await tools.execute(commandRequest); + assert.equal(resolved, 1); + assert.deepEqual('laneGit' in after && after.laneGit, { branch: 'main', head: await sha(source) }); +}); + +test('the probe is skipped when the command left too little of its settlement budget', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const timed = { ...commandRequest, timeoutMs: 1_000 }; + // Budget is timeoutMs plus a 5 s grace (6 s here), minus a 1 s settlement reserve; the clock jumps while the command runs. + for (const [elapsedMs, probed] of [ + [100, true], + [4_000, true], + [4_800, false], + [7_000, false], + ] as const) { + let clock = 0; + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(async () => { + clock += elapsedMs; + }), + now: () => clock, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + const result = await tools.execute(timed); + assert.equal(resolved, probed ? 1 : 0, `elapsed ${elapsedMs}`); + assert.equal('laneGit' in result, probed, `elapsed ${elapsedMs}`); + } +}); + +test('the probe budget follows the worker deadline, not a rebuilt timeout', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + // A full 31 s budget by timeout alone, but the worker has only 1.5 s left before it aborts. + for (const [remainingMs, probed] of [ + [10_000, true], + [1_500, true], + [1_200, false], + [500, false], + ] as const) { + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + now: () => 1_000, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + await tools.execute(commandRequest, undefined, { deadlineAtMs: 1_000 + remainingMs }); + assert.equal(resolved, probed ? 1 : 0, `remaining ${remainingMs}`); + } +}); + +test('a slow probe is cut short and never fails the command', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + probeTimeoutMs: 1, + resolveRoot: async (_request, signal) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + signal?.throwIfAborted(); + return source; + }, + }); + assert.deepEqual(await tools.execute(commandRequest), commandResult); +}); + +test('a HEAD that names a blob is reported as a null head, not as a commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const id = (await exec('bash', ['-c', `echo hello | git -C '${source}' hash-object -w --stdin`])).stdout.trim(); + await writeFile(join(source, '.git', 'HEAD'), `${id}\n`); + assert.deepEqual(await readLaneGit(source), { branch: null, head: null }); +}); + +test('branch names with Unicode control or format characters are not reported', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/\u202eabc']); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); + for (const name of ['a\u0085b', 'a\u202eb', 'a\u200bb', 'a\u2028b']) { + assert.equal(isWorkspaceLaneGit({ branch: name, head: null }), false, JSON.stringify(name)); + } + assert.equal(isWorkspaceLaneGit({ branch: 'feature/caf\u00e9-\u65e5\u672c', head: null }), true); +}); + +test('a resolver that ignores its signal cannot hold the command result past the probe limit', async () => { + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + probeTimeoutMs: 50, + resolveRoot: () => new Promise(() => {}), + }); + const started = Date.now(); + const result = await Promise.race([ + tools.execute(commandRequest), + new Promise<'hung'>((resolve) => setTimeout(() => resolve('hung'), 2_000)), + ]); + assert.notEqual(result, 'hung'); + assert.deepEqual(result, commandResult); + assert.ok(Date.now() - started < 1_000); +}); + +test('the worker reports lane_git active only when the registration that stuck advertised it', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + for (const secondFails of [false, true]) { + let registrations = 0; + const tools = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => source }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async () => { + registrations++; + if (secondFails && registrations === 2) return new Response('{}', { status: 500 }); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + supportedWorkspaceCommandResultFeatures: ['lane_git'], + }); + }, + }); + await worker.register(); + assert.equal(worker.commandResultFeatureActive('lane_git'), !secondFails, `secondFails=${secondFails}`); + } +}); + +test('a branch name with leading or trailing Unicode whitespace is reported as it is', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', '\u00a0main\u00a0']); + assert.deepEqual(await readLaneGit(source), { branch: '\u00a0main\u00a0', head: await sha(source) }); +}); + +test('a branch name that is not valid UTF-8 is reported as null, not as a lossy copy', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await writeFile(join(source, '.git', 'HEAD'), Buffer.concat([Buffer.from('ref: refs/heads/bad'), Buffer.from([0xff, 0x0a])])); + const state = await readLaneGit(source); + assert.equal(state?.branch, null); + assert.equal(JSON.stringify(state).includes('\ufffd'), false); +}); + +test('a valid non-ASCII branch name survives the byte level read', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/caf\u00e9-\u65e5\u672c']); + assert.equal((await readLaneGit(source))?.branch, 'feature/caf\u00e9-\u65e5\u672c'); +}); + +test('an own Git directory is accepted, and a commit made there keeps it accepted', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + assert.equal(await ownsGitMetadata(source), true); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + await exec('git', ['-C', source, 'pack-refs', '--all']); + assert.equal(await ownsGitMetadata(source), true); +}); + +test('a .git rewritten to point at another repository is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const victim = await repo(root, 'victim'); + await exec('git', ['-C', victim, 'checkout', '-b', 'victim-secret-branch']); + await rm(join(source, '.git'), { recursive: true }); + await writeFile(join(source, '.git'), `gitdir: ${join(victim, '.git')}\n`); + assert.equal(await ownsGitMetadata(source), false); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => ((await ownsGitMetadata(source)) ? source : undefined), + }); + const result = await tools.execute(commandRequest); + assert.equal('laneGit' in result, false); + assert.equal(JSON.stringify(result).includes('victim-secret-branch'), false); +}); + +test('Git storage redirected by a symlink, commondir or a swapped directory is not probed', async (t) => { + const root = await scratch(t); + const victim = await repo(root, 'victim'); + const cases: Array<[string, (source: string) => Promise]> = [ + ['.git symlink', async (s) => { await rm(join(s, '.git'), { recursive: true }); await symlink(join(victim, '.git'), join(s, '.git')); }], + ['commondir', async (s) => { await writeFile(join(s, '.git', 'commondir'), `${join(victim, '.git')}\n`); }], + ['HEAD symlink', async (s) => { await rm(join(s, '.git', 'HEAD')); await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'HEAD')); }], + ['refs symlink', async (s) => { await rm(join(s, '.git', 'refs'), { recursive: true }); await symlink(join(victim, '.git', 'refs'), join(s, '.git', 'refs')); }], + ['packed-refs symlink', async (s) => { await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'packed-refs')); }], + ['no .git', async (s) => { await rename(join(s, '.git'), join(root, `moved-${Math.random()}`)); }], + ]; + for (const [name, tamper] of cases) { + const source = await repo(root, `source-${name.replace(/\W/g, '')}`); + await tamper(source); + assert.equal(await ownsGitMetadata(source), false, name); + } +}); + +test('a nested ref symlinked to another checkout is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const firstCommit = await sha(source); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + // A clone shares history, so the commit its branch sits on also exists in the source's objects. + const other = join(root, 'other'); + await exec('git', ['clone', '-q', source, other]); + await exec('git', ['-C', other, 'checkout', '-q', '-b', 'other-wip', firstCommit]); + await symlink(join(other, '.git', 'refs', 'heads', 'other-wip'), join(source, '.git', 'refs', 'heads', 'leak')); + await writeFile(join(source, '.git', 'HEAD'), 'ref: refs/heads/leak\n'); + assert.equal(await ownsGitMetadata(source), false); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => ((await ownsGitMetadata(source)) ? source : undefined), + }); + const result = await tools.execute(commandRequest); + assert.equal('laneGit' in result, false); +}); + +test('a nested branch ref that is a plain file inside .git is still probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/deep/name']); + assert.equal(await ownsGitMetadata(source), true); + assert.equal((await readLaneGit(source))?.branch, 'feature/deep/name'); +}); + +test('a branch that shares its short name with a tag is reported by its real name', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'foo']); + await exec('git', ['-C', source, 'tag', 'foo']); + assert.equal((await readLaneGit(source))?.branch, 'foo'); +}); + +test('a HEAD that points outside refs/heads reports a null branch', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'update-ref', 'refs/remotes/origin/main', 'HEAD']); + await exec('git', ['-C', source, 'symbolic-ref', 'HEAD', 'refs/remotes/origin/main']); + assert.equal((await readLaneGit(source))?.branch, null); +}); + +test('a linked lane whose HEAD names a ref symlinked at another checkout is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const firstCommit = await sha(source); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + const other = join(root, 'other'); + await exec('git', ['clone', '-q', source, other]); + await exec('git', ['-C', other, 'checkout', '-q', '-b', 'other-wip', firstCommit]); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + const common = join(source, '.git'); + assert.equal(await ownsLinkedWorktreeMetadata(common, 'lane'), true); + await symlink(join(other, '.git', 'refs', 'heads', 'other-wip'), join(common, 'refs', 'heads', 'leak')); + await writeFile(join(common, 'worktrees', 'lane', 'HEAD'), 'ref: refs/heads/leak\n'); + assert.equal(await ownsLinkedWorktreeMetadata(common, 'lane'), false); +}); + +test('a linked lane with a symlinked metadata HEAD is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + const head = join(source, '.git', 'worktrees', 'lane', 'HEAD'); + await rm(head); + await symlink(join(source, '.git', 'HEAD'), head); + assert.equal(await ownsLinkedWorktreeMetadata(join(source, '.git'), 'lane'), false); +}); + +test('the refs walk stops as soon as the probe signal aborts', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + for (let i = 0; i < 30; i++) await mkdir(join(source, '.git', 'refs', 'heads', `dir${i}`), { recursive: true }); + let checks = 0; + const aborting = { + aborted: false, + throwIfAborted() { + if (++checks >= 3) throw new Error('aborted'); + }, + } as unknown as AbortSignal; + assert.equal(await ownsGitMetadata(source, aborting), false); + assert.equal(checks, 3); + const linked = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', linked]); + checks = 0; + assert.equal(await ownsLinkedWorktreeMetadata(join(source, '.git'), 'lane', aborting), false); + assert.equal(checks, 3); +}); + +test('the head is read from the branch ref the first read named, and only a stable branch is reported', async () => { + const head = 'a'.repeat(40); + const sequence = (values: Array) => { + let index = 0; + return async () => values[index++]; + }; + const refsRead: string[] = []; + const commit = (answer: string | null | undefined) => async (ref: string) => { + refsRead.push(ref); + return answer; + }; + // Stable branch: head comes from that branch's own ref, never from HEAD. + assert.deepEqual(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/a']), commit(head)), { branch: 'a', head }); + assert.deepEqual(refsRead, ['refs/heads/a']); + // A checkout away during the read is rejected. + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/b']), commit(head)), undefined); + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', null]), commit(head)), undefined); + assert.equal(await snapshotLaneGit(sequence([null, 'refs/heads/a']), commit(head)), undefined); + // Detached: HEAD is read twice and must agree, so a move between the reads is rejected. + refsRead.length = 0; + let calls = 0; + const moving = async (ref: string) => { + refsRead.push(ref); + return calls++ === 0 ? head : 'b'.repeat(40); + }; + assert.equal(await snapshotLaneGit(sequence([null, null]), moving), undefined); + assert.deepEqual(refsRead, ['HEAD', 'HEAD']); + assert.deepEqual(await snapshotLaneGit(sequence([null, null]), commit(head)), { branch: null, head }); + // HEAD on a non-branch ref reports no branch but still a head. + assert.deepEqual(await snapshotLaneGit(sequence(['refs/remotes/o/m', 'refs/remotes/o/m']), commit(head)), { branch: null, head }); + // Unreadable values omit the field. + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/a']), commit(undefined)), undefined); +}); + +test('a replace ref cannot turn a blob into a reported commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const commit = await sha(source); + const blob = (await exec('bash', ['-c', `echo hello | git -C '${source}' hash-object -w --stdin`])).stdout.trim(); + await mkdir(join(source, '.git', 'refs', 'replace'), { recursive: true }); + await writeFile(join(source, '.git', 'refs', 'replace', blob), `${commit}\n`); + await writeFile(join(source, '.git', 'refs', 'heads', 'main'), `${blob}\n`); + assert.deepEqual(await readLaneGit(source), { branch: 'main', head: null }); +}); + +test('redirected object storage is not probed, for source and linked checks alike', async (t) => { + const root = await scratch(t); + const victim = await repo(root, 'victim'); + const cases: Array<[string, (source: string) => Promise]> = [ + ['alternates', async (s) => { + await mkdir(join(s, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(s, '.git', 'objects', 'info', 'alternates'), `${join(victim, '.git', 'objects')}\n`); + }], + ['http-alternates', async (s) => { + await mkdir(join(s, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(s, '.git', 'objects', 'info', 'http-alternates'), 'https://example.invalid/\n'); + }], + ['objects symlink', async (s) => { + await rename(join(s, '.git', 'objects'), join(root, `moved-${Math.random()}`)); + await symlink(join(victim, '.git', 'objects'), join(s, '.git', 'objects')); + }], + ['fanout symlink', async (s) => { + await symlink(join(victim, '.git', 'objects', 'info'), join(s, '.git', 'objects', 'ab')); + }], + ['pack entry symlink', async (s) => { + await mkdir(join(s, '.git', 'objects', 'pack'), { recursive: true }); + await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'objects', 'pack', 'pack-x.pack')); + }], + ['reftable redirect', async (s) => { + await mkdir(join(s, '.git', 'reftable'), { recursive: true }); + }], + ]; + for (const [name, tamper] of cases) { + const source = await repo(root, `src-${name.replace(/\W/g, '')}`); + await tamper(source); + assert.equal(await ownsGitMetadata(source), false, `source ${name}`); + } + const common = join(root, 'common'); + await exec('git', ['init', '-q', '--initial-branch=main', common]); + await exec('git', ['-C', common, ...identity, 'commit', '--allow-empty', '-m', 'i']); + await exec('git', ['-C', common, 'worktree', 'add', '-b', 'lane', join(root, 'lane')]); + assert.equal(await ownsLinkedWorktreeMetadata(join(common, '.git'), 'lane'), true); + await mkdir(join(common, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(common, '.git', 'objects', 'info', 'alternates'), `${join(victim, '.git', 'objects')}\n`); + assert.equal(await ownsLinkedWorktreeMetadata(join(common, '.git'), 'lane'), false); +}); + +test('a repository with packed objects and refs is still probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + await exec('git', ['-C', source, 'gc', '-q']); + assert.equal(await ownsGitMetadata(source), true); + assert.equal((await readLaneGit(source))?.head, await sha(source)); +}); + +test('a promisor remote cannot make the probe fetch or run a command on the host', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const victim = await repo(root, 'victim'); + await exec('git', ['-C', victim, ...identity, 'commit', '--allow-empty', '-m', 'secret']); + const secret = await sha(victim); + const marker = join(root, 'ran-on-host'); + const config = [ + '[remote "origin"]', + `\turl = ${victim}`, + '\tpromisor = true', + '\tpartialclonefilter = blob:none', + `\tuploadpack = touch ${marker}; git-upload-pack`, + '[extensions]', + '\tpartialclone = origin', + '[core]', + '\trepositoryformatversion = 1', + '[protocol]', + '\tallow = always', + '[protocol "file"]', + '\tallow = always', + '', + ].join('\n'); + await writeFile(join(source, '.git', 'config'), config, { flag: 'a' }); + await writeFile(join(source, '.git', 'refs', 'heads', 'main'), `${secret}\n`); + const state = await readLaneGit(source); + assert.notEqual(state?.head, secret); + await assert.rejects(readFile(marker), { code: 'ENOENT' }); +}); + +test('a linked worktree .git file is not claimed by the source checkout check', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + assert.equal(await ownsGitMetadata(lane), false); + assert.equal((await readLaneGit(lane))?.branch, 'lane-branch'); +}); + +test('a laneGit supplied by the delegate is never forwarded, only the probe value is', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const forged = { branch: 'forged', head: 'f'.repeat(40) }; + const forging = { + ...delegate(), + async execute() { + return { ...commandResult, laneGit: forged }; + }, + }; + const unreadable = await new LaneGitWorkspaceTools({ delegate: forging, resolveRoot: async () => undefined }).execute(commandRequest); + assert.equal('laneGit' in unreadable, false); + const probed = await new LaneGitWorkspaceTools({ delegate: forging, resolveRoot: async () => source }).execute(commandRequest); + assert.deepEqual('laneGit' in probed && probed.laneGit, { branch: 'main', head: await sha(source) }); +}); + +test('the worker passes its real assignment deadline to the executor', async () => { + const seen: Array = []; + const base = delegate(); + const spying = { + ...base, + async execute(request: WorkspaceToolRequest, signal?: AbortSignal, context?: { deadlineAtMs?: number }) { + seen.push(context?.deadlineAtMs); + return base.execute(request); + }, + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: spying.capabilities }, + workspaceTools: spying, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async () => + Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + }), + }); + await worker.register(); + const before = Date.now(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 2_000, + executionKind: 'workspace_tool', + workspaceId: 'primary', + request: commandRequest, + }); + assert.equal(seen.length, 1); + const deadline = seen[0] as number; + assert.ok(deadline >= before + 1_500 && deadline <= Date.now() + 2_000, `deadline ${deadline - before}`); +}); + +// Compatibility: negotiation with old and new Code API servers. + +function quarantine() { + return { async assertAvailable() {}, async arm() {}, async clear() {}, async quarantine() {} }; +} + +async function run(t: TestContext, supported: boolean) { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => source }); + const registrations: Array<{ commandResultFeatures?: string[] }> = []; + const settlements: Array<{ result?: Record }> = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async (input, init) => { + const url = String(input); + const body = init?.body ? JSON.parse(String(init.body)) : {}; + if (url.endsWith('/bridge/workers/register')) { + registrations.push(body.capabilities.workspaceTools ?? {}); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + ...(supported ? { supportedWorkspaceCommandResultFeatures: ['lane_git'] } : {}), + }); + } + if (url.endsWith('/settle')) settlements.push(body); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.register(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + workspaceId: 'primary', + request: commandRequest, + }); + return { registrations, settlements }; +} + +test('a Code API without the feature still pairs, registers without it, and never receives laneGit', async (t) => { + const { registrations, settlements } = await run(t, false); + assert.ok(registrations.length > 0); + for (const registration of registrations) assert.equal('commandResultFeatures' in registration, false); + assert.equal(settlements.length, 1); + assert.equal(settlements[0]?.result?.exitCode, 0); + assert.equal('laneGit' in (settlements[0]?.result ?? {}), false); +}); + +test('a Code API that acknowledges the feature registers it and receives laneGit', async (t) => { + const { registrations, settlements } = await run(t, true); + assert.deepEqual(registrations.at(-1)?.commandResultFeatures, ['lane_git']); + assert.deepEqual(Object.keys((settlements[0]?.result?.laneGit as object) ?? {}).sort(), ['branch', 'head']); + assert.equal((settlements[0]?.result?.laneGit as { branch: string }).branch, 'main'); +}); + +test('a worker without the field still pairs and reports status', async () => { + const executor = { + capabilities: { protocolVersion: 1 as const, operations: ['read_file' as const], workspaces: [{ id: 'primary' }] }, + async execute(): Promise { + throw new Error('not executed'); + }, + }; + const registrations: unknown[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: executor.capabilities }, + workspaceTools: executor, + fetchImpl: async (_input, init) => { + registrations.push(JSON.parse(String(init?.body)).capabilities.workspaceTools); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file'], + }); + }, + }); + const registration = await worker.register(); + assert.equal(registration.workerId, 'vm-1'); + assert.ok(registrations.length > 0); + for (const advertised of registrations) { + assert.equal(JSON.stringify(advertised).includes('commandResultFeatures'), false); + assert.equal(JSON.stringify(advertised).includes('laneGit'), false); + } +}); diff --git a/packages/code/src/lane-git.ts b/packages/code/src/lane-git.ts new file mode 100644 index 00000000..f7aa3122 --- /dev/null +++ b/packages/code/src/lane-git.ts @@ -0,0 +1,297 @@ +import { lstat, opendir, realpath } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, boundedBranch, boundedHead } from './protocol.js'; +import { gitBytes } from './worktree-retirement.js'; + +import type { WorkspaceLaneGit, WorkspaceToolRequest, WorkspaceToolResult } from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +/** Longest the advisory probe may run, however much budget is left. */ +const LANE_GIT_PROBE_MAX_MS = 1_000; +/** Settlement budget the probe never touches, for validation and the settle request. */ +const LANE_GIT_SETTLEMENT_RESERVE_MS = 1_000; +/** Below this, a probe is not worth starting. */ +const LANE_GIT_PROBE_MIN_MS = 250; +/** Code API gives a command its own timeout plus this much before it expires the assignment. */ +const COMMAND_EXECUTION_GRACE_MS = 5_000; +const LANE_GIT_TIMEOUT_MS = 5_000; +const LANE_GIT_OUTPUT_LIMIT = 4096; + +/** + * A read-only probe must never reach the network or run a transport command, whatever the checkout's + * own config says. A command can rewrite `.git/config` to mark a remote as a promisor and point a ref + * at a missing object, which makes a plain object lookup fetch lazily and run `ext::`, `ssh` or + * `uploadpack` programs on the host. An empty `GIT_ALLOW_PROTOCOL` disallows every transport and + * overrides repository configuration; `GIT_NO_LAZY_FETCH` covers Git 2.44 and later directly. + */ +const LANE_GIT_PROBE_ENV = { GIT_ALLOW_PROTOCOL: '', GIT_NO_LAZY_FETCH: '1' }; + +/** Exit codes that mean "no value" rather than "could not read": detached HEAD, unborn branch. */ +const DETACHED_EXIT = 1; +const UNBORN_EXIT = 128; + +async function read( + root: string, + args: string[], + emptyExit: number, + signal?: AbortSignal, + timeoutMs: number = LANE_GIT_TIMEOUT_MS, +): Promise { + try { + const bytes = await gitBytes(root, args, signal, timeoutMs, LANE_GIT_OUTPUT_LIMIT, LANE_GIT_PROBE_ENV); + // Git permits ref names that are not UTF-8. Decoding them leniently would report a different, + // valid-looking name, so anything that is not strictly valid UTF-8 is reported as null. + let text: string; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + return null; + } + // Drop only Git's own trailing line terminator: a ref name may legally begin or end with + // other whitespace, and trimming it would report a different branch. + return text.replace(/\r?\n$/, ''); + } catch (error) { + signal?.throwIfAborted(); + return (error as { code?: unknown }).code === emptyExit ? null : undefined; + } +} + +/** + * The branch and head commit checked out in a lane root, or undefined when Git + * could not be read (timeout, missing binary, aborted). Detached HEAD and a + * repository without commits are ordinary states and report null. Values that + * fail the bounds are reported as null, never forwarded. Nothing else from the + * repository is read or returned. + */ +export async function readLaneGit( + root: string, + signal?: AbortSignal, + timeoutMs: number = LANE_GIT_TIMEOUT_MS, +): Promise { + return await snapshotLaneGit( + () => read(root, ['symbolic-ref', '--quiet', 'HEAD'], DETACHED_EXIT, signal, timeoutMs), + // `--no-replace-objects`: a `refs/replace` entry must not turn a blob into a reported commit. + (ref) => read(root, ['--no-replace-objects', 'rev-parse', '--verify', `${ref}^{commit}`], UNBORN_EXIT, signal, timeoutMs), + ); +} + +/** + * Two Git processes cannot see one atomic snapshot, so a checkout between them could pair one + * branch with another branch's commit, a tuple that was never checked out. The head is therefore + * read from the branch ref the first read named, not from `HEAD`, and the branch is read again + * afterwards: a checkout away during the read omits the field. A detached HEAD has no ref to pin, + * so it is read twice and must agree. A change that happens and reverts within one probe is not + * detectable without a lock, and is not worth one for an advisory field. + */ +export async function snapshotLaneGit( + readBranch: () => Promise, + readCommit: (ref: string) => Promise, +): Promise { + const before = await readBranch(); + if (before === undefined) return undefined; + const head = await readCommit(before ?? 'HEAD'); + const after = await readBranch(); + if (head === undefined || after === undefined || before !== after) return undefined; + if (before === null && (await readCommit('HEAD')) !== head) return undefined; + // `--short` shortens ambiguously (`heads/foo` when a tag `foo` exists). Take the full ref and strip + // exactly `refs/heads/`; a HEAD pointing anywhere else is not a branch. + const name = before?.startsWith('refs/heads/') ? before.slice('refs/heads/'.length) : null; + return { branch: boundedBranch(name), head: boundedHead(head) }; +} + +async function absent(path: string): Promise { + try { + await lstat(path); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT'; + } +} + +async function plain(path: string, kind: 'file' | 'directory'): Promise { + try { + const status = await lstat(path); + return !status.isSymbolicLink() && (kind === 'file' ? status.isFile() : status.isDirectory()); + } catch { + return false; + } +} + +/** Entries a checkout may have before the probe gives up on proving they are all plain. */ +const LANE_GIT_REFS_MAX_ENTRIES = 20_000; +const LANE_GIT_OBJECTS_MAX_ENTRIES = 100_000; + +/** Whether everything beneath a directory is a plain file or directory: no link can redirect a read. */ +async function treeIsLinkFree( + refs: string, + signal?: AbortSignal, + maxEntries = LANE_GIT_REFS_MAX_ENTRIES, +): Promise { + const pending = [refs]; + let seen = 0; + while (pending.length > 0) { + // Stop between directory operations: the probe timeout returns the command result, but only an + // abort observed here keeps an abandoned walk from running on in the background. + signal?.throwIfAborted(); + const current = pending.pop()!; + const directory = await opendir(current); + for await (const entry of directory) { + signal?.throwIfAborted(); + if (++seen > maxEntries) return false; + if (entry.isSymbolicLink()) return false; + // `Dirent.parentPath` needs Node 20.12; the package supports 20.11. + if (entry.isDirectory()) pending.push(join(current, entry.name)); + else if (!entry.isFile()) return false; + } + } + return true; +} + +/** + * The shared part of both ownership checks: the refs and object storage of a Git directory hold + * no link, no `alternates` redirect and no `reftable` backend, so the host-side reads cannot be + * steered at another repository's refs or objects. + */ +async function storageIsOwn(gitDir: string, signal?: AbortSignal): Promise { + const objects = join(gitDir, 'objects'); + return ( + (await plain(join(gitDir, 'refs'), 'directory')) && + (await treeIsLinkFree(join(gitDir, 'refs'), signal)) && + (await plain(objects, 'directory')) && + (await absent(join(objects, 'info', 'alternates'))) && + (await absent(join(objects, 'info', 'http-alternates'))) && + (await treeIsLinkFree(objects, signal, LANE_GIT_OBJECTS_MAX_ENTRIES)) && + (await absent(join(gitDir, 'reftable'))) && + ((await absent(join(gitDir, 'packed-refs'))) || (await plain(join(gitDir, 'packed-refs'), 'file'))) + ); +} + +/** + * Whether `/.git` is the checkout's own Git directory, so the host-side probe cannot be + * steered at another repository. A command can rewrite its own `.git` after it ran: replace it + * with a `gitdir:` file, add a `commondir` redirect, or link `HEAD`, `packed-refs`, `refs` or any + * ref beneath it to another checkout, and the unsandboxed probe would then report that checkout's + * branch and head. Every entry under `refs/` is therefore checked, up to a bound. + * The check runs after the command has finished, so it does not cover a process the command left + * running in the background. + * Anything else, including a `.git` file (a linked worktree or submodule as a source root), is + * not probed: the field is simply omitted. Linked worktree lanes are verified separately. + */ +export async function ownsGitMetadata(root: string, signal?: AbortSignal): Promise { + const dotGit = join(root, '.git'); + try { + if (!(await plain(dotGit, 'directory')) || (await realpath(dotGit)) !== join(await realpath(root), '.git')) { + return false; + } + return ( + (await plain(join(dotGit, 'HEAD'), 'file')) && + (await absent(join(dotGit, 'commondir'))) && + (await storageIsOwn(dotGit, signal)) + ); + } catch { + return false; + } +} + +/** + * The linked worktree counterpart of `ownsGitMetadata`. `verifyLinkedWorktree` proves the lane's + * `.git` pointer chain, but the probe also reads the lane's own `HEAD` and the refs it names from + * the shared common directory, and a command can write to both. Without this, a ref symlinked at + * another checkout and named by the lane's `HEAD` would be followed by the host-side probe. + */ +export async function ownsLinkedWorktreeMetadata(commonGitDir: string, name: string, signal?: AbortSignal): Promise { + const metadata = join(commonGitDir, 'worktrees', name); + try { + if (!(await plain(join(metadata, 'HEAD'), 'file'))) return false; + if (!(await absent(join(metadata, 'refs'))) && !(await treeIsLinkFree(join(metadata, 'refs'), signal))) return false; + return await storageIsOwn(commonGitDir, signal); + } catch { + return false; + } +} + +export interface LaneGitWorkspaceToolsOptions { + delegate: WorkspaceToolExecutor; + /** The lane root a request ran in (isolated worktree, linked worktree or source checkout), or undefined. */ + resolveRoot: (request: WorkspaceToolRequest, signal?: AbortSignal) => Promise; + /** Whether Code API negotiated `lane_git`. Git is not probed while this is false. Defaults to true. */ + isEnabled?: () => boolean; + /** Longest the probe may run. Defaults to 1 s; it is also cut to fit the command's remaining budget. */ + probeTimeoutMs?: number; + /** Clock, for tests. */ + now?: () => number; +} + +/** + * Attaches `laneGit` to every finished `execute_command` result, so a branch the + * agent created or switched is visible as soon as the command returns. Git is + * read once per command, never on a timer. The field is stripped by the worker + * unless Code API acknowledged the `lane_git` feature at registration. + */ +export class LaneGitWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + + constructor(private readonly options: LaneGitWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + const base = options.delegate.capabilities; + this.capabilities = base.operations.includes('execute_command') + ? { ...base, commandResultFeatures: ['lane_git'] } + : base; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + context?: { deadlineAtMs?: number }, + ): Promise { + const now = this.options.now ?? Date.now; + const startedAt = now(); + const delegated = await this.options.delegate.execute(request, signal, context); + if (request.operation !== 'execute_command' || delegated.operation !== 'execute_command') return delegated; + // Only the value this wrapper reads is trusted. A sandbox executor must not be able to supply + // its own branch and head, whether or not the probe below produces anything. + const { laneGit: _untrusted, ...result } = delegated; + if (this.options.isEnabled?.() === false) return result; + // The probe is advisory, so it only runs in budget the command left over, and never into the + // part reserved for settlement. The worker's real deadline accounts for time already spent on + // lease transport and credential refresh; without it, assume the full timeout plus the grace. + const deadlineAtMs = + context?.deadlineAtMs ?? + startedAt + (request.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS) + COMMAND_EXECUTION_GRACE_MS; + const probeMs = Math.min( + this.options.probeTimeoutMs ?? LANE_GIT_PROBE_MAX_MS, + deadlineAtMs - now() - LANE_GIT_SETTLEMENT_RESERVE_MS, + ); + if (probeMs < Math.min(LANE_GIT_PROBE_MIN_MS, this.options.probeTimeoutMs ?? LANE_GIT_PROBE_MIN_MS)) return result; + // The resolver may do filesystem work that cannot observe a signal, so the race, not the + // signal, is what bounds the wait. The signal still stops Git when the limit hits first. + const controller = new AbortController(); + const probeSignal = AbortSignal.any([...(signal ? [signal] : []), controller.signal]); + let timer: NodeJS.Timeout | undefined; + try { + const probe = (async (): Promise => { + try { + const root = await this.options.resolveRoot(request, probeSignal); + return root == null ? undefined : await readLaneGit(root, probeSignal, probeMs); + } catch { + return undefined; + } + })(); + const limit = new Promise((resolve) => { + timer = setTimeout(() => { + controller.abort(); + resolve(undefined); + }, probeMs); + }); + const laneGit = await Promise.race([probe, limit]); + return laneGit ? { ...result, laneGit } : result; + } catch { + // Git state is advisory; a failed read must never fail or delay the command's own result. + return result; + } finally { + clearTimeout(timer); + } + } +} diff --git a/packages/code/src/projects.ts b/packages/code/src/projects.ts index 1e2fcbd3..e91694f3 100644 --- a/packages/code/src/projects.ts +++ b/packages/code/src/projects.ts @@ -4,6 +4,8 @@ import { lstat, opendir, realpath } from 'node:fs/promises'; import { isAbsolute, relative, resolve, sep } from 'node:path'; import { promisify } from 'node:util'; +import { boundedBranch, boundedHead } from './protocol.js'; + const exec = promisify(execFile); const skipped = new Set(['node_modules', 'vendor']); @@ -221,16 +223,8 @@ export async function discoverProjects( ); const path = rel.split(sep).join('/') || '.'; const normalizedRemote = remote ? projectRemote(remote) : null; - const validBranch = - branch && - branch.length <= 256 && - !/[\x00-\x1f\x7f]/.test(branch) - ? branch - : null; - const validHead = - head && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(head) - ? head - : null; + const validBranch = boundedBranch(branch); + const validHead = boundedHead(head); if ( (remote !== null && normalizedRemote === null) || (branch !== null && validBranch === null) || diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 216784a4..1451986d 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -340,7 +340,50 @@ const WORKSPACE_EDIT_MATCH_STRATEGIES = new Set([ 'whitespace-normalized', 'indentation-flexible', ]); + +/** A branch name reportable to LibreChat: 1 to 256 chars, no control characters. */ +export function boundedBranch(value: unknown): string | null { + return typeof value === 'string' && + value.length > 0 && + value.length <= 256 && + // Git forbids only ASCII controls in ref names; the header must also never show Unicode + // controls, format characters (bidi overrides, zero-width) or line and paragraph separators. + !/[\x00-\x1f\x7f\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/u.test(value) + ? value + : null; +} + +/** A commit id reportable to LibreChat: 40 (SHA-1) or 64 (SHA-256) lowercase hex chars. */ +export function boundedHead(value: unknown): string | null { + return typeof value === 'string' && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(value) + ? value + : null; +} + export type WorkspaceListFileFeature = 'after_path'; +/** `lane_git`: `execute_command` results may carry `laneGit`. Sent only when Code API echoes it. */ +export type WorkspaceCommandResultFeature = 'lane_git'; + +/** Git state of the lane a command ran in. Never carries paths, remotes or repository content. */ +export interface WorkspaceLaneGit { + /** Checked-out branch, or null when HEAD is detached or the name is not reportable. */ + branch: string | null; + /** HEAD commit (40 or 64 lowercase hex), or null before the first commit or when unreportable. */ + head: string | null; +} + +export function isWorkspaceLaneGit(value: unknown): value is WorkspaceLaneGit { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const state = value as Record; + const keys = Object.keys(state); + return ( + keys.length === 2 && + keys.includes('branch') && + keys.includes('head') && + (state.branch === null || (state.branch !== undefined && boundedBranch(state.branch) === state.branch)) && + (state.head === null || (state.head !== undefined && boundedHead(state.head) === state.head)) + ); +} export type WorkspaceProgrammaticLanguage = 'bash'; export interface BridgeWorkspaceDescriptor { @@ -374,6 +417,8 @@ export interface BridgeWorkspaceToolCapabilities { editFileFeatures?: WorkspaceEditFileFeature[]; /** Omitted by workers that cannot continue a bounded file listing. */ listFileFeatures?: WorkspaceListFileFeature[]; + /** Omitted by workers that do not report lane Git state on command results. */ + commandResultFeatures?: WorkspaceCommandResultFeature[]; /** Languages that can execute PTC replay inside a selected workspace. */ programmaticLanguages?: WorkspaceProgrammaticLanguage[]; } @@ -625,6 +670,8 @@ export interface WorkspaceExecuteCommandResult { stderr: string; truncated: boolean; timedOut: boolean; + /** Present only when `commandResultFeatures` negotiated `lane_git` and Git was readable. */ + laneGit?: WorkspaceLaneGit; } export type WorkspaceToolRequest = @@ -789,6 +836,7 @@ const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'stderr', 'truncated', 'timedOut', + 'laneGit', ]); const WORKSPACE_SEARCH_MATCH_KEYS = new Set(['path', 'line', 'column', 'text']); @@ -836,6 +884,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceInstanceTypes?: ['git_worktree']; /** Scheduling scopes this Code API can admit as independent lanes. */ supportedWorkspaceScopes?: ['git_linked_worktree']; + /** Command result fields this Code API accepts in settlements. */ + supportedWorkspaceCommandResultFeatures?: WorkspaceCommandResultFeature[]; /** Added workspace tool error codes this Code API accepts in settlements. */ supportedWorkspaceToolErrorCodes?: WorkspaceToolErrorCode[]; } @@ -1717,6 +1767,7 @@ export function isWorkspaceToolResult( /^SIG[A-Z0-9]+$/.test(result.signal))) && typeof result.truncated === 'boolean' && typeof result.timedOut === 'boolean' && + (result.laneGit === undefined || isWorkspaceLaneGit(result.laneGit)) && (result.exitCode === null ? result.timedOut === true || result.signal !== undefined : result.timedOut === false && result.signal === undefined) @@ -1839,6 +1890,16 @@ export function isValidBridgeWorkspaceToolCapabilities( return false; } + if ( + capabilities.commandResultFeatures !== undefined && + (!Array.isArray(capabilities.commandResultFeatures) || + capabilities.commandResultFeatures.length !== 1 || + !capabilities.operations.includes('execute_command') || + capabilities.commandResultFeatures[0] !== 'lane_git') + ) { + return false; + } + if ( capabilities.programmaticLanguages !== undefined && (!Array.isArray(capabilities.programmaticLanguages) || diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 17a9a6e1..49b96eb1 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -18,6 +18,7 @@ import { signBridgeRequest } from './identity.js'; import { isWorkspaceToolRequest, WorkspaceToolError } from './workspace.js'; import type { + WorkspaceCommandResultFeature, BridgeAssignment, BridgeLeaseResponse, BridgeSandboxRequest, @@ -218,6 +219,11 @@ function workspaceCapabilitiesMatch( (feature, index) => feature === executor.listFileFeatures?.[index], ) ?? executor.listFileFeatures == null) && + advertised.commandResultFeatures?.length === executor.commandResultFeatures?.length && + (advertised.commandResultFeatures?.every( + (feature, index) => feature === executor.commandResultFeatures?.[index], + ) ?? + executor.commandResultFeatures == null) && advertised.programmaticLanguages?.length === executor.programmaticLanguages?.length && (advertised.programmaticLanguages?.every( @@ -314,6 +320,7 @@ function registrationCompatibleCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + commandResultFeatures: _commandResultFeatures, programmaticLanguages: _programmaticLanguages, ...compatibleWorkspaceTools } = workspaceTools; @@ -409,6 +416,9 @@ function supportedWorkspaceCapabilities( const listFileFeatures = desired.listFileFeatures?.filter((feature) => registration.supportedWorkspaceListFileFeatures?.includes(feature), ); + const commandResultFeatures = desired.commandResultFeatures?.filter((feature) => + registration.supportedWorkspaceCommandResultFeatures?.includes(feature), + ); const programmaticLanguages = desired.programmaticLanguages?.filter( (language) => registration.supportedWorkspaceProgrammaticLanguages?.includes(language), @@ -418,6 +428,7 @@ function supportedWorkspaceCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + commandResultFeatures: _commandResultFeatures, programmaticLanguages: _programmaticLanguages, ...compatibleDesired } = desired; @@ -439,6 +450,9 @@ function supportedWorkspaceCapabilities( ...(operations.includes('list_files') && listFileFeatures?.length ? { listFileFeatures } : {}), + ...(operations.includes('execute_command') && commandResultFeatures?.length + ? { commandResultFeatures } + : {}), ...(operations.includes('execute_command') && programmaticLanguages?.length ? { programmaticLanguages } @@ -468,6 +482,11 @@ export class BridgeWorker { private instructionMetadataSupported = true; /** Added error codes the current Code API registration accepts in settlements. */ private settlementErrorCodes: ReadonlySet = new Set(); + /** Whether the registration that is actually active advertised this command result feature. */ + commandResultFeatureActive(feature: WorkspaceCommandResultFeature): boolean { + return this.activeCapabilities.workspaceTools?.commandResultFeatures?.includes(feature) === true; + } + private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; private maintenanceOnly = false; @@ -1854,6 +1873,7 @@ export class BridgeWorker { payload = await this.options.workspaceTools.execute( workspaceRequest, executionController.signal, + { deadlineAtMs: localDeadlineAtMs }, ); if ( workspaceRequest.operation === 'list_files' && @@ -1864,6 +1884,14 @@ export class BridgeWorker { payload; payload = compatiblePayload; } + if ( + workspaceRequest.operation === 'execute_command' && + !advertised.commandResultFeatures?.includes('lane_git') && + 'laneGit' in payload + ) { + const { laneGit: _laneGit, ...compatiblePayload } = payload; + payload = compatiblePayload; + } workspaceMutationApplied = isMutation; if (isMutation && !isWorkspaceToolResult(workspaceRequest, payload)) { throw new BridgeProtocolError( diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 746c7249..d740d48a 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -100,6 +100,8 @@ export interface WorkspaceToolExecutor { execute( request: WorkspaceToolRequest, signal?: AbortSignal, + /** Local epoch ms after which the worker aborts this execution. Advisory work must finish well before it. */ + context?: { deadlineAtMs?: number }, ): Promise; } diff --git a/packages/code/src/worktree-retirement.ts b/packages/code/src/worktree-retirement.ts index 39a7b60c..2ca8b78f 100644 --- a/packages/code/src/worktree-retirement.ts +++ b/packages/code/src/worktree-retirement.ts @@ -158,7 +158,7 @@ function gitEnvironment(): NodeJS.ProcessEnv { }; } -async function git( +export async function git( cwd: string, args: string[], signal?: AbortSignal, @@ -173,6 +173,23 @@ async function git( return stdout; } +/** Same invocation as `git`, but the raw stdout bytes, for callers that must not accept lossy decoding. */ +export async function gitBytes( + cwd: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS, + maxBuffer = GIT_OUTPUT_LIMIT, + extraEnvironment: Record = {}, +): Promise { + const { stdout } = await execFileAsync( + 'git', + ['--no-optional-locks', '-C', cwd, '-c', 'core.fsmonitor=false', ...args], + { encoding: 'buffer', env: { ...gitEnvironment(), ...extraEnvironment }, maxBuffer, signal, timeout }, + ); + return stdout; +} + /** The commit a ref names, or undefined when it does not resolve. */ async function resolveCommit(cwd: string, ref: string, signal?: AbortSignal): Promise { try { diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index b04de55f..f7ba03b4 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -10,6 +10,7 @@ export const bridgeStore = new RedisBridgeStore( undefined, undefined, env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, + env.BRIDGE_LANE_GIT, ); export const bridgePairings = new RedisBridgePairingStore( connection, diff --git a/service/src/bridge/lane-git.test.ts b/service/src/bridge/lane-git.test.ts new file mode 100644 index 00000000..604f2070 --- /dev/null +++ b/service/src/bridge/lane-git.test.ts @@ -0,0 +1,251 @@ +import { afterEach, describe, expect, spyOn, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; + +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { laneGitPolicy } from './lane-git'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const incarnationId = 'incarnation-00000001'; +const head = 'a'.repeat(40); + +afterEach(async () => { + await redis.flushall(); +}); + +function capabilities(advertise: boolean) { + return { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary' }], + ...(advertise ? { commandResultFeatures: ['lane_git' as const] } : {}), + }, + }; +} + +const request = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'git checkout -b agent', +}; + +function commandResult(extra: Record = {}) { + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + ...extra, + }; +} + +/** Registers a worker, runs one command through the store, and returns what the caller receives. */ +async function run(opts: { enabled: boolean; advertise: boolean; workerResult: Record }) { + const store = new RedisBridgeStore(redis, undefined, undefined, 1, opts.enabled); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(opts.advertise), + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + const assignment = await store.lease('lane-worker', incarnationId, 1_000); + await store.settle('lane-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: opts.workerResult as never, + }); + // Read the durable copy before the dispatcher cleans it up; it must exist, or the + // "not stored" assertions below would pass vacuously. + const stored = await redis.get(`codeapi:bridge:v1:assignment:${assignment?.assignmentId}:settlement`); + expect(stored).not.toBeNull(); + const settlement = await completion; + return { settlement, stored: stored as string }; +} + +describe('laneGit on command settlements', () => { + test('a valid laneGit from an advertising worker reaches the caller unchanged', async () => { + const laneGit = { branch: 'librechat/conversation-abcd1234-0123', head }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect(settlement.status).toBe('fulfilled'); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + test('null branch and head (detached, no commit) are valid and preserved', async () => { + const laneGit = { branch: null, head: null }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + test('a 64 character head is accepted', async () => { + const laneGit = { branch: 'main', head: 'b'.repeat(64) }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + const invalid: Array<[string, unknown]> = [ + ['extra keys', { branch: 'main', head, path: '/srv/repo' }], + ['an over-long branch', { branch: 'x'.repeat(257), head }], + ['an empty branch', { branch: '', head }], + ['a control character in the branch', { branch: 'bad\nname', head }], + ['a bidi override in the branch', { branch: 'feature/\u202eabc', head }], + ['a zero-width character in the branch', { branch: 'fix\u200bme', head }], + ['uppercase hex', { branch: 'main', head: 'A'.repeat(40) }], + ['a short head', { branch: 'main', head: 'a'.repeat(39) }], + ['non-hex head', { branch: 'main', head: 'g'.repeat(40) }], + ['a missing head', { branch: 'main' }], + ['a non-object', 'main'], + ['an array', [null, null]], + ]; + for (const [name, laneGit] of invalid) { + test(`laneGit with ${name} is dropped and the command still completes`, async () => { + const { settlement, stored } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect(settlement.status).toBe('fulfilled'); + const result = (settlement as unknown as { result: Record }).result; + expect('laneGit' in result).toBe(false); + expect(result.exitCode).toBe(0); + expect(stored).not.toContain('laneGit'); + expect(stored).not.toContain('/srv/repo'); + }); + } + + test('laneGit from a worker that did not advertise lane_git is stripped', async () => { + const { settlement, stored } = await run({ + enabled: true, + advertise: false, + workerResult: commandResult({ laneGit: { branch: 'main', head } }), + }); + expect(settlement.status).toBe('fulfilled'); + expect('laneGit' in (settlement as { result: object }).result).toBe(false); + expect(stored).not.toContain('laneGit'); + }); + + test('laneGit is stripped when the deployment setting is off, even from an advertising worker', async () => { + const { settlement, stored } = await run({ + enabled: false, + advertise: true, + workerResult: commandResult({ laneGit: { branch: 'main', head } }), + }); + expect(settlement.status).toBe('fulfilled'); + expect('laneGit' in (settlement as { result: object }).result).toBe(false); + expect(stored).not.toContain('laneGit'); + }); + + test('an old worker without the field still registers and runs commands, with the setting on or off', async () => { + for (const enabled of [false, true]) { + const { settlement } = await run({ enabled, advertise: false, workerResult: commandResult() }); + expect(settlement.status).toBe('fulfilled'); + expect((settlement as { result: { exitCode: number } }).result.exitCode).toBe(0); + await redis.flushall(); + } + }); + + test('the policy never reports the value, only a fixed reason', () => { + const result = commandResult({ laneGit: { branch: 'secret-branch', head, path: '/srv/repo' } }); + const outcome = laneGitPolicy({ result, enabled: true, advertised: true }); + expect(outcome.dropped).toBe('invalid'); + expect(JSON.stringify(outcome.dropped)).not.toContain('secret-branch'); + }); +}); + +describe('retried settlements across replicas that disagree on the setting', () => { + // A replica commits a settlement and its response is lost; the worker retries the same body + // against another replica mid-rollout. The retry must be recognized as the same settlement. + for (const [first, second] of [ + [true, false], + [false, true], + ] as const) { + test(`commit with the setting ${first ? 'on' : 'off'}, retry on a replica with it ${second ? 'on' : 'off'}`, async () => { + const a = new RedisBridgeStore(redis, undefined, undefined, 1, first); + const b = new RedisBridgeStore(redis, undefined, undefined, 1, second); + await a.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(true), + }); + const completion = a.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + completion.catch(() => undefined); + const assignment = await a.lease('lane-worker', incarnationId, 1_000); + const settlement = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled' as const, + result: commandResult({ laneGit: { branch: 'main', head } }) as never, + }; + await a.settle('lane-worker', assignment?.assignmentId ?? '', settlement); + await expect(b.settle('lane-worker', assignment?.assignmentId ?? '', settlement)).resolves.toBeUndefined(); + await completion; + }); + } +}); + +describe('Redis reads while settling', () => { + async function readsDuring(withLaneGit: boolean): Promise { + const store = new RedisBridgeStore(redis, undefined, undefined, 1, true); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(true), + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + completion.catch(() => undefined); + const assignment = await store.lease('lane-worker', incarnationId, 1_000); + const spy = spyOn(redis, 'get'); + await store.settle('lane-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: commandResult(withLaneGit ? { laneGit: { branch: 'main', head } } : {}) as never, + }); + const calls = spy.mock.calls.length; + spy.mockRestore(); + await completion; + return calls; + } + + test('a settlement with laneGit costs no more Redis reads than one without', async () => { + const without = await readsDuring(false); + await redis.flushall(); + const withIt = await readsDuring(true); + expect(without).toBeGreaterThan(0); + expect(withIt).toBe(without); + }); +}); diff --git a/service/src/bridge/lane-git.ts b/service/src/bridge/lane-git.ts new file mode 100644 index 00000000..69ddf7b9 --- /dev/null +++ b/service/src/bridge/lane-git.ts @@ -0,0 +1,36 @@ +import { isWorkspaceLaneGit } from '../../../packages/code/src/protocol'; + +export type LaneGitDropReason = 'disabled' | 'not_advertised' | 'invalid'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Whether a fulfilled command result carries a `laneGit` key at all. */ +export function hasLaneGit(result: unknown): boolean { + return isRecord(result) && 'laneGit' in result; +} + +/** + * Decide what happens to the `laneGit` on an execute_command result. It is kept, + * unchanged, only when the deployment enables the feature, the worker advertised + * `lane_git`, and the value is exactly `{ branch, head }` within bounds. Anything + * else is dropped; the command result itself is never failed over it. The reason + * is a fixed label, never the value. + */ +export function laneGitPolicy(args: { + result: Record; + enabled: boolean; + advertised: boolean; +}): { result: Record; dropped?: LaneGitDropReason } { + const reason: LaneGitDropReason | undefined = !args.enabled + ? 'disabled' + : !args.advertised + ? 'not_advertised' + : isWorkspaceLaneGit(args.result.laneGit) + ? undefined + : 'invalid'; + if (reason === undefined) return { result: args.result }; + const { laneGit: _laneGit, ...rest } = args.result; + return { result: rest, dropped: reason }; +} diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 360414a0..5e76876e 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -167,6 +167,55 @@ describe('paired bridge HTTP API', () => { }); }); + for (const laneGit of [false, true]) { + test(`registration ${laneGit ? 'offers' : 'omits'} lane_git when the setting is ${laneGit ? 'on' : 'off'}`, async () => { + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis, undefined, undefined, 1, laneGit), + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/register`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'execute_command'], + workspaces: [{ id: 'primary' }], + ...(laneGit ? { commandResultFeatures: ['lane_git'] } : {}), + }, + }, + }), + }); + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body.supportedWorkspaceListFileFeatures).toEqual(['after_path']); + if (laneGit) expect(body.supportedWorkspaceCommandResultFeatures).toEqual(['lane_git']); + else expect('supportedWorkspaceCommandResultFeatures' in body).toBe(false); + }); + } + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 0bc20a1e..123f8c75 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -538,6 +538,9 @@ router.post( supportedWorkspaceEditFileModes: ['single', 'batch'], supportedWorkspaceEditFileFeatures: [...WORKSPACE_EDIT_FILE_FEATURES], supportedWorkspaceListFileFeatures: ['after_path'], + ...(options.store.laneGitEnabled + ? { supportedWorkspaceCommandResultFeatures: ['lane_git'] } + : {}), supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], supportedWorkspaceScopes: ['git_linked_worktree'], diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 542c5903..0ca74c45 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -21,6 +21,8 @@ import { workspaceIsolationParent, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; +import logger from '../logger'; +import { hasLaneGit, laneGitPolicy } from './lane-git'; import { BridgeAdmissionQueue, durableAdmissionFence } from './admission'; import { BridgeWorkspaceSlots } from './slots'; import type { StoredWorkspaceRequest } from '../workspace-tools/requests'; @@ -383,6 +385,12 @@ function assignmentKey(assignmentId: string): string { return `${PREFIX}:assignment:${assignmentId}`; } +function withoutLaneGit(settlement: AnyCodeBridgeSettlement): AnyCodeBridgeSettlement { + if (settlement.status !== 'fulfilled') return settlement; + const { laneGit: _laneGit, ...result } = settlement.result as unknown as Record; + return { ...settlement, result } as AnyCodeBridgeSettlement; +} + function settlementKey(assignmentId: string): string { return `${PREFIX}:assignment:${assignmentId}:settlement`; } @@ -472,6 +480,8 @@ export class RedisBridgeStore { private readonly workerTtlSeconds = DEFAULT_WORKER_TTL_SECONDS, private readonly redisCommandTimeoutMs = DEFAULT_REDIS_COMMAND_TIMEOUT_MS, private readonly maxWorkspaceLeaseSlots = 1, + /** Accept `laneGit` on command results and offer `lane_git` at registration. Off by default. */ + public readonly laneGitEnabled = false, ) { if ( !Number.isSafeInteger(maxWorkspaceLeaseSlots) || @@ -1791,12 +1801,25 @@ export class RedisBridgeStore { ); return; } - const serializedSettlement = JSON.stringify(settlement); + let serializedSettlement = JSON.stringify(settlement); const existingSettlement = await this.leaseCommand( this.redis.get(settlementKey(assignmentId)), signal, 'Bridge settlement existing read', ); + if ( + existingSettlement != null && + existingSettlement !== serializedSettlement && + hasLaneGit(settlement.status === 'fulfilled' ? settlement.result : undefined) + ) { + // A retry of a committed settlement must stay recognizable whichever way this replica's + // lane Git setting differs from the replica that committed it. + const stripped = withoutLaneGit(settlement); + if (existingSettlement === JSON.stringify(stripped)) { + settlement = stripped; + serializedSettlement = existingSettlement; + } + } if ( existingSettlement != null && existingSettlement !== serializedSettlement @@ -1856,6 +1879,13 @@ export class RedisBridgeStore { 'Bridge assignment has expired', ); } + // Uses the assignment and registration already read above: no extra round trips on the + // settlement path, which has only a short grace after the command ran. + const sanitized = this.sanitizeLaneGit(assignmentId, settlement, assignment, registration); + if (sanitized !== settlement) { + settlement = sanitized; + serializedSettlement = JSON.stringify(settlement); + } const ttlSeconds = assignmentOutcomeTtlSeconds(assignment); const settlementKeys = [ assignmentKey(assignmentId), @@ -1964,6 +1994,38 @@ export class RedisBridgeStore { } } + /** + * Validate the optional `laneGit` of a fulfilled execute_command settlement before it is + * stored or forwarded. An invalid, unexpected or unadvertised value is dropped, never an + * error: the command's own result must still reach the caller. Pure over records the caller + * already read, so it adds no Redis round trips. + */ + private sanitizeLaneGit( + assignmentId: string, + settlement: AnyCodeBridgeSettlement, + assignment: StoredAssignment, + registration: RegisteredBridgeWorker | undefined, + ): AnyCodeBridgeSettlement { + if ( + settlement.status !== 'fulfilled' || + !hasLaneGit(settlement.result) || + assignment.executionKind !== 'workspace_tool' || + (assignment.request as WorkspaceToolRequest).operation !== 'execute_command' + ) { + return settlement; + } + const { dropped } = laneGitPolicy({ + result: settlement.result as unknown as Record, + enabled: this.laneGitEnabled, + advertised: + registration?.capabilities.workspaceTools?.commandResultFeatures?.includes('lane_git') === true, + }); + if (dropped === undefined) return settlement; + // Reason only: never the branch, the head or anything else from the result. + logger.debug('Dropped lane Git from a command settlement', { reason: dropped, assignmentId }); + return withoutLaneGit(settlement); + } + async cancelled( workerId: string, incarnationId: string, diff --git a/service/src/config.ts b/service/src/config.ts index 7ba8b9b4..b3e397b8 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -420,6 +420,11 @@ export const env = { BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: Number( process.env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? 1, ), + /** + * Offer `lane_git` to workers and accept `laneGit` (`{ branch, head }`) on command results. + * Off by default: enable only after the LibreChat talking to this service accepts the field. + */ + BRIDGE_LANE_GIT: process.env.CODEAPI_BRIDGE_LANE_GIT === 'true', /** Outbound worker selected by the remote-bridge backend. */ BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', /** Static compatibility auth or short-lived proof-of-possession credentials. */ diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index 8b3abd1d..bafea2fe 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -15,6 +15,7 @@ function render(overrides) { CODEAPI_BRIDGE_WORKER_ID: '', CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', + CODEAPI_BRIDGE_LANE_GIT: '', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', @@ -34,6 +35,7 @@ for (const overrides of [ CODEAPI_BRIDGE_DYNAMIC_WORKERS: 'false', CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', + CODEAPI_BRIDGE_LANE_GIT: 'true', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', @@ -49,6 +51,7 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_AUTH_MODE, 'paired'); assert.equal(env.CODEAPI_BRIDGE_TOKEN, token); assert.equal(env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, overrides.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? '1'); + assert.equal(env.CODEAPI_BRIDGE_LANE_GIT, overrides.CODEAPI_BRIDGE_LANE_GIT ?? 'false'); assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); assert.equal(env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, overrides.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? ''); From 68c9f931b67fea8d8d96eb64b869dc6d70d04466 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 18:02:38 -0400 Subject: [PATCH 73/75] fix: Relay Loopback Resolvers Into KVM Guests (#314) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧭 fix: Relay Loopback Resolvers Into KVM Guests The KVM launcher forwards the runner's /etc/resolv.conf into the libkrun guest. Under Docker Compose that names the embedded resolver 127.0.0.11, but the TSI guest kernel keeps loopback traffic on its own loopback device, so every service-name lookup in the guest failed. Generated-file uploads and session marker reads to egress_gateway were dropped with ConnectionRefused or FailedToOpenSocket, while connections by IP worked. When the runner has a loopback nameserver, relay it over UDP and TCP port 53 from the runner's own IPv4 address with socat and hand the guest that address instead. Further loopback nameservers are dropped, routable resolvers such as Kubernetes cluster DNS are forwarded unchanged, and startup fails if the relay cannot listen or the runner has no routable address. * 🧭 fix: Bound the Guest DNS Relay and Relay IPv6 Loopback Give both relays an idle timeout and cap concurrent TCP children. socat 1.8 fails every UDP-RECVFROM child given max-children, so the UDP relay relies on its short idle timeout. Treat a `::1` nameserver as loopback too, relaying it over IPv6 from the same runner address instead of forwarding an address the guest would only answer locally. * 🧭 fix: Recognize Every Loopback Resolver Spelling Classify nameservers as loopback by value rather than by the canonical `::1` string: zero-padded or compressed IPv6 loopback and IPv4-mapped 127.0.0.0/8 are relayed too, forwarding over IPv6 to the address exactly as written. Routable IPv6 resolvers stay untouched. * 🧭 fix: Parse Nameserver Addresses Before Loopback Detection Replace string matching with real parsing. IPv6 addresses are expanded to their eight groups (compressed zeros, dotted IPv4 tails), and IPv4 addresses are read in the inet_aton(3) forms glibc accepts in resolv.conf. Then ::1 and IPv4-mapped or plain 127.0.0.0/8 are relayed in any spelling. Mapped addresses are relayed over IPv4 to their canonical form. Spellings of the same resolver collapse into one relayed entry, and addresses glibc rejects are forwarded unchanged. --- README.md | 11 +++ launcher/entrypoint.sh | 167 ++++++++++++++++++++++++++++++++++++++++- tests/kvm_guest_dns.sh | 115 +++++++++++++++++++++++++--- 3 files changed, 282 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index db4038c4..94ad3b9c 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,17 @@ read-only root disk does not need modification at boot. Rebuild the runner image to pick up this layout change. A missing resolver handoff fails startup rather than leaving the guest with an unrelated public DNS server. +The guest kernel keeps loopback traffic on its own loopback device instead of +proxying it through TSI, so it cannot reach a loopback resolver such as Docker's +embedded `127.0.0.11`. The launcher entrypoint relays the first loopback +nameserver (`127.0.0.0/8`, `::1` or IPv4-mapped `127.0.0.0/8`, in any spelling +glibc accepts) over UDP and TCP port 53 from the runner's +own IPv4 address with `socat`, and forwards that address to the guest in its +place; further loopback nameservers are dropped, and startup fails if the relay +cannot listen. Relay children exit when idle, and the TCP relay caps its +concurrent children. Routable nameservers, including Kubernetes cluster DNS, are forwarded +unchanged. + libkrun delivers the guest environment on the kernel command line, which only carries single-line printable ASCII and is capped at 2048 bytes by the guest kernel. The launcher entrypoint therefore forwards only the `nameserver`, diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index 304858e4..47fba9e6 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -12,15 +12,129 @@ set -e # joined by a separator that api/src/guest-dns.sh expands back into lines. RESOLV_FIELD_SEPARATOR='|' +# The guest kernel keeps loopback traffic on its own loopback device instead of +# proxying it through TSI, so a loopback resolver such as Docker's embedded +# 127.0.0.11 never answers inside the guest. Relay it from this container's own +# address, which the guest reaches through TSI, and hand the guest that address. +# The relay forks per datagram and per connection, so bound how long an idle +# child may live, and how many TCP children may run at once. socat 1.8 fails +# every UDP-RECVFROM child given max-children, so the UDP relay relies on its +# short idle timeout. Only the runner network reaches the relay: NsJail jobs in +# the guest run in their own network namespace. +RESOLVER_RELAY_READY_ATTEMPTS=50 +RESOLVER_RELAY_MAX_CHILDREN=64 +RESOLVER_RELAY_UDP_IDLE_SECONDS=5 +RESOLVER_RELAY_TCP_IDLE_SECONDS=30 + +# One inet_aton(3) part: hex with 0x, octal with a leading 0, else decimal. +inet_aton_part() { + case "$1" in + 0x*) [[ "$1" =~ ^0x[0-9a-f]+$ ]] && echo $((16#${1#0x})) ;; + 0*) [[ "$1" =~ ^0[0-7]*$ ]] && echo $((8#$1)) ;; + *) [[ "$1" =~ ^[0-9]+$ ]] && echo $((10#$1)) ;; + esac +} + +# Numeric value of an IPv4 address in the inet_aton(3) forms glibc accepts in +# resolv.conf: one to four parts, the last filling the remaining bytes. +ipv4_value() { + local parts part value=0 index last + IFS=. read -ra parts <<< "$1" + [[ "$1" != *. ]] && (( ${#parts[@]} >= 1 && ${#parts[@]} <= 4 )) || return 1 + last=$(( ${#parts[@]} - 1 )) + for ((index = 0; index < last; index++)); do + part="$(inet_aton_part "${parts[index]}")" || return 1 + (( part <= 255 )) || return 1 + value=$(( value | part << (24 - 8 * index) )) + done + part="$(inet_aton_part "${parts[last]}")" || return 1 + (( part < 1 << (32 - 8 * last) )) || return 1 + echo $(( value | part )) +} + +# The eight 16-bit groups of an IPv6 address in decimal, accepting compressed +# zeros and a dotted IPv4 tail. +ipv6_groups() { + local address="$1" head tail octets octet group groups=() tail_groups=() fill + if [[ "$address" =~ ^(.*:)([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + address="${BASH_REMATCH[1]}" + IFS=. read -ra octets <<< "${BASH_REMATCH[2]}" + for octet in "${octets[@]}"; do + [[ "$octet" =~ ^(0|[1-9][0-9]{0,2})$ ]] && (( octet <= 255 )) || return 1 + done + address+="$(printf '%x:%x' $(( octets[0] << 8 | octets[1] )) $(( octets[2] << 8 | octets[3] )))" + fi + [[ "$address" =~ ^[0-9a-f:]+$ && "$address" != :[!:]* && "$address" != *[!:]: && "$address" != *:::* ]] || return 1 + if [[ "$address" == *::* ]]; then + head="${address%%::*}" + tail="${address#*::}" + [[ "$tail" != *::* ]] || return 1 + [ -z "$head" ] || IFS=: read -ra groups <<< "$head" + [ -z "$tail" ] || IFS=: read -ra tail_groups <<< "$tail" + fill=$(( 8 - ${#groups[@]} - ${#tail_groups[@]} )) + (( fill >= 1 )) || return 1 + for ((; fill > 0; fill--)); do groups+=(0); done + groups+=("${tail_groups[@]}") + else + IFS=: read -ra groups <<< "$address" + fi + (( ${#groups[@]} == 8 )) || return 1 + for group in "${groups[@]}"; do + [[ "$group" =~ ^[0-9a-f]{1,4}$ ]] || return 1 + printf '%d ' $((16#$group)) + done +} + +# Canonical form of a loopback nameserver in any spelling glibc accepts: IPv4 +# 127.0.0.0/8, IPv6 ::1, or IPv4-mapped 127.0.0.0/8 (relayed as IPv4). Fails for +# every other address. +canonical_loopback() { + local address="${1,,}" text groups value + address="${address%%\%*}" + if [[ "$address" == *:* ]]; then + text="$(ipv6_groups "$address")" || return 1 + read -ra groups <<< "$text" + if [ "${groups[*]}" = '0 0 0 0 0 0 0 1' ]; then + echo ::1 + return 0 + fi + [ "${groups[*]:0:6}" = '0 0 0 0 0 65535' ] || return 1 + value=$(( groups[6] << 16 | groups[7] )) + else + value="$(ipv4_value "$address")" || return 1 + fi + (( value >> 24 == 127 )) || return 1 + printf '%d.%d.%d.%d' $(( value >> 24 )) $(( value >> 16 & 255 )) $(( value >> 8 & 255 )) $(( value & 255 )) +} + +loopback_nameserver() { + local line words + while IFS= read -r line || [ -n "$line" ]; do + read -ra words <<< "${line%$'\r'}" + if [ "${words[0]:-}" = nameserver ] && canonical_loopback "${words[1]:-}"; then + return 0 + fi + done < "$1" +} + +runner_address() { + getent ahostsv4 "$(cat /proc/sys/kernel/hostname)" | awk '$1 !~ /^127\./ { print $1; exit }' +} + encode_resolv_conf() { + local loopback="${1:-}" relay="${2:-}" local LC_ALL=C - local line words encoded='' + local line words canonical encoded='' while IFS= read -r line || [ -n "$line" ]; do line="${line%$'\r'}" if [[ ! "$line" =~ ^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]] ]]; then continue fi read -ra words <<< "$line" + if [ "${words[0]}" = nameserver ] && canonical="$(canonical_loopback "${words[1]}")"; then + [ "$canonical" = "$loopback" ] || continue + words[1]="$relay" + fi line="${words[*]}" if [[ "$line" == *[!' '-'~']* || "$line" == *[\"$RESOLV_FIELD_SEPARATOR]* ]]; then echo "ERROR: runner /etc/resolv.conf line cannot cross the kernel command line: $line" >&2 @@ -31,13 +145,62 @@ encode_resolv_conf() { printf '%s' "$encoded" } -SANDBOX_RESOLV_CONF="$(encode_resolv_conf < /etc/resolv.conf)" +proc_net_address() { + local a b c d + IFS=. read -r a b c d <<< "$1" + printf '%02X%02X%02X%02X:0035' "$d" "$c" "$b" "$a" +} + +start_resolver_relay() { + local nameserver="$1" address="$2" local_address udp_pid tcp_pid attempt + local udp_target="UDP4-SENDTO:$nameserver:53" tcp_target="TCP4:$nameserver:53" + if [ "$nameserver" = ::1 ]; then + udp_target='UDP6-SENDTO:[::1]:53' + tcp_target='TCP6:[::1]:53' + fi + socat -T "$RESOLVER_RELAY_UDP_IDLE_SECONDS" \ + "UDP4-RECVFROM:53,bind=$address,fork" "$udp_target" & + udp_pid=$! + socat -T "$RESOLVER_RELAY_TCP_IDLE_SECONDS" \ + "TCP4-LISTEN:53,bind=$address,reuseaddr,fork,max-children=$RESOLVER_RELAY_MAX_CHILDREN" "$tcp_target" & + tcp_pid=$! + local_address="$(proc_net_address "$address")" + for ((attempt = 0; attempt < RESOLVER_RELAY_READY_ATTEMPTS; attempt++)); do + if grep -q " $local_address " /proc/net/udp && grep -q " $local_address 00000000:0000 0A " /proc/net/tcp; then + echo "Relaying guest DNS from $address:53 to loopback resolver $nameserver" + return 0 + fi + if ! kill -0 "$udp_pid" "$tcp_pid" 2>/dev/null; then + break + fi + sleep 0.1 + done + kill "$udp_pid" "$tcp_pid" 2>/dev/null || true + echo "ERROR: guest DNS relay could not listen on $address:53 for loopback resolver $nameserver" >&2 + return 1 +} + +LOOPBACK_NAMESERVER="$(loopback_nameserver /etc/resolv.conf)" +RESOLVER_RELAY_ADDRESS='' +if [ -n "$LOOPBACK_NAMESERVER" ]; then + RESOLVER_RELAY_ADDRESS="$(runner_address)" + if [ -z "$RESOLVER_RELAY_ADDRESS" ]; then + echo "ERROR: runner has no non-loopback IPv4 address to relay loopback resolver $LOOPBACK_NAMESERVER" >&2 + exit 1 + fi +fi + +SANDBOX_RESOLV_CONF="$(encode_resolv_conf "$LOOPBACK_NAMESERVER" "$RESOLVER_RELAY_ADDRESS" < /etc/resolv.conf)" export SANDBOX_RESOLV_CONF if [[ "$RESOLV_FIELD_SEPARATOR$SANDBOX_RESOLV_CONF" != *"${RESOLV_FIELD_SEPARATOR}nameserver "[!\#]* ]]; then echo 'ERROR: runner /etc/resolv.conf has no nameserver' >&2 exit 1 fi +if [ -n "$LOOPBACK_NAMESERVER" ]; then + start_resolver_relay "$LOOPBACK_NAMESERVER" "$RESOLVER_RELAY_ADDRESS" +fi + if [ "${LAUNCHER_FILTER_VSOCK_ENOTCONN:-true}" = "true" ]; then # libkrun can emit this benign TSI/vsock teardown line after the guest has # already closed its side of the socket. It contains the word "error", so diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index 0e038a12..4ea99afb 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -80,31 +80,60 @@ printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$TEST_RESOLVER_OUTPUT" STUB cat > "$TEST_DIR/bin/getent" <<'STUB' #!/usr/bin/env bash -printf '192.0.2.99 stale-address\n' +[[ -z "${TEST_RUNNER_ADDRESS-192.0.2.99}" ]] || printf '%s stale-address\n' "${TEST_RUNNER_ADDRESS-192.0.2.99}" STUB -chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" +# The relay stub records its endpoints; readiness comes from the fake /proc/net. +cat > "$TEST_DIR/bin/socat" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$TEST_SOCAT_CALLS" +STUB +chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" "$TEST_DIR/bin/socat" +# 192.0.2.99:53 bound for UDP and listening for TCP, in /proc/net byte order. +mkdir "$TEST_DIR/proc-net-bound" "$TEST_DIR/proc-net-unbound" +printf ' sl local_address rem_address st\n 0: 630200C0:0035 00000000:0000 07 00000000:00000000\n' > "$TEST_DIR/proc-net-bound/udp" +printf ' sl local_address rem_address st\n 0: 630200C0:0035 00000000:0000 0A 00000000:00000000\n' > "$TEST_DIR/proc-net-bound/tcp" +printf ' sl local_address rem_address st\n' | tee "$TEST_DIR/proc-net-unbound/udp" > "$TEST_DIR/proc-net-unbound/tcp" # libkrun appends every guest environment entry to the kernel command line, # which linux-loader limits to single-line printable ASCII. Docker's generated # resolv.conf is multi-line with comments, so the entrypoint must forward only # the resolver directives, joined by the separator guest-dns.sh expands. run_entrypoint() { - local resolv_conf="$1" + local resolv_conf="$1" proc_net="${2:-$TEST_DIR/proc-net-bound}" sed -e "s|/usr/local/bin/launcher|$TEST_DIR/bin/launcher|g" \ -e "s|/etc/resolv.conf|$resolv_conf|g" \ + -e "s|/proc/net/|$proc_net/|g" \ + -e 's|RESOLVER_RELAY_READY_ATTEMPTS=50|RESOLVER_RELAY_READY_ATTEMPTS=3|' \ "$ROOT/launcher/entrypoint.sh" > "$TEST_DIR/entrypoint.sh" - rm -f "$TEST_DIR/forwarded" + rm -f "$TEST_DIR/forwarded" "$TEST_DIR/socat-calls" PATH="$TEST_DIR/bin:$PATH" \ EGRESS_GATEWAY_URL='https://egress_gateway:3190/base' \ FILE_SERVER_URL='http://[::1]:3000/base' \ SANDBOX_FORWARD_TARGET='tool_call_server:3033' \ LAUNCHER_FILTER_VSOCK_ENOTCONN=false \ TEST_RESOLVER_OUTPUT="$TEST_DIR/forwarded" \ + TEST_SOCAT_CALLS="$TEST_DIR/socat-calls" \ bash "$TEST_DIR/entrypoint.sh" } # Independent reference for the expected handoff: directives only, whitespace -# collapsed, joined by the separator. +# collapsed, joined by the separator. The first loopback nameserver becomes the +# relay address; other loopback nameservers are unreachable and dropped. encoded_reference() { - LC_ALL=C awk '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { sub(/\r$/, ""); $1 = $1; print }' "$1" | paste -sd '|' - + LC_ALL=C awk -v relay=192.0.2.99 '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { + sub(/\r$/, ""); $1 = $1 + address = tolower($2) + if ($1 == "nameserver" && (address ~ /^127\./ || address ~ /^[0:]*:0*1$/ || address ~ /^::ffff:127\./)) { if (loop == "") loop = $2; if ($2 != loop) next; $2 = relay } + print + }' "$1" | paste -sd '|' - +} +# The relays start in the background, so wait for both before comparing. Both +# bound idle children; the TCP listener also caps concurrent children. +assert_relayed() { + local udp_target="$1" tcp_target="$2" attempt + for ((attempt = 0; attempt < 50; attempt++)); do + [[ -e "$TEST_DIR/socat-calls" && "$(wc -l < "$TEST_DIR/socat-calls")" -ge 2 ]] && break + sleep 0.1 + done + [[ "$(LC_ALL=C sort "$TEST_DIR/socat-calls")" == "$(printf -- '-T 30 TCP4-LISTEN:53,bind=192.0.2.99,reuseaddr,fork,max-children=64 %s\n-T 5 UDP4-RECVFROM:53,bind=192.0.2.99,fork %s' "$tcp_target" "$udp_target")" ]] } cat > "$TEST_DIR/docker-resolv.conf" <<'RESOLV' # Generated by Docker Engine. @@ -119,24 +148,92 @@ options ndots:0 # Overrides: [] # Option ndots from: internal RESOLV +# The guest keeps loopback traffic local, so Docker's embedded resolver is +# relayed from the runner address and the guest gets that address instead. run_entrypoint "$TEST_DIR/docker-resolv.conf" -[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 127.0.0.11|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] [[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/docker-resolv.conf")" ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 [[ "$(wc -l < "$TEST_DIR/forwarded")" == 1 ]] if LC_ALL=C grep -q '[^ -~]' "$TEST_DIR/forwarded"; then echo 'forwarded resolver contains bytes the kernel command line rejects' >&2; exit 1 fi -# The guest restores exactly the directives Docker supplied. +# The guest restores the Docker directives with the relayed nameserver. rm -rf "$TEST_DIR/guest/run/codeapi-resolver" SANDBOX_RESOLV_CONF="$(cat "$TEST_DIR/forwarded")" configure_guest_dns "$TEST_DIR/guest" -printf 'nameserver 127.0.0.11\noptions ndots:0\n' > "$TEST_DIR/expected" +printf 'nameserver 192.0.2.99\noptions ndots:0\n' > "$TEST_DIR/expected" cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" +# Only one loopback resolver can be relayed; other loopback entries would time +# out in the guest, while routable nameservers keep their order. +printf 'nameserver 127.0.0.53\r\nnameserver 10.0.0.2\nnameserver\t127.0.0.11\nsearch example.internal\n' > "$TEST_DIR/multi-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/multi-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2|search example.internal' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/multi-loopback-resolv.conf")" ]] +assert_relayed UDP4-SENDTO:127.0.0.53:53 TCP4:127.0.0.53:53 + +# An IPv6 loopback resolver is just as unreachable from the guest; the IPv4 +# relay forwards to it over IPv6. +printf 'nameserver ::1\noptions ndots:0\n' > "$TEST_DIR/ipv6-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/ipv6-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/ipv6-loopback-resolv.conf")" ]] +assert_relayed 'UDP6-SENDTO:[::1]:53' 'TCP6:[::1]:53' +printf 'nameserver 127.0.0.11\nnameserver ::1\n' > "$TEST_DIR/mixed-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/mixed-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99' ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +# Every spelling glibc accepts for a loopback resolver is relayed to its +# canonical address: compressed or zero-padded ::1, IPv4-mapped 127/8 in any +# IPv6 form, and inet_aton(3) IPv4 forms. +while read -r spelling udp_target tcp_target; do + printf 'nameserver %s\nnameserver 10.0.0.2\n' "$spelling" > "$TEST_DIR/spelled-loopback-resolv.conf" + run_entrypoint "$TEST_DIR/spelled-loopback-resolv.conf" + [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2' ]] + assert_relayed "$udp_target" "$tcp_target" +done <<'SPELLINGS' +0:0:0:0:0:0:0:1 UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +0000::0001 UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +::1%lo UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +::FFFF:127.0.0.11 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0:0:0:0:0:ffff:127.0.0.11 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +::ffff:7f00:b UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +127.1 UDP4-SENDTO:127.0.0.1:53 TCP4:127.0.0.1:53 +2130706443 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0x7f.0.0.0xb UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0177.0.0.013 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +SPELLINGS +# Spellings of the same loopback resolver collapse into one relayed entry. +printf 'nameserver 127.0.0.11\nnameserver ::ffff:7f00:b\nnameserver 127.0.0.53\n' > "$TEST_DIR/same-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/same-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 192.0.2.99' ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +# Routable resolvers and addresses glibc rejects are forwarded unchanged, without +# a relay: 0127 is octal 87, and ::7f00:1 is not IPv4-mapped. +printf 'nameserver 2001:db8::1\nnameserver fd00::11\nnameserver ::ffff:10.0.0.2\nnameserver ::7f00:1\nnameserver 0127.0.0.1\nnameserver 127.0.0.256\nnameserver 1:2:3:4:5:6:7:8:\nnameserver 127..1\n' > "$TEST_DIR/routable-resolv.conf" +run_entrypoint "$TEST_DIR/routable-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 2001:db8::1|nameserver fd00::11|nameserver ::ffff:10.0.0.2|nameserver ::7f00:1|nameserver 0127.0.0.1|nameserver 127.0.0.256|nameserver 1:2:3:4:5:6:7:8:|nameserver 127..1' ]] +[[ ! -e "$TEST_DIR/socat-calls" ]] + +# A relay that never listens, or a runner without a routable address, fails +# startup instead of booting a guest that cannot resolve service names. +if run_entrypoint "$TEST_DIR/docker-resolv.conf" "$TEST_DIR/proc-net-unbound" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a listening resolver relay' >&2; exit 1 +fi +grep -q 'guest DNS relay could not listen on 192.0.2.99:53' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" ]] +if TEST_RUNNER_ADDRESS='' run_entrypoint "$TEST_DIR/docker-resolv.conf" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a resolver relay address' >&2; exit 1 +fi +grep -q 'no non-loopback IPv4 address to relay loopback resolver 127.0.0.11' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" && ! -e "$TEST_DIR/socat-calls" ]] + # Kubernetes resolvers: tabs, CRLF, trailing spaces, and unknown keywords are # normalized away; search domains and options survive intact. printf 'nameserver\t10.96.0.10 \r\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\n; resolver comment\nlookup file bind\noptions ndots:5\n' > "$TEST_DIR/k8s-resolv.conf" # leak-check:allow run_entrypoint "$TEST_DIR/k8s-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' ]] # leak-check:allow +[[ ! -e "$TEST_DIR/socat-calls" ]] # The runner's own resolver must round-trip through the same reference. run_entrypoint /etc/resolv.conf From a62d69e05a4541ac9a20722ac7ff3fcfecb6ed24 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 6 Oct 2026 01:21:55 -0400 Subject: [PATCH 74/75] =?UTF-8?q?=F0=9F=94=92=20Harden=20Runner=20Supervis?= =?UTF-8?q?ors=20with=20Anonymous=20Pipes=20(#315)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Harden runner supervisors with anonymous pipes and inherited seccomp * Load hosted-app netfilter modules in native CI and retain diagnostics * Keep relay idle on buffered pipe EOF and assert owned descriptor cleanup * Preserve supervisor DNS resolution without descriptor-send syscalls --- .github/workflows/ci.yml | 36 ++++++ api/Dockerfile | 16 ++- api/src/anonymous-pipes.c | 34 ++++++ api/src/entrypoint.sh | 4 +- api/src/hosted-app.ts | 26 ++++- api/src/nsjail.ts | 3 +- api/src/pipe-process.test.ts | 89 +++++++++++++++ api/src/pipe-process.ts | 116 ++++++++++++++++++++ api/src/session-checkpoint.ts | 2 +- api/src/supervisor-policy.c | 60 ++++++++++ api/src/tool-call-pipe-bridge.c | 40 ++++--- api/src/tool-call-pipe-broker.ts | 19 +++- docker/Dockerfile.worker-sandbox | 15 ++- docker/nsjail-anonymous-control.patch | 82 ++++++++++++++ docs/adr/0013-supervisor-anonymous-pipes.md | 65 +++++++++++ launcher/Dockerfile | 15 ++- tests/pipe_relay_backpressure.py | 55 ++++++++++ tests/supervisor_dns.py | 49 +++++++++ tests/supervisor_hosted_network.ts | 42 +++++++ tests/supervisor_policy.py | 51 +++++++++ tests/tool_call_runner.cjs | 2 +- 21 files changed, 784 insertions(+), 37 deletions(-) create mode 100644 api/src/anonymous-pipes.c create mode 100644 api/src/pipe-process.test.ts create mode 100644 api/src/pipe-process.ts create mode 100644 api/src/supervisor-policy.c create mode 100644 docker/nsjail-anonymous-control.patch create mode 100644 docs/adr/0013-supervisor-anonymous-pipes.md create mode 100644 tests/pipe_relay_backpressure.py create mode 100644 tests/supervisor_dns.py create mode 100644 tests/supervisor_hosted_network.ts create mode 100644 tests/supervisor_policy.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98ddb2ff..50804e27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,16 @@ jobs: - name: Install dependencies run: bun ci + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: 22 + + - name: Compile anonymous-pipe addon + run: | + mkdir -p .build + NODE_HEADERS="$(dirname "$(dirname "$(command -v node)")")/include/node" + gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I"$NODE_HEADERS" -o .build/anonymous-pipes.node src/anonymous-pipes.c + - name: Build API run: bun run build @@ -174,6 +184,32 @@ jobs: --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ --entrypoint /usr/local/bin/node \ launcher-guest-test /tests/pipe_runtime_image.cjs + - name: Load legacy netfilter modules for hosted-app isolation + run: sudo modprobe -a ip_tables ip6_tables iptable_filter ip6table_filter xt_conntrack xt_owner ipt_REJECT ip6t_REJECT + - name: Verify supervisor filter and anonymous stdio + run: | + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/supervisor_policy.py + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/pipe_relay_backpressure.py + docker run --rm --init --dns 127.0.0.1 --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/supervisor_dns.py + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + nsjail-ipc-test bun test /work/api/src/pipe-process.test.ts + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + launcher-guest-test node /tests/pipe_runtime_image.cjs + docker run --rm --init --cap-add NET_ADMIN --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + launcher-guest-test bun /work/tests/supervisor_hosted_network.ts + - name: Verify descriptor denial and runtime IPC with the installed filter run: | docker run --rm --init --security-opt seccomp=unconfined \ diff --git a/api/Dockerfile b/api/Dockerfile index f953f537..d630c547 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -26,18 +26,27 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup # Native policy/transport verification without building the full runner. FROM nsjail-builder AS nsjail-ipc-test +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node COPY docker/python-runtime /python-runtime RUN python3 /python-runtime/install.py COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node @@ -121,6 +130,9 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node RUN chmod +x /usr/sbin/nsjail # The per-invocation broker starts Node after restore, so its OpenSSL process diff --git a/api/src/anonymous-pipes.c b/api/src/anonymous-pipes.c new file mode 100644 index 00000000..524c3444 --- /dev/null +++ b/api/src/anonymous-pipes.c @@ -0,0 +1,34 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +static napi_value create_pipe(napi_env env, napi_callback_info info) { + (void)info; + int fds[2]; + if (pipe2(fds, O_CLOEXEC) != 0) { + napi_throw_error(env, NULL, strerror(errno)); + return NULL; + } + napi_value array, reader, writer; + if (napi_create_array_with_length(env, 2, &array) != napi_ok + || napi_create_int32(env, fds[0], &reader) != napi_ok + || napi_create_int32(env, fds[1], &writer) != napi_ok + || napi_set_element(env, array, 0, reader) != napi_ok + || napi_set_element(env, array, 1, writer) != napi_ok) { + close(fds[0]); close(fds[1]); + napi_throw_error(env, NULL, "cannot return anonymous pipe"); + return NULL; + } + return array; +} + +static napi_value initialize(napi_env env, napi_value exports) { + napi_value fn; + if (napi_create_function(env, "createPipe", NAPI_AUTO_LENGTH, create_pipe, NULL, &fn) != napi_ok + || napi_set_named_property(env, exports, "createPipe", fn) != napi_ok) return NULL; + return exports; +} +NAPI_MODULE(NODE_GYP_MODULE_NAME, initialize) diff --git a/api/src/entrypoint.sh b/api/src/entrypoint.sh index d670a359..0f93981b 100755 --- a/api/src/entrypoint.sh +++ b/api/src/entrypoint.sh @@ -178,7 +178,7 @@ if [ "$SANDBOX_USE_CGROUPV2" = "true" ]; then NSJAIL_CGROUP_ARGS=(--use_cgroupv2) fi -if timeout 10 "${NSJAIL_PATH:-/usr/sbin/nsjail}" --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \ +if timeout 10 /usr/local/bin/sandbox-supervisor-policy "${NSJAIL_PATH:-/usr/sbin/nsjail}" --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \ "${NSJAIL_CGROUP_ARGS[@]}" --log "$SMOKE_LOG" \ --user "65534:${SMOKE_OUTSIDE_UID}:1" --group "65534:${SMOKE_OUTSIDE_GID}:1" \ -s /usr/bin:/bin -s /usr/lib:/lib -s /usr/lib64:/lib64 \ @@ -201,4 +201,4 @@ rm -f "$SMOKE_STDERR" rm -rf "$SMOKE_DIR" echo "Starting sandbox API server..." -exec bun run /sandbox_api/.build/index.js +exec /usr/local/bin/sandbox-supervisor-policy bun run /sandbox_api/.build/index.js diff --git a/api/src/hosted-app.ts b/api/src/hosted-app.ts index 64a5aab8..0ce57370 100644 --- a/api/src/hosted-app.ts +++ b/api/src/hosted-app.ts @@ -1,10 +1,10 @@ -import { execFile, spawn, type ChildProcess, type SpawnOptions } from 'node:child_process'; +import { spawnPipeProcess as spawn } from './pipe-process'; +import type { ChildProcess, SpawnOptions } from 'node:child_process'; import { createHash } from 'node:crypto'; import * as fsp from 'node:fs/promises'; import * as net from 'node:net'; import * as path from 'node:path'; import type { Readable } from 'node:stream'; -import { promisify } from 'node:util'; import { config } from './config'; import { aggregateBashExtras, filterExtraEnvVars } from './job'; import { logger } from './logger'; @@ -12,7 +12,22 @@ import { getLatestRuntimeMatchingLanguageVersion, getRuntimes, type Runtime } fr import { getBoundSessionWorkspace, type SessionWorkspace } from './session-workspace'; import { ValidationError, validateFilePath } from './validation'; -const execFileAsync = promisify(execFile); +// Capture bounded diagnostics through an anonymous pipe; execFile's runtime +// capture would create socketpairs prohibited by the supervisor policy. +function execFileAsync(binary: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { stdio: ['ignore', 'ignore', 'pipe'] }); + let diagnostics = Buffer.alloc(0); + child.stderr.on('data', (chunk: Buffer) => { + diagnostics = Buffer.concat([diagnostics, chunk]).subarray(-8192); + }); + child.once('error', reject); + child.once('close', (code, signal) => { + if (code === 0) resolve(); + else reject(new Error(`${binary} failed (${signal ?? code}): ${diagnostics.toString().trim()}`)); + }); + }); +} const APP_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; const REVISION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; @@ -342,7 +357,10 @@ async function commandSucceeds(binary: string, args: string[]): Promise * packets for accepted inbound connections remain allowed by conntrack. */ export async function installHostedAppNetworkGuard(uid: number): Promise { - for (const binary of ['/usr/sbin/iptables', '/usr/sbin/ip6tables']) { + // nft's netlink transport needs sendmsg, which the supervisor denies. The + // packaged legacy backend uses getsockopt/setsockopt instead. Failure to + // install either family remains fatal before admitting any hosted app. + for (const binary of ['/usr/sbin/iptables-legacy', '/usr/sbin/ip6tables-legacy']) { if (!(await commandSucceeds(binary, ['-w', '5', '-L', HOSTED_APP_EGRESS_CHAIN]))) { await execFileAsync(binary, ['-w', '5', '-N', HOSTED_APP_EGRESS_CHAIN]); } diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 5c39884f..23c048a6 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -1,6 +1,7 @@ +import { spawnPipeProcess as spawn } from './pipe-process'; import * as fs from 'fs'; import * as path from 'path'; -import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { nanoid } from 'nanoid'; import { config } from './config'; import { logger } from './logger'; diff --git a/api/src/pipe-process.test.ts b/api/src/pipe-process.test.ts new file mode 100644 index 00000000..608002c8 --- /dev/null +++ b/api/src/pipe-process.test.ts @@ -0,0 +1,89 @@ +import { describe, test, expect } from 'bun:test'; +import { once } from 'node:events'; +import { pipeline } from 'node:stream/promises'; +import { Readable } from 'node:stream'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { spawnPipeProcess } from './pipe-process'; + +async function collect(stream: Readable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +} + +describe('anonymous process pipes', () => { + test('large concurrent streams preserve backpressure and output before close', async () => { + const bytes = Buffer.alloc(2 * 1024 * 1024, 'p'); + await Promise.all(Array.from({ length: 8 }, async () => { + const child = spawnPipeProcess('/bin/sh', ['-c', 'cat; printf stderr >&2']); + const closed = once(child, 'close'); + const output = collect(child.stdout); + const errors = collect(child.stderr); + await pipeline(Readable.from([bytes]), child.stdin); + const [code] = await closed; + expect(code).toBe(0); + expect(await output).toEqual(bytes); + expect((await errors).toString()).toBe('stderr'); + })); + }, 15000); + + test.skipIf(process.platform !== 'linux')('Linux children receive FIFO stdin, stdout and stderr', async () => { + const child = spawnPipeProcess('/bin/sh', ['-c', 'test -p /proc/self/fd/0 && test -p /proc/self/fd/1 && test -p /proc/self/fd/2']); + child.stdin.end(); + const [code] = await once(child, 'close'); + if (process.platform === 'linux') expect(code).toBe(0); + }); + + test('cancellation releases a blocked stdin writer', async () => { + const child = spawnPipeProcess('/bin/sleep', ['30']); + const closed = once(child, 'close'); + const writing = pipeline(Readable.from([Buffer.alloc(2 * 1024 * 1024)]), child.stdin); + // Observe immediately: cancellation may reject before the exit listener. + const rejected = writing.then(() => false, () => true); + await new Promise(resolve => setTimeout(resolve, 20)); + child.kill('SIGKILL'); + await closed; + expect(await rejected).toBe(true); + expect(child.stdin.destroyed).toBe(true); + }, 5000); + + test('spawn failures and killed children release every pipe', async () => { + const ownedPipes = new Set(); + const addon = process.platform === 'linux' + ? require(process.env.SANDBOX_PIPE_ADDON || path.resolve(__dirname, '..', '.build', 'anonymous-pipes.node')) : undefined; + const create = addon?.createPipe; + if (addon) addon.createPipe = () => { + const fds = create(); + for (const fd of fds) ownedPipes.add(fs.readlinkSync(`/proc/self/fd/${fd}`)); + return fds; + }; + try { + for (let n = 0; n < 12; n++) { + const missing = spawnPipeProcess('/missing-codeapi-executable'); + const missingClosed = new Promise(resolve => missing.once('close', () => resolve())); + const error = await new Promise(resolve => missing.once('error', resolve)); + expect(error.code).toBe('ENOENT'); + await missingClosed; + const child = spawnPipeProcess('/bin/sleep', ['30']); + const closed = once(child, 'close'); + child.kill('SIGKILL'); + await closed; + expect(child.stdin.destroyed).toBe(true); + expect(child.stdout.destroyed).toBe(true); + expect(child.stderr.destroyed).toBe(true); + } + } finally { + if (addon) addon.createPipe = create; + } + if (process.platform === 'linux') { + // Runtime watcher/pid descriptors can appear asynchronously. Assert the + // exact pipe inodes we allocated, including duplicates held by FileSink. + const survivors = fs.readdirSync('/proc/self/fd').flatMap(fd => { + try { const inode = fs.readlinkSync(`/proc/self/fd/${fd}`); return ownedPipes.has(inode) ? [inode] : []; } + catch { return []; } + }); + expect(survivors).toEqual([]); + } + }, 10000); +}); diff --git a/api/src/pipe-process.ts b/api/src/pipe-process.ts new file mode 100644 index 00000000..a4bc6b84 --- /dev/null +++ b/api/src/pipe-process.ts @@ -0,0 +1,116 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as net from 'node:net'; +import { spawn, type SpawnOptions, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { Readable, Writable } from 'node:stream'; + +let native: { createPipe(): [number, number] } | undefined; +function endpoints(): [number, number] { + native ??= require(process.env.SANDBOX_PIPE_ADDON || path.resolve(__dirname, '..', '.build', 'anonymous-pipes.node')); + return native!.createPipe(); +} +function close(fd: number): void { try { fs.closeSync(fd); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EBADF') throw error; } } +function readable(fd: number): Readable { + if (typeof Bun === 'undefined') return new net.Socket({ fd, readable: true, writable: false }); + const stream = Readable.fromWeb(Bun.file(fd).stream() as never); + stream.once('close', () => close(fd)); + return stream; +} +function writable(fd: number): Writable { + if (typeof Bun === 'undefined') return new net.Socket({ fd, readable: false, writable: true }); + const sink = Bun.file(fd).writer(); + return new Writable({ + write(chunk, _encoding, callback) { + try { sink.write(chunk); Promise.resolve(sink.flush()).then(() => callback(), callback); } + catch (error) { callback(error as Error); } + }, + final(callback) { + try { Promise.resolve(sink.end()).then(() => callback(), callback); } + catch (error) { callback(error as Error); } + }, + destroy(error, callback) { + // FileSink duplicates the supplied FD; end also releases that duplicate + // on spawn failure/cancellation, when _final was never reached. + try { void Promise.resolve(sink.end()).catch(() => {}); } catch {} + close(fd); callback(error); + }, + }); +} +/** Linux child stdio uses pipe2 endpoints, never the runtime's socketpairs. + * Non-Linux development keeps its native spawn implementation. */ +export function spawnPipeProcess(command: string, args: readonly string[] = [], options: SpawnOptions = {}): ChildProcessWithoutNullStreams { + if (process.platform !== 'linux') return spawn(command, args, options) as ChildProcessWithoutNullStreams; + const requested = options.stdio ?? ['pipe', 'pipe', 'pipe']; + const modes = typeof requested === 'string' ? [requested, requested, requested] : requested; + if (!Array.isArray(modes) || modes.slice(3).some(mode => typeof mode !== 'number')) throw new Error('unsupported anonymous-pipe process stdio'); + const childFds: number[] = []; + const parentFds: Array<[number, number]> = []; + const streams: Array = [null, null, null]; + let child: ReturnType | undefined; + try { + const stdio = Array.from({ length: Math.max(3, modes.length) }, (_, index) => index).map(index => { + const mode = modes[index] ?? 'pipe'; + if (mode !== 'pipe') { + if (mode !== 'ignore' && mode !== 'inherit' && typeof mode !== 'number') throw new Error('unsupported anonymous-pipe process stdio'); + return mode; + } + const [reader, writer] = endpoints(); + const parentFd = index === 0 ? writer : reader; + const childFd = index === 0 ? reader : writer; + childFds.push(childFd); + parentFds.push([index, parentFd]); + return childFd; + }); + child = spawn(command, args, { ...options, stdio }); + for (const [index, fd] of parentFds) streams[index] = index === 0 ? writable(fd) : readable(fd); + Object.defineProperties(child, { + stdin: { value: streams[0] }, stdout: { value: streams[1] }, stderr: { value: streams[2] }, + }); + // Integer stdio descriptors are not owned by ChildProcess. Reproduce its + // close-after-output semantics and release stdin when the child exits. + const outputClosed = Promise.all(streams.slice(1).map(stream => stream + ? new Promise(resolve => stream.once('close', resolve)) : undefined)); + const emit = child.emit; + child.emit = function (event: string | symbol, ...values: unknown[]): boolean { + if (event === 'close') { + void outputClosed.then(() => emit.call(this, event, ...values)); + return true; + } + return emit.call(this, event, ...values); + }; + child.once('exit', () => { + streams[0]?.destroy(); + // Like native spawn, drain output that has no consumer after exit. + for (const stream of streams.slice(1)) { + if (stream && stream.listenerCount('data') === 0 && stream.listenerCount('readable') === 0) (stream as Readable).resume(); + } + }); + child.once('error', () => { for (const stream of streams) stream?.destroy(); }); + return child as ChildProcessWithoutNullStreams; + } catch (error) { + child?.kill('SIGKILL'); + for (const [index, fd] of parentFds) { if (streams[index]) streams[index]!.destroy(); else close(fd); } + throw error; + } finally { for (const fd of childFds) close(fd); } +} + +export function toolCallPipes(): { child: [number, number]; reader: Readable; writer: Writable; closeChild(): void } { + const [requestReader, requestWriter] = endpoints(); + let responseReader: number, responseWriter: number; + try { [responseReader, responseWriter] = endpoints(); } + catch (error) { close(requestReader); close(requestWriter); throw error; } + let reader: Readable | undefined; + let writer: Writable | undefined; + try { + reader = readable(requestReader); + writer = writable(responseWriter); + let childClosed = false; + return { child: [requestWriter, responseReader], reader, writer, + closeChild() { if (!childClosed) { childClosed = true; close(requestWriter); close(responseReader); } } }; + } catch (error) { + if (reader) reader.destroy(); else close(requestReader); + if (writer) writer.destroy(); else close(responseWriter); + close(requestWriter); close(responseReader); + throw error; + } +} diff --git a/api/src/session-checkpoint.ts b/api/src/session-checkpoint.ts index 8ddd7390..dc642619 100644 --- a/api/src/session-checkpoint.ts +++ b/api/src/session-checkpoint.ts @@ -1,4 +1,4 @@ -import { spawn } from 'child_process'; +import { spawnPipeProcess as spawn } from './pipe-process'; import * as fsp from 'fs/promises'; import * as os from 'os'; import * as path from 'path'; diff --git a/api/src/supervisor-policy.c b/api/src/supervisor-policy.c new file mode 100644 index 00000000..b2091927 --- /dev/null +++ b/api/src/supervisor-policy.c @@ -0,0 +1,60 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) +#define NATIVE_ARCH AUDIT_ARCH_X86_64 +#elif defined(__aarch64__) +#define NATIVE_ARCH AUDIT_ARCH_AARCH64 +#else +#error Unsupported supervisor architecture +#endif +#define DENY(call) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_##call, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM) +int main(int argc, char **argv) { + if (argc < 2) return 125; + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, NATIVE_ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), +#ifdef __x86_64__ + BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x40000000, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), +#endif + DENY(socketpair), DENY(sendmsg), DENY(sendmmsg), + DENY(io_uring_setup), DENY(io_uring_enter), DENY(io_uring_register), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_socket, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AF_UNIX, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { .len = sizeof(instructions) / sizeof(instructions[0]), .filter = instructions }; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0 + || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program) != 0) { + perror("sandbox supervisor policy"); return 125; + } + /* glibc's resolver batches queries with sendmmsg. Use sequential sendto + * queries so hostname-based API/broker traffic needs no policy exception. */ + const char *existing = getenv("RES_OPTIONS"); + char *resolver_options = NULL; + if (asprintf(&resolver_options, "%s%ssingle-request", existing ? existing : "", + existing && *existing ? " " : "") < 0) { + perror("sandbox supervisor DNS options"); return 125; + } + if (setenv("RES_OPTIONS", resolver_options, 1) != 0) { + perror("sandbox supervisor DNS options"); free(resolver_options); return 125; + } + free(resolver_options); + execvp(argv[1], &argv[1]); + perror("sandbox supervisor exec"); return 125; +} diff --git a/api/src/tool-call-pipe-bridge.c b/api/src/tool-call-pipe-bridge.c index a3e9f084..af8c2af0 100644 --- a/api/src/tool-call-pipe-bridge.c +++ b/api/src/tool-call-pipe-bridge.c @@ -12,9 +12,9 @@ #include #endif -/* Node/Bun extra stdio uses socketpairs. Keep that trusted-side socket in - * this relay; only two anonymous pipe ends ever reach NsJail/the job. - * FD 3 in the job writes requests; FD 4 reads responses. */ +/* Both broker-facing and job-facing transports are anonymous pipes. + * FD 3 writes requests; FD 4 reads responses. The relay retains bounded + * buffering and detects lost readers even with no queued response. */ struct flow { int source, destination; unsigned char bytes[65536]; @@ -63,50 +63,57 @@ int main(int argc, char **argv) { perror("tool-call broker exec"); return 125; } - int ipc = fcntl(3, F_DUPFD_CLOEXEC, 5); + if (strcmp(argv[1], "--job-pipes") != 0 || argc < 3) return 125; + int request_upstream = fcntl(3, F_DUPFD_CLOEXEC, 5); + int response_upstream = fcntl(4, F_DUPFD_CLOEXEC, 5); + if (request_upstream < 0 || response_upstream < 0) return 125; + close(3); close(4); int requests[2], responses[2]; - if (ipc < 0 || pipe(requests) != 0 || pipe(responses) != 0) return 125; + if (pipe2(requests, O_CLOEXEC) != 0 || pipe2(responses, O_CLOEXEC) != 0) return 125; pid_t relay = getpid(); pid_t child = fork(); if (child < 0) return 125; if (child == 0) { if (parent_death(relay) != 0 || dup2(requests[1], 3) < 0 || dup2(responses[0], 4) < 0) _exit(125); - int fds[] = {ipc, requests[0], requests[1], responses[0], responses[1]}; + int fds[] = {request_upstream, response_upstream, requests[0], requests[1], responses[0], responses[1]}; for (unsigned i = 0; i < sizeof(fds) / sizeof(fds[0]); i++) { if (fds[i] > 4) close(fds[i]); } - execvp(argv[1], &argv[1]); + execvp(argv[2], &argv[2]); perror("tool-call pipe exec"); _exit(125); } - close(3); close(requests[1]); close(responses[0]); signal(SIGPIPE, SIG_IGN); - if (nonblock(ipc) < 0 || nonblock(requests[0]) < 0 || nonblock(responses[1]) < 0) { + if (nonblock(request_upstream) < 0 || nonblock(response_upstream) < 0 || nonblock(requests[0]) < 0 || nonblock(responses[1]) < 0) { kill(child, SIGKILL); waitpid(child, NULL, 0); return 125; } - struct flow request = {.source = requests[0], .destination = ipc}; - struct flow response = {.source = ipc, .destination = responses[1]}; + struct flow request = {.source = requests[0], .destination = request_upstream}; + struct flow response = {.source = response_upstream, .destination = responses[1]}; int status = 125 << 8; for (;;) { pid_t done = waitpid(child, &status, WNOHANG); if (done == child) break; if (done < 0 && errno != EINTR) return 125; struct pollfd pollfds[] = { - {.fd = ipc, .events = (response.length || response.source < 0 ? 0 : POLLIN) | (request.length ? POLLOUT : 0)}, - {.fd = request.source, .events = request.length ? 0 : POLLIN}, + /* HUP is reported even with events=0. Omit buffered sources + * until their destinations drain, or a closed writer spins poll. */ + {.fd = response.length ? -1 : response_upstream, .events = POLLIN}, + {.fd = request.length ? -1 : request.source, .events = POLLIN}, {.fd = responses[1], .events = response.length ? POLLOUT : 0}, + {.fd = request_upstream, .events = request.length ? POLLOUT : 0}, }; - int result = poll(pollfds, 3, 100); + int result = poll(pollfds, 4, 100); if (result < 0 && errno == EINTR) continue; if (result < 0 || (pollfds[0].revents & (POLLERR | POLLNVAL)) || (pollfds[1].revents & (POLLERR | POLLNVAL)) || (pollfds[2].revents & (POLLERR | POLLHUP | POLLNVAL)) - || transfer(&request, pollfds[1].revents & (POLLIN | POLLHUP), pollfds[0].revents & POLLOUT) < 0 + || (pollfds[3].revents & (POLLERR | POLLHUP | POLLNVAL)) + || transfer(&request, pollfds[1].revents & (POLLIN | POLLHUP), pollfds[3].revents & POLLOUT) < 0 || transfer(&response, pollfds[0].revents & (POLLIN | POLLHUP), pollfds[2].revents & POLLOUT) < 0 || response.source < 0) { /* EOF/error terminates the entire invocation, including an * abandoned tool call. Never leave an orphaned NsJail monitor. */ @@ -116,7 +123,8 @@ int main(int argc, char **argv) { break; } } - close(ipc); + close(request_upstream); + close(response_upstream); close(requests[0]); close(responses[1]); return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); diff --git a/api/src/tool-call-pipe-broker.ts b/api/src/tool-call-pipe-broker.ts index 4fb923cc..d81524c2 100644 --- a/api/src/tool-call-pipe-broker.ts +++ b/api/src/tool-call-pipe-broker.ts @@ -1,5 +1,5 @@ -import { spawn } from 'node:child_process'; -import type { Duplex } from 'node:stream'; +import { spawnPipeProcess as spawn, toolCallPipes } from './pipe-process'; +import { Duplex, Readable, Writable } from 'node:stream'; import * as http from 'node:http'; import * as https from 'node:https'; @@ -150,10 +150,17 @@ export function serveToolCallPipe(channel: Duplex, rawTarget: string, options: { if (require.main === module) { const target = process.env.SANDBOX_FORWARD_TARGET?.trim(); if (!target || !process.argv[2]) throw new Error('tool-call pipe broker is not configured'); - const child = spawn(process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge', process.argv.slice(2), { - stdio: [0, 1, 2, 'pipe'], - }); - const channel = child.stdio[3] as Duplex; + const pipes = toolCallPipes(); + let child: ReturnType; + try { + child = spawn(process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge', ['--job-pipes', ...process.argv.slice(2)], { + stdio: [0, 1, 2, ...pipes.child], + }); + } catch (error) { + pipes.reader.destroy(); pipes.writer.destroy(); + throw error; + } finally { pipes.closeChild(); } + const channel = Duplex.fromWeb({ readable: Readable.toWeb(pipes.reader), writable: Writable.toWeb(pipes.writer) }); const stop = serveToolCallPipe(channel, target, { requestTimeoutMs: Math.min(600_000, Math.max(5_000, Number(process.env.TCS_REQUEST_TIMEOUT_MS) || 300_000)), onFailure: () => child.kill('SIGKILL'), diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 0117132d..673dd42f 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -39,12 +39,20 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup @@ -148,6 +156,9 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node RUN chmod +x /usr/sbin/nsjail COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node diff --git a/docker/nsjail-anonymous-control.patch b/docker/nsjail-anonymous-control.patch new file mode 100644 index 00000000..bfce44bf --- /dev/null +++ b/docker/nsjail-anonymous-control.patch @@ -0,0 +1,82 @@ +--- a/subproc.cc ++++ b/subproc.cc +@@ -510,6 +510,12 @@ + } + + pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err) { ++ // This runner uses anonymous control pipes. Descriptor-passing user-mode ++ // networking and seccomp notification transports are intentionally unsupported. ++ if (nsj->njc.seccomp_unotify() || nsj->njc.has_user_net()) { ++ LOG_E("Descriptor-passing NsJail modes are disabled by runner hardening"); ++ return -1; ++ } + if (!net::limitConns(nsj, netfd)) { + return 0; + } +@@ -535,26 +541,34 @@ + LOG_D("Creating new process with clone flags:%s and exit_signal:SIGCHLD", + cloneFlagsToStr(flags).c_str()); + +- int sv[2]; +- if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == -1) { +- PLOG_E("socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC) failed"); ++ int ready[2], errors[2]; ++ if (pipe2(ready, O_CLOEXEC) == -1) { ++ PLOG_E("pipe2(ready) failed"); + return -1; + } +- int child_fd = sv[0]; +- int parent_fd = sv[1]; ++ if (pipe2(errors, O_CLOEXEC) == -1) { ++ close(ready[0]); close(ready[1]); ++ PLOG_E("pipe2(errors) failed"); ++ return -1; ++ } ++ int child_fd = ready[0]; ++ int parent_fd = ready[1]; + + pid_t pid = cloneProc(flags, SIGCHLD); + if (pid == 0) { + close(parent_fd); ++ close(errors[0]); + newProc(nsj, netfd, fd_in, fd_out, fd_err, child_fd); +- util::writeToFd(child_fd, &kSubprocErrorChar, sizeof(kSubprocErrorChar)); ++ util::writeToFd(errors[1], &kSubprocErrorChar, sizeof(kSubprocErrorChar)); + LOG_F("Launching child process failed"); + } + close(child_fd); ++ close(errors[1]); + if (pid == -1) { + auto saved_errno = errno; + PLOG_W("clone(flags=%s) failed", cloneFlagsToStr(flags).c_str()); + close(parent_fd); ++ close(errors[0]); + errno = saved_errno; + return pid; + } +@@ -562,20 +576,22 @@ + + if (!initParent(nsj, pid, parent_fd)) { + close(parent_fd); ++ close(errors[0]); + return -1; + } ++ close(parent_fd); + + char rcvChar; +- while (util::readFromFd(parent_fd, &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar)) { ++ while (util::readFromFd(errors[0], &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar)) { + if (rcvChar == kSubprocErrorChar) { + LOG_W("Received error message from the child process before it has been " + "executed"); +- close(parent_fd); ++ close(errors[0]); + return -1; + } + } + +- close(parent_fd); ++ close(errors[0]); + return pid; + } + diff --git a/docs/adr/0013-supervisor-anonymous-pipes.md b/docs/adr/0013-supervisor-anonymous-pipes.md new file mode 100644 index 00000000..0a905455 --- /dev/null +++ b/docs/adr/0013-supervisor-anonymous-pipes.md @@ -0,0 +1,65 @@ +# 0013: Remove Unix sockets from runner supervisors + +Status: Proposed + +## Context + +ADRs 0011 and 0012 deny sockets in untrusted jobs. The trusted API, broker, +relay, and NsJail monitor still process job-controlled data. Node/Bun's piped +subprocess stdio and NsJail's startup handshake create Unix socketpairs outside +the job filter. Supervisor hardening must replace those transports before it +can deny the same kernel entry points. + +## Decision + +Install an inherited seccomp filter before starting the API. Deny AF_UNIX +socket creation, every socketpair, sendmsg/sendmmsg (including inherited +sockets), and all io_uring entry points. Validate the native audit architecture +and reject x32 syscall encodings. TCP remains available for API requests, +hosted previews, and the broker's fixed upstream. This is a targeted deny filter, +not a complete supervisor syscall allowlist. Startup fails if installation fails. Force sequential resolver queries with +`RES_OPTIONS=single-request` (preserving existing options), so glibc DNS uses +sendto instead of its blocked sendmmsg batching. + +A small N-API addon creates CLOEXEC anonymous pipes. Linux subprocess launches +pass integer pipe ends to Node/Bun rather than requesting runtime-created pipes. +The adapter preserves output draining, backpressure, child close ordering, and +cleanup on exit, failed spawn, and cancellation. macOS development retains native +spawn; Linux container tests on the Mac verify the real filter and pipe adapter. + +The tool-call broker uses separate request/response pipes to its relay. The relay +retains bounded buffers, parent-death signals, and immediate termination when a +response reader disappears, even without a pending response. Buffered sources +are omitted from poll until drained so a closed writer cannot cause a CPU spin. +Untrusted jobs now +receive anonymous pipes for stdin/stdout/stderr as well as tool-call FDs 3/4. + +Patch the pinned NsJail build to replace its supervisor/child handshake with +separate readiness and error pipes. Reject optional seccomp notification and +user-mode networking modes that require descriptor passing. Build without libnl: +network-isolated jobs do not need interface movement or traffic-rule programming, +and this avoids a gratuitous netlink enumeration during namespace setup. +Unsupported interface/traffic-rule requests fail closed. + +Hosted-app egress uses the packaged iptables-legacy and ip6tables-legacy binaries, +whose transport does not require sendmsg. Both chains must install before any +app is admitted. IPv4/IPv6 legacy netfilter support is consequently required on +the runner kernel; missing support rejects hosted-app admission. + +## Consequences and validation + +All three runner image paths package the matching addon, policy binary, broker, +relay, and patched NsJail. Deploy rebuilt runner images and regenerate snapshots; +these internal processes and binaries are an atomic image upgrade. The external +service protocol and previously patched Python runtime are unchanged. + +The filter also applies to hosted apps and their descendants. Runtimes and +third-party libraries that require Unix sockets/socketpairs remain unsupported. +The trusted API retains its existing privileges and other syscalls; reducing +those privileges requires another containment boundary. Kernel patching remains +necessary. + +CI runs native filter, anonymous stdio, cleanup/backpressure, packaged broker, +and hosted-app egress/preview tests on amd64 and arm64. A local signed runner +canary additionally exercises normal namespaces with Python 3.14.4 asyncio, +spawn/fork multiprocessing, concurrent tool calls, and Bun 1.4.2 JS/TS execution. diff --git a/launcher/Dockerfile b/launcher/Dockerfile index 4acd2094..bad5cd2f 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -31,12 +31,20 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup @@ -68,6 +76,9 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node RUN chmod +x /usr/sbin/nsjail diff --git a/tests/pipe_relay_backpressure.py b/tests/pipe_relay_backpressure.py new file mode 100644 index 00000000..fb18803c --- /dev/null +++ b/tests/pipe_relay_backpressure.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""A closed request writer must not spin the trusted relay under backpressure.""" +import os +import signal +import subprocess +import sys +import tempfile +import time + +request_reader, request_writer = os.pipe() +response_reader, response_writer = os.pipe() +os.set_blocking(request_writer, False) +try: + while True: + os.write(request_writer, b'x' * 4096) +except BlockingIOError: + pass # The upstream deliberately does not drain requests. + +with tempfile.TemporaryDirectory() as directory: + marker = os.path.join(directory, 'ready') + job = os.path.join(directory, 'job.py') + with open(job, 'w') as output: + output.write(f"import os,time; os.write(3,b'x'*4096); os.close(3); open({marker!r},'w').close(); time.sleep(30)") + bootstrap = (f"import os; os.dup2({request_writer},3); os.dup2({response_reader},4); " + f"os.execv('/usr/local/bin/tool-call-pipe-bridge'," + f"['tool-call-pipe-bridge','--job-pipes',{sys.executable!r},{job!r}])") + relay = subprocess.Popen([sys.executable, '-c', bootstrap], + pass_fds=[request_writer, response_reader]) + try: + deadline = time.monotonic() + 5 + while not os.path.exists(marker) and time.monotonic() < deadline: + assert relay.poll() is None + time.sleep(0.01) + assert os.path.exists(marker), 'job did not close its request writer' + def cpu_seconds(): + with open(f'/proc/{relay.pid}/stat') as source: + fields = source.read().rsplit(')', 1)[1].split() + return (int(fields[11]) + int(fields[12])) / os.sysconf('SC_CLK_TCK') + before = cpu_seconds() + time.sleep(0.5) + used = cpu_seconds() - before + assert used < 0.1, f'relay spun on pipe HUP under backpressure: {used:.3f}s CPU' + assert relay.poll() is None + # Closing the broker response writer still terminates the invocation. + os.close(response_writer) + response_writer = -1 + assert relay.wait(timeout=2) == 137 + print('PASS: blocked relay stays idle after request EOF and terminates on broker EOF') + finally: + if relay.poll() is None: + relay.send_signal(signal.SIGKILL) + relay.wait(timeout=2) +for fd in [request_reader, request_writer, response_reader, response_writer]: + if fd >= 0: + os.close(fd) diff --git a/tests/supervisor_dns.py b/tests/supervisor_dns.py new file mode 100644 index 00000000..f2a9a291 --- /dev/null +++ b/tests/supervisor_dns.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Run with Docker --dns 127.0.0.1; no external DNS/network dependency.""" +import json +import os +import socket +import struct +import subprocess +import threading + +server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +server.bind(('127.0.0.1', 53)) +queries = [] +def serve(): + while True: + packet, peer = server.recvfrom(4096) + offset = 12 + labels = [] + while packet[offset]: + length = packet[offset] + labels.append(packet[offset + 1:offset + length + 1]) + offset += length + 1 + offset += 1 + kind, cls = struct.unpack('!HH', packet[offset:offset + 4]) + offset += 4 + queries.append((b'.'.join(labels), kind)) + address = socket.inet_pton(socket.AF_INET6 if kind == 28 else socket.AF_INET, + '::1' if kind == 28 else '127.0.0.1') + reply = packet[:2] + struct.pack('!HHHHH', 0x8180, 1, 1, 0, 0) + packet[12:offset] + reply += b'\xc0\x0c' + struct.pack('!HHIH', kind, cls, 60, len(address)) + address + server.sendto(reply, peer) +threading.Thread(target=serve, daemon=True).start() +program = """ +const assert=require('node:assert/strict'); +assert(process.env.RES_OPTIONS.includes('timeout:1')); +assert(process.env.RES_OPTIONS.includes('single-request')); +require('node:dns').lookup('supervisor-test.invalid.',{all:true},(error,addresses)=>{ + if(error) throw error; + assert(addresses.length>0); + assert(addresses.every(a=>a.address==='127.0.0.1'||a.address==='::1')); + console.log('PASS: hostname resolution under supervisor filter'); +}); +""" +for runtime in ['/usr/local/bin/node', '/usr/local/bin/bun']: + result = subprocess.run(['/usr/local/bin/sandbox-supervisor-policy', runtime, '-e', program], + env={**os.environ, 'RES_OPTIONS': 'timeout:1 attempts:1'}, + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, (runtime, result.stdout, result.stderr) + print(os.path.basename(runtime) + ': ' + result.stdout.strip()) +assert queries and all(name == b'supervisor-test.invalid' for name, _ in queries), queries diff --git a/tests/supervisor_hosted_network.ts b/tests/supervisor_hosted_network.ts new file mode 100644 index 00000000..e581576b --- /dev/null +++ b/tests/supervisor_hosted_network.ts @@ -0,0 +1,42 @@ +// Run as root with NET_ADMIN in the packaged guest, under supervisor-policy. +import { installHostedAppNetworkGuard } from '/work/api/src/hosted-app'; +import { spawnPipeProcess } from '/work/api/src/pipe-process'; +import { once } from 'node:events'; +import * as net from 'node:net'; +import assert from 'node:assert/strict'; + +const uid = 200123; +await installHostedAppNetworkGuard(uid); +const upstream = net.createServer(); +upstream.listen(0, '127.0.0.1'); +await once(upstream, 'listening'); +const port = (upstream.address() as net.AddressInfo).port; +const child = spawnPipeProcess(process.execPath, ['-e', ` +import net from 'node:net'; +import {once} from 'node:events'; +import assert from 'node:assert/strict'; +process.setgid(${uid}); process.setuid(${uid}); +const outbound = net.connect(${port}, '127.0.0.1'); +await assert.rejects(once(outbound, 'connect')); +outbound.destroy(); +const server = net.createServer(client => client.end('preview reply')); +server.listen(0, '127.0.0.1'); await once(server, 'listening'); +console.log(server.address().port); +await once(server, 'connection'); server.close(); +`]); +child.stdin.end(); +child.stderr.pipe(process.stderr); +const closed = once(child, 'close'); +let timer: ReturnType | undefined; +try { + timer = setTimeout(() => child.kill('SIGKILL'), 5000); + const [bytes] = await once(child.stdout, 'data'); + const preview = net.connect(Number(bytes.toString().trim()), '127.0.0.1'); + let reply = ''; + for await (const chunk of preview) reply += chunk.toString(); + assert.equal(reply, 'preview reply'); + assert.equal((await closed)[0], 0); + console.log('PASS: hosted-app egress is blocked and inbound preview replies work under supervisor policy'); +} finally { + clearTimeout(timer); child.kill('SIGKILL'); upstream.close(); +} diff --git a/tests/supervisor_policy.py b/tests/supervisor_policy.py new file mode 100644 index 00000000..d0fa0472 --- /dev/null +++ b/tests/supervisor_policy.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Exercise the installed supervisor filter, including inherited Unix sockets.""" +import ctypes +import errno +import os +import platform +import socket +import subprocess +import sys + +POLICY = '/usr/local/bin/sandbox-supervisor-policy' +if len(sys.argv) > 1 and sys.argv[1] == '--filtered': + def denied(action): + try: + action() + except OSError as error: + assert error.errno == errno.EPERM, error + else: + raise AssertionError('supervisor syscall allowed') + denied(lambda: socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) + denied(socket.socketpair) + inherited = socket.socket(fileno=int(sys.argv[2])) + denied(lambda: inherited.sendmsg([b'x'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, b'\0' * 4)])) + libc = ctypes.CDLL(None, use_errno=True) + calls = {'x86_64': [307, 425, 426, 427], 'aarch64': [269, 425, 426, 427]}[platform.machine()] + for number in calls: # sendmmsg and all io_uring entry points + assert libc.syscall(number, -1, 0, 0, 0, 0, 0) == -1 + assert ctypes.get_errno() == errno.EPERM + # TCP remains usable for the API listener and broker upstream. + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(('127.0.0.1', 0)); listener.listen() + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.connect(listener.getsockname()) + peer, _ = listener.accept() + client.sendall(b'tcp'); assert peer.recv(3) == b'tcp' + print('PASS: supervisor denies Unix sockets, inherited descriptor sends and io_uring; TCP works') +else: + left, right = socket.socketpair() + result = subprocess.run([POLICY, sys.executable, __file__, '--filtered', str(left.fileno())], + pass_fds=[left.fileno()], timeout=5) + assert result.returncode == 0, result.returncode + # The pipe-based NsJail handshake must handle child setup/exec failure, + # as well as the successful normal path tested by the runner canary. + command = [POLICY, '/nsjail/nsjail', '--mode', 'o', '--disable_clone_newnet', + '--disable_clone_newuser', '--disable_clone_newns', '--disable_clone_newpid', + '--disable_clone_newipc', '--disable_clone_newuts', '--disable_clone_newcgroup', + '--disable_proc', '--disable_rlimits', '--', '/missing-codeapi-executable'] + result = subprocess.run(command, capture_output=True, timeout=5) + assert result.returncode != 0 + assert b'Launching child process failed' in result.stderr, result.stderr + print('PASS: anonymous NsJail startup handshake reports child exec failure') diff --git a/tests/tool_call_runner.cjs b/tests/tool_call_runner.cjs index 4aab25e4..61ddcb9e 100644 --- a/tests/tool_call_runner.cjs +++ b/tests/tool_call_runner.cjs @@ -66,7 +66,7 @@ def _mp_echo(sender): if __name__ == '__main__': assert sys.version.split()[0] == ${JSON.stringify(version)} assert os.getuid() == 65534 - assert _stat.S_ISFIFO(os.fstat(3).st_mode) and _stat.S_ISFIFO(os.fstat(4).st_mode) + assert all(_stat.S_ISFIFO(os.fstat(fd).st_mode) for fd in range(5)) for _namespace, _outside in ${JSON.stringify(outerNamespaces)}.items(): assert os.readlink('/proc/self/ns/' + _namespace) != _outside, _namespace for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6): From c854ab5f37c656e88e6e4ad8b834cc8d17659151 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 6 Oct 2026 08:00:03 -0400 Subject: [PATCH 75/75] test: check parent-side nsjail startup failure (#318) --- tests/supervisor_policy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/supervisor_policy.py b/tests/supervisor_policy.py index d0fa0472..caab12c0 100644 --- a/tests/supervisor_policy.py +++ b/tests/supervisor_policy.py @@ -47,5 +47,7 @@ def denied(action): '--disable_proc', '--disable_rlimits', '--', '/missing-codeapi-executable'] result = subprocess.run(command, capture_output=True, timeout=5) assert result.returncode != 0 - assert b'Launching child process failed' in result.stderr, result.stderr + # The parent can kill the failed child before its final log is flushed. + # Its error-pipe diagnostic proves the startup handshake reported failure. + assert b'Received error message from the child process before it has been executed' in result.stderr, result.stderr print('PASS: anonymous NsJail startup handshake reports child exec failure')