diff --git a/apps/server/src/inference/InferenceProxy.test.ts b/apps/server/src/inference/InferenceProxy.test.ts index fb6aa6a2e408c..67e958139308f 100644 --- a/apps/server/src/inference/InferenceProxy.test.ts +++ b/apps/server/src/inference/InferenceProxy.test.ts @@ -9,6 +9,7 @@ import { ProjectId, ProviderInstanceId, ProviderDriverKind, + ServerSettingsError, isInferenceProviderConfig, resolveProviderInstanceEnabled, } from "@t3tools/contracts"; @@ -75,6 +76,7 @@ it.effect( keychainState.onRead = undefined; const config = yield* ServerConfig.ServerConfig; const state = yield* ServerSecretStore.ServerSecretStore; + const settings = yield* ServerSettings.ServerSettingsService; const savedPort = state.get("inference-proxy-state").pipe( Effect.map((value) => { const saved = JSON.parse(new TextDecoder().decode(Option.getOrThrow(value))) as { @@ -131,6 +133,13 @@ it.effect( const releaseClaudeUsage = yield* Deferred.make(); let lan = false; let rejectNextLaunch = false; + let rejectRouteStateWrite: "before" | "after" | undefined; + let rejectNextProviderWrite = false; + let providerWriteGate: + | { readonly entered: Deferred.Deferred; readonly release: Deferred.Deferred } + | undefined; + let providerCommitGate: typeof providerWriteGate; + let routeStateWriteGate: typeof providerWriteGate; let currentExit: Deferred.Deferred | undefined; let expectRecovery = false; const recovered = yield* Deferred.make(); @@ -419,6 +428,74 @@ it.effect( ); }); const service = InferenceProxy.layer.pipe( + Layer.provide( + Layer.succeed(ServerSettings.ServerSettingsService, { + ...settings, + reviseProviderInstance: (instanceId, revise) => + settings + .reviseProviderInstance(instanceId, (current, currentSettings) => + revise(current, currentSettings).pipe( + Effect.tap(() => + providerWriteGate === undefined + ? Effect.void + : Deferred.succeed(providerWriteGate.entered, undefined).pipe( + Effect.andThen(Deferred.await(providerWriteGate.release)), + ), + ), + Effect.tap(() => { + if (!rejectNextProviderWrite) return Effect.void; + rejectNextProviderWrite = false; + return Effect.fail( + new ServerSettingsError({ + settingsPath: config.settingsPath, + operation: "write-file", + cause: new Error("Provider settings could not be written"), + }), + ); + }), + ), + ) + .pipe( + Effect.tap(() => + providerCommitGate === undefined + ? Effect.void + : Deferred.succeed(providerCommitGate.entered, undefined).pipe( + Effect.andThen(Deferred.await(providerCommitGate.release)), + ), + ), + ), + }), + ), + Layer.provide( + Layer.succeed(ServerSecretStore.ServerSecretStore, { + ...state, + set: (name, value) => + Effect.suspend(() => { + if (name === "inference-proxy-state" && rejectRouteStateWrite) { + const failure = Effect.fail( + new ServerSecretStore.SecretStorePersistError({ + resource: name, + cause: new Error("Route state could not be written"), + }), + ); + if (rejectRouteStateWrite === "before") return failure; + rejectRouteStateWrite = undefined; + return state.set(name, value).pipe(Effect.andThen(failure)); + } + if (name === "inference-proxy-state" && routeStateWriteGate) { + const gate = routeStateWriteGate; + routeStateWriteGate = undefined; + return state + .set(name, value) + .pipe( + Effect.andThen(Deferred.succeed(gate.entered, undefined)), + Effect.andThen(Deferred.await(gate.release)), + ); + } + return state.set(name, value); + }), + }), + ), Layer.provide( Layer.mock(PeerLinks.PeerLinks)({ get: () => Effect.succeed(linked ? Option.some(link) : Option.none()), @@ -487,7 +564,6 @@ it.effect( Layer.provide(NetService.layer), Layer.fresh, ); - const settings = yield* ServerSettings.ServerSettingsService; const legacyRoutes = [ { driver: "codex" as const, @@ -1213,6 +1289,25 @@ it.effect( instanceId: ProviderInstanceId.make("codex"), instance: native, }); + const beforeFailedConnect = (yield* settings.getSettings).providerInstances[ + ProviderInstanceId.make("codex") + ]; + const beforeFailedConnectState = yield* state.get("inference-proxy-state"); + for (const failure of ["before", "after"] as const) { + rejectRouteStateWrite = failure; + yield* proxy.connect(connect).pipe(Effect.flip); + rejectRouteStateWrite = undefined; + expect( + (yield* settings.getSettings).providerInstances[ProviderInstanceId.make("codex")], + ).toEqual(beforeFailedConnect); + expect(yield* state.get("inference-proxy-state")).toEqual(beforeFailedConnectState); + } + rejectNextProviderWrite = true; + yield* proxy.connect(connect).pipe(Effect.flip); + expect( + (yield* settings.getSettings).providerInstances[ProviderInstanceId.make("codex")], + ).toEqual(beforeFailedConnect); + expect(yield* state.get("inference-proxy-state")).toEqual(beforeFailedConnectState); yield* proxy.connect(connect); let configured = (yield* settings.getSettings).providerInstances[ ProviderInstanceId.make("codex") @@ -1247,8 +1342,88 @@ it.effect( expect(yield* proxy.routeHealth(configured.config, local.key, "codex")).toMatchObject({ availableDrivers: expect.arrayContaining(["codex"]), }); + const priorRoute = yield* proxy.routeCandidates( + local.key, + hostId, + "codex", + "codex-auto-review", + ); + expect(priorRoute).toHaveLength(1); + providerWriteGate = { + entered: yield* Deferred.make(), + release: yield* Deferred.make(), + }; + const writeGate = providerWriteGate; + rejectNextProviderWrite = true; + const otherRoute = { + ...connect, + grant: { + ...local, + hostId: EnvironmentId.make("other-host"), + baseUrl: "https://fallback.example.ts.net/api/inference", + key: "different-primary-key", + }, + }; + const changingRoute = yield* proxy + .connect(otherRoute) + .pipe(Effect.exit, Effect.forkChild); + const priorRouteDuringChange = yield* Effect.gen(function* () { + yield* Deferred.await(writeGate.entered); + return yield* proxy + .hasRoutePool(local.key, hostId, "codex") + .pipe(Effect.forkChild({ startImmediately: true })); + }).pipe(Effect.ensuring(Deferred.succeed(writeGate.release, undefined))); + expect(yield* Fiber.join(changingRoute)).toMatchObject({ _tag: "Failure" }); + providerWriteGate = undefined; + remoteCatalogKeys.length = 0; + expect(yield* Fiber.join(priorRouteDuringChange)).toBe(true); + providerCommitGate = { + entered: yield* Deferred.make(), + release: yield* Deferred.make(), + }; + const commitGate = providerCommitGate; + const committedRoute = yield* proxy.connect(otherRoute).pipe(Effect.forkChild); + const [healthAfterSettingsCommit, routeAfterSettingsCommit] = yield* Effect.gen( + function* () { + yield* Deferred.await(commitGate.entered); + const committedConfig = (yield* settings.getSettings).providerInstances[ + ProviderInstanceId.make("codex") + ]?.config; + if (!isInferenceProviderConfig(committedConfig)) + throw new Error("Expected the newly committed inference route"); + const health = yield* proxy + .routeHealth(committedConfig, otherRoute.grant.key, "codex") + .pipe(Effect.forkChild({ startImmediately: true })); + const route = yield* proxy + .hasRoutePool(otherRoute.grant.key, otherRoute.grant.hostId, "codex") + .pipe(Effect.forkChild({ startImmediately: true })); + return [health, route] as const; + }, + ).pipe(Effect.ensuring(Deferred.succeed(commitGate.release, undefined))); + yield* Fiber.join(committedRoute); + expect(yield* Fiber.join(routeAfterSettingsCommit)).toBe(true); + expect(yield* Fiber.join(healthAfterSettingsCommit)).toMatchObject({ + availableDrivers: ["codex"], + }); + providerCommitGate = undefined; + yield* proxy.connect(connect); + remoteCatalogKeys.length = 0; const initialCatalog = catalog; catalog = [...catalog, { id: "gpt-local-new", owned_by: "openai" }]; + const beforeFailedRefresh = (yield* settings.getSettings).providerInstances[ + ProviderInstanceId.make("codex") + ]; + const beforeFailedRefreshState = yield* state.get("inference-proxy-state"); + for (const failure of ["before", "after", "provider"] as const) { + rejectRouteStateWrite = failure === "provider" ? undefined : failure; + rejectNextProviderWrite = failure === "provider"; + yield* proxy.refreshModelCatalog(true, ProviderInstanceId.make("codex")); + rejectRouteStateWrite = undefined; + expect( + (yield* settings.getSettings).providerInstances[ProviderInstanceId.make("codex")], + ).toEqual(beforeFailedRefresh); + expect(yield* state.get("inference-proxy-state")).toEqual(beforeFailedRefreshState); + } yield* proxy.refreshModelCatalog(); expect( (yield* settings.getSettings).providerInstances[ProviderInstanceId.make("codex")] @@ -1538,7 +1713,7 @@ it.effect( { id: "codex-auto-review", owned_by: "openai" }, { id: "fallback-only", owned_by: "openai" }, ]; - yield* proxy.registerFallback({ + const fallbackRoute = { ...connect, grant: { ...issued, @@ -1546,7 +1721,49 @@ it.effect( baseUrl: "https://fallback.example.ts.net/api/inference", alternates: [], }, - }); + }; + const beforeFailedFallback = (yield* settings.getSettings).providerInstances[ + ProviderInstanceId.make("codex") + ]; + const beforeFailedFallbackState = yield* state.get("inference-proxy-state"); + for (const failure of ["before", "after", "provider"] as const) { + rejectRouteStateWrite = failure === "provider" ? undefined : failure; + rejectNextProviderWrite = failure === "provider"; + yield* proxy.registerFallback(fallbackRoute).pipe(Effect.flip); + rejectRouteStateWrite = undefined; + expect( + (yield* settings.getSettings).providerInstances[ProviderInstanceId.make("codex")], + ).toEqual(beforeFailedFallback); + expect(yield* state.get("inference-proxy-state")).toEqual(beforeFailedFallbackState); + } + const fallbackWriteGate = { + entered: yield* Deferred.make(), + release: yield* Deferred.make(), + }; + routeStateWriteGate = fallbackWriteGate; + const interruptedFallback = yield* proxy + .registerFallback(fallbackRoute) + .pipe(Effect.forkChild); + yield* Deferred.await(fallbackWriteGate.entered).pipe( + Effect.andThen( + Fiber.interrupt(interruptedFallback).pipe( + Effect.forkChild({ startImmediately: true }), + ), + ), + Effect.ensuring(Deferred.succeed(fallbackWriteGate.release, undefined)), + ); + yield* Fiber.await(interruptedFallback); + const afterInterruptedFallback = (yield* settings.getSettings).providerInstances[ + ProviderInstanceId.make("codex") + ]?.config; + if (!isInferenceProviderConfig(afterInterruptedFallback)) + throw new Error("Expected the previous inference route to remain configured"); + expect( + new TextDecoder() + .decode(Option.getOrThrow(yield* state.get("inference-proxy-state"))) + .includes('"fallback-only"'), + ).toBe(afterInterruptedFallback.inferenceModels.includes("fallback-only")); + yield* proxy.registerFallback(fallbackRoute); catalog = primaryCatalog; fallbackCatalog = [ { id: "codex-auto-review", owned_by: "openai" }, diff --git a/apps/server/src/inference/InferenceProxy.ts b/apps/server/src/inference/InferenceProxy.ts index ba4004e94eac1..39a04203a050d 100644 --- a/apps/server/src/inference/InferenceProxy.ts +++ b/apps/server/src/inference/InferenceProxy.ts @@ -441,6 +441,7 @@ const make = Effect.gen(function* () { (exit) => (Exit.isSuccess(exit) ? "15 minutes" : 0), ); const mutex = yield* Semaphore.make(1); + const routeMutex = yield* Semaphore.make(1); const updateMutex = yield* Semaphore.make(1); const catalogMutex = yield* Semaphore.make(1); const children = yield* Queue.sliding(1); @@ -531,11 +532,48 @@ const make = Effect.gen(function* () { cached = Option.isSome(raw) ? yield* decodeSaved(new TextDecoder().decode(raw.value)) : empty; return cached; }); - const save = Effect.fn("InferenceProxy.save")(function* (next: Saved) { + const persist = Effect.fn("InferenceProxy.persist")(function* (next: Saved) { const state = running ? { ...next, port: running.port } : next; yield* store.set("inference-proxy-state", new TextEncoder().encode(yield* encodeSaved(state))); - cached = state; + return state; + }); + const save = Effect.fn("InferenceProxy.save")(function* (next: Saved) { + cached = yield* persist(next); }, Effect.uninterruptible); + const reviseRouteState = Effect.fn("InferenceProxy.reviseRouteState")( + function* (before: Saved, revise: (write: typeof save) => Effect.Effect) { + let attempted = false; + let next: Saved | undefined; + const result = yield* revise( + Effect.fnUntraced(function* (state: Saved) { + attempted = true; + next = yield* persist(state); + }), + ).pipe( + Effect.onError(() => + attempted + ? save(before).pipe( + Effect.catch(() => Effect.logWarning("Could not roll back inference route state.")), + Effect.ensuring( + Effect.sync(() => { + cached = before; + }), + ), + ) + : Effect.void, + ), + ); + if (next) cached = next; + return result; + }, + Effect.uninterruptible, + routeMutex.withPermit, + ); + const routeSnapshot = routeMutex.withPermit( + Effect.gen(function* () { + return { current: yield* settings.getSettings, state: yield* read }; + }), + ); const stop: Effect.Effect = Effect.gen(function* () { if (running) yield* Scope.close(running.scope, Exit.void); running = undefined; @@ -1679,7 +1717,7 @@ const make = Effect.gen(function* () { function* (key: string, hostId: EnvironmentId | undefined, driver: string | undefined) { if (!hostId || !driver) return false; const instanceId = defaultInstanceIdForDriver(ProviderDriverKind.make(driver)); - const current = yield* settings.getSettings; + const { current, state } = yield* routeSnapshot; const instance = current.providerInstances[instanceId]; if ( !instance || @@ -1691,7 +1729,7 @@ const make = Effect.gen(function* () { ) return false; return ( - (yield* read).routePools?.some( + state.routePools?.some( (entry) => entry.driver === driver && entry.primary.hostId === hostId && entry.primary.key === key, ) ?? false @@ -1716,7 +1754,7 @@ const make = Effect.gen(function* () { const target = yield* authorize(key); return target ? [{ ...target, models: [], unavailableModels: [] }] : null; } - const current = yield* settings.getSettings; + const { current, state } = yield* routeSnapshot; const active = Arr.findFirst(INFERENCE_DRIVERS, (candidateDriver) => { if (driver !== undefined && candidateDriver !== driver) return Option.none(); const instanceId = defaultInstanceIdForDriver(ProviderDriverKind.make(candidateDriver)); @@ -1731,7 +1769,7 @@ const make = Effect.gen(function* () { }); if (Option.isNone(active)) return null; const { driver: activeDriver, route } = active.value; - const pool = (yield* read).routePools?.find((entry) => entry.driver === activeDriver); + const pool = state.routePools?.find((entry) => entry.driver === activeDriver); if (!pool || pool.primary.hostId !== hostId || pool.primary.key !== key) { const target = yield* authorize(key, hostId, driver); return target @@ -1799,7 +1837,7 @@ const make = Effect.gen(function* () { const authorize = Effect.fn("InferenceProxy.authorize")( function* (key: string, hostId?: EnvironmentId, driver?: string) { if (hostId !== undefined) { - const current = yield* settings.getSettings; + const { current, state } = yield* routeSnapshot; const allowed = INFERENCE_DRIVERS.map((candidateDriver) => { const instanceId = defaultInstanceIdForDriver(ProviderDriverKind.make(candidateDriver)); const instance = current.providerInstances[instanceId]; @@ -1814,7 +1852,7 @@ const make = Effect.gen(function* () { : undefined; }).find((config) => config !== undefined); if (!allowed) return null; - const pool = (yield* read).routePools?.find( + const pool = state.routePools?.find( (entry) => (driver === undefined || entry.driver === driver) && entry.primary.hostId === hostId && @@ -1941,7 +1979,7 @@ const make = Effect.gen(function* () { const health = Effect.fn("InferenceProxy.health")( function* (key: string, hostId?: EnvironmentId, driver?: string) { const pool = hostId - ? (yield* read).routePools?.find( + ? (yield* routeMutex.withPermit(read)).routePools?.find( (entry) => (driver === undefined || entry.driver === driver) && entry.primary.hostId === hostId && @@ -1960,7 +1998,7 @@ const make = Effect.gen(function* () { const routeHealth = Effect.fn("InferenceProxy.routeHealth")( function* (route: InferenceProviderConfig, key: string, driver?: string) { - const pool = (yield* read).routePools?.find( + const pool = (yield* routeMutex.withPermit(read)).routePools?.find( (entry) => (driver === undefined || entry.driver === driver) && entry.primary.hostId === route.inferenceHostId && @@ -2133,15 +2171,8 @@ const make = Effect.gen(function* () { ...(existing?.alternates ?? []).filter((entry) => entry.hostId !== grant.hostId), candidate, ]; - yield* save({ - ...state, - routePools: [ - ...(state.routePools ?? []).filter((entry) => entry.driver !== driver), - { driver, primary, alternates }, - ], - }); - return yield* settings - .reviseProviderInstance( + return yield* reviseRouteState(state, (write) => + settings.reviseProviderInstance( instanceId, Effect.fnUntraced(function* (current, currentSettings) { if ( @@ -2153,6 +2184,13 @@ const make = Effect.gen(function* () { !(yield* matchesRouteKey(instanceId, current.config.inferenceKey, primaryKey)) ) return yield* failure("The inference route changed."); + yield* write({ + ...state, + routePools: [ + ...(state.routePools ?? []).filter((entry) => entry.driver !== driver), + { driver, primary, alternates }, + ], + }); return { instance: { ...current, @@ -2167,8 +2205,8 @@ const make = Effect.gen(function* () { result: undefined, }; }), - ) - .pipe(Effect.onError(() => save(state).pipe(Effect.orDie))); + ), + ); }), ); if (!revised) return; @@ -2183,64 +2221,58 @@ const make = Effect.gen(function* () { Effect.gen(function* () { const state = yield* read; const existing = state.routePools?.find((entry) => entry.driver === driver); - const updated = yield* settings.reviseProviderInstance( - instanceId, - Effect.fnUntraced(function* (current) { - const problem = - driver === "opencode" - ? openCodeRouteProblem( - current?.environment, - providerEnvironment.OPENCODE_CONFIG_CONTENT, - ) - : null; - if (problem) return yield* failure(problem); - const native = current ?? { driver: ProviderDriverKind.make(driver), config: {} }; - const { inferenceConnection: _previousConnection, ...nativeConfig } = providerConfig( - native.config, - ) - ? native.config - : {}; - const preserve = - existing?.primary.hostId === grant.hostId && - existing.primary.key === grant.key && - isInferenceProviderConfig(current?.config) && - current.config.inferenceHostId === grant.hostId && - (yield* matchesRouteKey(instanceId, current.config.inferenceKey, grant.key)); - const alternates = preserve ? existing.alternates : []; - return { - instance: { - ...native, - ...(preserveProviderEnabled && current ? {} : { enabled: true }), - config: { - ...nativeConfig, - inferenceEnabled: true, - inferenceHostId: grant.hostId, - inferenceHostLabel: local ? "" : hostLabel, - inferenceBaseUrl: loopbackUrl(grant.hostId), - inferenceKey: grant.key, - inferenceModels: [ - ...new Set([...models, ...alternates.flatMap((entry) => entry.models)]), - ], - inferenceHomePath: home, - ...(linked && connection ? { inferenceConnection: connection } : {}), + return yield* reviseRouteState(state, (write) => + settings.reviseProviderInstance( + instanceId, + Effect.fnUntraced(function* (current) { + const problem = + driver === "opencode" + ? openCodeRouteProblem( + current?.environment, + providerEnvironment.OPENCODE_CONFIG_CONTENT, + ) + : null; + if (problem) return yield* failure(problem); + const native = current ?? { driver: ProviderDriverKind.make(driver), config: {} }; + const { inferenceConnection: _previousConnection, ...nativeConfig } = + providerConfig(native.config) ? native.config : {}; + const preserve = + existing?.primary.hostId === grant.hostId && + existing.primary.key === grant.key && + isInferenceProviderConfig(current?.config) && + current.config.inferenceHostId === grant.hostId && + (yield* matchesRouteKey(instanceId, current.config.inferenceKey, grant.key)); + const alternates = preserve ? existing.alternates : []; + yield* write({ + ...state, + routePools: [ + ...(state.routePools ?? []).filter((entry) => entry.driver !== driver), + { driver, primary: candidate, alternates }, + ], + }); + return { + instance: { + ...native, + ...(preserveProviderEnabled && current ? {} : { enabled: true }), + config: { + ...nativeConfig, + inferenceEnabled: true, + inferenceHostId: grant.hostId, + inferenceHostLabel: local ? "" : hostLabel, + inferenceBaseUrl: loopbackUrl(grant.hostId), + inferenceKey: grant.key, + inferenceModels: [ + ...new Set([...models, ...alternates.flatMap((entry) => entry.models)]), + ], + inferenceHomePath: home, + ...(linked && connection ? { inferenceConnection: connection } : {}), + }, }, - }, - result: alternates, - }; - }), + result: undefined, + }; + }), + ), ); - yield* save({ - ...state, - routePools: [ - ...(state.routePools ?? []).filter((entry) => entry.driver !== driver), - { - driver, - primary: candidate, - alternates: updated.result, - }, - ], - }); - return updated; }), ); yield* registry.reconcile( @@ -2346,15 +2378,8 @@ const make = Effect.gen(function* () { ) ) return false; - if (poolChanged) - yield* save({ - ...before, - routePools: before.routePools.map((entry) => - entry.driver === pool.driver ? { ...pool, primary, alternates } : entry, - ), - }); - const revised = yield* settings - .reviseProviderInstance( + const revised = yield* reviseRouteState(before, (write) => + settings.reviseProviderInstance( instanceId, Effect.fnUntraced(function* (current, currentSettings) { if ( @@ -2371,16 +2396,20 @@ const make = Effect.gen(function* () { )) ) return yield* failure("The inference route changed during catalog refresh."); + if (poolChanged) + yield* write({ + ...before, + routePools: before.routePools!.map((entry) => + entry.driver === pool.driver ? { ...pool, primary, alternates } : entry, + ), + }); return { instance: { ...current, config: { ...current.config, inferenceModels } }, result: undefined, }; }), - ) - .pipe( - Effect.onError(() => (poolChanged ? save(before).pipe(Effect.orDie) : Effect.void)), - Effect.result, - ); + ), + ).pipe(Effect.result); return Result.isSuccess(revised); }).pipe(Effect.uninterruptible), ); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 79127e088f53d..81b2006083722 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -356,60 +356,93 @@ effectIt.effect("restores a missing worktree before a send only under the worksp }).pipe(Effect.provide(layerRestoreLeaseTest)), ); -effectIt.effect("restores into a leftover worktree folder that cannot itself be removed", () => { - const worktreePath = `${managedWorktreesDir}/project/feature-restore`; - // Windows refuses to remove a folder that is still some process's cwd. - const remove = vi.fn((target: string) => - target === worktreePath - ? Effect.fail( - PlatformError.systemError({ - _tag: "Busy", - module: "FileSystem", - method: "remove", - pathOrDescriptor: target, - }), - ) - : Effect.void, - ); - const createWorktree = vi.fn(() => Effect.succeed({} as never)); - return Effect.gen(function* () { - const { restore } = yield* ThreadWorktreeRestore; - yield* restore({ - id: ThreadId.make("thread_leftover_restore"), - projectId: ProjectId.make("project_leftover_restore"), - worktreePath, - branch: "feature/restore", - }); - expect(remove).toHaveBeenCalledWith(`${worktreePath}/node_modules`, { recursive: true }); - expect(createWorktree).toHaveBeenCalledTimes(1); - }).pipe( - Effect.provide( - layerThreadWorktreeRestore.pipe( - Layer.provide( - Layer.mergeAll( - Layer.succeed(FileSystem.FileSystem, { - exists: (target: string) => Effect.succeed(!target.endsWith("/.git")), - readDirectory: () => Effect.succeed(["node_modules"]), - stat: () => Effect.succeed({ type: "Directory" }), - realPath: (target: string) => Effect.succeed(normalize(target)), - remove, - } as never), - layerWorktreeLocation(), - Layer.mock(GitWorkflow.GitWorkflowService)({ - pruneWorktrees: () => Effect.void, - createWorktree, - listLocalBranchNames: () => Effect.succeed(["feature/restore"]), - }), - Layer.mock(ProjectService.ProjectService)({ - getById: () => - Effect.succeedSome({ workspaceRoot: "/tmp/provider-turn-start-project" } as never), +effectIt.effect.each(["unchanged", "user file", "git directory", "missing folder"] as const)( + "restores leftover worktree folders only while still safe, change=%s", + (change) => { + let afterProjectLookup = false; + let entries = ["node_modules"]; + const worktreePath = `${managedWorktreesDir}/project/feature-restore`; + // Windows refuses to remove a folder that is still some process's cwd. + const remove = vi.fn((target: string) => + target === worktreePath + ? Effect.fail( + PlatformError.systemError({ + _tag: "Busy", + module: "FileSystem", + method: "remove", + pathOrDescriptor: target, }), + ) + : Effect.void, + ); + const createWorktree = vi.fn(() => Effect.succeed({} as never)); + return Effect.gen(function* () { + const { restore } = yield* ThreadWorktreeRestore; + yield* restore({ + id: ThreadId.make("thread_leftover_restore"), + projectId: ProjectId.make("project_leftover_restore"), + worktreePath, + branch: "feature/restore", + }); + if (change !== "unchanged") { + expect(remove).not.toHaveBeenCalled(); + } else { + expect(remove).toHaveBeenCalledWith(`${worktreePath}/node_modules`, { recursive: true }); + } + expect(createWorktree).toHaveBeenCalledTimes(change === "unchanged" ? 1 : 0); + }).pipe( + Effect.provide( + layerThreadWorktreeRestore.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(FileSystem.FileSystem, { + exists: (target: string) => + Effect.sync(() => + target.endsWith("/.git") + ? afterProjectLookup && change === "git directory" + : !(afterProjectLookup && change === "missing folder"), + ), + readDirectory: () => + Effect.suspend(() => + afterProjectLookup && change === "missing folder" + ? Effect.fail( + PlatformError.systemError({ + _tag: "NotFound", + module: "FileSystem", + method: "readDirectory", + pathOrDescriptor: worktreePath, + }), + ) + : Effect.succeed(entries), + ), + stat: () => Effect.succeed({ type: "Directory" }), + realPath: (target: string) => Effect.succeed(normalize(target)), + remove, + } as never), + layerWorktreeLocation(), + Layer.mock(GitWorkflow.GitWorkflowService)({ + pruneWorktrees: () => Effect.void, + createWorktree, + listLocalBranchNames: () => Effect.succeed(["feature/restore"]), + }), + Layer.mock(ProjectService.ProjectService)({ + getById: () => + Effect.sync(() => { + afterProjectLookup = true; + if (change === "user file") entries = ["node_modules", "notes.md"]; + if (change === "git directory") entries = ["node_modules", ".git"]; + return Option.some({ + workspaceRoot: "/tmp/provider-turn-start-project", + } as never); + }), + }), + ), ), ), ), - ), - ); -}); + ); + }, +); it.each([ { label: "directly in the scratch root", folder: "/tmp/t3-home/scratch/notes", recreated: true }, diff --git a/apps/server/src/orchestration-v2/WorktreeBranchSync.test.ts b/apps/server/src/orchestration-v2/WorktreeBranchSync.test.ts index 14c5b5d2751d4..5e861d8619a67 100644 --- a/apps/server/src/orchestration-v2/WorktreeBranchSync.test.ts +++ b/apps/server/src/orchestration-v2/WorktreeBranchSync.test.ts @@ -1,41 +1,76 @@ import { assert, it } from "@effect/vitest"; import { + ProjectId, + RunId, ThreadId, + type OrchestrationProjectShell, type OrchestrationV2ThreadShell, type OrchestrationV2ServerCommand, } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Stream from "effect/Stream"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import { ChildProcessSpawner } from "effect/process"; import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProjectStore from "./ProjectStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; import * as WorktreeBranchSync from "./WorktreeBranchSync.ts"; +import * as WorktreeBranches from "./WorktreeBranches.ts"; -it.effect.each([false, true])( - "reconciles shared worktree branches without changing root checkouts or detached HEADs, conflict=%s", - (conflict) => { +it.effect.each(["safe", "conflict", "default", "held", "replaced", "busy", "unborn"] as const)( + "reconciles shared worktree branches safely for %s", + (scenario) => { + const writable = scenario === "safe" || scenario === "conflict"; + const branch = scenario === "default" ? "main" : "union"; const reconciled = Deferred.makeUnsafe(); const commands: OrchestrationV2ServerCommand[] = []; const thread = (id: string, branch: string, worktreePath: string | null) => - ({ id: ThreadId.make(id), branch, worktreePath }) as OrchestrationV2ThreadShell; + ({ + id: ThreadId.make(id), + projectId: ProjectId.make("project"), + branch, + worktreePath, + activeRunId: scenario === "busy" && id === "shared" ? RunId.make("running") : null, + archivedAt: id === "archived" ? DateTime.makeUnsafe(3) : null, + createdAt: DateTime.makeUnsafe(id === "generated" ? 0 : id === "shared" ? 1 : 2), + }) as OrchestrationV2ThreadShell; const snapshot = { schemaVersion: 2, snapshotSequence: 0, threads: [ thread("generated", "t3/unify-converter-branches", "/worktree"), thread("shared", "t3/bfc349ae", "/link/worktree"), - thread("current", "union", "/worktree"), + thread("current", branch, "/worktree"), thread("root", "main", null), thread("other", "feature", "/other"), ], archivedThreads: [thread("archived", "t3/bfc349ae", "/worktree")], }; + const dispatch = (command: OrchestrationV2ServerCommand) => + Effect.gen(function* () { + commands.push(command); + if ( + command.type === "thread.metadata.update" && + command.threadId === ThreadId.make("generated") && + scenario === "conflict" + ) { + return yield* new Orchestrator.OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: "Thread moved before branch metadata could be applied.", + }); + } + return { sequence: commands.length, storedEvents: [] }; + }); return Effect.gen(function* () { yield* Layer.build(WorktreeBranchSync.layer); yield* Deferred.await(reconciled); @@ -50,59 +85,93 @@ it.effect.each([false, true])( command.expectedWorktreePath, ]; }), - [ - ["generated", "union", "t3/unify-converter-branches", "/worktree"], - ["shared", "union", "t3/bfc349ae", "/link/worktree"], - ["archived", "union", "t3/bfc349ae", "/worktree"], - ], + writable + ? [ + ["generated", "union", "t3/unify-converter-branches", "/worktree"], + ["shared", "union", "t3/bfc349ae", "/link/worktree"], + ["archived", "union", "t3/bfc349ae", "/worktree"], + ] + : [], ); }).pipe( Effect.provide( - Layer.mergeAll( - FileSystem.layerNoop({ - realPath: (path) => Effect.succeed(path === "/link/worktree" ? "/worktree" : path), - }), - Layer.mock(ProjectionStore.ProjectionStoreV2)({ - getThreadWorktreePaths: () => - Effect.succeed( - [...snapshot.threads, ...snapshot.archivedThreads].flatMap((thread) => - thread.worktreePath === null - ? [] - : [{ threadId: thread.id, worktreePath: thread.worktreePath }], - ), - ), - }), - Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ - streamBranchChanges: Stream.fromIterable([null, "t3/bfc349ae", "union"]).pipe( - Stream.map((refName) => ({ - cwd: "/worktree", - refName, - })), - Stream.ensuring(Deferred.succeed(reconciled, undefined)), - ), - }), - Layer.mock(GitWorkflowService.GitWorkflowService)({ - currentBranch: () => Effect.succeed("union"), - }), - Layer.mock(ThreadManagement.ThreadManagementService)({ - getShellSnapshot: () => Effect.succeed(snapshot), - dispatch: (command) => - Effect.gen(function* () { - commands.push(command); - if ( - command.type === "thread.metadata.update" && - command.threadId === ThreadId.make("generated") && - conflict - ) { - return yield* new Orchestrator.OrchestratorDispatchError({ - commandId: command.commandId, - commandType: command.type, - cause: "Thread moved before branch metadata could be applied.", + WorktreeBranches.layer.pipe( + Layer.provideMerge( + Layer.mergeAll( + Path.layer, + FileSystem.layerNoop({ + realPath: (path) => Effect.succeed(path === "/link/worktree" ? "/worktree" : path), + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ + getShell: () => + Effect.succeedSome({ workspaceRoot: "/repo" } as OrchestrationProjectShell), + }), + Layer.mock(GitVcsDriver.GitVcsDriver)({ + resolvePrimaryRemoteName: () => Effect.succeed("origin"), + resolveDefaultBranchName: () => Effect.succeed("main"), + execute: ({ cwd, args }) => { + const stdout = + args[0] === "symbolic-ref" + ? `refs/heads/${branch}\n` + : args[1] === "--git-common-dir" + ? scenario === "replaced" && cwd !== "/repo" + ? "/other/.git\n" + : "/repo/.git\n" + : args[0] === "worktree" + ? `branch refs/heads/${branch}\n`.repeat(scenario === "held" ? 2 : 1) + : "commit\n"; + return Effect.succeed({ + stdout, + stderr: "", + exitCode: ChildProcessSpawner.ExitCode( + scenario === "unborn" && args[1] === "--verify" ? 1 : 0, + ), + stdoutTruncated: false, + stderrTruncated: false, }); - } - return { sequence: commands.length, storedEvents: [] }; + }, + }), + Layer.mock(Orchestrator.OrchestratorV2)({ + getShellSnapshot: (options) => + Effect.succeed({ + ...snapshot, + threads: snapshot.threads.filter( + (thread) => thread.worktreePath === options?.worktreePath, + ), + archivedThreads: snapshot.archivedThreads.filter( + (thread) => thread.worktreePath === options?.worktreePath, + ), + }), + dispatch, + }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getThreadWorktreePaths: () => + Effect.succeed( + [...snapshot.threads, ...snapshot.archivedThreads].flatMap((thread) => + thread.worktreePath === null + ? [] + : [{ threadId: thread.id, worktreePath: thread.worktreePath }], + ), + ), }), - }), + Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({ + streamBranchChanges: Stream.fromIterable([null, "t3/bfc349ae", branch]).pipe( + Stream.map((refName) => ({ + cwd: "/worktree", + refName, + })), + Stream.ensuring(Deferred.succeed(reconciled, undefined)), + ), + }), + Layer.mock(GitWorkflowService.GitWorkflowService)({ + currentBranch: () => Effect.succeed(branch), + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + getShellSnapshot: () => Effect.succeed(snapshot), + dispatch, + }), + ), + ), ), ), ); diff --git a/apps/server/src/orchestration-v2/WorktreeBranchSync.ts b/apps/server/src/orchestration-v2/WorktreeBranchSync.ts index 89ebf2d95df3b..069f7ec7d69c3 100644 --- a/apps/server/src/orchestration-v2/WorktreeBranchSync.ts +++ b/apps/server/src/orchestration-v2/WorktreeBranchSync.ts @@ -1,5 +1,3 @@ -import { CommandId } from "@t3tools/contracts"; -import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -10,10 +8,10 @@ import * as GitWorkflowService from "../git/GitWorkflowService.ts"; import { forkParked } from "../serverActivation.ts"; import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; -import * as ThreadManagement from "./ThreadManagementService.ts"; +import * as WorktreeBranches from "./WorktreeBranches.ts"; const makeReconcile = Effect.gen(function* () { - const threads = yield* ThreadManagement.ThreadManagementService; + const worktrees = yield* WorktreeBranches.WorktreeBranches; const git = yield* GitWorkflowService.GitWorkflowService; const projections = yield* ProjectionStore.ProjectionStoreV2; const fs = yield* FileSystem.FileSystem; @@ -31,40 +29,10 @@ const makeReconcile = Effect.gen(function* () { : yield* fs.realPath(worktreePath).pipe(Effect.orElseSucceed(() => null)); if (realPath === input.cwd) matchingPaths.add(worktreePath); } - const matchingThreads = new Map( - paths - .filter((thread) => matchingPaths.has(thread.worktreePath)) - .map((thread) => [thread.threadId, thread.worktreePath]), - ); - if (matchingThreads.size === 0) return; - const snapshot = yield* threads.getShellSnapshot({ threadIds: [...matchingThreads.keys()] }); - const sharing = [...snapshot.threads, ...snapshot.archivedThreads].filter( - (thread) => - thread.worktreePath === matchingThreads.get(thread.id) && thread.branch !== input.refName, - ); - if (sharing.length === 0) return; + if (matchingPaths.size === 0) return; // Status events may queue behind a generated rename or another checkout. if ((yield* git.currentBranch(input.cwd)) !== input.refName) return; - for (const thread of sharing) { - yield* threads - .dispatch({ - type: "thread.metadata.update", - commandId: CommandId.make(yield* randomUuidV4), - threadId: thread.id, - branch: input.refName, - expectedBranch: thread.branch, - expectedWorktreePath: thread.worktreePath, - }) - .pipe( - Effect.catchTags({ - OrchestratorDispatchError: (cause) => - Effect.logWarning("Worktree branch update was rejected", { - threadId: thread.id, - cause, - }), - }), - ); - } + yield* worktrees.sync({ worktreePath: input.cwd, threadPaths: [...matchingPaths] }); }); }); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index 659c65dc78dc8..80a136de3ebfe 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -408,7 +408,7 @@ export const layer = Layer.mergeAll( export const layerProduction = Layer.mergeAll( WorktreeBranchSync.layer.pipe( - Layer.provide(Layer.merge(layerThreadManagementProvided, ProjectionStore.layer)), + Layer.provide(Layer.merge(layerWorktreeBranchesProvided, ProjectionStore.layer)), ), layer, layerManagedProjectFoldersProvided, diff --git a/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts b/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts index 627f3ccfddb08..7c3ec79d56fdc 100644 --- a/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts +++ b/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts @@ -775,23 +775,38 @@ it.effect("reports a summary pass as pending until it settles", () => }).pipe(Effect.provide(makeLayer(citing().generate, optedIn))), ); -it.effect( - "recovers failed and legacy summary passes with bounded model-specific retry rounds", - () => { +it.effect.each([ + { work: "leaf", waitingMessages: 10 }, + { work: "rollup", waitingMessages: 0 }, +])( + "recovers failed and legacy $work passes with bounded model-specific retry rounds", + ({ work, waitingMessages }) => { + let gate: { entered: Deferred.Deferred; release: Deferred.Deferred } | undefined; const model = scripted(() => - Effect.fail( - new TextGenerationError({ operation: "generateConversationSummary", detail: "No model" }), - ), + Effect.gen(function* () { + if (gate !== undefined) { + yield* Deferred.succeed(gate.entered, undefined); + yield* Deferred.await(gate.release); + } + return yield* new TextGenerationError({ + operation: "generateConversationSummary", + detail: "No model", + }); + }), ); return Effect.gen(function* () { - yield* seed; + yield* seedThread(work === "rollup" ? 14 : MESSAGE_COUNT); + if (work === "rollup") { + for (let index = 0; index < 4; index++) + yield* persistSummary(index, [index * 2, index * 2 + 1]); + } assert.instanceOf(yield* summarize.pipe(Effect.flip), ConversationSummaryPassError); const result = yield* list; - assert.deepEqual(result.summaries, []); + assert.lengthOf(result.summaries, work === "rollup" ? 4 : 0); assert.deepInclude(result.lastPass, { result: "failed", reason: "No model", - waitingMessages: 10, + waitingMessages, }); const service = yield* ConversationSummaries.ConversationSummaries; const sql = yield* SqlClient.SqlClient; @@ -826,11 +841,24 @@ it.effect( `; yield* TestClock.adjust("2 days"); assert.deepEqual(yield* stalled(), []); + yield* sql` + UPDATE conversation_summary_passes + SET payload_json = json_set(payload_json, '$.retries', 5) WHERE thread_id = ${threadId} + `; + const finalRetry = (yield* stalled())[0]!; + assert.deepInclude(finalRetry, { threadId, runId, retry: 6 }); + gate = { entered: yield* Deferred.make(), release: yield* Deferred.make() }; + const pass = yield* service.summarizeThread(finalRetry).pipe(Effect.exit, Effect.forkChild); + yield* Deferred.await(gate.entered); const settings = yield* ServerSettings.ServerSettingsService; yield* settings.updateSettings({ projectSettingsOverrides: { [projectId]: { enableConversationSummaries: false } }, }); yield* settings.updateSettings(optedIn); + yield* Deferred.succeed(gate.release, undefined); + assert.isTrue(Exit.isFailure(yield* Fiber.join(pass))); + gate = undefined; + yield* TestClock.adjust("15 minutes"); assert.deepInclude((yield* stalled())[0], { threadId, runId, retry: 1 }); yield* sql` UPDATE conversation_summary_passes diff --git a/apps/server/src/project/conversationSummaries/ConversationSummaries.ts b/apps/server/src/project/conversationSummaries/ConversationSummaries.ts index fb12a0925e5d8..a0de2683bccb7 100644 --- a/apps/server/src/project/conversationSummaries/ConversationSummaries.ts +++ b/apps/server/src/project/conversationSummaries/ConversationSummaries.ts @@ -1211,13 +1211,16 @@ const make = Effect.gen(function* () { }; // Re-enabling summaries may clear the ledger while this pass runs. The reset wins, so only // this pass's own failures survive it. + let retryRound = retry ?? 0; const keepReset = Effect.fnUntraced(function* () { - if (prior?.chunkFailures === undefined) return; + if (prior?.chunkFailures === undefined && prior?.retries === undefined) return; const cleared = yield* sql<{ readonly cleared: number }>` - SELECT json_extract(payload_json, '$.chunkFailures') IS NULL AS cleared + SELECT (json_extract(payload_json, '$.chunkFailures') IS NULL + AND json_extract(payload_json, '$.retries') IS NULL) AS cleared FROM conversation_summary_passes WHERE thread_id = ${threadId} `; if (cleared[0]?.cleared !== 1) return; + retryRound = 0; chunkFailures.clear(); for (const failure of charges) charge(failure); }); @@ -1240,7 +1243,7 @@ const make = Effect.gen(function* () { ...open.filter((failure) => failure.failures > 0), ...failures.filter((failure) => failure.skipped === true).slice(-MAX_SKIPPED_CHUNKS), ].slice(-MAX_CHUNK_FAILURES); - const retries = result !== "failed" ? 0 : (retry ?? 0); + const retries = result !== "failed" ? 0 : retryRound; return recordPass(projectId, threadId, { runId, result, @@ -1915,7 +1918,6 @@ const make = Effect.gen(function* () { INNER JOIN orchestration_v2_projection_threads AS thread ON thread.thread_id = passes.thread_id WHERE json_extract(passes.payload_json, '$.result') = 'failed' - AND json_extract(passes.payload_json, '$.waitingMessages') > 0 AND ${eligibleProjects} AND EXISTS ( SELECT 1 FROM projection_projects AS project diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index e50c47b6646d6..c948cb465e55e 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -779,25 +779,40 @@ const checkpointCleanupFixture = Effect.fn("test.checkpointCleanupFixture")(func }); describe("checkpoint ref cleanup", () => { - it.live("prunes refs owned by deleted threads after their worktree is removed", () => - runCleanupTest( - Effect.gen(function* () { - const { service, repo, worktree, command, addThread, listRefs } = - yield* checkpointCleanupFixture(); - yield* addThread("deleted", { deleted: true }); - yield* addThread("imported-deleted", { deleted: true, legacyOnly: true }); - const retained = [ - ...(yield* addThread("active")), - ...(yield* addThread("archived", { archived: true })), - legacyCheckpointRef("unknown-owner", 0), - ]; - yield* command(repo, ["update-ref", retained.at(-1)!, "HEAD"]); - yield* command(repo, ["pack-refs", "--all"]); - yield* command(repo, ["worktree", "remove", worktree]); - yield* service.runNow; - expect(yield* listRefs()).toEqual(retained.sort()); - }).pipe(Effect.provide(Sqlite.layerMemory)), - ), + it.live.each([false, true])( + "prunes refs only when owned by this home, dev home=%s", + (worktreeHome) => + runCleanupTest( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const config = yield* ServerConfig.ServerConfig; + const checkout = yield* fs.makeTempDirectoryScoped({ + prefix: "checkpoint-dev-checkout-", + }); + yield* fs.writeFileString(`${checkout}/.git`, "gitdir: /repo/.git/worktrees/dev\n"); + const baseDir = worktreeHome ? `${checkout}/.t3` : config.baseDir; + const { service, repo, worktree, command, addThread, listRefs } = + yield* checkpointCleanupFixture().pipe( + Effect.provideService(ServerConfig.ServerConfig, { ...config, baseDir }), + ); + // A seeded dev home copies the real home's thread/scope IDs and repository paths. + // Deleting its copy must not delete the original thread's shared Git checkpoint refs. + const deleted = [ + ...(yield* addThread("deleted", { deleted: true })), + ...(yield* addThread("imported-deleted", { deleted: true, legacyOnly: true })), + ]; + const retained = [ + ...(yield* addThread("active")), + ...(yield* addThread("archived", { archived: true })), + legacyCheckpointRef("unknown-owner", 0), + ]; + yield* command(repo, ["update-ref", retained.at(-1)!, "HEAD"]); + yield* command(repo, ["pack-refs", "--all"]); + yield* command(repo, ["worktree", "remove", worktree]); + yield* service.runNow; + expect(yield* listRefs()).toEqual([...retained, ...(worktreeHome ? deleted : [])].sort()); + }).pipe(Effect.provide(Sqlite.layerMemory)), + ), ); it.live("keeps deleted fork ancestry while a live or archived descendant needs it", () => diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index 326b76dda42e4..e60ce8d398386 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -1661,6 +1661,9 @@ const make = Effect.gen(function* () { const reportRef = yield* SubscriptionRef.make(null); const cleanCheckpointRefs = Effect.fn("StorageCleanup.cleanCheckpointRefs")(function* () { + // Dev homes copy the real home's thread and scope IDs. Their tombstones cannot + // establish ownership of checkpoint refs in a shared repository. + if ((yield* resolveWorktreeT3Home(config.baseDir)) === path.resolve(config.baseDir)) return; // Forks display their source history even after its thread is deleted. Keep // that ancestry, including deleted intermediate forks and archived children. const rows = yield* sql<{ diff --git a/apps/server/src/workspace/restoreMissingWorktree.ts b/apps/server/src/workspace/restoreMissingWorktree.ts index c2c3c00eb7741..05a16be2bf94f 100644 --- a/apps/server/src/workspace/restoreMissingWorktree.ts +++ b/apps/server/src/workspace/restoreMissingWorktree.ts @@ -78,25 +78,28 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { ); // Only folders holding nothing but leftovers are cleared, never user files. - const isLeftoverFolder = (worktreePath: string) => + const readLeftoverEntries = (worktreePath: string) => fileSystem.exists(path.join(worktreePath, ".git")).pipe( Effect.flatMap((hasGit) => hasGit - ? Effect.succeed(false) - : fileSystem.readDirectory(worktreePath).pipe( - Effect.flatMap((entries) => - Effect.forEach(entries, (entry) => - LEFTOVER_WORKTREE_ENTRIES.has(entry) - ? fileSystem - .stat(path.join(worktreePath, entry)) - .pipe(Effect.map((info) => info.type === "Directory")) - : Effect.succeed(false), + ? Effect.undefined + : fileSystem + .readDirectory(worktreePath) + .pipe( + Effect.flatMap((entries) => + Effect.forEach(entries, (entry) => + LEFTOVER_WORKTREE_ENTRIES.has(entry) + ? fileSystem + .stat(path.join(worktreePath, entry)) + .pipe(Effect.map((info) => info.type === "Directory")) + : Effect.succeed(false), + ).pipe( + Effect.map((leftovers) => (leftovers.every(Boolean) ? entries : undefined)), + ), ), ), - Effect.map((leftovers) => leftovers.every(Boolean)), - ), ), - Effect.orElseSucceed(() => false), + Effect.orElseSucceed(() => undefined), ); const resolveRecreateBase = Effect.fn("restoreMissingWorktree.resolveRecreateBase")(function* ( @@ -170,7 +173,7 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { if (worktreePath === null) return; if (branch === null) return yield* recreateScratchFolder(thread, worktreePath); const exists = yield* fileSystem.exists(worktreePath).pipe(Effect.orElseSucceed(() => true)); - const leftover = exists && (yield* isLeftoverFolder(worktreePath)); + const leftover = exists && (yield* readLeftoverEntries(worktreePath)) !== undefined; if ((exists && !leftover) || !(yield* isManagedWorktreePath(worktreePath))) return; const project = yield* projects.getById(thread.projectId).pipe( Effect.map(Option.getOrUndefined), @@ -184,17 +187,18 @@ export const makeRestoreMissingWorktree = Effect.gen(function* () { }); // `git worktree add` refuses a non-empty target but accepts an empty one. Keep the folder // itself: Windows cannot remove a folder that is still a terminal's or provider's cwd. - const clearLeftovers = Effect.suspend(() => fileSystem.readDirectory(worktreePath)).pipe( - Effect.flatMap((entries) => - Effect.forEach( + yield* Effect.gen(function* () { + if (leftover) { + const entries = yield* readLeftoverEntries(worktreePath); + if (entries === undefined) return; + yield* Effect.forEach( entries, (entry) => fileSystem.remove(path.join(worktreePath, entry), { recursive: true }), { discard: true }, - ), - ), - ); - yield* (leftover ? clearLeftovers : Effect.void).pipe( - Effect.andThen(recreateMissingWorktree({ cwd: project.workspaceRoot, worktreePath, branch })), + ); + } + yield* recreateMissingWorktree({ cwd: project.workspaceRoot, worktreePath, branch }); + }).pipe( Effect.mapError((cause) => cause._tag === "WorktreeRecreateNoBaseError" ? cause diff --git a/evals/shared/proxy.test.ts b/evals/shared/proxy.test.ts new file mode 100644 index 0000000000000..05e22327a5e55 --- /dev/null +++ b/evals/shared/proxy.test.ts @@ -0,0 +1,47 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import { vi } from "vite-plus/test"; + +import { morphConfig } from "./proxy.ts"; + +it.live.each(["v1", "v2", "environment"] as const)( + "reads Morph credentials from %s without a model request", + (source) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const configHome = yield* fs.makeTempDirectoryScoped({ prefix: "morph-eval-config-" }); + const settings = { apiKey: "eval-config-key", baseURL: "https://config.invalid/v1/" }; + if (source !== "environment") { + yield* fs.makeDirectory(`${configHome}/opencode`); + yield* fs.writeFileString( + `${configHome}/opencode/opencode.json`, + JSON.stringify( + source === "v1" + ? { provider: { morph: { options: settings } } } + : { providers: { morph: { settings } } }, + ), + ); + } + vi.stubEnv("XDG_CONFIG_HOME", configHome); + vi.stubEnv("MORPH_API_KEY", source === "environment" ? "eval-environment-key" : undefined); + vi.stubEnv( + "MORPH_BASE_URL", + source === "environment" ? "https://environment.invalid/v1/" : undefined, + ); + try { + expect(morphConfig()).toEqual({ + url: + source === "environment" + ? "https://environment.invalid/v1" + : "https://config.invalid/v1", + headers: { + Authorization: `Bearer ${source === "environment" ? "eval-environment-key" : "eval-config-key"}`, + }, + }); + } finally { + vi.unstubAllEnvs(); + } + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/evals/shared/proxy.ts b/evals/shared/proxy.ts index fb4f123a2ad4e..3b11e78c9bad9 100644 --- a/evals/shared/proxy.ts +++ b/evals/shared/proxy.ts @@ -50,7 +50,10 @@ url = os.environ.get('MORPH_BASE_URL', 'https://api.morphllm.com/v1') if not key: home = pathlib.Path(os.environ.get('XDG_CONFIG_HOME') or pathlib.Path.home() / '.config') # OpenCode writes literal control characters inside strings, which strict parsing rejects. - settings = json.loads((home / 'opencode' / 'opencode.json').read_text(), strict=False)['providers']['morph']['settings'] + config = json.loads((home / 'opencode' / 'opencode.json').read_text(), strict=False) + settings = config.get('providers', {}).get('morph', {}).get('settings') + if settings is None: + settings = config.get('provider', {}).get('morph', {}).get('options', {}) key, url = settings.get('apiKey'), settings.get('baseURL', url) if not key: raise SystemExit('no morph key')