From 2b2a1bb55cb563ea4ae54c1c1091b56d93f8938f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:42:03 -0800 Subject: [PATCH] fix(server): preserve data across rollback and update recovery --- .../CheckpointRollbackService.test.ts | 147 ++++++++++++++++-- .../CheckpointRollbackService.ts | 21 ++- apps/server/src/serviceLauncher.test.ts | 30 ++-- apps/server/src/serviceLauncher.ts | 9 +- apps/server/src/vcs/GitVcsDriver.test.ts | 29 ++++ apps/server/src/vcs/GitVcsDriver.ts | 4 +- 6 files changed, 207 insertions(+), 33 deletions(-) diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index a517083a955c2..c09a66e502594 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -39,6 +39,7 @@ import * as CheckpointService from "./CheckpointService.ts"; import * as CheckpointRollbackService from "./CheckpointRollbackService.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; import * as EventSink from "./EventSink.ts"; +import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; @@ -178,9 +179,31 @@ it.effect.each([ { order: "cleanup-first", restoreFiles: false, leavesFolder: true }, { order: "cleanup-first", restoreFiles: true, leavesFolder: true, shared: true }, { order: "cleanup-first", restoreFiles: true, leavesFolder: true, createFails: true }, + { order: "cleanup-first", restoreFiles: true, leavesFolder: true, failureAfterCreate: "session" }, + { order: "cleanup-first", restoreFiles: true, leavesFolder: true, failureAfterCreate: "files" }, + { + order: "cleanup-first", + restoreFiles: true, + leavesFolder: true, + failureAfterCreate: "interrupt", + }, + { + order: "cleanup-first", + restoreFiles: true, + leavesFolder: true, + failureAfterCreate: "committed", + }, ])("locks rollback with %s", (row) => Effect.gen(function* () { - const { order, restoreFiles, unmanaged, leavesFolder, shared, createFails } = row; + const { + order, + restoreFiles, + unmanaged, + leavesFolder, + shared, + createFails, + failureAfterCreate, + } = row; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const workspaceRoot = yield* fileSystem.realPath( @@ -201,9 +224,30 @@ it.effect.each([ const branch = "feature/rollback"; const release = Deferred.makeUnsafe(); const openEntered = Deferred.makeUnsafe(); + const committed = yield* Deferred.make(); + const releaseCommit = yield* Deferred.make(); + const eventStore = yield* EventStore.EventStoreV2; + const persistedProjections = yield* ProjectionStore.ProjectionStoreV2; const calls: string[] = []; let cleanupEntered = false; - const providerThread = { id: providerThreadId, providerSessionId, providerInstanceId }; + const now = yield* DateTime.now; + const providerThread = { + id: providerThreadId, + driver: ProviderDriverKind.make("codex"), + providerSessionId, + providerInstanceId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; const projection = { thread: { id: threadId, @@ -266,15 +310,47 @@ it.effect.each([ yield* fileSystem.exists(path.join(worktreePath, ".git")).pipe(Effect.orDie), ); calls.push("files"); + if (failureAfterCreate === "files") + return yield* new CheckpointService.CheckpointRestoreError({ + scopeId, + checkpointId, + cause: "checkpoint ref is unavailable", + }); + yield* fileSystem + .writeFileString(path.join(worktreePath, "restored.txt"), "checkpoint") + .pipe(Effect.orDie); }), }), - Layer.mock(EventSink.EventSinkV2)({ - write: () => - Effect.sync(() => { - calls.push("projection"); - return []; + failureAfterCreate === "committed" + ? EventSink.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ProjectionStore.ProjectionStoreV2, persistedProjections), + Layer.succeed( + EventStore.EventStoreV2, + EventStore.EventStoreV2.of({ + ...eventStore, + publishCommitted: (events) => + eventStore.publishCommitted(events).pipe( + Effect.tap(() => { + calls.push("projection"); + return Deferred.succeed(committed, undefined).pipe( + Effect.andThen(Deferred.await(releaseCommit)), + ); + }), + ), + }), + ), + ), + ), + ) + : Layer.mock(EventSink.EventSinkV2)({ + write: () => + Effect.sync(() => { + calls.push("projection"); + return []; + }), }), - }), IdAllocator.layer, Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.succeed(projection.thread), @@ -310,6 +386,9 @@ it.effect.each([ }); } calls.push("session"); + if (failureAfterCreate === "session") + return yield* Effect.die("the provider rejected the session"); + if (failureAfterCreate === "interrupt") return yield* Effect.interrupt; yield* Deferred.succeed(openEntered, undefined); if (order === "rollback-first") yield* Deferred.await(release); return {} as never; @@ -356,6 +435,34 @@ it.effect.each([ if (order === "rollback-first") assert.isFalse(cleanupEntered); yield* Deferred.succeed(release, undefined); if (cleanup !== undefined) yield* Fiber.join(cleanup); + if (failureAfterCreate === "committed") { + yield* Deferred.await(committed).pipe( + Effect.andThen( + Fiber.interrupt(rollback).pipe(Effect.forkChild({ startImmediately: true })), + ), + Effect.ensuring(Deferred.succeed(releaseCommit, undefined)), + ); + assert.equal((yield* Fiber.await(rollback))._tag, "Failure"); + const sql = yield* SqlClient.SqlClient; + assert.deepEqual( + yield* sql`SELECT event_type FROM orchestration_events WHERE stream_id = ${threadId}`, + [{ event_type: "provider-thread.updated" }], + ); + assert.deepEqual( + yield* sql`SELECT json_extract(payload_json, '$.lastRunOrdinal') AS ordinal + FROM orchestration_v2_projection_provider_threads + WHERE provider_thread_id = ${providerThreadId}`, + [{ ordinal: null }], + ); + assert.isTrue(yield* fileSystem.exists(path.join(worktreePath, ".git"))); + assert.equal( + yield* fileSystem.readFileString(path.join(worktreePath, "restored.txt")), + "checkpoint", + ); + assert.isFalse(yield* fileSystem.exists(path.join(worktreePath, "unsaved.txt"))); + assert.deepEqual(yield* fileSystem.readDirectory(path.dirname(worktreePath)), ["feature"]); + return; + } if (unmanaged || shared) { const error = yield* Fiber.join(rollback).pipe(Effect.flip); assert.equal(error._tag, "CheckpointRollbackExecutionError"); @@ -366,9 +473,15 @@ it.effect.each([ } return; } - if (createFails) { - yield* Fiber.join(rollback).pipe(Effect.exit); - assert.deepEqual(calls, ["cleanup", "prune", "create"]); + if (createFails || failureAfterCreate !== undefined) { + assert.equal((yield* Fiber.join(rollback).pipe(Effect.exit))._tag, "Failure"); + assert.deepEqual(calls, [ + "cleanup", + "prune", + "create", + ...(failureAfterCreate === undefined ? [] : ["session"]), + ...(failureAfterCreate === "files" ? ["files"] : []), + ]); assert.equal( yield* fileSystem.readFileString(path.join(worktreePath, "unsaved.txt")), "keep", @@ -392,7 +505,17 @@ it.effect.each([ assert.isTrue(yield* fileSystem.exists(path.join(worktreePath, "unsaved.txt"))); } }).pipe(Effect.provide(testLayer)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + Layer.mergeAll(EventStore.layer, ProjectionStore.layer).pipe( + Layer.provideMerge(SqlitePersistence.layerMemory), + ), + ), + ), + ), ); it.effect.each([{ restoreFiles: true }, { restoreFiles: false }])( diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 14c5b904db556..7bd058a11bae0 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -239,6 +239,7 @@ export const layer: Layer.Layer< }); const { worktreePath, branch } = context.thread; + let rollbackCommitted = false; // Recreated only where the server would have made it, as turn start does. if (worktreePath != null && branch != null) { // A failed removal can leave the folder without its checkout; only a file restore replaces it. @@ -263,8 +264,8 @@ export const layer: Layer.Layer< ); if (leftover) { if (!(yield* isolated)) return yield* sharedWorkspace(); - // Set aside, not removed: a checkout that fails puts the folder and its files back. - yield* Effect.acquireUseRelease( + // Keep the original folder until the completed rollback is recorded. + yield* Effect.acquireRelease( Effect.gen(function* () { const aside = yield* fileSystem.makeTempDirectory({ directory: path.dirname(worktreePath), @@ -273,9 +274,8 @@ export const layer: Layer.Layer< yield* fileSystem.rename(worktreePath, path.join(aside, "folder")); return aside; }), - () => recreate, (aside, exit) => - (Exit.isSuccess(exit) + (Exit.isSuccess(exit) || rollbackCommitted ? Effect.void : fileSystem .remove(worktreePath, { recursive: true, force: true }) @@ -295,6 +295,7 @@ export const layer: Layer.Layer< ), ), ); + yield* recreate; } else { yield* recreate; } @@ -442,10 +443,18 @@ export const layer: Layer.Layer< ); } } - yield* eventSink.write({ events }); + // Cancellation after the durable commit must not restore the old folder. + yield* eventSink.write({ events }).pipe( + Effect.tap(() => + Effect.sync(() => { + rollbackCommitted = true; + }), + ), + Effect.uninterruptible, + ); }), ); - }); + }, Effect.scoped); return CheckpointRollbackServiceV2.of({ // The inner execute owns the rollback span; this wrapper selects the workspace lease. diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 2be2df6527b2b..1500d6466918d 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1485,9 +1485,13 @@ if (context.update?.status === "pending") { }), ); - it.effect.each(["wal", "delete"] as const)( - "backs up a %s-mode database (online when WAL) and restores its final state", - (journalMode) => + it.effect.each([ + { journalMode: "wal", canClone: false }, + { journalMode: "delete", canClone: false }, + { journalMode: "wal", canClone: true }, + ] as const)( + "backs up a $journalMode-mode database with clone support $canClone and restores its final state", + ({ journalMode, canClone }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -1535,6 +1539,7 @@ if (context.update?.status === "pending") { backupsAtStop: readdirSync(backupRoot).map((entry) => readdirSync(backupRoot + "/" + entry)), rows: db.prepare("SELECT count(*) AS n FROM events").get().n, })); + ${canClone ? 'process.kill(process.pid, "SIGKILL");' : ""} db.close(); process.exit(0); }); @@ -1562,7 +1567,7 @@ if (context.update?.status === "pending") { root, yield* Effect.promise(() => readServiceState(statePath)), { - backupSpaceProbe: { canClone: async () => false }, + backupSpaceProbe: { canClone: async () => canClone }, }, ); yield* Effect.promise(() => @@ -1576,13 +1581,9 @@ if (context.update?.status === "pending") { assert.equal(state.update?.status, "rolled-back"); const observed = JSON.parse(yield* fs.readFileString(observationsPath)); // A WAL database's main file is copied before the server is asked to stop; others fall back offline. - assert.deepEqual(observed.backupsAtStop, journalMode === "wal" ? [["database"]] : []); + const online = journalMode === "wal" && !canClone; + assert.deepEqual(observed.backupsAtStop, online ? [["database"]] : []); assert.isAbove(observed.rows, 501); - // Only the online copy holds no -shm: the offline fallback copies the one the server left. - assert.deepEqual( - JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)), - journalMode === "wal" ? ["database", "database-wal"] : ["database"], - ); const restored = new NodeSqlite.DatabaseSync(databasePath); try { assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok"); @@ -1601,6 +1602,15 @@ if (context.update?.status === "pending") { } finally { restored.close(); } + // Only the online copy holds no -shm: the offline fallback copies the one the server left. + assert.deepEqual( + JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)), + online + ? ["database", "database-wal"] + : journalMode === "wal" + ? ["database", "database-shm", "database-wal"] + : ["database"], + ); }), ); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 77453fe8d6bcd..5936b1f0001a2 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -371,9 +371,10 @@ const CLONED_BACKUP_HEADROOM_FRACTION = 0.1; const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2; async function statDatabaseFiles(dbPath: string): Promise> { + const databasePath = await NodeFSP.realpath(dbPath); const stats: Array = []; for (const suffix of DB_FILE_SUFFIXES) { - const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => { + const stat = await NodeFSP.stat(`${databasePath}${suffix}`).catch((cause: unknown) => { if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw cause; }); @@ -400,8 +401,9 @@ async function backupCanClone(dbPath: string, backupRoot: string): Promise + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-symbolic-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const branchRef = (yield* git(["symbolic-ref", "HEAD"])).stdout.trim(); + const originalHead = (yield* git(["rev-parse", "HEAD"])).stdout; + yield* git(["symbolic-ref", checkpointRef, branchRef]); + + if (operation === "capture") + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + else yield* driver.checkpoints.deleteCheckpointRefs({ cwd, checkpointRefs: [checkpointRef] }); + + const branch = yield* driver.execute({ + operation: "checkpoint-test.read-branch", + cwd, + args: ["rev-parse", "--verify", branchRef], + allowNonZeroExit: true, + }); + assert.equal(branch.stdout, originalHead); + assert.equal(branch.exitCode, 0); + const refs = (yield* git(["for-each-ref", "--format=%(refname)", "refs/t3/"])).stdout; + assert.equal(refs, operation === "capture" ? `${checkpointRef}\n` : ""); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("restores and deletes checkpoints inside populated submodules", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index d7b72ff4318e2..e1417ade17e47 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -1762,7 +1762,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( yield* execute({ operation, cwd: input.cwd, - args: [...durableWrite, "update-ref", input.checkpointRef, commitOid], + args: [...durableWrite, "update-ref", "--no-deref", input.checkpointRef, commitOid], }); if (untracked !== undefined) { yield* untrackedBlobs.remember({ @@ -2264,7 +2264,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( execute({ operation: "GitVcsDriver.checkpoints.deleteCheckpointRefs", cwd: input.cwd, - args: ["update-ref", "-d", checkpointRef], + args: ["update-ref", "--no-deref", "-d", checkpointRef], allowNonZeroExit: true, }), { discard: true },