diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts index a517083a955c2..b12e9c0c5f8d2 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts @@ -1,4 +1,5 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, it, vi } from "@effect/vitest"; import { CheckpointId, @@ -32,6 +33,8 @@ import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerConfig from "../config.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as GitWorkflow from "../git/GitWorkflowService.ts"; +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; +import * as VcsProcess from "../vcs/VcsProcess.ts"; import { withTurnStartReservation, withWorkspaceLease } from "../workspace/workspaceLease.ts"; import { ProviderWorkspaceMissingError } from "../provider/Errors.ts"; import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts"; @@ -39,6 +42,7 @@ import * as CheckpointService from "./CheckpointService.ts"; import * as CheckpointRollbackService from "./CheckpointRollbackService.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; import * as EventSink from "./EventSink.ts"; +import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; @@ -178,9 +182,31 @@ it.effect.each([ { order: "cleanup-first", restoreFiles: false, leavesFolder: true }, { order: "cleanup-first", restoreFiles: true, leavesFolder: true, shared: true }, { order: "cleanup-first", restoreFiles: true, leavesFolder: true, createFails: true }, + { order: "cleanup-first", restoreFiles: true, leavesFolder: true, failureAfterCreate: "session" }, + { order: "cleanup-first", restoreFiles: true, leavesFolder: true, failureAfterCreate: "files" }, + { + order: "cleanup-first", + restoreFiles: true, + leavesFolder: true, + failureAfterCreate: "interrupt", + }, + { + order: "cleanup-first", + restoreFiles: true, + leavesFolder: true, + failureAfterCreate: "committed", + }, ])("locks rollback with %s", (row) => Effect.gen(function* () { - const { order, restoreFiles, unmanaged, leavesFolder, shared, createFails } = row; + const { + order, + restoreFiles, + unmanaged, + leavesFolder, + shared, + createFails, + failureAfterCreate, + } = row; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const workspaceRoot = yield* fileSystem.realPath( @@ -201,9 +227,30 @@ it.effect.each([ const branch = "feature/rollback"; const release = Deferred.makeUnsafe(); const openEntered = Deferred.makeUnsafe(); + const committed = yield* Deferred.make(); + const releaseCommit = yield* Deferred.make(); + const eventStore = yield* EventStore.EventStoreV2; + const persistedProjections = yield* ProjectionStore.ProjectionStoreV2; const calls: string[] = []; let cleanupEntered = false; - const providerThread = { id: providerThreadId, providerSessionId, providerInstanceId }; + const now = yield* DateTime.now; + const providerThread = { + id: providerThreadId, + driver: ProviderDriverKind.make("codex"), + providerSessionId, + providerInstanceId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle" as const, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; const projection = { thread: { id: threadId, @@ -266,15 +313,47 @@ it.effect.each([ yield* fileSystem.exists(path.join(worktreePath, ".git")).pipe(Effect.orDie), ); calls.push("files"); + if (failureAfterCreate === "files") + return yield* new CheckpointService.CheckpointRestoreError({ + scopeId, + checkpointId, + cause: "checkpoint ref is unavailable", + }); + yield* fileSystem + .writeFileString(path.join(worktreePath, "restored.txt"), "checkpoint") + .pipe(Effect.orDie); }), }), - Layer.mock(EventSink.EventSinkV2)({ - write: () => - Effect.sync(() => { - calls.push("projection"); - return []; + failureAfterCreate === "committed" + ? EventSink.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(ProjectionStore.ProjectionStoreV2, persistedProjections), + Layer.succeed( + EventStore.EventStoreV2, + EventStore.EventStoreV2.of({ + ...eventStore, + publishCommitted: (events) => + eventStore.publishCommitted(events).pipe( + Effect.tap(() => { + calls.push("projection"); + return Deferred.succeed(committed, undefined).pipe( + Effect.andThen(Deferred.await(releaseCommit)), + ); + }), + ), + }), + ), + ), + ), + ) + : Layer.mock(EventSink.EventSinkV2)({ + write: () => + Effect.sync(() => { + calls.push("projection"); + return []; + }), }), - }), IdAllocator.layer, Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.succeed(projection.thread), @@ -310,6 +389,9 @@ it.effect.each([ }); } calls.push("session"); + if (failureAfterCreate === "session") + return yield* Effect.die("the provider rejected the session"); + if (failureAfterCreate === "interrupt") return yield* Effect.interrupt; yield* Deferred.succeed(openEntered, undefined); if (order === "rollback-first") yield* Deferred.await(release); return {} as never; @@ -356,6 +438,34 @@ it.effect.each([ if (order === "rollback-first") assert.isFalse(cleanupEntered); yield* Deferred.succeed(release, undefined); if (cleanup !== undefined) yield* Fiber.join(cleanup); + if (failureAfterCreate === "committed") { + yield* Deferred.await(committed).pipe( + Effect.andThen( + Fiber.interrupt(rollback).pipe(Effect.forkChild({ startImmediately: true })), + ), + Effect.ensuring(Deferred.succeed(releaseCommit, undefined)), + ); + assert.equal((yield* Fiber.await(rollback))._tag, "Failure"); + const sql = yield* SqlClient.SqlClient; + assert.deepEqual( + yield* sql`SELECT event_type FROM orchestration_events WHERE stream_id = ${threadId}`, + [{ event_type: "provider-thread.updated" }], + ); + assert.deepEqual( + yield* sql`SELECT json_extract(payload_json, '$.lastRunOrdinal') AS ordinal + FROM orchestration_v2_projection_provider_threads + WHERE provider_thread_id = ${providerThreadId}`, + [{ ordinal: null }], + ); + assert.isTrue(yield* fileSystem.exists(path.join(worktreePath, ".git"))); + assert.equal( + yield* fileSystem.readFileString(path.join(worktreePath, "restored.txt")), + "checkpoint", + ); + assert.isFalse(yield* fileSystem.exists(path.join(worktreePath, "unsaved.txt"))); + assert.deepEqual(yield* fileSystem.readDirectory(path.dirname(worktreePath)), ["feature"]); + return; + } if (unmanaged || shared) { const error = yield* Fiber.join(rollback).pipe(Effect.flip); assert.equal(error._tag, "CheckpointRollbackExecutionError"); @@ -366,9 +476,15 @@ it.effect.each([ } return; } - if (createFails) { - yield* Fiber.join(rollback).pipe(Effect.exit); - assert.deepEqual(calls, ["cleanup", "prune", "create"]); + if (createFails || failureAfterCreate !== undefined) { + assert.equal((yield* Fiber.join(rollback).pipe(Effect.exit))._tag, "Failure"); + assert.deepEqual(calls, [ + "cleanup", + "prune", + "create", + ...(failureAfterCreate === undefined ? [] : ["session"]), + ...(failureAfterCreate === "files" ? ["files"] : []), + ]); assert.equal( yield* fileSystem.readFileString(path.join(worktreePath, "unsaved.txt")), "keep", @@ -392,7 +508,17 @@ it.effect.each([ assert.isTrue(yield* fileSystem.exists(path.join(worktreePath, "unsaved.txt"))); } }).pipe(Effect.provide(testLayer)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + Layer.mergeAll(EventStore.layer, ProjectionStore.layer).pipe( + Layer.provideMerge(SqlitePersistence.layerMemory), + ), + ), + ), + ), ); it.effect.each([{ restoreFiles: true }, { restoreFiles: false }])( @@ -528,9 +654,17 @@ it.effect.each([{ restoreFiles: true }, { restoreFiles: false }])( /** `restartedAtRun` starts a fresh native conversation at that run, as a context restart does. */ const seedRollbackHistory = Effect.fn(function* ( - options: { readonly restartedAtRun?: number } = {}, + options: { + readonly restartedAtRun?: number; + readonly cwd?: string; + readonly checkpoints?: Pick< + CheckpointService.CheckpointServiceV2Shape, + "restore" | "deleteStaleRefs" + >; + } = {}, ) { const store = yield* ProjectionStore.ProjectionStoreV2; + const eventSink = yield* EventSink.EventSinkV2; const now = yield* DateTime.now; const threadId = ThreadId.make("rollback-history"); const providerThreadId = ProviderThreadId.make("rollback-history-provider"); @@ -570,7 +704,7 @@ const seedRollbackHistory = Effect.fn(function* ( runtimeMode: "full-access", interactionMode: "default", branch: null, - worktreePath: process.cwd(), + worktreePath: options.cwd ?? process.cwd(), activeProviderThreadId: providerThreadId, lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, forkedFrom: null, @@ -620,7 +754,7 @@ const seedRollbackHistory = Effect.fn(function* ( kind: id === scopeId ? "root_run" : "subagent", ordinalWithinParent: 0, advancesAppRunCount: id === scopeId, - cwd: process.cwd(), + cwd: options.cwd ?? process.cwd(), createdAt: now, }, }); @@ -766,30 +900,24 @@ const seedRollbackHistory = Effect.fn(function* ( ThreadCommandExecutor.layer, IdAllocator.layer, Layer.mock(CheckpointService.CheckpointServiceV2)({ - restore: ({ checkpoint }) => - Effect.sync(() => { - restored.push(checkpoint.id); - }), - deleteStaleRefs: ({ checkpoints }) => - Effect.sync(() => { - deletedRefs.push(...checkpoints.map((checkpoint) => checkpoint.id)); - }), + restore: (input) => + (options.checkpoints?.restore(input) ?? Effect.void).pipe( + Effect.tap(() => Effect.sync(() => restored.push(input.checkpoint.id))), + ), + deleteStaleRefs: (input) => + (options.checkpoints?.deleteStaleRefs(input) ?? Effect.void).pipe( + Effect.tap(() => + Effect.sync(() => + deletedRefs.push(...input.checkpoints.map((checkpoint) => checkpoint.id)), + ), + ), + ), }), Layer.mock(EventSink.EventSinkV2)({ write: ({ events }) => - Effect.gen(function* () { - for (const event of events) yield* store.apply(event); - committed.push(...events); - return []; - }).pipe( - Effect.mapError( - (cause) => - new EventSink.EventSinkWriteError({ - eventCount: events.length, - cause, - }), - ), - ), + eventSink + .write({ events }) + .pipe(Effect.tap(() => Effect.sync(() => committed.push(...events)))), }), Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open: (input) => @@ -837,9 +965,123 @@ const seedRollbackHistory = Effect.fn(function* ( }; }); -it.layer(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)))( +const layerRollbackHistory = EventSink.layer.pipe( + Layer.provideMerge(Layer.mergeAll(EventStore.layer, ProjectionStore.layer)), +); +it.layer(layerRollbackHistory.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)))( "rollback history reads", (it) => { + it.effect.each(["commit", "ref cleanup"] as const)( + "preserves recovery checkpoints when rollback %s fails", + (failure) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sql = yield* SqlClient.SqlClient; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-rollback-refs-" }); + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const checkpointRef = (ordinal: number) => + CheckpointRef.make(`refs/t3/rollback-history/${ordinal}`); + yield* driver.execute({ operation: "test.init", cwd, args: ["init"] }); + yield* fs.writeFileString(path.join(cwd, "work.txt"), "rollback target"); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: checkpointRef(2) }); + yield* fs.writeFileString(path.join(cwd, "work.txt"), "later saved work"); + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef: checkpointRef(6) }); + for (const ordinal of [3, 4, 5]) { + yield* driver.execute({ + operation: "test.create-checkpoint-ref", + cwd, + args: ["update-ref", checkpointRef(ordinal), checkpointRef(6)], + }); + } + const history = yield* seedRollbackHistory({ + cwd, + checkpoints: { + restore: ({ scope, checkpoint }) => + driver.checkpoints + .restoreCheckpoint({ cwd: scope.cwd, checkpointRef: checkpoint.ref }) + .pipe( + Effect.asVoid, + Effect.mapError( + (cause) => + new CheckpointService.CheckpointRestoreError({ + scopeId: scope.id, + checkpointId: checkpoint.id, + cause, + }), + ), + ), + deleteStaleRefs: ({ scope, checkpoints }) => + driver.checkpoints + .deleteCheckpointRefs({ + cwd: scope.cwd, + checkpointRefs: checkpoints.map((checkpoint) => checkpoint.ref), + }) + .pipe( + Effect.mapError( + (cause) => + new CheckpointService.CheckpointDeleteStaleRefsError({ + scopeId: scope.id, + checkpointIds: checkpoints.map((checkpoint) => checkpoint.id), + cause, + }), + ), + ), + }, + }); + if (failure === "commit") { + yield* sql`CREATE TEMP TRIGGER fail_rollback_commit + BEFORE UPDATE ON orchestration_v2_projection_checkpoints + WHEN NEW.status = 'stale' + BEGIN SELECT RAISE(ABORT, 'rollback storage unavailable'); END`; + yield* Effect.addFinalizer(() => + sql`DROP TRIGGER IF EXISTS fail_rollback_commit`.pipe(Effect.orDie), + ); + } else { + yield* fs.writeFileString(path.join(cwd, ".git", `${checkpointRef(6)}.lock`), "held"); + } + + const result = yield* Effect.exit(history.rollback(2)); + const checkpoints = yield* sql<{ status: string }>` + SELECT status FROM orchestration_v2_projection_checkpoints + WHERE thread_id = ${history.threadId} AND ordinal_within_scope BETWEEN 3 AND 6 + ORDER BY ordinal_within_scope`; + assert.deepEqual( + checkpoints.map((checkpoint) => checkpoint.status), + Array.from({ length: 4 }, () => (failure === "commit" ? "ready" : "stale")), + ); + assert.equal(result._tag, failure === "commit" ? "Failure" : "Success"); + assert.isTrue( + yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef: checkpointRef(6) }), + ); + assert.equal(yield* fs.readFileString(path.join(cwd, "work.txt")), "rollback target"); + if (failure === "commit") { + assert.deepEqual( + yield* sql`SELECT event_type FROM orchestration_events WHERE stream_id = ${history.threadId}`, + [], + ); + yield* driver.checkpoints.restoreCheckpoint({ cwd, checkpointRef: checkpointRef(6) }); + assert.equal(yield* fs.readFileString(path.join(cwd, "work.txt")), "later saved work"); + yield* sql`DROP TRIGGER fail_rollback_commit`; + yield* history.rollback(2); + assert.isFalse( + yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef: checkpointRef(6) }), + ); + } else { + assert.isFalse( + yield* driver.checkpoints.hasCheckpointRef({ cwd, checkpointRef: checkpointRef(3) }), + ); + } + }).pipe( + Effect.scoped, + Effect.provide( + VcsProcess.layer.pipe( + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + it.effect("rolls back later terminal runs despite obsolete unrelated SQLite payloads", () => Effect.gen(function* () { const history = yield* seedRollbackHistory(); diff --git a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts index 14c5b904db556..dcbede160c148 100644 --- a/apps/server/src/orchestration-v2/CheckpointRollbackService.ts +++ b/apps/server/src/orchestration-v2/CheckpointRollbackService.ts @@ -239,6 +239,7 @@ export const layer: Layer.Layer< }); const { worktreePath, branch } = context.thread; + let rollbackCommitted = false; // Recreated only where the server would have made it, as turn start does. if (worktreePath != null && branch != null) { // A failed removal can leave the folder without its checkout; only a file restore replaces it. @@ -263,8 +264,8 @@ export const layer: Layer.Layer< ); if (leftover) { if (!(yield* isolated)) return yield* sharedWorkspace(); - // Set aside, not removed: a checkout that fails puts the folder and its files back. - yield* Effect.acquireUseRelease( + // Keep the original folder until the completed rollback is recorded. + yield* Effect.acquireRelease( Effect.gen(function* () { const aside = yield* fileSystem.makeTempDirectory({ directory: path.dirname(worktreePath), @@ -273,9 +274,8 @@ export const layer: Layer.Layer< yield* fileSystem.rename(worktreePath, path.join(aside, "folder")); return aside; }), - () => recreate, (aside, exit) => - (Exit.isSuccess(exit) + (Exit.isSuccess(exit) || rollbackCommitted ? Effect.void : fileSystem .remove(worktreePath, { recursive: true, force: true }) @@ -295,6 +295,7 @@ export const layer: Layer.Layer< ), ), ); + yield* recreate; } else { yield* recreate; } @@ -369,9 +370,6 @@ export const layer: Layer.Layer< providerThreadTurns, }); if (input.restoreFiles !== false) yield* checkpoints.restore({ scope, checkpoint }); - if (staleCheckpoints.length > 0) { - yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }); - } // Lock against full-row thread writes that could restore the old native session after rollback. yield* threadCommands.withLock( @@ -442,10 +440,30 @@ export const layer: Layer.Layer< ); } } - yield* eventSink.write({ events }); + // Cancellation after the durable commit must not restore the old folder. + yield* eventSink.write({ events }).pipe( + Effect.tap(() => + Effect.sync(() => { + rollbackCommitted = true; + }), + ), + Effect.uninterruptible, + ); + // Ready checkpoints must remain recoverable if the rollback transaction fails. + if (staleCheckpoints.length > 0) { + yield* checkpoints.deleteStaleRefs({ scope, checkpoints: staleCheckpoints }).pipe( + Effect.catch((cause) => + Effect.logWarning("Rollback left stale checkpoint refs", { + threadId: input.threadId, + scopeId: scope.id, + cause, + }), + ), + ); + } }), ); - }); + }, Effect.scoped); return CheckpointRollbackServiceV2.of({ // The inner execute owns the rollback span; this wrapper selects the workspace lease. diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 2be2df6527b2b..3fcbce75b5432 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1406,6 +1406,94 @@ if (context.update?.status === "pending") { }), ); + it.effect.each([false, true])( + "preserves the live database when a backup has lost its main file, restore pending %s", + (restoring) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-partial-backup-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const backupDir = path.join(root, "runtime", "db-backup", "partial-update"); + const trialStarted = path.join(root, "trial-started"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + { + using database = new NodeSqlite.DatabaseSync(databasePath); + database.exec("CREATE TABLE saved (value TEXT); INSERT INTO saved VALUES ('keep');"); + } + // A killed recursive discard can leave its directory after deleting the backup file. + yield* fs.makeDirectory(backupDir, { recursive: true }); + if (restoring) yield* fs.writeFileString(path.join(backupDir, ".restore-pending"), ""); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + 'process.send("active"); setInterval(() => {}, 1_000);', + ); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.1.0"), + `require("node:fs").writeFileSync(${JSON.stringify(trialStarted)}, ""); process.exit(1);`, + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "partial-update", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: databasePath, + status: "pending", + }, + }), + ); + const active = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => { + spawnHook.current = (child) => + child.on("message", (message) => message === "active" && active.resolve()); + }), + () => + Effect.sync(() => { + spawnHook.current = undefined; + }), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + const running = launcher.run().then( + () => undefined, + (error: unknown) => error, + ); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + await launcher.stop("SIGTERM"); + await running; + }), + ); + const result = yield* Effect.promise(() => Promise.race([running, active.promise])); + + assert.isTrue(yield* fs.exists(databasePath)); + using restored = new NodeSqlite.DatabaseSync(databasePath, { readOnly: true }); + assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok"); + assert.deepEqual(restored.prepare("SELECT value FROM saved").all(), [{ value: "keep" }]); + assert.isFalse(yield* fs.exists(trialStarted)); + assert.include( + result instanceof Error ? result.message : "", + "Database backup is incomplete", + ); + assert.equal( + (yield* Effect.promise(() => readServiceState(statePath))).update?.status, + "pending", + ); + assert.isTrue(yield* fs.exists(backupDir)); + }), + ); + it.effect.each(["file", "symlink", "dangling symlink"] as const)( "restores the database when a migrating trial exits, with the database a %s", (kind) => @@ -1485,9 +1573,13 @@ if (context.update?.status === "pending") { }), ); - it.effect.each(["wal", "delete"] as const)( - "backs up a %s-mode database (online when WAL) and restores its final state", - (journalMode) => + it.effect.each([ + { journalMode: "wal", canClone: false }, + { journalMode: "delete", canClone: false }, + { journalMode: "wal", canClone: true }, + ] as const)( + "backs up a $journalMode-mode database with clone support $canClone and restores its final state", + ({ journalMode, canClone }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -1535,6 +1627,7 @@ if (context.update?.status === "pending") { backupsAtStop: readdirSync(backupRoot).map((entry) => readdirSync(backupRoot + "/" + entry)), rows: db.prepare("SELECT count(*) AS n FROM events").get().n, })); + ${canClone ? 'process.kill(process.pid, "SIGKILL");' : ""} db.close(); process.exit(0); }); @@ -1562,7 +1655,7 @@ if (context.update?.status === "pending") { root, yield* Effect.promise(() => readServiceState(statePath)), { - backupSpaceProbe: { canClone: async () => false }, + backupSpaceProbe: { canClone: async () => canClone }, }, ); yield* Effect.promise(() => @@ -1576,13 +1669,9 @@ if (context.update?.status === "pending") { assert.equal(state.update?.status, "rolled-back"); const observed = JSON.parse(yield* fs.readFileString(observationsPath)); // A WAL database's main file is copied before the server is asked to stop; others fall back offline. - assert.deepEqual(observed.backupsAtStop, journalMode === "wal" ? [["database"]] : []); + const online = journalMode === "wal" && !canClone; + assert.deepEqual(observed.backupsAtStop, online ? [["database"]] : []); assert.isAbove(observed.rows, 501); - // Only the online copy holds no -shm: the offline fallback copies the one the server left. - assert.deepEqual( - JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)), - journalMode === "wal" ? ["database", "database-wal"] : ["database"], - ); const restored = new NodeSqlite.DatabaseSync(databasePath); try { assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok"); @@ -1601,6 +1690,15 @@ if (context.update?.status === "pending") { } finally { restored.close(); } + // Only the online copy holds no -shm: the offline fallback copies the one the server left. + assert.deepEqual( + JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)), + online + ? ["database", "database-wal"] + : journalMode === "wal" + ? ["database", "database-shm", "database-wal"] + : ["database"], + ); }), ); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 77453fe8d6bcd..94c937bdcf286 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -282,6 +282,14 @@ async function pathExists(target: string): Promise { } } +async function databaseBackupExists(backupDir: string): Promise { + if (!(await pathExists(backupDir))) return false; + if (!(await pathExists(databaseBackupFile(backupDir, "")))) { + throw new Error(`Database backup is incomplete: ${backupDir} has no main database file.`); + } + return true; +} + const MANAGED_SERVICE_ENV_NAMES = new Set([ "PATH", "T3CODE_HOME", @@ -371,9 +379,10 @@ const CLONED_BACKUP_HEADROOM_FRACTION = 0.1; const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2; async function statDatabaseFiles(dbPath: string): Promise> { + const databasePath = await NodeFSP.realpath(dbPath); const stats: Array = []; for (const suffix of DB_FILE_SUFFIXES) { - const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => { + const stat = await NodeFSP.stat(`${databasePath}${suffix}`).catch((cause: unknown) => { if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw cause; }); @@ -400,8 +409,9 @@ async function backupCanClone(dbPath: string, backupRoot: string): Promise = {}, ): Promise<(() => Promise) | undefined> { const backupDir = databaseBackupDir(baseDir, pending.id); - if (await pathExists(backupDir)) return undefined; + if (await databaseBackupExists(backupDir)) return undefined; const { canClone } = { ...filesystemBackupSpaceProbe, ...probe }; if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true)) return undefined; @@ -546,15 +556,16 @@ async function backupDatabaseOnce( signal: AbortSignal, ): Promise { const backupDir = databaseBackupDir(baseDir, pending.id); - if (await pathExists(backupDir)) return; + if (await databaseBackupExists(backupDir)) return; const stagingDir = `${backupDir}.staging`; await NodeFSP.rm(stagingDir, { recursive: true, force: true }); await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 }); try { + const databasePath = await NodeFSP.realpath(pending.dbPath); for (const suffix of DB_FILE_SUFFIXES) { signal.throwIfAborted(); - const source = `${pending.dbPath}${suffix}`; + const source = `${databasePath}${suffix}`; if (suffix !== "" && !(await pathExists(source))) continue; const destination = databaseBackupFile(stagingDir, suffix); await copyDatabaseFile(source, destination, signal); @@ -616,7 +627,7 @@ async function restoreDatabaseBackup( pending: PendingServiceUpdate, ): Promise { const backupDir = databaseBackupDir(baseDir, pending.id); - if (!(await pathExists(backupDir))) return; + if (!(await databaseBackupExists(backupDir))) return; await markDatabaseRestorePending(backupDir); const directories = new Set([NodePath.dirname(pending.dbPath)]); diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts index 977a9078b9b01..a4875c501b904 100644 --- a/apps/server/src/vcs/GitVcsDriver.test.ts +++ b/apps/server/src/vcs/GitVcsDriver.test.ts @@ -2027,6 +2027,35 @@ it.effect("sparse restore rewrites manual skip-worktree files and keeps exclusio }).pipe(Effect.scoped, Effect.provide(layerGitContract)), ); +it.effect.each(["capture", "delete"] as const)( + "checkpoint %s changes only the named ref when it is symbolic", + (operation) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const driver = yield* GitVcsDriver.makeVcsDriverShape(); + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-checkpoint-symbolic-" }); + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd); + const branchRef = (yield* git(["symbolic-ref", "HEAD"])).stdout.trim(); + const originalHead = (yield* git(["rev-parse", "HEAD"])).stdout; + yield* git(["symbolic-ref", checkpointRef, branchRef]); + + if (operation === "capture") + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef }); + else yield* driver.checkpoints.deleteCheckpointRefs({ cwd, checkpointRefs: [checkpointRef] }); + + const branch = yield* driver.execute({ + operation: "checkpoint-test.read-branch", + cwd, + args: ["rev-parse", "--verify", branchRef], + allowNonZeroExit: true, + }); + assert.equal(branch.stdout, originalHead); + assert.equal(branch.exitCode, 0); + const refs = (yield* git(["for-each-ref", "--format=%(refname)", "refs/t3/"])).stdout; + assert.equal(refs, operation === "capture" ? `${checkpointRef}\n` : ""); + }).pipe(Effect.scoped, Effect.provide(layerGitContract)), +); + it.effect("restores and deletes checkpoints inside populated submodules", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts index d7b72ff4318e2..e1417ade17e47 100644 --- a/apps/server/src/vcs/GitVcsDriver.ts +++ b/apps/server/src/vcs/GitVcsDriver.ts @@ -1762,7 +1762,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( yield* execute({ operation, cwd: input.cwd, - args: [...durableWrite, "update-ref", input.checkpointRef, commitOid], + args: [...durableWrite, "update-ref", "--no-deref", input.checkpointRef, commitOid], }); if (untracked !== undefined) { yield* untrackedBlobs.remember({ @@ -2264,7 +2264,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* ( execute({ operation: "GitVcsDriver.checkpoints.deleteCheckpointRefs", cwd: input.cwd, - args: ["update-ref", "-d", checkpointRef], + args: ["update-ref", "--no-deref", "-d", checkpointRef], allowNonZeroExit: true, }), { discard: true },