Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions apps/server/src/vcs/GitVcsDriver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,41 @@ it.effect.each(["interrupt", "timeout", "success"] as const)(
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
);

it.effect.each(["darwin", "win32"] as const)(
"checkpoint on $0 captures a big file staged after small new files",
(platform) =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const driver = yield* GitVcsDriver.makeVcsDriverShape();
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-big-file-" });
const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
yield* git(["config", "core.bigFileThreshold", "1m"]);
yield* fs.writeFileString(path.join(cwd, "a-note.txt"), "small\n");
const crypto = yield* SecureRandom.SecureRandom;
yield* fs.writeFile(
path.join(cwd, "b-media.bin"),
yield* crypto.randomBytes(2 * 1024 * 1024),
);
// More embedded repositories than staging's recovery scans, so a failed add is final.
const nested = path.join(cwd, "nested-0");
yield* fs.makeDirectory(nested);
yield* makeCheckpointFixture(driver, nested);
for (let index = 1; index <= 64; index++)
yield* fs.copy(nested, path.join(cwd, `nested-${index}`));
const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
Effect.provideService(HostProcess.Platform, platform),
);

yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });

assert.strictEqual(
(yield* git(["rev-parse", `${checkpointRef}:b-media.bin`])).stdout.trim(),
(yield* git(["hash-object", "b-media.bin"])).stdout.trim(),
);
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
);

it.effect.each(["failure", "interrupt"] as const)(
"checkpoint publication $0 preserves existing refs and permits retry",
(stop) =>
Expand Down
25 changes: 18 additions & 7 deletions apps/server/src/vcs/GitVcsDriver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1035,14 +1035,16 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
// Set fsync first: Git 2.36 ignores unknown batch without resetting the method.
// Batch fully flushes refs and equals fsync on macOS HFS+/APFS and Windows NTFS/ReFS.
// Keep per-object fsync elsewhere; VcsProcess retries unsupported macOS full flushes with plain fsync.
const durableWrite = [
const perObjectFsync = [
"-c",
"core.fsync=objects,reference",
"-c",
"core.fsyncMethod=fsync",
...(hostPlatform === "darwin" || hostPlatform === "win32"
? ["-c", "core.fsyncMethod=batch"]
: []),
] as const;
const batchFsync = hostPlatform === "darwin" || hostPlatform === "win32";
const durableWrite = [
...perObjectFsync,
...(batchFsync ? ["-c", "core.fsyncMethod=batch"] : []),
] as const;

// Checkpoints must handle existing paths beyond Windows MAX_PATH, even when
Expand Down Expand Up @@ -1433,13 +1435,17 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
? CheckpointUntrackedBlobs.NOTHING
: yield* untrackedBlobs.reuse({ ...untracked, indexConfig });

const stageFiles = (exclusions: ReadonlyArray<string>, ignoreErrors = false) =>
const stageFiles = (
exclusions: ReadonlyArray<string>,
ignoreErrors = false,
fsync: ReadonlyArray<string> = durableWrite,
) =>
execute({
operation,
cwd: input.cwd,
args: [
...indexConfig,
...durableWrite,
...fsync,
...longPaths,
// safecrlf=true refuses LF files under CRLF conversion; capture them as Git's default does.
"-c",
Expand All @@ -1465,6 +1471,11 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
);
const stagingAt = yield* Clock.currentTimeMillis;
yield* stageFiles([]).pipe(
// Git 2.43-2.56.1 under batch fsync loses a big file's pack when a loose object precedes it.
Effect.catchTags({
VcsProcessExitError: (error) =>
batchFsync ? stageFiles([], false, perObjectFsync) : Effect.fail(error),
}),
Effect.catchTags({
VcsProcessExitError: (error) =>
Effect.gen(function* () {
Expand Down Expand Up @@ -1511,7 +1522,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
exclusions.push(`:(exclude,literal)${entry}`);
}
}
return yield* stageFiles(exclusions, true);
return yield* stageFiles(exclusions, true, perObjectFsync);
}).pipe(
Effect.timeoutOrElse({
duration: CHECKPOINT_RECOVERY_TIMEOUT,
Expand Down
Loading