Skip to content

test: align eval host startup and cleanup deadlines #64

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vriesd merged 1 commit into from
Sep 6, 2026
Merged

test: align eval host startup and cleanup deadlines #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions tests/eval-host-config.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ test("eval reviewer configuration keeps provenance and tuple options aligned", (

test("eval host writes reviewer model through native plugin tuple options", async () => {
const repositoryRoot = join(import.meta.dir, "..");
const scratch = await mkdtemp(join(tmpdir(), "flow-eval-host-config-test-"));
const toolchain = currentBunToolchain(packageJson.packageManager);
const scratch = await mkdtemp(join(tmpdir(), "flow-eval-host-config-test-"));
const previous = process.env.FLOW_EVAL_NO_AUTH_COPY;
process.env.FLOW_EVAL_NO_AUTH_COPY = "1";
let host: EvalHost | null = null;
Expand All @@ -41,6 +41,8 @@ test("eval host writes reviewer model through native plugin tuple options", asyn
opencodeVersion: packageJson.devDependencies["@opencode-ai/plugin"],
files: { "package.json": '{"name":"eval-host-config-test"}\n' },
reviewer: { model: "provider/reviewer", steps: 80 },
// Let startup own cancellation/cleanup before the outer test expires.
signal: AbortSignal.timeout(180_000),

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 6, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve the host's full startup window

On a slow filesystem, this signal can cancel an otherwise valid startup early: its 180-second clock begins when EvalHost.start is invoked, but the harness does not begin its own STARTUP_TIMEOUT_MS deadline until after credential copying, fixture creation, synchronous Git commands, package-cache copying, and process spawning. Consequently, that setup time is subtracted from the host's intended 180-second health/readiness allowance, making this integration test flaky in environments where startup legitimately approaches the internal deadline; give the cancellation signal additional headroom while keeping it below the 240-second outer timeout.

Useful? React with 👍 / 👎.

});

expect(
Expand All @@ -55,9 +57,13 @@ test("eval host writes reviewer model through native plugin tuple options", asyn
],
});
} finally {
await host?.stop();
if (previous === undefined) delete process.env.FLOW_EVAL_NO_AUTH_COPY;
else process.env.FLOW_EVAL_NO_AUTH_COPY = previous;
await rm(scratch, { recursive: true, force: true });
try {
await host?.stop();
} finally {
if (previous === undefined) delete process.env.FLOW_EVAL_NO_AUTH_COPY;
else process.env.FLOW_EVAL_NO_AUTH_COPY = previous;
await rm(scratch, { recursive: true, force: true });
}
}
}, 30_000);
// The host already permits 180s startup; allow packaging and cleanup as well.
}, 240_000);
30 changes: 24 additions & 6 deletions tests/eval-reporting.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1187,7 +1187,12 @@ describe("eval campaign cancellation", () => {
});
}

for (const phase of ["health", "cache", "cache-failure"] as const) {
for (const phase of [
"health",
"readiness",
"cache",
"cache-failure",
] as const) {
test(`cleans startup scratch and temporary credentials on ${phase}`, async () => {
const fixture = await mkdtemp(join(tmpdir(), "flow-eval-start-test-"));
const source = join(fixture, "opencode", "auth.json");
Expand All @@ -1201,18 +1206,27 @@ describe("eval campaign cancellation", () => {
process.env.XDG_DATA_HOME = fixture;
delete process.env.FLOW_EVAL_NO_AUTH_COPY;
const controller = new AbortController();
const reason = new CampaignCancelled(130);
const reason =
phase === "readiness"
? new DOMException("Startup deadline expired", "TimeoutError")
: new CampaignCancelled(130);
let scratch = "";
let healthSignal: AbortSignal | null | undefined;
let requestSignal: AbortSignal | null | undefined;
const stop = EvalHost.prototype.stop;
const stopping = spyOn(EvalHost.prototype, "stop").mockImplementation(
function (this: EvalHost) {
scratch = join(this.project, "..");
return stop.call(this);
},
);
const requests = mockFetch(async (_input, init) => {
healthSignal = init?.signal;
const requests = mockFetch(async (input, init) => {
if (phase === "readiness" && String(input).endsWith("/global/health"))
return Response.json({ healthy: true });
if (phase === "readiness") {
expect(String(input)).toEndWith("/session");
expect(init?.method).toBe("POST");
}
requestSignal = init?.signal;
queueMicrotask(() => controller.abort(reason));
return new Promise(() => {});
});
Expand All @@ -1236,12 +1250,16 @@ describe("eval campaign cancellation", () => {
signal: controller.signal,
});
if (phase === "cache-failure") await expect(starting).rejects.toThrow();
else if (phase === "readiness")
await expect(starting).rejects.toThrow("Startup deadline expired");
else await expect(starting).rejects.toBe(reason);
expect(scratch).not.toBe("");
await expect(readdir(scratch)).rejects.toThrow();
expect(await readFile(source, "utf8")).toBe(credentials);
if (phase === "health") expect(healthSignal?.aborted).toBe(true);
if (phase === "health" || phase === "readiness")
expect(requestSignal?.aborted).toBe(true);
else expect(requests).not.toHaveBeenCalled();
if (phase === "readiness") expect(requests).toHaveBeenCalledTimes(2);
} finally {
requests.mockRestore();
stopping.mockRestore();
Expand Down