From e5b4b91fec49ea1252c4c0a3e5387fc61a86ed44 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Wed, 18 Nov 2026 07:33:33 +0200 Subject: [PATCH 1/3] ci(scient): quarantine exactly matched flaky Oh My Pi test failures A row of scripts/scient/ci-quarantine.json names one test and a signature from the failure's own output; a matching failure passes the bucket with a warning and a job-summary line until the row expires, 7 days after it was quarantined. A release run takes no rows. KNOWN_FAILURES is unchanged. First row: browser-open-options downloads (a stray .crdownload in the second tab's folder), seen on main 11-16 and PR #70 on 11-17. Co-Authored-By: Claude Opus 5.5 --- SCIENT.md | 16 +++++ scripts/scient/ci-quarantine.json | 13 ++++ scripts/scient/upstream-tests.test.ts | 69 +++++++++++++++++++- scripts/scient/upstream-tests.ts | 90 ++++++++++++++++++++++++--- 4 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 scripts/scient/ci-quarantine.json diff --git a/SCIENT.md b/SCIENT.md index c4e799d480..30f2d7ca47 100644 --- a/SCIENT.md +++ b/SCIENT.md @@ -1060,6 +1060,22 @@ Scient's own tests live in `test/scient/` folders and run on macOS with the plat The macOS native test process preloads `scripts/scient/macos-spelling-ready.ts`. AppKit can return no spans when its first XPC request times out during spelling-service startup; the preflight requires the service to identify a known misspelling within three probes before running the unchanged tests. Persistent unavailability fails qualification. This provisions the CI fixture; it does not retry failed tests, change runtime results, or qualify cold-service startup behavior. +### Flake quarantine + +`KNOWN_FAILURES` is for tests that fail on untouched Oh My Pi. A flaky test that fails on Scient's tree, on content that also passed, can be quarantined instead. The rule is the one Scient Desktop's `Check` follows (owner decision, 2026-11-17). + +- **Exact match.** A row names one test and a signature: text the failure's own output must contain, between the test before it and its `(fail)` line. The same test failing another way still fails the job, and so does everything `KNOWN_FAILURES` never excuses (a crash, an error outside any test, a report that does not add up). +- **Still visible.** The test still runs. Each quarantined failure is printed as a warning and listed in the job summary with its row. +- **Expiry.** Every row has an owner and expires 7 days after it was quarantined. From the next day (UTC) its failures fail the job again, unless the cause is fixed (the fix removes the row) or the owner extends it. +- **Releases.** A release run (`release: true`) takes no rows, so a release ships only from a run without a quarantined failure. +- **The rows** are in `scripts/scient/ci-quarantine.json`; each row's id and expiry must also be in the table below (a test checks this). The daily CI care task adds each occurrence (date, run link, running count) to its row. + +| Row | Test | Signature | Owner | Quarantined → expires | Occurrences (running count) | Cause | +| --- | --- | --- | --- | --- | --- | --- | +| `browser-open-options-downloads` | `coding-agent` `browser-open-options.test.ts` › saves each tab's downloads into its own downloads directory, also after another tab closes | `.crdownload",` in the failed `readdir(second)` diff | Browser tool | 2026-11-18 → 2026-11-25 | 11-16 [main](https://github.com/ScientFactory/scient-agent/actions/runs/43954563060) (1); 11-17 [PR #70](https://github.com/ScientFactory/scient-agent/actions/runs/44078095729) (2) | Open. Upstream's test and code, unchanged by Scient. In 30 local runs Chrome's own download never left a `.crdownload` behind at completion, but Chrome also saved downloads that no page started (no `downloadWillBegin`) into the folder it was pointed at last, the second tab's here. Not yet reproduced in the test itself. | + +The other two browser tests that failed `main` on 2026-11-16 need no row. "does not call a cancelled run's page listeners" (`browser-nav-frames-dialogs.test.ts`) is in `KNOWN_FAILURES` for 18.22.0. The relay default-tab test (`browser-relay-tab-isolation`) was fixed by #75 and has not failed since. + ## Definition of done Report work as done only when all of these hold on the final head. Without a pull request, the handback carries what the pull request would. diff --git a/scripts/scient/ci-quarantine.json b/scripts/scient/ci-quarantine.json new file mode 100644 index 0000000000..1c4bb2cd7e --- /dev/null +++ b/scripts/scient/ci-quarantine.json @@ -0,0 +1,13 @@ +{ + "schemaVersion": 1, + "rows": [ + { + "id": "browser-open-options-downloads", + "test": "packages/coding-agent/test/tools/browser-open-options.test.ts > browser open options > saves each tab's downloads into its own downloads directory, also after another tab closes", + "signature": ".crdownload\",", + "owner": "Browser tool", + "quarantined": "2026-11-18", + "expires": "2026-11-25" + } + ] +} diff --git a/scripts/scient/upstream-tests.test.ts b/scripts/scient/upstream-tests.test.ts index b7e9482dae..abb27c51bc 100644 --- a/scripts/scient/upstream-tests.test.ts +++ b/scripts/scient/upstream-tests.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "bun:test"; -import { judge } from "./upstream-tests"; +import * as path from "node:path"; +import { judge, type Quarantine, type QuarantineRow } from "./upstream-tests"; // Shaped as `scripts/ci-test-ts.ts --full` prints a failed run, with // Bun 1.4's own output inside each chunk. @@ -44,6 +45,7 @@ describe("a failed bucket", () => { it("passes when every failure is a known upstream failure", () => { expect(judge(1, run([knownOnly]), KNOWN)).toEqual({ known: [`packages/coding-agent/${FILE} > ${KNOWN_NAME}`], + quarantined: [], problems: [], }); }); @@ -111,3 +113,68 @@ describe("a failed bucket", () => { expect(judge(2, run([knownOnly]), KNOWN).problems).toEqual(["the runner exited 2, not with failed tests"]); }); }); + +describe("the flake quarantine", () => { + const FLAKY = "saves each tab's downloads"; + const row: QuarantineRow = { + id: "downloads", + test: `packages/coding-agent/${FILE} > ${FLAKY}`, + signature: '.crdownload",', + owner: "Browser tool", + quarantined: "2026-11-18", + expires: "2026-11-25", + }; + const strayFile = ["error: expect(received).toEqual(expected)", "", "- []", "+ [", '+ "0a1d.crdownload",', "+ ]"]; + const flaky = (error: string[]) => + chunk(LABEL, 1, [ + "", + `${FILE}:`, + "(pass) an earlier test [1.00ms]", + ...error, + `(fail) ${FLAKY} [2199.58ms]`, + ...summary(110, 1), + ]); + const on = (today: string): Quarantine => ({ rows: [row], today }); + + it("lets a failure through when it shows the row's signature, and reports it", () => { + expect(judge(1, run([flaky(strayFile)]), new Set(), on("2026-11-25"))).toEqual({ + known: [], + quarantined: [{ test: row.test, row: "downloads", owner: "Browser tool", expires: "2026-11-25" }], + problems: [], + }); + }); + + it("blocks the same test failing another way", () => { + const timeout = ["error: Test timed out after 30000ms"]; + expect(judge(1, run([flaky(timeout)]), new Set(), on("2026-11-20")).problems).toEqual([`failed: ${row.test}`]); + }); + + it("reads the signature only from the failing test's own output", () => { + const body = ["", `${FILE}:`, ...strayFile, "(pass) an earlier test [1.00ms]", `(fail) ${FLAKY} [2.00ms]`]; + const judged = judge(1, run([chunk(LABEL, 1, [...body, ...summary(110, 1)])]), new Set(), on("2026-11-20")); + expect(judged.problems).toEqual([`failed: ${row.test}`]); + }); + + it("blocks again from the day after the row expires, and without rows", () => { + expect(judge(1, run([flaky(strayFile)]), new Set(), on("2026-11-26")).problems).toEqual([`failed: ${row.test}`]); + expect(judge(1, run([flaky(strayFile)]), new Set()).problems).toEqual([`failed: ${row.test}`]); + }); + + it("keeps every row exact, a week long and documented in SCIENT.md", async () => { + const root = path.resolve(import.meta.dir, "..", ".."); + const register = await Bun.file(path.join(import.meta.dir, "ci-quarantine.json")).json(); + const scientMd = await Bun.file(path.join(root, "SCIENT.md")).text(); + const ids = new Set(); + for (const entry of register.rows as QuarantineRow[]) { + expect(ids.has(entry.id)).toBe(false); + ids.add(entry.id); + expect(entry.test).toMatch(/^packages\/[^ ]+\.test\.[cm]?[jt]sx? > .+ > .+/); + expect(entry.signature.trim().length).toBeGreaterThan(5); + expect(entry.owner.length).toBeGreaterThan(0); + const week = new Date(`${entry.quarantined}T00:00:00Z`).getTime() + 7 * 86_400_000; + expect(entry.expires <= new Date(week).toISOString().slice(0, 10)).toBe(true); + expect(scientMd).toContain(`| \`${entry.id}\``); + expect(scientMd).toContain(entry.expires); + } + }); +}); diff --git a/scripts/scient/upstream-tests.ts b/scripts/scient/upstream-tests.ts index 45cf6af940..f3869f150d 100644 --- a/scripts/scient/upstream-tests.ts +++ b/scripts/scient/upstream-tests.ts @@ -7,7 +7,9 @@ * A failed bucket passes only when every failure is one `KNOWN_FAILURES` lists * for the exact Oh My Pi commit `scient.json` names: a test that fails on that * commit untouched, on the runner this workflow uses. The entries lapse at the - * next alignment, which checks them again. Anything else fails the run: an + * next alignment, which checks them again. A flaky test can also pass while an + * unexpired row of `ci-quarantine.json` names it and its failure shows the row's + * signature; a release run takes no quarantine. Anything else fails the run: an * unlisted test, a chunk that crashed or was killed, an error outside any test, * and a report this cannot reconcile in full. * @@ -18,6 +20,7 @@ * * bun scripts/scient/upstream-tests.ts */ +import * as fs from "node:fs/promises"; import * as path from "node:path"; /** A test that fails on untouched Oh My Pi at `commit`, with the evidence. */ @@ -53,13 +56,46 @@ const KNOWN_FAILURES: KnownFailure[] = [ }, ]; +/** + * A flaky test whose failure does not fail the run until `expires`, a week after `quarantined`. Only a failure whose + * own output contains `signature` matches, so the same test failing another way still blocks. + */ +export interface QuarantineRow { + id: string; + /** `/ > > `. */ + test: string; + signature: string; + owner: string; + /** `YYYY-MM-DD`, UTC. */ + quarantined: string; + /** `YYYY-MM-DD`, UTC; the row blocks again from the next day. */ + expires: string; +} + +/** The rows in force and today's UTC date; a release run passes no rows. */ +export interface Quarantine { + rows: readonly QuarantineRow[]; + today: string; +} + +/** A failure a quarantine row let through. */ +export interface QuarantinedFailure { + test: string; + row: string; + owner: string; + expires: string; +} + /** Why a bucket's failure is or is not acceptable. */ export interface Judgement { /** Known failures that occurred; the run passes when `problems` is empty. */ known: string[]; + quarantined: QuarantinedFailure[]; problems: string[]; } +const NO_QUARANTINE: Quarantine = { rows: [], today: "" }; + const CHUNK_HEADER = /^==> \[\d+\/\d+\] (.+) \((?:ok|FAILED exit (\d+)), [\d.]+s\)$/; const FAILED_FOOTER = /^(\d+) of \d+ test chunk\(s\) FAILED$/; @@ -68,9 +104,15 @@ const FAILED_FOOTER = /^(\d+) of \d+ test chunk\(s\) FAILED$/; * output: a `==> [i/n]