From 6487edf8b097e4a82d0dc4b1eff82e6dba4f62fa Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Sun, 6 Sep 2026 00:39:18 +0200 Subject: [PATCH] fix: distinguish temporary homes in qualification evidence --- evals/qualification-bundle.ts | 7 +- tests/qualification-bundle.test.ts | 149 ++++++++++++++++++++++++++++- 2 files changed, 153 insertions(+), 3 deletions(-) diff --git a/evals/qualification-bundle.ts b/evals/qualification-bundle.ts index 7c30f13a..cd735a56 100644 --- a/evals/qualification-bundle.ts +++ b/evals/qualification-bundle.ts @@ -165,8 +165,13 @@ function assertSafeText(text: string): void { throw new Error("Qualification bundle contains secret-shaped evidence."); if (/\b(?:ses_[A-Za-z0-9]+|(?:session|review):[A-Za-z0-9-]+)\b/.test(text)) throw new Error("Qualification bundle contains a raw session identifier."); + // Match Unix homes at path starts, not beneath another root such as /tmp. + // Decode escaped slashes for inspection only; retained bytes stay untouched. + const paths = text.replace(/\\+\//g, "/"); if ( - /(?:\/Users\/[^/\s]+|\/home\/[^/\s]+|[A-Za-z]:\\Users\\[^\\\s]+)/.test(text) + /(?:(?:^|[\s"'`=():,;<>[\]{}!?|]|\\[nrtbf]|file:\/\/[^/\s"'<>]*)[*_~]*\/+(?:Users|home)\/[^/\s]+|[A-Za-z]:\\+Users\\+[^\\\s]+)/.test( + paths, + ) ) throw new Error("Qualification bundle contains an absolute user path."); } diff --git a/tests/qualification-bundle.test.ts b/tests/qualification-bundle.test.ts index a6a4ecba..044549cc 100644 --- a/tests/qualification-bundle.test.ts +++ b/tests/qualification-bundle.test.ts @@ -29,6 +29,8 @@ afterEach(async () => { }); const json = (value: unknown) => Buffer.from(canonicalJson(value)); +const temporaryToolOutput = + "/tmp/flow-eval-Vj7Io4/home/.local/share/opencode/tool-output/tool_123.txt"; function tarball( content = "safe artifact\n", name = "package/readme.txt", @@ -121,6 +123,138 @@ describe("qualification bundle", () => { expect(read.files).toHaveLength(input().files.length); }); + test("R29-01 preserves temporary HOME transcript and artifact bytes", async () => { + const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-")); + temporary.push(outputRoot); + const transcript = json({ + gradeInput: { + schemaVersion: 1, + table: `|Path|\n|---|\n|${temporaryToolOutput}|`, + messages: [ + { + tool: { + path: temporaryToolOutput, + error: JSON.stringify({ + message: `Permission denied reading '${temporaryToolOutput}'`, + permission: { patterns: [temporaryToolOutput] }, + }), + escapedError: JSON.stringify({ + path: temporaryToolOutput, + }).replaceAll("/", "\\/"), + }, + }, + ], + }, + }); + const artifact = tarball(`Tool output: ${temporaryToolOutput}\n`); + const fixture = input(); + const written = await writeQualificationBundle({ + input: { + ...fixture, + files: fixture.files.map((file) => + file.role === "transcript" + ? { ...file, bytes: transcript } + : file.role === "artifact" + ? { ...file, bytes: artifact } + : file, + ), + }, + outputRoot, + }); + const read = await readQualificationBundle(written.path); + expect( + read.files.find(({ ref }) => ref.role === "transcript")?.bytes, + ).toEqual(transcript); + expect( + read.files.find(({ ref }) => ref.role === "artifact")?.bytes, + ).toEqual(artifact); + }); + + test.each([ + ["transcript", "/home/alice/private.txt"], + ["transcript", "/Users/alice/private.txt"], + ["transcript", "C:\\Users\\alice\\private.txt"], + ["transcript", "/home/.alice/private.txt"], + ["transcript", "/Users/.alice/private.txt"], + ["transcript", "C:\\Users\\.alice\\private.txt"], + ["transcript", "Permission denied reading '/home/alice/private.txt'"], + [ + "transcript", + JSON.stringify({ error: { path: "C:\\Users\\alice\\private.txt" } }), + ], + ["transcript", "file:///home/alice/private.txt"], + ["transcript", "file://localhost/Users/alice/private.txt"], + ["transcript", "//home/alice/private.txt"], + ["transcript", "Permission denied:\t/home/alice/private.txt"], + ["transcript", "See **/home/alice/private.txt**"], + ["transcript", "See __/Users/alice/private.txt__"], + ["transcript", `${temporaryToolOutput}\nSee **/Users/alice/private.txt**`], + [ + "transcript", + JSON.stringify({ path: "/home/alice/private.txt" }).replaceAll( + "/", + "\\/", + ), + ], + [ + "transcript", + JSON.stringify({ path: "/Users/alice/private.txt" }).replaceAll( + "/", + "\\/", + ), + ], + [ + "transcript", + `${temporaryToolOutput}\n${JSON.stringify({ path: "/home/alice/private.txt" }).replaceAll("/", "\\/")}`, + ], + ["artifact", "See **/home/alice/private.txt**"], + ["transcript", "|Path|\n|---|\n|/home/alice/private.txt|"], + ["transcript", "|Path|\n|---|\n|/Users/alice/private.txt|"], + ["artifact", "|Path|\n|---|\n|/home/alice/private.txt|"], + ["artifact", "|Path|\n|---|\n|/Users/alice/private.txt|"], + [ + "transcript", + `|Path|\n|---|\n|${temporaryToolOutput}|\n|/home/alice/private.txt|`, + ], + [ + "transcript", + `|Path|\n|---|\n|${temporaryToolOutput}|\n|/Users/alice/private.txt|`, + ], + [ + "artifact", + `|Path|\n|---|\n|${temporaryToolOutput}|\n|/home/alice/private.txt|`, + ], + [ + "artifact", + `|Path|\n|---|\n|${temporaryToolOutput}|\n|/Users/alice/private.txt|`, + ], + ["transcript", `${temporaryToolOutput}\n/home/alice/private.txt`], + ["artifact", `${temporaryToolOutput}\n/home/alice/private.txt`], + ])("R29-01 rejects private user paths in %s: %s", async (role, evidence) => { + const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-")); + temporary.push(outputRoot); + const fixture = input(); + await expect( + writeQualificationBundle({ + input: { + ...fixture, + files: fixture.files.map((file) => + file.role === role + ? { + ...file, + bytes: + role === "artifact" + ? tarball(evidence) + : json({ output: evidence }), + } + : file, + ), + }, + outputRoot, + }), + ).rejects.toThrow(/absolute user path/i); + }); + test("publishes no readable seal after interruption and resumes", async () => { const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-")); temporary.push(outputRoot); @@ -162,7 +296,13 @@ describe("qualification bundle", () => { ...unsafe, files: unsafe.files.map((file, index) => index === 0 - ? { ...file, bytes: json({ sessionId: "ses_rawSecret" }) } + ? { + ...file, + bytes: json({ + path: temporaryToolOutput, + sessionId: "ses_rawSecret", + }), + } : file, ), }, @@ -195,7 +335,12 @@ describe("qualification bundle", () => { ...assignedSecret, files: assignedSecret.files.map((file) => file.role === "artifact" - ? { ...file, bytes: tarball("api_key=super-secret-value") } + ? { + ...file, + bytes: tarball( + `${temporaryToolOutput}\napi_key=super-secret-value`, + ), + } : file, ), },