Skip to content

fix: distinguish temporary homes in qualification evidence #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vriesd merged 1 commit into from
Sep 5, 2026
Merged

fix: distinguish temporary homes in qualification evidence #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion evals/qualification-bundle.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,13 @@ function assertSafeText(text: string): void {
throw new Error("Qualification bundle contains secret-shaped evidence.");
if (/\b(?:ses_[A-Za-z0-9]+|(?:session|review):[A-Za-z0-9-]+)\b/.test(text))
throw new Error("Qualification bundle contains a raw session identifier.");
// Match Unix homes at path starts, not beneath another root such as /tmp.
// Decode escaped slashes for inspection only; retained bytes stay untouched.
const paths = text.replace(/\\+\//g, "/");
if (
/(?:\/Users\/[^/\s]+|\/home\/[^/\s]+|[A-Za-z]:\\Users\\[^\\\s]+)/.test(text)
/(?:(?:^|[\s"'`=():,;<>[\]{}!?|]|\\[nrtbf]|file:\/\/[^/\s"'<>]*)[*_~]*\/+(?:Users|home)\/[^/\s]+|[A-Za-z]:\\+Users\\+[^\\\s]+)/.test(

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 5, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Strip ANSI prefixes before checking home paths

When a transcript or tar member contains colorized terminal output such as \u001b[31m/home/alice/private.txt, the ANSI sequence's final m immediately precedes the path and is not an accepted boundary, so this regex does not reject the private absolute path. The previous substring-based guard caught this case, and tool output can realistically contain ANSI formatting, allowing a user-home path to be sealed into qualification evidence.

Useful? React with 👍 / 👎.

paths,
)
)
throw new Error("Qualification bundle contains an absolute user path.");
}
Expand Down
149 changes: 147 additions & 2 deletions tests/qualification-bundle.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ afterEach(async () => {
});

const json = (value: unknown) => Buffer.from(canonicalJson(value));
const temporaryToolOutput =
"/tmp/flow-eval-Vj7Io4/home/.local/share/opencode/tool-output/tool_123.txt";
function tarball(
content = "safe artifact\n",
name = "package/readme.txt",
Expand Down Expand Up @@ -121,6 +123,138 @@ describe("qualification bundle", () => {
expect(read.files).toHaveLength(input().files.length);
});

test("R29-01 preserves temporary HOME transcript and artifact bytes", async () => {
const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-"));
temporary.push(outputRoot);
const transcript = json({
gradeInput: {
schemaVersion: 1,
table: `|Path|\n|---|\n|${temporaryToolOutput}|`,
messages: [
{
tool: {
path: temporaryToolOutput,
error: JSON.stringify({
message: `Permission denied reading '${temporaryToolOutput}'`,
permission: { patterns: [temporaryToolOutput] },
}),
escapedError: JSON.stringify({
path: temporaryToolOutput,
}).replaceAll("/", "\\/"),
},
},
],
},
});
const artifact = tarball(`Tool output: ${temporaryToolOutput}\n`);
const fixture = input();
const written = await writeQualificationBundle({
input: {
...fixture,
files: fixture.files.map((file) =>
file.role === "transcript"
? { ...file, bytes: transcript }
: file.role === "artifact"
? { ...file, bytes: artifact }
: file,
),
},
outputRoot,
});
const read = await readQualificationBundle(written.path);
expect(
read.files.find(({ ref }) => ref.role === "transcript")?.bytes,
).toEqual(transcript);
expect(
read.files.find(({ ref }) => ref.role === "artifact")?.bytes,
).toEqual(artifact);
});

test.each([
["transcript", "/home/alice/private.txt"],
["transcript", "/Users/alice/private.txt"],
["transcript", "C:\\Users\\alice\\private.txt"],
["transcript", "/home/.alice/private.txt"],
["transcript", "/Users/.alice/private.txt"],
["transcript", "C:\\Users\\.alice\\private.txt"],
["transcript", "Permission denied reading '/home/alice/private.txt'"],
[
"transcript",
JSON.stringify({ error: { path: "C:\\Users\\alice\\private.txt" } }),
],
["transcript", "file:///home/alice/private.txt"],
["transcript", "file://localhost/Users/alice/private.txt"],
["transcript", "//home/alice/private.txt"],
["transcript", "Permission denied:\t/home/alice/private.txt"],
["transcript", "See **/home/alice/private.txt**"],
["transcript", "See __/Users/alice/private.txt__"],
["transcript", `${temporaryToolOutput}\nSee **/Users/alice/private.txt**`],
[
"transcript",
JSON.stringify({ path: "/home/alice/private.txt" }).replaceAll(
"/",
"\\/",
),
],
[
"transcript",
JSON.stringify({ path: "/Users/alice/private.txt" }).replaceAll(
"/",
"\\/",
),
],
[
"transcript",
`${temporaryToolOutput}\n${JSON.stringify({ path: "/home/alice/private.txt" }).replaceAll("/", "\\/")}`,
],
["artifact", "See **/home/alice/private.txt**"],
["transcript", "|Path|\n|---|\n|/home/alice/private.txt|"],
["transcript", "|Path|\n|---|\n|/Users/alice/private.txt|"],
["artifact", "|Path|\n|---|\n|/home/alice/private.txt|"],
["artifact", "|Path|\n|---|\n|/Users/alice/private.txt|"],
[
"transcript",
`|Path|\n|---|\n|${temporaryToolOutput}|\n|/home/alice/private.txt|`,
],
[
"transcript",
`|Path|\n|---|\n|${temporaryToolOutput}|\n|/Users/alice/private.txt|`,
],
[
"artifact",
`|Path|\n|---|\n|${temporaryToolOutput}|\n|/home/alice/private.txt|`,
],
[
"artifact",
`|Path|\n|---|\n|${temporaryToolOutput}|\n|/Users/alice/private.txt|`,
],
["transcript", `${temporaryToolOutput}\n/home/alice/private.txt`],
["artifact", `${temporaryToolOutput}\n/home/alice/private.txt`],
])("R29-01 rejects private user paths in %s: %s", async (role, evidence) => {
const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-"));
temporary.push(outputRoot);
const fixture = input();
await expect(
writeQualificationBundle({
input: {
...fixture,
files: fixture.files.map((file) =>
file.role === role
? {
...file,
bytes:
role === "artifact"
? tarball(evidence)
: json({ output: evidence }),
}
: file,
),
},
outputRoot,
}),
).rejects.toThrow(/absolute user path/i);
});

test("publishes no readable seal after interruption and resumes", async () => {
const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-"));
temporary.push(outputRoot);
Expand Down Expand Up @@ -162,7 +296,13 @@ describe("qualification bundle", () => {
...unsafe,
files: unsafe.files.map((file, index) =>
index === 0
? { ...file, bytes: json({ sessionId: "ses_rawSecret" }) }
? {
...file,
bytes: json({
path: temporaryToolOutput,
sessionId: "ses_rawSecret",
}),
}
: file,
),
},
Expand Down Expand Up @@ -195,7 +335,12 @@ describe("qualification bundle", () => {
...assignedSecret,
files: assignedSecret.files.map((file) =>
file.role === "artifact"
? { ...file, bytes: tarball("api_key=super-secret-value") }
? {
...file,
bytes: tarball(
`${temporaryToolOutput}\napi_key=super-secret-value`,
),
}
: file,
),
},
Expand Down