Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 17 additions & 6 deletions packages/core/src/testing-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ import { ExpectedOperationError } from "@mobile-dev/shared/error-reporting.ts";
/** One step of the run, as the panel shows it against a line of the preview. */
export interface TestRunStep {
/** The 1-based line in the test file the step came from. */
line: number;
line?: number;
/** Position among this test's awaited calls when its runner couldn't locate the source file. */
stepIndex?: number;
/** The runner's own name for the call, e.g. `locator.tap`. */
api: string;
label: string;
Expand Down Expand Up @@ -88,7 +90,9 @@ function errorMessageOf(value: unknown): string | undefined {
export function reportSteps(report: unknown, file: string): TestRunStep[] {
const run = record(record(report)?.run);
const steps: TestRunStep[] = [];
for (const entry of list(run?.results)) {
const results = list(run?.results);
const requestedFile = file.replaceAll("\\", "/");
for (const entry of results) {
const result = record(entry);
if (!result) continue;
const attempts = list(result.attempts);
Expand All @@ -97,11 +101,18 @@ export function reportSteps(report: unknown, file: string): TestRunStep[] {
const step = record(item);
const source = record(step?.source);
const line = typeof source?.line === "number" ? source.line : 0;
// A step the runner attributes to another file is another test's; `app.open` has no line.
if (!step || line <= 0 || (str(source?.file) && str(source?.file) !== file)) continue;
const sourceFile = str(source?.file).replaceAll("\\", "/");
if (!step) continue;
// On Windows e2e can attribute every call to "unknown:1". Its index still identifies
// the call in a single test, just as it does for the panel's live progress.
const location = sourceFile === "unknown" && results.length === 1
&& str(result.file).replaceAll("\\", "/") === requestedFile && typeof step.index === "number"
? { stepIndex: step.index }
: line > 0 && (!sourceFile || sourceFile === requestedFile) ? { line } : undefined;
if (!location) continue;
const error = errorMessageOf(step.error);
steps.push({
line, api: str(step.api), label: str(step.label), status: str(step.status) || "unknown",
...location, api: str(step.api), label: str(step.label), status: str(step.status) || "unknown",
...(typeof step.durationMs === "number" ? { durationMs: step.durationMs } : {}),
...(error ? { error } : {}),
});
Expand Down Expand Up @@ -335,7 +346,7 @@ export function runnerEnv(environment: NodeJS.ProcessEnv = process.env): NodeJS.
* progress spinner must never be the reason Run stopped working.
*/
function refusedTheConfig(result: TestRunResult): boolean {
return result.steps.length === 0 && /config/i.test(result.output ?? "");
return result.status !== "passed" && result.steps.length === 0 && /config/i.test(result.output ?? "");
}

export async function runRecordedTest(request: TestRunRequest, deps: TestRunDeps = {}): Promise<TestRunResult> {
Expand Down
30 changes: 29 additions & 1 deletion packages/core/test/testing-run.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,20 @@ test("a retried test is read from the attempt that decided it", () => {
assert.deepEqual(reportSteps(raw, FILE).map(step => step.status), ["passed"]);
});

test("the final runner report retains step indexes for one selected test without source locations", () => {
const raw = report([{ line: 1, status: "passed" }, { line: 1, status: "failed" }]);
const result = raw.run.results[0]!;
Object.assign(result, { file: FILE.replaceAll("/", "\\") });
for (const step of result.attempts[0]!.steps) step.source.file = "unknown";
assert.deepEqual(reportSteps(raw, FILE).map(step => [step.stepIndex, step.line, step.status]),
[[0, undefined, "passed"], [1, undefined, "failed"]]);
assert.deepEqual(reportSteps(raw, "tests/another.e2e.ts"), []);
Object.assign(result, { file: FILE });
assert.deepEqual(reportSteps(raw, FILE).map(step => step.stepIndex), [0, 1]);
raw.run.results.push(result);
assert.deepEqual(reportSteps(raw, FILE), [], "step indexes restart for each test");
});

test("the report is found even when something else wrote to the same stream", () => {
const document = JSON.stringify(report([{ line: 8, status: "passed" }]));
assert.notEqual(parseReport(`npm warn exec something\n${document}\n`), undefined);
Expand Down Expand Up @@ -177,7 +191,7 @@ test("the runner is not handed the app's pinned agent-device state directory", (
test("the generated config adds a reporter and changes nothing else", () => {
const source = progressConfigSource("/app/e2e.config.ts");
// The project's own config is spread, so its targets, agents and workers all stand.
assert.match(source, /import base from "file:\/\/\/app\/e2e\.config\.ts";/);
assert.match(source, /import base from "file:\/\/\/(?:[A-Za-z]:\/)?app\/e2e\.config\.ts";/);
assert.match(source, /\.\.\.resolved,/);
// A config can export a function; the runner resolves one, so the generated config does too.
assert.match(source, /typeof base === "function" \? await base\(\) : base/);
Expand Down Expand Up @@ -263,6 +277,20 @@ test("a test that really failed is not retried into a different answer", async (
assert.equal(result.status, "failed");
});

test("a passing test with no source locations runs only once when its output mentions config", async () => {
const root = await project();
const output = JSON.stringify({ ...report([]), config: "e2e.config.ts" });
let runs = 0;
const spawn = ((command: string, args: readonly string[], options: { cwd?: string }) => {
runs++;
return spawner(output)(command, args, options as never);
}) as never;
const result = await runRecordedTest({ projectPath: root, relativePath: FILE, configPath: join(root, "e2e.config.ts") },
{ spawner: spawn, onProgress: () => undefined });
assert.equal(result.status, "passed");
assert.equal(runs, 1);
});

test("the generated config is syntactically whole, line by line", () => {
// It is source written by source, so an escape that survives one level and not the other produces
// a file the runner refuses with `CONFIG_LOAD_FAILED: Unterminated string literal` — and a run
Expand Down
48 changes: 20 additions & 28 deletions packages/ui/src/testing-panel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,21 @@ export class TestingPanel {
&& !this.snapshot.loadingTest && !this.snapshot.agent?.active && !this.snapshot.running && !this.snapshot.saving);
}

private finishRun(data: Record<string, unknown>, calls: readonly number[]) {
const steps = (Array.isArray(data.steps) ? data.steps : []).flatMap(item => {
const step = object(item);
if (!step) return [];
const line = typeof step.line === "number" ? step.line : typeof step.stepIndex === "number" ? calls[step.stepIndex] : undefined;
if (line === undefined || line <= 0) return [];
return [{ line, status: text(step.status), ...(text(step.error) ? { error: text(step.error) } : {}) }];
});
const status = text(data.status) || "failed";
testRecorder.finishRun({ status, steps,
...(text(data.output) ? { output: text(data.output) } : {}),
...(typeof data.durationMs === "number" && Number.isFinite(data.durationMs) ? { durationMs: data.durationMs } : {}) });
countUiEvent(status === "passed" ? "ui.testing.run_passed" : "ui.testing.run_failed");
}

/** Runs a temporary sibling copy so Play and Run never save the working copy implicitly. */
async runStoredTest(untilLine?: number) {
if (!this.canRunStored) return;
Expand All @@ -1018,15 +1033,7 @@ export class TestingPanel {
...(this.snapshot.configPath ? { configPath: this.snapshot.configPath } : {}),
...(this.snapshot.targetName ? { target: this.snapshot.targetName } : {}),
}, 15 * 60_000);
const data = object(response.structuredContent) ?? {};
const steps = (Array.isArray(data.steps) ? data.steps : []).map(entry => object(entry))
.filter((entry): entry is Record<string, unknown> => entry !== null)
.map(entry => ({ line: typeof entry.line === "number" ? entry.line : 0, status: text(entry.status), ...(text(entry.error) ? { error: text(entry.error) } : {}) }))
.filter(step => step.line > 0);
testRecorder.finishRun({ status: text(data.status) || "failed", steps,
...(text(data.output) ? { output: text(data.output) } : {}),
...(typeof data.durationMs === "number" ? { durationMs: data.durationMs } : {}) });
countUiEvent(text(data.status) === "passed" ? "ui.testing.run_passed" : "ui.testing.run_failed");
this.finishRun(object(response.structuredContent) ?? {}, callLines);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
testRecorder.failRun(message);
Expand Down Expand Up @@ -1090,32 +1097,17 @@ export class TestingPanel {
const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes();
// The run's own tool call is one request and one response, so progress comes from polling a
// second tool while it is in flight. See `mobile_testing_run_progress`.
const poll = setInterval(() => { void this.pollRunProgress(); }, 400);
const callLines = testRecorder.getSnapshot().sourceOverride !== undefined
? storedTestCallLines(testRecorder.source) : testRecorder.emitted.calls;
const poll = setInterval(() => { void this.pollRunProgress(callLines); }, 400);
try {
const response = await this.call("mobile_testing_run_test", {
projectPath: this.snapshot.projectPath, relativePath,
...(this.snapshot.configPath ? { configPath: this.snapshot.configPath } : {}),
...(this.snapshot.targetName ? { target: this.snapshot.targetName } : {}),
...(until ? { untilSource: until.source } : {}),
}, 15 * 60_000);
const data = object(response.structuredContent) ?? {};
const steps = (Array.isArray(data.steps) ? data.steps : []).map(entry => object(entry))
.filter((entry): entry is Record<string, unknown> => entry !== null)
.map(entry => ({
line: typeof entry.line === "number" ? entry.line : 0,
status: text(entry.status),
...(text(entry.error) ? { error: text(entry.error) } : {}),
}))
.filter(step => step.line > 0);
// The runner's own number for the run, which is not the timing below: that one brackets the
// whole call, spawning the CLI and resolving the project included.
const durationMs = typeof data.durationMs === "number" && Number.isFinite(data.durationMs) ? data.durationMs : undefined;
testRecorder.finishRun({
status: text(data.status) || "failed", steps,
...(text(data.output) ? { output: text(data.output) } : {}),
...(durationMs === undefined ? {} : { durationMs }),
});
countUiEvent(text(data.status) === "passed" ? "ui.testing.run_passed" : "ui.testing.run_failed");
this.finishRun(object(response.structuredContent) ?? {}, callLines);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
testRecorder.failRun(message);
Expand Down
25 changes: 24 additions & 1 deletion packages/ui/test/testing-panel.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,13 +150,36 @@ test("a project test polls live progress against its own source lines", async ()
assert.ok(calls.some(call => call.name === "mobile_testing_run_progress"), "project runs poll while the runner is in flight");
assert.deepEqual(testRecorder.getSnapshot().run?.lines, { 2: "passed", 3: "running" },
"runner indexes map to the awaited calls in the saved project's source");
finishRun({ status: "passed", exitCode: 0, steps: [{ line: 2, status: "passed" }, { line: 3, status: "passed" }] });
finishRun({ status: "passed", exitCode: 0, steps: [{ stepIndex: 0, status: "passed" }, { stepIndex: 1, status: "passed" }] });
await running;
assert.deepEqual(testRecorder.getSnapshot().run?.lines, { 2: "passed", 3: "passed" },
"a runner that omitted source locations still marks the final result against its awaited calls");

testRecorder.clearRun();
panel.dispose();
});

test("step indexes in an edited recording use the saved source's awaited calls", async () => {
const panel = new TestingPanel(host({
mobile_testing_write_file: { written: true, existed: false, path: `${PROJECT}/tests/flow.e2e.ts` },
mobile_testing_run_test: { status: "failed", steps: [
{ stepIndex: 1, status: "failed", error: "Button missing" }, { stepIndex: 0, status: "passed" },
] },
}, []), deviceApps);
panel.setAvailable(true);
(panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT, fileName: "flow" });
testRecorder.editSource('test("flow", async ({ app, screen }) => {\n await app.open();\n\n await screen.getByText("Done").tap();\n});');
try {
await panel.runTest();
assert.deepEqual(testRecorder.getSnapshot().run?.lines, { 2: "passed", 4: "failed" });
assert.equal(testRecorder.getSnapshot().run?.error, "Button missing");
} finally {
testRecorder.clearRun();
testRecorder.clearSteps();
panel.dispose();
}
});

test("Play until here runs the flow cut short, and never writes the test", async () => {
const calls: { name: string; arguments: Record<string, unknown> }[] = [];
const panel = new TestingPanel(host({
Expand Down
Loading