From 9fa0709d00fd87e37ab09a899e0f7830c3019813 Mon Sep 17 00:00:00 2001 From: stanleykim-ux Date: Thu, 10 Sep 2026 15:39:50 -0400 Subject: [PATCH 1/3] feat(results): unique test results per run via --run-name Every run now writes into test-results/ (Playwright output plus appwright's video store) and playwright-report/, so concurrent runs on one machine no longer clobber each other. The name comes from the appwright CLI's new --run-name flag or APPWRIGHT_RUN_NAME, and defaults to ---. The config re-reads the name from the environment because Playwright evaluates it again in every worker. The video store moves out of playwright-report/, which the html reporter deletes before it copies attachments. Co-Authored-By: Claude Fable 5.1 --- .changeset/named-test-runs.md | 5 + README.md | 11 ++ docs/basics.md | 4 + docs/config.md | 36 ++++++ src/bin/args.ts | 54 ++++++++ src/bin/index.ts | 40 +++--- src/config.ts | 24 ++-- src/fixture/workerInfo.ts | 2 - src/global-setup.ts | 36 ++++-- src/run-name.ts | 205 ++++++++++++++++++++++++++++++ src/tests/bin-args.spec.ts | 63 ++++++++++ src/tests/run-name.spec.ts | 229 ++++++++++++++++++++++++++++++++++ src/utils.ts | 10 +- 13 files changed, 674 insertions(+), 45 deletions(-) create mode 100644 .changeset/named-test-runs.md create mode 100644 src/bin/args.ts create mode 100644 src/run-name.ts create mode 100644 src/tests/bin-args.spec.ts create mode 100644 src/tests/run-name.spec.ts diff --git a/.changeset/named-test-runs.md b/.changeset/named-test-runs.md new file mode 100644 index 00000000..c8b3693c --- /dev/null +++ b/.changeset/named-test-runs.md @@ -0,0 +1,5 @@ +--- +'@tulip/appwright': minor +--- + +Every run now writes its results into its own folders: `test-results/` for Playwright's test output and appwright's video store, and `playwright-report/` for the HTML report, so concurrent runs on one machine (for example iOS and Android side by side) no longer overwrite each other. Name the run with `appwright test --run-name ` or `APPWRIGHT_RUN_NAME`; otherwise the folder is named `---<4 random chars>`. diff --git a/README.md b/README.md index eae0ffb0..fb10f9cb 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,17 @@ To run on several local devices or emulators at once, list them under `device.de project config and raise `workers` up to that number; each Playwright worker then drives its own device. See [Running on multiple local devices](docs/config.md#running-on-multiple-local-devices). +Every run writes its results into its own folders, `test-results/` and +`playwright-report/`, so runs started side by side do not overwrite each other. Name a run +with `--run-name`; without it the folder is called `---<4 random chars>`. + +```sh +npx appwright test --project android --run-name nightly +npx playwright show-report playwright-report/nightly +``` + +See [Test results per run](docs/config.md#test-results-per-run). + #### Run tests on BrowserStack Appwright supports BrowserStack out of the box. To run tests on BrowserStack, configure diff --git a/docs/basics.md b/docs/basics.md index cc560d81..2a2050f9 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -99,4 +99,8 @@ npx appwright test --project ios Above commands will trigger runs on android and iOS emulators based on the above configuration. +Each run writes its results into `test-results/` and its HTML report into +`playwright-report/`. Pass `--run-name ` to choose the folder name; see +[Test results per run](config.md#test-results-per-run). + Once the test is completed, the report is launched automatically in the browser. \ No newline at end of file diff --git a/docs/config.md b/docs/config.md index 44d627e9..07fa5e68 100644 --- a/docs/config.md +++ b/docs/config.md @@ -271,3 +271,39 @@ export default defineConfig({ Find the UDIDs with `xcrun xctrace list devices`. Both devices must be connected and trusted before the run starts; Appwright does not boot or pair physical devices. + +## Test results per run + +Every run writes into its own folders, so two runs started side by side on one machine (for +example iOS and Android at the same time) never overwrite each other's output: + +``` +test-results// Playwright output: per-test artifacts, .last-run.json +test-results//videos-store/ Appwright worker videos and worker-info files +playwright-report// HTML report +``` + +Open a report with `npx playwright show-report playwright-report/`. Global setup logs the +folders of the current run when it starts. + +### Naming a run + +- `npx appwright test --project android --run-name nightly` names the run `nightly`. The flag is + handled by the appwright CLI and is not passed on to Playwright. +- `APPWRIGHT_RUN_NAME=nightly npx appwright test --project android` does the same through the + environment, which is convenient in CI. The flag wins when both are given. +- Without either, the run is named `---<4 random chars>` in local time, + for example `android-20260910-143201-k3x9`. Several `--project` values are joined with `+`. + +Names are used as folder names, so anything other than letters, digits, `.`, `_`, `-` and `+` is +replaced with `-`, and leading dots are removed. + +### Interaction with Playwright options + +- A custom `outputDir` or html `outputFolder` in your config is kept as the base folder; the run + name is nested under it (`/`). +- Playwright's own `--output ` flag still overrides `outputDir` completely, as it always has. +- `--last-failed` reads `.last-run.json` from the run's output folder. To rerun the failures of an + earlier run, pass the same `--run-name` again. +- Run folders are never deleted automatically. Remove `test-results/` and `playwright-report/` when + you want the disk space back. diff --git a/src/bin/args.ts b/src/bin/args.ts new file mode 100644 index 00000000..13c858b6 --- /dev/null +++ b/src/bin/args.ts @@ -0,0 +1,54 @@ +import { + extractRunNameArg, + generateRunName, + parseProjectsFromArgv, + RUN_NAME_ENV, + sanitizeRunName, +} from '../run-name'; + +export const DEFAULT_CONFIG_FILE = 'appwright.config.ts'; + +export type Invocation = { + /** Arguments to hand to `npx playwright`, with appwright-only flags removed. */ + pwArgs: string[]; + /** Extra environment for the Playwright process. */ + env: Record; + runName: string; +}; + +function hasConfigFlag(args: readonly string[]): boolean { + return args.some( + (arg) => arg === '--config' || arg === '-c' || arg.startsWith('--config=') || arg.startsWith('-c='), + ); +} + +/** + * Turns the appwright CLI arguments into a Playwright invocation. + * + * - `--run-name ` is removed (Playwright would reject it) and passed on as `APPWRIGHT_RUN_NAME`. + * - Run name precedence: flag, then `APPWRIGHT_RUN_NAME` already in the environment, then a + * generated `--` default. + * - `--config appwright.config.ts` is appended when no config flag is given. + */ +export function prepareInvocation( + argv: readonly string[], + env: NodeJS.ProcessEnv = process.env, +): Invocation { + const { runName: fromFlag, rest } = extractRunNameArg(argv); + const pwArgs = [...rest]; + if (!hasConfigFlag(pwArgs)) { + pwArgs.push('--config', DEFAULT_CONFIG_FILE); + } + + const fromEnv = env[RUN_NAME_ENV]; + let runName: string; + if (fromFlag !== undefined) { + runName = sanitizeRunName(fromFlag); + } else if (fromEnv !== undefined && fromEnv.trim() !== '') { + runName = sanitizeRunName(fromEnv); + } else { + runName = generateRunName(parseProjectsFromArgv(pwArgs)); + } + + return { pwArgs, env: { [RUN_NAME_ENV]: runName }, runName }; +} diff --git a/src/bin/index.ts b/src/bin/index.ts index a8ffc661..0d279c57 100644 --- a/src/bin/index.ts +++ b/src/bin/index.ts @@ -1,32 +1,31 @@ #!/usr/bin/env node -import { spawn } from "child_process"; -import { logger } from "../logger"; +import { spawn } from 'child_process'; -function cmd( - command: string[], - options: { env?: Record }, -): Promise { +import { logger } from '../logger'; +import { prepareInvocation } from './args'; + +function cmd(command: string[], options: { env?: Record }): Promise { let errorLogs: string[] = []; return new Promise((resolveFunc, rejectFunc) => { let p = spawn(command[0]!, command.slice(1), { env: { ...process.env, ...options.env }, }); - p.stdout.on("data", (x) => { + p.stdout.on('data', (x) => { const log = x.toString(); - if (log.includes("Error")) { + if (log.includes('Error')) { errorLogs.push(log); } process.stdout.write(log); }); - p.stderr.on("data", (x) => { + p.stderr.on('data', (x) => { const log = x.toString(); process.stderr.write(x.toString()); errorLogs.push(log); }); - p.on("exit", (code) => { + p.on('exit', (code) => { if (code != 0) { // assuming last log is the error message before exiting - rejectFunc(errorLogs.slice(-3).join("\n")); + rejectFunc(errorLogs.slice(-3).join('\n')); } else { resolveFunc(code!); } @@ -34,20 +33,17 @@ function cmd( }); } -async function runPlaywrightCmd(args: string) { - const pwRunCmd = `npx playwright ${args}`; - return cmd(pwRunCmd.split(" "), {}); -} - (async function main() { - const defaultConfigFile = `appwright.config.ts`; - const pwOptions = process.argv.slice(2); - if (!pwOptions.includes("--config")) { - pwOptions.push(`--config`); - pwOptions.push(defaultConfigFile); + let invocation: ReturnType; + try { + invocation = prepareInvocation(process.argv.slice(2)); + } catch (error: any) { + logger.error(error?.message ?? String(error)); + process.exit(1); } + logger.log(`Run name: ${invocation.runName}`); try { - await runPlaywrightCmd(pwOptions.join(" ")); + await cmd(['npx', 'playwright', ...invocation.pwArgs], { env: invocation.env }); } catch (error: any) { logger.error(`Error while running playwright test: ${error}`); process.exit(1); diff --git a/src/config.ts b/src/config.ts index e765d6ba..d3e2793e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,6 +7,13 @@ import { } from '@playwright/test'; import { logger } from './logger'; +import { + applyRunNameToReporters, + DEFAULT_OUTPUT_DIR, + normalizeReporters, + resolveRunName, + runOutputDir, +} from './run-name'; import { AppwrightConfig } from './types'; const resolveGlobalSetup = () => { @@ -21,6 +28,8 @@ const resolveVideoReporter = () => { return path.join(directory, 'reporter.js'); }; +const defaultReporters: ReporterDescription[] = [['list'], ['html', { open: 'always' }]]; + const defaultConfig: PlaywrightTestConfig = { globalSetup: resolveGlobalSetup(), testDir: './tests', @@ -32,7 +41,6 @@ const defaultConfig: PlaywrightTestConfig = { // For local-device / emulator runs, `workers` must not exceed the number of entries in // `device.devices`: each worker drives its own device (slot = parallelIndex). workers: 2, - reporter: [['list'], ['html', { open: 'always' }]], use: { // TODO: Use this for actions actionTimeout: 20_000, @@ -53,16 +61,16 @@ export function defineConfig(config: PlaywrightTestConfig) { ); delete config.globalSetup; } - let reporterConfig: ReporterDescription[]; - if (config.reporter) { - reporterConfig = config.reporter as ReporterDescription[]; - } else { - reporterConfig = [['list'], ['html', { open: 'always' }]]; - } + // Every run writes into its own folders (`test-results/`, `playwright-report/`) so + // concurrent runs on one machine do not clobber each other. The name comes from the appwright CLI + // (`--run-name` / APPWRIGHT_RUN_NAME) or is generated here and inherited by the worker processes. + const runName = resolveRunName(); + const reporterConfig = normalizeReporters(config.reporter) ?? defaultReporters; return defineConfigPlaywright({ ...defaultConfig, ...config, - reporter: [[resolveVideoReporter()], ...reporterConfig], + outputDir: runOutputDir(runName, config.outputDir ?? DEFAULT_OUTPUT_DIR), + reporter: [[resolveVideoReporter()], ...applyRunNameToReporters(reporterConfig, runName)], use: { ...defaultConfig.use, expectTimeout: config.use?.expectTimeout diff --git a/src/fixture/workerInfo.ts b/src/fixture/workerInfo.ts index b3577bda..6d1921da 100644 --- a/src/fixture/workerInfo.ts +++ b/src/fixture/workerInfo.ts @@ -33,8 +33,6 @@ export class WorkerInfoStore { if (!fs.existsSync(this.basePath)) { fs.mkdirSync(this.basePath, { recursive: true }); } - // TODO: can we make this file path unique for a session? - // will avoidd ios/android running into issues when running concurrently on local fs.writeFileSync( path.join(this.basePath, `worker-info-${idx}.json`), JSON.stringify(contents, null, 2), diff --git a/src/global-setup.ts b/src/global-setup.ts index 97fa897c..cc566bea 100644 --- a/src/global-setup.ts +++ b/src/global-setup.ts @@ -1,4 +1,5 @@ import { type ChildProcess } from 'child_process'; +import path from 'path'; import { type FullConfig } from '@playwright/test'; @@ -11,23 +12,32 @@ import { } from './providers/appium'; import { shutdownBootedEmulators } from './providers/emulator/boot'; import { APPIUM_PORT_ENV, assertWorkersFitDevices } from './providers/slots'; +import { parseProjectsFromArgv, resolveRunName } from './run-name'; import { AppwrightConfig, EmulatorConfig, LocalDeviceConfig, Platform } from './types'; const LOCAL_PROVIDERS = ['local-device', 'emulator']; +/** + * One log line saying where this run's results go, so that interleaved output from several + * concurrent runs in one terminal can be told apart. + */ +function logRunLocation(config: FullConfig, projects: string[]) { + const runName = resolveRunName(); + const selected = config.projects.filter((project) => projects.includes(project.name)); + const outputDirs = [...new Set(selected.map((project) => project.outputDir))].map((dir) => + path.relative(process.cwd(), dir), + ); + const htmlReporter = config.reporter.find(([name]) => name === 'html'); + const reportDir: string | undefined = htmlReporter?.[1]?.outputFolder; + const parts = [`test output in ${outputDirs.join(', ')}`]; + if (reportDir) { + parts.push(`HTML report in ${path.relative(process.cwd(), reportDir)}`); + } + logger.log(`Run "${runName}": ${parts.join(', ')}`); +} + async function globalSetup(config: FullConfig) { - const args = process.argv; - const projects: string[] = []; - args.forEach((arg, index) => { - if (arg === '--project') { - const project = args[index + 1]; - if (project) { - projects.push(project); - } else { - throw new Error('Project name is required with --project flag'); - } - } - }); + const projects = parseProjectsFromArgv(process.argv); if (projects.length == 0) { // Capability to run all projects is not supported currently @@ -37,6 +47,8 @@ async function globalSetup(config: FullConfig) { ); } + logRunLocation(config, projects); + // One Appium server is shared by every local project selected for this run. let appiumProcess: ChildProcess | undefined; let usedEmulatorProvider = false; diff --git a/src/run-name.ts b/src/run-name.ts new file mode 100644 index 00000000..48d00f7f --- /dev/null +++ b/src/run-name.ts @@ -0,0 +1,205 @@ +import crypto from 'crypto'; +import path from 'path'; + +import type { PlaywrightTestConfig, ReporterDescription } from '@playwright/test'; + +/** + * Environment variable through which the appwright CLI (or the first process that calls + * `resolveRunName`) tells every other process of a run which name the run has. + * + * Playwright evaluates the config file again inside every worker process, so the name is stored + * here once and re-read everywhere instead of being generated again per process. + */ +export const RUN_NAME_ENV = 'APPWRIGHT_RUN_NAME'; + +/** Playwright's default `outputDir`; per-run folders are nested under it. */ +export const DEFAULT_OUTPUT_DIR = 'test-results'; +/** The html reporter's default `outputFolder`; per-run folders are nested under it. */ +export const DEFAULT_REPORT_DIR = 'playwright-report'; +/** Sub-folder of the run's output dir that holds worker videos and worker-info files. */ +export const VIDEOS_STORE_DIR = 'videos-store'; + +export const RUN_NAME_FLAG = '--run-name'; +const PROJECT_FLAG = '--project'; + +/** + * Characters allowed in a run name. Everything else is replaced with `-` so the name is always a + * safe single path segment on every OS. `+` is kept so several projects can be joined as `a+b`. + */ +const DISALLOWED_CHARS = /[^A-Za-z0-9._+-]+/g; + +const SUFFIX_ALPHABET = '0123456789abcdefghijklmnopqrstuvwxyz'; + +type FlagValues = { values: string[]; rest: string[] }; + +/** + * Removes every ` value` / `=value` occurrence from `argv`. Throws when the flag is + * present without a value (a following argument that starts with `-` does not count as a value). + */ +function takeFlag(argv: readonly string[], flag: string, missingValueMessage: string): FlagValues { + const values: string[] = []; + const rest: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]!; + if (arg === flag) { + const next = argv[i + 1]; + if (next === undefined || next.startsWith('-')) { + throw new Error(missingValueMessage); + } + values.push(next); + i++; + } else if (arg.startsWith(`${flag}=`)) { + const value = arg.slice(flag.length + 1); + if (value === '') { + throw new Error(missingValueMessage); + } + values.push(value); + } else { + rest.push(arg); + } + } + return { values, rest }; +} + +/** + * Names of the projects selected with `--project` on the command line. Playwright does not expose + * the selection to the config or to `process.env`, so it is read from argv (the same way + * globalSetup has always done it). + */ +export function parseProjectsFromArgv(argv: readonly string[]): string[] { + return takeFlag(argv, PROJECT_FLAG, `Project name is required with ${PROJECT_FLAG} flag`).values; +} + +/** + * Splits `--run-name ` (or `--run-name=`) out of the CLI arguments. `rest` is argv + * without the flag, ready to be handed to Playwright, which rejects unknown options. + */ +export function extractRunNameArg(argv: readonly string[]): { runName?: string; rest: string[] } { + const { values, rest } = takeFlag( + argv, + RUN_NAME_FLAG, + `${RUN_NAME_FLAG} requires a value, e.g. \`${RUN_NAME_FLAG} nightly\``, + ); + const runName = values.length > 0 ? values[values.length - 1] : undefined; + return runName === undefined ? { rest } : { runName, rest }; +} + +/** + * Makes a run name safe to use as a folder name: trims it, replaces disallowed characters with + * `-`, and strips leading dots (so `..` can never escape the output folder). + */ +export function sanitizeRunName(raw: string): string { + const cleaned = raw.trim().replace(DISALLOWED_CHARS, '-').replace(/^\.+/, ''); + if (cleaned === '') { + throw new Error( + `Run name "${raw}" contains no usable characters. Use letters, digits, ".", "_" or "-".`, + ); + } + return cleaned; +} + +/** `length` random lowercase base36 characters. */ +export function randomSuffix(length = 4): string { + const bytes = crypto.randomBytes(length); + let out = ''; + for (const byte of bytes) { + out += SUFFIX_ALPHABET[byte % SUFFIX_ALPHABET.length]; + } + return out; +} + +/** `YYYYMMDD-HHmmss` in local time, so the folder name matches the clock the user is looking at. */ +export function formatTimestamp(date: Date): string { + const pad = (n: number) => String(n).padStart(2, '0'); + const ymd = `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}`; + const hms = `${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`; + return `${ymd}-${hms}`; +} + +/** + * Default run name: `---<4 random chars>`, e.g. + * `android-20260910-143201-k3x9`. Several projects are joined with `+`; with none, `run` is used. + */ +export function generateRunName( + projects: readonly string[], + now: Date = new Date(), + suffix: string = randomSuffix(), +): string { + const prefix = projects.length > 0 ? projects.join('+') : 'run'; + return sanitizeRunName(`${prefix}-${formatTimestamp(now)}-${suffix}`); +} + +/** + * The run name for the current process tree. Reads `APPWRIGHT_RUN_NAME` when set (by the appwright + * CLI, by CI, or by an earlier call in this process), otherwise generates a default from the + * `--project` arguments and stores it in the environment so worker processes inherit it. + * + * Idempotent: every call in a process, and in every child process, returns the same value. + */ +export function resolveRunName(argv: readonly string[] = process.argv): string { + const fromEnv = process.env[RUN_NAME_ENV]; + let name: string; + if (fromEnv !== undefined && fromEnv.trim() !== '') { + name = sanitizeRunName(fromEnv); + } else { + let projects: string[] = []; + try { + projects = parseProjectsFromArgv(argv); + } catch { + // A malformed --project is Playwright's error to report, not the run name's. + } + name = generateRunName(projects); + } + process.env[RUN_NAME_ENV] = name; + return name; +} + +/** Playwright `outputDir` for a run: `/`. */ +export function runOutputDir(runName: string, base: string = DEFAULT_OUTPUT_DIR): string { + return path.join(base, runName); +} + +/** html reporter `outputFolder` for a run: `/`. */ +export function runReportDir(runName: string, base: string = DEFAULT_REPORT_DIR): string { + return path.join(base, runName); +} + +/** + * Playwright accepts `reporter: 'html'` as well as `reporter: [['html', {...}]]`. Returns the + * array form, or `undefined` when no reporter was configured. + */ +export function normalizeReporters( + reporter: PlaywrightTestConfig['reporter'], +): ReporterDescription[] | undefined { + if (reporter === undefined) { + return undefined; + } + if (typeof reporter === 'string') { + return [[reporter]]; + } + return reporter as ReporterDescription[]; +} + +/** + * Points every `html` reporter entry at `/`. + * Other reporters and the order of the list are left untouched. + */ +export function applyRunNameToReporters( + reporters: readonly ReporterDescription[], + runName: string, +): ReporterDescription[] { + return reporters.map((entry): ReporterDescription => { + const [name, options] = Array.isArray(entry) ? entry : [entry, undefined]; + if (name !== 'html') { + return entry; + } + const htmlOptions = (options ?? {}) as { outputFolder?: string }; + return [ + 'html', + { + ...htmlOptions, + outputFolder: runReportDir(runName, htmlOptions.outputFolder ?? DEFAULT_REPORT_DIR), + }, + ]; + }); +} diff --git a/src/tests/bin-args.spec.ts b/src/tests/bin-args.spec.ts new file mode 100644 index 00000000..e4614804 --- /dev/null +++ b/src/tests/bin-args.spec.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from 'vitest'; + +import { DEFAULT_CONFIG_FILE, prepareInvocation } from '../bin/args'; +import { RUN_NAME_ENV } from '../run-name'; + +describe('prepareInvocation', () => { + test('strips --run-name and passes it on through the environment', () => { + const result = prepareInvocation(['test', '--run-name', 'smoke', '--project', 'ios'], {}); + expect(result.runName).toBe('smoke'); + expect(result.env).toEqual({ [RUN_NAME_ENV]: 'smoke' }); + expect(result.pwArgs).toEqual(['test', '--project', 'ios', '--config', DEFAULT_CONFIG_FILE]); + }); + + test('accepts the --run-name= form and sanitizes the value', () => { + const result = prepareInvocation(['test', '--run-name=nightly run', '--project', 'ios'], {}); + expect(result.runName).toBe('nightly-run'); + expect(result.pwArgs).not.toContain('--run-name=nightly run'); + }); + + test('appends the default config only when no config flag is given', () => { + expect(prepareInvocation(['test'], {}).pwArgs).toEqual(['test', '--config', DEFAULT_CONFIG_FILE]); + expect(prepareInvocation(['test', '--config', 'custom.ts'], {}).pwArgs).toEqual([ + 'test', + '--config', + 'custom.ts', + ]); + expect(prepareInvocation(['test', '-c', 'custom.ts'], {}).pwArgs).toEqual([ + 'test', + '-c', + 'custom.ts', + ]); + expect(prepareInvocation(['test', '--config=custom.ts'], {}).pwArgs).toEqual([ + 'test', + '--config=custom.ts', + ]); + }); + + test('prefers the flag over the environment', () => { + const result = prepareInvocation(['test', '--run-name', 'flag'], { [RUN_NAME_ENV]: 'env' }); + expect(result.runName).toBe('flag'); + }); + + test('falls back to APPWRIGHT_RUN_NAME from the environment', () => { + const result = prepareInvocation(['test', '--project', 'ios'], { [RUN_NAME_ENV]: 'from-ci' }); + expect(result.runName).toBe('from-ci'); + expect(result.env).toEqual({ [RUN_NAME_ENV]: 'from-ci' }); + }); + + test('generates -- when neither is given', () => { + const result = prepareInvocation(['test', '--project', 'android'], {}); + expect(result.runName).toMatch(/^android-\d{8}-\d{6}-[0-9a-z]{4}$/); + expect(result.env[RUN_NAME_ENV]).toBe(result.runName); + }); + + test('treats a blank environment value as unset', () => { + const result = prepareInvocation(['test'], { [RUN_NAME_ENV]: '' }); + expect(result.runName).toMatch(/^run-\d{8}-\d{6}-[0-9a-z]{4}$/); + }); + + test('throws when --run-name has no value', () => { + expect(() => prepareInvocation(['test', '--run-name'], {})).toThrow(/--run-name requires a value/); + }); +}); diff --git a/src/tests/run-name.spec.ts b/src/tests/run-name.spec.ts new file mode 100644 index 00000000..f13d3082 --- /dev/null +++ b/src/tests/run-name.spec.ts @@ -0,0 +1,229 @@ +import path from 'path'; + +import { afterEach, beforeEach, describe, expect, test } from 'vitest'; + +import { + applyRunNameToReporters, + extractRunNameArg, + formatTimestamp, + generateRunName, + normalizeReporters, + parseProjectsFromArgv, + randomSuffix, + resolveRunName, + RUN_NAME_ENV, + runOutputDir, + runReportDir, + sanitizeRunName, +} from '../run-name'; + +describe('sanitizeRunName', () => { + test('keeps letters, digits, dots, underscores, dashes and plus signs', () => { + expect(sanitizeRunName('android-20260910-143201-k3x9')).toBe('android-20260910-143201-k3x9'); + expect(sanitizeRunName('ios+android_v1.2')).toBe('ios+android_v1.2'); + }); + + test('replaces runs of other characters with a single dash', () => { + expect(sanitizeRunName('nightly run #3 (ios)')).toBe('nightly-run-3-ios-'); + expect(sanitizeRunName('a/b\\c:d')).toBe('a-b-c-d'); + }); + + test('strips surrounding whitespace and leading dots', () => { + expect(sanitizeRunName(' smoke ')).toBe('smoke'); + expect(sanitizeRunName('..hidden')).toBe('hidden'); + expect(sanitizeRunName('../escape')).toBe('-escape'); + }); + + test('throws when nothing usable is left', () => { + expect(() => sanitizeRunName('')).toThrow(/no usable characters/); + expect(() => sanitizeRunName(' ')).toThrow(/no usable characters/); + expect(() => sanitizeRunName('...')).toThrow(/no usable characters/); + }); +}); + +describe('formatTimestamp', () => { + test('formats local time as YYYYMMDD-HHmmss with zero padding', () => { + expect(formatTimestamp(new Date(2026, 8, 10, 14, 32, 1))).toBe('20260910-143201'); + expect(formatTimestamp(new Date(2026, 0, 5, 3, 4, 9))).toBe('20260105-030409'); + }); +}); + +describe('randomSuffix', () => { + test('returns lowercase base36 characters of the requested length', () => { + expect(randomSuffix()).toMatch(/^[0-9a-z]{4}$/); + expect(randomSuffix(8)).toMatch(/^[0-9a-z]{8}$/); + }); +}); + +describe('generateRunName', () => { + const now = new Date(2026, 8, 10, 14, 32, 1); + + test('is ---', () => { + expect(generateRunName(['android'], now, 'k3x9')).toBe('android-20260910-143201-k3x9'); + }); + + test('joins several projects with a plus sign', () => { + expect(generateRunName(['ios', 'android'], now, 'k3x9')).toBe('ios+android-20260910-143201-k3x9'); + }); + + test('falls back to "run" when no project is selected', () => { + expect(generateRunName([], now, 'k3x9')).toBe('run-20260910-143201-k3x9'); + }); + + test('sanitizes project names', () => { + expect(generateRunName(['my project'], now, 'k3x9')).toBe('my-project-20260910-143201-k3x9'); + }); + + test('uses the current time and a random suffix by default', () => { + expect(generateRunName(['ios'])).toMatch(/^ios-\d{8}-\d{6}-[0-9a-z]{4}$/); + }); +}); + +describe('parseProjectsFromArgv', () => { + test('reads --project values in both flag forms', () => { + expect(parseProjectsFromArgv(['test', '--project', 'android'])).toEqual(['android']); + expect(parseProjectsFromArgv(['test', '--project=ios'])).toEqual(['ios']); + expect(parseProjectsFromArgv(['--project', 'a', '--project=b'])).toEqual(['a', 'b']); + }); + + test('returns an empty list when the flag is absent', () => { + expect(parseProjectsFromArgv(['test', '--grep', 'login'])).toEqual([]); + }); + + test('throws when the flag has no value', () => { + expect(() => parseProjectsFromArgv(['test', '--project'])).toThrow(/Project name is required/); + expect(() => parseProjectsFromArgv(['test', '--project', '--headed'])).toThrow( + /Project name is required/, + ); + expect(() => parseProjectsFromArgv(['test', '--project='])).toThrow(/Project name is required/); + }); +}); + +describe('extractRunNameArg', () => { + test('returns the name and argv without the flag', () => { + expect(extractRunNameArg(['test', '--run-name', 'smoke', '--project', 'ios'])).toEqual({ + runName: 'smoke', + rest: ['test', '--project', 'ios'], + }); + expect(extractRunNameArg(['test', '--run-name=smoke'])).toEqual({ + runName: 'smoke', + rest: ['test'], + }); + }); + + test('the last occurrence wins', () => { + expect(extractRunNameArg(['--run-name', 'a', '--run-name', 'b'])).toEqual({ + runName: 'b', + rest: [], + }); + }); + + test('leaves argv alone when the flag is absent', () => { + expect(extractRunNameArg(['test', '--project', 'ios'])).toEqual({ + rest: ['test', '--project', 'ios'], + }); + }); + + test('throws when the value is missing', () => { + expect(() => extractRunNameArg(['test', '--run-name'])).toThrow(/--run-name requires a value/); + expect(() => extractRunNameArg(['test', '--run-name', '--project', 'ios'])).toThrow( + /--run-name requires a value/, + ); + expect(() => extractRunNameArg(['test', '--run-name='])).toThrow(/--run-name requires a value/); + }); +}); + +describe('resolveRunName', () => { + const original = process.env[RUN_NAME_ENV]; + + beforeEach(() => { + delete process.env[RUN_NAME_ENV]; + }); + + afterEach(() => { + if (original === undefined) { + delete process.env[RUN_NAME_ENV]; + } else { + process.env[RUN_NAME_ENV] = original; + } + }); + + test('uses the environment value when set, sanitized', () => { + process.env[RUN_NAME_ENV] = 'nightly run'; + expect(resolveRunName(['--project', 'ios'])).toBe('nightly-run'); + expect(process.env[RUN_NAME_ENV]).toBe('nightly-run'); + }); + + test('generates a default from --project and stores it in the environment', () => { + const name = resolveRunName(['node', 'playwright', 'test', '--project', 'android']); + expect(name).toMatch(/^android-\d{8}-\d{6}-[0-9a-z]{4}$/); + expect(process.env[RUN_NAME_ENV]).toBe(name); + }); + + test('is idempotent within a process', () => { + const first = resolveRunName(['--project', 'ios']); + const second = resolveRunName(['--project', 'android']); + expect(second).toBe(first); + }); + + test('treats a blank environment value as unset', () => { + process.env[RUN_NAME_ENV] = ' '; + expect(resolveRunName([])).toMatch(/^run-\d{8}-\d{6}-[0-9a-z]{4}$/); + }); + + test('ignores a malformed --project and still produces a name', () => { + expect(resolveRunName(['test', '--project'])).toMatch(/^run-\d{8}-\d{6}-[0-9a-z]{4}$/); + }); +}); + +describe('runOutputDir / runReportDir', () => { + test('nest the run name under the default folders', () => { + expect(runOutputDir('smoke')).toBe(path.join('test-results', 'smoke')); + expect(runReportDir('smoke')).toBe(path.join('playwright-report', 'smoke')); + }); + + test('nest the run name under a custom base', () => { + expect(runOutputDir('smoke', 'out')).toBe(path.join('out', 'smoke')); + expect(runReportDir('smoke', '/abs/report')).toBe(path.join('/abs/report', 'smoke')); + }); +}); + +describe('normalizeReporters', () => { + test('wraps a bare reporter name', () => { + expect(normalizeReporters('html')).toEqual([['html']]); + }); + + test('passes arrays through and leaves undefined alone', () => { + expect(normalizeReporters([['list'], ['html', { open: 'never' }]])).toEqual([ + ['list'], + ['html', { open: 'never' }], + ]); + expect(normalizeReporters(undefined)).toBeUndefined(); + }); +}); + +describe('applyRunNameToReporters', () => { + test('adds outputFolder to an html entry without options', () => { + expect(applyRunNameToReporters([['html']], 'smoke')).toEqual([ + ['html', { outputFolder: path.join('playwright-report', 'smoke') }], + ]); + }); + + test('keeps existing html options and nests a custom outputFolder', () => { + expect( + applyRunNameToReporters([['html', { open: 'never', outputFolder: 'reports' }]], 'smoke'), + ).toEqual([['html', { open: 'never', outputFolder: path.join('reports', 'smoke') }]]); + }); + + test('leaves other reporters and their order untouched', () => { + const custom = '/abs/path/to/reporter.js'; + expect( + applyRunNameToReporters([[custom], ['list'], ['html', { open: 'always' }], ['json']], 'smoke'), + ).toEqual([ + [custom], + ['list'], + ['html', { open: 'always', outputFolder: path.join('playwright-report', 'smoke') }], + ['json'], + ]); + }); +}); diff --git a/src/utils.ts b/src/utils.ts index 1ec9b50e..86443f52 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -3,6 +3,8 @@ import path from 'path'; import test from '@playwright/test'; +import { resolveRunName, runOutputDir, VIDEOS_STORE_DIR } from './run-name'; + export function boxedStep(target: Function, context: ClassMethodDecoratorContext) { return function replacementMethod( this: { @@ -75,8 +77,14 @@ export function longestDeterministicGroup(pattern: RegExp): string | undefined { return longestString; } +/** + * Folder for this run's worker videos and worker-info files: `test-results//videos-store`. + * It lives inside Playwright's per-run output dir (which Playwright clears at the start of a run) + * rather than inside the html report folder, which the html reporter deletes before it copies + * attachments. + */ export function basePath() { - return path.join(process.cwd(), 'playwright-report', 'data', 'videos-store'); + return path.join(process.cwd(), runOutputDir(resolveRunName()), VIDEOS_STORE_DIR); } export function isNoSuchWindowError(error: unknown): boolean { From 7f26ee171ad1a91502f768e074fd30090b7879c0 Mon Sep 17 00:00:00 2001 From: stanleykim-ux Date: Wed, 16 Sep 2026 14:35:46 -0400 Subject: [PATCH 2/3] fix(results): video store follows a custom outputDir; document layout change Review follow-ups on #3: - basePath() assumed the default `test-results` base, so with a custom `outputDir` the video store landed outside Playwright's output dir and was never cleared between runs. defineConfig now publishes the resolved outputDir through APPWRIGHT_OUTPUT_DIR and basePath() derives from it. Its doc comment was inaccurate and is corrected. - Changeset gains a migration note, and docs an "Upgrading from a flat layout" section, for the results moving one level deeper. - Document that only the html reporter is namespaced automatically, with a recipe for json/junit/blob; `resolveRunName` is exported so consumers can build their own per-run paths. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/named-test-runs.md | 7 ++++ docs/config.md | 33 ++++++++++++++++++ src/config.ts | 6 +++- src/index.ts | 1 + src/run-name.ts | 32 +++++++++++++++++ src/tests/run-name.spec.ts | 65 +++++++++++++++++++++++++++++++++-- src/utils.ts | 9 +++-- 7 files changed, 147 insertions(+), 6 deletions(-) diff --git a/.changeset/named-test-runs.md b/.changeset/named-test-runs.md index c8b3693c..d9349b66 100644 --- a/.changeset/named-test-runs.md +++ b/.changeset/named-test-runs.md @@ -3,3 +3,10 @@ --- Every run now writes its results into its own folders: `test-results/` for Playwright's test output and appwright's video store, and `playwright-report/` for the HTML report, so concurrent runs on one machine (for example iOS and Android side by side) no longer overwrite each other. Name the run with `appwright test --run-name ` or `APPWRIGHT_RUN_NAME`; otherwise the folder is named `---<4 random chars>`. + +**Migration.** Results move one level deeper: `test-results/` is now +`test-results//`, and the HTML report is at `playwright-report//index.html`. +Anything that reads those paths — CI artifact globs, `playwright show-report`, scripts that open +the report — needs the run folder in the path. Pass `--run-name ` (or set +`APPWRIGHT_RUN_NAME`) to make it a fixed, known value. Appwright's video store also moves from +`playwright-report/data/videos-store` to `test-results//videos-store`. diff --git a/docs/config.md b/docs/config.md index 07fa5e68..0942dfc2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -305,5 +305,38 @@ replaced with `-`, and leading dots are removed. - Playwright's own `--output ` flag still overrides `outputDir` completely, as it always has. - `--last-failed` reads `.last-run.json` from the run's output folder. To rerun the failures of an earlier run, pass the same `--run-name` again. +- Only the html reporter's folder is namespaced automatically. Reporters that write to a path you + choose (`json`, `junit`) or to their own folder (`blob`) keep writing exactly where their options + say, so two concurrent runs still overwrite each other there. Put the run name in the path + yourself when you need those side by side: + + ```ts + import { defineConfig, resolveRunName } from "appwright"; + + const run = resolveRunName(); + + export default defineConfig({ + reporter: [ + ["list"], + ["html"], + ["json", { outputFile: `test-results/${run}/results.json` }], + ], + // ... + }); + ``` + - Run folders are never deleted automatically. Remove `test-results/` and `playwright-report/` when you want the disk space back. + +### Upgrading from a flat layout + +Before this change every run wrote straight into `test-results/` and `playwright-report/`. Results +are now one level deeper, under the run folder. Update anything that reads a fixed path — CI +artifact globs, `npx playwright show-report`, scripts that open `playwright-report/index.html` — to +include the run folder, and pass `--run-name ` when you want that folder to have a known, +stable name: + +```sh +npx appwright test --project android --run-name ci +npx playwright show-report playwright-report/ci +``` diff --git a/src/config.ts b/src/config.ts index d3e2793e..82378a8e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,6 +11,7 @@ import { applyRunNameToReporters, DEFAULT_OUTPUT_DIR, normalizeReporters, + publishOutputDir, resolveRunName, runOutputDir, } from './run-name'; @@ -66,10 +67,13 @@ export function defineConfig(config: PlaywrightTestConfig) { // (`--run-name` / APPWRIGHT_RUN_NAME) or is generated here and inherited by the worker processes. const runName = resolveRunName(); const reporterConfig = normalizeReporters(config.reporter) ?? defaultReporters; + // Published so that folders derived from the output dir (the video store) follow a custom + // `outputDir` instead of assuming the default base. + const outputDir = publishOutputDir(runOutputDir(runName, config.outputDir ?? DEFAULT_OUTPUT_DIR)); return defineConfigPlaywright({ ...defaultConfig, ...config, - outputDir: runOutputDir(runName, config.outputDir ?? DEFAULT_OUTPUT_DIR), + outputDir, reporter: [[resolveVideoReporter()], ...applyRunNameToReporters(reporterConfig, runName)], use: { ...defaultConfig.use, diff --git a/src/index.ts b/src/index.ts index c7464930..4739b1a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ export { expect, test } from './fixture'; export { defineConfig } from './config'; export { Device } from './device'; +export { resolveRunName } from './run-name'; export { WebView } from './webView'; export * from './types'; diff --git a/src/run-name.ts b/src/run-name.ts index 48d00f7f..59b6ff0e 100644 --- a/src/run-name.ts +++ b/src/run-name.ts @@ -12,6 +12,13 @@ import type { PlaywrightTestConfig, ReporterDescription } from '@playwright/test */ export const RUN_NAME_ENV = 'APPWRIGHT_RUN_NAME'; +/** + * Environment variable through which `defineConfig` publishes the run's resolved Playwright + * `outputDir` to the rest of appwright, so that folders derived from it (the video store) follow a + * custom `outputDir` instead of assuming the default base. + */ +export const OUTPUT_DIR_ENV = 'APPWRIGHT_OUTPUT_DIR'; + /** Playwright's default `outputDir`; per-run folders are nested under it. */ export const DEFAULT_OUTPUT_DIR = 'test-results'; /** The html reporter's default `outputFolder`; per-run folders are nested under it. */ @@ -159,6 +166,31 @@ export function runOutputDir(runName: string, base: string = DEFAULT_OUTPUT_DIR) return path.join(base, runName); } +/** + * The run's Playwright `outputDir` for the current process. + * + * `defineConfig` resolves it (honouring a custom `outputDir` in the consumer config) and publishes + * it through {@link OUTPUT_DIR_ENV}; Playwright re-evaluates the config in every worker, so the + * value is available wherever appwright runs. Falls back to the default base for processes that + * never loaded an appwright config. + * + * Note that Playwright's `--output ` flag overrides `outputDir` after the config has been + * evaluated, so it is not reflected here. + */ +export function resolveOutputDir(): string { + const fromEnv = process.env[OUTPUT_DIR_ENV]; + if (fromEnv !== undefined && fromEnv.trim() !== '') { + return fromEnv; + } + return runOutputDir(resolveRunName()); +} + +/** Stores the run's resolved `outputDir` so every process of this run derives folders from it. */ +export function publishOutputDir(outputDir: string): string { + process.env[OUTPUT_DIR_ENV] = outputDir; + return outputDir; +} + /** html reporter `outputFolder` for a run: `/`. */ export function runReportDir(runName: string, base: string = DEFAULT_REPORT_DIR): string { return path.join(base, runName); diff --git a/src/tests/run-name.spec.ts b/src/tests/run-name.spec.ts index f13d3082..7431138b 100644 --- a/src/tests/run-name.spec.ts +++ b/src/tests/run-name.spec.ts @@ -8,14 +8,18 @@ import { formatTimestamp, generateRunName, normalizeReporters, + OUTPUT_DIR_ENV, parseProjectsFromArgv, + publishOutputDir, randomSuffix, + resolveOutputDir, resolveRunName, RUN_NAME_ENV, runOutputDir, runReportDir, sanitizeRunName, } from '../run-name'; +import { basePath } from '../utils'; describe('sanitizeRunName', () => { test('keeps letters, digits, dots, underscores, dashes and plus signs', () => { @@ -63,7 +67,9 @@ describe('generateRunName', () => { }); test('joins several projects with a plus sign', () => { - expect(generateRunName(['ios', 'android'], now, 'k3x9')).toBe('ios+android-20260910-143201-k3x9'); + expect(generateRunName(['ios', 'android'], now, 'k3x9')).toBe( + 'ios+android-20260910-143201-k3x9', + ); }); test('falls back to "run" when no project is selected', () => { @@ -218,7 +224,10 @@ describe('applyRunNameToReporters', () => { test('leaves other reporters and their order untouched', () => { const custom = '/abs/path/to/reporter.js'; expect( - applyRunNameToReporters([[custom], ['list'], ['html', { open: 'always' }], ['json']], 'smoke'), + applyRunNameToReporters( + [[custom], ['list'], ['html', { open: 'always' }], ['json']], + 'smoke', + ), ).toEqual([ [custom], ['list'], @@ -227,3 +236,55 @@ describe('applyRunNameToReporters', () => { ]); }); }); + +describe('resolveOutputDir', () => { + const originalOutputDir = process.env[OUTPUT_DIR_ENV]; + const originalRunName = process.env[RUN_NAME_ENV]; + + beforeEach(() => { + delete process.env[OUTPUT_DIR_ENV]; + process.env[RUN_NAME_ENV] = 'smoke'; + }); + + afterEach(() => { + restore(OUTPUT_DIR_ENV, originalOutputDir); + restore(RUN_NAME_ENV, originalRunName); + }); + + function restore(key: string, value: string | undefined) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + + test('falls back to the default base when no config has been evaluated', () => { + expect(resolveOutputDir()).toBe(path.join('test-results', 'smoke')); + }); + + test('uses the output dir published by defineConfig', () => { + publishOutputDir(path.join('custom-output', 'smoke')); + expect(resolveOutputDir()).toBe(path.join('custom-output', 'smoke')); + }); + + test('ignores a blank value', () => { + process.env[OUTPUT_DIR_ENV] = ' '; + expect(resolveOutputDir()).toBe(path.join('test-results', 'smoke')); + }); + + test('basePath follows a custom output dir', () => { + publishOutputDir(path.join('custom-output', 'smoke')); + expect(basePath()).toBe(path.resolve(process.cwd(), 'custom-output', 'smoke', 'videos-store')); + }); + + test('basePath defaults to the run folder under test-results', () => { + expect(basePath()).toBe(path.resolve(process.cwd(), 'test-results', 'smoke', 'videos-store')); + }); + + test('basePath honours an absolute output dir', () => { + const absolute = path.resolve(path.sep, 'tmp', 'appwright-out', 'smoke'); + publishOutputDir(absolute); + expect(basePath()).toBe(path.join(absolute, 'videos-store')); + }); +}); diff --git a/src/utils.ts b/src/utils.ts index 86443f52..383d71cd 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -3,7 +3,7 @@ import path from 'path'; import test from '@playwright/test'; -import { resolveRunName, runOutputDir, VIDEOS_STORE_DIR } from './run-name'; +import { resolveOutputDir, VIDEOS_STORE_DIR } from './run-name'; export function boxedStep(target: Function, context: ClassMethodDecoratorContext) { return function replacementMethod( @@ -78,13 +78,16 @@ export function longestDeterministicGroup(pattern: RegExp): string | undefined { } /** - * Folder for this run's worker videos and worker-info files: `test-results//videos-store`. + * Folder for this run's worker videos and worker-info files, `/videos-store` — by + * default `test-results//videos-store`, and nested under a custom `outputDir` when the + * consumer config sets one. + * * It lives inside Playwright's per-run output dir (which Playwright clears at the start of a run) * rather than inside the html report folder, which the html reporter deletes before it copies * attachments. */ export function basePath() { - return path.join(process.cwd(), runOutputDir(resolveRunName()), VIDEOS_STORE_DIR); + return path.resolve(process.cwd(), resolveOutputDir(), VIDEOS_STORE_DIR); } export function isNoSuchWindowError(error: unknown): boolean { From b7f4828c100bdb57d1d9affad150ef23f2845157 Mon Sep 17 00:00:00 2001 From: stanleykim-ux Date: Wed, 16 Sep 2026 15:08:19 -0400 Subject: [PATCH 3/3] fix(results): namespace the blob reporter's output dir per run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blob reporter calls removeFolders() on its output dir every time it writes, so with the default `blob-report` the second of two concurrent runs deleted the first run's report outright — even though the file names differ (they carry a hash of --project/--grep). Verified: two runs left only one zip behind; with the run nested in, both survive. blob joins html as a reporter that owns a whole folder and so gets `/`. json and junit keep writing to the outputFile the consumer chose, since moving a path CI reads would be worse than the collision. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/named-test-runs.md | 7 ++++-- docs/config.md | 15 +++++++++--- src/run-name.ts | 46 ++++++++++++++++++++++++++--------- src/tests/run-name.spec.ts | 36 ++++++++++++++++++++++++--- 4 files changed, 83 insertions(+), 21 deletions(-) diff --git a/.changeset/named-test-runs.md b/.changeset/named-test-runs.md index d9349b66..b5e22cef 100644 --- a/.changeset/named-test-runs.md +++ b/.changeset/named-test-runs.md @@ -2,11 +2,14 @@ '@tulip/appwright': minor --- -Every run now writes its results into its own folders: `test-results/` for Playwright's test output and appwright's video store, and `playwright-report/` for the HTML report, so concurrent runs on one machine (for example iOS and Android side by side) no longer overwrite each other. Name the run with `appwright test --run-name ` or `APPWRIGHT_RUN_NAME`; otherwise the folder is named `---<4 random chars>`. +Every run now writes its results into its own folders: `test-results/` for Playwright's test output and appwright's video store, `playwright-report/` for the HTML report, and `blob-report/` when the blob reporter is enabled, so concurrent runs on one machine (for example iOS and Android side by side) no longer overwrite each other. Name the run with `appwright test --run-name ` or `APPWRIGHT_RUN_NAME`; otherwise the folder is named `---<4 random chars>`. **Migration.** Results move one level deeper: `test-results/` is now `test-results//`, and the HTML report is at `playwright-report//index.html`. Anything that reads those paths — CI artifact globs, `playwright show-report`, scripts that open the report — needs the run folder in the path. Pass `--run-name ` (or set `APPWRIGHT_RUN_NAME`) to make it a fixed, known value. Appwright's video store also moves from -`playwright-report/data/videos-store` to `test-results//videos-store`. +`playwright-report/data/videos-store` to `test-results//videos-store`, and blob reports move +from `blob-report/` to `blob-report//` (merge with +`npx playwright merge-reports blob-report/`). The `json` and `junit` reporters are left where +their `outputFile` points. diff --git a/docs/config.md b/docs/config.md index 0942dfc2..fdaaa6f9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -281,6 +281,7 @@ example iOS and Android at the same time) never overwrite each other's output: test-results// Playwright output: per-test artifacts, .last-run.json test-results//videos-store/ Appwright worker videos and worker-info files playwright-report// HTML report +blob-report// Blob report, when the blob reporter is enabled ``` Open a report with `npx playwright show-report playwright-report/`. Global setup logs the @@ -305,10 +306,16 @@ replaced with `-`, and leading dots are removed. - Playwright's own `--output ` flag still overrides `outputDir` completely, as it always has. - `--last-failed` reads `.last-run.json` from the run's output folder. To rerun the failures of an earlier run, pass the same `--run-name` again. -- Only the html reporter's folder is namespaced automatically. Reporters that write to a path you - choose (`json`, `junit`) or to their own folder (`blob`) keep writing exactly where their options - say, so two concurrent runs still overwrite each other there. Put the run name in the path - yourself when you need those side by side: +- Reporters that own a whole folder are namespaced automatically: the html reporter + (`playwright-report/`) and the blob reporter (`blob-report/`). Both wipe their folder + when they write a report, so without this the second run to finish would delete the first run's + report. Merge blob reports from a run with + `npx playwright merge-reports blob-report/`, or collect the `.zip` files of several runs + into one folder first. +- Reporters that write a single file to a path you chose (`json`, `junit`) are left exactly where + their options point, because silently moving a path your CI reads would be worse than the + collision it avoids. Two concurrent runs do still overwrite each other there, so put the run name + in the path yourself when you need those side by side: ```ts import { defineConfig, resolveRunName } from "appwright"; diff --git a/src/run-name.ts b/src/run-name.ts index 59b6ff0e..bab7bab0 100644 --- a/src/run-name.ts +++ b/src/run-name.ts @@ -23,6 +23,8 @@ export const OUTPUT_DIR_ENV = 'APPWRIGHT_OUTPUT_DIR'; export const DEFAULT_OUTPUT_DIR = 'test-results'; /** The html reporter's default `outputFolder`; per-run folders are nested under it. */ export const DEFAULT_REPORT_DIR = 'playwright-report'; +/** The blob reporter's default `outputDir`; per-run folders are nested under it. */ +export const DEFAULT_BLOB_DIR = 'blob-report'; /** Sub-folder of the run's output dir that holds worker videos and worker-info files. */ export const VIDEOS_STORE_DIR = 'videos-store'; @@ -196,6 +198,11 @@ export function runReportDir(runName: string, base: string = DEFAULT_REPORT_DIR) return path.join(base, runName); } +/** blob reporter `outputDir` for a run: `/`. */ +export function runBlobDir(runName: string, base: string = DEFAULT_BLOB_DIR): string { + return path.join(base, runName); +} + /** * Playwright accepts `reporter: 'html'` as well as `reporter: [['html', {...}]]`. Returns the * array form, or `undefined` when no reporter was configured. @@ -213,8 +220,13 @@ export function normalizeReporters( } /** - * Points every `html` reporter entry at `/`. - * Other reporters and the order of the list are left untouched. + * Nests the run name into the output folder of every reporter that owns a whole directory: + * `html` (`outputFolder`) and `blob` (`outputDir`). Both wipe that directory when they write, so + * without this two concurrent runs destroy each other's report. + * + * Reporters that write a single file to a path the consumer picked (`json`, `junit`) are left + * alone: silently moving a path that CI reads would be worse than the collision it avoids. + * The order of the list is preserved. */ export function applyRunNameToReporters( reporters: readonly ReporterDescription[], @@ -222,16 +234,26 @@ export function applyRunNameToReporters( ): ReporterDescription[] { return reporters.map((entry): ReporterDescription => { const [name, options] = Array.isArray(entry) ? entry : [entry, undefined]; - if (name !== 'html') { - return entry; + if (name === 'html') { + const htmlOptions = (options ?? {}) as { outputFolder?: string }; + return [ + 'html', + { + ...htmlOptions, + outputFolder: runReportDir(runName, htmlOptions.outputFolder ?? DEFAULT_REPORT_DIR), + }, + ]; + } + if (name === 'blob') { + const blobOptions = (options ?? {}) as { outputDir?: string }; + return [ + 'blob', + { + ...blobOptions, + outputDir: runBlobDir(runName, blobOptions.outputDir ?? DEFAULT_BLOB_DIR), + }, + ]; } - const htmlOptions = (options ?? {}) as { outputFolder?: string }; - return [ - 'html', - { - ...htmlOptions, - outputFolder: runReportDir(runName, htmlOptions.outputFolder ?? DEFAULT_REPORT_DIR), - }, - ]; + return entry; }); } diff --git a/src/tests/run-name.spec.ts b/src/tests/run-name.spec.ts index 7431138b..0b386949 100644 --- a/src/tests/run-name.spec.ts +++ b/src/tests/run-name.spec.ts @@ -15,6 +15,7 @@ import { resolveOutputDir, resolveRunName, RUN_NAME_ENV, + runBlobDir, runOutputDir, runReportDir, sanitizeRunName, @@ -182,6 +183,16 @@ describe('resolveRunName', () => { }); }); +describe('runBlobDir', () => { + test('nests the run name under the default base', () => { + expect(runBlobDir('smoke')).toBe(path.join('blob-report', 'smoke')); + }); + + test('nests the run name under a custom base', () => { + expect(runBlobDir('smoke', 'blobs')).toBe(path.join('blobs', 'smoke')); + }); +}); + describe('runOutputDir / runReportDir', () => { test('nest the run name under the default folders', () => { expect(runOutputDir('smoke')).toBe(path.join('test-results', 'smoke')); @@ -221,18 +232,37 @@ describe('applyRunNameToReporters', () => { ).toEqual([['html', { open: 'never', outputFolder: path.join('reports', 'smoke') }]]); }); - test('leaves other reporters and their order untouched', () => { + test('adds outputDir to a blob entry without options', () => { + expect(applyRunNameToReporters([['blob']], 'smoke')).toEqual([ + ['blob', { outputDir: path.join('blob-report', 'smoke') }], + ]); + }); + + test('keeps existing blob options and nests a custom outputDir', () => { + expect( + applyRunNameToReporters([['blob', { fileName: 'r.zip', outputDir: 'blobs' }]], 'smoke'), + ).toEqual([['blob', { fileName: 'r.zip', outputDir: path.join('blobs', 'smoke') }]]); + }); + + test('leaves single-file reporters and the order untouched', () => { const custom = '/abs/path/to/reporter.js'; expect( applyRunNameToReporters( - [[custom], ['list'], ['html', { open: 'always' }], ['json']], + [ + [custom], + ['list'], + ['html', { open: 'always' }], + ['json', { outputFile: 'results.json' }], + ['junit', { outputFile: 'results.xml' }], + ], 'smoke', ), ).toEqual([ [custom], ['list'], ['html', { open: 'always', outputFolder: path.join('playwright-report', 'smoke') }], - ['json'], + ['json', { outputFile: 'results.json' }], + ['junit', { outputFile: 'results.xml' }], ]); }); });