feat: unique test results per run (--run-name)

.changeset/named-test-runs.md

@@ -0,0 +1,15 @@
+---
+'@tulip/appwright': minor
+---
+
+Every run now writes its results into its own folders: `test-results/<run>` for Playwright's test output and appwright's video store, `playwright-report/<run>` for the HTML report, and `blob-report/<run>` when the blob reporter is enabled, so concurrent runs on one machine (for example iOS and Android side by side) no longer overwrite each other. Name the run with `appwright test --run-name <name>` or `APPWRIGHT_RUN_NAME`; otherwise the folder is named `<project>-<YYYYMMDD>-<HHmmss>-<4 random chars>`.
+
+**Migration.** Results move one level deeper: `test-results/<file>` is now
+`test-results/<run>/<file>`, and the HTML report is at `playwright-report/<run>/index.html`.
+Anything that reads those paths — CI artifact globs, `playwright show-report`, scripts that open
+the report — needs the run folder in the path. Pass `--run-name <name>` (or set
+`APPWRIGHT_RUN_NAME`) to make it a fixed, known value. Appwright's video store also moves from
+`playwright-report/data/videos-store` to `test-results/<run>/videos-store`, and blob reports move
+from `blob-report/` to `blob-report/<run>/` (merge with
+`npx playwright merge-reports blob-report/<run>`). The `json` and `junit` reporters are left where
+their `outputFile` points.

README.md

@@ -142,6 +142,17 @@ To run on several local devices or emulators at once, list them under `device.de
 project config and raise `workers` up to that number; each Playwright worker then drives its own
 device. See [Running on multiple local devices](docs/config.md#running-on-multiple-local-devices).
 
+Every run writes its results into its own folders, `test-results/<run>` and
+`playwright-report/<run>`, so runs started side by side do not overwrite each other. Name a run
+with `--run-name`; without it the folder is called `<project>-<YYYYMMDD>-<HHmmss>-<4 random chars>`.
+
+```sh
+npx appwright test --project android --run-name nightly
+npx playwright show-report playwright-report/nightly
+```
+
+See [Test results per run](docs/config.md#test-results-per-run).
+
 #### Run tests on BrowserStack
 
 Appwright supports BrowserStack out of the box. To run tests on BrowserStack, configure

docs/basics.md

@@ -99,4 +99,8 @@ npx appwright test --project ios
 
 Above commands will trigger runs on android and iOS emulators based on the above configuration.
 
+Each run writes its results into `test-results/<run>` and its HTML report into
+`playwright-report/<run>`. Pass `--run-name <name>` to choose the folder name; see
+[Test results per run](config.md#test-results-per-run).
+
 Once the test is completed, the report is launched automatically in the browser.

docs/config.md

@@ -271,3 +271,79 @@ export default defineConfig({
 
 Find the UDIDs with `xcrun xctrace list devices`. Both devices must be connected and trusted before
 the run starts; Appwright does not boot or pair physical devices.
+
+## Test results per run
+
+Every run writes into its own folders, so two runs started side by side on one machine (for
+example iOS and Android at the same time) never overwrite each other's output:
+
+```
+test-results/<run>/               Playwright output: per-test artifacts, .last-run.json
+test-results/<run>/videos-store/  Appwright worker videos and worker-info files
+playwright-report/<run>/          HTML report
+blob-report/<run>/                Blob report, when the blob reporter is enabled
+```
+
+Open a report with `npx playwright show-report playwright-report/<run>`. Global setup logs the
+folders of the current run when it starts.
+
+### Naming a run
+
+- `npx appwright test --project android --run-name nightly` names the run `nightly`. The flag is
+  handled by the appwright CLI and is not passed on to Playwright.
+- `APPWRIGHT_RUN_NAME=nightly npx appwright test --project android` does the same through the
+  environment, which is convenient in CI. The flag wins when both are given.
+- Without either, the run is named `<project>-<YYYYMMDD>-<HHmmss>-<4 random chars>` in local time,
+  for example `android-20260910-143201-k3x9`. Several `--project` values are joined with `+`.
+
+Names are used as folder names, so anything other than letters, digits, `.`, `_`, `-` and `+` is
+replaced with `-`, and leading dots are removed.
+
+### Interaction with Playwright options
+
+- A custom `outputDir` or html `outputFolder` in your config is kept as the base folder; the run
+  name is nested under it (`<outputDir>/<run>`).
+- Playwright's own `--output <dir>` flag still overrides `outputDir` completely, as it always has.
+- `--last-failed` reads `.last-run.json` from the run's output folder. To rerun the failures of an
+  earlier run, pass the same `--run-name` again.
+- Reporters that own a whole folder are namespaced automatically: the html reporter
+  (`playwright-report/<run>`) and the blob reporter (`blob-report/<run>`). Both wipe their folder
+  when they write a report, so without this the second run to finish would delete the first run's
+  report. Merge blob reports from a run with
+  `npx playwright merge-reports blob-report/<run>`, or collect the `.zip` files of several runs
+  into one folder first.
+- Reporters that write a single file to a path you chose (`json`, `junit`) are left exactly where
+  their options point, because silently moving a path your CI reads would be worse than the
+  collision it avoids. Two concurrent runs do still overwrite each other there, so put the run name
+  in the path yourself when you need those side by side:
+
+  ```ts
+  import { defineConfig, resolveRunName } from "appwright";
+
+  const run = resolveRunName();
+
+  export default defineConfig({
+    reporter: [
+      ["list"],
+      ["html"],
+      ["json", { outputFile: `test-results/${run}/results.json` }],
+    ],
+    // ...
+  });
+  ```
+
+- Run folders are never deleted automatically. Remove `test-results/` and `playwright-report/` when
+  you want the disk space back.
+
+### Upgrading from a flat layout
+
+Before this change every run wrote straight into `test-results/` and `playwright-report/`. Results
+are now one level deeper, under the run folder. Update anything that reads a fixed path — CI
+artifact globs, `npx playwright show-report`, scripts that open `playwright-report/index.html` — to
+include the run folder, and pass `--run-name <name>` when you want that folder to have a known,
+stable name:
+
+```sh
+npx appwright test --project android --run-name ci
+npx playwright show-report playwright-report/ci
+```

src/bin/args.ts

@@ -0,0 +1,54 @@
+import {
+  extractRunNameArg,
+  generateRunName,
+  parseProjectsFromArgv,
+  RUN_NAME_ENV,
+  sanitizeRunName,
+} from '../run-name';
+
+export const DEFAULT_CONFIG_FILE = 'appwright.config.ts';
+
+export type Invocation = {
+  /** Arguments to hand to `npx playwright`, with appwright-only flags removed. */
+  pwArgs: string[];
+  /** Extra environment for the Playwright process. */
+  env: Record<string, string>;
+  runName: string;
+};
+
+function hasConfigFlag(args: readonly string[]): boolean {
+  return args.some(
+    (arg) => arg === '--config' || arg === '-c' || arg.startsWith('--config=') || arg.startsWith('-c='),
+  );
+}
+
+/**
+ * Turns the appwright CLI arguments into a Playwright invocation.
+ *
+ * - `--run-name <name>` is removed (Playwright would reject it) and passed on as `APPWRIGHT_RUN_NAME`.
+ * - Run name precedence: flag, then `APPWRIGHT_RUN_NAME` already in the environment, then a
+ *   generated `<project>-<timestamp>-<suffix>` default.
+ * - `--config appwright.config.ts` is appended when no config flag is given.
+ */
+export function prepareInvocation(
+  argv: readonly string[],
+  env: NodeJS.ProcessEnv = process.env,
+): Invocation {
+  const { runName: fromFlag, rest } = extractRunNameArg(argv);
+  const pwArgs = [...rest];
+  if (!hasConfigFlag(pwArgs)) {
+    pwArgs.push('--config', DEFAULT_CONFIG_FILE);
+  }
+
+  const fromEnv = env[RUN_NAME_ENV];
+  let runName: string;
+  if (fromFlag !== undefined) {
+    runName = sanitizeRunName(fromFlag);
+  } else if (fromEnv !== undefined && fromEnv.trim() !== '') {
+    runName = sanitizeRunName(fromEnv);
+  } else {
+    runName = generateRunName(parseProjectsFromArgv(pwArgs));
+  }
+
+  return { pwArgs, env: { [RUN_NAME_ENV]: runName }, runName };
+}

src/bin/index.ts

@@ -1,53 +1,49 @@
 #!/usr/bin/env node
-import { spawn } from "child_process";
-import { logger } from "../logger";
+import { spawn } from 'child_process';
 
-function cmd(
-  command: string[],
-  options: { env?: Record<string, string> },
-): Promise<number> {
+import { logger } from '../logger';
+import { prepareInvocation } from './args';
+
+function cmd(command: string[], options: { env?: Record<string, string> }): Promise<number> {
   let errorLogs: string[] = [];
   return new Promise((resolveFunc, rejectFunc) => {
     let p = spawn(command[0]!, command.slice(1), {
       env: { ...process.env, ...options.env },
     });
-    p.stdout.on("data", (x) => {
+    p.stdout.on('data', (x) => {
       const log = x.toString();
-      if (log.includes("Error")) {
+      if (log.includes('Error')) {
         errorLogs.push(log);
       }
       process.stdout.write(log);
     });
-    p.stderr.on("data", (x) => {
+    p.stderr.on('data', (x) => {
       const log = x.toString();
       process.stderr.write(x.toString());
       errorLogs.push(log);
     });
-    p.on("exit", (code) => {
+    p.on('exit', (code) => {
       if (code != 0) {
         // assuming last log is the error message before exiting
-        rejectFunc(errorLogs.slice(-3).join("\n"));
+        rejectFunc(errorLogs.slice(-3).join('\n'));
       } else {
         resolveFunc(code!);
       }
     });
   });
 }
 
-async function runPlaywrightCmd(args: string) {
-  const pwRunCmd = `npx playwright ${args}`;
-  return cmd(pwRunCmd.split(" "), {});
-}
-
 (async function main() {
-  const defaultConfigFile = `appwright.config.ts`;
-  const pwOptions = process.argv.slice(2);
-  if (!pwOptions.includes("--config")) {
-    pwOptions.push(`--config`);
-    pwOptions.push(defaultConfigFile);
+  let invocation: ReturnType<typeof prepareInvocation>;
+  try {
+    invocation = prepareInvocation(process.argv.slice(2));
+  } catch (error: any) {
+    logger.error(error?.message ?? String(error));
+    process.exit(1);
   }
+  logger.log(`Run name: ${invocation.runName}`);
   try {
-    await runPlaywrightCmd(pwOptions.join(" "));
+    await cmd(['npx', 'playwright', ...invocation.pwArgs], { env: invocation.env });
   } catch (error: any) {
     logger.error(`Error while running playwright test: ${error}`);
     process.exit(1);

src/config.ts

@@ -7,6 +7,14 @@ import {
 } from '@playwright/test';
 
 import { logger } from './logger';
+import {
+  applyRunNameToReporters,
+  DEFAULT_OUTPUT_DIR,
+  normalizeReporters,
+  publishOutputDir,
+  resolveRunName,
+  runOutputDir,
+} from './run-name';
 import { AppwrightConfig } from './types';
 
 const resolveGlobalSetup = () => {
@@ -21,6 +29,8 @@ const resolveVideoReporter = () => {
   return path.join(directory, 'reporter.js');
 };
 
+const defaultReporters: ReporterDescription[] = [['list'], ['html', { open: 'always' }]];
+
 const defaultConfig: PlaywrightTestConfig<AppwrightConfig> = {
   globalSetup: resolveGlobalSetup(),
   testDir: './tests',
@@ -32,7 +42,6 @@ const defaultConfig: PlaywrightTestConfig<AppwrightConfig> = {
   // For local-device / emulator runs, `workers` must not exceed the number of entries in
   // `device.devices`: each worker drives its own device (slot = parallelIndex).
   workers: 2,
-  reporter: [['list'], ['html', { open: 'always' }]],
   use: {
     // TODO: Use this for actions
     actionTimeout: 20_000,
@@ -53,16 +62,19 @@ export function defineConfig(config: PlaywrightTestConfig<AppwrightConfig>) {
     );
     delete config.globalSetup;
   }
-  let reporterConfig: ReporterDescription[];
-  if (config.reporter) {
-    reporterConfig = config.reporter as ReporterDescription[];
-  } else {
-    reporterConfig = [['list'], ['html', { open: 'always' }]];
-  }
+  // Every run writes into its own folders (`test-results/<run>`, `playwright-report/<run>`) so
+  // concurrent runs on one machine do not clobber each other. The name comes from the appwright CLI
+  // (`--run-name` / APPWRIGHT_RUN_NAME) or is generated here and inherited by the worker processes.
+  const runName = resolveRunName();
+  const reporterConfig = normalizeReporters(config.reporter) ?? defaultReporters;
+  // Published so that folders derived from the output dir (the video store) follow a custom
+  // `outputDir` instead of assuming the default base.
+  const outputDir = publishOutputDir(runOutputDir(runName, config.outputDir ?? DEFAULT_OUTPUT_DIR));
   return defineConfigPlaywright<AppwrightConfig>({
     ...defaultConfig,
     ...config,
-    reporter: [[resolveVideoReporter()], ...reporterConfig],
+    outputDir,
+    reporter: [[resolveVideoReporter()], ...applyRunNameToReporters(reporterConfig, runName)],
     use: {
       ...defaultConfig.use,
       expectTimeout: config.use?.expectTimeout

src/fixture/workerInfo.ts

@@ -33,8 +33,6 @@ export class WorkerInfoStore {
     if (!fs.existsSync(this.basePath)) {
       fs.mkdirSync(this.basePath, { recursive: true });
     }
-    // TODO: can we make this file path unique for a session?
-    // will avoidd ios/android running into issues when running concurrently on local
     fs.writeFileSync(
       path.join(this.basePath, `worker-info-${idx}.json`),
       JSON.stringify(contents, null, 2),

src/global-setup.ts

@@ -1,4 +1,5 @@
 import { type ChildProcess } from 'child_process';
+import path from 'path';
 
 import { type FullConfig } from '@playwright/test';
 
@@ -11,23 +12,32 @@ import {
 } from './providers/appium';
 import { shutdownBootedEmulators } from './providers/emulator/boot';
 import { APPIUM_PORT_ENV, assertWorkersFitDevices } from './providers/slots';
+import { parseProjectsFromArgv, resolveRunName } from './run-name';
 import { AppwrightConfig, EmulatorConfig, LocalDeviceConfig, Platform } from './types';
 
 const LOCAL_PROVIDERS = ['local-device', 'emulator'];
 
+/**
+ * One log line saying where this run's results go, so that interleaved output from several
+ * concurrent runs in one terminal can be told apart.
+ */
+function logRunLocation(config: FullConfig<AppwrightConfig>, projects: string[]) {
+  const runName = resolveRunName();
+  const selected = config.projects.filter((project) => projects.includes(project.name));
+  const outputDirs = [...new Set(selected.map((project) => project.outputDir))].map((dir) =>
+    path.relative(process.cwd(), dir),
+  );
+  const htmlReporter = config.reporter.find(([name]) => name === 'html');
+  const reportDir: string | undefined = htmlReporter?.[1]?.outputFolder;
+  const parts = [`test output in ${outputDirs.join(', ')}`];
+  if (reportDir) {
+    parts.push(`HTML report in ${path.relative(process.cwd(), reportDir)}`);
+  }
+  logger.log(`Run "${runName}": ${parts.join(', ')}`);
+}
+
 async function globalSetup(config: FullConfig<AppwrightConfig>) {
-  const args = process.argv;
-  const projects: string[] = [];
-  args.forEach((arg, index) => {
-    if (arg === '--project') {
-      const project = args[index + 1];
-      if (project) {
-        projects.push(project);
-      } else {
-        throw new Error('Project name is required with --project flag');
-      }
-    }
-  });
+  const projects = parseProjectsFromArgv(process.argv);
 
   if (projects.length == 0) {
     // Capability to run all projects is not supported currently
@@ -37,6 +47,8 @@ async function globalSetup(config: FullConfig<AppwrightConfig>) {
     );
   }
 
+  logRunLocation(config, projects);
+
   // One Appium server is shared by every local project selected for this run.
   let appiumProcess: ChildProcess | undefined;
   let usedEmulatorProvider = false;

src/index.ts

@@ -1,5 +1,6 @@
 export { expect, test } from './fixture';
 export { defineConfig } from './config';
 export { Device } from './device';
+export { resolveRunName } from './run-name';
 export { WebView } from './webView';
 export * from './types';