feat(reporter): and printOnlyFailures and --reporter-only-failures

docs/src/test-api/class-fullconfig.md

@@ -107,7 +107,7 @@ See [`property: TestConfig.quiet`].
 
 ## property: FullConfig.reporter
 * since: v1.10
-- type: <[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html"|"blob"|"perfetto">>
+- type: <[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"failures"|"line"|"github"|"json"|"junit"|"null"|"html"|"blob"|"perfetto">>
   - `0` <[string]> Reporter name or module or file path
   - `1` <[Object]> An object with reporter options if any
 

docs/src/test-api/class-testconfig.md

@@ -449,7 +449,7 @@ export default defineConfig({
 
 ## property: TestConfig.reporter
 * since: v1.10
-- type: ?<[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html"|"blob"|"perfetto">>
+- type: ?<[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"failures"|"line"|"github"|"json"|"junit"|"null"|"html"|"blob"|"perfetto">>
   - `0` <[string]> Reporter name or module or file path
   - `1` <[Object]> An object with reporter options if any
 

docs/src/test-cli-js.md

@@ -102,7 +102,8 @@ npx playwright test --ui
 | `--project <project-name...>` | Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects). |
 | `--quiet` | Suppress stdio. |
 | `--repeat-each <N>` | Run each test `N` times (default: 1). |
-| `--reporter <reporter>` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. |
+| `--reporter <reporter>` | Reporter to use, comma-separated, can be "dot", "line", "list", "failures", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. |
+| `--reporter-only-failures` | Include only failed, flaky, and interrupted tests in JSON, JUnit, and HTML reports. |
 | `--retries <retries>` | Maximum retry count for flaky tests, zero for no retries (default: no retries). |
 | `--run-agents <mode>` | Run agents to generate the code for `page.perform`. Possible values are "missing", "all" and "none" (default: "none"). See [test-agents](./test-agents.md). |
 | `--shard <shard>` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". |
@@ -321,7 +322,8 @@ npx playwright merge-reports ./reports
 | Option | Description |
 | :--- | :--- |
 | `-c, --config <file>` | Configuration file. Can be used to specify additional configuration for the output report |
-| `--reporter <reporter>` | Reporter to use, comma-separated, can be "list", "line", "dot", "json", "junit", "null", "github", "html", "blob", "perfetto" (default: "list" locally and "dot" on CI) |
+| `--reporter <reporter>` | Reporter to use, comma-separated, can be "list", "line", "dot", "failures", "json", "junit", "null", "github", "html", "blob", "perfetto" (default: "list" locally and "dot" on CI) |
+| `--reporter-only-failures` | Include only failed, flaky, and interrupted tests in JSON, JUnit, and HTML reports. |
 
 ### Clear Cache
 

docs/src/test-reporter-api/class-reporter.md

@@ -74,6 +74,8 @@ export default defineConfig({
 });
 ```
 
+When `--reporter-only-failures` is set, custom reporters receive `onlyFailures: true` in their constructor options.
+
 Here is a typical order of reporter calls:
 * [`method: Reporter.onBegin`] is called once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite].
 * [`method: Reporter.onTestBegin`] is called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will get final `status` once the test finishes.

docs/src/test-reporters-js.md

@@ -50,6 +50,35 @@ export default defineConfig({
 });
 ```
 
+### Report only failures
+
+Use `onlyFailures` with the JSON, JUnit, or HTML reporter to include only failed, flaky, and interrupted tests.  Every attempt of an included test is retained, including successful retries:
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  reporter: [['html', { onlyFailures: true }]],
+});
+```
+
+The command line flag enables this option for all configured reporters that support it:
+
+```bash
+npx playwright test --reporter=failures,html --reporter-only-failures
+npx playwright merge-reports --reporter=html --reporter-only-failures ./blob-report
+```
+
+Use the `failures` reporter for live terminal diagnostics without progress output.  For example, retain a complete HTML report alongside concise terminal output:
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  reporter: [['failures'], ['html']],
+});
+```
+
 ## Built-in reporters
 
 All built-in reporters show detailed information about failures, and mostly differ in verbosity for successful runs.
@@ -223,6 +252,33 @@ Dot report supports the following configuration options and environment variable
 | `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise.
 | `NO_COLOR` | | Whether to disable colored output ([no-color.org](https://no-color.org/)). Any non-empty value disables colors. | unset
 
+### Failures reporter
+
+The failures reporter prints failures and successful retries as they occur, followed by a summary.
+
+```bash
+npx playwright test --reporter=failures
+```
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  reporter: 'failures',
+});
+```
+
+Use [`property: TestConfig.quiet`] to suppress test stdout and stderr.
+
+Failures report supports the following configuration options and environment variables:
+
+| Environment Variable Name | Reporter Config Option | Description | Default |
+|---|---|---|---|
+| `PLAYWRIGHT_FAILURES_OMIT_TAGS` | `omitTags` | Whether to omit test tags appended to test titles. | `false` |
+| `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. | `true` when terminal is in TTY mode, `false` otherwise |
+| `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise |
+| `NO_COLOR` | | Whether to disable colored output. | unset |
+
 ### HTML reporter
 
 HTML reporter produces a self-contained folder that contains report for the test run that can be served as a web page.

packages/playwright/src/cli/reportActions.ts

@@ -29,7 +29,8 @@ export async function showReport(report: string | undefined, host: string, port:
 
 export async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) {
   const configFile = opts.config;
-  const config = configFile ? await configLoader.loadConfigFromFile(configFile) : await configLoader.loadEmptyConfigForMergeReports();
+  const overrides = { reporterOnlyFailures: opts.reporterOnlyFailures ? true : undefined };
+  const config = configFile ? await configLoader.loadConfigFromFile(configFile, overrides) : await configLoader.loadEmptyConfigForMergeReports(overrides);
 
   const dir = path.resolve(process.cwd(), reportDir || '');
   const dirStat = await fs.promises.stat(dir).catch(e => null);

packages/playwright/src/cli/testActions.ts

@@ -121,6 +121,7 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOve
     repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined,
     retries: options.retries ? parseInt(options.retries, 10) : undefined,
     reporter: resolveReporterOption(options.reporter),
+    reporterOnlyFailures: options.reporterOnlyFailures ? true : undefined,
     additionalReporters: resolveReporterOption(options.addReporter),
     shard: resolveShardOption(options.shard),
     shuffle: resolveShuffleOption(options.shuffle),

packages/playwright/src/common/config.ts

@@ -293,7 +293,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] |
   return reporters;
 }
 
-export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto', 'coverage'] as const;
+export const builtInReporters = ['list', 'line', 'dot', 'failures', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto', 'coverage'] as const;
 export type BuiltInReporter = typeof builtInReporters[number];
 
 export type ContextReuseMode = 'none' | 'when-possible';

packages/playwright/src/common/configLoader.ts

@@ -366,7 +366,7 @@ export async function loadConfigFromFile(configFile: string | undefined, overrid
   return await loadConfig(resolveConfigLocation(configFile), overrides, ignoreDeps);
 }
 
-export async function loadEmptyConfigForMergeReports() {
+export async function loadEmptyConfigForMergeReports(overrides?: ConfigCLIOverrides) {
   // Merge reports is "different" for no good reason. It should not pick up local config from the cwd.
-  return await loadConfig({ configDir: process.cwd() });
+  return await loadConfig({ configDir: process.cwd() }, overrides);
 }

packages/playwright/src/common/ipc.ts

@@ -36,6 +36,7 @@ export type ConfigCLIOverrides = {
   repeatEach?: number;
   retries?: number;
   reporter?: ReporterDescription[];
+  reporterOnlyFailures?: boolean;
   additionalReporters?: ReporterDescription[];
   shard?: { current: number, total: number };
   shuffle?: string;

packages/playwright/src/program.ts

@@ -129,6 +129,7 @@ function addMergeReportsCommand(program: Command) {
   });
   command.option('-c, --config <file>', `Configuration file. Can be used to specify additional configuration for the output report.`);
   command.option('--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${config.defaultReporter}")`);
+  command.option('--reporter-only-failures', 'Only include tests with failures in JSON, JUnit, and HTML reports');
   command.addHelpText('afterAll', `
 Arguments [dir]:
   Directory containing blob reports.
@@ -230,6 +231,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
   ['--quiet', { description: `Suppress stdio` }],
   ['--repeat-each <N>', { description: `Run each test N times (default: 1)` }],
   ['--reporter <reporter>', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${config.defaultReporter}")` }],
+  ['--reporter-only-failures', { description: `Only include tests with failures in JSON, JUnit, and HTML reports` }],
   ['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],
   ['--run-agents <mode>', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }],
   ['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],

packages/playwright/src/reporters/base.ts

@@ -51,6 +51,7 @@ type TestSummary = {
 
 export type CommonReporterOptions = {
   configDir: string,
+  onlyFailures?: boolean,
   _mode?: 'list' | 'test' | 'merge',
   _commandHash?: string,
 };
@@ -389,6 +390,30 @@ export class TerminalReporter implements ReporterV2 {
   }
 }
 
+export function isFailure(test: TestCase): boolean {
+  const outcome = test.outcome();
+  return outcome === 'unexpected' || outcome === 'flaky' || test.results.some(result => result.status === 'interrupted');
+}
+
+export type TestFilter = (test: TestCase) => boolean;
+
+export function createTestFilter(options: CommonReporterOptions): TestFilter | undefined {
+  return options.onlyFailures ? isFailure : undefined;
+}
+
+export function* visitTests(suite: Suite, filter?: TestFilter): Generator<TestCase> {
+  for (const entry of suite.entries()) {
+    if (entry.type !== 'test')
+      yield* visitTests(entry, filter);
+    else if (!filter || filter(entry))
+      yield entry;
+  }
+}
+
+export function filterTests<T extends Suite | TestCase>(entries: T[], filter?: TestFilter): T[] {
+  return filter ? entries.filter(entry => entry.type === 'test' ? filter(entry) : !visitTests(entry, filter).next().done) : entries;
+}
+
 function formatResultErrors(screen: Screen, test: TestCase, result: TestResult): string {
   const lines: string[] = [];
   if (test.outcome() === 'unexpected') {

packages/playwright/src/reporters/failures.ts

@@ -0,0 +1,114 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { msToString } from '@isomorphic/formatUtils';
+import { getAsBooleanFromENV } from '@utils/env';
+
+import { markErrorsAsReported, TerminalReporter } from './base';
+
+import type { FailuresReporterOptions } from '../../types/test';
+import type { FullResult, TestCase, TestError, TestResult } from '../../types/testReporter';
+import type { CommonReporterOptions, TerminalReporterOptions } from './base';
+
+class FailuresReporter extends TerminalReporter {
+  private _failureIndex = new Map<TestCase, number>();
+  private _needNewLine = false;
+
+  constructor(options?: FailuresReporterOptions & CommonReporterOptions & TerminalReporterOptions) {
+    super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_FAILURES_OMIT_TAGS', options?.omitTags), lastResult: true });
+  }
+
+  override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
+    super.onStdOut(chunk, test, result);
+    this._dumpToStdio(chunk, this.screen.stdout);
+  }
+
+  override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) {
+    super.onStdErr(chunk, test, result);
+    this._dumpToStdio(chunk, this.screen.stderr);
+  }
+
+  private _dumpToStdio(chunk: string | Buffer, stream: NodeJS.WriteStream) {
+    if (this.config.quiet)
+      return;
+    stream.write(chunk);
+    if (chunk.length)
+      this._needNewLine = !chunk.toString().endsWith('\n');
+  }
+
+  override onTestEnd(test: TestCase, result: TestResult) {
+    super.onTestEnd(test, result);
+    if (result.status !== 'skipped' && result.status !== test.expectedStatus) {
+      this._printFailure(test);
+    } else if (result.retry && result.status === 'passed') {
+      this._maybeWriteNewLine();
+      this.writeLine(this.screen.colors.green(this.formatTestHeader(test, { indent: '  ', index: this._failureIndex.get(test) })));
+      this.writeLine(this.screen.colors.green(`    Passed on retry #${result.retry} (${msToString(result.duration)}).`));
+      this.writeLine();
+    }
+  }
+
+  private _printFailure(test: TestCase) {
+    let index = this._failureIndex.get(test);
+    if (index === undefined) {
+      index = this._failureIndex.size + 1;
+      this._failureIndex.set(test, index);
+    }
+    const message = this.formatFailure(test, index);
+    if (message.trim()) {
+      this._maybeWriteNewLine();
+      this.writeLine(message);
+    }
+  }
+
+  override onError(error: TestError) {
+    super.onError(error);
+    this._maybeWriteNewLine();
+    this.writeLine(this.formatError(error).message);
+  }
+
+  async onTestPaused(test: TestCase, result: TestResult) {
+    // Without TTY, the user cannot interrupt the pause.
+    if (!process.stdin.isTTY && !process.env.PW_TEST_DEBUG_REPORTERS)
+      return;
+
+    this._maybeWriteNewLine();
+    if (test.outcome() === 'unexpected') {
+      this._printFailure(test);
+      markErrorsAsReported(result);
+      this.writeLine(this.screen.colors.yellow('    Paused on error. Press Ctrl+C to end.'));
+    } else {
+      this.writeLine(this.screen.colors.yellow(this.formatTestHeader(test, { indent: '  ' })));
+      this.writeLine(this.screen.colors.yellow('    Paused at test end. Press Ctrl+C to end.'));
+    }
+    await new Promise<void>(() => {});
+  }
+
+  override async onEnd(result: FullResult) {
+    await super.onEnd(result);
+    this._maybeWriteNewLine();
+    this.epilogue(false);
+  }
+
+  private _maybeWriteNewLine() {
+    if (this._needNewLine) {
+      this.writeLine();
+      this._needNewLine = false;
+    }
+  }
+}
+
+export default FailuresReporter;

packages/playwright/src/reporters/html.ts

@@ -36,11 +36,12 @@ import { extractZip } from '@utils/third_party/extractZip';
 // `import('vite')`) is DCE'd in release builds.
 declare const __PW_HMR__: boolean;
 
-import { CommonReporterOptions, formatError, formatResultFailure, internalScreen } from './base';
+import { createTestFilter, filterTests, formatError, formatResultFailure, internalScreen } from './base';
 import * as babel from '../transform/babelBundle';
 import { resolveReporterOutputPath, stripAnsiEscapes } from '../util';
 
 import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2';
+import type { CommonReporterOptions, TestFilter } from './base';
 import type { HtmlReporterOptions as HtmlReporterConfigOptions, Metadata, TestAnnotation } from '../../types/test';
 import type * as api from '../../types/testReporter';
 import type { HTMLReport, HTMLReportOptions, Location, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types';
@@ -164,7 +165,7 @@ class HtmlReporter implements ReporterV2 {
       noSnippets,
       noCopyPrompt,
       mergeFiles,
-    });
+    }, createTestFilter(this._options));
     this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors, this._machines);
   }
 
@@ -339,13 +340,15 @@ class HtmlBuilder {
   private _attachmentsBaseURL: string;
   private _options: HTMLReportOptions;
   private _doNotInlineAssets: boolean;
+  private _testFilter: TestFilter | undefined;
 
-  constructor(yazl: typeof import('yazl'), config: api.FullConfig, outputDir: string, attachmentsBaseURL: string, doNotInlineAssets: boolean, options: HTMLReportOptions) {
+  constructor(yazl: typeof import('yazl'), config: api.FullConfig, outputDir: string, attachmentsBaseURL: string, doNotInlineAssets: boolean, options: HTMLReportOptions, testFilter?: TestFilter) {
     this._dataZipFile = new yazl.ZipFile();
     this._config = config;
     this._reportFolder = outputDir;
     this._options = options;
     this._doNotInlineAssets = doNotInlineAssets;
+    this._testFilter = testFilter;
     fs.mkdirSync(this._reportFolder, { recursive: true });
     this._attachmentsBaseURL = attachmentsBaseURL;
   }
@@ -354,7 +357,7 @@ class HtmlBuilder {
     const data: DataMap = new Map();
     for (const projectSuite of projectSuites) {
       const projectName = projectSuite.project()!.name;
-      for (const fileSuite of projectSuite.suites) {
+      for (const fileSuite of filterTests(projectSuite.suites, this._testFilter)) {
         const fileName = this._relativeLocation(fileSuite.location)!.file;
         this._createEntryForSuite(data, projectName, fileSuite, fileName, true);
       }
@@ -505,7 +508,7 @@ class HtmlBuilder {
 
   private _processSuite(suite: api.Suite, projectName: string, path: string[], deep: boolean, outTests: TestEntry[]) {
     const newPath = [...path, suite.title];
-    suite.entries().forEach(e => {
+    filterTests(suite.entries(), this._testFilter).forEach(e => {
       if (e.type === 'test')
         outTests.push(this._createTestEntry(e, projectName, newPath));
       else if (deep)