diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index 596cda945ab44..0e49d2e04d26a 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -107,7 +107,7 @@ See [`property: TestConfig.quiet`]. ## property: FullConfig.reporter * since: v1.10 -- type: <[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html"|"blob"|"perfetto">> +- type: <[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"failures"|"line"|"github"|"json"|"junit"|"null"|"html"|"blob"|"perfetto">> - `0` <[string]> Reporter name or module or file path - `1` <[Object]> An object with reporter options if any diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 39b0f1f53a053..1191937e4add4 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -449,7 +449,7 @@ export default defineConfig({ ## property: TestConfig.reporter * since: v1.10 -- type: ?<[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html"|"blob"|"perfetto">> +- type: ?<[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"failures"|"line"|"github"|"json"|"junit"|"null"|"html"|"blob"|"perfetto">> - `0` <[string]> Reporter name or module or file path - `1` <[Object]> An object with reporter options if any diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index fadedf6b741dd..f7509beaee429 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -102,7 +102,8 @@ npx playwright test --ui | `--project ` | Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects). | | `--quiet` | Suppress stdio. | | `--repeat-each ` | Run each test `N` times (default: 1). | -| `--reporter ` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. | +| `--reporter ` | Reporter to use, comma-separated, can be "dot", "line", "list", "failures", or others (default: "list" locally and "dot" on CI). You can also pass a path to a custom reporter file. | +| `--reporter-only-failures` | Include only failed, flaky, and interrupted tests in JSON, JUnit, and HTML reports. | | `--retries ` | Maximum retry count for flaky tests, zero for no retries (default: no retries). | | `--run-agents ` | Run agents to generate the code for `page.perform`. Possible values are "missing", "all" and "none" (default: "none"). See [test-agents](./test-agents.md). | | `--shard ` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". | @@ -321,7 +322,8 @@ npx playwright merge-reports ./reports | Option | Description | | :--- | :--- | | `-c, --config ` | Configuration file. Can be used to specify additional configuration for the output report | -| `--reporter ` | Reporter to use, comma-separated, can be "list", "line", "dot", "json", "junit", "null", "github", "html", "blob", "perfetto" (default: "list" locally and "dot" on CI) | +| `--reporter ` | Reporter to use, comma-separated, can be "list", "line", "dot", "failures", "json", "junit", "null", "github", "html", "blob", "perfetto" (default: "list" locally and "dot" on CI) | +| `--reporter-only-failures` | Include only failed, flaky, and interrupted tests in JSON, JUnit, and HTML reports. | ### Clear Cache diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 68a2a21dab9b5..8b92b9e326281 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -74,6 +74,8 @@ export default defineConfig({ }); ``` +When `--reporter-only-failures` is set, custom reporters receive `onlyFailures: true` in their constructor options. + Here is a typical order of reporter calls: * [`method: Reporter.onBegin`] is called once with a root suite that contains all other suites and tests. Learn more about [suites hierarchy][Suite]. * [`method: Reporter.onTestBegin`] is called for each test run. It is given a [TestCase] that is executed, and a [TestResult] that is almost empty. Test result will be populated while the test runs (for example, with steps and stdio) and will get final `status` once the test finishes. diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index e78805094e179..1ec83d6cfd0d0 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -50,6 +50,35 @@ export default defineConfig({ }); ``` +### Report only failures + +Use `onlyFailures` with the JSON, JUnit, or HTML reporter to include only failed, flaky, and interrupted tests. Every attempt of an included test is retained, including successful retries: + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['html', { onlyFailures: true }]], +}); +``` + +The command line flag enables this option for all configured reporters that support it: + +```bash +npx playwright test --reporter=failures,html --reporter-only-failures +npx playwright merge-reports --reporter=html --reporter-only-failures ./blob-report +``` + +Use the `failures` reporter for live terminal diagnostics without progress output. For example, retain a complete HTML report alongside concise terminal output: + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['failures'], ['html']], +}); +``` + ## Built-in reporters All built-in reporters show detailed information about failures, and mostly differ in verbosity for successful runs. @@ -223,6 +252,33 @@ Dot report supports the following configuration options and environment variable | `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise. | `NO_COLOR` | | Whether to disable colored output ([no-color.org](https://no-color.org/)). Any non-empty value disables colors. | unset +### Failures reporter + +The failures reporter prints failures and successful retries as they occur, followed by a summary. + +```bash +npx playwright test --reporter=failures +``` + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: 'failures', +}); +``` + +Use [`property: TestConfig.quiet`] to suppress test stdout and stderr. + +Failures report supports the following configuration options and environment variables: + +| Environment Variable Name | Reporter Config Option | Description | Default | +|---|---|---|---| +| `PLAYWRIGHT_FAILURES_OMIT_TAGS` | `omitTags` | Whether to omit test tags appended to test titles. | `false` | +| `PLAYWRIGHT_FORCE_TTY` | | Whether to produce output suitable for a live terminal. | `true` when terminal is in TTY mode, `false` otherwise | +| `FORCE_COLOR` | | Whether to produce colored output. | `true` when terminal is in TTY mode, `false` otherwise | +| `NO_COLOR` | | Whether to disable colored output. | unset | + ### HTML reporter HTML reporter produces a self-contained folder that contains report for the test run that can be served as a web page. diff --git a/packages/playwright/src/cli/reportActions.ts b/packages/playwright/src/cli/reportActions.ts index 3b8943ebc5243..599de78429498 100644 --- a/packages/playwright/src/cli/reportActions.ts +++ b/packages/playwright/src/cli/reportActions.ts @@ -29,7 +29,8 @@ export async function showReport(report: string | undefined, host: string, port: export async function mergeReports(reportDir: string | undefined, opts: { [key: string]: any }) { const configFile = opts.config; - const config = configFile ? await configLoader.loadConfigFromFile(configFile) : await configLoader.loadEmptyConfigForMergeReports(); + const overrides = { reporterOnlyFailures: opts.reporterOnlyFailures ? true : undefined }; + const config = configFile ? await configLoader.loadConfigFromFile(configFile, overrides) : await configLoader.loadEmptyConfigForMergeReports(overrides); const dir = path.resolve(process.cwd(), reportDir || ''); const dirStat = await fs.promises.stat(dir).catch(e => null); diff --git a/packages/playwright/src/cli/testActions.ts b/packages/playwright/src/cli/testActions.ts index b6fd25facb05c..c9f529e450dce 100644 --- a/packages/playwright/src/cli/testActions.ts +++ b/packages/playwright/src/cli/testActions.ts @@ -121,6 +121,7 @@ function overridesFromOptions(options: { [key: string]: any }): ipc.ConfigCLIOve repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined, retries: options.retries ? parseInt(options.retries, 10) : undefined, reporter: resolveReporterOption(options.reporter), + reporterOnlyFailures: options.reporterOnlyFailures ? true : undefined, additionalReporters: resolveReporterOption(options.addReporter), shard: resolveShardOption(options.shard), shuffle: resolveShuffleOption(options.shuffle), diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index bb6f1b2bc49ba..c27be2079f431 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -293,7 +293,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] | return reporters; } -export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto', 'coverage'] as const; +export const builtInReporters = ['list', 'line', 'dot', 'failures', 'json', 'junit', 'null', 'github', 'html', 'blob', 'perfetto', 'coverage'] as const; export type BuiltInReporter = typeof builtInReporters[number]; export type ContextReuseMode = 'none' | 'when-possible'; diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts index 5e74dc254a8e3..5d931a847ae44 100644 --- a/packages/playwright/src/common/configLoader.ts +++ b/packages/playwright/src/common/configLoader.ts @@ -366,7 +366,7 @@ export async function loadConfigFromFile(configFile: string | undefined, overrid return await loadConfig(resolveConfigLocation(configFile), overrides, ignoreDeps); } -export async function loadEmptyConfigForMergeReports() { +export async function loadEmptyConfigForMergeReports(overrides?: ConfigCLIOverrides) { // Merge reports is "different" for no good reason. It should not pick up local config from the cwd. - return await loadConfig({ configDir: process.cwd() }); + return await loadConfig({ configDir: process.cwd() }, overrides); } diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 512a919b5d285..f282d85bc358f 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -36,6 +36,7 @@ export type ConfigCLIOverrides = { repeatEach?: number; retries?: number; reporter?: ReporterDescription[]; + reporterOnlyFailures?: boolean; additionalReporters?: ReporterDescription[]; shard?: { current: number, total: number }; shuffle?: string; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index 9042f965224aa..95db695b62d98 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -129,6 +129,7 @@ function addMergeReportsCommand(program: Command) { }); command.option('-c, --config ', `Configuration file. Can be used to specify additional configuration for the output report.`); command.option('--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${config.defaultReporter}")`); + command.option('--reporter-only-failures', 'Only include tests with failures in JSON, JUnit, and HTML reports'); command.addHelpText('afterAll', ` Arguments [dir]: Directory containing blob reports. @@ -230,6 +231,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?: ['--quiet', { description: `Suppress stdio` }], ['--repeat-each ', { description: `Run each test N times (default: 1)` }], ['--reporter ', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${config.defaultReporter}")` }], + ['--reporter-only-failures', { description: `Only include tests with failures in JSON, JUnit, and HTML reports` }], ['--retries ', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }], ['--run-agents ', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }], ['--shard ', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }], diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 00bbb0172bed9..87ce46efb7bae 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -51,6 +51,7 @@ type TestSummary = { export type CommonReporterOptions = { configDir: string, + onlyFailures?: boolean, _mode?: 'list' | 'test' | 'merge', _commandHash?: string, }; @@ -389,6 +390,30 @@ export class TerminalReporter implements ReporterV2 { } } +export function isFailure(test: TestCase): boolean { + const outcome = test.outcome(); + return outcome === 'unexpected' || outcome === 'flaky' || test.results.some(result => result.status === 'interrupted'); +} + +export type TestFilter = (test: TestCase) => boolean; + +export function createTestFilter(options: CommonReporterOptions): TestFilter | undefined { + return options.onlyFailures ? isFailure : undefined; +} + +export function* visitTests(suite: Suite, filter?: TestFilter): Generator { + for (const entry of suite.entries()) { + if (entry.type !== 'test') + yield* visitTests(entry, filter); + else if (!filter || filter(entry)) + yield entry; + } +} + +export function filterTests(entries: T[], filter?: TestFilter): T[] { + return filter ? entries.filter(entry => entry.type === 'test' ? filter(entry) : !visitTests(entry, filter).next().done) : entries; +} + function formatResultErrors(screen: Screen, test: TestCase, result: TestResult): string { const lines: string[] = []; if (test.outcome() === 'unexpected') { diff --git a/packages/playwright/src/reporters/failures.ts b/packages/playwright/src/reporters/failures.ts new file mode 100644 index 0000000000000..cf063e5d09b15 --- /dev/null +++ b/packages/playwright/src/reporters/failures.ts @@ -0,0 +1,114 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { msToString } from '@isomorphic/formatUtils'; +import { getAsBooleanFromENV } from '@utils/env'; + +import { markErrorsAsReported, TerminalReporter } from './base'; + +import type { FailuresReporterOptions } from '../../types/test'; +import type { FullResult, TestCase, TestError, TestResult } from '../../types/testReporter'; +import type { CommonReporterOptions, TerminalReporterOptions } from './base'; + +class FailuresReporter extends TerminalReporter { + private _failureIndex = new Map(); + private _needNewLine = false; + + constructor(options?: FailuresReporterOptions & CommonReporterOptions & TerminalReporterOptions) { + super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_FAILURES_OMIT_TAGS', options?.omitTags), lastResult: true }); + } + + override onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult) { + super.onStdOut(chunk, test, result); + this._dumpToStdio(chunk, this.screen.stdout); + } + + override onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult) { + super.onStdErr(chunk, test, result); + this._dumpToStdio(chunk, this.screen.stderr); + } + + private _dumpToStdio(chunk: string | Buffer, stream: NodeJS.WriteStream) { + if (this.config.quiet) + return; + stream.write(chunk); + if (chunk.length) + this._needNewLine = !chunk.toString().endsWith('\n'); + } + + override onTestEnd(test: TestCase, result: TestResult) { + super.onTestEnd(test, result); + if (result.status !== 'skipped' && result.status !== test.expectedStatus) { + this._printFailure(test); + } else if (result.retry && result.status === 'passed') { + this._maybeWriteNewLine(); + this.writeLine(this.screen.colors.green(this.formatTestHeader(test, { indent: ' ', index: this._failureIndex.get(test) }))); + this.writeLine(this.screen.colors.green(` Passed on retry #${result.retry} (${msToString(result.duration)}).`)); + this.writeLine(); + } + } + + private _printFailure(test: TestCase) { + let index = this._failureIndex.get(test); + if (index === undefined) { + index = this._failureIndex.size + 1; + this._failureIndex.set(test, index); + } + const message = this.formatFailure(test, index); + if (message.trim()) { + this._maybeWriteNewLine(); + this.writeLine(message); + } + } + + override onError(error: TestError) { + super.onError(error); + this._maybeWriteNewLine(); + this.writeLine(this.formatError(error).message); + } + + async onTestPaused(test: TestCase, result: TestResult) { + // Without TTY, the user cannot interrupt the pause. + if (!process.stdin.isTTY && !process.env.PW_TEST_DEBUG_REPORTERS) + return; + + this._maybeWriteNewLine(); + if (test.outcome() === 'unexpected') { + this._printFailure(test); + markErrorsAsReported(result); + this.writeLine(this.screen.colors.yellow(' Paused on error. Press Ctrl+C to end.')); + } else { + this.writeLine(this.screen.colors.yellow(this.formatTestHeader(test, { indent: ' ' }))); + this.writeLine(this.screen.colors.yellow(' Paused at test end. Press Ctrl+C to end.')); + } + await new Promise(() => {}); + } + + override async onEnd(result: FullResult) { + await super.onEnd(result); + this._maybeWriteNewLine(); + this.epilogue(false); + } + + private _maybeWriteNewLine() { + if (this._needNewLine) { + this.writeLine(); + this._needNewLine = false; + } + } +} + +export default FailuresReporter; diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index da7ad610da842..647107d6e8500 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -36,11 +36,12 @@ import { extractZip } from '@utils/third_party/extractZip'; // `import('vite')`) is DCE'd in release builds. declare const __PW_HMR__: boolean; -import { CommonReporterOptions, formatError, formatResultFailure, internalScreen } from './base'; +import { createTestFilter, filterTests, formatError, formatResultFailure, internalScreen } from './base'; import * as babel from '../transform/babelBundle'; import { resolveReporterOutputPath, stripAnsiEscapes } from '../util'; import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2'; +import type { CommonReporterOptions, TestFilter } from './base'; import type { HtmlReporterOptions as HtmlReporterConfigOptions, Metadata, TestAnnotation } from '../../types/test'; import type * as api from '../../types/testReporter'; import type { HTMLReport, HTMLReportOptions, Location, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types'; @@ -164,7 +165,7 @@ class HtmlReporter implements ReporterV2 { noSnippets, noCopyPrompt, mergeFiles, - }); + }, createTestFilter(this._options)); this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors, this._machines); } @@ -339,13 +340,15 @@ class HtmlBuilder { private _attachmentsBaseURL: string; private _options: HTMLReportOptions; private _doNotInlineAssets: boolean; + private _testFilter: TestFilter | undefined; - constructor(yazl: typeof import('yazl'), config: api.FullConfig, outputDir: string, attachmentsBaseURL: string, doNotInlineAssets: boolean, options: HTMLReportOptions) { + constructor(yazl: typeof import('yazl'), config: api.FullConfig, outputDir: string, attachmentsBaseURL: string, doNotInlineAssets: boolean, options: HTMLReportOptions, testFilter?: TestFilter) { this._dataZipFile = new yazl.ZipFile(); this._config = config; this._reportFolder = outputDir; this._options = options; this._doNotInlineAssets = doNotInlineAssets; + this._testFilter = testFilter; fs.mkdirSync(this._reportFolder, { recursive: true }); this._attachmentsBaseURL = attachmentsBaseURL; } @@ -354,7 +357,7 @@ class HtmlBuilder { const data: DataMap = new Map(); for (const projectSuite of projectSuites) { const projectName = projectSuite.project()!.name; - for (const fileSuite of projectSuite.suites) { + for (const fileSuite of filterTests(projectSuite.suites, this._testFilter)) { const fileName = this._relativeLocation(fileSuite.location)!.file; this._createEntryForSuite(data, projectName, fileSuite, fileName, true); } @@ -505,7 +508,7 @@ class HtmlBuilder { private _processSuite(suite: api.Suite, projectName: string, path: string[], deep: boolean, outTests: TestEntry[]) { const newPath = [...path, suite.title]; - suite.entries().forEach(e => { + filterTests(suite.entries(), this._testFilter).forEach(e => { if (e.type === 'test') outTests.push(this._createTestEntry(e, projectName, newPath)); else if (deep) diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index 406df9ba87d55..a04406920595e 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -20,10 +20,11 @@ import path from 'path'; import { MultiMap } from '@isomorphic/multimap'; import { toPosixPath } from '@utils/fileUtils'; -import { formatError, nonTerminalScreen, prepareErrorStack, resolveOutputFile, CommonReporterOptions } from './base'; +import { createTestFilter, filterTests, formatError, nonTerminalScreen, prepareErrorStack, resolveOutputFile, visitTests } from './base'; import { config } from '../common'; import type { ReporterV2 } from './reporterV2'; +import type { CommonReporterOptions, TestFilter } from './base'; import type { JsonReporterOptions } from '../../types/test'; import type { FullConfig, FullResult, JSONReport, JSONReportError, JSONReportSpec, JSONReportSuite, JSONReportTest, JSONReportTestResult, JSONReportTestStep, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; @@ -32,9 +33,11 @@ class JSONReporter implements ReporterV2 { suite!: Suite; private _errors: TestError[] = []; private _resolvedOutputFile: string | undefined; + private _testFilter: TestFilter | undefined; constructor(options: JsonReporterOptions & CommonReporterOptions) { this._resolvedOutputFile = resolveOutputFile('JSON', options)?.outputFile; + this._testFilter = createTestFilter(options); } version(): 'v2' { @@ -92,7 +95,7 @@ class JSONReporter implements ReporterV2 { flaky: 0, }, }; - for (const test of this.suite.allTests()) + for (const test of visitTests(this.suite, this._testFilter)) ++report.stats[test.outcome()]; return report; } @@ -162,13 +165,15 @@ class JSONReporter implements ReporterV2 { } private _serializeSuite(projectId: string, projectName: string, suite: Suite): null | JSONReportSuite { - if (!suite.allTests().length) + const entries = filterTests(suite.entries(), this._testFilter); + const suites = entries.filter(entry => entry.type !== 'test').map(suite => this._serializeSuite(projectId, projectName, suite)).filter(s => s) as JSONReportSuite[]; + const tests = entries.filter(entry => entry.type === 'test'); + if (!tests.length && !suites.length) return null; - const suites = suite.suites.map(suite => this._serializeSuite(projectId, projectName, suite)).filter(s => s) as JSONReportSuite[]; return { title: suite.title, ...this._relativeLocation(suite.location), - specs: suite.tests.map(test => this._serializeTestSpec(projectId, projectName, test)), + specs: tests.map(test => this._serializeTestSpec(projectId, projectName, test)), suites: suites.length ? suites : undefined, }; } diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts index 82df28228344b..cc4cca59f1089 100644 --- a/packages/playwright/src/reporters/junit.ts +++ b/packages/playwright/src/reporters/junit.ts @@ -19,10 +19,11 @@ import path from 'path'; import { getAsBooleanFromENV } from '@utils/env'; -import { CommonReporterOptions, formatFailure, nonTerminalScreen, resolveOutputFile } from './base'; +import { createTestFilter, filterTests, formatFailure, nonTerminalScreen, resolveOutputFile, visitTests } from './base'; import { stripAnsiEscapes } from '../util'; import type { ReporterV2 } from './reporterV2'; +import type { CommonReporterOptions, TestFilter } from './base'; import type { JUnitReporterOptions } from '../../types/test'; import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '../../types/testReporter'; @@ -40,11 +41,13 @@ class JUnitReporter implements ReporterV2 { private includeProjectInTestName = false; private includeRetries = false; private omitTags = false; + private testFilter: TestFilter | undefined; constructor(options: JUnitReporterOptions & CommonReporterOptions) { + this.testFilter = createTestFilter(options); this.stripANSIControlSequences = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_STRIP_ANSI', !!options.stripANSIControlSequences); this.includeProjectInTestName = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_PROJECT_IN_TEST_NAME', !!options.includeProjectInTestName); - this.includeRetries = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_RETRIES', !!options.includeRetries); + this.includeRetries = !!options.onlyFailures || getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_RETRIES', !!options.includeRetries); this.omitTags = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_OMIT_TAGS', !!options.omitTags); this.configDir = options.configDir; this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile; @@ -70,7 +73,7 @@ class JUnitReporter implements ReporterV2 { async onEnd(result: FullResult) { const children: XMLEntry[] = []; for (const projectSuite of this.suite.suites) { - for (const fileSuite of projectSuite.suites) + for (const fileSuite of filterTests(projectSuite.suites, this.testFilter)) children.push(await this._buildTestSuite(projectSuite.title, fileSuite)); } const tokens: string[] = []; @@ -110,7 +113,7 @@ class JUnitReporter implements ReporterV2 { const children: XMLEntry[] = []; const testCaseNamePrefix = projectName && this.includeProjectInTestName ? `[${projectName}] ` : ''; - for (const test of suite.allTests()){ + for (const test of visitTests(suite, this.testFilter)) { ++tests; if (test.outcome() === 'skipped') ++skipped; diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index 34cff70b12c59..7b9bef3c18f7d 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -22,6 +22,7 @@ import { BlobReporter } from '../reporters/blob'; import CoverageReporter from '../reporters/coverage'; import DotReporter from '../reporters/dot'; import EmptyReporter from '../reporters/empty'; +import FailuresReporter from '../reporters/failures'; import GitHubReporter from '../reporters/github'; import HtmlReporter from '../reporters/html'; import JSONReporter from '../reporters/json'; @@ -44,6 +45,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | 'blob': BlobReporter, 'perfetto': PerfettoReporter, 'dot': mode === 'list' ? ListModeReporter : DotReporter, + 'failures': mode === 'list' ? ListModeReporter : FailuresReporter, 'line': mode === 'list' ? ListModeReporter : LineReporter, 'list': mode === 'list' ? ListModeReporter : ListReporter, 'github': GitHubReporter, @@ -58,9 +60,9 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | const reportOptions = reporterCommandOptions(config, mode, runOptions); for (const r of descriptions) { const [name, arg] = r; - const options = { ...reportOptions, ...arg }; + const options = resolveReporterOptions(reportOptions, { ...reportOptions, ...arg }); if (name in defaultReporters) { - reporters.push(new defaultReporters[name as keyof typeof defaultReporters](options)); + reporters.push(new defaultReporters[name as commonConfig.BuiltInReporter](options)); } else { const reporterConstructor = await loadReporter(config, name); reporters.push(wrapReporterAsV2(new reporterConstructor(options))); @@ -69,7 +71,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | if (process.env.PW_TEST_REPORTER) { const name = process.env.PW_TEST_REPORTER; if (name in defaultReporters) { - reporters.push(new defaultReporters[name as keyof typeof defaultReporters](reportOptions)); + reporters.push(new defaultReporters[name as commonConfig.BuiltInReporter](reportOptions)); } else { const reporterConstructor = await loadReporter(config, name); reporters.push(wrapReporterAsV2(new reporterConstructor(reportOptions))); @@ -109,9 +111,19 @@ function reporterCommandOptions(config: FullConfigInternal, mode: 'list' | 'test configDir: config.configDir, _mode: mode, _commandHash: computeCommandHash(config, runOptions), + onlyFailures: mode !== 'list' && config.configCLIOverrides.reporterOnlyFailures, }; } +function resolveReporterOptions(commonOptions: CommonReporterOptions, options: CommonReporterOptions): CommonReporterOptions { + let onlyFailures = options.onlyFailures; + if (commonOptions._mode === 'list') + onlyFailures = false; + else if (commonOptions.onlyFailures) + onlyFailures = true; + return { ...options, onlyFailures }; +} + function computeCommandHash(config: FullConfigInternal, runOptions?: TestRunOptions) { const parts = []; // Include project names for readability. diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 1b6826812a531..fdd2114fc7e7d 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -20,13 +20,15 @@ export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; export type DotReporterOptions = { omitTags?: boolean }; +export type FailuresReporterOptions = { omitTags?: boolean }; export type LineReporterOptions = { omitTags?: boolean }; export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: boolean, omitTags?: boolean, printWorkerIndex?: boolean }; export type GitHubReporterOptions = { omitTags?: boolean }; -export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; -export type JsonReporterOptions = { outputFile?: string }; +export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean, onlyFailures?: boolean }; +export type JsonReporterOptions = { outputFile?: string, onlyFailures?: boolean }; export type PerfettoReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { + onlyFailures?: boolean; outputFolder?: string; open?: 'always' | 'never' | 'on-failure'; host?: string; @@ -42,6 +44,7 @@ export type HtmlReporterOptions = { export type ReporterDescription = Readonly< ['blob'] | ['blob', BlobReporterOptions] | ['dot'] | ['dot', DotReporterOptions] | + ['failures'] | ['failures', FailuresReporterOptions] | ['line'] | ['line', LineReporterOptions] | ['list'] | ['list', ListReporterOptions] | ['github'] | ['github', GitHubReporterOptions] | @@ -961,7 +964,7 @@ interface TestConfig { * ``` * */ - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'blob'|'perfetto', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'failures'|'line'|'github'|'json'|'junit'|'null'|'html'|'blob'|'perfetto', string> | ReporterDescription[]; /** * Global options for all tests, for example * [testOptions.browserName](https://playwright.dev/docs/api/class-testoptions#test-options-browser-name). Learn more diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index a7962739df19e..12a5618bb1b63 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -92,6 +92,8 @@ export interface FullResult { * }); * ``` * + * When `--reporter-only-failures` is set, custom reporters receive `onlyFailures: true` in their constructor options. + * * Here is a typical order of reporter calls: * - [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin) is called * once with a root suite that contains all other suites and tests. Learn more about diff --git a/tests/playwright-test/reporter-only-failures.spec.ts b/tests/playwright-test/reporter-only-failures.spec.ts new file mode 100644 index 0000000000000..6442b2b31320a --- /dev/null +++ b/tests/playwright-test/reporter-only-failures.spec.ts @@ -0,0 +1,402 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; +import xml2js from 'xml2js'; +import { test, expect, stripAnsi } from './playwright-test-fixtures'; +import { extractZip } from '../../packages/utils/third_party/extractZip'; + +import type { HTMLReport, TestFile } from '../../packages/html-reporter/src/types'; +import type { JSONReport, JSONReportSpec, JSONReportSuite } from '@playwright/test/reporter'; + +const passingTest = ` + import { test } from '@playwright/test'; + test('passes', () => {}); +`; + +const testFiles = { + 'passing.test.ts': ` + import fs from 'fs'; + import { test } from '@playwright/test'; + test('passes', async ({}, testInfo) => { + fs.writeFileSync('passing-ran.txt', 'ran'); + const attachmentPath = testInfo.outputPath('passing.txt'); + fs.writeFileSync(attachmentPath, 'passing attachment'); + await testInfo.attach('passing', { path: attachmentPath }); + }); + `, + 'failures.test.ts': ` + import fs from 'fs'; + import { test, expect } from '@playwright/test'; + test.describe('failures', () => { + test('fails', async ({}, testInfo) => { + const attachmentPath = testInfo.outputPath('failure.txt'); + fs.writeFileSync(attachmentPath, 'failure attachment'); + await testInfo.attach('failure', { path: attachmentPath }); + expect(1).toBe(2); + }); + test('flaky', ({}, testInfo) => { + console.log('flaky attempt ' + testInfo.retry); + console.error('flaky stderr ' + testInfo.retry); + if (!testInfo.retry) + throw new Error('flaky failure'); + }); + test('unexpectedly passes', () => { + test.fail(); + }); + }); + test.describe('ignored', () => { + test.skip('skipped', () => {}); + test('expected failure', () => { + test.fail(); + throw new Error('expected failure'); + }); + }); + `, +}; + +test('--reporter-only-failures leaves terminal reporters unchanged', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': passingTest, + }, { 'reporter': 'list,line,dot', 'reporter-only-failures': true }, { PLAYWRIGHT_FORCE_TTY: '0' }); + expect(result.exitCode).toBe(0); + expect(result.output.match(/Running 1 test using/g)).toHaveLength(3); + expect(result.output.match(/› passes/g)).toHaveLength(2); + expect(result.output).toContain('·'); +}); + +test('failures reporter accounts for passing and failing test durations', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { + reporter: 'failures', + fullyParallel: false, + reportSlowTests: { max: 1, threshold: 300 }, + }; + `, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('passes', async () => { + await new Promise(resolve => setTimeout(resolve, 200)); + }); + test('fails', async () => { + await new Promise(resolve => setTimeout(resolve, 200)); + throw new Error('failure'); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toContain('Slow test file: a.test.ts'); +}); + +test('failures reporter respects quiet and reports errors outside tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { + reporter: 'failures', + quiet: true, + globalTeardown: './global-teardown.ts', + }; + `, + 'global-teardown.ts': ` + export default () => { + throw new Error('global teardown failed'); + }; + `, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('passes', () => { + process.stdout.write('test stdout'); + process.stderr.write('test stderr'); + }); + test.skip('skipped', () => {}); + test('expected failure', () => { + test.fail(); + throw new Error('expected failure'); + }); + `, + }, {}, { PLAYWRIGHT_FORCE_TTY: '1' }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(0); + expect(result.passed).toBe(2); + expect(result.output).not.toContain('test stdout'); + expect(result.output).not.toContain('test stderr'); + expect(result.output.match(/Error: global teardown failed/g)).toHaveLength(1); + expect(result.output).toContain('1 error was not a part of any test'); + expect(result.output).toContain('1 skipped'); + expect(result.output).not.toContain('Running 1 test using'); + expect(result.output).not.toMatch(/› (?:passes|skipped|expected failure)/); + expect(result.rawOutput).not.toContain('\u001B[1A'); +}); + +test('failures reporter avoids automatic terminal output', async ({ runInlineTest }) => { + const files = { + 'playwright.config.ts': ` + export default { reporter: [['json', { outputFile: 'report.json' }]] }; + `, + 'a.test.ts': passingTest, + }; + const fallback = await runInlineTest(files, { 'reporter-only-failures': true }, { CI: undefined }); + expect(fallback.exitCode).toBe(0); + expect(fallback.output).toContain('Running 1 test using'); + expect(fallback.output).toContain('› passes'); + const added = await runInlineTest(files, { 'reporter-only-failures': true, 'add-reporter': 'failures' }, { CI: undefined }); + expect(added.exitCode).toBe(0); + expect(added.output).toMatch(/^ 1 passed \([^)]+\)\n$/); +}); + +test('onlyFailures can be configured independently for each reporter', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + export default { + reporter: [ + ['failures'], + ['json', { onlyFailures: true, outputFile: 'failures.json' }], + ['json', { outputFile: 'complete.json' }], + ], + }; + `, + 'a.test.ts': passingTest + ` + test('fails', () => { + throw new Error('failure'); + }); + `, + }, { workers: 1 }); + expect(result.exitCode).toBe(1); + expect(result.output).not.toContain('› passes'); + expect(result.passed).toBe(1); + const failures: JSONReport = JSON.parse(fs.readFileSync(testInfo.outputPath('failures.json'), 'utf8')); + const complete: JSONReport = JSON.parse(fs.readFileSync(testInfo.outputPath('complete.json'), 'utf8')); + expect(jsonTestSpecs(failures.suites).map(spec => spec.title)).toEqual(['fails']); + expect(jsonTestSpecs(complete.suites).map(spec => spec.title)).toEqual(['passes', 'fails']); +}); + +for (const merge of [false, true]) { + test(`--reporter-only-failures filters final reports and preserves blobs when ${merge ? 'merging' : 'running'}`, async ({ runInlineTest, mergeReports }, testInfo) => { + const result = await runInlineTest({ + ...testFiles, + 'playwright.config.ts': ` + export default { + retries: 1, + reporter: [ + ['failures'], + ['json', { outputFile: 'report.json' }], + ['junit', { outputFile: 'report.xml', includeRetries: false }], + ['html', { open: 'never' }], + ['blob', { outputDir: 'complete-blob-report' }], + ['perfetto', { outputFile: 'perfetto.json' }], + ['./integrity-reporter.ts', { onlyFailures: false }], + ], + }; + `, + 'integrity-reporter.ts': ` + export default class { + constructor(options: { onlyFailures?: boolean }) { + console.log('%%onlyFailures: ' + options.onlyFailures); + } + onBegin(config, suite) { + this.suite = suite; + console.log('%%begin: ' + suite.allTests().length); + } + onTestBegin(test) { + console.log('%%begin: ' + test.title); + } + onTestEnd(test) { + console.log('%%end: ' + test.title); + } + onEnd() { + console.log('%%end: ' + this.suite.allTests().length); + } + } + `, + }, merge ? { reporter: 'blob', workers: 1 } : { 'reporter-only-failures': true, 'workers': 1 }, { PLAYWRIGHT_JUNIT_INCLUDE_RETRIES: '0' }); + expect(result.exitCode).toBe(1); + let output = result.output; + if (merge) { + const merged = await mergeReports(testInfo.outputPath('blob-report'), { PLAYWRIGHT_JUNIT_INCLUDE_RETRIES: '0' }, { + cwd: testInfo.outputPath(), + additionalArgs: ['--config', testInfo.outputPath('playwright.config.ts'), '--reporter-only-failures'], + }); + expect(merged.exitCode).toBe(0); + output = stripAnsi(merged.output); + } + expect(output).toContain('2 passed'); + expect(output).toContain('2 failed'); + expect(output).toContain('1 flaky'); + expect(output).toContain('Passed on retry #1'); + expect(output.match(/^ Error: expect\(received\)/gm)).toHaveLength(2); + expect(output.match(/^ Error: flaky failure/gm)).toHaveLength(1); + expect(output).toContain('Expected: 2'); + expect(output).toContain('Received: 1'); + expect(output).toMatch(/> +\d+ \| +expect\(1\)\.toBe\(2\);/); + expect(output).toMatch(/at .*failures\.test\.ts:\d+:\d+/); + expect(output).not.toContain('Running 6 tests'); + expect(output).not.toContain('passing.test.ts'); + expect(output).toContain('%%begin: 6'); + expect(output).toContain('%%end: 6'); + expect(output).toContain('%%onlyFailures: true'); + expect(fs.existsSync(testInfo.outputPath('passing-ran.txt'))).toBe(true); + + const titles = ['fails', 'flaky', 'unexpectedly passes']; + const allTitles = [...titles, 'passes', 'skipped', 'expected failure'].sort(); + for (const title of allTitles) { + expect(output).toContain(`%%begin: ${title}`); + expect(output).toContain(`%%end: ${title}`); + } + for (const retry of [0, 1]) { + expect(output).toContain(`flaky attempt ${retry}`); + expect(output).toContain(`flaky stderr ${retry}`); + } + const json: JSONReport = JSON.parse(fs.readFileSync(testInfo.outputPath('report.json'), 'utf8')); + const specs = jsonTestSpecs(json.suites); + expect(specs.map(spec => spec.title).sort()).toEqual(titles); + expect(json.stats).toMatchObject({ expected: 0, skipped: 0, unexpected: 2, flaky: 1 }); + for (const spec of specs) { + const statuses = spec.title === 'flaky' ? ['failed', 'passed'] : spec.title === 'fails' ? ['failed', 'failed'] : ['passed', 'passed']; + expect(spec.tests[0].results.map(result => result.status)).toEqual(statuses); + } + + const xml = await xml2js.parseStringPromise(fs.readFileSync(testInfo.outputPath('report.xml'), 'utf8')); + expect(xml.testsuites.$.tests).toBe('3'); + expect(xml.testsuites.testsuite).toHaveLength(1); + expect(xml.testsuites.testsuite[0].testcase.map((entry: { $: { name: string } }) => entry.$.name).sort()).toEqual(titles.map(title => `failures › ${title}`)); + const junitTests = xml.testsuites.testsuite[0].testcase; + expect(junitTests[0].failure).toHaveLength(1); + expect(junitTests[0].rerunFailure).toHaveLength(1); + expect(junitTests[1].flakyError).toHaveLength(1); + expect(junitTests[1].flakyError[0]['system-out'][0]).toContain('flaky attempt 0'); + expect(junitTests[1]['system-out'][0]).toContain('flaky attempt 1'); + + const html = fs.readFileSync(testInfo.outputPath('playwright-report', 'index.html'), 'utf8'); + const data = html.match(/