diff --git a/.changeset/fail-on-zero-executed-tests.md b/.changeset/fail-on-zero-executed-tests.md new file mode 100644 index 0000000000..71adbce1f8 --- /dev/null +++ b/.changeset/fail-on-zero-executed-tests.md @@ -0,0 +1,8 @@ +--- +'@web/test-runner-core': major +'@web/test-runner': major +--- + +A test run now fails when all test files loaded but no tests were executed, instead of silently passing with exit code 0. Skipped tests count as executed, and watch mode and manual mode are unaffected. + +Restore the old behavior with the new `passWithNoTests` option or the `--pass-with-no-tests` CLI flag. Note that test sessions which do not report test results, for example from a custom test framework, are now treated as having executed zero tests. diff --git a/docs/docs/test-runner/cli-and-configuration.md b/docs/docs/test-runner/cli-and-configuration.md index 7e718f000b..5b986dcdec 100644 --- a/docs/docs/test-runner/cli-and-configuration.md +++ b/docs/docs/test-runner/cli-and-configuration.md @@ -16,6 +16,7 @@ The test runner can be configured using CLI flags, or with a configuration file. | static-logging | boolean | Disables rendering a progress bar dynamically to the terminal. | | manual | boolean | Starts test runner in manual testing mode. Ignores browsers option and prints manual testing URL. | | open | boolean | Opens browser for manual testing. Requires the manual option to be set. | +| pass-with-no-tests | boolean | pass the test run when no tests were executed | | port | number | Port to bind the server on. | | groups | string | pattern of where to read test group config files from | | group | string | runs tests only for the test group with this name | @@ -188,6 +189,8 @@ interface TestRunnerConfig { manual?: boolean; /** Opens browser for manual testing. Requires the manual option to be set. */ open?: boolean; + /** Passes the test run when no tests were executed. By default a test run fails when none of the test files executed any tests. */ + passWithNoTests?: boolean; // how long a browser can take to start up before failing. defaults to 30000 (30 sec) browserStartTimeout?: number; diff --git a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts index 6e694df72b..6094c461fd 100644 --- a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts +++ b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts @@ -63,6 +63,8 @@ export interface TestRunnerCoreConfig { manual?: boolean; /** Opens browser for manual testing. Requires the manual option to be set. */ open?: boolean; + /** Passes the test run when no tests were executed. By default a test run fails when none of the test files executed any tests. */ + passWithNoTests?: boolean; debug?: boolean; mimeTypes?: Record; diff --git a/packages/test-runner-core/src/runner/TestRunner.ts b/packages/test-runner-core/src/runner/TestRunner.ts index 31f3f15a39..66ef12ba4d 100644 --- a/packages/test-runner-core/src/runner/TestRunner.ts +++ b/packages/test-runner-core/src/runner/TestRunner.ts @@ -7,6 +7,7 @@ import { type TestSession } from '../test-session/TestSession.js'; import { TestSessionManager } from '../test-session/TestSessionManager.js'; import { SESSION_STATUS } from '../test-session/TestSessionStatus.js'; import { EventEmitter } from '../utils/EventEmitter.js'; +import { getExecutedTestCount } from '../utils/getExecutedTestCount.js'; import { createDebugSessions } from './createDebugSessions.js'; import { createTestSessions } from './createSessionGroups.js'; import { createSessionUrl } from './createSessionUrl.js'; @@ -222,7 +223,13 @@ export class TestRunner extends EventEmitter { if (!this.config.watch) { setTimeout(async () => { - this.passed = passedCoverage && Array.from(this.sessions.failed()).length === 0; + const noFailedSessions = Array.from(this.sessions.failed()).length === 0; + const hasExecutedTests = this.countExecutedTests() > 0; + // manual mode is for interactive testing and does not require executed tests. + this.passed = + passedCoverage && + noFailedSessions && + (this.config.manual || this.config.passWithNoTests || hasExecutedTests); this.emit('finished', this.passed); }); } @@ -231,4 +238,14 @@ export class TestRunner extends EventEmitter { this.stop(error); } } + + private countExecutedTests(): number { + let count = 0; + for (const session of this.sessions.all()) { + if (session.testResults) { + count += getExecutedTestCount(session.testResults); + } + } + return count; + } } diff --git a/packages/test-runner-core/src/utils/getExecutedTestCount.ts b/packages/test-runner-core/src/utils/getExecutedTestCount.ts new file mode 100644 index 0000000000..4cdd497903 --- /dev/null +++ b/packages/test-runner-core/src/utils/getExecutedTestCount.ts @@ -0,0 +1,15 @@ +import { type TestSuiteResult } from '../test-session/TestSession.js'; + +/** + * Counts the tests executed as part of a test result tree, including + * skipped tests. + */ +export function getExecutedTestCount(testResults: TestSuiteResult): number { + let count = testResults.tests.length; + + for (const childSuite of testResults.suites) { + count += getExecutedTestCount(childSuite); + } + + return count; +} diff --git a/packages/test-runner-core/test/src/runner/TestRunner.test.ts b/packages/test-runner-core/test/src/runner/TestRunner.test.ts index 29aa9b5633..b6a7f6f835 100644 --- a/packages/test-runner-core/test/src/runner/TestRunner.test.ts +++ b/packages/test-runner-core/test/src/runner/TestRunner.test.ts @@ -11,6 +11,8 @@ import type { Logger } from '../../../dist/logger/Logger.js'; import { TestRunner } from '../../../dist/runner/TestRunner.js'; import { SESSION_STATUS } from '../../../dist/test-session/TestSessionStatus.js'; +import { type TestSuiteResult } from '../../../dist/test-session/TestSession.js'; + function createBrowserStub(): BrowserLauncher { return { name: 'myBrowser', @@ -72,6 +74,23 @@ async function createTestRunner( return { runner, browser }; } +function createTestResults(...tests: TestSuiteResult['tests']): TestSuiteResult { + return { name: 'root', suites: [], tests }; +} + +const passedTest: TestSuiteResult['tests'][number] = { + name: 'my test', + passed: true, + skipped: false, +}; + +const failedTest: TestSuiteResult['tests'][number] = { + name: 'my test', + passed: false, + skipped: false, + error: { message: 'test failed' }, +}; + describe('TestRunner', function () { it('can run a single test file', async () => { const { runner, browser } = await createTestRunner(); @@ -101,7 +120,10 @@ describe('TestRunner', function () { await runner.start(); const sessions = Array.from(runner.sessions.all()); - runner.sessions.updateStatus({ ...sessions[0], passed: true }, SESSION_STATUS.TEST_FINISHED); + runner.sessions.updateStatus( + { ...sessions[0], passed: true, testResults: createTestResults(passedTest) }, + SESSION_STATUS.TEST_FINISHED, + ); const passed = await stopped; @@ -126,7 +148,10 @@ describe('TestRunner', function () { await runner.start(); const sessions = Array.from(runner.sessions.all()); - runner.sessions.updateStatus({ ...sessions[0], passed: false }, SESSION_STATUS.TEST_FINISHED); + runner.sessions.updateStatus( + { ...sessions[0], passed: false, testResults: createTestResults(failedTest) }, + SESSION_STATUS.TEST_FINISHED, + ); const passed = await stopped; assert.equal(browser.stopSession.mock.callCount(), 1, 'browser session is stopped'); @@ -134,6 +159,85 @@ describe('TestRunner', function () { assert.equal(passed, false, 'test runner quits with false'); }); + it('fails the test run when no tests were executed', async () => { + const { runner } = await createTestRunner(); + let resolveStopped: (passed: boolean) => void; + const stopped = new Promise(resolve => { + resolveStopped = resolve; + }); + runner.on('finished', () => { + runner.stop(); + }); + runner.on('stopped', passed => { + resolveStopped(passed); + }); + + await runner.start(); + + const sessions = Array.from(runner.sessions.all()); + runner.sessions.updateStatus( + { ...sessions[0], passed: true, testResults: createTestResults() }, + SESSION_STATUS.TEST_FINISHED, + ); + const passed = await stopped; + + assert.equal(passed, false, 'test runner quits with false when no tests were executed'); + }); + + it('passes the test run when no tests were executed and passWithNoTests is enabled', async () => { + const { runner } = await createTestRunner({ passWithNoTests: true }); + let resolveStopped: (passed: boolean) => void; + const stopped = new Promise(resolve => { + resolveStopped = resolve; + }); + runner.on('finished', () => { + runner.stop(); + }); + runner.on('stopped', passed => { + resolveStopped(passed); + }); + + await runner.start(); + + const sessions = Array.from(runner.sessions.all()); + runner.sessions.updateStatus( + { ...sessions[0], passed: true, testResults: createTestResults() }, + SESSION_STATUS.TEST_FINISHED, + ); + const passed = await stopped; + + assert.equal(passed, true, 'test runner quits with true when passWithNoTests is enabled'); + }); + + it('counts skipped tests as executed tests', async () => { + const { runner } = await createTestRunner(); + let resolveStopped: (passed: boolean) => void; + const stopped = new Promise(resolve => { + resolveStopped = resolve; + }); + runner.on('finished', () => { + runner.stop(); + }); + runner.on('stopped', passed => { + resolveStopped(passed); + }); + + await runner.start(); + + const sessions = Array.from(runner.sessions.all()); + runner.sessions.updateStatus( + { + ...sessions[0], + passed: true, + testResults: createTestResults({ name: 'skipped test', passed: false, skipped: true }), + }, + SESSION_STATUS.TEST_FINISHED, + ); + const passed = await stopped; + + assert.equal(passed, true, 'test runner quits with true when only skipped tests ran'); + }); + describe('groups', () => { it('can create a group in addition to the default group', async () => { const { runner } = await createTestRunner(undefined, [ diff --git a/packages/test-runner/src/config/parseConfig.ts b/packages/test-runner/src/config/parseConfig.ts index 9963aa359a..0c438dcea4 100644 --- a/packages/test-runner/src/config/parseConfig.ts +++ b/packages/test-runner/src/config/parseConfig.ts @@ -81,6 +81,7 @@ const booleanSettings = [ 'staticLogging', 'manual', 'open', + 'passWithNoTests', 'debug', ]; diff --git a/packages/test-runner/src/config/readCliArgs.ts b/packages/test-runner/src/config/readCliArgs.ts index d5305ed698..60804b8123 100644 --- a/packages/test-runner/src/config/readCliArgs.ts +++ b/packages/test-runner/src/config/readCliArgs.ts @@ -16,6 +16,7 @@ export interface TestRunnerCliArgs extends Partial< | 'staticLogging' | 'manual' | 'open' + | 'passWithNoTests' | 'port' | 'preserveSymlinks' | 'nodeResolve' @@ -87,6 +88,12 @@ const options: OptionDefinition[] = [ type: Boolean, description: 'Opens browser for manual testing. Requires the manual option to be set.', }, + { + name: 'pass-with-no-tests', + type: Boolean, + description: + 'Passes the test run when no tests were executed. By default the test run fails when none of the test files executed any tests.', + }, { name: 'port', type: Number, diff --git a/packages/test-runner/src/reporter/getTestProgress.ts b/packages/test-runner/src/reporter/getTestProgress.ts index 1a60944069..f5d13b965e 100644 --- a/packages/test-runner/src/reporter/getTestProgress.ts +++ b/packages/test-runner/src/reporter/getTestProgress.ts @@ -77,6 +77,14 @@ export function getTestProgressReport(config: TestRunnerCoreConfig, args: TestPr const finishedFiles = new Set(); let failedTestCount = 0; let failed = false; + let executedTestCount = 0; + + for (const session of sessions.all()) { + if (session.testResults) { + const parsed = getPassedFailedSkippedCount(session.testResults); + executedTestCount += parsed.passed + parsed.failed + parsed.skipped; + } + } const longestBrowser = [...browserNames].sort((a, b) => b.length - a.length)[0]; const minWidth = longestBrowser ? longestBrowser.length + 1 : 0; @@ -167,6 +175,15 @@ export function getTestProgressReport(config: TestRunnerCoreConfig, args: TestPr } else { entries.push(bold(red(`Failed to run any tests.`))); } + } else if (executedTestCount === 0 && !config.passWithNoTests) { + entries.push( + bold(red(`Finished running tests in ${duration}s, but no tests were executed.`)), + ); + entries.push( + bold( + `A test run without executed tests fails. Add tests to your test files, or enable the "passWithNoTests" option to pass on test runs without executed tests.`, + ), + ); } else { entries.push(bold(`Finished running tests in ${duration}s, all tests passed! 🎉`)); }