feat(test-runner): fail test runs with zero executed tests

.changeset/fail-on-zero-executed-tests.md

@@ -0,0 +1,8 @@
+---
+'@web/test-runner-core': major
+'@web/test-runner': major
+---
+
+A test run now fails when all test files loaded but no tests were executed, instead of silently passing with exit code 0. Skipped tests count as executed, and watch mode and manual mode are unaffected.
+
+Restore the old behavior with the new `passWithNoTests` option or the `--pass-with-no-tests` CLI flag. Note that test sessions which do not report test results, for example from a custom test framework, are now treated as having executed zero tests.

docs/docs/test-runner/cli-and-configuration.md

@@ -16,6 +16,7 @@ The test runner can be configured using CLI flags, or with a configuration file.
 | static-logging      | boolean           | Disables rendering a progress bar dynamically to the terminal.                                                        |
 | manual              | boolean           | Starts test runner in manual testing mode. Ignores browsers option and prints manual testing URL.                     |
 | open                | boolean           | Opens browser for manual testing. Requires the manual option to be set.                                               |
+| pass-with-no-tests  | boolean           | pass the test run when no tests were executed                                                                         |
 | port                | number            | Port to bind the server on.                                                                                           |
 | groups              | string            | pattern of where to read test group config files from                                                                 |
 | group               | string            | runs tests only for the test group with this name                                                                     |
@@ -188,6 +189,8 @@ interface TestRunnerConfig {
   manual?: boolean;
   /** Opens browser for manual testing. Requires the manual option to be set. */
   open?: boolean;
+  /** Passes the test run when no tests were executed. By default a test run fails when none of the test files executed any tests. */
+  passWithNoTests?: boolean;
 
   // how long a browser can take to start up before failing. defaults to 30000 (30 sec)
   browserStartTimeout?: number;

packages/test-runner-core/src/config/TestRunnerCoreConfig.ts

@@ -63,6 +63,8 @@ export interface TestRunnerCoreConfig {
   manual?: boolean;
   /** Opens browser for manual testing. Requires the manual option to be set. */
   open?: boolean;
+  /** Passes the test run when no tests were executed. By default a test run fails when none of the test files executed any tests. */
+  passWithNoTests?: boolean;
 
   debug?: boolean;
   mimeTypes?: Record<string, string>;

packages/test-runner-core/src/runner/TestRunner.ts

@@ -7,6 +7,7 @@ import { type TestSession } from '../test-session/TestSession.js';
 import { TestSessionManager } from '../test-session/TestSessionManager.js';
 import { SESSION_STATUS } from '../test-session/TestSessionStatus.js';
 import { EventEmitter } from '../utils/EventEmitter.js';
+import { getExecutedTestCount } from '../utils/getExecutedTestCount.js';
 import { createDebugSessions } from './createDebugSessions.js';
 import { createTestSessions } from './createSessionGroups.js';
 import { createSessionUrl } from './createSessionUrl.js';
@@ -222,7 +223,13 @@ export class TestRunner extends EventEmitter<EventMap> {
 
         if (!this.config.watch) {
           setTimeout(async () => {
-            this.passed = passedCoverage && Array.from(this.sessions.failed()).length === 0;
+            const noFailedSessions = Array.from(this.sessions.failed()).length === 0;
+            const hasExecutedTests = this.countExecutedTests() > 0;
+            // manual mode is for interactive testing and does not require executed tests.
+            this.passed =
+              passedCoverage &&
+              noFailedSessions &&
+              (this.config.manual || this.config.passWithNoTests || hasExecutedTests);
             this.emit('finished', this.passed);
           });
         }
@@ -231,4 +238,14 @@ export class TestRunner extends EventEmitter<EventMap> {
       this.stop(error);
     }
   }
+
+  private countExecutedTests(): number {
+    let count = 0;
+    for (const session of this.sessions.all()) {
+      if (session.testResults) {
+        count += getExecutedTestCount(session.testResults);
+      }
+    }
+    return count;
+  }
 }

packages/test-runner-core/src/utils/getExecutedTestCount.ts

@@ -0,0 +1,15 @@
+import { type TestSuiteResult } from '../test-session/TestSession.js';
+
+/**
+ * Counts the tests executed as part of a test result tree, including
+ * skipped tests.
+ */
+export function getExecutedTestCount(testResults: TestSuiteResult): number {
+  let count = testResults.tests.length;
+
+  for (const childSuite of testResults.suites) {
+    count += getExecutedTestCount(childSuite);
+  }
+
+  return count;
+}

packages/test-runner-core/test/src/runner/TestRunner.test.ts

@@ -11,6 +11,8 @@ import type { Logger } from '../../../dist/logger/Logger.js';
 import { TestRunner } from '../../../dist/runner/TestRunner.js';
 import { SESSION_STATUS } from '../../../dist/test-session/TestSessionStatus.js';
 
+import { type TestSuiteResult } from '../../../dist/test-session/TestSession.js';
+
 function createBrowserStub(): BrowserLauncher {
   return {
     name: 'myBrowser',
@@ -72,6 +74,23 @@ async function createTestRunner(
   return { runner, browser };
 }
 
+function createTestResults(...tests: TestSuiteResult['tests']): TestSuiteResult {
+  return { name: 'root', suites: [], tests };
+}
+
+const passedTest: TestSuiteResult['tests'][number] = {
+  name: 'my test',
+  passed: true,
+  skipped: false,
+};
+
+const failedTest: TestSuiteResult['tests'][number] = {
+  name: 'my test',
+  passed: false,
+  skipped: false,
+  error: { message: 'test failed' },
+};
+
 describe('TestRunner', function () {
   it('can run a single test file', async () => {
     const { runner, browser } = await createTestRunner();
@@ -101,7 +120,10 @@ describe('TestRunner', function () {
     await runner.start();
 
     const sessions = Array.from(runner.sessions.all());
-    runner.sessions.updateStatus({ ...sessions[0], passed: true }, SESSION_STATUS.TEST_FINISHED);
+    runner.sessions.updateStatus(
+      { ...sessions[0], passed: true, testResults: createTestResults(passedTest) },
+      SESSION_STATUS.TEST_FINISHED,
+    );
 
     const passed = await stopped;
 
@@ -126,14 +148,96 @@ describe('TestRunner', function () {
     await runner.start();
 
     const sessions = Array.from(runner.sessions.all());
-    runner.sessions.updateStatus({ ...sessions[0], passed: false }, SESSION_STATUS.TEST_FINISHED);
+    runner.sessions.updateStatus(
+      { ...sessions[0], passed: false, testResults: createTestResults(failedTest) },
+      SESSION_STATUS.TEST_FINISHED,
+    );
     const passed = await stopped;
 
     assert.equal(browser.stopSession.mock.callCount(), 1, 'browser session is stopped');
     assert.equal(browser.stop.mock.callCount(), 1, 'browser is stopped');
     assert.equal(passed, false, 'test runner quits with false');
   });
 
+  it('fails the test run when no tests were executed', async () => {
+    const { runner } = await createTestRunner();
+    let resolveStopped: (passed: boolean) => void;
+    const stopped = new Promise<boolean>(resolve => {
+      resolveStopped = resolve;
+    });
+    runner.on('finished', () => {
+      runner.stop();
+    });
+    runner.on('stopped', passed => {
+      resolveStopped(passed);
+    });
+
+    await runner.start();
+
+    const sessions = Array.from(runner.sessions.all());
+    runner.sessions.updateStatus(
+      { ...sessions[0], passed: true, testResults: createTestResults() },
+      SESSION_STATUS.TEST_FINISHED,
+    );
+    const passed = await stopped;
+
+    assert.equal(passed, false, 'test runner quits with false when no tests were executed');
+  });
+
+  it('passes the test run when no tests were executed and passWithNoTests is enabled', async () => {
+    const { runner } = await createTestRunner({ passWithNoTests: true });
+    let resolveStopped: (passed: boolean) => void;
+    const stopped = new Promise<boolean>(resolve => {
+      resolveStopped = resolve;
+    });
+    runner.on('finished', () => {
+      runner.stop();
+    });
+    runner.on('stopped', passed => {
+      resolveStopped(passed);
+    });
+
+    await runner.start();
+
+    const sessions = Array.from(runner.sessions.all());
+    runner.sessions.updateStatus(
+      { ...sessions[0], passed: true, testResults: createTestResults() },
+      SESSION_STATUS.TEST_FINISHED,
+    );
+    const passed = await stopped;
+
+    assert.equal(passed, true, 'test runner quits with true when passWithNoTests is enabled');
+  });
+
+  it('counts skipped tests as executed tests', async () => {
+    const { runner } = await createTestRunner();
+    let resolveStopped: (passed: boolean) => void;
+    const stopped = new Promise<boolean>(resolve => {
+      resolveStopped = resolve;
+    });
+    runner.on('finished', () => {
+      runner.stop();
+    });
+    runner.on('stopped', passed => {
+      resolveStopped(passed);
+    });
+
+    await runner.start();
+
+    const sessions = Array.from(runner.sessions.all());
+    runner.sessions.updateStatus(
+      {
+        ...sessions[0],
+        passed: true,
+        testResults: createTestResults({ name: 'skipped test', passed: false, skipped: true }),
+      },
+      SESSION_STATUS.TEST_FINISHED,
+    );
+    const passed = await stopped;
+
+    assert.equal(passed, true, 'test runner quits with true when only skipped tests ran');
+  });
+
   describe('groups', () => {
     it('can create a group in addition to the default group', async () => {
       const { runner } = await createTestRunner(undefined, [

packages/test-runner/src/config/parseConfig.ts

@@ -81,6 +81,7 @@ const booleanSettings = [
   'staticLogging',
   'manual',
   'open',
+  'passWithNoTests',
   'debug',
 ];
 

packages/test-runner/src/config/readCliArgs.ts

@@ -16,6 +16,7 @@ export interface TestRunnerCliArgs extends Partial<
     | 'staticLogging'
     | 'manual'
     | 'open'
+    | 'passWithNoTests'
     | 'port'
     | 'preserveSymlinks'
     | 'nodeResolve'
@@ -87,6 +88,12 @@ const options: OptionDefinition[] = [
     type: Boolean,
     description: 'Opens browser for manual testing. Requires the manual option to be set.',
   },
+  {
+    name: 'pass-with-no-tests',
+    type: Boolean,
+    description:
+      'Passes the test run when no tests were executed. By default the test run fails when none of the test files executed any tests.',
+  },
   {
     name: 'port',
     type: Number,

packages/test-runner/src/reporter/getTestProgress.ts

@@ -77,6 +77,14 @@ export function getTestProgressReport(config: TestRunnerCoreConfig, args: TestPr
   const finishedFiles = new Set<string>();
   let failedTestCount = 0;
   let failed = false;
+  let executedTestCount = 0;
+
+  for (const session of sessions.all()) {
+    if (session.testResults) {
+      const parsed = getPassedFailedSkippedCount(session.testResults);
+      executedTestCount += parsed.passed + parsed.failed + parsed.skipped;
+    }
+  }
 
   const longestBrowser = [...browserNames].sort((a, b) => b.length - a.length)[0];
   const minWidth = longestBrowser ? longestBrowser.length + 1 : 0;
@@ -167,6 +175,15 @@ export function getTestProgressReport(config: TestRunnerCoreConfig, args: TestPr
         } else {
           entries.push(bold(red(`Failed to run any tests.`)));
         }
+      } else if (executedTestCount === 0 && !config.passWithNoTests) {
+        entries.push(
+          bold(red(`Finished running tests in ${duration}s, but no tests were executed.`)),
+        );
+        entries.push(
+          bold(
+            `A test run without executed tests fails. Add tests to your test files, or enable the "passWithNoTests" option to pass on test runs without executed tests.`,
+          ),
+        );
       } else {
         entries.push(bold(`Finished running tests in ${duration}s, all tests passed! 🎉`));
       }