fix(test-runner-core): stop queued sessions after shutdown

.changeset/stopped-sessions-stay-queued.md

@@ -0,0 +1,5 @@
+---
+'@web/test-runner-core': patch
+---
+
+Prevent queued test sessions from starting when page teardown finishes after the test runner has stopped.

packages/test-runner-core/src/runner/TestScheduler.ts

@@ -14,6 +14,7 @@ export class TestScheduler {
   private browsers: BrowserLauncher[];
   private finishedBrowsers = new Set<BrowserLauncher>();
   private stopPromises = new Set<Promise<unknown>>();
+  private stopped = false;
   private browserStartTimeoutMsg: string;
 
   constructor(
@@ -70,12 +71,17 @@ export class TestScheduler {
   }
 
   stop(): Promise<unknown> {
+    this.stopped = true;
     this.timeoutHandler.clearAllTimeouts();
     return Promise.all(Array.from(this.stopPromises));
   }
 
   /** Runs the next batch of scheduled sessions, if any. */
   private runNextScheduled() {
+    if (this.stopped) {
+      return;
+    }
+
     let runningBrowsers = 0;
 
     for (const browser of this.browsers) {

packages/test-runner-core/test/src/runner/TestRunner.test.ts

@@ -1,10 +1,14 @@
 import assert from 'node:assert/strict';
 import path from 'node:path';
 import { describe, it, mock } from 'node:test';
+import { setTimeout } from 'node:timers/promises';
 
 import portfinder from 'portfinder';
 
-import type { BrowserLauncher } from '../../../dist/browser-launcher/BrowserLauncher.js';
+import type {
+  BrowserLauncher,
+  SessionResult,
+} from '../../../dist/browser-launcher/BrowserLauncher.js';
 import type { TestRunnerCoreConfig } from '../../../dist/config/TestRunnerCoreConfig.js';
 import type { TestRunnerGroupConfig } from '../../../dist/index.js';
 import type { Logger } from '../../../dist/logger/Logger.js';
@@ -134,6 +138,94 @@ describe('TestRunner', function () {
     assert.equal(passed, false, 'test runner quits with false');
   });
 
+  for (const stopRunner of [false, true]) {
+    it(
+      stopRunner
+        ? 'does not start queued sessions after stopping during page teardown'
+        : 'starts queued sessions after page teardown while running',
+      async () => {
+        const { runner, browser } = await createTestRunner({
+          watch: true,
+          concurrency: 1,
+          files: ['a-1.test.js', 'a-2.test.js'].map(file =>
+            path.resolve(import.meta.dirname, '..', '..', 'fixtures', 'group-a', file),
+          ),
+        });
+        const [firstSession, queuedSession] = Array.from(runner.sessions.all());
+        const pageStopped = Promise.withResolvers<SessionResult>();
+        let stopping: Promise<void> | undefined;
+
+        function waitForFinished(sessionId: string) {
+          return new Promise<void>(resolve => {
+            const onStatus = (session: typeof firstSession) => {
+              if (session.id === sessionId && session.status === SESSION_STATUS.FINISHED) {
+                runner.sessions.off('session-status-updated', onStatus);
+                resolve();
+              }
+            };
+            runner.sessions.on('session-status-updated', onStatus);
+          });
+        }
+
+        browser.stopSession.mock.mockImplementation((sessionId: string) =>
+          sessionId === firstSession.id ? pageStopped.promise : Promise.resolve({}),
+        );
+
+        try {
+          await runner.start();
+          await setTimeout();
+          assert.equal(browser.startSession.mock.callCount(), 1, 'only the first page starts');
+          assert.equal(runner.sessions.get(queuedSession.id)!.status, SESSION_STATUS.SCHEDULED);
+
+          const finished = waitForFinished(firstSession.id);
+          runner.sessions.updateStatus(
+            { ...runner.sessions.get(firstSession.id)!, passed: true },
+            SESSION_STATUS.TEST_FINISHED,
+          );
+          assert.equal(browser.stopSession.mock.callCount(), 1, 'page teardown has started');
+          if (stopRunner) {
+            stopping = runner.stop();
+          }
+          pageStopped.resolve({});
+          await finished;
+          await setTimeout();
+          await stopping;
+
+          assert.equal(
+            browser.startSession.mock.callCount(),
+            stopRunner ? 1 : 2,
+            stopRunner
+              ? 'stopped runner must not launch queued sessions'
+              : 'active runner starts the next page',
+          );
+          assert.equal(
+            runner.sessions.get(queuedSession.id)!.status,
+            stopRunner ? SESSION_STATUS.SCHEDULED : SESSION_STATUS.INITIALIZING,
+          );
+        } finally {
+          pageStopped.resolve({});
+          for (const session of runner.sessions.all()) {
+            if (session.status === SESSION_STATUS.TEST_FINISHED) {
+              await waitForFinished(session.id);
+            } else if (
+              session.status === SESSION_STATUS.INITIALIZING ||
+              session.status === SESSION_STATUS.TEST_STARTED
+            ) {
+              const finished = waitForFinished(session.id);
+              runner.sessions.updateStatus(
+                { ...session, passed: true },
+                SESSION_STATUS.TEST_FINISHED,
+              );
+              await finished;
+            }
+          }
+          await stopping;
+          await runner.stop();
+        }
+      },
+    );
+  }
+
   describe('groups', () => {
     it('can create a group in addition to the default group', async () => {
       const { runner } = await createTestRunner(undefined, [