test: add packaged VS Code E2E smoke coverage

.github/actions/e2e/action.yml

@@ -0,0 +1,84 @@
+name: Extension end-to-end tests
+description: Test an existing VSIX in isolated VS Code and optionally run the browser suite.
+
+inputs:
+  vsix:
+    description: Exact VSIX produced by the caller's packaging step.
+    required: true
+  vscode-version:
+    description: Exact VS Code version or latest; empty uses engines.vscode's minimum.
+    default: ''
+  browser-tests:
+    description: Also run the browser-only Playwright suite.
+    default: 'true'
+
+runs:
+  using: composite
+  steps:
+    - name: Cache desktop downloads
+      uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5
+      with:
+        path: .vscode-test/cache
+        key: vscode-${{ runner.os }}-${{ runner.arch }}-${{ inputs.vscode-version || 'minimum' }}-${{ hashFiles('package.json', 'package-lock.json') }}-${{ inputs.vscode-version == 'latest' && github.run_id || 'pinned' }}
+        restore-keys: |
+          vscode-${{ runner.os }}-${{ runner.arch }}-${{ inputs.vscode-version || 'minimum' }}-${{ hashFiles('package.json', 'package-lock.json') }}-
+
+    - name: Install Linux display and browser libraries
+      if: runner.os == 'Linux'
+      shell: bash
+      run: npx playwright install-deps chromium
+
+    - name: Install browser test Chromium
+      if: inputs.browser-tests == 'true'
+      shell: bash
+      run: npx playwright install chromium
+
+    - name: Browser webview tests
+      if: inputs.browser-tests == 'true'
+      shell: bash
+      run: npm run test:e2e
+
+    - name: Packaged VSIX smoke tests
+      shell: bash
+      env:
+        TEST_VSIX: ${{ inputs.vsix }}
+        TEST_VSCODE_VERSION: ${{ inputs.vscode-version }}
+      run: |
+        args=(--vsix "$TEST_VSIX")
+        if [ -n "$TEST_VSCODE_VERSION" ]; then
+          args+=(--vscode-version "$TEST_VSCODE_VERSION")
+        fi
+        if [ "$RUNNER_OS" = Linux ]; then
+          xvfb-run -a --server-args="-screen 0 1280x900x24" npm run test:vscode -- "${args[@]}"
+        else
+          npm run test:vscode -- "${args[@]}"
+        fi
+
+    - name: Upload test diagnostics
+      id: diagnostics
+      if: always()
+      uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+      with:
+        name: e2e-${{ runner.os }}-${{ inputs.vscode-version || 'minimum' }}
+        path: test-results/
+        if-no-files-found: warn
+        retention-days: 7
+
+    - name: Summarize desktop snapshots
+      if: always()
+      shell: bash
+      env:
+        E2E_ARTIFACT_URL: ${{ steps.diagnostics.outputs.artifact-url }}
+      run: |
+        shopt -s nullglob
+        reports=(test-results/vscode/*/snapshots.md)
+        if [ "${#reports[@]}" -eq 0 ]; then
+          printf '## VS Code smoke snapshots\n\nNo desktop snapshot report was produced. Check the test-step logs.\n' >> "$GITHUB_STEP_SUMMARY"
+        else
+          for report in "${reports[@]}"; do
+            cat "$report" >> "$GITHUB_STEP_SUMMARY"
+          done
+        fi
+        if [ -n "$E2E_ARTIFACT_URL" ]; then
+          printf '\n[Download screenshots and test diagnostics](%s)\n\nExtract the artifact and open `vscode/<run>/gallery.html` to browse the milestone snapshots. Individual PNGs are under each suite'\''s `snapshots` directory; failure screenshots remain under `screenshots`.\n' "$E2E_ARTIFACT_URL" >> "$GITHUB_STEP_SUMMARY"
+        fi

.github/workflows/ci.yml

@@ -12,6 +12,7 @@ permissions:
 jobs:
   verify:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
 
     steps:
       - name: Checkout
@@ -32,14 +33,23 @@ jobs:
       - name: Build
         run: npm run build
 
-      - name: Check bundle size
-        run: npm run check-size
-
       - name: Package VSIX
         run: npm run package
 
+      - name: Check bundle and VSIX size
+        run: npm run check-size
+
+      - name: Capture VSIX path
+        id: vsix
+        run: echo "path=$(node -p 'const p = require("./package.json"); `${p.name}-${p.version}.vsix`')" >> "$GITHUB_OUTPUT"
+
+      - name: End-to-end tests
+        uses: ./.github/actions/e2e
+        with:
+          vsix: ${{ steps.vsix.outputs.path }}
+
       - name: Upload VSIX artifact
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: vsix
-          path: '*.vsix'
+          path: ${{ steps.vsix.outputs.path }}

.github/workflows/release.yml

@@ -12,6 +12,7 @@ permissions:
 jobs:
   release:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
 
     steps:
       - name: Checkout
@@ -39,22 +40,27 @@ jobs:
       - name: Run build
         run: npm run build
 
-      - name: Run tests
-        run: npm test
+      - name: Run checks
+        run: npm run check
 
       - name: Package VSIX
         run: npm run package
 
       - name: Capture VSIX path
         id: vsix
         run: |
-          VSIX_PATH="$(find . -maxdepth 1 -name '*.vsix' -print -quit)"
-          if [ -z "$VSIX_PATH" ]; then
-            echo "No VSIX package was produced"
-            exit 1
-          fi
+          VSIX_PATH="$(node -p 'const p = require("./package.json"); `${p.name}-${p.version}.vsix`')"
+          test -f "$VSIX_PATH"
           echo "path=$VSIX_PATH" >> "$GITHUB_OUTPUT"
 
+      - name: Check bundle and VSIX size
+        run: npm run check-size
+
+      - name: End-to-end release gate
+        uses: ./.github/actions/e2e
+        with:
+          vsix: ${{ steps.vsix.outputs.path }}
+
       - name: Create GitHub release
         if: startsWith(github.ref, 'refs/tags/')
         uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
@@ -68,4 +74,3 @@ jobs:
         with:
           name: vsix
           path: ${{ steps.vsix.outputs.path }}
-

.github/workflows/vscode-compatibility.yml

@@ -0,0 +1,52 @@
+name: VS Code compatibility
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/vscode-compatibility.yml'
+      - '.github/actions/e2e/**'
+      - 'tests/vscode/**'
+  schedule:
+    - cron: '0 7 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  smoke:
+    name: ${{ matrix.os }} / VS Code ${{ matrix.version || 'minimum' }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        version: ['', latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Setup Node.js
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Package VSIX
+        run: npm run package
+
+      - name: Capture VSIX path
+        id: vsix
+        shell: bash
+        run: echo "path=$(node -p 'const p = require("./package.json"); `${p.name}-${p.version}.vsix`')" >> "$GITHUB_OUTPUT"
+
+      - name: Desktop compatibility tests
+        uses: ./.github/actions/e2e
+        with:
+          vsix: ${{ steps.vsix.outputs.path }}
+          vscode-version: ${{ matrix.version }}
+          browser-tests: 'false'

.vscodeignore

@@ -10,6 +10,7 @@ eslint.config.mjs
 
 # Dev state
 .vscode/**
+.vscode-test/**
 .github/**
 .git/**
 .gitignore
@@ -21,6 +22,7 @@ eslint.config.mjs
 **/*.map
 *.vsix
 dist/**/*.map
+test-results/**
 
 # Docs scaffolding (only publish README + LICENSE + dist)
 README.github.md

CONTRIBUTING.md

@@ -28,6 +28,130 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio
 Please use [GitHub Issues](https://github.com/microsoft/ai-engineer-coach/issues) to report bugs or
 request features. Before filing a new issue, please check if one already exists.
 
+## End-to-End Testing
+
+Use Node 22 (the CI version) or a newer supported Node release. Install the locked dependencies
+with `npm ci`. The existing database integration tests also require the `sqlite3` command on
+your `PATH`. There are three complementary test layers:
+
+| Layer | Command | What it verifies |
+| --- | --- | --- |
+| Unit and fixture tests | `npm test` | Parsers, rules, calculations, and synthetic-data invariants |
+| Browser webview tests | `npm run build` then `npm run test:e2e` | Dashboard interactions with mocked host responses |
+| Packaged VS Code tests | `npm run test:vscode` | Installation, activation, workers, real webview messaging, navigation, reload, and empty data |
+
+Before the first browser run, install its browser with `npx playwright install chromium`.
+On a Linux machine without a display, install the desktop dependencies with
+`npx playwright install-deps chromium`, then run `xvfb-run -a npm run test:vscode`.
+Windows and macOS run the same npm command without the virtual display.
+
+### Testing the shipped extension
+
+`npm run test:vscode` compiles the test code, packages the extension through the normal packaging
+script, and installs that VSIX into a disposable VS Code installation. It never installs into
+your normal editor. To test an already-built artifact without rebuilding it, use:
+
+```sh
+npm run test:vscode -- --vsix ai-engineer-coach-0.1.0.vsix
+```
+
+Substitute the filename produced by `npm run package` when the package version changes.
+The runner records the package's SHA-256 hash and checks that the artifact did not change.
+CI and releases pass the exact package path this way; publishing happens only after the
+browser and desktop suites pass, and the release uses that same VSIX.
+
+By default, desktop tests use the minimum version in `package.json`'s `engines.vscode`.
+Use `--vscode-version 1.125.0` to select an explicit compatible version, or
+`--vscode-version latest` to check the current stable release. An incompatible pinned version
+fails before launching. The [compatibility workflow](.github/workflows/vscode-compatibility.yml)
+runs the minimum and latest versions on Linux, Windows, and macOS weekly, on manual dispatch,
+and on PRs that change the desktop harness, its fixtures, or the shared E2E action.
+
+The desktop runner uses [ExTester](https://github.com/redhat-developer/vscode-extension-tester).
+Its actively tested editor-version window moves with new releases; older versions are
+best-effort. Keep the minimum-version tests rather than silently increasing our public engine
+requirement to accommodate the runner. Dependency updates use the existing Dependabot workflow.
+
+### Synthetic data and isolation
+
+The checked-in data is [`tests/vscode/data/projects.json`](tests/vscode/data/projects.json):
+fictional project names, prompts, source code, tests, model IDs, and a seven-day activity schedule.
+It is entirely synthetic, not sampled or anonymized from anyone's personal logs. Three projects
+cover TypeScript (`inventory-api`), Python (`sensor-pipeline`), and Go (`release-worker`).
+
+[`tests/vscode/fixtures.ts`](tests/vscode/fixtures.ts) materializes those definitions into raw
+`chatSessions` and `chatEditingSessions` JSON in the platform's normal discovery layout,
+entirely inside a fresh temporary home. Initially there are **21 sessions, 63 requests, 315 added
+lines and 294 net lines** across seven active days, including weekday, weekend and evening activity.
+Each session records file creation, a one-line correction, and test creation; the same code also
+appears in responses to exercise edit/code-block deduplication. The reload test adds one session,
+then verifies **22 sessions, 66 requests and 330 added lines**, including a second reload without
+duplicates. The empty-data scenario starts a separate process with no session sources.
+
+The data is varied but deterministic rather than randomly changing on every build. The most
+recent activity day is the day before the run, with earlier days and local hours taken from the
+schedule, so charts remain populated. The unit suite checks exact code totals, all seven heatmap
+days, overlapping Timeline sessions, and the three project summaries through the real parser
+and analyzers. Desktop tests do not replace the parser, workers, VS Code API, or message bridge.
+They also check that input log files remain byte-for-byte unchanged except for the test's
+explicit addition. Personal home, AppData, Copilot data, temporary directories, extension
+directories, and VS Code profiles are isolated; no sign-in or live model call is required.
+
+Only editor/driver download archives are cached under `.vscode-test/cache`. Each invocation
+creates fresh profiles, fixtures, and application caches and removes them after the run.
+Diagnostics remain under `test-results/vscode`, including runtime and driver logs, failure
+screenshots, and a package-hash summary. Browser traces, screenshots, and the HTML report
+are stored separately under `test-results/browser` and `test-results/browser-report`.
+GitHub Actions uploads these diagnostics even when a test fails.
+
+The pipeline logs the generated counts and includes a **Synthetic test data** section in its
+summary and gallery. Downloaded artifacts include each suite's `dataset.json`, the checked-in
+definitions under `fixtures/projects.json`, the actual initial input copies under
+`fixtures/initial`, and the planned additional files under `fixtures/reload`. These are generated
+test inputs, not user logs; the reload files are kept separate from the initial dataset.
+
+Successful milestones also capture real VS Code screenshots: dashboard totals and activity, Timeline
+session details, nonzero Output by project, Patterns Work Hours and Projects, reload, reopening,
+and the expected empty-data state. In a GitHub Actions
+run, open the **Summary** and follow **Download screenshots and test diagnostics**. Extract the
+artifact, then open `vscode/<run>/gallery.html`; its images are embedded, so it works offline.
+The gallery includes captions, suite outcomes, dataset counts, the editor target, and the tested VSIX hash.
+Individual PNGs are in each suite's `snapshots` directory. Locally, the same gallery is under
+`test-results/vscode/<run>/gallery.html`.
+
+These screenshots document the synthetic-data run, not pixel-comparison baselines. The capture
+helper waits for chart pixels to settle so it does not show an in-progress animation. Captures
+are preserved even if a later assertion fails; the report still marks that suite as failed.
+Existing automatic failure screenshots remain separate.
+
+### Maintaining regression coverage
+
+Add the regression test in the same PR as the feature or fix. Prefer the smallest layer that
+can reproduce the behavior: parser/calculation changes belong in Vitest, dashboard interaction
+changes in Playwright, and packaging or host/webview boundary changes in the desktop suite.
+Do not duplicate every browser scenario in VS Code.
+
+Add lowercase, hyphenated `*.spec.ts` files directly under
+[`tests/vscode/suite`](tests/vscode/suite). The runner discovers new source suites automatically;
+each file gets a separate process, profile, and baseline synthetic data. `empty.spec.ts` is the
+no-data exception. Reuse the [`Dashboard`](tests/vscode/dashboard.ts) helper and small synthetic
+fixtures. Keep selectors in the helper, wait for observable readiness instead of fixed sleeps,
+and assert exact results.
+After a meaningful assertion, use `await dashboard.snapshot('01-feature', 'What the screenshot demonstrates')`
+to add a named milestone to the gallery. Use a unique lowercase, hyphenated name within each suite.
+The desktop suite is sequential, has no automatic test retries, and rejects focused, skipped, or
+empty test runs. Inherited `MOCHA_GREP` and `MOCHA_INVERT` filters are removed so a nonempty subset
+cannot pass as the full suite. Reload replaces the webview's frames, so the dashboard helper waits
+for a different document and reacquires detached frames within a single deadline, including
+ExTester's wrapped stale-outer-frame error. It does not repeat the reload command or suppress
+unexpected driver failures.
+Its TypeScript and fixture tests are included in `npm run check`.
+
+Use the shared [E2E action](.github/actions/e2e/action.yml) for pipeline changes so PR, release,
+and compatibility runs do not acquire different setup procedures. Investigate failures using
+the saved screenshots and logs; do not bypass the release gate or disable an operating-system
+security feature globally to get a green run.
+
 ## Security
 
 If you discover a security vulnerability, please follow the instructions in [SECURITY.md](SECURITY.md).

eslint.config.mjs

@@ -13,7 +13,7 @@ export default tseslint.config(
   eslint.configs.recommended,
   ...tseslint.configs.recommendedTypeChecked,
   {
-    files: ["src/**/*.ts"],
+    files: ["src/**/*.ts", "tests/vscode/**/*.ts"],
     languageOptions: {
       parserOptions: {
         projectService: true,
@@ -83,6 +83,6 @@ export default tseslint.config(
     },
   },
   {
-    ignores: ["dist/", "node_modules/", "**/*.mjs", "tests/"],
+    ignores: ["dist/", "node_modules/", ".vscode-test/", "**/*.mjs", "tests/e2e/"],
   }
 );

knip.json

@@ -7,9 +7,11 @@
     "src/core/warm-up-worker.ts",
     "src/core/parse-worker.ts",
     "src/core/cache-write-worker.ts",
-    "src/core/parser.bench.ts"
+    "src/core/parser.bench.ts",
+    "tests/vscode/run.ts",
+    "tests/vscode/suite/**/*.spec.ts"
   ],
-  "project": ["src/**/*.ts"],
+  "project": ["src/**/*.ts", "tests/vscode/**/*.ts"],
   "ignore": [
     "src/webview/page-insights.ts",
     "src/webview/page-rule-editor.ts",