Feat/smoke verdict

.github/workflows/self-test.yml

@@ -0,0 +1,190 @@
+# Runs the Pester tests for the scripts behind the composite actions in this repository.
+name: Self Test
+
+on:
+  pull_request:
+    branches: [ "main" ]
+  push:
+    branches: [ "main" ]
+
+jobs:
+  pester:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+    - name: Run Pester tests
+      shell: pwsh
+      run: |
+        if (-not (Get-Module -ListAvailable Pester | Where-Object { $_.Version -ge [version]'5.5.0' })) {
+          # windows-latest ships Pester 3.4 in-box, signed by a publisher not already trusted on
+          # the runner; without this flag the install prompts and the non-interactive step hangs.
+          Install-Module Pester -MinimumVersion 5.5.0 -Scope CurrentUser -Force -SkipPublisherCheck
+        }
+        Import-Module Pester -MinimumVersion 5.5.0
+        $config = New-PesterConfiguration
+        $config.Run.Path = 'actions'
+        $config.Run.Exit = $true
+        $config.Output.Verbosity = 'Detailed'
+        Invoke-Pester -Configuration $config
+
+  launcher-e2e:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+    steps:
+    - name: Checkout
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+    # Three fixtures, each with its own releases_dir/output_dir: a legacy app that never speaks
+    # the smoke protocol, and two smoke-aware apps that report PASSED and FAILED. Running the
+    # real composite action against each is what proves the launcher's exit status and verdict
+    # wiring works under GitHub's actual pwsh step wrapper, not just under Pester.
+    - name: Build Linux fixtures
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        mkdir -p fixtures/legacy/releases fixtures/pass/releases fixtures/fail/releases
+
+        cat > fixtures/legacy/releases/Fixture.AppImage <<'SCRIPT'
+        #!/bin/bash
+        sleep 120
+        SCRIPT
+
+        cat > fixtures/pass/releases/Fixture.AppImage <<'SCRIPT'
+        #!/bin/bash
+        echo "HERMES_SMOKE_START: Fixture 1.0.0 Linux x64"
+        sleep 0.2
+        echo "HERMES_SMOKE_MILESTONE: window-shown 5ms"
+        sleep 0.2
+        echo "HERMES_SMOKE_RESULT: PASSED (0 checks)"
+        exit 0
+        SCRIPT
+
+        cat > fixtures/fail/releases/Fixture.AppImage <<'SCRIPT'
+        #!/bin/bash
+        echo "HERMES_SMOKE_START: Fixture 1.0.0 Linux x64"
+        sleep 0.2
+        echo "HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 0 errors)"
+        exit 1
+        SCRIPT
+
+        chmod +x fixtures/legacy/releases/Fixture.AppImage fixtures/pass/releases/Fixture.AppImage fixtures/fail/releases/Fixture.AppImage
+
+    - name: Build macOS fixtures
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        for name in legacy pass fail; do
+          mkdir -p "fixtures/$name/releases" "fixtures/$name/build/Fixture.app/Contents/MacOS"
+          cat > "fixtures/$name/build/Fixture.app/Contents/Info.plist" <<'PLIST'
+        <?xml version="1.0" encoding="UTF-8"?>
+        <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+        <plist version="1.0">
+        <dict>
+          <key>CFBundleExecutable</key>
+          <string>Fixture</string>
+          <key>CFBundleIdentifier</key>
+          <string>com.mythetech.smoke.fixture</string>
+          <key>CFBundlePackageType</key>
+          <string>APPL</string>
+        </dict>
+        </plist>
+        PLIST
+        done
+
+        cat > fixtures/legacy/build/Fixture.app/Contents/MacOS/Fixture <<'SCRIPT'
+        #!/bin/bash
+        sleep 120
+        SCRIPT
+
+        cat > fixtures/pass/build/Fixture.app/Contents/MacOS/Fixture <<'SCRIPT'
+        #!/bin/bash
+        echo "HERMES_SMOKE_START: Fixture 1.0.0 macOS arm64"
+        sleep 0.2
+        echo "HERMES_SMOKE_MILESTONE: window-shown 5ms"
+        sleep 0.2
+        echo "HERMES_SMOKE_RESULT: PASSED (0 checks)"
+        exit 0
+        SCRIPT
+
+        cat > fixtures/fail/build/Fixture.app/Contents/MacOS/Fixture <<'SCRIPT'
+        #!/bin/bash
+        echo "HERMES_SMOKE_START: Fixture 1.0.0 macOS arm64"
+        sleep 0.2
+        echo "HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 0 errors)"
+        exit 1
+        SCRIPT
+
+        for name in legacy pass fail; do
+          chmod +x "fixtures/$name/build/Fixture.app/Contents/MacOS/Fixture"
+          (cd "fixtures/$name/build" && ditto -c -k --keepParent Fixture.app "../releases/Fixture-Portable.zip")
+        done
+
+    - name: Run legacy fixture
+      id: legacy
+      uses: ./actions/smoke-test
+      with:
+        app_name: Fixture
+        platform: ${{ runner.os }}
+        releases_dir: fixtures/legacy/releases
+        output_dir: fixtures/legacy/output
+        timeout: '20'
+
+    - name: Run passing fixture
+      id: pass
+      uses: ./actions/smoke-test
+      with:
+        app_name: Fixture
+        platform: ${{ runner.os }}
+        releases_dir: fixtures/pass/releases
+        output_dir: fixtures/pass/output
+        timeout: '20'
+
+    - name: Run failing fixture
+      id: fail
+      continue-on-error: true
+      uses: ./actions/smoke-test
+      with:
+        app_name: Fixture
+        platform: ${{ runner.os }}
+        releases_dir: fixtures/fail/releases
+        output_dir: fixtures/fail/output
+        timeout: '20'
+
+    - name: Assert outputs
+      shell: bash
+      run: |
+        status=0
+
+        if [ "${{ steps.legacy.outputs.result }}" != "passed" ]; then
+          echo "::error::legacy fixture expected result=passed, got '${{ steps.legacy.outputs.result }}'"
+          status=1
+        fi
+
+        if [ "${{ steps.pass.outputs.result }}" != "passed" ]; then
+          echo "::error::passing fixture expected result=passed, got '${{ steps.pass.outputs.result }}'"
+          status=1
+        fi
+
+        if [ "${{ steps.fail.outputs.result }}" != "failed" ]; then
+          echo "::error::failing fixture expected result=failed, got '${{ steps.fail.outputs.result }}'"
+          status=1
+        fi
+
+        exit $status

README.md

@@ -171,6 +171,62 @@ Configure these at the **organization level** for sharing across repos:
 
 ---
 
+### `desktop-smoke-test.yml`
+
+Runs unit tests and smoke tests on demand, without signing or publishing. Builds the same
+unsigned Velopack packages as `desktop-publish.yml`, then launches them on each selected
+platform. Useful for checking a large change before cutting a release. No secrets are needed.
+
+```yaml
+name: Smoke Test
+on:
+  workflow_dispatch:
+    inputs:
+      platforms:
+        description: 'Platforms to smoke test'
+        type: choice
+        options: [all, windows, macos, linux]
+        default: all
+
+jobs:
+  smoke-test:
+    uses: mythetech/workflows/.github/workflows/desktop-smoke-test.yml@main
+    with:
+      app_name: "Horizon"
+      project_path: "Horizon/Horizon.csproj"
+      test_project: "Horizon.Test/Horizon.Test.csproj"
+      icon_windows: "Horizon/wwwroot/logo.ico"
+      icon_linux: "Horizon/wwwroot/logo.png"
+      icon_macos: "Horizon/wwwroot/logo.icns"
+      platforms: ${{ inputs.platforms }}
+```
+
+Unit tests run through `pr-test.yml`, so VSTest/MTP detection works the same as on pull
+requests. Tests and smoke tests run in parallel.
+
+#### Inputs
+
+| Input | Required | Default | Description |
+|-------|----------|---------|-------------|
+| `app_name` | **Yes** | - | Application name (e.g., "Horizon") |
+| `project_path` | **Yes** | - | Path to main .csproj |
+| `icon_windows` | **Yes** | - | Windows icon (.ico) |
+| `icon_linux` | **Yes** | - | Linux icon (.png) |
+| `icon_macos` | **Yes** | - | macOS icon (.icns) |
+| `platforms` | No | `all` | `all`, `windows`, `macos` or `linux` |
+| `dotnet_version` | No | `10.0.x` | .NET SDK version. Ignored when `global_json_file` is set |
+| `global_json_file` | No | - | Path to a `global.json` pinning the SDK |
+| `enable_tests` | No | `true` | Run unit tests alongside the smoke tests |
+| `test_project` | No | - | Path to test project. If omitted, runs `dotnet test` in root |
+| `test_command` | No | - | Override the entire test command |
+| `test_runner` | No | `auto` | `auto`, `vstest` or `mtp` |
+| `enable_coverage` | No | `false` | Collect code coverage for the unit tests |
+
+The packages are unsigned, so the macOS signing-specific bundle restructuring from
+`desktop-publish.yml` is not exercised here.
+
+---
+
 ## Composite Actions
 
 ### `actions/macos-sign`

actions/smoke-test/Start-SmokeRun.Tests.ps1

@@ -0,0 +1,129 @@
+# Copyright (c) Mythetech. Licensed under the MIT License.
+
+BeforeAll {
+    . (Join-Path $PSScriptRoot 'Start-SmokeRun.ps1')
+}
+
+Describe 'Get-SmokeEnvironment' {
+    It 'turns smoke mode on with the budget and result path' {
+        $environment = Get-SmokeEnvironment -TimeoutSeconds 45 -ResultPath '/tmp/out/result.json'
+
+        $environment['HERMES_SMOKE_TEST'] | Should -Be '1'
+        $environment['HERMES_SMOKE_TEST_TIMEOUT'] | Should -Be '45'
+        $environment['HERMES_SMOKE_TEST_RESULT'] | Should -Be '/tmp/out/result.json'
+    }
+}
+
+Describe 'Get-SmokeLaunchPlan' {
+    It 'launches macOS apps through open with explicit environment and output paths' {
+        $environment = [ordered]@{ HERMES_SMOKE_TEST = '1'; HERMES_SMOKE_TEST_TIMEOUT = '60' }
+
+        $plan = Get-SmokeLaunchPlan -Platform macOS -TargetPath '/work/app/My App.app' `
+            -StdoutPath '/work/out/app-stdout.log' -StderrPath '/work/out/app-stderr.log' -Environment $environment
+
+        $plan.FilePath | Should -Be 'open'
+        $plan.RedirectOutput | Should -BeFalse
+        $plan.ArgumentList | Should -Be @(
+            '-W', '-n',
+            '--env', 'HERMES_SMOKE_TEST=1',
+            '--env', 'HERMES_SMOKE_TEST_TIMEOUT=60',
+            '--stdout', '/work/out/app-stdout.log',
+            '--stderr', '/work/out/app-stderr.log',
+            '/work/app/My App.app'
+        )
+    }
+
+    It 'keeps a path with spaces as a single argument' {
+        $plan = Get-SmokeLaunchPlan -Platform macOS -TargetPath '/work/app/Http Platypus.app' `
+            -StdoutPath '/o/out.log' -StderrPath '/o/err.log' -Environment ([ordered]@{})
+
+        $plan.ArgumentList[-1] | Should -Be '/work/app/Http Platypus.app'
+    }
+
+    It 'runs Windows and Linux targets directly with redirected output' {
+        foreach ($platform in 'Windows', 'Linux') {
+            $plan = Get-SmokeLaunchPlan -Platform $platform -TargetPath '/work/app/App' `
+                -StdoutPath '/o/out.log' -StderrPath '/o/err.log' -Environment ([ordered]@{})
+
+            $plan.FilePath | Should -Be '/work/app/App'
+            $plan.RedirectOutput | Should -BeTrue
+            @($plan.ArgumentList).Count | Should -Be 0
+        }
+    }
+}
+
+Describe 'Test-SmokeStarted' {
+    It 'is false when the log does not exist yet' {
+        Test-SmokeStarted -LogPath (Join-Path $TestDrive ([guid]::NewGuid())) | Should -BeFalse
+    }
+
+    It 'is true once the start line is in the log, even with Windows line endings' {
+        $logPath = Join-Path $TestDrive 'stdout-crlf.log'
+        [System.IO.File]::WriteAllText($logPath, "starting up`r`nHERMES_SMOKE_START: App 1.0.0 Windows x64`r`n")
+
+        Test-SmokeStarted -LogPath $logPath | Should -BeTrue
+    }
+
+    It 'ignores the marker when it is not at the start of a line' {
+        $logPath = Join-Path $TestDrive 'stdout-embedded.log'
+        Set-Content -LiteralPath $logPath -Value 'echo HERMES_SMOKE_START: nope'
+
+        Test-SmokeStarted -LogPath $logPath | Should -BeFalse
+    }
+}
+
+Describe 'Get-SmokeRunOutcome' {
+    It 'classifies an app that exited before the start window as legacy, not alive' {
+        $outcome = Get-SmokeRunOutcome -Platform Linux -Started $false -Mode 'verdict' -TimedOut $false `
+            -LegacyAlive $false -ExitCode 1 -StartedAfterExit $false
+
+        $outcome.mode | Should -Be 'legacy'
+        $outcome.legacyAlive | Should -BeFalse
+        $outcome.exitCode | Should -BeNullOrEmpty
+    }
+
+    It 'keeps a legacy app alive when the start window elapsed with the process still running' {
+        $outcome = Get-SmokeRunOutcome -Platform Linux -Started $false -Mode 'legacy' -TimedOut $false `
+            -LegacyAlive $true -ExitCode 0 -StartedAfterExit $false
+
+        $outcome.mode | Should -Be 'legacy'
+        $outcome.legacyAlive | Should -BeTrue
+        $outcome.exitCode | Should -BeNullOrEmpty
+    }
+
+    It 'never gives a liveness pass when the start line only becomes readable after the legacy stop' {
+        $outcome = Get-SmokeRunOutcome -Platform Linux -Started $false -Mode 'legacy' -TimedOut $false `
+            -LegacyAlive $true -ExitCode 0 -StartedAfterExit $true
+
+        $outcome.mode | Should -Be 'verdict'
+        $outcome.timedOut | Should -BeTrue
+        $outcome.legacyAlive | Should -BeFalse
+        $outcome.exitCode | Should -BeNullOrEmpty
+    }
+
+    It 'nulls the exit code for a verdict-mode timeout' {
+        $outcome = Get-SmokeRunOutcome -Platform Windows -Started $true -Mode 'verdict' -TimedOut $true `
+            -LegacyAlive $false -ExitCode 0 -StartedAfterExit $false
+
+        $outcome.mode | Should -Be 'verdict'
+        $outcome.exitCode | Should -BeNullOrEmpty
+    }
+
+    It 'keeps the process exit code for a normal verdict exit on Windows and Linux' {
+        foreach ($platform in 'Windows', 'Linux') {
+            $outcome = Get-SmokeRunOutcome -Platform $platform -Started $true -Mode 'verdict' -TimedOut $false `
+                -LegacyAlive $false -ExitCode 3 -StartedAfterExit $false
+
+            $outcome.mode | Should -Be 'verdict'
+            $outcome.exitCode | Should -Be 3
+        }
+    }
+
+    It 'always nulls the exit code on macOS even on a normal verdict exit' {
+        $outcome = Get-SmokeRunOutcome -Platform macOS -Started $true -Mode 'verdict' -TimedOut $false `
+            -LegacyAlive $false -ExitCode 0 -StartedAfterExit $false
+
+        $outcome.mode | Should -Be 'verdict'
+        $outcome.exitCode | Should -BeNullOrEmpty
+    }
+}