diff --git a/.github/workflows/self-test.yml b/.github/workflows/self-test.yml new file mode 100644 index 0000000..43da8ed --- /dev/null +++ b/.github/workflows/self-test.yml @@ -0,0 +1,190 @@ +# Runs the Pester tests for the scripts behind the composite actions in this repository. +name: Self Test + +on: + pull_request: + branches: [ "main" ] + push: + branches: [ "main" ] + +jobs: + pester: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Run Pester tests + shell: pwsh + run: | + if (-not (Get-Module -ListAvailable Pester | Where-Object { $_.Version -ge [version]'5.5.0' })) { + # windows-latest ships Pester 3.4 in-box, signed by a publisher not already trusted on + # the runner; without this flag the install prompts and the non-interactive step hangs. + Install-Module Pester -MinimumVersion 5.5.0 -Scope CurrentUser -Force -SkipPublisherCheck + } + Import-Module Pester -MinimumVersion 5.5.0 + $config = New-PesterConfiguration + $config.Run.Path = 'actions' + $config.Run.Exit = $true + $config.Output.Verbosity = 'Detailed' + Invoke-Pester -Configuration $config + + launcher-e2e: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + # Three fixtures, each with its own releases_dir/output_dir: a legacy app that never speaks + # the smoke protocol, and two smoke-aware apps that report PASSED and FAILED. Running the + # real composite action against each is what proves the launcher's exit status and verdict + # wiring works under GitHub's actual pwsh step wrapper, not just under Pester. + - name: Build Linux fixtures + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + + mkdir -p fixtures/legacy/releases fixtures/pass/releases fixtures/fail/releases + + cat > fixtures/legacy/releases/Fixture.AppImage <<'SCRIPT' + #!/bin/bash + sleep 120 + SCRIPT + + cat > fixtures/pass/releases/Fixture.AppImage <<'SCRIPT' + #!/bin/bash + echo "HERMES_SMOKE_START: Fixture 1.0.0 Linux x64" + sleep 0.2 + echo "HERMES_SMOKE_MILESTONE: window-shown 5ms" + sleep 0.2 + echo "HERMES_SMOKE_RESULT: PASSED (0 checks)" + exit 0 + SCRIPT + + cat > fixtures/fail/releases/Fixture.AppImage <<'SCRIPT' + #!/bin/bash + echo "HERMES_SMOKE_START: Fixture 1.0.0 Linux x64" + sleep 0.2 + echo "HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 0 errors)" + exit 1 + SCRIPT + + chmod +x fixtures/legacy/releases/Fixture.AppImage fixtures/pass/releases/Fixture.AppImage fixtures/fail/releases/Fixture.AppImage + + - name: Build macOS fixtures + if: runner.os == 'macOS' + shell: bash + run: | + set -euo pipefail + + for name in legacy pass fail; do + mkdir -p "fixtures/$name/releases" "fixtures/$name/build/Fixture.app/Contents/MacOS" + cat > "fixtures/$name/build/Fixture.app/Contents/Info.plist" <<'PLIST' + + + + + CFBundleExecutable + Fixture + CFBundleIdentifier + com.mythetech.smoke.fixture + CFBundlePackageType + APPL + + + PLIST + done + + cat > fixtures/legacy/build/Fixture.app/Contents/MacOS/Fixture <<'SCRIPT' + #!/bin/bash + sleep 120 + SCRIPT + + cat > fixtures/pass/build/Fixture.app/Contents/MacOS/Fixture <<'SCRIPT' + #!/bin/bash + echo "HERMES_SMOKE_START: Fixture 1.0.0 macOS arm64" + sleep 0.2 + echo "HERMES_SMOKE_MILESTONE: window-shown 5ms" + sleep 0.2 + echo "HERMES_SMOKE_RESULT: PASSED (0 checks)" + exit 0 + SCRIPT + + cat > fixtures/fail/build/Fixture.app/Contents/MacOS/Fixture <<'SCRIPT' + #!/bin/bash + echo "HERMES_SMOKE_START: Fixture 1.0.0 macOS arm64" + sleep 0.2 + echo "HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 0 errors)" + exit 1 + SCRIPT + + for name in legacy pass fail; do + chmod +x "fixtures/$name/build/Fixture.app/Contents/MacOS/Fixture" + (cd "fixtures/$name/build" && ditto -c -k --keepParent Fixture.app "../releases/Fixture-Portable.zip") + done + + - name: Run legacy fixture + id: legacy + uses: ./actions/smoke-test + with: + app_name: Fixture + platform: ${{ runner.os }} + releases_dir: fixtures/legacy/releases + output_dir: fixtures/legacy/output + timeout: '20' + + - name: Run passing fixture + id: pass + uses: ./actions/smoke-test + with: + app_name: Fixture + platform: ${{ runner.os }} + releases_dir: fixtures/pass/releases + output_dir: fixtures/pass/output + timeout: '20' + + - name: Run failing fixture + id: fail + continue-on-error: true + uses: ./actions/smoke-test + with: + app_name: Fixture + platform: ${{ runner.os }} + releases_dir: fixtures/fail/releases + output_dir: fixtures/fail/output + timeout: '20' + + - name: Assert outputs + shell: bash + run: | + status=0 + + if [ "${{ steps.legacy.outputs.result }}" != "passed" ]; then + echo "::error::legacy fixture expected result=passed, got '${{ steps.legacy.outputs.result }}'" + status=1 + fi + + if [ "${{ steps.pass.outputs.result }}" != "passed" ]; then + echo "::error::passing fixture expected result=passed, got '${{ steps.pass.outputs.result }}'" + status=1 + fi + + if [ "${{ steps.fail.outputs.result }}" != "failed" ]; then + echo "::error::failing fixture expected result=failed, got '${{ steps.fail.outputs.result }}'" + status=1 + fi + + exit $status diff --git a/README.md b/README.md index 3d55605..d9ea432 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,62 @@ Configure these at the **organization level** for sharing across repos: --- +### `desktop-smoke-test.yml` + +Runs unit tests and smoke tests on demand, without signing or publishing. Builds the same +unsigned Velopack packages as `desktop-publish.yml`, then launches them on each selected +platform. Useful for checking a large change before cutting a release. No secrets are needed. + +```yaml +name: Smoke Test +on: + workflow_dispatch: + inputs: + platforms: + description: 'Platforms to smoke test' + type: choice + options: [all, windows, macos, linux] + default: all + +jobs: + smoke-test: + uses: mythetech/workflows/.github/workflows/desktop-smoke-test.yml@main + with: + app_name: "Horizon" + project_path: "Horizon/Horizon.csproj" + test_project: "Horizon.Test/Horizon.Test.csproj" + icon_windows: "Horizon/wwwroot/logo.ico" + icon_linux: "Horizon/wwwroot/logo.png" + icon_macos: "Horizon/wwwroot/logo.icns" + platforms: ${{ inputs.platforms }} +``` + +Unit tests run through `pr-test.yml`, so VSTest/MTP detection works the same as on pull +requests. Tests and smoke tests run in parallel. + +#### Inputs + +| Input | Required | Default | Description | +|-------|----------|---------|-------------| +| `app_name` | **Yes** | - | Application name (e.g., "Horizon") | +| `project_path` | **Yes** | - | Path to main .csproj | +| `icon_windows` | **Yes** | - | Windows icon (.ico) | +| `icon_linux` | **Yes** | - | Linux icon (.png) | +| `icon_macos` | **Yes** | - | macOS icon (.icns) | +| `platforms` | No | `all` | `all`, `windows`, `macos` or `linux` | +| `dotnet_version` | No | `10.0.x` | .NET SDK version. Ignored when `global_json_file` is set | +| `global_json_file` | No | - | Path to a `global.json` pinning the SDK | +| `enable_tests` | No | `true` | Run unit tests alongside the smoke tests | +| `test_project` | No | - | Path to test project. If omitted, runs `dotnet test` in root | +| `test_command` | No | - | Override the entire test command | +| `test_runner` | No | `auto` | `auto`, `vstest` or `mtp` | +| `enable_coverage` | No | `false` | Collect code coverage for the unit tests | + +The packages are unsigned, so the macOS signing-specific bundle restructuring from +`desktop-publish.yml` is not exercised here. + +--- + ## Composite Actions ### `actions/macos-sign` diff --git a/actions/smoke-test/Start-SmokeRun.Tests.ps1 b/actions/smoke-test/Start-SmokeRun.Tests.ps1 new file mode 100644 index 0000000..185d697 --- /dev/null +++ b/actions/smoke-test/Start-SmokeRun.Tests.ps1 @@ -0,0 +1,129 @@ +# Copyright (c) Mythetech. Licensed under the MIT License. + +BeforeAll { + . (Join-Path $PSScriptRoot 'Start-SmokeRun.ps1') +} + +Describe 'Get-SmokeEnvironment' { + It 'turns smoke mode on with the budget and result path' { + $environment = Get-SmokeEnvironment -TimeoutSeconds 45 -ResultPath '/tmp/out/result.json' + + $environment['HERMES_SMOKE_TEST'] | Should -Be '1' + $environment['HERMES_SMOKE_TEST_TIMEOUT'] | Should -Be '45' + $environment['HERMES_SMOKE_TEST_RESULT'] | Should -Be '/tmp/out/result.json' + } +} + +Describe 'Get-SmokeLaunchPlan' { + It 'launches macOS apps through open with explicit environment and output paths' { + $environment = [ordered]@{ HERMES_SMOKE_TEST = '1'; HERMES_SMOKE_TEST_TIMEOUT = '60' } + + $plan = Get-SmokeLaunchPlan -Platform macOS -TargetPath '/work/app/My App.app' ` + -StdoutPath '/work/out/app-stdout.log' -StderrPath '/work/out/app-stderr.log' -Environment $environment + + $plan.FilePath | Should -Be 'open' + $plan.RedirectOutput | Should -BeFalse + $plan.ArgumentList | Should -Be @( + '-W', '-n', + '--env', 'HERMES_SMOKE_TEST=1', + '--env', 'HERMES_SMOKE_TEST_TIMEOUT=60', + '--stdout', '/work/out/app-stdout.log', + '--stderr', '/work/out/app-stderr.log', + '/work/app/My App.app' + ) + } + + It 'keeps a path with spaces as a single argument' { + $plan = Get-SmokeLaunchPlan -Platform macOS -TargetPath '/work/app/Http Platypus.app' ` + -StdoutPath '/o/out.log' -StderrPath '/o/err.log' -Environment ([ordered]@{}) + + $plan.ArgumentList[-1] | Should -Be '/work/app/Http Platypus.app' + } + + It 'runs Windows and Linux targets directly with redirected output' { + foreach ($platform in 'Windows', 'Linux') { + $plan = Get-SmokeLaunchPlan -Platform $platform -TargetPath '/work/app/App' ` + -StdoutPath '/o/out.log' -StderrPath '/o/err.log' -Environment ([ordered]@{}) + + $plan.FilePath | Should -Be '/work/app/App' + $plan.RedirectOutput | Should -BeTrue + @($plan.ArgumentList).Count | Should -Be 0 + } + } +} + +Describe 'Test-SmokeStarted' { + It 'is false when the log does not exist yet' { + Test-SmokeStarted -LogPath (Join-Path $TestDrive ([guid]::NewGuid())) | Should -BeFalse + } + + It 'is true once the start line is in the log, even with Windows line endings' { + $logPath = Join-Path $TestDrive 'stdout-crlf.log' + [System.IO.File]::WriteAllText($logPath, "starting up`r`nHERMES_SMOKE_START: App 1.0.0 Windows x64`r`n") + + Test-SmokeStarted -LogPath $logPath | Should -BeTrue + } + + It 'ignores the marker when it is not at the start of a line' { + $logPath = Join-Path $TestDrive 'stdout-embedded.log' + Set-Content -LiteralPath $logPath -Value 'echo HERMES_SMOKE_START: nope' + + Test-SmokeStarted -LogPath $logPath | Should -BeFalse + } +} + +Describe 'Get-SmokeRunOutcome' { + It 'classifies an app that exited before the start window as legacy, not alive' { + $outcome = Get-SmokeRunOutcome -Platform Linux -Started $false -Mode 'verdict' -TimedOut $false ` + -LegacyAlive $false -ExitCode 1 -StartedAfterExit $false + + $outcome.mode | Should -Be 'legacy' + $outcome.legacyAlive | Should -BeFalse + $outcome.exitCode | Should -BeNullOrEmpty + } + + It 'keeps a legacy app alive when the start window elapsed with the process still running' { + $outcome = Get-SmokeRunOutcome -Platform Linux -Started $false -Mode 'legacy' -TimedOut $false ` + -LegacyAlive $true -ExitCode 0 -StartedAfterExit $false + + $outcome.mode | Should -Be 'legacy' + $outcome.legacyAlive | Should -BeTrue + $outcome.exitCode | Should -BeNullOrEmpty + } + + It 'never gives a liveness pass when the start line only becomes readable after the legacy stop' { + $outcome = Get-SmokeRunOutcome -Platform Linux -Started $false -Mode 'legacy' -TimedOut $false ` + -LegacyAlive $true -ExitCode 0 -StartedAfterExit $true + + $outcome.mode | Should -Be 'verdict' + $outcome.timedOut | Should -BeTrue + $outcome.legacyAlive | Should -BeFalse + $outcome.exitCode | Should -BeNullOrEmpty + } + + It 'nulls the exit code for a verdict-mode timeout' { + $outcome = Get-SmokeRunOutcome -Platform Windows -Started $true -Mode 'verdict' -TimedOut $true ` + -LegacyAlive $false -ExitCode 0 -StartedAfterExit $false + + $outcome.mode | Should -Be 'verdict' + $outcome.exitCode | Should -BeNullOrEmpty + } + + It 'keeps the process exit code for a normal verdict exit on Windows and Linux' { + foreach ($platform in 'Windows', 'Linux') { + $outcome = Get-SmokeRunOutcome -Platform $platform -Started $true -Mode 'verdict' -TimedOut $false ` + -LegacyAlive $false -ExitCode 3 -StartedAfterExit $false + + $outcome.mode | Should -Be 'verdict' + $outcome.exitCode | Should -Be 3 + } + } + + It 'always nulls the exit code on macOS even on a normal verdict exit' { + $outcome = Get-SmokeRunOutcome -Platform macOS -Started $true -Mode 'verdict' -TimedOut $false ` + -LegacyAlive $false -ExitCode 0 -StartedAfterExit $false + + $outcome.mode | Should -Be 'verdict' + $outcome.exitCode | Should -BeNullOrEmpty + } +} diff --git a/actions/smoke-test/Start-SmokeRun.ps1 b/actions/smoke-test/Start-SmokeRun.ps1 new file mode 100644 index 0000000..f13fc27 --- /dev/null +++ b/actions/smoke-test/Start-SmokeRun.ps1 @@ -0,0 +1,296 @@ +# Copyright (c) Mythetech. Licensed under the MIT License. +# Launches a packaged desktop app in Hermes smoke mode and records how the run ended in run.json. +# Dot-source to load the functions only (the tests do); run as a script to perform a run. +[CmdletBinding()] +param( + [string] $AppName, + [string] $Platform, + [string] $ReleasesDir = 'releases', + [string] $OutputDir = 'smoke-output', + [int] $TimeoutSeconds = 60 +) + +Set-StrictMode -Version Latest + +# A cold single-file launch on a macOS runner can take longer than 15 seconds before the Hermes +# builder prints HERMES_SMOKE_START, so the window is wider than the spec's 15. Only apps that +# predate smoke mode pay for it. +$script:SmokeStartWindowSeconds = 30 +$script:CiGraceSeconds = 30 + +function Get-SmokeEnvironment { + param([Parameter(Mandatory)] [int] $TimeoutSeconds, [Parameter(Mandatory)] [string] $ResultPath) + + [ordered]@{ + HERMES_SMOKE_TEST = '1' + HERMES_SMOKE_TEST_TIMEOUT = "$TimeoutSeconds" + HERMES_SMOKE_TEST_RESULT = $ResultPath + } +} + +function Get-SmokeLaunchPlan { + param( + [Parameter(Mandatory)] [ValidateSet('Windows', 'macOS', 'Linux')] [string] $Platform, + [Parameter(Mandatory)] [string] $TargetPath, + [Parameter(Mandatory)] [string] $StdoutPath, + [Parameter(Mandatory)] [string] $StderrPath, + [Parameter(Mandatory)] [System.Collections.IDictionary] $Environment + ) + + if ($Platform -eq 'macOS') { + # open hands the app to LaunchServices, which ignores the caller's environment and stdout, + # so both go through open's own flags. -W waits for the app to quit; -n forces a new + # instance so the flags apply. + $arguments = [System.Collections.Generic.List[string]]::new() + $arguments.Add('-W') + $arguments.Add('-n') + foreach ($key in $Environment.Keys) { + $arguments.Add('--env') + $arguments.Add("$key=$($Environment[$key])") + } + $arguments.Add('--stdout') + $arguments.Add($StdoutPath) + $arguments.Add('--stderr') + $arguments.Add($StderrPath) + $arguments.Add($TargetPath) + + return [pscustomobject]@{ FilePath = 'open'; ArgumentList = $arguments.ToArray(); RedirectOutput = $false } + } + + return [pscustomobject]@{ FilePath = $TargetPath; ArgumentList = @(); RedirectOutput = $true } +} + +function Test-SmokeStarted { + param([Parameter(Mandatory)] [string] $LogPath) + + if (-not (Test-Path -LiteralPath $LogPath)) { + return $false + } + + try { + # The app is still writing the log; open it shared so this read never blocks the writer. + $stream = [System.IO.File]::Open($LogPath, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite) + try { + $text = [System.IO.StreamReader]::new($stream).ReadToEnd() + } + finally { + $stream.Dispose() + } + } + catch [System.IO.IOException] { + return $false + } + + return $text -match '(?m)^HERMES_SMOKE_START:' +} + +function Find-SmokeTarget { + param([string] $Platform, [string] $AppName, [string] $ReleasesDir, [string] $WorkDir) + + New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null + + switch ($Platform) { + 'Windows' { + $zip = Get-ChildItem -Path $ReleasesDir -Filter '*Portable*.zip' -Recurse | Select-Object -First 1 + if (-not $zip) { $zip = Get-ChildItem -Path $ReleasesDir -Filter '*.zip' -Recurse | Select-Object -First 1 } + if (-not $zip) { throw "No zip archive found in $ReleasesDir" } + Expand-Archive -Path $zip.FullName -DestinationPath $WorkDir -Force + + if (-not (Get-ChildItem -Path $WorkDir -Filter 'Microsoft.Web.WebView2*.dll' -Recurse)) { + Write-Warning 'WebView2 assemblies were not found in the package' + } + + # Velopack puts a launcher stub at the root; the real app lives in current/. + $exe = Get-ChildItem -Path (Join-Path $WorkDir 'current') -Filter "$AppName.exe" -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $exe) { $exe = Get-ChildItem -Path $WorkDir -Filter "$AppName.exe" -Recurse | Select-Object -First 1 } + if (-not $exe) { throw "$AppName.exe not found in the package" } + return $exe.FullName + } + 'macOS' { + $zip = Get-ChildItem -Path $ReleasesDir -Filter '*Portable*.zip' -Recurse | Select-Object -First 1 + if (-not $zip) { $zip = Get-ChildItem -Path $ReleasesDir -Filter '*.zip' -Recurse | Select-Object -First 1 } + if (-not $zip) { throw "No zip archive found in $ReleasesDir" } + # unzip keeps the symlinks and executable bits an app bundle needs; Expand-Archive does not. + & unzip -q $zip.FullName -d $WorkDir | Out-Null + if ($LASTEXITCODE -ne 0) { throw "unzip failed for $($zip.FullName)" } + + $bundle = Get-ChildItem -Path $WorkDir -Filter '*.app' -Directory -Recurse | Select-Object -First 1 + if (-not $bundle) { throw "No .app bundle found in $($zip.FullName)" } + & xattr -cr $bundle.FullName | Out-Null + return $bundle.FullName + } + 'Linux' { + $appImage = Get-ChildItem -Path $ReleasesDir -Filter '*.AppImage' -Recurse | Select-Object -First 1 + if (-not $appImage) { throw "No AppImage found in $ReleasesDir" } + & chmod +x $appImage.FullName | Out-Null + return $appImage.FullName + } + default { throw "Unknown platform '$Platform'" } + } +} + +function Start-SmokeProcess { + param($Plan, [string] $StdoutPath, [string] $StderrPath, [System.Collections.IDictionary] $Environment) + + foreach ($key in $Environment.Keys) { + Set-Item -Path "env:$key" -Value $Environment[$key] + } + + if ($Plan.RedirectOutput) { + $process = Start-Process -FilePath $Plan.FilePath -RedirectStandardOutput $StdoutPath -RedirectStandardError $StderrPath -PassThru + } + else { + # ProcessStartInfo.ArgumentList quotes each argument correctly; Start-Process joins them with spaces. + $startInfo = [System.Diagnostics.ProcessStartInfo]::new($Plan.FilePath) + foreach ($argument in $Plan.ArgumentList) { $startInfo.ArgumentList.Add($argument) } + $startInfo.UseShellExecute = $false + $process = [System.Diagnostics.Process]::Start($startInfo) + } + + # Reading Handle now keeps ExitCode available after the process ends on Windows. + $null = $process.Handle + return $process +} + +function Stop-SmokeApp { + param([string] $Platform, [string] $AppName, [System.Diagnostics.Process] $Process) + + switch ($Platform) { + 'Windows' { + $app = Get-Process -Name $AppName -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($app) { + $null = $app.CloseMainWindow() + if (-not $app.WaitForExit(10000)) { $app.Kill($true) } + } + } + 'macOS' { + # A hung app can leave osascript waiting forever for the Apple Event reply; bound it + # so a stuck quit still falls through to pkill instead of hanging the whole step. + & osascript -e 'with timeout of 5 seconds' -e "quit app `"$AppName`"" -e 'end timeout' 2>&1 | Out-Null + & pkill -x $AppName 2>&1 | Out-Null + Start-Sleep -Seconds 2 + & pkill -KILL -x $AppName 2>&1 | Out-Null + } + } + + if (-not $Process.HasExited) { $Process.Kill($true) } + $null = $Process.WaitForExit(10000) + + # pkill exits 1 when nothing matches, which is the expected outcome once quit already worked. + # GitHub's pwsh step wrapper turns a stale $LASTEXITCODE into a failed step, so clear it here. + $global:LASTEXITCODE = 0 +} + +function Get-SmokeRunOutcome { + param( + [Parameter(Mandatory)] [string] $Platform, + [Parameter(Mandatory)] [bool] $Started, + [Parameter(Mandatory)] [ValidateSet('verdict', 'legacy')] [string] $Mode, + [Parameter(Mandatory)] [bool] $TimedOut, + [Parameter(Mandatory)] [bool] $LegacyAlive, + [Nullable[int]] $ExitCode = $null, + [Parameter(Mandatory)] [bool] $StartedAfterExit + ) + + if (-not $Started -and $StartedAfterExit) { + if ($Mode -eq 'legacy') { + # The start line was there but unreadable while the app ran; never pass a smoke-aware app on liveness. + $Mode = 'verdict' + $TimedOut = $true + $LegacyAlive = $false + } + $Started = $true + } + + if (-not $Started -and $Mode -eq 'verdict' -and -not $TimedOut) { + # Exited without ever printing the start line: a crash, judged by the legacy rules. + $Mode = 'legacy' + $LegacyAlive = $false + } + + $finalExitCode = if ($Platform -eq 'macOS' -or $TimedOut -or $Mode -eq 'legacy') { $null } else { $ExitCode } + + [ordered]@{ mode = $Mode; exitCode = $finalExitCode; timedOut = $TimedOut; legacyAlive = $LegacyAlive } +} + +function Invoke-SmokeRun { + param([string] $AppName, [string] $Platform, [string] $ReleasesDir, [string] $OutputDir, [int] $TimeoutSeconds) + + $outputPath = (New-Item -ItemType Directory -Force -Path $OutputDir).FullName + $stdoutPath = Join-Path $outputPath 'app-stdout.log' + $stderrPath = Join-Path $outputPath 'app-stderr.log' + $resultPath = Join-Path $outputPath 'result.json' + $runPath = Join-Path $outputPath 'run.json' + $workDir = Join-Path (Get-Location) 'smoke-app' + + # A second run reusing OutputDir/WorkDir (retries in the same job) must never let a prior + # run's leftovers produce a false verdict or make unzip/Expand-Archive prompt or merge into + # an existing bundle. + foreach ($path in @($stdoutPath, $stderrPath, $resultPath, $runPath)) { + Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue + } + if (Test-Path -LiteralPath $workDir) { + Remove-Item -LiteralPath $workDir -Recurse -Force + } + + $target = Find-SmokeTarget -Platform $Platform -AppName $AppName -ReleasesDir $ReleasesDir -WorkDir $workDir + $environment = Get-SmokeEnvironment -TimeoutSeconds $TimeoutSeconds -ResultPath $resultPath + if ($Platform -eq 'Windows') { + # Single-file extraction under %TEMP% trips Windows Defender on hosted runners. + $extractDir = (New-Item -ItemType Directory -Force -Path (Join-Path (Get-Location) 'dotnet-extract')).FullName + $environment['DOTNET_BUNDLE_EXTRACT_BASE_DIR'] = $extractDir + } + + $plan = Get-SmokeLaunchPlan -Platform $Platform -TargetPath $target -StdoutPath $stdoutPath -StderrPath $stderrPath -Environment $environment + Write-Output "Launching $target" + $process = Start-SmokeProcess -Plan $plan -StdoutPath $stdoutPath -StderrPath $stderrPath -Environment $environment + + $deadline = $TimeoutSeconds + $script:CiGraceSeconds + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + $started = $false + $mode = 'verdict' + $timedOut = $false + $legacyAlive = $false + + while (-not $process.HasExited) { + if (-not $started -and (Test-SmokeStarted -LogPath $stdoutPath)) { + $started = $true + Write-Output 'The app printed HERMES_SMOKE_START; waiting for its verdict' + } + + $elapsed = $stopwatch.Elapsed.TotalSeconds + if (-not $started -and $elapsed -ge $script:SmokeStartWindowSeconds) { + $mode = 'legacy' + $legacyAlive = $true + Write-Output "No HERMES_SMOKE_START after $($script:SmokeStartWindowSeconds)s; checking liveness only" + Stop-SmokeApp -Platform $Platform -AppName $AppName -Process $process + break + } + + if ($elapsed -ge $deadline) { + $timedOut = $true + Write-Output "The app did not exit within ${deadline}s; stopping it" + Stop-SmokeApp -Platform $Platform -AppName $AppName -Process $process + break + } + + Start-Sleep -Milliseconds 500 + } + + $outcome = Get-SmokeRunOutcome -Platform $Platform -Started $started -Mode $mode -TimedOut $timedOut ` + -LegacyAlive $legacyAlive -ExitCode $process.ExitCode -StartedAfterExit (Test-SmokeStarted -LogPath $stdoutPath) + + $outcome | ConvertTo-Json | Set-Content -LiteralPath $runPath + + Write-Output "Run finished: mode=$($outcome.mode) timedOut=$($outcome.timedOut) exitCode=$($outcome.exitCode)" +} + +if ($MyInvocation.InvocationName -ne '.') { + $ErrorActionPreference = 'Stop' + Invoke-SmokeRun -AppName $AppName -Platform $Platform -ReleasesDir $ReleasesDir -OutputDir $OutputDir -TimeoutSeconds $TimeoutSeconds + + # The launcher's job is to record the run; Invoke-SmokeVerdict.ps1 judges it. A stale + # $LASTEXITCODE from a native tool called during the run (for example Stop-SmokeApp's pkill) + # would otherwise make GitHub's pwsh step wrapper fail this step even on a recorded pass. + exit 0 +} diff --git a/actions/smoke-test/action.yml b/actions/smoke-test/action.yml index 2143867..ece20e6 100644 --- a/actions/smoke-test/action.yml +++ b/actions/smoke-test/action.yml @@ -1,5 +1,5 @@ name: 'Smoke Test Desktop App' -description: 'Verifies that a desktop application can launch and display a window' +description: 'Launches a packaged desktop app in Hermes smoke mode and fails unless it reports a passing verdict' inputs: app_name: @@ -12,262 +12,104 @@ inputs: description: 'Directory containing the release artifacts' required: true default: 'releases' + timeout: + description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT' + required: false + default: '60' + require_verdict: + description: 'Fail when the app predates smoke mode instead of falling back to a liveness check' + required: false + default: 'false' + output_dir: + description: 'Directory for the app log, result JSON, run record and failure screenshot' + required: false + default: 'smoke-output' + +outputs: + result: + description: 'passed or failed' + value: ${{ steps.verdict.outputs.result }} + reason: + description: 'One-line explanation of the verdict' + value: ${{ steps.verdict.outputs.reason }} runs: using: 'composite' steps: - # Windows smoke test - - name: Smoke Test (Windows) - if: inputs.platform == 'Windows' - shell: pwsh - run: | - # Find and extract portable zip (simpler than installer for CI) - $zip = Get-ChildItem -Path "${{ inputs.releases_dir }}" -Filter "*Portable*.zip" -Recurse | Select-Object -First 1 - if (-not $zip) { - # Fall back to any zip - $zip = Get-ChildItem -Path "${{ inputs.releases_dir }}" -Filter "*.zip" -Recurse | Select-Object -First 1 - } - if (-not $zip) { throw "No zip archive found in ${{ inputs.releases_dir }}/" } - - Write-Output "Extracting: $($zip.FullName)" - Expand-Archive -Path $zip.FullName -DestinationPath "app" -Force - - # List contents to diagnose missing assemblies - Write-Output "=== Package contents (app/current/) ===" - Get-ChildItem -Path "app/current" -Recurse -ErrorAction SilentlyContinue | ForEach-Object { - Write-Output " $($_.FullName.Replace((Get-Location).Path + '\app\current\', ''))" - } - Write-Output "=== End package contents ===" - - # Check specifically for WebView2 assemblies - $webview2 = Get-ChildItem -Path "app" -Filter "Microsoft.Web.WebView2*.dll" -Recurse - if ($webview2) { - Write-Output "Found WebView2 assemblies:" - $webview2 | ForEach-Object { Write-Output " $($_.FullName)" } - } else { - Write-Warning "WebView2 assemblies NOT found in package" - } - - # Prefer the real app in current/ (Velopack packages have a launcher stub at root) - $exe = Get-ChildItem -Path "app/current" -Filter "${{ inputs.app_name }}.exe" -ErrorAction SilentlyContinue | Select-Object -First 1 - if (-not $exe) { - $exe = Get-ChildItem -Path "app" -Filter "${{ inputs.app_name }}.exe" -Recurse | Select-Object -First 1 - } - if (-not $exe) { throw "${{ inputs.app_name }}.exe not found in package" } - - Write-Output "Launching: $($exe.FullName)" - - # Set extraction directory for single-file apps to avoid Windows Defender issues with %TEMP% - $extractDir = Join-Path $PWD "dotnet-extract" - New-Item -ItemType Directory -Force -Path $extractDir | Out-Null - [System.Environment]::SetEnvironmentVariable("DOTNET_BUNDLE_EXTRACT_BASE_DIR", $extractDir, "Process") - Write-Output "Set DOTNET_BUNDLE_EXTRACT_BASE_DIR=$extractDir" - - # Use ProcessStartInfo to ensure environment variables are inherited - $psi = New-Object System.Diagnostics.ProcessStartInfo - $psi.FileName = $exe.FullName - $psi.UseShellExecute = $false - $psi.RedirectStandardError = $true - $psi.RedirectStandardOutput = $true - $psi.Environment["DOTNET_BUNDLE_EXTRACT_BASE_DIR"] = $extractDir - $psi.Environment["HERMES_SMOKE_TEST"] = "1" - $launchedProcess = [System.Diagnostics.Process]::Start($psi) - Write-Output "Launched process PID: $($launchedProcess.Id)" - - # Poll for the process to appear (startup can be slow on CI) - $maxWait = 60 - $elapsed = 0 - $appProcess = $null - while ($elapsed -lt $maxWait) { - Start-Sleep -Seconds 5 - $elapsed += 5 - - # Check if the launched process crashed or exited early - if ($launchedProcess.HasExited -and -not $appProcess) { - Write-Warning "Launched process exited with code $($launchedProcess.ExitCode)" - $stderr = $launchedProcess.StandardError.ReadToEnd() - $stdout = $launchedProcess.StandardOutput.ReadToEnd() - if ($stderr) { Write-Output "=== STDERR ===$([Environment]::NewLine)$stderr" } - if ($stdout) { Write-Output "=== STDOUT ===$([Environment]::NewLine)$stdout" } - Write-Error "${{ inputs.app_name }} process exited unexpectedly with code $($launchedProcess.ExitCode)" - exit 1 - } - - $appProcess = Get-Process -Name "${{ inputs.app_name }}" -ErrorAction SilentlyContinue - if ($appProcess) { - Write-Output "${{ inputs.app_name }} process found after $elapsed seconds" - break - } - Write-Output "Waiting for ${{ inputs.app_name }} process... ($elapsed/$maxWait seconds)" - } - if (-not $appProcess) { - Write-Output "=== Running processes ===" - Get-Process | Where-Object { $_.MainWindowTitle -or $_.ProcessName -match 'dotnet|siren|${{ inputs.app_name }}' } | - Format-Table ProcessName, Id, MainWindowTitle -AutoSize | Out-String | Write-Output - Write-Output "=== All dotnet-related processes ===" - Get-Process | Where-Object { $_.Path -and $_.Path -like "*app*" } | - Format-Table ProcessName, Id, Path -AutoSize | Out-String | Write-Output - Write-Error "${{ inputs.app_name }} process not found after $maxWait seconds" - exit 1 - } - - Write-Output "Found ${{ inputs.app_name }} process (PID: $($appProcess.Id))" - - if (-not $appProcess.MainWindowHandle -or $appProcess.MainWindowHandle -eq 0) { - Write-Warning "Main window handle not found, but process is running" - } else { - Write-Output "Window title: $($appProcess.MainWindowTitle)" - } - - # Clean shutdown - $appProcess.CloseMainWindow() | Out-Null - if (-not $appProcess.WaitForExit(10000)) { - Write-Warning "App did not close gracefully, killing..." - $appProcess.Kill() - } - Write-Output "Windows smoke test PASSED" - - # macOS smoke test - - name: Smoke Test (macOS) - if: inputs.platform == 'macOS' - shell: bash - run: | - # Find portable app zip - ZIP_FILE=$(find "${{ inputs.releases_dir }}" -name "*Portable*.zip" 2>/dev/null | head -1) - if [ -z "$ZIP_FILE" ]; then - # Fall back to any zip - ZIP_FILE=$(find "${{ inputs.releases_dir }}" -name "*.zip" 2>/dev/null | head -1) - fi - if [ -z "$ZIP_FILE" ]; then - echo "No zip archive found" - ls -la "${{ inputs.releases_dir }}/" - exit 1 - fi - - echo "Extracting: $ZIP_FILE" - unzip -q "$ZIP_FILE" -d app - - APP_BUNDLE=$(find app -name "*.app" -type d | head -1) - if [ -z "$APP_BUNDLE" ]; then - echo "No .app bundle found in archive" - ls -laR app/ - exit 1 - fi - - # Clear quarantine flag (required for CI) - xattr -cr "$APP_BUNDLE" - - echo "Launching: $APP_BUNDLE" - open "$APP_BUNDLE" - - # Poll for the process to appear - MAX_WAIT=60 - ELAPSED=0 - while [ $ELAPSED -lt $MAX_WAIT ]; do - sleep 5 - ELAPSED=$((ELAPSED + 5)) - if pgrep -x "${{ inputs.app_name }}" > /dev/null; then - echo "${{ inputs.app_name }} process found after $ELAPSED seconds" - break - fi - echo "Waiting for ${{ inputs.app_name }} process... ($ELAPSED/$MAX_WAIT seconds)" - done - if ! pgrep -x "${{ inputs.app_name }}" > /dev/null; then - echo "Application not running after $MAX_WAIT seconds" - exit 1 - fi - - # Give the window time to fully render before checking via AppleScript - sleep 10 - - # Verify window exists via AppleScript - osascript -e 'tell application "System Events" to get name of first window of process "${{ inputs.app_name }}"' || { - echo "Warning: Could not verify window via AppleScript, but process is running" - } - - # Clean shutdown - osascript -e 'quit app "${{ inputs.app_name }}"' || true - sleep 2 - echo "macOS smoke test PASSED" - - # Linux dependencies - name: Install Linux Dependencies if: inputs.platform == 'Linux' shell: bash run: | sudo apt-get update - sudo apt-get install -y xvfb libgtk-3-0 libwebkit2gtk-4.1-0 xdotool libfuse2 libnotify4 + sudo apt-get install -y xvfb libgtk-3-0 libwebkit2gtk-4.1-0 xdotool libfuse2 libnotify4 scrot - # Linux smoke test - - name: Smoke Test (Linux) + - name: Start Virtual Display (Linux) if: inputs.platform == 'Linux' shell: bash run: | - # Find AppImage - APPIMAGE=$(find "${{ inputs.releases_dir }}" -name "*.AppImage" | head -1) - if [ -z "$APPIMAGE" ]; then - echo "No AppImage found" - ls -la "${{ inputs.releases_dir }}/" - exit 1 - fi - chmod +x "$APPIMAGE" - - # Start virtual display - export DISPLAY=:99 Xvfb :99 -screen 0 1920x1080x24 & - XVFB_PID=$! sleep 2 - echo "Launching: $APPIMAGE" - "$APPIMAGE" & - APP_PID=$! - sleep 20 - - # Check if still running - if ! kill -0 $APP_PID 2>/dev/null; then - echo "Application crashed" - kill $XVFB_PID 2>/dev/null || true - exit 1 - fi - - # Verify window using xdotool - WINDOW_ID=$(xdotool search --name "${{ inputs.app_name }}" 2>/dev/null | head -1) - if [ -z "$WINDOW_ID" ]; then - echo "Warning: Could not find ${{ inputs.app_name }} window via xdotool, but process is running" - else - echo "Found window ID: $WINDOW_ID" - fi - - # Cleanup - kill $APP_PID 2>/dev/null || true - kill $XVFB_PID 2>/dev/null || true - echo "Linux smoke test PASSED" + - name: Run App in Smoke Mode + shell: pwsh + env: + DISPLAY: ':99' + SMOKE_APP_NAME: ${{ inputs.app_name }} + SMOKE_PLATFORM: ${{ inputs.platform }} + SMOKE_RELEASES_DIR: ${{ inputs.releases_dir }} + SMOKE_OUTPUT_DIR: ${{ inputs.output_dir }} + SMOKE_TIMEOUT: ${{ inputs.timeout }} + run: | + & (Join-Path $env:GITHUB_ACTION_PATH 'Start-SmokeRun.ps1') ` + -AppName $env:SMOKE_APP_NAME ` + -Platform $env:SMOKE_PLATFORM ` + -ReleasesDir $env:SMOKE_RELEASES_DIR ` + -OutputDir $env:SMOKE_OUTPUT_DIR ` + -TimeoutSeconds ([int]$env:SMOKE_TIMEOUT) + + - name: Decide Verdict + id: verdict + if: ${{ !cancelled() }} + shell: pwsh + env: + SMOKE_PLATFORM: ${{ inputs.platform }} + SMOKE_OUTPUT_DIR: ${{ inputs.output_dir }} + SMOKE_REQUIRE_VERDICT: ${{ inputs.require_verdict }} + run: | + & (Join-Path $env:GITHUB_ACTION_PATH '..' 'smoke-verdict' 'Invoke-SmokeVerdict.ps1') ` + -OutputDir $env:SMOKE_OUTPUT_DIR ` + -RequireVerdict $env:SMOKE_REQUIRE_VERDICT ` + -Platform $env:SMOKE_PLATFORM - # Screenshot on failure - name: Screenshot on Failure (Windows) if: failure() && inputs.platform == 'Windows' shell: pwsh + env: + SMOKE_OUTPUT_DIR: ${{ inputs.output_dir }} run: | + New-Item -ItemType Directory -Force -Path $env:SMOKE_OUTPUT_DIR | Out-Null Add-Type -AssemblyName System.Windows.Forms Add-Type -AssemblyName System.Drawing $screen = [System.Windows.Forms.Screen]::PrimaryScreen $bitmap = New-Object System.Drawing.Bitmap($screen.Bounds.Width, $screen.Bounds.Height) $graphics = [System.Drawing.Graphics]::FromImage($bitmap) $graphics.CopyFromScreen($screen.Bounds.Location, [System.Drawing.Point]::Empty, $screen.Bounds.Size) - $bitmap.Save("screenshot-failure.png") - Write-Output "Screenshot saved to screenshot-failure.png" + $bitmap.Save((Join-Path $env:SMOKE_OUTPUT_DIR 'screenshot-failure.png')) - name: Screenshot on Failure (macOS) if: failure() && inputs.platform == 'macOS' shell: bash + env: + SMOKE_OUTPUT_DIR: ${{ inputs.output_dir }} run: | - screencapture -x screenshot-failure.png || true - echo "Screenshot saved to screenshot-failure.png" + mkdir -p "$SMOKE_OUTPUT_DIR" + screencapture -x "$SMOKE_OUTPUT_DIR/screenshot-failure.png" || true - name: Screenshot on Failure (Linux) if: failure() && inputs.platform == 'Linux' shell: bash + env: + SMOKE_OUTPUT_DIR: ${{ inputs.output_dir }} run: | - sudo apt-get install -y scrot || true - DISPLAY=:99 scrot screenshot-failure.png || true - echo "Screenshot saved to screenshot-failure.png" + mkdir -p "$SMOKE_OUTPUT_DIR" + DISPLAY=:99 scrot "$SMOKE_OUTPUT_DIR/screenshot-failure.png" || true diff --git a/actions/smoke-verdict/Invoke-SmokeVerdict.Tests.ps1 b/actions/smoke-verdict/Invoke-SmokeVerdict.Tests.ps1 new file mode 100644 index 0000000..2d1b60a --- /dev/null +++ b/actions/smoke-verdict/Invoke-SmokeVerdict.Tests.ps1 @@ -0,0 +1,109 @@ +# Copyright (c) Mythetech. Licensed under the MIT License. + +BeforeAll { + $script:Command = Join-Path $PSScriptRoot 'Invoke-SmokeVerdict.ps1' + + function New-RunDirectory { + param([string[]] $Log, [hashtable] $Run) + $dir = Join-Path $TestDrive ([guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $dir | Out-Null + if ($null -ne $Log) { Set-Content -LiteralPath (Join-Path $dir 'app-stdout.log') -Value $Log } + if ($null -ne $Run) { $Run | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $dir 'run.json') } + return $dir + } + + function Read-GitHubOutput { + param([string] $Path) + $values = @{} + foreach ($line in Get-Content -LiteralPath $Path) { + $name, $value = $line -split '=', 2 + $values[$name] = $value + } + return $values + } +} + +Describe 'Invoke-SmokeVerdict.ps1' { + BeforeEach { + # A real runner already has these set for the job; overwrite and restore them rather than + # deleting them, so the suite behaves the same locally and in self-test.yml. + $script:PreviousGitHubOutput = $env:GITHUB_OUTPUT + $script:PreviousGitHubStepSummary = $env:GITHUB_STEP_SUMMARY + + $script:OutputFile = Join-Path $TestDrive ([guid]::NewGuid().ToString('N') + '-output.txt') + New-Item -ItemType File -Path $script:OutputFile | Out-Null + $env:GITHUB_OUTPUT = $script:OutputFile + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive ([guid]::NewGuid().ToString('N') + '-summary.md') + New-Item -ItemType File -Path $env:GITHUB_STEP_SUMMARY | Out-Null + } + + AfterEach { + $env:GITHUB_OUTPUT = $script:PreviousGitHubOutput + $env:GITHUB_STEP_SUMMARY = $script:PreviousGitHubStepSummary + } + + It 'treats the string false as false for require_verdict' { + $dir = New-RunDirectory -Log @('app output') -Run @{ mode = 'legacy'; exitCode = $null; timedOut = $false; legacyAlive = $true } + + & $script:Command -OutputDir $dir -RequireVerdict 'false' -FailOnFailed 'false' + + (Read-GitHubOutput $script:OutputFile)['result'] | Should -Be 'passed' + } + + It 'treats the string true as true for require_verdict' { + $dir = New-RunDirectory -Log @('app output') -Run @{ mode = 'legacy'; exitCode = $null; timedOut = $false; legacyAlive = $true } + + & $script:Command -OutputDir $dir -RequireVerdict 'true' -FailOnFailed 'false' + + (Read-GitHubOutput $script:OutputFile)['result'] | Should -Be 'failed' + } + + It 'reports no verdict when the log file is missing' { + $dir = New-RunDirectory -Log $null -Run @{ mode = 'verdict'; exitCode = $null; timedOut = $true; legacyAlive = $false } + + & $script:Command -OutputDir $dir -RequireVerdict 'false' -FailOnFailed 'false' + + $outputs = Read-GitHubOutput $script:OutputFile + $outputs['result'] | Should -Be 'failed' + $outputs['reason'] | Should -Be 'The app was killed after the CI timeout without printing a verdict (no milestone was reached)' + } + + It 'reports FAILED with a launcher-failure reason when run.json itself is missing' { + $dir = New-RunDirectory -Log @('some partial output') -Run $null + + & $script:Command -OutputDir $dir -FailOnFailed 'false' + + $outputs = Read-GitHubOutput $script:OutputFile + $outputs['result'] | Should -Be 'failed' + $outputs['reason'] | Should -Be 'The launcher did not record a run (it failed before or while starting the app)' + } + + It 'writes the summary to the step summary file' { + $dir = New-RunDirectory -Log @('HERMES_SMOKE_RESULT: PASSED (0 checks)') -Run @{ mode = 'verdict'; exitCode = 0; timedOut = $false; legacyAlive = $false } + + & $script:Command -OutputDir $dir -Platform 'Windows' -FailOnFailed 'false' + + Get-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Raw | Should -Match '### Smoke test \(Windows\): PASSED' + } + + It 'prints the last 50 lines of stdout and stderr when the verdict fails' { + $dir = New-RunDirectory -Log @('HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 0 errors)') -Run @{ mode = 'verdict'; exitCode = 1; timedOut = $false; legacyAlive = $false } + Set-Content -LiteralPath (Join-Path $dir 'app-stderr.log') -Value @('a stderr line') + + $output = & $script:Command -OutputDir $dir -FailOnFailed 'false' + + $output | Should -Contain '::group::App stdout (last 50 lines)' + $output | Should -Contain 'HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 0 errors)' + $output | Should -Contain '::group::App stderr (last 50 lines)' + $output | Should -Contain 'a stderr line' + } + + It 'exits 1 on a failed verdict when failing is enabled' { + $global:LASTEXITCODE = 0 + $dir = New-RunDirectory -Log @('HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 0 errors)') -Run @{ mode = 'verdict'; exitCode = 1; timedOut = $false; legacyAlive = $false } + + & $script:Command -OutputDir $dir -FailOnFailed 'true' | Out-Null + + $LASTEXITCODE | Should -Be 1 + } +} diff --git a/actions/smoke-verdict/Invoke-SmokeVerdict.ps1 b/actions/smoke-verdict/Invoke-SmokeVerdict.ps1 new file mode 100644 index 0000000..e45d5c0 --- /dev/null +++ b/actions/smoke-verdict/Invoke-SmokeVerdict.ps1 @@ -0,0 +1,76 @@ +# Copyright (c) Mythetech. Licensed under the MIT License. +# Decides a smoke run's verdict from the files Start-SmokeRun.ps1 (or Hermes CI) left in OutputDir: +# app-stdout.log, optionally result.json, and run.json, a JSON object +# { "mode": "verdict"|"legacy", "exitCode": int|null, "timedOut": bool, "legacyAlive": bool } +# written by Start-SmokeRun.ps1. +[CmdletBinding()] +param( + [Parameter(Mandatory)] [string] $OutputDir, + # Strings, not bools: YAML inputs arrive as text and [bool]'false' is $true. + [string] $RequireVerdict = 'false', + [string] $FailOnFailed = 'true', + [string] $Platform = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +. (Join-Path $PSScriptRoot 'SmokeVerdict.ps1') + +$logPath = Join-Path $OutputDir 'app-stdout.log' +$errPath = Join-Path $OutputDir 'app-stderr.log' +$resultPath = Join-Path $OutputDir 'result.json' +$runPath = Join-Path $OutputDir 'run.json' + +$logLines = if (Test-Path -LiteralPath $logPath) { @(Get-Content -LiteralPath $logPath) } else { @() } +$resultJson = if (Test-Path -LiteralPath $resultPath) { [string](Get-Content -LiteralPath $resultPath -Raw) } else { '' } + +$verdict = if (Test-Path -LiteralPath $runPath) { + $run = Get-Content -LiteralPath $runPath -Raw | ConvertFrom-Json + Get-SmokeVerdict ` + -LogLines $logLines ` + -ResultJson $resultJson ` + -Mode $run.mode ` + -ExitCode $run.exitCode ` + -TimedOut ([bool]$run.timedOut) ` + -LegacyAlive ([bool]$run.legacyAlive) ` + -RequireVerdict ($RequireVerdict -eq 'true') +} else { + # No run.json means the launcher itself never got that far (for example Find-SmokeTarget + # threw before a process was even started); defaulting to verdict mode would blame the app + # for something the launcher never gave it the chance to do. + New-SmokeVerdict -Passed $false -Source 'none' ` + -Reason 'The launcher did not record a run (it failed before or while starting the app)' +} + +$summary = Format-SmokeSummary -Verdict $verdict -Platform $Platform +Write-Output $summary + +if ($env:GITHUB_STEP_SUMMARY) { + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Value $summary +} + +if ($env:GITHUB_OUTPUT) { + $result = if ($verdict.Passed) { 'passed' } else { 'failed' } + Add-Content -LiteralPath $env:GITHUB_OUTPUT -Value "result=$result" + Add-Content -LiteralPath $env:GITHUB_OUTPUT -Value "reason=$($verdict.Reason -replace '[\r\n]+', ' ')" +} + +foreach ($warning in $verdict.Warnings) { + Write-Output "::warning::$warning" +} + +if (-not $verdict.Passed) { + foreach ($stream in @(@{ Name = 'stdout'; Path = $logPath }, @{ Name = 'stderr'; Path = $errPath })) { + if (Test-Path -LiteralPath $stream.Path) { + Write-Output "::group::App $($stream.Name) (last 50 lines)" + Get-Content -LiteralPath $stream.Path -Tail 50 | ForEach-Object { Write-Output $_ } + Write-Output '::endgroup::' + } + } +} + +if (-not $verdict.Passed -and $FailOnFailed -eq 'true') { + Write-Output "::error::Smoke test failed: $($verdict.Reason)" + exit 1 +} diff --git a/actions/smoke-verdict/SmokeVerdict.Tests.ps1 b/actions/smoke-verdict/SmokeVerdict.Tests.ps1 new file mode 100644 index 0000000..d6ca147 --- /dev/null +++ b/actions/smoke-verdict/SmokeVerdict.Tests.ps1 @@ -0,0 +1,162 @@ +# Copyright (c) Mythetech. Licensed under the MIT License. + +BeforeAll { + . (Join-Path $PSScriptRoot 'SmokeVerdict.ps1') + + function New-ResultJson { + param([string] $Result, [object[]] $Checks = @(), [object[]] $Errors = @(), [string] $TimedOutWaitingFor = $null) + [ordered]@{ + schema = 1 + result = $Result + timedOutWaitingFor = $TimedOutWaitingFor + checks = $Checks + errors = $Errors + } | ConvertTo-Json -Depth 5 + } +} + +Describe 'Get-SmokeVerdict in verdict mode' { + It 'passes from a passing result file' { + $json = New-ResultJson -Result 'passed' -Checks @(@{ name = 'framework/initialization'; status = 'passed' }) + + $verdict = Get-SmokeVerdict -Mode verdict -ResultJson $json -ExitCode 0 + + $verdict.Passed | Should -BeTrue + $verdict.Source | Should -Be 'result-file' + $verdict.Reason | Should -Be 'All 1 checks passed' + } + + It 'explains a failing result file with its failed checks, errors and timeout' { + $json = New-ResultJson -Result 'failed' -TimedOutWaitingFor 'app-ready' ` + -Checks @(@{ name = 'horizon/workspace-store'; status = 'failed' }, @{ name = 'framework/storage'; status = 'passed' }) ` + -Errors @(@{ source = 'blazor'; type = 'InvalidOperationException'; message = 'boom' }) + + $verdict = Get-SmokeVerdict -Mode verdict -ResultJson $json + + $verdict.Passed | Should -BeFalse + $verdict.Reason | Should -Be '1/2 checks failed (horizon/workspace-store), 1 error, timed out waiting for app-ready' + } + + It 'falls back to the last result line when there is no result file' { + $log = @('HERMES_SMOKE_START: App 1.0.0 Linux x64', 'HERMES_SMOKE_RESULT: PASSED (2 checks)') + + $verdict = Get-SmokeVerdict -Mode verdict -LogLines $log -ExitCode 0 + + $verdict.Passed | Should -BeTrue + $verdict.Source | Should -Be 'log' + $verdict.Reason | Should -Be 'PASSED (2 checks)' + } + + It 'fails from a failed result line' { + $log = @('HERMES_SMOKE_RESULT: FAILED (1/3 checks failed, 0 errors)') + + $verdict = Get-SmokeVerdict -Mode verdict -LogLines $log + + $verdict.Passed | Should -BeFalse + $verdict.Reason | Should -Be 'FAILED (1/3 checks failed, 0 errors)' + } + + It 'matches lines that end with a Windows carriage return' { + $log = @("HERMES_SMOKE_MILESTONE: window-shown 10ms`r", "HERMES_SMOKE_RESULT: PASSED (0 checks)`r") + + $verdict = Get-SmokeVerdict -Mode verdict -LogLines $log -ExitCode 0 + + $verdict.Passed | Should -BeTrue + $verdict.Milestones | Should -Be @('window-shown') + } + + It 'falls back to the log when the result file is empty or truncated' { + $log = @('HERMES_SMOKE_RESULT: PASSED (1 checks)') + + (Get-SmokeVerdict -Mode verdict -LogLines $log -ResultJson '' -ExitCode 0).Source | Should -Be 'log' + (Get-SmokeVerdict -Mode verdict -LogLines $log -ResultJson '{"schema":1,"res' -ExitCode 0).Source | Should -Be 'log' + } + + It 'fails with the last milestone when the app was killed without a verdict' { + $log = @('HERMES_SMOKE_START: App 1.0.0 macOS arm64', 'HERMES_SMOKE_MILESTONE: window-shown 400ms', 'HERMES_SMOKE_MILESTONE: first-render 900ms') + + $verdict = Get-SmokeVerdict -Mode verdict -LogLines $log -TimedOut $true + + $verdict.Passed | Should -BeFalse + $verdict.Source | Should -Be 'none' + $verdict.Reason | Should -Be 'The app was killed after the CI timeout without printing a verdict (last milestone reached: first-render)' + } + + It 'fails when the app exited without a verdict or any milestone' { + $verdict = Get-SmokeVerdict -Mode verdict -LogLines @('HERMES_SMOKE_START: App 1.0.0 Windows x64') -ExitCode 1 + + $verdict.Passed | Should -BeFalse + $verdict.Reason | Should -Be 'The app exited without printing a verdict (no milestone was reached)' + } + + It 'fails a passing verdict when the app crashed during shutdown' { + $verdict = Get-SmokeVerdict -Mode verdict -LogLines @('HERMES_SMOKE_RESULT: PASSED (1 checks)') -ExitCode 139 + + $verdict.Passed | Should -BeFalse + $verdict.Reason | Should -Be 'The app reported PASSED but exited with code 139 (crashed during shutdown)' + } + + It 'fails a passing verdict when the app hung during shutdown' { + $verdict = Get-SmokeVerdict -Mode verdict -LogLines @('HERMES_SMOKE_RESULT: PASSED (1 checks)') -TimedOut $true + + $verdict.Passed | Should -BeFalse + $verdict.Reason | Should -Be 'The app reported PASSED but did not exit before the CI timeout (hung during shutdown)' + } + + It 'collects failed checks and errors from the log for the summary' { + $log = @( + 'HERMES_SMOKE_CHECK_FAIL: horizon/workspace-store 10002ms - Timed out after 10s', + 'HERMES_SMOKE_ERROR: blazor: UnhandledException: Theme palette was null', + 'HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 1 error)' + ) + + $verdict = Get-SmokeVerdict -Mode verdict -LogLines $log + + $verdict.FailedChecks | Should -Be @('horizon/workspace-store 10002ms - Timed out after 10s') + $verdict.Errors | Should -Be @('blazor: UnhandledException: Theme palette was null') + } +} + +Describe 'Get-SmokeVerdict in legacy mode' { + It 'passes with a warning when the process stayed alive' { + $verdict = Get-SmokeVerdict -Mode legacy -LegacyAlive $true + + $verdict.Passed | Should -BeTrue + $verdict.Source | Should -Be 'legacy' + $verdict.Warnings | Should -Be @('This build does not support smoke mode (no HERMES_SMOKE_START); only liveness was verified') + } + + It 'fails when the process exited before the liveness check' { + $verdict = Get-SmokeVerdict -Mode legacy -LegacyAlive $false + + $verdict.Passed | Should -BeFalse + $verdict.Reason | Should -Be 'The app exited before the liveness check, without printing HERMES_SMOKE_START' + } + + It 'fails when a verdict is required' { + $verdict = Get-SmokeVerdict -Mode legacy -LegacyAlive $true -RequireVerdict $true + + $verdict.Passed | Should -BeFalse + $verdict.Reason | Should -Be 'The app never printed HERMES_SMOKE_START, so this build does not support smoke mode, and require_verdict is set' + } +} + +Describe 'Format-SmokeSummary' { + It 'lists the reason, milestones, failed checks, errors and warnings' { + $log = @( + 'HERMES_SMOKE_MILESTONE: window-shown 400ms', + 'HERMES_SMOKE_CHECK_FAIL: app/check 5ms - boom', + 'HERMES_SMOKE_ERROR: log: Error: bad', + 'HERMES_SMOKE_RESULT: FAILED (1/1 checks failed, 1 error)' + ) + $verdict = Get-SmokeVerdict -Mode verdict -LogLines $log + + $summary = Format-SmokeSummary -Verdict $verdict -Platform 'Linux' + + $summary | Should -Match '### Smoke test \(Linux\): FAILED' + $summary | Should -Match 'FAILED \(1/1 checks failed, 1 error\)' + $summary | Should -Match 'Milestones: window-shown' + $summary | Should -Match '- Failed check: app/check 5ms - boom' + $summary | Should -Match '- Error: log: Error: bad' + } +} diff --git a/actions/smoke-verdict/SmokeVerdict.ps1 b/actions/smoke-verdict/SmokeVerdict.ps1 new file mode 100644 index 0000000..a465d81 --- /dev/null +++ b/actions/smoke-verdict/SmokeVerdict.ps1 @@ -0,0 +1,181 @@ +# Copyright (c) Mythetech. Licensed under the MIT License. +# Verdict rules for Hermes smoke runs, shared by the smoke-test action and Hermes CI. +# Dot-source this file; Invoke-SmokeVerdict.ps1 is the command-line wrapper. + +function Get-SmokeLineValues { + param([string[]] $LogLines, [Parameter(Mandatory)] [string] $Prefix) + + # Scoped to this function, not file level: dot-sourcing this file (Hermes CI does) must not + # turn on strict mode in the caller's own scope. + Set-StrictMode -Version Latest + + foreach ($line in $LogLines) { + # Logs captured on Windows keep a trailing carriage return on every line. + $trimmed = $line.TrimEnd("`r") + if ($trimmed.StartsWith($Prefix, [StringComparison]::Ordinal)) { + $trimmed.Substring($Prefix.Length).Trim() + } + } +} + +function New-SmokeVerdict { + param( + [bool] $Passed, + [string] $Source, + [string] $Reason, + [string[]] $Milestones = @(), + [string[]] $FailedChecks = @(), + [string[]] $Errors = @(), + [string[]] $Warnings = @() + ) + + Set-StrictMode -Version Latest + + [pscustomobject]@{ + Passed = $Passed + Source = $Source + Reason = $Reason + Milestones = @($Milestones) + FailedChecks = @($FailedChecks) + Errors = @($Errors) + LastMilestone = $(if (@($Milestones).Count -gt 0) { @($Milestones)[-1] } else { $null }) + Warnings = @($Warnings) + } +} + +function Get-SmokeResultFileReason { + param([Parameter(Mandatory)] $Result) + + Set-StrictMode -Version Latest + + $checks = @($Result.checks) + if ($Result.result -eq 'passed') { + return "All $($checks.Count) checks passed" + } + + $parts = [System.Collections.Generic.List[string]]::new() + $failed = @($checks | Where-Object { $_.status -eq 'failed' } | ForEach-Object { $_.name }) + if ($failed.Count -gt 0) { + $parts.Add("$($failed.Count)/$($checks.Count) checks failed ($($failed -join ', '))") + } + + $errorCount = @($Result.errors).Count + if ($errorCount -gt 0) { + $parts.Add($(if ($errorCount -eq 1) { '1 error' } else { "$errorCount errors" })) + } + + if ($Result.timedOutWaitingFor) { + $parts.Add("timed out waiting for $($Result.timedOutWaitingFor)") + } + + if ($parts.Count -eq 0) { + $parts.Add('The app reported a failed run') + } + + return ($parts -join ', ') +} + +function Get-SmokeVerdict { + [CmdletBinding()] + param( + [string[]] $LogLines = @(), + [string] $ResultJson = '', + [Parameter(Mandatory)] [ValidateSet('verdict', 'legacy')] [string] $Mode, + [Nullable[int]] $ExitCode = $null, + [bool] $TimedOut = $false, + [bool] $LegacyAlive = $false, + [bool] $RequireVerdict = $false + ) + + Set-StrictMode -Version Latest + + $milestones = @(Get-SmokeLineValues -LogLines $LogLines -Prefix 'HERMES_SMOKE_MILESTONE:' | ForEach-Object { ($_ -split '\s+')[0] }) + $failedChecks = @(Get-SmokeLineValues -LogLines $LogLines -Prefix 'HERMES_SMOKE_CHECK_FAIL:') + $errors = @(Get-SmokeLineValues -LogLines $LogLines -Prefix 'HERMES_SMOKE_ERROR:') + $resultLines = @(Get-SmokeLineValues -LogLines $LogLines -Prefix 'HERMES_SMOKE_RESULT:') + $details = @{ Milestones = $milestones; FailedChecks = $failedChecks; Errors = $errors } + + if ($Mode -eq 'legacy') { + if ($RequireVerdict) { + return New-SmokeVerdict -Passed $false -Source 'legacy' @details ` + -Reason 'The app never printed HERMES_SMOKE_START, so this build does not support smoke mode, and require_verdict is set' + } + if ($LegacyAlive) { + return New-SmokeVerdict -Passed $true -Source 'legacy' @details ` + -Reason 'The app stayed alive through the liveness check' ` + -Warnings @('This build does not support smoke mode (no HERMES_SMOKE_START); only liveness was verified') + } + return New-SmokeVerdict -Passed $false -Source 'legacy' @details ` + -Reason 'The app exited before the liveness check, without printing HERMES_SMOKE_START' + } + + $passed = $null + $source = $null + $reason = $null + + if (-not [string]::IsNullOrWhiteSpace($ResultJson)) { + try { + $result = $ResultJson | ConvertFrom-Json -ErrorAction Stop + $reason = Get-SmokeResultFileReason -Result $result + $passed = $result.result -eq 'passed' + $source = 'result-file' + } + catch { + # A truncated or malformed file means the app died while writing it; the log still has the verdict line. + $passed = $null + } + } + + if ($null -eq $passed -and $resultLines.Count -gt 0) { + $line = $resultLines[-1] + $passed = $line.StartsWith('PASSED', [StringComparison]::Ordinal) + $source = 'log' + $reason = $line + } + + if ($null -eq $passed) { + $how = if ($TimedOut) { 'was killed after the CI timeout' } else { 'exited' } + $where = if ($milestones.Count -gt 0) { "last milestone reached: $($milestones[-1])" } else { 'no milestone was reached' } + return New-SmokeVerdict -Passed $false -Source 'none' @details -Reason "The app $how without printing a verdict ($where)" + } + + if ($passed -and $TimedOut) { + return New-SmokeVerdict -Passed $false -Source $source @details ` + -Reason 'The app reported PASSED but did not exit before the CI timeout (hung during shutdown)' + } + + if ($passed -and $null -ne $ExitCode -and $ExitCode -ne 0) { + return New-SmokeVerdict -Passed $false -Source $source @details ` + -Reason "The app reported PASSED but exited with code $ExitCode (crashed during shutdown)" + } + + return New-SmokeVerdict -Passed $passed -Source $source -Reason $reason @details +} + +function Format-SmokeSummary { + param([Parameter(Mandatory)] $Verdict, [string] $Platform = '') + + Set-StrictMode -Version Latest + + $status = if ($Verdict.Passed) { 'PASSED' } else { 'FAILED' } + $title = if ($Platform) { "### Smoke test ($Platform): $status" } else { "### Smoke test: $status" } + + $lines = [System.Collections.Generic.List[string]]::new() + $lines.Add($title) + $lines.Add('') + $lines.Add($Verdict.Reason) + + if (@($Verdict.Milestones).Count -gt 0) { + $lines.Add('') + $lines.Add("Milestones: $(@($Verdict.Milestones) -join ', ')") + } + + if (@($Verdict.FailedChecks).Count + @($Verdict.Errors).Count + @($Verdict.Warnings).Count -gt 0) { + $lines.Add('') + } + foreach ($check in $Verdict.FailedChecks) { $lines.Add("- Failed check: $check") } + foreach ($errorLine in $Verdict.Errors) { $lines.Add("- Error: $errorLine") } + foreach ($warning in $Verdict.Warnings) { $lines.Add("- Warning: $warning") } + + return ($lines -join [Environment]::NewLine) +} diff --git a/actions/smoke-verdict/action.yml b/actions/smoke-verdict/action.yml new file mode 100644 index 0000000..02b551c --- /dev/null +++ b/actions/smoke-verdict/action.yml @@ -0,0 +1,48 @@ +name: 'Hermes Smoke Verdict' +description: 'Decides pass or fail for a Hermes smoke run from its log, result JSON and run record' + +inputs: + output_dir: + description: >- + Directory holding app-stdout.log, optionally app-stderr.log and result.json, and run.json + ({ "mode": "verdict"|"legacy", "exitCode": int|null, "timedOut": bool, "legacyAlive": bool }) + as written by Start-SmokeRun.ps1 + required: true + require_verdict: + description: 'Fail when the app predates smoke mode instead of accepting a liveness check' + required: false + default: 'false' + fail_on_failed: + description: 'Fail the step when the verdict is FAILED' + required: false + default: 'true' + platform: + description: 'Platform name for the job summary heading' + required: false + default: '' + +outputs: + result: + description: 'passed or failed' + value: ${{ steps.verdict.outputs.result }} + reason: + description: 'One-line explanation of the verdict' + value: ${{ steps.verdict.outputs.reason }} + +runs: + using: 'composite' + steps: + - name: Decide Verdict + id: verdict + shell: pwsh + env: + SMOKE_OUTPUT_DIR: ${{ inputs.output_dir }} + SMOKE_REQUIRE_VERDICT: ${{ inputs.require_verdict }} + SMOKE_FAIL_ON_FAILED: ${{ inputs.fail_on_failed }} + SMOKE_PLATFORM: ${{ inputs.platform }} + run: | + & (Join-Path $env:GITHUB_ACTION_PATH 'Invoke-SmokeVerdict.ps1') ` + -OutputDir $env:SMOKE_OUTPUT_DIR ` + -RequireVerdict $env:SMOKE_REQUIRE_VERDICT ` + -FailOnFailed $env:SMOKE_FAIL_ON_FAILED ` + -Platform $env:SMOKE_PLATFORM