From 43bc828827590019366e762d155913a5f4ad0adb Mon Sep 17 00:00:00 2001 From: Tom Brewer Date: Sat, 26 Sep 2026 20:31:39 -0600 Subject: [PATCH 1/5] feat: read the smoke verdict in desktop-smoke-test --- .github/workflows/desktop-smoke-test.yml | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/desktop-smoke-test.yml b/.github/workflows/desktop-smoke-test.yml index 2d7d4ee..cdce498 100644 --- a/.github/workflows/desktop-smoke-test.yml +++ b/.github/workflows/desktop-smoke-test.yml @@ -27,6 +27,8 @@ # icon_linux: "Horizon/wwwroot/logo.png" # icon_macos: "Horizon/wwwroot/logo.icns" # platforms: ${{ inputs.platforms }} +# smoke_timeout: 60 # optional, seconds +# require_verdict: false # optional, set true once the app is on Hermes smoke mode # # To pin the SDK from the repository's own global.json instead of dotnet_version: # global_json_file: "global.json" @@ -88,6 +90,14 @@ on: description: 'Enable code coverage collection for the unit tests' type: boolean default: false + smoke_timeout: + description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT' + type: number + default: 60 + require_verdict: + description: 'Fail apps that predate Hermes smoke mode instead of accepting a liveness check' + type: boolean + default: false # Restore credentials for the Mythetech GitHub Packages feed. A consuming repository commits a # nuget.config whose packageSourceCredentials read these two variables, so no step here has to @@ -198,17 +208,21 @@ jobs: run: vpk pack -u "${{ inputs.app_name }}" -v 0.0.${{ github.run_number }} -o "releases/${{ matrix.platform }}" -p "publish/${{ matrix.rid }}" --icon "${{ steps.icon.outputs.path }}" - name: Run Smoke Test - timeout-minutes: 3 - uses: mythetech/workflows/actions/smoke-test@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit + timeout-minutes: 6 + uses: mythetech/workflows/actions/smoke-test@586f3f549771d0d523007ee4bb8b7cf3fa9d39a9 # pinned, bump on action edit with: app_name: ${{ inputs.app_name }} platform: ${{ matrix.platform }} releases_dir: releases/${{ matrix.platform }} + timeout: ${{ inputs.smoke_timeout }} + require_verdict: ${{ inputs.require_verdict }} + output_dir: smoke-output - - name: Upload Failure Artifacts - if: failure() + - name: Upload Smoke Output + if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: smoke-test-failure-${{ matrix.platform }} - path: screenshot-failure.png + name: smoke-output-${{ matrix.platform }} + path: smoke-output retention-days: 14 + if-no-files-found: ignore From e41cac944fdb442e328603ba4001e89b6d139b90 Mon Sep 17 00:00:00 2001 From: Tom Brewer Date: Sat, 26 Sep 2026 20:33:02 -0600 Subject: [PATCH 2/5] feat: release only after desktop smoke tests pass --- .github/workflows/desktop-publish.yml | 94 ++++++++++++++++++--------- 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/.github/workflows/desktop-publish.yml b/.github/workflows/desktop-publish.yml index 54d3ee7..f1aa943 100644 --- a/.github/workflows/desktop-publish.yml +++ b/.github/workflows/desktop-publish.yml @@ -89,9 +89,17 @@ on: type: boolean default: true enable_smoke_tests: - description: 'Enable smoke tests after publishing' + description: 'Run smoke tests on the packaged builds before releasing; a failed smoke test stops the release' type: boolean default: true + smoke_timeout: + description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT' + type: number + default: 60 + require_verdict: + description: 'Fail apps that predate Hermes smoke mode instead of accepting a liveness check' + type: boolean + default: false enable_blob_upload: description: 'Enable uploading to Azure Blob Storage' type: boolean @@ -491,22 +499,6 @@ jobs: if: matrix.os == 'macos-latest' && inputs.enable_signing && always() run: security delete-keychain $RUNNER_TEMP/app-signing.keychain-db - # ==================== - # UPLOAD TO BLOB STORAGE - # ==================== - - name: Upload to Azure Blob Storage - if: inputs.enable_blob_upload - uses: mythetech/workflows/actions/blob-upload@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit - with: - app_name: ${{ inputs.app_name }} - version: ${{ needs.test.outputs.version }} - platform: ${{ matrix.platform }} - release_dir: releases/${{ matrix.platform }} - storage_account: ${{ inputs.storage_account }} - storage_container: ${{ inputs.storage_container }} - releases_container: ${{ inputs.releases_container }} - sas_token: ${{ secrets.BLOB_SAS_TOKEN }} - - name: Upload Release Artifacts uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: @@ -531,7 +523,7 @@ jobs: - os: ubuntu-latest platform: Linux runs-on: ${{ matrix.os }} - timeout-minutes: 3 + timeout-minutes: 6 steps: - name: Download Build Artifacts @@ -550,7 +542,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: mythetech/workflows - ref: f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit + ref: 586f3f549771d0d523007ee4bb8b7cf3fa9d39a9 # pinned, bump on action edit path: .workflows - name: Run Smoke Test @@ -559,18 +551,62 @@ jobs: app_name: ${{ inputs.app_name }} platform: ${{ matrix.platform }} releases_dir: releases + timeout: ${{ inputs.smoke_timeout }} + require_verdict: ${{ inputs.require_verdict }} + output_dir: smoke-output - - name: Delete publish artifact (on success) - if: success() - uses: geekyeggo/delete-artifact@176a747ab7e287e3ff4787bf8a148716375ca118 # v6 + - name: Upload Smoke Output + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: smoke-output-${{ matrix.platform }} + path: smoke-output + retention-days: 14 + if-no-files-found: ignore + + # ==================== + # RELEASE + # ==================== + # Uploads only after every platform's smoke test passed (or smoke tests are disabled), so a + # build that fails its smoke test never reaches users or the auto-updater. + release: + needs: [test, publish, smoke-test] + if: ${{ !cancelled() && inputs.enable_blob_upload && needs.publish.result == 'success' && (needs.smoke-test.result == 'success' || needs.smoke-test.result == 'skipped') }} + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + platform: Windows + - os: macos-latest + platform: macOS + - os: ubuntu-latest + platform: Linux + runs-on: ${{ matrix.os }} + permissions: + contents: read + + steps: + - name: Download Build Artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: ${{ inputs.app_name }}-${{ matrix.platform }}-${{ needs.test.outputs.version }} - failOnError: false + path: releases/${{ matrix.platform }} - - name: Upload Failure Artifacts - if: failure() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + - name: Upload to Azure Blob Storage + uses: mythetech/workflows/actions/blob-upload@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit with: - name: smoke-test-failure-${{ matrix.platform }} - path: screenshot-failure.png - retention-days: 14 + app_name: ${{ inputs.app_name }} + version: ${{ needs.test.outputs.version }} + platform: ${{ matrix.platform }} + release_dir: releases/${{ matrix.platform }} + storage_account: ${{ inputs.storage_account }} + storage_container: ${{ inputs.storage_container }} + releases_container: ${{ inputs.releases_container }} + sas_token: ${{ secrets.BLOB_SAS_TOKEN }} + + - name: Delete publish artifact + uses: geekyeggo/delete-artifact@176a747ab7e287e3ff4787bf8a148716375ca118 # v6 + with: + name: ${{ inputs.app_name }}-${{ matrix.platform }}-${{ needs.test.outputs.version }} + failOnError: false From ee56b6e615f3a129b81cf8b47029073e577c1fa3 Mon Sep 17 00:00:00 2001 From: Tom Brewer Date: Sat, 26 Sep 2026 20:33:43 -0600 Subject: [PATCH 3/5] docs: smoke verdict, smoke-verdict action, and release ordering --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9ea432..c214cbb 100644 --- a/README.md +++ b/README.md @@ -142,9 +142,15 @@ jobs: | `storage_container` | **Yes** | - | Container for releases | | `releases_container` | No | `releases` | Container for Velopack auto-update | | `enable_signing` | No | `true` | Enable code signing | -| `enable_smoke_tests` | No | `true` | Enable smoke tests | +| `enable_smoke_tests` | No | `true` | Smoke test the packaged builds before releasing; a failed smoke test stops the release | +| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`) | +| `require_verdict` | No | `false` | Fail apps that predate Hermes smoke mode instead of accepting a liveness check | | `enable_blob_upload` | No | `true` | Enable Azure uploads | +Jobs run in the order `test`, `publish` (build, sign, pack), `smoke-test`, `release` (Azure Blob +upload). `release` waits for every platform's smoke test, so a build that fails smoke testing is +never uploaded. With `enable_smoke_tests: false` the release runs straight after `publish`. + #### Required Secrets Configure these at the **organization level** for sharing across repos: @@ -221,6 +227,8 @@ requests. Tests and smoke tests run in parallel. | `test_command` | No | - | Override the entire test command | | `test_runner` | No | `auto` | `auto`, `vstest` or `mtp` | | `enable_coverage` | No | `false` | Collect code coverage for the unit tests | +| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`) | +| `require_verdict` | No | `false` | Fail apps that predate Hermes smoke mode instead of accepting a liveness check | The packages are unsigned, so the macOS signing-specific bundle restructuring from `desktop-publish.yml` is not exercised here. @@ -246,7 +254,10 @@ Signs and notarizes a macOS application bundle. ### `actions/smoke-test` -Verifies that a desktop application can launch and display a window. +Launches a packaged app with `HERMES_SMOKE_TEST=1` and judges the run by the verdict the app +prints (`HERMES_SMOKE_RESULT`) or writes (`HERMES_SMOKE_TEST_RESULT` JSON). Apps built on a Hermes +version without smoke mode never print `HERMES_SMOKE_START`; for those the action falls back to a +liveness check with a warning, unless `require_verdict` is `true`. ```yaml - uses: mythetech/workflows/actions/smoke-test@main @@ -254,8 +265,35 @@ Verifies that a desktop application can launch and display a window. app_name: "MyApp" platform: "Windows" # or "macOS" or "Linux" releases_dir: "releases" + timeout: "60" # optional + require_verdict: "false" # optional + output_dir: "smoke-output" ``` +`output_dir` receives `app-stdout.log`, `app-stderr.log`, `result.json`, `run.json`, and a +screenshot on failure. Upload it with `if: always()`. + +### `actions/smoke-verdict` + +The verdict step on its own, for pipelines that launch the app themselves. `output_dir` must hold +`app-stdout.log` and `run.json`, and optionally `app-stderr.log` and `result.json`. `run.json` is +required; without it the action reports a launcher failure rather than guessing at the app's own +verdict. `actions/smoke-test` writes `run.json` via `Start-SmokeRun.ps1`, in the shape +`{ "mode": "verdict" | "legacy", "exitCode": , "timedOut": , "legacyAlive": }`. +Pipelines that do not want to write `run.json` can dot-source `actions/smoke-verdict/SmokeVerdict.ps1` +and call `Get-SmokeVerdict` directly, which is what Hermes CI does. + +```yaml +- uses: mythetech/workflows/actions/smoke-verdict@main + with: + output_dir: "smoke-output" + fail_on_failed: "true" + require_verdict: "false" # optional + platform: "Windows" # optional, job summary heading +``` + +Outputs: `result` (`passed` or `failed`) and `reason`. + ### `actions/blob-upload` Uploads release artifacts to Azure Blob Storage. From ea885abac1d7a061584e99b40a4ac260ca0d25a3 Mon Sep 17 00:00:00 2001 From: Tom Brewer Date: Sat, 26 Sep 2026 21:00:40 -0600 Subject: [PATCH 4/5] fix: restore the AppImage executable bit before the blob upload upload-artifact does not preserve file modes, so the Linux AppImage loses the executable bit vpk set during the round trip through the publish artifact. Restore it in the release job before blob-upload zips it, or the downloaded AppImage would not run. --- .github/workflows/desktop-publish.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/desktop-publish.yml b/.github/workflows/desktop-publish.yml index f1aa943..e31bc87 100644 --- a/.github/workflows/desktop-publish.yml +++ b/.github/workflows/desktop-publish.yml @@ -93,7 +93,7 @@ on: type: boolean default: true smoke_timeout: - description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT' + description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT; keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup' type: number default: 60 require_verdict: @@ -593,6 +593,12 @@ jobs: name: ${{ inputs.app_name }}-${{ matrix.platform }}-${{ needs.test.outputs.version }} path: releases/${{ matrix.platform }} + # upload-artifact does not keep file modes and blob-upload zips the AppImage as it is on disk, + # so restore the executable bit vpk set, or the Linux download would not run. + - name: Restore AppImage executable bit + if: matrix.platform == 'Linux' + run: chmod +x releases/Linux/*.AppImage + - name: Upload to Azure Blob Storage uses: mythetech/workflows/actions/blob-upload@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit with: From 1692798f1dd6b39ecb93373b7baaa01246a2bd09 Mon Sep 17 00:00:00 2001 From: Tom Brewer Date: Sat, 26 Sep 2026 21:00:44 -0600 Subject: [PATCH 5/5] docs: clarify smoke_timeout limits, release re-run recovery, and smoke-test output_dir Note that smoke_timeout should stay at 180 or below since the smoke job has a fixed 6-minute timeout that also covers setup, explain how to recover when release holds back all platforms on one failure, mark output_dir as optional in the smoke-test example, and note that result.json is only written by apps on a smoke-aware Hermes. --- .github/workflows/desktop-smoke-test.yml | 2 +- README.md | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/desktop-smoke-test.yml b/.github/workflows/desktop-smoke-test.yml index cdce498..5f94299 100644 --- a/.github/workflows/desktop-smoke-test.yml +++ b/.github/workflows/desktop-smoke-test.yml @@ -91,7 +91,7 @@ on: type: boolean default: false smoke_timeout: - description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT' + description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT; keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup' type: number default: 60 require_verdict: diff --git a/README.md b/README.md index c214cbb..8a18e68 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ jobs: | `releases_container` | No | `releases` | Container for Velopack auto-update | | `enable_signing` | No | `true` | Enable code signing | | `enable_smoke_tests` | No | `true` | Smoke test the packaged builds before releasing; a failed smoke test stops the release | -| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`) | +| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`); keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup | | `require_verdict` | No | `false` | Fail apps that predate Hermes smoke mode instead of accepting a liveness check | | `enable_blob_upload` | No | `true` | Enable Azure uploads | @@ -151,6 +151,11 @@ Jobs run in the order `test`, `publish` (build, sign, pack), `smoke-test`, `rele upload). `release` waits for every platform's smoke test, so a build that fails smoke testing is never uploaded. With `enable_smoke_tests: false` the release runs straight after `publish`. +Because `release` waits for every platform, one platform's failed publish or smoke test holds back +the upload for all platforms. Use "Re-run failed jobs" to recover while the publish artifacts still +exist (they are kept for 3 days); after that, re-run all jobs. With `enable_blob_upload: false`, +`release` is skipped and the publish artifacts expire on their own. + #### Required Secrets Configure these at the **organization level** for sharing across repos: @@ -227,7 +232,7 @@ requests. Tests and smoke tests run in parallel. | `test_command` | No | - | Override the entire test command | | `test_runner` | No | `auto` | `auto`, `vstest` or `mtp` | | `enable_coverage` | No | `false` | Collect code coverage for the unit tests | -| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`) | +| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`); keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup | | `require_verdict` | No | `false` | Fail apps that predate Hermes smoke mode instead of accepting a liveness check | The packages are unsigned, so the macOS signing-specific bundle restructuring from @@ -267,11 +272,11 @@ liveness check with a warning, unless `require_verdict` is `true`. releases_dir: "releases" timeout: "60" # optional require_verdict: "false" # optional - output_dir: "smoke-output" + output_dir: "smoke-output" # optional ``` -`output_dir` receives `app-stdout.log`, `app-stderr.log`, `result.json`, `run.json`, and a -screenshot on failure. Upload it with `if: always()`. +`output_dir` receives `app-stdout.log`, `app-stderr.log`, `run.json`, `result.json` (apps on a +smoke-aware Hermes only), and a screenshot on failure. Upload it with `if: always()`. ### `actions/smoke-verdict`