Feat/smoke before release

.github/workflows/desktop-publish.yml

@@ -89,9 +89,17 @@ on:
         type: boolean
         default: true
       enable_smoke_tests:
-        description: 'Enable smoke tests after publishing'
+        description: 'Run smoke tests on the packaged builds before releasing; a failed smoke test stops the release'
         type: boolean
         default: true
+      smoke_timeout:
+        description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT; keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup'
+        type: number
+        default: 60
+      require_verdict:
+        description: 'Fail apps that predate Hermes smoke mode instead of accepting a liveness check'
+        type: boolean
+        default: false
       enable_blob_upload:
         description: 'Enable uploading to Azure Blob Storage'
         type: boolean
@@ -491,22 +499,6 @@ jobs:
       if: matrix.os == 'macos-latest' && inputs.enable_signing && always()
       run: security delete-keychain $RUNNER_TEMP/app-signing.keychain-db
 
-    # ====================
-    # UPLOAD TO BLOB STORAGE
-    # ====================
-    - name: Upload to Azure Blob Storage
-      if: inputs.enable_blob_upload
-      uses: mythetech/workflows/actions/blob-upload@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit
-      with:
-        app_name: ${{ inputs.app_name }}
-        version: ${{ needs.test.outputs.version }}
-        platform: ${{ matrix.platform }}
-        release_dir: releases/${{ matrix.platform }}
-        storage_account: ${{ inputs.storage_account }}
-        storage_container: ${{ inputs.storage_container }}
-        releases_container: ${{ inputs.releases_container }}
-        sas_token: ${{ secrets.BLOB_SAS_TOKEN }}
-
     - name: Upload Release Artifacts
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
@@ -531,7 +523,7 @@ jobs:
           - os: ubuntu-latest
             platform: Linux
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 3
+    timeout-minutes: 6
 
     steps:
     - name: Download Build Artifacts
@@ -550,7 +542,7 @@ jobs:
       uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         repository: mythetech/workflows
-        ref: f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit
+        ref: 586f3f549771d0d523007ee4bb8b7cf3fa9d39a9 # pinned, bump on action edit
         path: .workflows
 
     - name: Run Smoke Test
@@ -559,18 +551,68 @@ jobs:
         app_name: ${{ inputs.app_name }}
         platform: ${{ matrix.platform }}
         releases_dir: releases
+        timeout: ${{ inputs.smoke_timeout }}
+        require_verdict: ${{ inputs.require_verdict }}
+        output_dir: smoke-output
 
-    - name: Delete publish artifact (on success)
-      if: success()
-      uses: geekyeggo/delete-artifact@176a747ab7e287e3ff4787bf8a148716375ca118 # v6
+    - name: Upload Smoke Output
+      if: always()
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      with:
+        name: smoke-output-${{ matrix.platform }}
+        path: smoke-output
+        retention-days: 14
+        if-no-files-found: ignore
+
+  # ====================
+  # RELEASE
+  # ====================
+  # Uploads only after every platform's smoke test passed (or smoke tests are disabled), so a
+  # build that fails its smoke test never reaches users or the auto-updater.
+  release:
+    needs: [test, publish, smoke-test]
+    if: ${{ !cancelled() && inputs.enable_blob_upload && needs.publish.result == 'success' && (needs.smoke-test.result == 'success' || needs.smoke-test.result == 'skipped') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows-latest
+            platform: Windows
+          - os: macos-latest
+            platform: macOS
+          - os: ubuntu-latest
+            platform: Linux
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+
+    steps:
+    - name: Download Build Artifacts
+      uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
       with:
         name: ${{ inputs.app_name }}-${{ matrix.platform }}-${{ needs.test.outputs.version }}
-        failOnError: false
+        path: releases/${{ matrix.platform }}
 
-    - name: Upload Failure Artifacts
-      if: failure()
-      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+    # upload-artifact does not keep file modes and blob-upload zips the AppImage as it is on disk,
+    # so restore the executable bit vpk set, or the Linux download would not run.
+    - name: Restore AppImage executable bit
+      if: matrix.platform == 'Linux'
+      run: chmod +x releases/Linux/*.AppImage
+
+    - name: Upload to Azure Blob Storage
+      uses: mythetech/workflows/actions/blob-upload@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit
       with:
-        name: smoke-test-failure-${{ matrix.platform }}
-        path: screenshot-failure.png
-        retention-days: 14
+        app_name: ${{ inputs.app_name }}
+        version: ${{ needs.test.outputs.version }}
+        platform: ${{ matrix.platform }}
+        release_dir: releases/${{ matrix.platform }}
+        storage_account: ${{ inputs.storage_account }}
+        storage_container: ${{ inputs.storage_container }}
+        releases_container: ${{ inputs.releases_container }}
+        sas_token: ${{ secrets.BLOB_SAS_TOKEN }}
+
+    - name: Delete publish artifact
+      uses: geekyeggo/delete-artifact@176a747ab7e287e3ff4787bf8a148716375ca118 # v6
+      with:
+        name: ${{ inputs.app_name }}-${{ matrix.platform }}-${{ needs.test.outputs.version }}
+        failOnError: false

.github/workflows/desktop-smoke-test.yml

@@ -27,6 +27,8 @@
 #       icon_linux: "Horizon/wwwroot/logo.png"
 #       icon_macos: "Horizon/wwwroot/logo.icns"
 #       platforms: ${{ inputs.platforms }}
+#       smoke_timeout: 60        # optional, seconds
+#       require_verdict: false   # optional, set true once the app is on Hermes smoke mode
 #
 # To pin the SDK from the repository's own global.json instead of dotnet_version:
 #       global_json_file: "global.json"
@@ -88,6 +90,14 @@ on:
         description: 'Enable code coverage collection for the unit tests'
         type: boolean
         default: false
+      smoke_timeout:
+        description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT; keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup'
+        type: number
+        default: 60
+      require_verdict:
+        description: 'Fail apps that predate Hermes smoke mode instead of accepting a liveness check'
+        type: boolean
+        default: false
 
 # Restore credentials for the Mythetech GitHub Packages feed. A consuming repository commits a
 # nuget.config whose packageSourceCredentials read these two variables, so no step here has to
@@ -198,17 +208,21 @@ jobs:
       run: vpk pack -u "${{ inputs.app_name }}" -v 0.0.${{ github.run_number }} -o "releases/${{ matrix.platform }}" -p "publish/${{ matrix.rid }}" --icon "${{ steps.icon.outputs.path }}"
 
     - name: Run Smoke Test
-      timeout-minutes: 3
-      uses: mythetech/workflows/actions/smoke-test@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit
+      timeout-minutes: 6
+      uses: mythetech/workflows/actions/smoke-test@586f3f549771d0d523007ee4bb8b7cf3fa9d39a9 # pinned, bump on action edit
       with:
         app_name: ${{ inputs.app_name }}
         platform: ${{ matrix.platform }}
         releases_dir: releases/${{ matrix.platform }}
+        timeout: ${{ inputs.smoke_timeout }}
+        require_verdict: ${{ inputs.require_verdict }}
+        output_dir: smoke-output
 
-    - name: Upload Failure Artifacts
-      if: failure()
+    - name: Upload Smoke Output
+      if: always()
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
-        name: smoke-test-failure-${{ matrix.platform }}
-        path: screenshot-failure.png
+        name: smoke-output-${{ matrix.platform }}
+        path: smoke-output
         retention-days: 14
+        if-no-files-found: ignore

README.md

@@ -142,9 +142,20 @@ jobs:
 | `storage_container` | **Yes** | - | Container for releases |
 | `releases_container` | No | `releases` | Container for Velopack auto-update |
 | `enable_signing` | No | `true` | Enable code signing |
-| `enable_smoke_tests` | No | `true` | Enable smoke tests |
+| `enable_smoke_tests` | No | `true` | Smoke test the packaged builds before releasing; a failed smoke test stops the release |
+| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`); keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup |
+| `require_verdict` | No | `false` | Fail apps that predate Hermes smoke mode instead of accepting a liveness check |
 | `enable_blob_upload` | No | `true` | Enable Azure uploads |
 
+Jobs run in the order `test`, `publish` (build, sign, pack), `smoke-test`, `release` (Azure Blob
+upload). `release` waits for every platform's smoke test, so a build that fails smoke testing is
+never uploaded. With `enable_smoke_tests: false` the release runs straight after `publish`.
+
+Because `release` waits for every platform, one platform's failed publish or smoke test holds back
+the upload for all platforms. Use "Re-run failed jobs" to recover while the publish artifacts still
+exist (they are kept for 3 days); after that, re-run all jobs. With `enable_blob_upload: false`,
+`release` is skipped and the publish artifacts expire on their own.
+
 #### Required Secrets
 
 Configure these at the **organization level** for sharing across repos:
@@ -221,6 +232,8 @@ requests. Tests and smoke tests run in parallel.
 | `test_command` | No | - | Override the entire test command |
 | `test_runner` | No | `auto` | `auto`, `vstest` or `mtp` |
 | `enable_coverage` | No | `false` | Collect code coverage for the unit tests |
+| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`); keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup |
+| `require_verdict` | No | `false` | Fail apps that predate Hermes smoke mode instead of accepting a liveness check |
 
 The packages are unsigned, so the macOS signing-specific bundle restructuring from
 `desktop-publish.yml` is not exercised here.
@@ -246,16 +259,46 @@ Signs and notarizes a macOS application bundle.
 
 ### `actions/smoke-test`
 
-Verifies that a desktop application can launch and display a window.
+Launches a packaged app with `HERMES_SMOKE_TEST=1` and judges the run by the verdict the app
+prints (`HERMES_SMOKE_RESULT`) or writes (`HERMES_SMOKE_TEST_RESULT` JSON). Apps built on a Hermes
+version without smoke mode never print `HERMES_SMOKE_START`; for those the action falls back to a
+liveness check with a warning, unless `require_verdict` is `true`.
 
 ```yaml
 - uses: mythetech/workflows/actions/smoke-test@main
   with:
     app_name: "MyApp"
     platform: "Windows"  # or "macOS" or "Linux"
     releases_dir: "releases"
+    timeout: "60"             # optional
+    require_verdict: "false"  # optional
+    output_dir: "smoke-output"  # optional
+```
+
+`output_dir` receives `app-stdout.log`, `app-stderr.log`, `run.json`, `result.json` (apps on a
+smoke-aware Hermes only), and a screenshot on failure. Upload it with `if: always()`.
+
+### `actions/smoke-verdict`
+
+The verdict step on its own, for pipelines that launch the app themselves. `output_dir` must hold
+`app-stdout.log` and `run.json`, and optionally `app-stderr.log` and `result.json`. `run.json` is
+required; without it the action reports a launcher failure rather than guessing at the app's own
+verdict. `actions/smoke-test` writes `run.json` via `Start-SmokeRun.ps1`, in the shape
+`{ "mode": "verdict" | "legacy", "exitCode": <int or null>, "timedOut": <bool>, "legacyAlive": <bool> }`.
+Pipelines that do not want to write `run.json` can dot-source `actions/smoke-verdict/SmokeVerdict.ps1`
+and call `Get-SmokeVerdict` directly, which is what Hermes CI does.
+
+```yaml
+- uses: mythetech/workflows/actions/smoke-verdict@main
+  with:
+    output_dir: "smoke-output"
+    fail_on_failed: "true"
+    require_verdict: "false"  # optional
+    platform: "Windows"  # optional, job summary heading
 ```
 
+Outputs: `result` (`passed` or `failed`) and `reason`.
+
 ### `actions/blob-upload`
 
 Uploads release artifacts to Azure Blob Storage.