diff --git a/.github/workflows/desktop-publish.yml b/.github/workflows/desktop-publish.yml index 54d3ee7..e31bc87 100644 --- a/.github/workflows/desktop-publish.yml +++ b/.github/workflows/desktop-publish.yml @@ -89,9 +89,17 @@ on: type: boolean default: true enable_smoke_tests: - description: 'Enable smoke tests after publishing' + description: 'Run smoke tests on the packaged builds before releasing; a failed smoke test stops the release' type: boolean default: true + smoke_timeout: + description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT; keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup' + type: number + default: 60 + require_verdict: + description: 'Fail apps that predate Hermes smoke mode instead of accepting a liveness check' + type: boolean + default: false enable_blob_upload: description: 'Enable uploading to Azure Blob Storage' type: boolean @@ -491,22 +499,6 @@ jobs: if: matrix.os == 'macos-latest' && inputs.enable_signing && always() run: security delete-keychain $RUNNER_TEMP/app-signing.keychain-db - # ==================== - # UPLOAD TO BLOB STORAGE - # ==================== - - name: Upload to Azure Blob Storage - if: inputs.enable_blob_upload - uses: mythetech/workflows/actions/blob-upload@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit - with: - app_name: ${{ inputs.app_name }} - version: ${{ needs.test.outputs.version }} - platform: ${{ matrix.platform }} - release_dir: releases/${{ matrix.platform }} - storage_account: ${{ inputs.storage_account }} - storage_container: ${{ inputs.storage_container }} - releases_container: ${{ inputs.releases_container }} - sas_token: ${{ secrets.BLOB_SAS_TOKEN }} - - name: Upload Release Artifacts uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: @@ -531,7 +523,7 @@ jobs: - os: ubuntu-latest platform: Linux runs-on: ${{ matrix.os }} - timeout-minutes: 3 + timeout-minutes: 6 steps: - name: Download Build Artifacts @@ -550,7 +542,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: mythetech/workflows - ref: f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit + ref: 586f3f549771d0d523007ee4bb8b7cf3fa9d39a9 # pinned, bump on action edit path: .workflows - name: Run Smoke Test @@ -559,18 +551,68 @@ jobs: app_name: ${{ inputs.app_name }} platform: ${{ matrix.platform }} releases_dir: releases + timeout: ${{ inputs.smoke_timeout }} + require_verdict: ${{ inputs.require_verdict }} + output_dir: smoke-output - - name: Delete publish artifact (on success) - if: success() - uses: geekyeggo/delete-artifact@176a747ab7e287e3ff4787bf8a148716375ca118 # v6 + - name: Upload Smoke Output + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: smoke-output-${{ matrix.platform }} + path: smoke-output + retention-days: 14 + if-no-files-found: ignore + + # ==================== + # RELEASE + # ==================== + # Uploads only after every platform's smoke test passed (or smoke tests are disabled), so a + # build that fails its smoke test never reaches users or the auto-updater. + release: + needs: [test, publish, smoke-test] + if: ${{ !cancelled() && inputs.enable_blob_upload && needs.publish.result == 'success' && (needs.smoke-test.result == 'success' || needs.smoke-test.result == 'skipped') }} + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + platform: Windows + - os: macos-latest + platform: macOS + - os: ubuntu-latest + platform: Linux + runs-on: ${{ matrix.os }} + permissions: + contents: read + + steps: + - name: Download Build Artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: ${{ inputs.app_name }}-${{ matrix.platform }}-${{ needs.test.outputs.version }} - failOnError: false + path: releases/${{ matrix.platform }} - - name: Upload Failure Artifacts - if: failure() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + # upload-artifact does not keep file modes and blob-upload zips the AppImage as it is on disk, + # so restore the executable bit vpk set, or the Linux download would not run. + - name: Restore AppImage executable bit + if: matrix.platform == 'Linux' + run: chmod +x releases/Linux/*.AppImage + + - name: Upload to Azure Blob Storage + uses: mythetech/workflows/actions/blob-upload@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit with: - name: smoke-test-failure-${{ matrix.platform }} - path: screenshot-failure.png - retention-days: 14 + app_name: ${{ inputs.app_name }} + version: ${{ needs.test.outputs.version }} + platform: ${{ matrix.platform }} + release_dir: releases/${{ matrix.platform }} + storage_account: ${{ inputs.storage_account }} + storage_container: ${{ inputs.storage_container }} + releases_container: ${{ inputs.releases_container }} + sas_token: ${{ secrets.BLOB_SAS_TOKEN }} + + - name: Delete publish artifact + uses: geekyeggo/delete-artifact@176a747ab7e287e3ff4787bf8a148716375ca118 # v6 + with: + name: ${{ inputs.app_name }}-${{ matrix.platform }}-${{ needs.test.outputs.version }} + failOnError: false diff --git a/.github/workflows/desktop-smoke-test.yml b/.github/workflows/desktop-smoke-test.yml index 2d7d4ee..5f94299 100644 --- a/.github/workflows/desktop-smoke-test.yml +++ b/.github/workflows/desktop-smoke-test.yml @@ -27,6 +27,8 @@ # icon_linux: "Horizon/wwwroot/logo.png" # icon_macos: "Horizon/wwwroot/logo.icns" # platforms: ${{ inputs.platforms }} +# smoke_timeout: 60 # optional, seconds +# require_verdict: false # optional, set true once the app is on Hermes smoke mode # # To pin the SDK from the repository's own global.json instead of dotnet_version: # global_json_file: "global.json" @@ -88,6 +90,14 @@ on: description: 'Enable code coverage collection for the unit tests' type: boolean default: false + smoke_timeout: + description: 'Smoke run budget in seconds, passed to the app as HERMES_SMOKE_TEST_TIMEOUT; keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup' + type: number + default: 60 + require_verdict: + description: 'Fail apps that predate Hermes smoke mode instead of accepting a liveness check' + type: boolean + default: false # Restore credentials for the Mythetech GitHub Packages feed. A consuming repository commits a # nuget.config whose packageSourceCredentials read these two variables, so no step here has to @@ -198,17 +208,21 @@ jobs: run: vpk pack -u "${{ inputs.app_name }}" -v 0.0.${{ github.run_number }} -o "releases/${{ matrix.platform }}" -p "publish/${{ matrix.rid }}" --icon "${{ steps.icon.outputs.path }}" - name: Run Smoke Test - timeout-minutes: 3 - uses: mythetech/workflows/actions/smoke-test@f61f736e5f147fc8aec6cc245365adbab4fa280b # pinned, bump on action edit + timeout-minutes: 6 + uses: mythetech/workflows/actions/smoke-test@586f3f549771d0d523007ee4bb8b7cf3fa9d39a9 # pinned, bump on action edit with: app_name: ${{ inputs.app_name }} platform: ${{ matrix.platform }} releases_dir: releases/${{ matrix.platform }} + timeout: ${{ inputs.smoke_timeout }} + require_verdict: ${{ inputs.require_verdict }} + output_dir: smoke-output - - name: Upload Failure Artifacts - if: failure() + - name: Upload Smoke Output + if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: smoke-test-failure-${{ matrix.platform }} - path: screenshot-failure.png + name: smoke-output-${{ matrix.platform }} + path: smoke-output retention-days: 14 + if-no-files-found: ignore diff --git a/README.md b/README.md index d9ea432..8a18e68 100644 --- a/README.md +++ b/README.md @@ -142,9 +142,20 @@ jobs: | `storage_container` | **Yes** | - | Container for releases | | `releases_container` | No | `releases` | Container for Velopack auto-update | | `enable_signing` | No | `true` | Enable code signing | -| `enable_smoke_tests` | No | `true` | Enable smoke tests | +| `enable_smoke_tests` | No | `true` | Smoke test the packaged builds before releasing; a failed smoke test stops the release | +| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`); keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup | +| `require_verdict` | No | `false` | Fail apps that predate Hermes smoke mode instead of accepting a liveness check | | `enable_blob_upload` | No | `true` | Enable Azure uploads | +Jobs run in the order `test`, `publish` (build, sign, pack), `smoke-test`, `release` (Azure Blob +upload). `release` waits for every platform's smoke test, so a build that fails smoke testing is +never uploaded. With `enable_smoke_tests: false` the release runs straight after `publish`. + +Because `release` waits for every platform, one platform's failed publish or smoke test holds back +the upload for all platforms. Use "Re-run failed jobs" to recover while the publish artifacts still +exist (they are kept for 3 days); after that, re-run all jobs. With `enable_blob_upload: false`, +`release` is skipped and the publish artifacts expire on their own. + #### Required Secrets Configure these at the **organization level** for sharing across repos: @@ -221,6 +232,8 @@ requests. Tests and smoke tests run in parallel. | `test_command` | No | - | Override the entire test command | | `test_runner` | No | `auto` | `auto`, `vstest` or `mtp` | | `enable_coverage` | No | `false` | Collect code coverage for the unit tests | +| `smoke_timeout` | No | `60` | Smoke run budget in seconds (`HERMES_SMOKE_TEST_TIMEOUT`); keep it at 180 or below, because the smoke job has a fixed 6-minute timeout that also covers setup | +| `require_verdict` | No | `false` | Fail apps that predate Hermes smoke mode instead of accepting a liveness check | The packages are unsigned, so the macOS signing-specific bundle restructuring from `desktop-publish.yml` is not exercised here. @@ -246,7 +259,10 @@ Signs and notarizes a macOS application bundle. ### `actions/smoke-test` -Verifies that a desktop application can launch and display a window. +Launches a packaged app with `HERMES_SMOKE_TEST=1` and judges the run by the verdict the app +prints (`HERMES_SMOKE_RESULT`) or writes (`HERMES_SMOKE_TEST_RESULT` JSON). Apps built on a Hermes +version without smoke mode never print `HERMES_SMOKE_START`; for those the action falls back to a +liveness check with a warning, unless `require_verdict` is `true`. ```yaml - uses: mythetech/workflows/actions/smoke-test@main @@ -254,8 +270,35 @@ Verifies that a desktop application can launch and display a window. app_name: "MyApp" platform: "Windows" # or "macOS" or "Linux" releases_dir: "releases" + timeout: "60" # optional + require_verdict: "false" # optional + output_dir: "smoke-output" # optional +``` + +`output_dir` receives `app-stdout.log`, `app-stderr.log`, `run.json`, `result.json` (apps on a +smoke-aware Hermes only), and a screenshot on failure. Upload it with `if: always()`. + +### `actions/smoke-verdict` + +The verdict step on its own, for pipelines that launch the app themselves. `output_dir` must hold +`app-stdout.log` and `run.json`, and optionally `app-stderr.log` and `result.json`. `run.json` is +required; without it the action reports a launcher failure rather than guessing at the app's own +verdict. `actions/smoke-test` writes `run.json` via `Start-SmokeRun.ps1`, in the shape +`{ "mode": "verdict" | "legacy", "exitCode": , "timedOut": , "legacyAlive": }`. +Pipelines that do not want to write `run.json` can dot-source `actions/smoke-verdict/SmokeVerdict.ps1` +and call `Get-SmokeVerdict` directly, which is what Hermes CI does. + +```yaml +- uses: mythetech/workflows/actions/smoke-verdict@main + with: + output_dir: "smoke-output" + fail_on_failed: "true" + require_verdict: "false" # optional + platform: "Windows" # optional, job summary heading ``` +Outputs: `result` (`passed` or `failed`) and `reason`. + ### `actions/blob-upload` Uploads release artifacts to Azure Blob Storage.