diff --git a/.github/workflows/e2e-android-rntester.yml b/.github/workflows/e2e-android-rntester.yml index f3ff2fc0dc9..62fa4c4759f 100644 --- a/.github/workflows/e2e-android-rntester.yml +++ b/.github/workflows/e2e-android-rntester.yml @@ -14,14 +14,17 @@ on: default: 0 outputs: status: - description: 'The result of the E2E tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the E2E tests failed, empty otherwise" + value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }} jobs: test: runs-on: 4-core-ubuntu + # Matrix legs write differently named outputs so they are combined + # instead of the last leg to finish overwriting the others. outputs: - status: ${{ steps.report-status.outputs.status }} + debug_status: ${{ steps.report-status.outputs.debug_status }} + release_status: ${{ steps.report-status.outputs.release_status }} strategy: fail-fast: false matrix: @@ -59,11 +62,11 @@ jobs: echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" - name: Run E2E Tests id: run-tests - if: steps.test-state.outputs.should-run == 'true' + if: ${{ steps.test-state.outputs.should-run == 'true' && !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/maestro-android timeout-minutes: 90 - with: + with: &test-inputs app-path: ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.flavor }}/app-arm64-v8a-${{ matrix.flavor }}.apk app-id: com.facebook.react.uiapp maestro-flow: ./packages/rn-tester/.maestro @@ -75,6 +78,14 @@ jobs: # Running the debug ARM64 APK through native translation can delay # React commits long enough for Maestro to miss taps and assertions. exclude-tags: ${{ matrix.flavor == 'debug' && 'android-release-only' || '' }} + - name: Run E2E Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ steps.test-state.outputs.should-run == 'true' && inputs.fail-on-error }} + uses: ./.github/actions/maestro-android + timeout-minutes: 90 + with: *test-inputs - name: Store per-flow test state if: always() uses: ./.github/actions/upload-artifact @@ -85,23 +96,15 @@ jobs: path: /tmp/maestro-android-state/results.json - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: test - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} + env: + FLAVOR: ${{ matrix.flavor }} run: | - if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 + flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]") + case "$flavor" in + debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output." + exit 1 + ;; + esac diff --git a/.github/workflows/e2e-android-templateapp.yml b/.github/workflows/e2e-android-templateapp.yml index eb9518af734..5fdc3b517de 100644 --- a/.github/workflows/e2e-android-templateapp.yml +++ b/.github/workflows/e2e-android-templateapp.yml @@ -14,14 +14,17 @@ on: default: 0 outputs: status: - description: 'The result of the E2E tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the E2E tests failed, empty otherwise" + value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }} jobs: test: runs-on: 4-core-ubuntu + # Matrix legs write differently named outputs so they are combined + # instead of the last leg to finish overwriting the others. outputs: - status: ${{ steps.report-status.outputs.status }} + debug_status: ${{ steps.report-status.outputs.debug_status }} + release_status: ${{ steps.report-status.outputs.release_status }} strategy: fail-fast: false matrix: @@ -81,11 +84,11 @@ jobs: echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" - name: Run E2E Tests id: run-tests - if: steps.test-state.outputs.should-run == 'true' + if: ${{ steps.test-state.outputs.should-run == 'true' && !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/maestro-android timeout-minutes: 90 - with: + with: &test-inputs app-path: /tmp/android-templateapp/app-${{ matrix.flavor }}.apk app-id: com.rntestproject maestro-flow: ./scripts/e2e/.maestro/ @@ -94,6 +97,14 @@ jobs: emulator-api-level: '35' emulator-target: google_apis emulator-arch: x86_64 + - name: Run E2E Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ steps.test-state.outputs.should-run == 'true' && inputs.fail-on-error }} + uses: ./.github/actions/maestro-android + timeout-minutes: 90 + with: *test-inputs - name: Store per-flow test state if: always() uses: ./.github/actions/upload-artifact @@ -104,23 +115,15 @@ jobs: path: /tmp/maestro-android-state/results.json - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: test - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} + env: + FLAVOR: ${{ matrix.flavor }} run: | - if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 + flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]") + case "$flavor" in + debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output." + exit 1 + ;; + esac diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index c841c409b9e..b69ab633feb 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -11,14 +11,17 @@ on: default: false outputs: status: - description: 'The result of the E2E tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the E2E tests failed, empty otherwise" + value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }} jobs: test: runs-on: macos-26-large + # Matrix legs write differently named outputs so they are combined + # instead of the last leg to finish overwriting the others. outputs: - status: ${{ steps.report-status.outputs.status }} + debug_status: ${{ steps.report-status.outputs.debug_status }} + release_status: ${{ steps.report-status.outputs.release_status }} strategy: fail-fast: false matrix: @@ -37,35 +40,34 @@ jobs: run: ls -lR /tmp/RNTesterBuild - name: Run E2E Tests id: run-tests + if: ${{ !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/maestro-ios - with: + with: &test-inputs app-path: '/tmp/RNTesterBuild/RNTester.app' app-id: com.meta.RNTester.localDevelopment maestro-flow: ./packages/rn-tester/.maestro/ flavor: ${{ matrix.flavor }} device-model: iPhone-17-Pro device-os: iOS-26-2 + - name: Run E2E Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ inputs.fail-on-error }} + uses: ./.github/actions/maestro-ios + with: *test-inputs - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: test - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} + env: + FLAVOR: ${{ matrix.flavor }} run: | - if [[ "${{ needs.test.result }}" != "success" || - "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 + flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]") + case "$flavor" in + debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output." + exit 1 + ;; + esac diff --git a/.github/workflows/e2e-ios-templateapp.yml b/.github/workflows/e2e-ios-templateapp.yml index e91b5ba0f72..283a5d6ee1a 100644 --- a/.github/workflows/e2e-ios-templateapp.yml +++ b/.github/workflows/e2e-ios-templateapp.yml @@ -11,8 +11,8 @@ on: default: false outputs: status: - description: 'The result of the E2E tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the E2E tests failed, empty otherwise" + value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }} jobs: build: @@ -100,8 +100,11 @@ jobs: test: needs: build runs-on: macos-26-large + # Matrix legs write differently named outputs so they are combined + # instead of the last leg to finish overwriting the others. outputs: - status: ${{ steps.report-status.outputs.status }} + debug_status: ${{ steps.report-status.outputs.debug_status }} + release_status: ${{ steps.report-status.outputs.release_status }} strategy: fail-fast: false matrix: @@ -149,35 +152,33 @@ jobs: node ./scripts/e2e/init-project-e2e.js --projectName RNTestProject --currentBranch $BRANCH --directory /tmp/RNTestProject --pathToLocalReactNative $REACT_NATIVE_PKG - name: Run E2E Tests id: run-tests + if: ${{ !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/maestro-ios - with: + with: &test-inputs app-path: '/tmp/RNTestProjectBuild/RNTestProject.app' app-id: org.reactjs.native.example.RNTestProject maestro-flow: ./scripts/e2e/.maestro/ flavor: ${{ matrix.flavor }} working-directory: /tmp/RNTestProject + - name: Run E2E Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ inputs.fail-on-error }} + uses: ./.github/actions/maestro-ios + with: *test-inputs - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: [build, test] - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} + env: + FLAVOR: ${{ matrix.flavor }} run: | - if [[ "${{ needs.build.result }}" != "success" || - "${{ needs.test.result }}" != "success" || - "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 + flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]") + case "$flavor" in + debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output." + exit 1 + ;; + esac diff --git a/.github/workflows/fantom-tests.yml b/.github/workflows/fantom-tests.yml index 82cdd462e6d..85baed28f90 100644 --- a/.github/workflows/fantom-tests.yml +++ b/.github/workflows/fantom-tests.yml @@ -11,8 +11,8 @@ on: default: false outputs: status: - description: 'The result of the Fantom tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the Fantom tests failed, empty otherwise" + value: ${{ jobs.test.outputs.status }} jobs: test: @@ -28,27 +28,16 @@ jobs: uses: actions/checkout@v6 - name: Run Fantom Tests id: run-tests + if: ${{ !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/run-fantom-tests + - name: Run Fantom Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ inputs.fail-on-error }} + uses: ./.github/actions/run-fantom-tests - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: test - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check - run: | - if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 1d475353c8a..752635c79e5 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -199,13 +199,13 @@ jobs: test_e2e_ios_rntester_retry_1: needs: test_e2e_ios_rntester - if: ${{ always() && needs.test_e2e_ios_rntester.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_ios_rntester.result == 'failure' || needs.test_e2e_ios_rntester.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-ios-rntester.yml secrets: inherit test_e2e_ios_rntester_retry_2: needs: test_e2e_ios_rntester_retry_1 - if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_ios_rntester_retry_1.result == 'failure' || needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-ios-rntester.yml with: fail-on-error: true @@ -226,13 +226,13 @@ jobs: test_e2e_ios_templateapp_retry_1: needs: test_e2e_ios_templateapp - if: ${{ always() && needs.test_e2e_ios_templateapp.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_ios_templateapp.result == 'failure' || needs.test_e2e_ios_templateapp.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-ios-templateapp.yml secrets: inherit test_e2e_ios_templateapp_retry_2: needs: test_e2e_ios_templateapp_retry_1 - if: ${{ always() && needs.test_e2e_ios_templateapp_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_ios_templateapp_retry_1.result == 'failure' || needs.test_e2e_ios_templateapp_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-ios-templateapp.yml with: fail-on-error: true @@ -286,7 +286,7 @@ jobs: test_e2e_android_templateapp_retry_1: needs: test_e2e_android_templateapp - if: ${{ always() && needs.test_e2e_android_templateapp.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_android_templateapp.result == 'failure' || needs.test_e2e_android_templateapp.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-android-templateapp.yml with: retry-attempt: 1 @@ -294,7 +294,7 @@ jobs: test_e2e_android_templateapp_retry_2: needs: test_e2e_android_templateapp_retry_1 - if: ${{ always() && needs.test_e2e_android_templateapp_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_android_templateapp_retry_1.result == 'failure' || needs.test_e2e_android_templateapp_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-android-templateapp.yml with: fail-on-error: true @@ -332,13 +332,13 @@ jobs: run_fantom_tests_retry_1: needs: run_fantom_tests - if: ${{ always() && needs.run_fantom_tests.outputs.status == 'failure' }} + if: ${{ always() && (needs.run_fantom_tests.result == 'failure' || needs.run_fantom_tests.outputs.status == 'failure') }} uses: ./.github/workflows/fantom-tests.yml secrets: inherit run_fantom_tests_retry_2: needs: run_fantom_tests_retry_1 - if: ${{ always() && needs.run_fantom_tests_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.run_fantom_tests_retry_1.result == 'failure' || needs.run_fantom_tests_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/fantom-tests.yml with: fail-on-error: true @@ -402,7 +402,7 @@ jobs: test_e2e_android_rntester_retry_1: needs: test_e2e_android_rntester - if: ${{ always() && needs.test_e2e_android_rntester.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_android_rntester.result == 'failure' || needs.test_e2e_android_rntester.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-android-rntester.yml with: retry-attempt: 1 @@ -410,7 +410,7 @@ jobs: test_e2e_android_rntester_retry_2: needs: test_e2e_android_rntester_retry_1 - if: ${{ always() && needs.test_e2e_android_rntester_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_android_rntester_retry_1.result == 'failure' || needs.test_e2e_android_rntester_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-android-rntester.yml with: fail-on-error: true