From 939abb1def1cfaaf0a395319a79d4449067e6d39 Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 29 Oct 2026 17:09:32 +0000 Subject: [PATCH 1/6] Retry failing iOS E2E flows once and stop rerunning real failures Each RNTester iOS flow had 5 attempts inside the job, and the whole job (both flavors, every flow, all setup) was retried up to 3 times on top, so a flow could run 15 times. The runner also stopped at the first flow that exhausted its attempts, hiding later failures. - Give each flow 2 attempts, rebooting the simulator before the retry. - Keep running the remaining flows after a failure and report every failed flow at the end, with a step summary. - Persist per-flow results between CI attempts (same format as the Android runner) so retries skip flows that already passed. - Only retry the workflow when a job stopped before running every flow. A flow that failed both attempts fails the run immediately. - Upload the recording, JUnit report and Maestro debug output for failed attempts only. --- .github/actions/maestro-ios/action.yml | 24 +- .../__tests__/maestro-ios-test.js | 221 +++++++++----- .github/workflow-scripts/maestro-ios.js | 275 ++++++++++++++---- .github/workflows/e2e-ios-rntester.yml | 85 +++++- .github/workflows/test-all.yml | 10 +- 5 files changed, 454 insertions(+), 161 deletions(-) diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index b5e4036d6280..19ddea8417f6 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -26,6 +26,10 @@ inputs: required: false default: '' description: Maestro device OS name, such as iOS-26-2 + test-state-path: + required: false + default: '' + description: The path used to persist per-flow test results between retries runs: using: composite @@ -73,24 +77,14 @@ runs: "${{ inputs.flavor }}" \ "${{ inputs.working-directory }}" \ "${{ inputs.device-model }}" \ - "${{ inputs.device-os }}" - - name: Store video record - if: always() - uses: ./.github/actions/upload-artifact - with: - name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.flavor }}_NewArch - overwrite: true - path: | - video_record_1.mov - video_record_2.mov - video_record_3.mov - video_record_4.mov - video_record_5.mov - report.xml + "${{ inputs.device-os }}" \ + "${{ inputs.test-state-path }}" - name: Store Logs - if: failure() && steps.run-tests.outcome == 'failure' + if: always() uses: ./.github/actions/upload-artifact with: name: maestro-logs-${{ inputs.app-id }}-${{ inputs.flavor }}-NewArch overwrite: true + if-no-files-found: ignore + # Contains the recording, report and debug output of failed attempts only path: /tmp/MaestroLogs diff --git a/.github/workflow-scripts/__tests__/maestro-ios-test.js b/.github/workflow-scripts/__tests__/maestro-ios-test.js index f5f9686e4e09..f01ae9451195 100644 --- a/.github/workflow-scripts/__tests__/maestro-ios-test.js +++ b/.github/workflow-scripts/__tests__/maestro-ios-test.js @@ -11,21 +11,32 @@ jest.mock('child_process', () => ({ execSync: jest.fn(), spawn: jest.fn(), })); -jest.mock('fs', () => ({ - existsSync: jest.fn(), - lstatSync: jest.fn(), - readdirSync: jest.fn(), -})); const childProcess = require('child_process'); const {EventEmitter} = require('events'); const fs = require('fs'); +const os = require('os'); +const path = require('path'); -const {executeFlows, findAvailableSimulator} = require('../maestro-ios'); +const { + collectFlows, + executeFlowSuite, + findAvailableSimulator, + loadState, + runMaestroFlow, +} = require('../maestro-ios'); describe('Maestro iOS runner', () => { + let temporaryDirectory; + beforeEach(() => { jest.resetAllMocks(); + temporaryDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), 'maestro-ios-test-'), + ); + jest.spyOn(console, 'error').mockImplementation(() => {}); + jest.spyOn(console, 'info').mockImplementation(() => {}); + jest.spyOn(console, 'log').mockImplementation(() => {}); childProcess.spawn.mockImplementation(() => { const recordingProcess = new EventEmitter(); recordingProcess.pid = 1; @@ -37,112 +48,170 @@ describe('Maestro iOS runner', () => { }); }); - it('executes each YAML flow separately and skips other files', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => path === 'flows/', - })); - fs.readdirSync.mockReturnValue(['second.yaml', 'image.png', 'first.yml']); + afterEach(() => { + jest.restoreAllMocks(); + fs.rmSync(temporaryDirectory, {recursive: true, force: true}); + }); - await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); + it('collects YAML flows in a stable order and skips helpers and other files', () => { + const helpersDirectory = path.join(temporaryDirectory, 'helpers'); + const nestedDirectory = path.join(temporaryDirectory, 'nested'); + fs.mkdirSync(helpersDirectory); + fs.mkdirSync(nestedDirectory); + fs.writeFileSync(path.join(temporaryDirectory, 'second.yaml'), 'appId: x'); + fs.writeFileSync(path.join(temporaryDirectory, 'image.png'), 'not a flow'); + fs.writeFileSync(path.join(temporaryDirectory, 'first.yml'), 'appId: x'); + fs.writeFileSync(path.join(helpersDirectory, 'search.yml'), 'appId: x'); + fs.writeFileSync(path.join(nestedDirectory, 'third.yml'), 'appId: x'); - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - expect(childProcess.execSync.mock.calls[0][0]).toContain( - 'test "flows/first.yml"', - ); - expect(childProcess.execSync.mock.calls[1][0]).toContain( - 'test "flows/second.yaml"', - ); + expect(collectFlows(temporaryDirectory)).toEqual([ + path.join(temporaryDirectory, 'first.yml'), + path.join(nestedDirectory, 'third.yml'), + path.join(temporaryDirectory, 'second.yaml'), + ]); }); - it('retries only the failing flow', async () => { - fs.existsSync.mockReturnValue(false); - childProcess.execSync.mockImplementationOnce(() => { - throw new Error('Maestro driver failed'); + it('retries a failing flow once on a rebooted simulator and keeps going', async () => { + const flows = ['first.yml', 'second.yml', 'third.yml'].map(file => + path.join(temporaryDirectory, file), + ); + const statePath = path.join(temporaryDirectory, 'state', 'results.json'); + const executeFlow = jest.fn(async (appId, udid, flow) => { + if (flow.endsWith('second.yml')) { + throw new Error('failed assertion'); + } }); + const resetSimulator = jest.fn(); - await executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'); + await expect( + executeFlowSuite({ + flows, + appId: 'com.example', + udid: 'device-id', + state: loadState(statePath), + statePath, + executeFlow, + resetSimulator, + }), + ).rejects.toThrow('1 Maestro flow(s) failed'); - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - for (const call of childProcess.execSync.mock.calls) { - expect(call[0]).toContain('test "flow.yml"'); - } + expect(executeFlow.mock.calls.map(call => [call[2], call[3]])).toEqual([ + [flows[0], 1], + [flows[1], 1], + [flows[1], 2], + [flows[2], 1], + ]); + expect(resetSimulator).toHaveBeenCalledTimes(1); + expect(resetSimulator).toHaveBeenCalledWith('device-id'); + expect(Object.values(loadState(statePath).flows)).toEqual([ + {status: 'passed', attempts: 1}, + {status: 'failed', attempts: 2, error: 'failed assertion'}, + {status: 'passed', attempts: 1}, + ]); }); - it('waits for the recorder to exit before starting the next flow', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => path === 'flows/', - })); - fs.readdirSync.mockReturnValue(['first.yml', 'second.yml']); + it('passes a flow that succeeds on its retry', async () => { + const flow = path.join(temporaryDirectory, 'flaky.yml'); + const statePath = path.join(temporaryDirectory, 'results.json'); + const executeFlow = jest + .fn() + .mockRejectedValueOnce(new Error('flaky')) + .mockResolvedValueOnce(); + + await executeFlowSuite({ + flows: [flow], + appId: 'com.example', + udid: 'device-id', + state: loadState(statePath), + statePath, + executeFlow, + resetSimulator: jest.fn(), + }); + + expect(Object.values(loadState(statePath).flows)).toEqual([ + {status: 'passed', attempts: 2}, + ]); + }); + + it('only runs flows that have not passed in a previous CI attempt', async () => { + const flows = ['first.yml', 'second.yml'].map(file => + path.join(temporaryDirectory, file), + ); + const statePath = path.join(temporaryDirectory, 'results.json'); + await expect( + executeFlowSuite({ + flows, + appId: 'com.example', + udid: 'device-id', + state: loadState(statePath), + statePath, + executeFlow: jest.fn(async (appId, udid, flow) => { + if (flow.endsWith('second.yml')) { + throw new Error('failed assertion'); + } + }), + resetSimulator: jest.fn(), + }), + ).rejects.toThrow(); + + const retry = jest.fn(); + await executeFlowSuite({ + flows, + appId: 'com.example', + udid: 'device-id', + state: loadState(statePath), + statePath, + executeFlow: retry, + resetSimulator: jest.fn(), + }); + expect(retry).toHaveBeenCalledTimes(1); + expect(retry.mock.calls[0][2]).toBe(flows[1]); + }); + + it('waits for the recorder to exit before returning', async () => { const recordingProcess = new EventEmitter(); recordingProcess.pid = 1; recordingProcess.kill = jest.fn(() => true); childProcess.spawn.mockReturnValueOnce(recordingProcess); - const execution = executeFlows( + let finished = false; + const execution = runMaestroFlow( 'com.example', 'device-id', - 'flows/', - 'Hermes', - ); + 'flows/first.yml', + 1, + ).then(() => { + finished = true; + }); await new Promise(resolve => jest.requireActual('timers').setImmediate(resolve), ); expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); - expect(childProcess.execSync).toHaveBeenCalledTimes(1); - expect(childProcess.spawn).toHaveBeenCalledTimes(1); + expect(childProcess.execSync.mock.calls[0][0]).toContain( + 'test "flows/first.yml"', + ); + expect(finished).toBe(false); recordingProcess.emit('exit', 0, null); await execution; - - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - expect(childProcess.spawn).toHaveBeenCalledTimes(2); + expect(finished).toBe(true); }); - it('skips helper directories while recursing into flow directories', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => !path.endsWith('.yml'), - })); - fs.readdirSync.mockImplementation(path => - path === 'flows/' ? ['helpers', 'nested'] : ['flow.yml'], - ); - - await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); - - expect(fs.readdirSync).not.toHaveBeenCalledWith('flows/helpers'); - expect(childProcess.execSync).toHaveBeenCalledTimes(1); - expect(childProcess.execSync.mock.calls[0][0]).toContain( - 'test "flows/nested/flow.yml"', - ); - }); - - it('rejects after exhausting retries and stops every recorder', async () => { - const consoleError = jest - .spyOn(console, 'error') - .mockImplementation(() => {}); - fs.existsSync.mockReturnValue(false); + it('stops the recorder when Maestro fails', async () => { const error = new Error('Maestro driver failed'); childProcess.execSync.mockImplementation(() => { throw error; }); await expect( - executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'), + runMaestroFlow('com.example', 'device-id', 'flow.yml', 2), ).rejects.toBe(error); - expect(childProcess.execSync).toHaveBeenCalledTimes(5); - expect(childProcess.spawn).toHaveBeenCalledTimes(5); - for (const {value: recordingProcess} of childProcess.spawn.mock.results) { - expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); - } - expect(consoleError).toHaveBeenCalledWith( - 'Failed to execute flow flow.yml after 5 attempts.', - ); + const {value: recordingProcess} = childProcess.spawn.mock.results[0]; + expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); }); it('selects an iPhone Pro simulator from the latest runtime', () => { diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js index a9782ec6506f..b39051f65272 100644 --- a/.github/workflow-scripts/maestro-ios.js +++ b/.github/workflow-scripts/maestro-ios.js @@ -9,10 +9,11 @@ const childProcess = require('child_process'); const fs = require('fs'); +const path = require('path'); const usage = ` === Usage === -node maestro-ios.js [device_model] [device_os] +node maestro-ios.js [device_model] [device_os] [test_state_path] @param {string} appPath - Path to the app APK @param {string} appId - App ID that needs to be launched @@ -22,10 +23,16 @@ node maestro-ios.js ({flow, ...state.flows[flow]})); + const counts = results.reduce( + (result, flow) => { + result[flow.status] += 1; + return result; + }, + {passed: 0, failed: 0, pending: 0}, + ); + const rows = results + .map( + result => + `| ${result.status} | \`${result.flow.replaceAll('|', '\\|')}\` | ${result.attempts} |`, + ) + .join('\n'); + + return `### iOS Maestro E2E results + +Passed: ${counts.passed} · Failed: ${counts.failed} · Pending: ${counts.pending} + +| Status | Flow | Attempts | +| --- | --- | ---: | +${rows} +`; +} + +function writeResultsSummary(flowKeys, state) { + const summary = formatResults(flowKeys, state); + console.info(`\n${summary}`); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + } +} + +async function runMaestroFlow(appId, udid, flow, attempt) { + const outputName = `${path.basename(flow, path.extname(flow))}-attempt-${attempt}`; + const outputPath = path.join(MAESTRO_LOG_DIRECTORY, outputName); + fs.mkdirSync(outputPath, {recursive: true}); + + const videoPath = path.join(outputPath, 'recording.mov'); + const recProcess = startVideoRecording(udid, videoPath); try { const timeout = 1000 * 60 * 10; // 10 minutes - const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit -e APP_ID="${appId}"`; - console.info(`Executing flow: ${flow} (attempt ${currentAttempt})`); + const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit --output "${outputPath}/report.xml" --debug-output "${outputPath}" --flatten-debug-output -e APP_ID="${appId}"`; + console.info(`Executing flow: ${flow} (attempt ${attempt})`); console.log(command); childProcess.execSync(`MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 ${command}`, { stdio: 'inherit', timeout, }); - - await stopVideoRecording(recProcess); } catch (error) { await stopVideoRecording(recProcess); + throw error; + } - if (currentAttempt < MAX_ATTEMPTS) { - console.info(`Retrying flow: ${flow}`); - await executeFlowWithRetries( - appId, - udid, - flow, - jsengine, - currentAttempt + 1, - ); - } else { + await stopVideoRecording(recProcess); + // Only failed attempts are worth keeping in the uploaded logs. + fs.rmSync(outputPath, {recursive: true, force: true}); +} + +async function executeFlowSuite({ + flows, + appId, + udid, + state, + statePath, + executeFlow = runMaestroFlow, + resetSimulator = rebootSimulator, +}) { + const flowKeys = flows.map(getFlowKey); + + for (const flow of flowKeys) { + state.flows[flow] ??= {status: 'pending', attempts: 0}; + } + saveState(statePath, state); + + const failedFlows = []; + for (let index = 0; index < flows.length; index++) { + const flow = flows[index]; + const flowKey = flowKeys[index]; + const result = state.flows[flowKey]; + + if (result.status === 'passed') { + console.info(`Skipping previously passed flow: ${flow}`); + continue; + } + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + if (attempt > 1) { + console.info(`Retrying flow: ${flow}`); + resetSimulator(udid); + } + result.attempts += 1; + try { + await executeFlow(appId, udid, flow, attempt); + result.status = 'passed'; + delete result.error; + break; + } catch (error) { + result.status = 'failed'; + result.error = error instanceof Error ? error.message : String(error); + } finally { + saveState(statePath, state); + } + } + + if (result.status === 'failed') { + failedFlows.push(flowKey); console.error( `Failed to execute flow ${flow} after ${MAX_ATTEMPTS} attempts.`, ); - throw error; } } -} -async function executeFlows(appId, udid, maestroFlow, jsengine) { - if (!fs.existsSync(maestroFlow) || !fs.lstatSync(maestroFlow).isDirectory()) { - await executeFlowWithRetries(appId, udid, maestroFlow, jsengine, 1); - return; - } + writeResultsSummary(flowKeys, state); - for (const file of fs.readdirSync(maestroFlow).sort()) { - const filePath = `${maestroFlow.replace(/\/$/, '')}/${file}`; - if (fs.lstatSync(filePath).isDirectory()) { - // Fragments pulled in via `runFlow`; they have no `launchApp` of their - // own and fail when run standalone. - if (file === 'helpers') { - continue; - } - await executeFlows(appId, udid, filePath, jsengine); - } else if (file.endsWith('.yml') || file.endsWith('.yaml')) { - await executeFlowWithRetries(appId, udid, filePath, jsengine, 1); - } + if (failedFlows.length > 0) { + throw new Error( + `${failedFlows.length} Maestro flow(s) failed:\n${failedFlows.join('\n')}`, + ); } } async function main(args = process.argv.slice(2)) { - if (args.length < 6 || args.length > 8) { + if (args.length < 6 || args.length > 9) { throw new Error(`Invalid number of arguments.\n${usage}`); } @@ -249,6 +387,7 @@ async function main(args = process.argv.slice(2)) { const workingDirectory = args[5]; const deviceModel = args[6] || null; const deviceOS = args[7] || null; + const statePath = args[8] || DEFAULT_STATE_PATH; console.info('\n=============================='); console.info('Running tests for iOS with the following parameters:'); @@ -260,22 +399,40 @@ async function main(args = process.argv.slice(2)) { console.info(`WORKING_DIRECTORY: ${workingDirectory}`); console.info(`DEVICE_MODEL: ${deviceModel ?? ''}`); console.info(`DEVICE_OS: ${deviceOS ?? ''}`); + console.info(`TEST_STATE_PATH: ${statePath}`); console.info('==============================\n'); + const flows = collectFlows(maestroFlow); + const state = loadState(statePath); + const simulator = findAvailableSimulator(deviceModel, deviceOS); launchSimulator(simulator); installAppOnSimulator(appPath, simulator.udid); bringSimulatorInForeground(); await launchAppOnSimulator(appId, simulator.udid, isDebug); - await executeFlows(appId, simulator.udid, maestroFlow, jsengine); + console.info(`Start testing ${flows.length} flow(s)`); + await executeFlowSuite({ + flows, + appId, + udid: simulator.udid, + state, + statePath, + }); console.log('Test finished'); } if (require.main === module) { - main(); + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); } module.exports = { - executeFlows, + collectFlows, + executeFlowSuite, findAvailableSimulator, + formatResults, + loadState, + runMaestroFlow, }; diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index c841c409b9ea..f5c7c0c1bbad 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -9,16 +9,25 @@ on: fail-on-error: type: boolean default: false + retry-attempt: + type: number + default: 0 outputs: status: description: 'The result of the E2E tests (success or failure)' value: ${{ jobs.report.outputs.status }} + retryable: + description: 'Whether the failure came from a job that did not finish running every flow' + value: ${{ jobs.report.outputs.retryable }} jobs: test: runs-on: macos-26-large outputs: - status: ${{ steps.report-status.outputs.status }} + # Matrix legs only keep the last value written to an output name, so each + # flavor writes its own. + debug-result: ${{ steps.classify.outputs.debug-result }} + release-result: ${{ steps.classify.outputs.release-result }} strategy: fail-fast: false matrix: @@ -26,17 +35,38 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Download previous per-flow test state + if: ${{ inputs.retry-attempt > 0 }} + continue-on-error: true + uses: ./.github/actions/download-artifact + with: + name: e2e_ios_rntester_state_${{ matrix.flavor }}_NewArch + path: /tmp/maestro-ios-state + - name: Check for unfinished E2E flows + id: test-state + shell: bash + run: | + SHOULD_RUN=true + if [[ -f /tmp/maestro-ios-state/results.json ]] && \ + jq -e '(.flows | length > 0) and all(.flows[]; .status == "passed")' /tmp/maestro-ios-state/results.json > /dev/null; then + SHOULD_RUN=false + fi + echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" - name: Setup Node.js + if: steps.test-state.outputs.should-run == 'true' uses: ./.github/actions/setup-node - name: Download App + if: steps.test-state.outputs.should-run == 'true' uses: ./.github/actions/download-artifact with: name: RNTesterApp-NewArch-${{ matrix.flavor }} path: /tmp/RNTesterBuild/RNTester.app - name: Check downloaded folder content + if: steps.test-state.outputs.should-run == 'true' run: ls -lR /tmp/RNTesterBuild - name: Run E2E Tests id: run-tests + if: steps.test-state.outputs.should-run == 'true' continue-on-error: true uses: ./.github/actions/maestro-ios with: @@ -46,10 +76,36 @@ jobs: flavor: ${{ matrix.flavor }} device-model: iPhone-17-Pro device-os: iOS-26-2 - - name: Report status - id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT + test-state-path: /tmp/maestro-ios-state/results.json + - name: Store per-flow test state + if: always() + uses: ./.github/actions/upload-artifact + with: + name: e2e_ios_rntester_state_${{ matrix.flavor }}_NewArch + overwrite: true + if-no-files-found: warn + path: /tmp/maestro-ios-state/results.json + - name: Classify result + id: classify + if: always() + shell: bash + # passed: every flow passed. + # failed: a flow failed all of its attempts; retrying the job would only + # repeat the same failure. + # incomplete: the job stopped before running every flow (runner or + # setup failure); a retry can pick up the remaining flows. + run: | + STATE=/tmp/maestro-ios-state/results.json + if [[ "${{ steps.test-state.outputs.should-run }}" == "false" || "${{ steps.run-tests.outcome }}" == "success" ]]; then + RESULT=passed + elif [[ -f "$STATE" ]] && jq -e 'any(.flows[]; .status == "failed")' "$STATE" > /dev/null; then + RESULT=failed + else + RESULT=incomplete + fi + FLAVOR=$(echo "${{ matrix.flavor }}" | tr '[:upper:]' '[:lower:]') + echo "Result: $RESULT" + echo "$FLAVOR-result=$RESULT" >> "$GITHUB_OUTPUT" report: needs: test @@ -57,15 +113,26 @@ jobs: runs-on: ubuntu-latest outputs: status: ${{ steps.check.outputs.status }} + retryable: ${{ steps.check.outputs.retryable }} steps: - id: check + env: + DEBUG_RESULT: ${{ needs.test.outputs.debug-result }} + RELEASE_RESULT: ${{ needs.test.outputs.release-result }} + # A missing result means the leg did not get as far as classifying + # itself, which is treated like an incomplete run. run: | - if [[ "${{ needs.test.result }}" != "success" || - "${{ needs.test.outputs.status }}" == "failure" ]]; then + echo "Debug: ${DEBUG_RESULT:-}, Release: ${RELEASE_RESULT:-}" + if [[ "$DEBUG_RESULT" == "passed" && "$RELEASE_RESULT" == "passed" ]]; then + echo "status=success" >> $GITHUB_OUTPUT + echo "retryable=false" >> $GITHUB_OUTPUT + elif [[ "$DEBUG_RESULT" == "failed" || "$RELEASE_RESULT" == "failed" ]]; then echo "status=failure" >> $GITHUB_OUTPUT + echo "retryable=false" >> $GITHUB_OUTPUT else - echo "status=success" >> $GITHUB_OUTPUT + echo "status=failure" >> $GITHUB_OUTPUT + echo "retryable=true" >> $GITHUB_OUTPUT fi - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} + if: ${{ steps.check.outputs.status == 'failure' && (inputs.fail-on-error || steps.check.outputs.retryable == 'false') }} run: exit 1 diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index a9a3fc90350c..22cd50681ff0 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -197,18 +197,24 @@ jobs: uses: ./.github/workflows/e2e-ios-rntester.yml secrets: inherit + # Each flow already gets a retry inside the job, so a flow that failed is + # reported straight away. These retries only re-run jobs that stopped before + # running every flow, and they skip flows that already passed. test_e2e_ios_rntester_retry_1: needs: test_e2e_ios_rntester - if: ${{ always() && needs.test_e2e_ios_rntester.outputs.status == 'failure' }} + if: ${{ always() && needs.test_e2e_ios_rntester.outputs.retryable == 'true' }} uses: ./.github/workflows/e2e-ios-rntester.yml + with: + retry-attempt: 1 secrets: inherit test_e2e_ios_rntester_retry_2: needs: test_e2e_ios_rntester_retry_1 - if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure' }} + if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.retryable == 'true' }} uses: ./.github/workflows/e2e-ios-rntester.yml with: fail-on-error: true + retry-attempt: 2 secrets: inherit test_visual_ios_rntester: From 7f6ddd3bb40ba8fe126857a003c0b2d0e8145427 Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 29 Oct 2026 17:09:46 +0000 Subject: [PATCH 2/6] Cancel superseded Test All runs on pull requests Test All had no concurrency group, so every push to a pull request left the previous run going to completion. Over Oct 27-29 about 40% of the macOS minutes spent on iOS E2E went to runs whose pull request had already received a newer push. Cancel in-progress runs of the same pull request. Push and workflow_dispatch runs get a unique group and are never cancelled. --- .github/workflows/test-all.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 22cd50681ff0..602371059d8f 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -11,6 +11,13 @@ on: permissions: contents: read +concurrency: + # A new push to a pull request makes the previous run obsolete, so cancel it + # instead of spending (macOS) runner time on it. Runs on branches are never + # cancelled: each one gets its own group. + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: set_release_type: runs-on: ubuntu-latest From ae06dad1245c7e45b56a97a85b35491fa297c3c7 Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 29 Oct 2026 17:13:10 +0000 Subject: [PATCH 3/6] Run iOS E2E flows in a single Maestro invocation The runner started a new `maestro test` process for every flow. Each one paid for JVM startup and an XCTest driver reinstall (~24s) plus stopping the screen recording (~3s): about 21 minutes of every ~75 minute RNTester job. - Run every pending flow in one `maestro test` invocation and read the per-flow results from its JUnit report. Flows missing from the report (Maestro crashed or timed out) count as failed. - Retry each failed flow on its own, on a rebooted simulator, with a screen recording. Passing runs don't produce recordings any more; failures keep Maestro's screenshots and debug output. - Add an `exclude-tags` input, and skip the `android-only` flows in RNTester. They only contain an Android-gated `runFlow`, but still paid the per-flow startup cost on iOS. --- .github/actions/maestro-ios/action.yml | 9 +- .../__tests__/maestro-ios-test.js | 106 +++++++--- .github/workflow-scripts/maestro-ios.js | 181 +++++++++++++++--- .github/workflows/e2e-ios-rntester.yml | 1 + 4 files changed, 248 insertions(+), 49 deletions(-) diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index 19ddea8417f6..8e1a11c9a259 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -30,6 +30,10 @@ inputs: required: false default: '' description: The path used to persist per-flow test results between retries + exclude-tags: + required: false + default: '' + description: Comma-separated flow tags to exclude from the test run runs: using: composite @@ -78,7 +82,8 @@ runs: "${{ inputs.working-directory }}" \ "${{ inputs.device-model }}" \ "${{ inputs.device-os }}" \ - "${{ inputs.test-state-path }}" + "${{ inputs.test-state-path }}" \ + "${{ inputs.exclude-tags }}" - name: Store Logs if: always() uses: ./.github/actions/upload-artifact @@ -86,5 +91,5 @@ runs: name: maestro-logs-${{ inputs.app-id }}-${{ inputs.flavor }}-NewArch overwrite: true if-no-files-found: ignore - # Contains the recording, report and debug output of failed attempts only + # Only failed runs leave their report, debug output and recording here path: /tmp/MaestroLogs diff --git a/.github/workflow-scripts/__tests__/maestro-ios-test.js b/.github/workflow-scripts/__tests__/maestro-ios-test.js index f01ae9451195..636278940167 100644 --- a/.github/workflow-scripts/__tests__/maestro-ios-test.js +++ b/.github/workflow-scripts/__tests__/maestro-ios-test.js @@ -21,11 +21,27 @@ const path = require('path'); const { collectFlows, executeFlowSuite, + filterFlowsByTags, findAvailableSimulator, loadState, + parseJUnitReport, runMaestroFlow, } = require('../maestro-ios'); +// Simulates a single Maestro run in which the flows matching `failing` fail. +function suiteResults(failing = () => false) { + return jest.fn(async (appId, udid, flows) => { + return new Map( + flows.map(flow => [ + path.relative(process.cwd(), flow), + failing(flow) + ? {passed: false, error: 'failed assertion'} + : {passed: true}, + ]), + ); + }); +} + describe('Maestro iOS runner', () => { let temporaryDirectory; @@ -71,14 +87,54 @@ describe('Maestro iOS runner', () => { ]); }); - it('retries a failing flow once on a rebooted simulator and keeps going', async () => { + it('excludes flows with matching tags', () => { + const androidOnlyFlow = path.join(temporaryDirectory, 'android-only.yml'); + const regularFlow = path.join(temporaryDirectory, 'regular.yml'); + fs.writeFileSync( + androidOnlyFlow, + 'appId: x\ntags:\n - android-only\n---\n- launchApp\n', + ); + fs.writeFileSync(regularFlow, 'appId: x\n---\n- launchApp\n'); + + expect( + filterFlowsByTags([androidOnlyFlow, regularFlow], ['android-only']), + ).toEqual([regularFlow]); + }); + + it('parses flow results from a Maestro JUnit report', () => { + const report = ` + + + + + Assertion is false: "Cheesecake" is visible + + + + +`; + + expect(parseJUnitReport(report)).toEqual( + new Map([ + ['flows/first.yml', {passed: true, error: 'SUCCESS'}], + [ + 'flows/second.yml', + {passed: false, error: 'Assertion is false: "Cheesecake" is visible'}, + ], + ['flows/third.yml', {passed: true, error: 'WARNING'}], + ]), + ); + }); + + it('runs all flows together, then retries each failure once on a rebooted simulator', async () => { const flows = ['first.yml', 'second.yml', 'third.yml'].map(file => path.join(temporaryDirectory, file), ); const statePath = path.join(temporaryDirectory, 'state', 'results.json'); + const executeSuite = suiteResults(flow => !flow.endsWith('first.yml')); const executeFlow = jest.fn(async (appId, udid, flow) => { if (flow.endsWith('second.yml')) { - throw new Error('failed assertion'); + throw new Error('failed again'); } }); const resetSimulator = jest.fn(); @@ -90,33 +146,32 @@ describe('Maestro iOS runner', () => { udid: 'device-id', state: loadState(statePath), statePath, + executeSuite, executeFlow, resetSimulator, }), ).rejects.toThrow('1 Maestro flow(s) failed'); + expect(executeSuite).toHaveBeenCalledTimes(1); + expect(executeSuite.mock.calls[0][2]).toEqual(flows); expect(executeFlow.mock.calls.map(call => [call[2], call[3]])).toEqual([ - [flows[0], 1], - [flows[1], 1], [flows[1], 2], - [flows[2], 1], + [flows[2], 2], ]); - expect(resetSimulator).toHaveBeenCalledTimes(1); + expect(resetSimulator).toHaveBeenCalledTimes(2); expect(resetSimulator).toHaveBeenCalledWith('device-id'); expect(Object.values(loadState(statePath).flows)).toEqual([ {status: 'passed', attempts: 1}, - {status: 'failed', attempts: 2, error: 'failed assertion'}, - {status: 'passed', attempts: 1}, + {status: 'failed', attempts: 2, error: 'failed again'}, + {status: 'passed', attempts: 2}, ]); }); - it('passes a flow that succeeds on its retry', async () => { - const flow = path.join(temporaryDirectory, 'flaky.yml'); + it('does not retry when every flow passes', async () => { + const flow = path.join(temporaryDirectory, 'first.yml'); const statePath = path.join(temporaryDirectory, 'results.json'); - const executeFlow = jest - .fn() - .mockRejectedValueOnce(new Error('flaky')) - .mockResolvedValueOnce(); + const executeFlow = jest.fn(); + const resetSimulator = jest.fn(); await executeFlowSuite({ flows: [flow], @@ -124,12 +179,15 @@ describe('Maestro iOS runner', () => { udid: 'device-id', state: loadState(statePath), statePath, + executeSuite: suiteResults(), executeFlow, - resetSimulator: jest.fn(), + resetSimulator, }); + expect(executeFlow).not.toHaveBeenCalled(); + expect(resetSimulator).not.toHaveBeenCalled(); expect(Object.values(loadState(statePath).flows)).toEqual([ - {status: 'passed', attempts: 2}, + {status: 'passed', attempts: 1}, ]); }); @@ -145,28 +203,26 @@ describe('Maestro iOS runner', () => { udid: 'device-id', state: loadState(statePath), statePath, - executeFlow: jest.fn(async (appId, udid, flow) => { - if (flow.endsWith('second.yml')) { - throw new Error('failed assertion'); - } - }), + executeSuite: suiteResults(flow => flow.endsWith('second.yml')), + executeFlow: jest.fn().mockRejectedValue(new Error('failed again')), resetSimulator: jest.fn(), }), ).rejects.toThrow(); - const retry = jest.fn(); + const executeSuite = suiteResults(); await executeFlowSuite({ flows, appId: 'com.example', udid: 'device-id', state: loadState(statePath), statePath, - executeFlow: retry, + executeSuite, + executeFlow: jest.fn(), resetSimulator: jest.fn(), }); - expect(retry).toHaveBeenCalledTimes(1); - expect(retry.mock.calls[0][2]).toBe(flows[1]); + expect(executeSuite).toHaveBeenCalledTimes(1); + expect(executeSuite.mock.calls[0][2]).toEqual([flows[1]]); }); it('waits for the recorder to exit before returning', async () => { diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js index b39051f65272..fb2c26836d1b 100644 --- a/.github/workflow-scripts/maestro-ios.js +++ b/.github/workflow-scripts/maestro-ios.js @@ -13,7 +13,7 @@ const path = require('path'); const usage = ` === Usage === -node maestro-ios.js [device_model] [device_os] [test_state_path] +node maestro-ios.js [device_model] [device_os] [test_state_path] [exclude_tags] @param {string} appPath - Path to the app APK @param {string} appId - App ID that needs to be launched @@ -24,6 +24,7 @@ node maestro-ios.js { + const tags = getFlowTags(flow); + const excludedTag = tags.find(tag => excludeTags.includes(tag)); + if (excludedTag != null) { + console.info(`Skipping flow tagged ${excludedTag}: ${flow}`); + return false; + } + return true; + }); +} + +function decodeXmlEntities(value) { + return value + .replaceAll('"', '"') + .replaceAll(''', "'") + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('&', '&'); +} + +// Maps each flow key in a Maestro JUnit report to whether it passed, plus the +// failure message for flows that did not. +function parseJUnitReport(contents) { + const results = new Map(); + const testCasePattern = /]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g; + for (const [, attributes, body = ''] of contents.matchAll(testCasePattern)) { + const attribute = name => + attributes.match(new RegExp(`\\b${name}="([^"]*)"`))?.[1]; + const file = attribute('file'); + if (file == null) { + continue; + } + const status = attribute('status'); + const failure = body.match(/]*>([\s\S]*?)<\/failure>/)?.[1]; + results.set(getFlowKey(decodeXmlEntities(file)), { + passed: status === 'SUCCESS' || status === 'WARNING', + error: failure != null ? decodeXmlEntities(failure.trim()) : status, + }); + } + return results; +} + +function maestroCommand(udid, flows, appId, outputPath) { + const flowArguments = flows.map(flow => `"${flow}"`).join(' '); + return `MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 $HOME/.maestro/bin/maestro --udid="${udid}" test ${flowArguments} --format junit --output "${outputPath}/report.xml" --debug-output "${outputPath}" --flatten-debug-output -e APP_ID="${appId}"`; +} + +// Runs every flow in a single Maestro invocation, so the CLI and the XCTest +// driver start once instead of once per flow. Flows missing from the report +// (e.g. because Maestro crashed or timed out) count as failed. +function runMaestroSuite(appId, udid, flows) { + const outputPath = path.join(MAESTRO_LOG_DIRECTORY, 'suite'); + fs.mkdirSync(outputPath, {recursive: true}); + + const command = maestroCommand(udid, flows, appId, outputPath); + console.info(`Executing ${flows.length} flow(s) in a single Maestro run`); + console.log(command); + let runError = null; + try { + childProcess.execSync(command, { + stdio: 'inherit', + timeout: flows.length * FLOW_TIMEOUT_MS, + }); + } catch (error) { + runError = error; + } + + const reportPath = path.join(outputPath, 'report.xml'); + const report = fs.existsSync(reportPath) + ? parseJUnitReport(fs.readFileSync(reportPath, 'utf8')) + : new Map(); + const missingError = + runError instanceof Error + ? runError.message + : 'Flow did not report a result'; + const results = new Map( + flows.map(flow => [ + getFlowKey(flow), + report.get(getFlowKey(flow)) ?? {passed: false, error: missingError}, + ]), + ); + if ([...results.values()].every(result => result.passed)) { + fs.rmSync(outputPath, {recursive: true, force: true}); + } + return results; +} + async function runMaestroFlow(appId, udid, flow, attempt) { const outputName = `${path.basename(flow, path.extname(flow))}-attempt-${attempt}`; const outputPath = path.join(MAESTRO_LOG_DIRECTORY, outputName); @@ -293,13 +404,12 @@ async function runMaestroFlow(appId, udid, flow, attempt) { const videoPath = path.join(outputPath, 'recording.mov'); const recProcess = startVideoRecording(udid, videoPath); try { - const timeout = 1000 * 60 * 10; // 10 minutes - const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit --output "${outputPath}/report.xml" --debug-output "${outputPath}" --flatten-debug-output -e APP_ID="${appId}"`; + const command = maestroCommand(udid, [flow], appId, outputPath); console.info(`Executing flow: ${flow} (attempt ${attempt})`); console.log(command); - childProcess.execSync(`MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 ${command}`, { + childProcess.execSync(command, { stdio: 'inherit', - timeout, + timeout: FLOW_TIMEOUT_MS, }); } catch (error) { await stopVideoRecording(recProcess); @@ -317,6 +427,7 @@ async function executeFlowSuite({ udid, state, statePath, + executeSuite = runMaestroSuite, executeFlow = runMaestroFlow, resetSimulator = rebootSimulator, }) { @@ -327,30 +438,50 @@ async function executeFlowSuite({ } saveState(statePath, state); - const failedFlows = []; - for (let index = 0; index < flows.length; index++) { - const flow = flows[index]; - const flowKey = flowKeys[index]; - const result = state.flows[flowKey]; - - if (result.status === 'passed') { + const flowsToRun = flows.filter((flow, index) => { + if (state.flows[flowKeys[index]].status === 'passed') { console.info(`Skipping previously passed flow: ${flow}`); - continue; + return false; } + return true; + }); - for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { - if (attempt > 1) { - console.info(`Retrying flow: ${flow}`); - resetSimulator(udid); + // First attempt: every flow in one Maestro run. + if (flowsToRun.length > 0) { + const results = await executeSuite(appId, udid, flowsToRun); + for (const flow of flowsToRun) { + const result = state.flows[getFlowKey(flow)]; + const {passed, error} = results.get(getFlowKey(flow)); + result.attempts += 1; + result.status = passed ? 'passed' : 'failed'; + if (passed) { + delete result.error; + } else { + result.error = error; } + } + saveState(statePath, state); + } + + // Retries: each failed flow on its own, on a rebooted simulator. + const failedFlows = []; + for (const flow of flowsToRun) { + const flowKey = getFlowKey(flow); + const result = state.flows[flowKey]; + + for ( + let attempt = 2; + attempt <= MAX_ATTEMPTS && result.status === 'failed'; + attempt++ + ) { + console.info(`Retrying flow: ${flow}`); + resetSimulator(udid); result.attempts += 1; try { await executeFlow(appId, udid, flow, attempt); result.status = 'passed'; delete result.error; - break; } catch (error) { - result.status = 'failed'; result.error = error instanceof Error ? error.message : String(error); } finally { saveState(statePath, state); @@ -375,7 +506,7 @@ async function executeFlowSuite({ } async function main(args = process.argv.slice(2)) { - if (args.length < 6 || args.length > 9) { + if (args.length < 6 || args.length > 10) { throw new Error(`Invalid number of arguments.\n${usage}`); } @@ -388,6 +519,10 @@ async function main(args = process.argv.slice(2)) { const deviceModel = args[6] || null; const deviceOS = args[7] || null; const statePath = args[8] || DEFAULT_STATE_PATH; + const excludeTags = (args[9] ?? '') + .split(',') + .map(tag => tag.trim()) + .filter(Boolean); console.info('\n=============================='); console.info('Running tests for iOS with the following parameters:'); @@ -400,9 +535,10 @@ async function main(args = process.argv.slice(2)) { console.info(`DEVICE_MODEL: ${deviceModel ?? ''}`); console.info(`DEVICE_OS: ${deviceOS ?? ''}`); console.info(`TEST_STATE_PATH: ${statePath}`); + console.info(`EXCLUDE_TAGS: ${excludeTags.join(',') || ''}`); console.info('==============================\n'); - const flows = collectFlows(maestroFlow); + const flows = filterFlowsByTags(collectFlows(maestroFlow), excludeTags); const state = loadState(statePath); const simulator = findAvailableSimulator(deviceModel, deviceOS); @@ -431,8 +567,9 @@ if (require.main === module) { module.exports = { collectFlows, executeFlowSuite, + filterFlowsByTags, findAvailableSimulator, - formatResults, loadState, + parseJUnitReport, runMaestroFlow, }; diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index f5c7c0c1bbad..4fba45819de8 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -77,6 +77,7 @@ jobs: device-model: iPhone-17-Pro device-os: iOS-26-2 test-state-path: /tmp/maestro-ios-state/results.json + exclude-tags: android-only - name: Store per-flow test state if: always() uses: ./.github/actions/upload-artifact From 79a54ab35eec955973d40ad0c930a30d1eafc81b Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 29 Oct 2026 17:13:52 +0000 Subject: [PATCH 4/6] Run the FlatList maintainVisibleContentPosition E2E flows in Release only on iOS The 24 FlatList maintainVisibleContentPosition flows are half of the RNTester Maestro suite (~37 minutes per iOS job). Android already runs them against the release build only. On iOS about 88% of their failed attempts over Oct 27-29 were in Debug (133 vs 17). Rename their `android-release-only` tag to `release-only` and exclude it from the Debug job on both platforms. --- .../__tests__/maestro-android-test.js | 12 +++--------- .github/workflows/e2e-android-rntester.yml | 2 +- .github/workflows/e2e-ios-rntester.yml | 6 +++++- .../.maestro/flatlist-append-maintainvisible.yml | 2 +- .../flatlist-complex-mutations-maintainvisible.yml | 2 +- .../flatlist-delete-anchor-maintainvisible.yml | 2 +- .../flatlist-delete-middle-maintainvisible.yml | 2 +- .../.maestro/flatlist-empty-list-maintainvisible.yml | 2 +- .../flatlist-first-prepend-maintainvisible.yml | 2 +- ...atlist-horizontal-add50-reset-maintainvisible.yml | 2 +- .../flatlist-horizontal-inverted-maintainvisible.yml | 2 +- ...t-horizontal-inverted-recycle-maintainvisible.yml | 2 +- .../.maestro/flatlist-horizontal-maintainvisible.yml | 2 +- .../flatlist-horizontal-recycle-maintainvisible.yml | 2 +- .../.maestro/flatlist-inverted-maintainvisible.yml | 2 +- .../flatlist-inverted-recycle-maintainvisible.yml | 2 +- .../rn-tester/.maestro/flatlist-maintainvisible.yml | 4 ++-- .../flatlist-momentum-scroll-maintainvisible.yml | 2 +- .../flatlist-orientation-maintainvisible.yml | 2 +- .../flatlist-prepend-delete-maintainvisible.yml | 2 +- .../flatlist-pull-to-refresh-maintainvisible.yml | 2 +- .../flatlist-rapid-prepends-maintainvisible.yml | 2 +- .../.maestro/flatlist-recycle-maintainvisible.yml | 2 +- .../flatlist-scrolltooffset-maintainvisible.yml | 2 +- .../.maestro/flatlist-throttle-maintainvisible.yml | 2 +- ...variable-height-first-prepend-maintainvisible.yml | 2 +- .../flatlist-variable-height-maintainvisible.yml | 2 +- 27 files changed, 34 insertions(+), 36 deletions(-) diff --git a/.github/workflow-scripts/__tests__/maestro-android-test.js b/.github/workflow-scripts/__tests__/maestro-android-test.js index c07def601b2e..bf353061d97d 100644 --- a/.github/workflow-scripts/__tests__/maestro-android-test.js +++ b/.github/workflow-scripts/__tests__/maestro-android-test.js @@ -47,22 +47,16 @@ describe('Maestro Android runner', () => { }); it('excludes flows with matching tags', () => { - const releaseOnlyFlow = path.join( - temporaryDirectory, - 'android-release-only.yml', - ); + const releaseOnlyFlow = path.join(temporaryDirectory, 'release-only.yml'); const regularFlow = path.join(temporaryDirectory, 'regular.yml'); fs.writeFileSync( releaseOnlyFlow, - 'appId: x\ntags:\n - android-release-only\n---\n- launchApp\n', + 'appId: x\ntags:\n - release-only\n---\n- launchApp\n', ); fs.writeFileSync(regularFlow, 'appId: x\n---\n- launchApp\n'); expect( - filterFlowsByTags( - [releaseOnlyFlow, regularFlow], - ['android-release-only'], - ), + filterFlowsByTags([releaseOnlyFlow, regularFlow], ['release-only']), ).toEqual([regularFlow]); }); diff --git a/.github/workflows/e2e-android-rntester.yml b/.github/workflows/e2e-android-rntester.yml index f3ff2fc0dc98..ef91256ce372 100644 --- a/.github/workflows/e2e-android-rntester.yml +++ b/.github/workflows/e2e-android-rntester.yml @@ -74,7 +74,7 @@ jobs: # These timing-sensitive flows remain covered by the release APK. # Running the debug ARM64 APK through native translation can delay # React commits long enough for Maestro to miss taps and assertions. - exclude-tags: ${{ matrix.flavor == 'debug' && 'android-release-only' || '' }} + exclude-tags: ${{ matrix.flavor == 'debug' && 'release-only' || '' }} - name: Store per-flow test state if: always() uses: ./.github/actions/upload-artifact diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index 4fba45819de8..b904cc43f6a8 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -77,7 +77,11 @@ jobs: device-model: iPhone-17-Pro device-os: iOS-26-2 test-state-path: /tmp/maestro-ios-state/results.json - exclude-tags: android-only + # The `release-only` flows (FlatList maintainVisibleContentPosition) + # are half of the suite and test scroll behavior that doesn't depend + # on the build flavor. Nearly 90% of their failed attempts were in + # Debug, so like on Android they only run against Release. + exclude-tags: ${{ matrix.flavor == 'Debug' && 'android-only,release-only' || 'android-only' }} - name: Store per-flow test state if: always() uses: ./.github/actions/upload-artifact diff --git a/packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml index 47a3097fe686..e477f1ce16fc 100644 --- a/packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml @@ -2,7 +2,7 @@ # Appending items should NOT affect scroll offset (delta ~0) appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml index 72781f1a9a30..188ccf4570ba 100644 --- a/packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml @@ -2,7 +2,7 @@ # Tests prepend + append + delete in sequence appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml index 7e6dd3a5f6cb..162f8c1f812f 100644 --- a/packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml @@ -2,7 +2,7 @@ # When the anchor item (first visible) is deleted, MVCP should select a new anchor appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml index 8ca036af0204..03620b2dd224 100644 --- a/packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml @@ -2,7 +2,7 @@ # When items are deleted from the middle, MVCP should adjust scroll offset appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml index be79c35b7a62..f2c549e53550 100644 --- a/packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml @@ -4,7 +4,7 @@ # correction. appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml index 8305cbc874c4..b7d032a94b9d 100644 --- a/packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml @@ -2,7 +2,7 @@ # Single prepend with fixed-height items: delta should be ~44px (40px height + 4px margin) appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-add50-reset-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-add50-reset-maintainvisible.yml index 04d9c3c20728..44b9e863afaf 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-add50-reset-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-add50-reset-maintainvisible.yml @@ -3,7 +3,7 @@ # reset returns offset to 0. appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-inverted-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-inverted-maintainvisible.yml index c53fcd8db233..bc629d5155d2 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-inverted-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-inverted-maintainvisible.yml @@ -3,7 +3,7 @@ # In inverted mode, prepending adds items to the right end appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-inverted-recycle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-inverted-recycle-maintainvisible.yml index ce9234cae135..2d8e39090ddc 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-inverted-recycle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-inverted-recycle-maintainvisible.yml @@ -2,7 +2,7 @@ # Items are 200px wide + 4px margin = 204px each appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-maintainvisible.yml index 8258d1172304..e625f861ace2 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-maintainvisible.yml @@ -2,7 +2,7 @@ # Horizontal: items are 200px wide, delta should be ~204px (200px + 4px margin) appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-horizontal-recycle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-horizontal-recycle-maintainvisible.yml index c63531017e39..b80c7f975b6d 100644 --- a/packages/rn-tester/.maestro/flatlist-horizontal-recycle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-horizontal-recycle-maintainvisible.yml @@ -2,7 +2,7 @@ # Items are 200px wide + 4px margin = 204px each appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-inverted-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-inverted-maintainvisible.yml index 28f585f95f44..132565d9ccee 100644 --- a/packages/rn-tester/.maestro/flatlist-inverted-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-inverted-maintainvisible.yml @@ -3,7 +3,7 @@ # not logical order. Prepending shifts anchor view frame downward in both modes. appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-inverted-recycle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-inverted-recycle-maintainvisible.yml index ded9fe4c9878..ed0041bdedb5 100644 --- a/packages/rn-tester/.maestro/flatlist-inverted-recycle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-inverted-recycle-maintainvisible.yml @@ -4,7 +4,7 @@ # In inverted mode, items display in reverse order but MVCP delta behavior is the same appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-maintainvisible.yml index d0e05fc6287d..f767288857cc 100644 --- a/packages/rn-tester/.maestro/flatlist-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-maintainvisible.yml @@ -8,7 +8,7 @@ # - Delta should equal height of prepended items (~44px per item with margin) appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait @@ -90,7 +90,7 @@ tags: # Test that user scroll is not interrupted during prepend appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp - assertVisible: 'Components' diff --git a/packages/rn-tester/.maestro/flatlist-momentum-scroll-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-momentum-scroll-maintainvisible.yml index 820c6586db20..cb1baa52e5ac 100644 --- a/packages/rn-tester/.maestro/flatlist-momentum-scroll-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-momentum-scroll-maintainvisible.yml @@ -3,7 +3,7 @@ # post-prepend. MVCP correction runs asynchronously in didMountItems. appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-orientation-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-orientation-maintainvisible.yml index 8ecefd77df59..5f1a5f00c258 100644 --- a/packages/rn-tester/.maestro/flatlist-orientation-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-orientation-maintainvisible.yml @@ -7,7 +7,7 @@ # to work correctly after the change. appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-prepend-delete-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-prepend-delete-maintainvisible.yml index ad116f7a61e4..c42ca0bee0aa 100644 --- a/packages/rn-tester/.maestro/flatlist-prepend-delete-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-prepend-delete-maintainvisible.yml @@ -5,7 +5,7 @@ # the first visible view, but re-ordering edge case is untested. appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-pull-to-refresh-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-pull-to-refresh-maintainvisible.yml index 31445c02d9ac..a4d561a73bc7 100644 --- a/packages/rn-tester/.maestro/flatlist-pull-to-refresh-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-pull-to-refresh-maintainvisible.yml @@ -2,7 +2,7 @@ # Simulates scroll-to-top then prepend (like pull-to-refresh with new items) appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp - setOrientation: portrait diff --git a/packages/rn-tester/.maestro/flatlist-rapid-prepends-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-rapid-prepends-maintainvisible.yml index c7434fb6a12a..ac203a99c831 100644 --- a/packages/rn-tester/.maestro/flatlist-rapid-prepends-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-rapid-prepends-maintainvisible.yml @@ -3,7 +3,7 @@ # promptly, blocking render window updates. All prepends fired without waiting. appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-recycle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-recycle-maintainvisible.yml index 16cb6cf3132b..69072f4afc52 100644 --- a/packages/rn-tester/.maestro/flatlist-recycle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-recycle-maintainvisible.yml @@ -3,7 +3,7 @@ # With windowSize=3, only ~3 pages of items are rendered appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-scrolltooffset-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-scrolltooffset-maintainvisible.yml index db298cc2b2ff..c3736d816762 100644 --- a/packages/rn-tester/.maestro/flatlist-scrolltooffset-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-scrolltooffset-maintainvisible.yml @@ -4,7 +4,7 @@ # of the scrollTo target). appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-throttle-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-throttle-maintainvisible.yml index 5b0e4103f5eb..02ffcf6c982b 100644 --- a/packages/rn-tester/.maestro/flatlist-throttle-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-throttle-maintainvisible.yml @@ -2,7 +2,7 @@ # Throttle affects timing but not final delta: should be ~44px appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-variable-height-first-prepend-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-variable-height-first-prepend-maintainvisible.yml index 385b1d20952b..2c69fd301fcb 100644 --- a/packages/rn-tester/.maestro/flatlist-variable-height-first-prepend-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-variable-height-first-prepend-maintainvisible.yml @@ -2,7 +2,7 @@ # Single prepend with variable height: delta should be 28-112px appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait diff --git a/packages/rn-tester/.maestro/flatlist-variable-height-maintainvisible.yml b/packages/rn-tester/.maestro/flatlist-variable-height-maintainvisible.yml index da034ec4b2b6..26b4d01ed965 100644 --- a/packages/rn-tester/.maestro/flatlist-variable-height-maintainvisible.yml +++ b/packages/rn-tester/.maestro/flatlist-variable-height-maintainvisible.yml @@ -2,7 +2,7 @@ # Delta should be between 28-112px (random height from [30,50,70,90,110]) appId: ${APP_ID} tags: - - android-release-only + - release-only --- - launchApp # Change to portrait From 4aeb269791bc0c3933c6a116ae7ed2e1839617b2 Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 29 Oct 2026 17:16:48 +0000 Subject: [PATCH 5/6] Overlap simulator boot with the rest of the iOS E2E setup The iOS E2E action did all of its setup one step after another before touching the simulator. `xcrun simctl install` then waited ~2 minutes for the simulator to finish booting. - Start booting the simulator as the first step (new `boot` subcommand of maestro-ios.js), so it boots while Maestro, Java, the node modules and Metro are being set up. - Drop the `idb-companion` install. Maestro removed its IDB dependency on iOS in 1.x (see "Removed IDB on iOS" in its changelog) and 2.6.1 doesn't reference it. - Drop the second Node.js setup: both callers (RNTester and the template app) already set up Node.js before using the action. --- .github/actions/maestro-ios/action.yml | 15 +++++++------- .github/workflow-scripts/maestro-ios.js | 26 +++++++++++++++++++++---- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index 8e1a11c9a259..f7f3feac346a 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -37,22 +37,23 @@ inputs: runs: using: composite + # Expects Node.js to be set up by the calling job. steps: + # Booting takes a couple of minutes; it finishes while the next steps run. + - name: Boot simulator + shell: bash + run: | + node .github/workflow-scripts/maestro-ios.js boot \ + "${{ inputs.device-model }}" \ + "${{ inputs.device-os }}" - name: Installing Maestro shell: bash run: export MAESTRO_VERSION=2.6.1; curl -Ls "https://get.maestro.mobile.dev" | bash - - name: Installing Maestro dependencies - shell: bash - run: | - brew tap facebook/fb - brew install facebook/fb/idb-companion - name: Set up JDK 17 uses: actions/setup-java@v5 with: java-version: '17' distribution: 'zulu' - - name: Setup node.js - uses: ./.github/actions/setup-node - name: Run yarn install uses: ./.github/actions/yarn-install - name: Start Metro in Debug diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js index fb2c26836d1b..bbc870dd56b1 100644 --- a/.github/workflow-scripts/maestro-ios.js +++ b/.github/workflow-scripts/maestro-ios.js @@ -25,6 +25,11 @@ node maestro-ios.js 2) { + throw new Error(`Invalid number of arguments.\n${usage}`); + } + launchSimulator(findAvailableSimulator(args[0] || null, args[1] || null)); +} + if (require.main === module) { - main().catch(error => { - console.error(error); - process.exitCode = 1; - }); + const args = process.argv.slice(2); + const command = args[0] === 'boot' ? () => boot(args.slice(1)) : main; + Promise.resolve() + .then(command) + .catch(error => { + console.error(error); + process.exitCode = 1; + }); } module.exports = { From 23b9b776feed2561fca2effa998f223af1dec25c Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 29 Oct 2026 17:17:23 +0000 Subject: [PATCH 6/6] Fail iOS RNTester E2E straight away when the app artifact is missing test_e2e_ios_rntester only checks that test_ios_rntester succeeded, but test_ios_rntester uses continue-on-error, so it reports success even when the leg that uploads RNTesterApp-NewArch- failed. In run 40709683649 the Release app was never uploaded and the E2E job still started (and retried) three times. Download the app as the first step after checkout, and treat a missing app as a non-retryable failure with an error that points to the build job, instead of an incomplete run that triggers the workflow retries. --- .github/workflows/e2e-ios-rntester.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index b904cc43f6a8..9acfe081b7fa 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -52,10 +52,9 @@ jobs: SHOULD_RUN=false fi echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" - - name: Setup Node.js - if: steps.test-state.outputs.should-run == 'true' - uses: ./.github/actions/setup-node + # First, so that a missing app fails the job before any other setup. - name: Download App + id: download-app if: steps.test-state.outputs.should-run == 'true' uses: ./.github/actions/download-artifact with: @@ -64,6 +63,9 @@ jobs: - name: Check downloaded folder content if: steps.test-state.outputs.should-run == 'true' run: ls -lR /tmp/RNTesterBuild + - name: Setup Node.js + if: steps.test-state.outputs.should-run == 'true' + uses: ./.github/actions/setup-node - name: Run E2E Tests id: run-tests if: steps.test-state.outputs.should-run == 'true' @@ -95,14 +97,19 @@ jobs: if: always() shell: bash # passed: every flow passed. - # failed: a flow failed all of its attempts; retrying the job would only - # repeat the same failure. + # failed: a flow failed all of its attempts, or the app to test is + # missing; retrying the job would only repeat the same failure. # incomplete: the job stopped before running every flow (runner or # setup failure); a retry can pick up the remaining flows. run: | STATE=/tmp/maestro-ios-state/results.json if [[ "${{ steps.test-state.outputs.should-run }}" == "false" || "${{ steps.run-tests.outcome }}" == "success" ]]; then RESULT=passed + elif [[ "${{ steps.download-app.outcome }}" == "failure" ]]; then + # test_ios_rntester uses continue-on-error, so it reports success + # even when the build that uploads this app failed. + echo "::error::RNTesterApp-NewArch-${{ matrix.flavor }} is missing. Check the test_ios_rntester (${{ matrix.flavor }}, frameworks: false) job." + RESULT=failed elif [[ -f "$STATE" ]] && jq -e 'any(.flows[]; .status == "failed")' "$STATE" > /dev/null; then RESULT=failed else