diff --git a/.github/workflows/tests_bidi.yml b/.github/workflows/tests_bidi.yml index 342207e7ab362..1e68db76efad9 100644 --- a/.github/workflows/tests_bidi.yml +++ b/.github/workflows/tests_bidi.yml @@ -24,10 +24,8 @@ env: jobs: test_bidi: name: BiDi - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} runs-on: ubuntu-24.04 permissions: - id-token: write # This is required for OIDC login (azure/login) to succeed contents: read # This is required for actions/checkout to succeed strategy: fail-fast: false @@ -76,19 +74,53 @@ jobs: with: job_name: ${{ matrix.channel }} - - name: Azure Login - if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} - uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3.0.2 - with: - client-id: ${{ secrets.AZURE_BLOB_REPORTS_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_BLOB_REPORTS_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_BLOB_REPORTS_SUBSCRIPTION_ID }} + - name: Add report to the job summary + if: ${{ !cancelled() && hashFiles('test-results/report.md') != '' }} + run: cat test-results/report.md >> "$GITHUB_STEP_SUMMARY" - - name: Upload report.csv to Azure - if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} + publish_reports: + name: Publish reports + needs: test_bidi + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.repository == 'microsoft/playwright' }} + runs-on: ubuntu-24.04 + permissions: + contents: write # This is required to push to the bidi-reports branch + steps: + - name: Download csv reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: csv-report-* + path: csv-reports + - name: Publish the reports to the bidi-reports branch run: | - REPORT_DIR='bidi-reports' - azcopy cp "./test-results/report.csv" "https://mspwblobreport.blob.core.windows.net/\$web/$REPORT_DIR/${{ matrix.channel }}.csv" - echo "Report url: https://mspwblobreport.z1.web.core.windows.net/$REPORT_DIR/${{ matrix.channel }}.csv" + git init -q "$RUNNER_TEMP/reports" + cd "$RUNNER_TEMP/reports" + git remote add origin "https://x-access-token:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" + # Continue the branch history, or start it on the very first run. An + # initial commit with no fetched parent is what makes the branch orphan. + git fetch --depth=1 origin bidi-reports && git reset --hard FETCH_HEAD || true + echo 'Nightly BiDi test reports, published by .github/workflows/tests_bidi.yml.' > README.md + channels=() + for dir in "$GITHUB_WORKSPACE"/csv-reports/csv-report-*/; do + channel=${dir%/} + channel=${channel##*/csv-report-} + cp "$dir/report.csv" "$channel.csv" + channels+=("$channel") + done + git add -A + if git diff --cached --quiet; then + echo "No changes since the last run." + exit 0 + fi + git commit -q -m "bidi: results for ${GITHUB_SHA:0:9}" + git push origin HEAD:bidi-reports + for channel in "${channels[@]}"; do + echo "Report url: https://github.com/$GITHUB_REPOSITORY/blob/bidi-reports/$channel.csv" + echo "Raw report url: https://raw.githubusercontent.com/$GITHUB_REPOSITORY/bidi-reports/$channel.csv" + done env: - AZCOPY_AUTO_LOGIN_TYPE: AZCLI + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GIT_AUTHOR_NAME: microsoft-playwright-automation[bot] + GIT_AUTHOR_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: microsoft-playwright-automation[bot] + GIT_COMMITTER_EMAIL: 203992400+microsoft-playwright-automation[bot]@users.noreply.github.com diff --git a/tests/bidi/csvReporter.ts b/tests/bidi/csvReporter.ts index b84ca9e81d11f..d5ef17cdf0131 100644 --- a/tests/bidi/csvReporter.ts +++ b/tests/bidi/csvReporter.ts @@ -24,9 +24,12 @@ import path from 'path'; type ReporterOptions = { outputFile?: string, + markdownFile?: string, configDir: string, }; +const header = ['Test Name', 'Expected Status', 'Status', 'Error Message']; + class CsvReporter implements Reporter { private _suite: Suite; private _options: ReporterOptions; @@ -41,7 +44,7 @@ class CsvReporter implements Reporter { } onEnd(result: FullResult) { - const rows = [['Test Name', 'Expected Status', 'Status', 'Error Message']]; + const rows: string[][] = []; for (const project of this._suite.suites) { for (const file of project.suites) { for (const test of file.allTests()) { @@ -51,7 +54,7 @@ class CsvReporter implements Reporter { continue; const row = []; const [, , , ...titles] = test.titlePath(); - row.push(csvEscape(`${file.title} › ${titles.join(' › ')}`)); + row.push(`${file.title} › ${titles.join(' › ')}`); row.push(test.expectedStatus); row.push(test.outcome()); if (fixme) { @@ -60,11 +63,11 @@ class CsvReporter implements Reporter { const result = test.results.find(r => r.error); if (result) { const errorMessage = stripAnsi(result.error?.message.replace(/\s+/g, ' ').trim().substring(0, 1024) ?? ''); - row.push(csvEscape(errorMessage)); + row.push(errorMessage); } else { const fail = test.annotations.find(a => a.type === 'fail'); if (fail) - row.push(csvEscape(`Should have failed: ${fail.description}`)); + row.push(`Should have failed: ${fail.description}`); else row.push(''); } @@ -73,11 +76,16 @@ class CsvReporter implements Reporter { } } } - const csv = rows.map(r => r.join(',')).join('\n'); const reportFile = path.resolve(this._options.configDir, this._options.outputFile || 'test-results.csv'); + const markdownFile = this._options.markdownFile && path.resolve(this._options.configDir, this._options.markdownFile); this._pendingWrite = (async () => { await fs.promises.mkdir(path.dirname(reportFile), { recursive: true }); + const csv = [header, ...rows].map(r => r.map(csvEscape).join(',')).join('\n'); await fs.promises.writeFile(reportFile, csv); + if (markdownFile) { + await fs.promises.mkdir(path.dirname(markdownFile), { recursive: true }); + await fs.promises.writeFile(markdownFile, markdownTable(rows)); + } })(); } @@ -96,4 +104,24 @@ function csvEscape(str) { return str; } +// GitHub job summaries are capped at 1MiB, so keep the rendered table bounded. +const maxMarkdownRows = 500; + +function markdownTable(rows: string[][]): string { + const lines = [ + `### ${rows.length} failing tests`, + '', + `| ${header.join(' | ')} |`, + `| ${header.map(() => '---').join(' | ')} |`, + ...rows.slice(0, maxMarkdownRows).map(row => `| ${row.map(markdownEscape).join(' | ')} |`), + ]; + if (rows.length > maxMarkdownRows) + lines.push('', `_...and ${rows.length - maxMarkdownRows} more, see the csv report._`); + return lines.join('\n') + '\n'; +} + +function markdownEscape(str: string): string { + return str.replace(/[\\|`<>]/g, c => '\\' + c); +} + export default CsvReporter; diff --git a/tests/bidi/playwright.config.ts b/tests/bidi/playwright.config.ts index afa43752e13da..83996f2bde224 100644 --- a/tests/bidi/playwright.config.ts +++ b/tests/bidi/playwright.config.ts @@ -46,7 +46,7 @@ const reporters = () => { hasDebugOutput ? ['list'] : ['dot'], ['blob'], ['../config/parquetReporter.ts'], - ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv') }], + ['./csvReporter', { outputFile: path.join(outputDir, 'report.csv'), markdownFile: path.join(outputDir, 'report.md') }], ['./expectationReporter', { rebase: false }], ] : [ ['html', { open: 'on-failure' }],