From 603cd761b2e6b2e9ab9e93b5acd5c9663e68b394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9D=BC=ED=98=81?= Date: Sun, 4 Oct 2026 06:55:40 +0000 Subject: [PATCH 1/2] =?UTF-8?q?chore(ci):=20merge=20queue=20=EB=B0=A9?= =?UTF-8?q?=EC=B6=9C=EC=9D=B4=20GMD=20=EC=9D=B8=ED=94=84=EB=9D=BC=20?= =?UTF-8?q?=EC=8B=A4=ED=8C=A8=EB=BF=90=EC=9D=B4=EB=A9=B4=20=EA=B0=99?= =?UTF-8?q?=EC=9D=80=20head=20=EB=8B=B9=20=ED=95=9C=20=EB=B2=88=20?= =?UTF-8?q?=EC=9E=AC=ED=88=AC=EC=9E=85=ED=95=9C=EB=8B=A4=20(#2250)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scripts/handle-merge-queue-dequeue.mjs | 64 ++++++++++++++++--- .../handle-merge-queue-dequeue.test.mjs | 54 ++++++++++++++++ 2 files changed, 108 insertions(+), 10 deletions(-) diff --git a/.github/scripts/handle-merge-queue-dequeue.mjs b/.github/scripts/handle-merge-queue-dequeue.mjs index b1498e601..706d1c04e 100644 --- a/.github/scripts/handle-merge-queue-dequeue.mjs +++ b/.github/scripts/handle-merge-queue-dequeue.mjs @@ -10,6 +10,11 @@ // 한 번만 재투입한다. 0902 에 감시 루프가 90초마다 맹목 재투입해 30분을 헛돈 사고(#1638·#1639)가 // «한 번까지» 규칙의 근거다. 횟수는 이 스크립트가 남기는 마커 코멘트로 센다. // +// 예외는 증명된 인프라 실패다(#2250). GMD 분류기(classify-android-managed-device-failure.mjs)가 테스트가 돌기 전에 +// 죽었다고 판정하면 그 run 에 재시도 마커 아티팩트를 남긴다. 실패 job 이 전부 그런 job 이면 PR 코드가 실행되기 +// 전에 끝난 것이므로 조용한 방출과 같이 같은 head 당 한 번 재투입한다. 머지 큐는 실패한 필수 체크로 바로 방출하므로 +// job 재실행(android-managed-device-retry.yml)으로는 복구되지 않고 재투입만이 수단이다. +// // 판정 순서는 «실패 job 이 있는가» 가 먼저다. payload 의 reason 문자열은 GitHub 이 열거값을 문서화하지 // 않아 정본으로 삼지 않는다 — 실패 job 은 데이터로 확인하고, reason 은 «사람이 뺐다·이미 머지됐다» 를 // 거르는 데만 쓴다. 그 밖의 reason 은 전부 조용한 방출로 본다. @@ -31,6 +36,11 @@ export const REQUEUE_MARKER_PREFIX = "`; } @@ -75,14 +105,16 @@ export function countRequeuesForHead(comments, headSha) { /** * 판정표. - * 1. 실패 job 이 있다 → comment-failure (재투입 없음) + * 1. 실패 job 이 있고 그중 하나라도 인프라로 증명되지 않았다 → comment-failure (재투입 없음) * 2. reason 이 MANUAL·ALREADY_MERGED·MERGE → none - * 3. 같은 head 에서 이미 재투입했다 → comment-give-up - * 4. 그 밖 → requeue + * 3. 실패 job 이 전부 증명된 인프라 실패다 → 같은 head 첫 번째면 requeue, 이미 재투입했으면 comment-failure + * 4. 같은 head 에서 이미 재투입했다 → comment-give-up + * 5. 그 밖 → requeue */ export function decide({ reason, failedJobs, requeueCount }) { - if (failedJobs.length > 0) return "comment-failure"; + if (failedJobs.length > 0 && !onlyInfrastructureFailures(failedJobs)) return "comment-failure"; if (NO_ACTION_REASONS.has(String(reason ?? "").toUpperCase())) return "none"; + if (failedJobs.length > 0) return requeueCount >= 1 ? "comment-failure" : "requeue"; if (requeueCount >= 1) return "comment-give-up"; return "requeue"; } @@ -98,7 +130,7 @@ export function manualEnqueueCommand({ repository, number, stack = false }) { } export function renderFailureComment({ reason, headSha, failedJobs, repository, number, stack = false }) { - const lines = failedJobs.map((job) => `- [${job.runName} / ${job.jobName}](${job.jobUrl}) — ${job.conclusion}`); + const lines = failedJobs.map((job) => `- [${job.runName} / ${job.jobName}](${job.jobUrl}) — ${job.conclusion}${job.infrastructure ? " (인프라 실패로 분류됨)" : ""}`); return [ "### merge queue 방출 — merge group CI 실패", "", @@ -111,12 +143,16 @@ export function renderFailureComment({ reason, headSha, failedJobs, repository, ].join("\n"); } -export function renderRequeueComment({ reason, headSha }) { +export function renderRequeueComment({ reason, headSha, failedJobs = [] }) { + const why = failedJobs.length > 0 + ? "merge group 실패 job 이 전부 테스트 전 인프라 실패로 분류됨 → 한 번 재투입했다." + : "merge group 실패 job 없음 → 조용한 방출로 보고 한 번 재투입했다."; return [ requeueMarker(headSha), "### merge queue 방출 — 재투입했다", "", - `사유 \`${reason}\` · head \`${shortSha(headSha)}\` · merge group 실패 job 없음 → 조용한 방출로 보고 한 번 재투입했다.`, + `사유 \`${reason}\` · head \`${shortSha(headSha)}\` · ${why}`, + ...failedJobs.map((job) => `- [${job.runName} / ${job.jobName}](${job.jobUrl}) — ${job.conclusion}`), "같은 head 에서 또 방출되면 재투입하지 않고 여기에 남긴다.", ].join("\n"); } @@ -223,11 +259,19 @@ export async function fetchQueueFailedJobs(api, repository, { baseRef, number, s .filter((run) => !since || typeof run.created_at !== "string" || run.created_at >= since) .slice(0, QUEUE_RUNS_TO_INSPECT); const jobsByRunId = new Map(); + const artifactsByRunId = new Map(); for (const run of runs) { const jobs = await api(`/repos/${repository}/actions/runs/${run.id}/jobs?per_page=100`); jobsByRunId.set(run.id, jobs.jobs ?? []); + // 마커를 남길 수 있는 run 만 아티팩트를 본다 — 실패한 GMD job 이 없으면 볼 필요가 없다. + const hasFailedManagedDeviceJob = (jobs.jobs ?? []).some((job) => + FAILED_JOB_CONCLUSIONS.has(job.conclusion) && [...MANAGED_DEVICE_RETRY_JOBS.values()].includes(job.name)); + if (hasFailedManagedDeviceJob) { + const artifacts = await api(`/repos/${repository}/actions/runs/${run.id}/artifacts?per_page=100`); + artifactsByRunId.set(run.id, artifacts.artifacts ?? []); + } } - return collectFailedJobs({ runs, jobsByRunId, baseRef, number, since }); + return collectFailedJobs({ runs, jobsByRunId, artifactsByRunId, baseRef, number, since }); } export async function fetchComments(api, repository, number) { @@ -327,7 +371,7 @@ export async function handleDequeue({ await postComment(api, repository, number, renderStackUpperComment({ reason, headSha, repository, number })); } else if (action === "requeue") { // 마커를 먼저 남긴다 — 재투입 뒤 코멘트가 실패하면 다음 방출에서 두 번째 재투입이 나간다. - await postComment(api, repository, number, renderRequeueComment({ reason, headSha })); + await postComment(api, repository, number, renderRequeueComment({ reason, headSha, failedJobs })); if (stack) { const enqueueResult = await enqueueStacked(api, repository, number, headSha, { sleep }); if (enqueueResult.unresolved) { diff --git a/.github/scripts/handle-merge-queue-dequeue.test.mjs b/.github/scripts/handle-merge-queue-dequeue.test.mjs index d498d0bac..670a79d9a 100644 --- a/.github/scripts/handle-merge-queue-dequeue.test.mjs +++ b/.github/scripts/handle-merge-queue-dequeue.test.mjs @@ -7,6 +7,7 @@ import { collectFailedJobs, countRequeuesForHead, decide, + provenInfrastructureJobNames, enqueueStacked, handleDequeue, isStackBottom, @@ -47,6 +48,24 @@ test("실패 job 없는 방출은 같은 head 에서 한 번만 재투입하고, assert.equal(decide({ reason: "ROLL_BACK", failedJobs: [], requeueCount: 3 }), "comment-give-up"); }); +test("실패 job 이 전부 증명된 인프라 실패면 같은 head 에서 한 번 재투입하고, 두 번째는 링크만 남긴다", () => { + const infra = [{ jobName: "Pixel 2 API 30 androidTest", conclusion: "failure", infrastructure: true }]; + assert.equal(decide({ reason: "CI_FAILURE", failedJobs: infra, requeueCount: 0 }), "requeue"); + assert.equal(decide({ reason: "CI_FAILURE", failedJobs: infra, requeueCount: 1 }), "comment-failure"); + assert.equal(decide({ reason: "MANUAL", failedJobs: infra, requeueCount: 0 }), "none"); + const mixed = [...infra, { jobName: "Run Unit Tests", conclusion: "failure", infrastructure: false }]; + assert.equal(decide({ reason: "CI_FAILURE", failedJobs: mixed, requeueCount: 0 }), "comment-failure"); +}); + +test("재시도 마커는 같은 run·attempt 이름만 인정한다 — 다른 run·만료·모르는 device 는 증명이 아니다", () => { + const run = { id: 37178469871, run_attempt: 1 }; + assert.deepEqual([...provenInfrastructureJobNames(run, [{ name: "android-managed-device-retry-api30-37178469871-1" }])], ["Pixel 2 API 30 androidTest"]); + assert.equal(provenInfrastructureJobNames(run, [{ name: "android-managed-device-retry-api30-37178469871-2" }]).size, 0); + assert.equal(provenInfrastructureJobNames(run, [{ name: "android-managed-device-retry-api30-1-1" }]).size, 0); + assert.equal(provenInfrastructureJobNames(run, [{ name: "android-managed-device-retry-api30-37178469871-1", expired: true }]).size, 0); + assert.equal(provenInfrastructureJobNames(run, [{ name: "android-managed-device-retry-api26-37178469871-1" }]).size, 0); +}); + // ---- 실패 job 수집 ---- test("이 PR 의 큐 브랜치 run 에서 failure·timed_out job 만 모은다 — 다른 PR 의 run, cancelled job 은 뺀다", () => { @@ -165,6 +184,7 @@ function fakeApi({ liveAfter = live, runs = [], jobs = {}, + artifacts = {}, comments = [], mergeAsync = { status: "pending", @@ -199,6 +219,8 @@ function fakeApi({ return respond(mergeAsyncPolls[Math.min(pollReads++, mergeAsyncPolls.length - 1)]); } if (apiPath.startsWith(`/repos/${REPO}/actions/runs?`)) return { workflow_runs: runs }; + const artifactsMatch = /\/actions\/runs\/(\d+)\/artifacts/.exec(apiPath); + if (artifactsMatch) return { artifacts: artifacts[artifactsMatch[1]] ?? [] }; const jobsMatch = /\/actions\/runs\/(\d+)\/jobs/.exec(apiPath); if (jobsMatch) return { jobs: jobs[jobsMatch[1]] ?? [] }; if (apiPath.includes("/comments") && (options.method ?? "GET") === "GET") return comments; @@ -524,6 +546,38 @@ test("CI 실패: 실패 job 링크 코멘트만, 재투입 없음", async () => assert.ok(!api.calls.some((call) => call.body?.query?.includes("enqueuePullRequest"))); }); +test("GMD 인프라 실패만으로 방출되면 마커 코멘트 뒤 재투입한다 — 마커 없는 같은 job 은 실패 코멘트만", async () => { + const runs = [{ id: 9, run_attempt: 1, name: "Android Managed Device Test", html_url: "https://x/runs/9", head_branch: queueBranchPrefix("develop", 1509) + "abc", created_at: "2026-09-04T08:05:00Z" }]; + const jobs = { 9: [ + { name: "Pixel 2 API 30 androidTest", conclusion: "failure", html_url: "https://x/jobs/91" }, + { name: "Pixel 2 API 34 accessibility smoke", conclusion: "success", html_url: "https://x/jobs/92" }, + ] }; + const proven = fakeApi({ live: openLive, runs, jobs, artifacts: { 9: [{ name: "android-managed-device-retry-api30-9-1" }] } }); + const result = await handleDequeue({ api: proven, repository: REPO, number: 1509, reason: "CI_FAILURE", logger: silent }); + assert.equal(result.action, "requeue"); + assert.equal(enqueueMutations(proven).length, 1); + const comment = commentPosts(proven)[0].body.body; + assert.ok(comment.startsWith(requeueMarker(HEAD))); + assert.match(comment, /인프라 실패로 분류됨/); + assert.match(comment, /https:\/\/x\/jobs\/91/); + const writes = proven.calls.filter((call) => call.method === "POST"); + assert.ok(writes.findIndex((call) => call.apiPath.endsWith("/comments")) < writes.findIndex((call) => call.body?.query?.includes("enqueuePullRequest"))); + + const unproven = fakeApi({ live: openLive, runs, jobs }); + const failed = await handleDequeue({ api: unproven, repository: REPO, number: 1509, reason: "CI_FAILURE", logger: silent }); + assert.equal(failed.action, "comment-failure"); + assert.equal(enqueueMutations(unproven).length, 0); + + const second = fakeApi({ + live: openLive, runs, jobs, + artifacts: { 9: [{ name: "android-managed-device-retry-api30-9-1" }] }, + comments: [{ body: renderRequeueComment({ reason: "CI_FAILURE", headSha: HEAD }) }], + }); + const again = await handleDequeue({ api: second, repository: REPO, number: 1509, reason: "CI_FAILURE", logger: silent }); + assert.equal(again.action, "comment-failure"); + assert.equal(enqueueMutations(second).length, 0); +}); + test("dry-run 은 판정만 하고 아무것도 쓰지 않는다 — 스택 PR 도 같다", async () => { for (const live of [openLive, stackedLive, stackUpperLive]) { const api = fakeApi({ live }); From 15a48cdee7c597158d21f7a6e46f57d31e52d5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9D=BC=ED=98=81?= Date: Sun, 4 Oct 2026 06:59:39 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix(ci):=20API30=20=EC=A0=84=EC=B2=B4=20and?= =?UTF-8?q?roidTest=20=EB=A0=88=EC=9D=B8=EC=97=90=EB=8F=84=20APK=20?= =?UTF-8?q?=ED=8C=A8=ED=82=A4=EC=A7=95=EC=9A=A9=204Gi=20=ED=9E=99=EC=9D=84?= =?UTF-8?q?=20=EC=A4=80=EB=8B=A4=20(#2250)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/android-managed-device.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android-managed-device.yml b/.github/workflows/android-managed-device.yml index aeea5b05c..333d6fd48 100644 --- a/.github/workflows/android-managed-device.yml +++ b/.github/workflows/android-managed-device.yml @@ -647,8 +647,11 @@ jobs: local test_arguments=() jvm_arguments=() # #1824 merge-group API34: cold APK packaging exhausted the 2Gi heap # in ApkFlinger. The 15Gi runner still had 7Gi available, so reserve - # 4Gi for this lane only and preserve the existing UTF-8 JVM flag. - if [[ "$DEVICE_TASK" == "pixel2Api34DebugAndroidTest" ]]; then + # 4Gi and preserve the existing UTF-8 JVM flag. + # #2250 merge-group API30 (#1837 큐): 같은 ApkFlinger OOM 이 :app:packageDebug 에서 + # 시스템 이미지 설치와 겹쳐 났다. 전체 androidTest 레인도 같은 4Gi 를 쓴다. + if [[ "$DEVICE_TASK" == "pixel2Api34DebugAndroidTest" || + "$DEVICE_TASK" == "pixel2Api30DebugAndroidTest" ]]; then jvm_arguments+=("-Dorg.gradle.jvmargs=-Xmx4096m -Dfile.encoding=UTF-8") fi if [[ -n "$selectors" ]]; then