diff --git a/package.json b/package.json index 0cec864785150..c3c4aaf49dba3 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "knip:production": "knip --production --preprocessor ./scripts/knip-schemas.ts", "lint:mobile": "node scripts/mobile-native-static-check.ts", "mobile:install": "node scripts/mobile-native-client.ts ensure", + "mobile:audit": "node scripts/mobile-audit.ts", "mobile:behavior": "node scripts/mobile-behavior.ts", "mobile:e2e": "node scripts/mobile-e2e.ts", "scenarios": "vp run --filter @t3tools/scripts scenarios", diff --git a/scripts/mobile-audit.ts b/scripts/mobile-audit.ts new file mode 100644 index 0000000000000..dfb30070de2e2 --- /dev/null +++ b/scripts/mobile-audit.ts @@ -0,0 +1,343 @@ +#!/usr/bin/env node +// @effect-diagnostics nodeBuiltinImport:off - Host-side mobile accessibility audit CLI uses Node subprocess and filesystem APIs directly. + +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeProcess from "node:process"; +import * as NodeURL from "node:url"; + +import { + renderBeforeAfterComparison, + renderSingleAnnotatedImage, +} from "./mobile-audit/annotate.ts"; +import { + buildOverlayBoxes, + diffAuditCaptures, + findingTag, + isPlatformNoise, + renderMarkdownReport, + resolveViewport, + unitForPlatform, + type AuditCapture, + type AuditFinding, + type AuditPlatform, + type RawSnapshotData, + type ScreenDiff, +} from "./mobile-audit/diff.ts"; + +const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), ".."); +const DEFAULT_AGENT_DEVICE_ENTRY = NodePath.join( + REPO_ROOT, + "scripts/node_modules/agent-device/bin/agent-device.mjs", +); + +function usage(): never { + NodeProcess.stderr.write(`Usage: + vp run mobile:audit capture --platform --screen --out [--agent-device ] [-- ] + vp run mobile:audit compare --before --after --out [--before-label ] [--after-label ] [--all] [--fail-on-regression] + vp run mobile:audit report --dir --out +`); + NodeProcess.exit(2); +} + +function parseFlags(argv: ReadonlyArray) { + const separator = argv.indexOf("--"); + const own = separator >= 0 ? argv.slice(0, separator) : argv; + const passThrough = separator >= 0 ? argv.slice(separator + 1) : []; + const flags = new Map(); + for (let i = 0; i < own.length; i++) { + const token = own[i]!; + if (!token.startsWith("--")) continue; + const eq = token.indexOf("="); + if (eq >= 0) { + flags.set(token.slice(2, eq), token.slice(eq + 1)); + } else { + const next = own[i + 1]; + if (next !== undefined && !next.startsWith("--")) { + flags.set(token.slice(2), next); + i += 1; + } else { + flags.set(token.slice(2), true); + } + } + } + return { flags, passThrough }; +} + +function requiredString(flags: Map, name: string): string { + const value = flags.get(name); + if (typeof value !== "string" || value.trim().length === 0) { + NodeProcess.stderr.write(`Missing required flag: --${name}\n`); + usage(); + } + return value; +} + +function execAgentDevice(bin: string | undefined, args: ReadonlyArray): string { + const file = bin ?? NodeProcess.execPath; + const fullArgs = bin ? args : [DEFAULT_AGENT_DEVICE_ENTRY, ...args]; + return NodeChildProcess.execFileSync(file, fullArgs, { + encoding: "utf8", + maxBuffer: 50 * 1024 * 1024, + }); +} + +function parseJsonPayload(raw: string, label: string): T { + const parsed = JSON.parse(raw) as { + readonly success?: boolean; + readonly data?: T; + readonly error?: { readonly message?: string; readonly hint?: string }; + }; + if (parsed.success === false) { + throw new Error( + `${label} failed: ${parsed.error?.message ?? "unknown error"}${parsed.error?.hint ? ` (${parsed.error.hint})` : ""}`, + ); + } + return (parsed.data ?? parsed) as T; +} + +async function runCapture(argv: ReadonlyArray) { + const { flags, passThrough } = parseFlags(argv); + const platformRaw = requiredString(flags, "platform"); + if (platformRaw !== "ios" && platformRaw !== "android") { + throw new Error(`Invalid --platform "${platformRaw}" (expected ios or android).`); + } + const platform: AuditPlatform = platformRaw; + const screen = requiredString(flags, "screen"); + const outRoot = NodePath.resolve(requiredString(flags, "out")); + const agentDevice = + typeof flags.get("agent-device") === "string" + ? (flags.get("agent-device") as string) + : undefined; + + const platformDir = NodePath.join(outRoot, platform); + await NodeFSP.mkdir(platformDir, { recursive: true }); + + const pngPath = NodePath.join(platformDir, `${screen}.png`); + const screenshotArgs = + platform === "ios" + ? ["screenshot", "--pixel-density", "3", pngPath, ...passThrough] + : ["screenshot", pngPath, ...passThrough]; + execAgentDevice(agentDevice, screenshotArgs); + + const auditRaw = execAgentDevice(agentDevice, ["audit", "--json", ...passThrough]); + const auditData = parseJsonPayload<{ + readonly findings?: ReadonlyArray; + readonly guidance?: Readonly>; + readonly limitations?: ReadonlyArray; + }>(auditRaw, "agent-device audit"); + + const snapRaw = execAgentDevice(agentDevice, ["snapshot", "--raw", "--json", ...passThrough]); + const snapData = parseJsonPayload(snapRaw, "agent-device snapshot"); + + const capture: AuditCapture = { + platform, + screen, + viewport: resolveViewport({}, snapData), + pixelDensity: platform === "android" ? (snapData.androidSnapshot?.pixelDensity ?? 1) : 1, + findings: auditData.findings ?? [], + guidance: auditData.guidance, + limitations: auditData.limitations, + }; + + await NodeFSP.writeFile(NodePath.join(platformDir, `${screen}.raw.json`), snapRaw); + await NodeFSP.writeFile( + NodePath.join(platformDir, `${screen}.audit.json`), + `${JSON.stringify(capture, null, 2)}\n`, + ); + + const unit = unitForPlatform(platform); + NodeProcess.stdout.write( + `[${platform}] ${screen}: ${capture.findings.length} finding(s) -> ${pngPath}\n`, + ); + for (const finding of capture.findings) { + NodeProcess.stdout.write( + ` [${finding.kind}] ${finding.role} ${findingTag(finding, capture.pixelDensity, unit)} ${finding.label ? JSON.stringify(finding.label) : "(unnamed)"}\n`, + ); + } +} + +async function listCapturedScreens( + rootDir: string, +): Promise> { + const results: Array<{ readonly platform: AuditPlatform; readonly screen: string }> = []; + for (const platform of ["ios", "android"] as const) { + const dir = NodePath.join(rootDir, platform); + if (!NodeFS.existsSync(dir)) continue; + const files = (await NodeFSP.readdir(dir)).filter((f) => f.endsWith(".audit.json")).sort(); + for (const file of files) { + results.push({ platform, screen: file.replace(/\.audit\.json$/, "") }); + } + } + return results; +} + +async function readCaptureSet(rootDir: string, platform: AuditPlatform, screen: string) { + const base = NodePath.join(rootDir, platform, screen); + const parsed = JSON.parse( + await NodeFSP.readFile(`${base}.audit.json`, "utf8"), + ) as Partial; + const capture: AuditCapture = { + platform: parsed.platform ?? platform, + screen: parsed.screen ?? screen, + viewport: parsed.viewport, + pixelDensity: parsed.pixelDensity ?? 1, + findings: parsed.findings ?? [], + guidance: parsed.guidance, + limitations: parsed.limitations, + }; + const rawPath = `${base}.raw.json`; + const raw = NodeFS.existsSync(rawPath) + ? parseJsonPayload(await NodeFSP.readFile(rawPath, "utf8"), rawPath) + : undefined; + return { + capture, + raw, + pngPath: `${base}.png`, + }; +} + +async function runCompare(argv: ReadonlyArray) { + const { flags } = parseFlags(argv); + const beforeDir = NodePath.resolve(requiredString(flags, "before")); + const afterDir = NodePath.resolve(requiredString(flags, "after")); + const outDir = NodePath.resolve(requiredString(flags, "out")); + const beforeLabel = + typeof flags.get("before-label") === "string" + ? (flags.get("before-label") as string) + : "BEFORE (base)"; + const afterLabel = + typeof flags.get("after-label") === "string" + ? (flags.get("after-label") as string) + : "AFTER (head)"; + const renderAll = flags.get("all") === true; + const failOnRegression = flags.get("fail-on-regression") === true; + + await NodeFSP.mkdir(outDir, { recursive: true }); + const beforeScreens = await listCapturedScreens(beforeDir); + const afterScreens = await listCapturedScreens(afterDir); + const afterSet = new Set(afterScreens.map((s) => `${s.platform}/${s.screen}`)); + const pairs = beforeScreens.filter((s) => afterSet.has(`${s.platform}/${s.screen}`)); + + if (pairs.length === 0) { + throw new Error( + `No matching /.audit.json captures in both ${beforeDir} and ${afterDir}.`, + ); + } + + const diffs: Array = []; + for (const pair of pairs) { + const before = await readCaptureSet(beforeDir, pair.platform, pair.screen); + const after = await readCaptureSet(afterDir, pair.platform, pair.screen); + const diff = diffAuditCaptures({ + before: before.capture, + after: after.capture, + beforeRaw: before.raw, + afterRaw: after.raw, + }); + + const hasDelta = diff.fixed.length > 0 || diff.regressions.length > 0; + let imagePath: string | undefined; + if ( + (hasDelta || renderAll) && + NodeFS.existsSync(before.pngPath) && + NodeFS.existsSync(after.pngPath) + ) { + const { beforeBoxes, afterBoxes } = buildOverlayBoxes( + diff, + after.capture.pixelDensity || 1, + before.capture.pixelDensity || 1, + ); + const outPng = NodePath.join(outDir, `${pair.platform}-${pair.screen}.png`); + await renderBeforeAfterComparison({ + beforePng: before.pngPath, + beforeBoxes, + beforeTitle: `${beforeLabel} · ${diff.beforeCount} finding(s)`, + beforeViewport: resolveViewport(before.capture, before.raw), + afterPng: after.pngPath, + afterBoxes, + afterTitle: `${afterLabel} · ${diff.afterCount} finding(s) (${diff.fixed.length} fixed, ${diff.regressions.length} new)`, + afterBannerBg: diff.regressions.length > 0 ? "#b91c1c" : "#15803d", + afterViewport: resolveViewport(after.capture, after.raw), + outPath: outPng, + }); + imagePath = outPng; + } + + diffs.push({ ...diff, ...(imagePath ? { imagePath } : {}) }); + } + + const markdown = renderMarkdownReport(diffs); + await NodeFSP.writeFile( + NodePath.join(outDir, "diff.json"), + `${JSON.stringify(diffs, null, 2)}\n`, + ); + await NodeFSP.writeFile(NodePath.join(outDir, "report.md"), markdown); + NodeProcess.stdout.write(markdown); + + const totalRegressions = diffs.reduce((n, d) => n + d.regressions.length, 0); + if (failOnRegression && totalRegressions > 0) { + NodeProcess.exit(1); + } +} + +async function runReport(argv: ReadonlyArray) { + const { flags } = parseFlags(argv); + const dir = NodePath.resolve(requiredString(flags, "dir")); + const outDir = NodePath.resolve(requiredString(flags, "out")); + await NodeFSP.mkdir(outDir, { recursive: true }); + + const screens = await listCapturedScreens(dir); + const lines: string[] = [ + "### Mobile accessibility audit report (`agent-device audit`)", + "", + "| Platform | Screen | Findings |", + "| --- | --- | ---: |", + ]; + + for (const entry of screens) { + const { capture, raw, pngPath } = await readCaptureSet(dir, entry.platform, entry.screen); + const viewport = resolveViewport(capture, raw); + const findings = capture.findings.filter( + (f) => !isPlatformNoise(f, capture.platform, viewport), + ); + lines.push(`| ${entry.platform} | \`${entry.screen}\` | ${findings.length} |`); + + if (findings.length > 0 && NodeFS.existsSync(pngPath)) { + const unit = unitForPlatform(entry.platform); + const boxes = findings + .filter((f) => f.rect !== undefined) + .map((f) => ({ + ...f.rect!, + tag: findingTag(f, capture.pixelDensity || 1, unit), + color: "#ef4444", + })); + const outPng = NodePath.join(outDir, `${entry.platform}-${entry.screen}.png`); + await renderSingleAnnotatedImage({ + pngPath, + boxes, + bannerText: `${entry.platform} · ${entry.screen} · ${findings.length} finding(s)`, + bannerBg: "#b91c1c", + viewport, + outPath: outPng, + }); + } + } + + const markdown = `${lines.join("\n")}\n`; + await NodeFSP.writeFile(NodePath.join(outDir, "report.md"), markdown); + NodeProcess.stdout.write(markdown); +} + +const [subcommand, ...rest] = NodeProcess.argv.slice(2); +if (subcommand === "capture") { + await runCapture(rest); +} else if (subcommand === "compare") { + await runCompare(rest); +} else if (subcommand === "report") { + await runReport(rest); +} else { + usage(); +} diff --git a/scripts/mobile-audit/annotate.ts b/scripts/mobile-audit/annotate.ts new file mode 100644 index 0000000000000..359aed53006f7 --- /dev/null +++ b/scripts/mobile-audit/annotate.ts @@ -0,0 +1,181 @@ +// @effect-diagnostics nodeBuiltinImport:off - Host-side image rendering uses Node filesystem APIs directly. + +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import sharp from "sharp"; + +import type { OverlayBox } from "./diff.ts"; + +function escapeXml(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +async function renderAnnotatedPanel(input: { + readonly pngPath: string; + readonly boxes: ReadonlyArray; + readonly bannerText: string; + readonly bannerBg: string; + readonly viewport?: { readonly width: number; readonly height: number } | undefined; + readonly targetWidth: number; +}): Promise<{ readonly buffer: Buffer; readonly width: number; readonly height: number }> { + const meta = await sharp(input.pngPath).metadata(); + const srcWidth = meta.width ?? input.targetWidth; + const srcHeight = meta.height ?? input.targetWidth * 2; + const coordScaleX = input.viewport?.width ? srcWidth / input.viewport.width : 1; + const coordScaleY = input.viewport?.height ? srcHeight / input.viewport.height : 1; + + const scale = input.targetWidth / srcWidth; + const targetHeight = Math.round(srcHeight * scale); + const bannerHeight = 44; + + const rectsSvg = input.boxes + .map((box) => { + const rx = Math.round(box.x * coordScaleX * scale); + const ry = Math.round(box.y * coordScaleY * scale); + const rw = Math.max(8, Math.round(box.width * coordScaleX * scale)); + const rh = Math.max(8, Math.round(box.height * coordScaleY * scale)); + const stroke = box.color; + const fill = stroke === "#16a34a" ? "rgba(22,163,74,0.14)" : "rgba(239,68,68,0.16)"; + const tag = box.tag; + const tagW = Math.min( + input.targetWidth - 12, + Math.max(46, Math.round(tag.length * 7.2 + 14)), + ); + const tagH = 20; + let tx = Math.min(input.targetWidth - tagW - 4, Math.max(4, rx)); + let ty = ry - tagH - 2; + if (ty < 4) ty = Math.min(targetHeight - tagH - 4, ry + rh + 2); + if (box.tagSide === "right") { + tx = Math.min(input.targetWidth - tagW - 4, rx + rw - tagW); + } else if (box.tagSide === "below") { + ty = Math.min(targetHeight - tagH - 4, ry + rh + 2); + } else if (box.tagSide === "inside-right") { + tx = Math.min(input.targetWidth - tagW - 4, rx + rw - tagW - 4); + ty = Math.max(4, ry + Math.round((rh - tagH) / 2)); + } + return [ + ``, + tag + ? `` + + `${escapeXml(tag)}` + : "", + ].join(""); + }) + .join("\n"); + + const overlaySvg = Buffer.from( + `${rectsSvg}`, + ); + + const annotatedImg = await sharp(input.pngPath) + .resize(input.targetWidth, targetHeight) + .composite([{ input: overlaySvg, top: 0, left: 0 }]) + .png() + .toBuffer(); + + const headerSvg = Buffer.from( + `` + + `` + + `${escapeXml(input.bannerText)}` + + ``, + ); + + const combined = await sharp({ + create: { + width: input.targetWidth, + height: targetHeight + bannerHeight, + channels: 4, + background: { r: 24, g: 24, b: 27, alpha: 1 }, + }, + }) + .composite([ + { input: headerSvg, top: 0, left: 0 }, + { input: annotatedImg, top: bannerHeight, left: 0 }, + ]) + .png() + .toBuffer(); + + return { buffer: combined, width: input.targetWidth, height: targetHeight + bannerHeight }; +} + +export async function renderSingleAnnotatedImage(input: { + readonly pngPath: string; + readonly boxes: ReadonlyArray; + readonly bannerText: string; + readonly bannerBg: string; + readonly viewport?: { readonly width: number; readonly height: number } | undefined; + readonly outPath: string; + readonly targetWidth?: number | undefined; +}): Promise { + const panel = await renderAnnotatedPanel({ + pngPath: input.pngPath, + boxes: input.boxes, + bannerText: input.bannerText, + bannerBg: input.bannerBg, + viewport: input.viewport, + targetWidth: input.targetWidth ?? 540, + }); + await NodeFSP.mkdir(NodePath.dirname(input.outPath), { recursive: true }); + await NodeFSP.writeFile(input.outPath, panel.buffer); +} + +export async function renderBeforeAfterComparison(input: { + readonly beforePng: string; + readonly beforeBoxes: ReadonlyArray; + readonly beforeTitle: string; + readonly beforeViewport?: { readonly width: number; readonly height: number } | undefined; + readonly afterPng: string; + readonly afterBoxes: ReadonlyArray; + readonly afterTitle: string; + readonly afterBannerBg?: string | undefined; + readonly afterViewport?: { readonly width: number; readonly height: number } | undefined; + readonly outPath: string; + readonly panelWidth?: number | undefined; +}): Promise<{ readonly width: number; readonly height: number }> { + const targetWidth = input.panelWidth ?? 540; + const [left, right] = await Promise.all([ + renderAnnotatedPanel({ + pngPath: input.beforePng, + boxes: input.beforeBoxes, + bannerText: input.beforeTitle, + bannerBg: "#b91c1c", + viewport: input.beforeViewport, + targetWidth, + }), + renderAnnotatedPanel({ + pngPath: input.afterPng, + boxes: input.afterBoxes, + bannerText: input.afterTitle, + bannerBg: input.afterBannerBg ?? "#15803d", + viewport: input.afterViewport, + targetWidth, + }), + ]); + + const gap = 16; + const pad = 16; + const width = left.width + right.width + gap + pad * 2; + const height = Math.max(left.height, right.height) + pad * 2; + + await NodeFSP.mkdir(NodePath.dirname(input.outPath), { recursive: true }); + await sharp({ + create: { + width, + height, + channels: 4, + background: { r: 24, g: 24, b: 27, alpha: 1 }, + }, + }) + .composite([ + { input: left.buffer, top: pad, left: pad }, + { input: right.buffer, top: pad, left: pad + left.width + gap }, + ]) + .png() + .toFile(input.outPath); + + return { width, height }; +} diff --git a/scripts/mobile-audit/diff.test.ts b/scripts/mobile-audit/diff.test.ts new file mode 100644 index 0000000000000..8acd32a05c1f0 --- /dev/null +++ b/scripts/mobile-audit/diff.test.ts @@ -0,0 +1,208 @@ +// @effect-diagnostics nodeBuiltinImport:off - Image rendering test writes temporary PNGs directly. + +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import sharp from "sharp"; +import { describe, expect, it } from "vite-plus/test"; + +import { renderBeforeAfterComparison } from "./annotate.ts"; +import { + buildOverlayBoxes, + diffAuditCaptures, + isPlatformNoise, + renderMarkdownReport, + type AuditCapture, + type RawSnapshotData, +} from "./diff.ts"; + +describe("mobile-audit diff", () => { + it("filters iOS UISwitch frames, dictation chrome, and scroll-edge clipped rows", () => { + const viewport = { width: 402, height: 874 }; + expect( + isPlatformNoise( + { + ref: "@e1", + kind: "small-target", + role: "switch", + label: "Enable cups", + detail: { unit: "pt", width: 63, height: 28, recommendedSize: 44 }, + }, + "ios", + viewport, + ), + ).toBe(true); + + expect( + isPlatformNoise( + { + ref: "@e2", + kind: "ambiguous-name", + role: "button", + label: "Dictate", + identifier: "dictation", + }, + "ios", + viewport, + ), + ).toBe(true); + + expect( + isPlatformNoise( + { + ref: "@e3", + kind: "small-target", + role: "button", + label: "Bottom row", + rect: { x: 0, y: 860, width: 402, height: 14 }, + }, + "ios", + viewport, + ), + ).toBe(true); + + expect( + isPlatformNoise( + { + ref: "@e4", + kind: "small-target", + role: "text-field", + label: "Address", + rect: { x: 111, y: 431, width: 260, height: 23 }, + detail: { unit: "pt", width: 260, height: 23.3, recommendedSize: 44 }, + }, + "ios", + viewport, + ), + ).toBe(false); + }); + + it("classifies fixed, regressed, and unchanged findings and locates fixed after bounds", () => { + const before: AuditCapture = { + platform: "android", + screen: "connections-new", + viewport: { width: 1080, height: 2424 }, + pixelDensity: 2.625, + findings: [ + { + ref: "@e1", + kind: "small-target", + role: "text-field", + label: "Address", + rect: { x: 293, y: 1055, width: 707, height: 61 }, + detail: { unit: "dp", width: 269.3, height: 23.2, recommendedSize: 48 }, + }, + { + ref: "@e2", + kind: "missing-name", + role: "button", + rect: { x: 11, y: 74, width: 126, height: 126 }, + }, + ], + }; + + const after: AuditCapture = { + platform: "android", + screen: "connections-new", + viewport: { width: 1080, height: 2424 }, + pixelDensity: 2.625, + findings: [ + { + ref: "@e2", + kind: "missing-name", + role: "button", + rect: { x: 11, y: 74, width: 126, height: 126 }, + }, + { + ref: "@e9", + kind: "small-target", + role: "button", + label: "Scan QR code", + rect: { x: 100, y: 400, width: 100, height: 90 }, + detail: { unit: "dp", width: 38.1, height: 34.3, recommendedSize: 48 }, + }, + ], + }; + + const afterRaw: RawSnapshotData = { + viewport: { width: 1080, height: 2424 }, + androidSnapshot: { pixelDensity: 2.625 }, + nodes: [ + { + index: 10, + type: "android.widget.EditText", + label: "Address", + rect: { x: 293, y: 1055, width: 707, height: 126 }, + }, + ], + }; + + const diff = diffAuditCaptures({ before, after, afterRaw }); + expect(diff.fixed).toHaveLength(1); + expect(diff.fixed[0]?.before.label).toBe("Address"); + expect(diff.fixed[0]?.afterMeasurement).toBe("48dp ✓"); + expect(diff.regressions).toHaveLength(1); + expect(diff.regressions[0]?.label).toBe("Scan QR code"); + expect(diff.unchanged).toHaveLength(1); + + const { beforeBoxes, afterBoxes } = buildOverlayBoxes(diff, 2.625); + expect(beforeBoxes).toHaveLength(1); + expect(beforeBoxes[0]?.tag).toBe("23.2dp"); + expect(afterBoxes).toHaveLength(2); + expect(afterBoxes.map((b) => b.tag)).toEqual(["48dp ✓", "NEW: 34.3dp"]); + + const md = renderMarkdownReport([{ ...diff, imagePath: "android-connections-new.png" }]); + expect(md).toContain("**1 fixed**, **1 regressions**"); + expect(md).toContain("![android connections-new before vs after](android-connections-new.png)"); + }); + + it("composites a side-by-side annotated PNG for Retina 3x iOS screenshots", async () => { + const tmp = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mobile-audit-test-")); + try { + const beforePng = NodePath.join(tmp, "before.png"); + const afterPng = NodePath.join(tmp, "after.png"); + const outPng = NodePath.join(tmp, "comparison.png"); + + await sharp({ + create: { + width: 1206, + height: 2622, + channels: 4, + background: { r: 255, g: 255, b: 255, alpha: 1 }, + }, + }) + .png() + .toFile(beforePng); + await sharp({ + create: { + width: 1206, + height: 2622, + channels: 4, + background: { r: 255, g: 255, b: 255, alpha: 1 }, + }, + }) + .png() + .toFile(afterPng); + + const result = await renderBeforeAfterComparison({ + beforePng, + beforeBoxes: [{ x: 111, y: 431, width: 260, height: 23, tag: "23.3pt", color: "#ef4444" }], + beforeTitle: "BEFORE (main)", + beforeViewport: { width: 402, height: 874 }, + afterPng, + afterBoxes: [{ x: 111, y: 431, width: 260, height: 48, tag: "48pt ✓", color: "#16a34a" }], + afterTitle: "AFTER (PR)", + afterViewport: { width: 402, height: 874 }, + outPath: outPng, + }); + + const outMeta = await sharp(outPng).metadata(); + expect(outMeta.format).toBe("png"); + expect(outMeta.width).toBe(result.width); + expect(outMeta.height).toBe(result.height); + expect(outMeta.width).toBe(1128); + } finally { + await NodeFSP.rm(tmp, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/mobile-audit/diff.ts b/scripts/mobile-audit/diff.ts new file mode 100644 index 0000000000000..c64b0138d0801 --- /dev/null +++ b/scripts/mobile-audit/diff.ts @@ -0,0 +1,434 @@ +export type AuditPlatform = "ios" | "android"; + +export interface AuditRect { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface SmallTargetDetail { + readonly unit: string; + readonly width: number; + readonly height: number; + readonly recommendedSize: number; +} + +export interface AmbiguousNameDetail { + readonly name: string; + readonly count: number; +} + +export interface AuditFinding { + readonly ref: string; + readonly kind: "small-target" | "missing-name" | "ambiguous-name" | "covered-control" | string; + readonly severity?: string; + readonly role: string; + readonly label?: string; + readonly identifier?: string; + readonly rect?: AuditRect; + readonly detail?: SmallTargetDetail | AmbiguousNameDetail | Record; +} + +export interface RawSnapshotNode { + readonly index: number; + readonly parentIndex?: number; + readonly ref?: string; + readonly type?: string; + readonly role?: string; + readonly label?: string; + readonly identifier?: string; + readonly contentDescription?: string; + readonly text?: string; + readonly rect?: AuditRect; + readonly enabled?: boolean; + readonly hittable?: boolean; + readonly visibleToUser?: boolean; + readonly keyboardOwned?: boolean; + readonly systemBar?: string; +} + +export interface RawSnapshotData { + readonly nodes: ReadonlyArray; + readonly viewport?: { readonly width: number; readonly height: number }; + readonly androidSnapshot?: { readonly pixelDensity?: number; readonly backend?: string }; +} + +export interface AuditCapture { + readonly platform: AuditPlatform; + readonly screen: string; + readonly viewport?: { readonly width: number; readonly height: number } | undefined; + readonly pixelDensity: number; + readonly findings: ReadonlyArray; + readonly guidance?: Readonly> | undefined; + readonly limitations?: ReadonlyArray | undefined; +} + +export interface FixedFinding { + readonly before: AuditFinding; + readonly afterNode?: RawSnapshotNode | undefined; + readonly afterMeasurement?: string | undefined; +} + +export interface ScreenDiff { + readonly platform: AuditPlatform; + readonly screen: string; + readonly pixelDensity: number; + readonly beforeCount: number; + readonly afterCount: number; + readonly regressions: ReadonlyArray; + readonly fixed: ReadonlyArray; + readonly unchanged: ReadonlyArray; + readonly filteredNoiseCount: number; +} + +export interface OverlayBox { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + readonly tag: string; + readonly color: string; + readonly tagSide?: "above" | "below" | "right" | "inside-right"; +} + +const VIEWPORT_EDGE_CLIP_TOLERANCE = 10; + +export function unitForPlatform(platform: AuditPlatform): "pt" | "dp" { + return platform === "ios" ? "pt" : "dp"; +} + +export function resolveViewport( + capture: Pick, + raw?: RawSnapshotData, +): { readonly width: number; readonly height: number } | undefined { + if (capture.viewport?.width && capture.viewport?.height) return capture.viewport; + if (raw?.viewport?.width && raw?.viewport?.height) return raw.viewport; + const rootRect = raw?.nodes[0]?.rect; + if (rootRect && rootRect.width > 0 && rootRect.height > 0) { + return { width: rootRect.width, height: rootRect.height }; + } + return undefined; +} + +export function formatHeight(rect: AuditRect, density: number, unit: "pt" | "dp"): string { + const h = Math.round((rect.height / density) * 10) / 10; + return `${h}${unit}`; +} + +function isSmallTargetDetail(detail: AuditFinding["detail"]): detail is SmallTargetDetail { + return ( + typeof detail === "object" && + detail !== null && + typeof (detail as SmallTargetDetail).width === "number" && + typeof (detail as SmallTargetDetail).height === "number" + ); +} + +function isAmbiguousNameDetail(detail: AuditFinding["detail"]): detail is AmbiguousNameDetail { + return ( + typeof detail === "object" && + detail !== null && + typeof (detail as AmbiguousNameDetail).name === "string" && + typeof (detail as AmbiguousNameDetail).count === "number" + ); +} + +/** + * Filters known platform-chrome and viewport-clipping artifacts so regressions stay high-signal: + * - iOS 26/27 native UISwitch accessibility frames report 63×28pt. + * - iOS system keyboard dictation/predictive bar buttons when not tagged keyboardOwned. + * - Rows clipped at the top or bottom scroll-viewport edge. + */ +export function isPlatformNoise( + finding: AuditFinding, + platform: AuditPlatform, + viewport?: { readonly width: number; readonly height: number }, +): boolean { + if (platform === "ios") { + if ( + finding.kind === "small-target" && + finding.role === "switch" && + isSmallTargetDetail(finding.detail) && + finding.detail.width >= 50 && + finding.detail.height <= 32 + ) { + return true; + } + if (finding.identifier === "dictation" || finding.label === "Dictate") { + return true; + } + } + if (finding.kind === "small-target" && finding.rect && viewport) { + const bottomEdge = finding.rect.y + finding.rect.height; + const clippedAtBottom = + bottomEdge >= viewport.height - VIEWPORT_EDGE_CLIP_TOLERANCE && + finding.rect.height < viewport.height * 0.04; + const clippedAtTop = finding.rect.y <= 0 && finding.rect.height < viewport.height * 0.04; + if (clippedAtBottom || clippedAtTop) return true; + } + return false; +} + +/** Stable identity key for matching findings across before/after runs. */ +export function findingSignature(finding: AuditFinding): string { + const name = ( + finding.label ?? + (isAmbiguousNameDetail(finding.detail) ? finding.detail.name : undefined) ?? + finding.identifier ?? + "" + ) + .trim() + .toLowerCase(); + if (name) return `${finding.kind}|${finding.role}|${name}`; + const rx = finding.rect ? Math.round(finding.rect.x / 20) * 20 : -1; + const ry = finding.rect ? Math.round(finding.rect.y / 20) * 20 : -1; + return `${finding.kind}|${finding.role}|@${rx},${ry}`; +} + +function roleMatchesNode(role: string, node: RawSnapshotNode): boolean { + const type = (node.type ?? "").toLowerCase(); + switch (role) { + case "button": + return type.includes("button"); + case "text-field": + return type.includes("edittext") || type.includes("textfield") || type.includes("textview"); + case "search-field": + return type.includes("search"); + case "radio": + return type.includes("radio"); + case "switch": + return type.includes("switch"); + case "link": + return type.includes("link"); + default: + return true; + } +} + +function findMatchingAfterNode( + finding: AuditFinding, + afterRaw?: RawSnapshotData, +): RawSnapshotNode | undefined { + if (!afterRaw) return undefined; + const targetLabel = ( + finding.label ?? (isAmbiguousNameDetail(finding.detail) ? finding.detail.name : undefined) + )?.trim(); + const candidates = afterRaw.nodes.filter((node) => { + if (!node.rect || node.rect.width <= 0 || node.rect.height <= 0) return false; + if (!roleMatchesNode(finding.role, node)) return false; + if (targetLabel) { + return ( + node.label?.trim() === targetLabel || + node.contentDescription?.trim() === targetLabel || + node.text?.trim() === targetLabel + ); + } + if (finding.rect) { + return ( + Math.abs(node.rect.x - finding.rect.x) <= 40 && Math.abs(node.rect.y - finding.rect.y) <= 40 + ); + } + return false; + }); + if (candidates.length === 0) return undefined; + if (!finding.rect || candidates.length === 1) return candidates[0]; + return candidates.reduce((best, curr) => { + const bestDist = + Math.abs(best.rect!.x - finding.rect!.x) + Math.abs(best.rect!.y - finding.rect!.y); + const currDist = + Math.abs(curr.rect!.x - finding.rect!.x) + Math.abs(curr.rect!.y - finding.rect!.y); + return currDist < bestDist ? curr : best; + }); +} + +export function diffAuditCaptures(input: { + readonly before: AuditCapture; + readonly after: AuditCapture; + readonly beforeRaw?: RawSnapshotData | undefined; + readonly afterRaw?: RawSnapshotData | undefined; +}): ScreenDiff { + const beforeVp = resolveViewport(input.before, input.beforeRaw); + const afterVp = resolveViewport(input.after, input.afterRaw); + + let filteredNoiseCount = 0; + const beforeFindings = input.before.findings.filter((f) => { + if (isPlatformNoise(f, input.before.platform, beforeVp)) { + filteredNoiseCount += 1; + return false; + } + return true; + }); + const afterFindings = input.after.findings.filter((f) => { + if (isPlatformNoise(f, input.after.platform, afterVp)) { + filteredNoiseCount += 1; + return false; + } + return true; + }); + + // Multiset diff by signature so fixing 1 of 2 duplicate buttons still shows up accurately. + const remainingAfter = new Map(); + for (const f of afterFindings) { + const key = findingSignature(f); + const list = remainingAfter.get(key) ?? []; + list.push(f); + remainingAfter.set(key, list); + } + + const unchanged: AuditFinding[] = []; + const fixed: FixedFinding[] = []; + const unit = unitForPlatform(input.after.platform); + const density = input.after.pixelDensity || 1; + + for (const bf of beforeFindings) { + const key = findingSignature(bf); + const bucket = remainingAfter.get(key); + if (bucket && bucket.length > 0) { + bucket.shift(); + unchanged.push(bf); + } else { + const afterNode = findMatchingAfterNode(bf, input.afterRaw); + const afterMeasurement = + bf.kind === "ambiguous-name" + ? "1 btn ✓" + : afterNode?.rect + ? `${formatHeight(afterNode.rect, density, unit)} ✓` + : "fixed ✓"; + fixed.push({ before: bf, afterNode, afterMeasurement }); + } + } + + const regressions: AuditFinding[] = []; + for (const bucket of remainingAfter.values()) { + regressions.push(...bucket); + } + + return { + platform: input.after.platform, + screen: input.after.screen, + pixelDensity: density, + beforeCount: beforeFindings.length, + afterCount: afterFindings.length, + regressions, + fixed, + unchanged, + filteredNoiseCount, + }; +} + +function dedupeBoxes(boxes: ReadonlyArray): OverlayBox[] { + const seen = new Set(); + return boxes.filter((box) => { + const key = `${Math.round(box.x / 4)}:${Math.round(box.y / 4)}:${Math.round(box.width / 4)}:${Math.round(box.height / 4)}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +export function findingTag(finding: AuditFinding, density: number, unit: "pt" | "dp"): string { + if (finding.kind === "small-target") { + if (isSmallTargetDetail(finding.detail)) { + return `${finding.detail.height}${finding.detail.unit}`; + } + if (finding.rect) return formatHeight(finding.rect, density, unit); + return "small"; + } + if (finding.kind === "ambiguous-name") { + const count = isAmbiguousNameDetail(finding.detail) ? finding.detail.count : 2; + return `×${count} dup`; + } + if (finding.kind === "missing-name") return "no label"; + if (finding.kind === "covered-control") return "covered"; + return finding.kind; +} + +export function buildOverlayBoxes( + diff: ScreenDiff, + density: number, + beforeDensity = density, +): { readonly beforeBoxes: OverlayBox[]; readonly afterBoxes: OverlayBox[] } { + const unit = unitForPlatform(diff.platform); + const beforeBoxes: OverlayBox[] = []; + const afterBoxes: OverlayBox[] = []; + + for (const item of diff.fixed) { + if (item.before.rect) { + beforeBoxes.push({ + ...item.before.rect, + tag: findingTag(item.before, beforeDensity, unit), + color: "#ef4444", + }); + } + if (item.afterNode?.rect) { + afterBoxes.push({ + ...item.afterNode.rect, + tag: item.afterMeasurement ?? "fixed ✓", + color: "#16a34a", + }); + } + } + + for (const reg of diff.regressions) { + if (reg.rect) { + afterBoxes.push({ + ...reg.rect, + tag: `NEW: ${findingTag(reg, density, unit)}`, + color: "#ef4444", + }); + } + } + + return { + beforeBoxes: dedupeBoxes(beforeBoxes), + afterBoxes: dedupeBoxes(afterBoxes), + }; +} + +export function renderMarkdownReport( + diffs: ReadonlyArray, +): string { + const totalRegressions = diffs.reduce((n, d) => n + d.regressions.length, 0); + const totalFixed = diffs.reduce((n, d) => n + d.fixed.length, 0); + const lines: string[] = [ + "### Mobile accessibility audit (`agent-device audit`)", + "", + `Summary: **${totalFixed} fixed**, **${totalRegressions} regressions** across ${diffs.length} screen(s).`, + "", + "| Platform | Screen | Before | After | Fixed | Regressions |", + "| --- | --- | ---: | ---: | ---: | ---: |", + ]; + + for (const d of diffs) { + lines.push( + `| ${d.platform} | \`${d.screen}\` | ${d.beforeCount} | ${d.afterCount} | ${d.fixed.length} | ${d.regressions.length} |`, + ); + } + + for (const d of diffs) { + if (d.fixed.length === 0 && d.regressions.length === 0) continue; + lines.push("", `#### ${d.platform} · \`${d.screen}\``); + if (d.regressions.length > 0) { + lines.push("", "**Regressions:**"); + for (const r of d.regressions) { + lines.push( + `- \`${r.kind}\` (${r.role}) ${r.label ? `\`${r.label}\`` : "(unnamed)"} — ${findingTag(r, d.pixelDensity || 1, unitForPlatform(d.platform))}`, + ); + } + } + if (d.fixed.length > 0) { + lines.push("", "**Fixed:**"); + for (const f of d.fixed) { + lines.push( + `- \`${f.before.kind}\` (${f.before.role}) ${f.before.label ? `\`${f.before.label}\`` : "(unnamed)"} → ${f.afterMeasurement ?? "fixed"}`, + ); + } + } + if (d.imagePath) { + lines.push("", `![${d.platform} ${d.screen} before vs after](${d.imagePath})`); + } + } + + return `${lines.join("\n")}\n`; +}