diff --git a/packages/runtime-playground/src/artifact-bundle-builder.ts b/packages/runtime-playground/src/artifact-bundle-builder.ts index b6701909..ab3d9dc5 100644 --- a/packages/runtime-playground/src/artifact-bundle-builder.ts +++ b/packages/runtime-playground/src/artifact-bundle-builder.ts @@ -266,7 +266,10 @@ export class ArtifactBundleBuilder { ...source.browserManifestFiles(), ...source.observationManifestFiles(), ...commandArtifactManifestFiles(source.artifactRoot, source.commands), - ...phpunitCompletedResults.map(({ path }) => artifactManifestFile(join(source.artifactRoot, path), "test-results", "text/plain")), + ...phpunitCompletedResults.flatMap(({ path, junitPath }) => [ + artifactManifestFile(join(source.artifactRoot, path), "test-results", "text/plain"), + ...(junitPath ? [artifactManifestFile(join(source.artifactRoot, junitPath), "test-results", "application/junit+xml")] : []), + ]), ...source.pluginCheckManifestFiles(), ...source.themeCheckManifestFiles(), ...runtimeSnapshotFiles, diff --git a/packages/runtime-playground/src/phpunit-command-handlers.ts b/packages/runtime-playground/src/phpunit-command-handlers.ts index 24ce9ef9..2f8e7649 100644 --- a/packages/runtime-playground/src/phpunit-command-handlers.ts +++ b/packages/runtime-playground/src/phpunit-command-handlers.ts @@ -31,6 +31,8 @@ export interface PhpunitRunCodeOptions { * die() or exit(). */ resultFile?: string + /** Private JUnit report captured after PHPUnit finishes, including failures. */ + junitFile?: string } export type PhpunitMultisitePreinstallCodeOptions = Pick @@ -328,6 +330,8 @@ function phpunitArgsPhp(functionName: string, logFunction: string): string { function ${functionName}(array $argv) { $arguments = array('colors' => 'never', 'testdox' => true, 'verbose' => false, 'cacheResult' => false, 'cacheResultFile' => ${functionName}_private_cache_result_file(), 'extensions' => array()); + global $junit_file; + $arguments['junitLogfile'] = $junit_file; $selected_testsuites = array(); $args = array_slice($argv, 1); for ($i = 0; $i < count($args); $i++) { @@ -524,6 +528,8 @@ $plugin_slug = ${JSON.stringify(options.pluginSlug)}; $plugin_path = '/wordpress/wp-content/plugins/' . $plugin_slug; $runtime_cwd = ${JSON.stringify(options.cwd || `/wordpress/wp-content/plugins/${options.pluginSlug}`)}; $result_file = ${JSON.stringify(options.resultFile ?? PLUGIN_PHPUNIT_RESULT_FILE)}; +$junit_file = ${JSON.stringify(options.junitFile ?? "/tmp/wp-codebox-phpunit-junit.xml")}; +@unlink($junit_file); $current_stage = 'preboot'; $pg_stage_output_buffering = false; $autoload_file = ${JSON.stringify(options.autoloadFile)}; diff --git a/packages/runtime-playground/src/phpunit-test-results.ts b/packages/runtime-playground/src/phpunit-test-results.ts index 7f7d3f7c..9fb7f67b 100644 --- a/packages/runtime-playground/src/phpunit-test-results.ts +++ b/packages/runtime-playground/src/phpunit-test-results.ts @@ -1,4 +1,4 @@ -import { readdir, readFile } from "node:fs/promises" +import { readdir, readFile, stat } from "node:fs/promises" import { join, relative } from "node:path" import type { ArtifactTestResults, ExecutionResult } from "@automattic/wp-codebox-core" @@ -19,6 +19,7 @@ export interface PhpunitCompletedResult { export interface CapturedPhpunitCompletedResult { path: string result: PhpunitCompletedResult + junitPath?: string } export function parsePhpunitCompletedResult(log: string): PhpunitCompletedResult | undefined { @@ -82,7 +83,11 @@ export async function readCapturedPhpunitCompletedResults(artifactRoot: string): for (const path of paths) { const result = parsePhpunitCompletedResult(await readFile(path, "utf8")) - if (result) results.push({ path: relative(artifactRoot, path), result }) + if (result) { + const relativePath = relative(artifactRoot, path) + const junitPath = phpunitJunitPath(relativePath) + results.push({ path: relativePath, result, ...(await artifactFileExists(join(artifactRoot, junitPath)) ? { junitPath } : {}) }) + } } return results } @@ -92,12 +97,13 @@ export function buildPhpunitTestResults(commands: ExecutionResult[], completed: const rawLogReferences = [ { path: "commands.jsonl", kind: "commands-jsonl" }, { path: "logs/commands.log", kind: "commands-log" }, - ...completed.flatMap(({ path }) => [ + ...completed.flatMap(({ path, junitPath }) => [ { path, kind: "phpunit-result" }, { path: phpunitDiagnosticPath(path), kind: "phpunit-output" }, + ...(junitPath ? [{ path: junitPath, kind: "phpunit-junit" }] : []), ]), ] - const suites: ArtifactTestResults["suites"] = completed.map(({ path, result }, index) => ({ + const suites: ArtifactTestResults["suites"] = completed.map(({ path, result, junitPath }, index) => ({ name: completed.length === 1 ? "wordpress.phpunit" : `wordpress.phpunit:${index + 1}`, status: result.status, tests: result.total, @@ -105,7 +111,7 @@ export function buildPhpunitTestResults(commands: ExecutionResult[], completed: failed: result.failed, skipped: result.skipped, unknown: 0, - rawLogReferences: [{ path, kind: "phpunit-result" }, { path: phpunitDiagnosticPath(path), kind: "phpunit-output" }, { path: "logs/commands.log", kind: "commands-log" }], + rawLogReferences: [{ path, kind: "phpunit-result" }, { path: phpunitDiagnosticPath(path), kind: "phpunit-output" }, ...(junitPath ? [{ path: junitPath, kind: "phpunit-junit" }] : []), { path: "logs/commands.log", kind: "commands-log" }], })) for (let index = completed.length; index < phpunitCommands.length; index += 1) { suites.push({ @@ -178,3 +184,15 @@ function completedResult(total: number, assertions: number, failures: number, er export function phpunitDiagnosticPath(completedResultPath: string): string { return completedResultPath.replace(/\.wp-codebox-result\.txt$/, ".pg-test-result.txt") } + +export function phpunitJunitPath(completedResultPath: string): string { + return completedResultPath.replace(/\.wp-codebox-result\.txt$/, ".wp-codebox-junit.xml") +} + +async function artifactFileExists(path: string): Promise { + try { + return (await stat(path)).isFile() + } catch { + return false + } +} diff --git a/packages/runtime-playground/src/playground-command-errors.ts b/packages/runtime-playground/src/playground-command-errors.ts index 8ff678e1..12a65dd4 100644 --- a/packages/runtime-playground/src/playground-command-errors.ts +++ b/packages/runtime-playground/src/playground-command-errors.ts @@ -121,7 +121,7 @@ export function completedPlaygroundCommandError(command: string, cause: unknown) return new PlaygroundCommandError(command, { cause: commandCause, exitCode: playgroundNonzeroExitCode(commandCause) ?? playgroundNonzeroExitCode(cause) ?? 1, - errors: diagnostics.length > 0 ? diagnostics.join("\n") : errorMessage(commandCause), + errors: diagnostics.length > 0 ? diagnostics.join("\n") : truncateDiagnostic(redactDiagnosticText(errorMessage(commandCause))) ?? "Playground command failed without diagnostics.", text: "", }) } diff --git a/packages/runtime-playground/src/runtime-diagnostics.ts b/packages/runtime-playground/src/runtime-diagnostics.ts index 73e32040..f9b9b636 100644 --- a/packages/runtime-playground/src/runtime-diagnostics.ts +++ b/packages/runtime-playground/src/runtime-diagnostics.ts @@ -1,5 +1,5 @@ import { basename, dirname, join } from "node:path" -import { captureArtifactFile, type MountSpec } from "@automattic/wp-codebox-core" +import { captureArtifactFile, DEFAULT_CAPTURED_ARTIFACT_MAX_BYTES, type CapturedArtifactFile, type MountSpec } from "@automattic/wp-codebox-core" import type { PlaygroundCliServer } from "./preview-server.js" import { extractPhpunitFailureMessage } from "./playground-command-errors.js" import { PHPUNIT_COMPLETED_RESULT_PREFIX, parsePhpunitCompletedResult, type PhpunitCompletedResult } from "./phpunit-test-results.js" @@ -17,6 +17,45 @@ export async function persistPluginPhpunitResult(server: PlaygroundCliServer, vf await persistPhpunitResult(server, vfsPath, join(artifactRoot, "files", "phpunit", ...(namespace ? [namespace] : []), ".pg-test-result.txt")) } +export async function clearPluginPhpunitJunitResult(server: PlaygroundCliServer, vfsPath: string): Promise { + if (!server.playground?.unlink) return + + try { + await server.playground.unlink(vfsPath) + } catch { + // A missing report is expected before the first PHPUnit command. + } +} + +export async function persistPluginPhpunitJunitResult(server: PlaygroundCliServer, vfsPath: string, artifactRoot: string, namespace?: string): Promise { + const artifactPath = join("files", "phpunit", ...(namespace ? [namespace] : []), ".wp-codebox-junit.xml") + if (!server.playground.readFileAsText) { + return { schema: "wp-codebox/captured-artifact-file/v1", status: "failed", path: artifactPath, reason: "runtime-read-unavailable" } + } + + try { + // Playground exposes only whole-file reads; this caps retained host output, not VFS read allocation. + const contents = await server.playground.readFileAsText(vfsPath) + return await captureArtifactFile({ + root: join(artifactRoot, "files", "phpunit", ...(namespace ? [namespace] : [])), + path: ".wp-codebox-junit.xml", + kind: "test-results", + contentType: "application/junit+xml", + contents, + maxBytes: 8 * DEFAULT_CAPTURED_ARTIFACT_MAX_BYTES, + redaction: { policy: "applied", sensitive: true, reason: "PHPUnit JUnit failure details are redacted and bounded before private artifact capture." }, + provenance: { source: "wordpress-playground", operation: "persist-phpunit-junit-result", id: vfsPath }, + }) + } catch { + return { schema: "wp-codebox/captured-artifact-file/v1", status: "failed", path: artifactPath, reason: "runtime-read-failed" } + } +} + +export function phpunitJunitCaptureDiagnostic(capture: CapturedArtifactFile): string | undefined { + if (capture.status === "captured") return undefined + return `JUnit artifact capture ${capture.status}: ${capture.reason ?? "unknown"}.` +} + export async function persistPluginPhpunitCompletedResult(artifactRoot: string, result: PhpunitCompletedResult, namespace?: string): Promise { const hostPath = join(artifactRoot, "files", "phpunit", ...(namespace ? [namespace] : []), ".wp-codebox-result.txt") const { passed: _passed, failed: _failed, ...aggregate } = result diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index 9a225a5f..a6a1c9af 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -53,7 +53,7 @@ import { import { bootstrapAbilityPhpCode, bootstrapPhpCode, phpCodeFromArgs, splitLeadingStrictTypesDeclare } from "./php-bootstrap.js" import { assertPlaygroundResponseOk, attachPlaygroundDiagnostics, completedPlaygroundCommandError, playgroundCommandDiagnosticText, type PlaygroundRunResponse } from "./playground-command-errors.js" import type { PlaygroundCliServer } from "./preview-server.js" -import { persistCorePhpunitResult, persistPluginPhpunitCompletedResult, persistPluginPhpunitResult, persistVfsDiagnosticFileToHost, readCorePhpunitDiagnostic, readPluginPhpunitCompletedResult, readPluginPhpunitDiagnostic, readPluginPhpunitDiscoveryResult } from "./runtime-diagnostics.js" +import { clearPluginPhpunitJunitResult, persistCorePhpunitResult, persistPluginPhpunitCompletedResult, persistPluginPhpunitJunitResult, persistPluginPhpunitResult, persistVfsDiagnosticFileToHost, phpunitJunitCaptureDiagnostic, readCorePhpunitDiagnostic, readPluginPhpunitCompletedResult, readPluginPhpunitDiagnostic, readPluginPhpunitDiscoveryResult } from "./runtime-diagnostics.js" import { phpunitExecutionSemantics, requiresManagedMultisitePreinstall } from "./phpunit-command-semantics.js" import { parsePhpunitOutput } from "./phpunit-test-results.js" import { runRuntimeExternalHttpLoad, waitForRuntimePreviewReady, type RuntimeExternalHttpLoadResult } from "./external-http-load.js" @@ -939,6 +939,9 @@ export async function runPhpunitCommand({ spec: ExecutionSpec }): Promise { const args = spec.args ?? [] + const processIdentity = boundedProcessIdentity(spec.processIdentity) + const junitFile = processIdentity ? `/tmp/wp-codebox-phpunit-junit-${processIdentity}.xml` : "/tmp/wp-codebox-phpunit-junit.xml" + await clearPluginPhpunitJunitResult(server, junitFile) const phpunitXmlArg = argValue(args, "phpunit-xml") const explicitCode = argValue(args, "code") || argValue(args, "code-file") const pluginSlug = argValue(args, "plugin-slug")?.trim() || "" @@ -962,7 +965,6 @@ export async function runPhpunitCommand({ } const autoloadFile = argValue(args, "autoload-file")?.trim() || (bootstrapMode === "project" ? "" : "/wp-codebox-vendor/autoload.php") const autoloadFileRole = argValue(args, "autoload-file-role")?.trim() === "harness" ? "harness" : undefined - const processIdentity = boundedProcessIdentity(spec.processIdentity) const managedMultisitePreinstalled = !explicitCode && !discoveryOnly && requiresManagedMultisitePreinstall(args, runtimeSpec) const resultFile = processIdentity ? `/tmp/wp-codebox-phpunit-result-${processIdentity}.txt` : PLUGIN_PHPUNIT_RESULT_FILE const diagnosticHostFile = `/wordpress/wp-content/plugins/${pluginSlug}/.pg-test-result${processIdentity ? `-${processIdentity}` : ""}.txt` @@ -1000,6 +1002,7 @@ export async function runPhpunitCommand({ databaseType, managedMultisitePreinstalled, resultFile, + junitFile, }) if (!explicitCode && !pluginSlug) { throw new Error("wordpress.phpunit requires plugin-slug= when code/code-file is not provided") @@ -1020,13 +1023,17 @@ export async function runPhpunitCommand({ } response = await runPlaygroundCommand("wordpress.phpunit", server, { code: bootstrapPhpCode(runtimeSpec, code, bootstrapArgs, undefined, resultFile) }) } catch (error) { + // Capture before the runtime teardown can reset the command VFS. + const junitCapture = await persistPluginPhpunitJunitResult(server, junitFile, artifactRoot, processIdentity) await persistPluginPhpunitResult(server, resultFile, artifactRoot, processIdentity) await persistVfsDiagnosticFileToHost(server, resultFile, diagnosticHostFile, mounts) await captureCommandResultPaths(server, spec, artifactRoot) const completed = await readPluginPhpunitCompletedResult(server, resultFile) ?? parsePhpunitOutput(playgroundCommandDiagnosticText(error)) if (completed) { await persistPluginPhpunitCompletedResult(artifactRoot, completed, processIdentity) - throw attachPlaygroundDiagnostics(completedPlaygroundCommandError("wordpress.phpunit", error), "wordpress.phpunit completed result", `completed ${completed.status}; total ${completed.total}; failed ${completed.failed}; skipped ${completed.skipped}; failures ${completed.failures}; errors ${completed.errors}`) + const completedError = attachPlaygroundDiagnostics(completedPlaygroundCommandError("wordpress.phpunit", error), "wordpress.phpunit completed result", `completed ${completed.status}; total ${completed.total}; failed ${completed.failed}; skipped ${completed.skipped}; failures ${completed.failures}; errors ${completed.errors}`) + const diagnostic = phpunitJunitCaptureDiagnostic(junitCapture) + throw diagnostic ? attachPlaygroundDiagnostics(completedError, "wordpress.phpunit JUnit artifact", diagnostic) : completedError } const structured = await readPluginPhpunitDiagnostic(server, resultFile) if (structured) { @@ -1035,6 +1042,7 @@ export async function runPhpunitCommand({ throw error } + const junitCapture = await persistPluginPhpunitJunitResult(server, junitFile, artifactRoot, processIdentity) await persistPluginPhpunitResult(server, resultFile, artifactRoot, processIdentity) await persistVfsDiagnosticFileToHost(server, resultFile, diagnosticHostFile, mounts) const structured = await readPluginPhpunitDiagnostic(server, resultFile) @@ -1046,10 +1054,12 @@ export async function runPhpunitCommand({ try { assertPlaygroundResponseOk("wordpress.phpunit", response) } catch (error) { + const diagnostic = phpunitJunitCaptureDiagnostic(junitCapture) if (structured) { - throw attachPlaygroundDiagnostics(error, "wordpress.phpunit structured diagnostics", structured) + const structuredError = attachPlaygroundDiagnostics(error, "wordpress.phpunit structured diagnostics", structured) + throw diagnostic ? attachPlaygroundDiagnostics(structuredError, "wordpress.phpunit JUnit artifact", diagnostic) : structuredError } - throw error + throw diagnostic ? attachPlaygroundDiagnostics(error, "wordpress.phpunit JUnit artifact", diagnostic) : error } if (discoveryOnly) { diff --git a/tests/phpunit-runtime-failure-diagnostics.test.ts b/tests/phpunit-runtime-failure-diagnostics.test.ts index 47fa4c3b..5e262242 100644 --- a/tests/phpunit-runtime-failure-diagnostics.test.ts +++ b/tests/phpunit-runtime-failure-diagnostics.test.ts @@ -32,6 +32,7 @@ await assert.rejects( (error: Error) => { assert.match(error.message, /wordpress\.phpunit failed with exit code 1/) assert.match(error.message, /wordpress\.phpunit structured diagnostics/) + assert.match(error.message, /wordpress\.phpunit JUnit artifact/) assert.match(error.message, /Bootstrap failed with token: \[redacted\]/) assert.match(error.message, /\[diagnostic truncated\]/) assert.doesNotMatch(error.message, new RegExp(secret)) @@ -39,6 +40,20 @@ await assert.rejects( }, ) +const clearedJunitFiles: string[] = [] +await assert.rejects( + () => runPhpunitCommand({ + artifactRoot, + mounts: [], + runPlaygroundCommand: async () => ({ exitCode: 0, errors: "", text: "" }), + runtimeSpec: wordpressRuntimeSpec({ commands: ["wordpress.phpunit"] }), + server: { playground: { unlink: async (path: string) => { clearedJunitFiles.push(path) } } } as never, + spec: { command: "wordpress.phpunit", args: ["discovery-only=true", "code=assertTrue(true); } public function test_fails(): void { $this->assertTrue(false); } public function test_errors(): void { throw new RuntimeException('fixture error'); } public function test_skips(): void { $this->markTestSkipped('fixture skipped'); } }\n") + await writeFile(join(plugin, "tests", "ReadonlyCacheTest.php"), "assertTrue(true); } public static function failure_cases(): array { return array_map(static fn(int $index): array => ['fixture-case-' . $index, $index], range(1, 49)); } /** @dataProvider failure_cases */ public function test_large_failure_output(string $identity): void { $this->fail($identity . ' Authorization: Bearer literal-credential-for-redaction ' . str_repeat('response-cap-detail ', 256)); } public function test_error_output(): void { throw new RuntimeException('fixture-error-50 credential=literal-credential-for-redaction ' . str_repeat('response-cap-detail ', 256)); } public function test_fixture_skip_51(): void { $this->markTestSkipped('fixture-skip-51'); } }\n") const failedOutput = await runFailedRecipe() assert.equal(failedOutput.success, false) assert.match(failedOutput.error?.message ?? "", /failureClassification=runtime-command-failure/) assert.doesNotMatch(failedOutput.error?.message ?? "", /crashed before producing a structured response/) + assert(Buffer.byteLength(failedOutput.error?.message ?? "") <= 50_000, "failed command response must remain bounded") + assert.doesNotMatch(failedOutput.error?.message ?? "", /literal-credential-for-redaction/) const failedRuntime = JSON.parse(await readFile(join(failingArtifactsPath, "latest-runtime.json"), "utf8")) as { paths?: { runtimeDirectory?: string } } const failingEvidence = await readTestResults(failingArtifactsPath, failedRuntime.paths?.runtimeDirectory) assert.equal(failingEvidence.status, "failed") - assert.deepEqual(failingEvidence.summary, { total: 4, passed: 1, failed: 2, skipped: 1, unknown: 0 }) + assert.deepEqual(failingEvidence.summary, { total: 52, passed: 1, failed: 50, skipped: 1, unknown: 0 }) assert(failingEvidence.rawLogReferences.some((reference) => reference.path === "files/phpunit/.pg-test-result.txt")) + assert(failingEvidence.rawLogReferences.some((reference) => reference.path === "files/phpunit/.wp-codebox-junit.xml")) + const junitPath = join(failingArtifactsPath, failedRuntime.paths?.runtimeDirectory ?? "", "files/phpunit/.wp-codebox-junit.xml") + const junit = await readFile(junitPath, "utf8") + assert(Buffer.byteLength(junit) > 20_000, "JUnit artifact must retain failure detail beyond the bounded command response") + for (let index = 1; index <= 49; index += 1) { + assert.match(junit, new RegExp(`fixture-case-${index}`), `JUnit artifact must retain fixture-case-${index}`) + } + assert.match(junit, /fixture-error-50/) + assert.match(junit, /name="test_fixture_skip_51" class="ReadonlyCacheTest"/) + assert.doesNotMatch(junit, /literal-credential-for-redaction/) + const manifest = JSON.parse(await readFile(join(failingArtifactsPath, failedRuntime.paths?.runtimeDirectory ?? "", "manifest.json"), "utf8")) as { files?: Array<{ path?: string, contentType?: string, sha256?: { value?: string } }> } + const junitManifest = manifest.files?.find((file) => file.path === "files/phpunit/.wp-codebox-junit.xml") + assert.equal(junitManifest?.contentType, "application/junit+xml") + assert.match(junitManifest?.sha256?.value ?? "", /^[a-f0-9]{64}$/) } finally { await rm(root, { recursive: true, force: true }) }