Retain complete PHPUnit failure artifacts

packages/runtime-playground/src/artifact-bundle-builder.ts

@@ -266,7 +266,10 @@ export class ArtifactBundleBuilder {
       ...source.browserManifestFiles(),
       ...source.observationManifestFiles(),
       ...commandArtifactManifestFiles(source.artifactRoot, source.commands),
-      ...phpunitCompletedResults.map(({ path }) => artifactManifestFile(join(source.artifactRoot, path), "test-results", "text/plain")),
+      ...phpunitCompletedResults.flatMap(({ path, junitPath }) => [
+        artifactManifestFile(join(source.artifactRoot, path), "test-results", "text/plain"),
+        ...(junitPath ? [artifactManifestFile(join(source.artifactRoot, junitPath), "test-results", "application/junit+xml")] : []),
+      ]),
       ...source.pluginCheckManifestFiles(),
       ...source.themeCheckManifestFiles(),
       ...runtimeSnapshotFiles,

packages/runtime-playground/src/phpunit-command-handlers.ts

@@ -31,6 +31,8 @@ export interface PhpunitRunCodeOptions {
    * die() or exit().
    */
   resultFile?: string
+  /** Private JUnit report captured after PHPUnit finishes, including failures. */
+  junitFile?: string
 }
 
 export type PhpunitMultisitePreinstallCodeOptions = Pick<PhpunitRunCodeOptions, "testsDir" | "env" | "wpConfigDefines" | "databaseType" | "resultFile">
@@ -328,6 +330,8 @@ function phpunitArgsPhp(functionName: string, logFunction: string): string {
 
 function ${functionName}(array $argv) {
     $arguments = array('colors' => 'never', 'testdox' => true, 'verbose' => false, 'cacheResult' => false, 'cacheResultFile' => ${functionName}_private_cache_result_file(), 'extensions' => array());
+    global $junit_file;
+    $arguments['junitLogfile'] = $junit_file;
     $selected_testsuites = array();
     $args = array_slice($argv, 1);
     for ($i = 0; $i < count($args); $i++) {
@@ -524,6 +528,8 @@ $plugin_slug = ${JSON.stringify(options.pluginSlug)};
 $plugin_path = '/wordpress/wp-content/plugins/' . $plugin_slug;
 $runtime_cwd = ${JSON.stringify(options.cwd || `/wordpress/wp-content/plugins/${options.pluginSlug}`)};
 $result_file = ${JSON.stringify(options.resultFile ?? PLUGIN_PHPUNIT_RESULT_FILE)};
+$junit_file = ${JSON.stringify(options.junitFile ?? "/tmp/wp-codebox-phpunit-junit.xml")};
+@unlink($junit_file);
 $current_stage = 'preboot';
 $pg_stage_output_buffering = false;
 $autoload_file = ${JSON.stringify(options.autoloadFile)};

packages/runtime-playground/src/phpunit-test-results.ts

@@ -1,4 +1,4 @@
-import { readdir, readFile } from "node:fs/promises"
+import { readdir, readFile, stat } from "node:fs/promises"
 import { join, relative } from "node:path"
 import type { ArtifactTestResults, ExecutionResult } from "@automattic/wp-codebox-core"
 
@@ -19,6 +19,7 @@ export interface PhpunitCompletedResult {
 export interface CapturedPhpunitCompletedResult {
   path: string
   result: PhpunitCompletedResult
+  junitPath?: string
 }
 
 export function parsePhpunitCompletedResult(log: string): PhpunitCompletedResult | undefined {
@@ -82,7 +83,11 @@ export async function readCapturedPhpunitCompletedResults(artifactRoot: string):
 
   for (const path of paths) {
     const result = parsePhpunitCompletedResult(await readFile(path, "utf8"))
-    if (result) results.push({ path: relative(artifactRoot, path), result })
+    if (result) {
+      const relativePath = relative(artifactRoot, path)
+      const junitPath = phpunitJunitPath(relativePath)
+      results.push({ path: relativePath, result, ...(await artifactFileExists(join(artifactRoot, junitPath)) ? { junitPath } : {}) })
+    }
   }
   return results
 }
@@ -92,20 +97,21 @@ export function buildPhpunitTestResults(commands: ExecutionResult[], completed:
   const rawLogReferences = [
     { path: "commands.jsonl", kind: "commands-jsonl" },
     { path: "logs/commands.log", kind: "commands-log" },
-    ...completed.flatMap(({ path }) => [
+    ...completed.flatMap(({ path, junitPath }) => [
       { path, kind: "phpunit-result" },
       { path: phpunitDiagnosticPath(path), kind: "phpunit-output" },
+      ...(junitPath ? [{ path: junitPath, kind: "phpunit-junit" }] : []),
     ]),
   ]
-  const suites: ArtifactTestResults["suites"] = completed.map(({ path, result }, index) => ({
+  const suites: ArtifactTestResults["suites"] = completed.map(({ path, result, junitPath }, index) => ({
     name: completed.length === 1 ? "wordpress.phpunit" : `wordpress.phpunit:${index + 1}`,
     status: result.status,
     tests: result.total,
     passed: result.passed,
     failed: result.failed,
     skipped: result.skipped,
     unknown: 0,
-    rawLogReferences: [{ path, kind: "phpunit-result" }, { path: phpunitDiagnosticPath(path), kind: "phpunit-output" }, { path: "logs/commands.log", kind: "commands-log" }],
+    rawLogReferences: [{ path, kind: "phpunit-result" }, { path: phpunitDiagnosticPath(path), kind: "phpunit-output" }, ...(junitPath ? [{ path: junitPath, kind: "phpunit-junit" }] : []), { path: "logs/commands.log", kind: "commands-log" }],
   }))
   for (let index = completed.length; index < phpunitCommands.length; index += 1) {
     suites.push({
@@ -178,3 +184,15 @@ function completedResult(total: number, assertions: number, failures: number, er
 export function phpunitDiagnosticPath(completedResultPath: string): string {
   return completedResultPath.replace(/\.wp-codebox-result\.txt$/, ".pg-test-result.txt")
 }
+
+export function phpunitJunitPath(completedResultPath: string): string {
+  return completedResultPath.replace(/\.wp-codebox-result\.txt$/, ".wp-codebox-junit.xml")
+}
+
+async function artifactFileExists(path: string): Promise<boolean> {
+  try {
+    return (await stat(path)).isFile()
+  } catch {
+    return false
+  }
+}

packages/runtime-playground/src/playground-command-errors.ts

@@ -121,7 +121,7 @@ export function completedPlaygroundCommandError(command: string, cause: unknown)
   return new PlaygroundCommandError(command, {
     cause: commandCause,
     exitCode: playgroundNonzeroExitCode(commandCause) ?? playgroundNonzeroExitCode(cause) ?? 1,
-    errors: diagnostics.length > 0 ? diagnostics.join("\n") : errorMessage(commandCause),
+    errors: diagnostics.length > 0 ? diagnostics.join("\n") : truncateDiagnostic(redactDiagnosticText(errorMessage(commandCause))) ?? "Playground command failed without diagnostics.",
     text: "",
   })
 }

packages/runtime-playground/src/runtime-diagnostics.ts

@@ -1,5 +1,5 @@
 import { basename, dirname, join } from "node:path"
-import { captureArtifactFile, type MountSpec } from "@automattic/wp-codebox-core"
+import { captureArtifactFile, DEFAULT_CAPTURED_ARTIFACT_MAX_BYTES, type CapturedArtifactFile, type MountSpec } from "@automattic/wp-codebox-core"
 import type { PlaygroundCliServer } from "./preview-server.js"
 import { extractPhpunitFailureMessage } from "./playground-command-errors.js"
 import { PHPUNIT_COMPLETED_RESULT_PREFIX, parsePhpunitCompletedResult, type PhpunitCompletedResult } from "./phpunit-test-results.js"
@@ -17,6 +17,45 @@ export async function persistPluginPhpunitResult(server: PlaygroundCliServer, vf
   await persistPhpunitResult(server, vfsPath, join(artifactRoot, "files", "phpunit", ...(namespace ? [namespace] : []), ".pg-test-result.txt"))
 }
 
+export async function clearPluginPhpunitJunitResult(server: PlaygroundCliServer, vfsPath: string): Promise<void> {
+  if (!server.playground?.unlink) return
+
+  try {
+    await server.playground.unlink(vfsPath)
+  } catch {
+    // A missing report is expected before the first PHPUnit command.
+  }
+}
+
+export async function persistPluginPhpunitJunitResult(server: PlaygroundCliServer, vfsPath: string, artifactRoot: string, namespace?: string): Promise<CapturedArtifactFile> {
+  const artifactPath = join("files", "phpunit", ...(namespace ? [namespace] : []), ".wp-codebox-junit.xml")
+  if (!server.playground.readFileAsText) {
+    return { schema: "wp-codebox/captured-artifact-file/v1", status: "failed", path: artifactPath, reason: "runtime-read-unavailable" }
+  }
+
+  try {
+    // Playground exposes only whole-file reads; this caps retained host output, not VFS read allocation.
+    const contents = await server.playground.readFileAsText(vfsPath)
+    return await captureArtifactFile({
+      root: join(artifactRoot, "files", "phpunit", ...(namespace ? [namespace] : [])),
+      path: ".wp-codebox-junit.xml",
+      kind: "test-results",
+      contentType: "application/junit+xml",
+      contents,
+      maxBytes: 8 * DEFAULT_CAPTURED_ARTIFACT_MAX_BYTES,
+      redaction: { policy: "applied", sensitive: true, reason: "PHPUnit JUnit failure details are redacted and bounded before private artifact capture." },
+      provenance: { source: "wordpress-playground", operation: "persist-phpunit-junit-result", id: vfsPath },
+    })
+  } catch {
+    return { schema: "wp-codebox/captured-artifact-file/v1", status: "failed", path: artifactPath, reason: "runtime-read-failed" }
+  }
+}
+
+export function phpunitJunitCaptureDiagnostic(capture: CapturedArtifactFile): string | undefined {
+  if (capture.status === "captured") return undefined
+  return `JUnit artifact capture ${capture.status}: ${capture.reason ?? "unknown"}.`
+}
+
 export async function persistPluginPhpunitCompletedResult(artifactRoot: string, result: PhpunitCompletedResult, namespace?: string): Promise<void> {
   const hostPath = join(artifactRoot, "files", "phpunit", ...(namespace ? [namespace] : []), ".wp-codebox-result.txt")
   const { passed: _passed, failed: _failed, ...aggregate } = result

packages/runtime-playground/src/wordpress-command-runners.ts

@@ -53,7 +53,7 @@ import {
 import { bootstrapAbilityPhpCode, bootstrapPhpCode, phpCodeFromArgs, splitLeadingStrictTypesDeclare } from "./php-bootstrap.js"
 import { assertPlaygroundResponseOk, attachPlaygroundDiagnostics, completedPlaygroundCommandError, playgroundCommandDiagnosticText, type PlaygroundRunResponse } from "./playground-command-errors.js"
 import type { PlaygroundCliServer } from "./preview-server.js"
-import { persistCorePhpunitResult, persistPluginPhpunitCompletedResult, persistPluginPhpunitResult, persistVfsDiagnosticFileToHost, readCorePhpunitDiagnostic, readPluginPhpunitCompletedResult, readPluginPhpunitDiagnostic, readPluginPhpunitDiscoveryResult } from "./runtime-diagnostics.js"
+import { clearPluginPhpunitJunitResult, persistCorePhpunitResult, persistPluginPhpunitCompletedResult, persistPluginPhpunitJunitResult, persistPluginPhpunitResult, persistVfsDiagnosticFileToHost, phpunitJunitCaptureDiagnostic, readCorePhpunitDiagnostic, readPluginPhpunitCompletedResult, readPluginPhpunitDiagnostic, readPluginPhpunitDiscoveryResult } from "./runtime-diagnostics.js"
 import { phpunitExecutionSemantics, requiresManagedMultisitePreinstall } from "./phpunit-command-semantics.js"
 import { parsePhpunitOutput } from "./phpunit-test-results.js"
 import { runRuntimeExternalHttpLoad, waitForRuntimePreviewReady, type RuntimeExternalHttpLoadResult } from "./external-http-load.js"
@@ -939,6 +939,9 @@ export async function runPhpunitCommand({
   spec: ExecutionSpec
 }): Promise<string | RuntimeCommandResultEnvelope> {
   const args = spec.args ?? []
+  const processIdentity = boundedProcessIdentity(spec.processIdentity)
+  const junitFile = processIdentity ? `/tmp/wp-codebox-phpunit-junit-${processIdentity}.xml` : "/tmp/wp-codebox-phpunit-junit.xml"
+  await clearPluginPhpunitJunitResult(server, junitFile)
   const phpunitXmlArg = argValue(args, "phpunit-xml")
   const explicitCode = argValue(args, "code") || argValue(args, "code-file")
   const pluginSlug = argValue(args, "plugin-slug")?.trim() || ""
@@ -962,7 +965,6 @@ export async function runPhpunitCommand({
   }
   const autoloadFile = argValue(args, "autoload-file")?.trim() || (bootstrapMode === "project" ? "" : "/wp-codebox-vendor/autoload.php")
   const autoloadFileRole = argValue(args, "autoload-file-role")?.trim() === "harness" ? "harness" : undefined
-  const processIdentity = boundedProcessIdentity(spec.processIdentity)
   const managedMultisitePreinstalled = !explicitCode && !discoveryOnly && requiresManagedMultisitePreinstall(args, runtimeSpec)
   const resultFile = processIdentity ? `/tmp/wp-codebox-phpunit-result-${processIdentity}.txt` : PLUGIN_PHPUNIT_RESULT_FILE
   const diagnosticHostFile = `/wordpress/wp-content/plugins/${pluginSlug}/.pg-test-result${processIdentity ? `-${processIdentity}` : ""}.txt`
@@ -1000,6 +1002,7 @@ export async function runPhpunitCommand({
     databaseType,
     managedMultisitePreinstalled,
     resultFile,
+    junitFile,
   })
   if (!explicitCode && !pluginSlug) {
     throw new Error("wordpress.phpunit requires plugin-slug=<slug> when code/code-file is not provided")
@@ -1020,13 +1023,17 @@ export async function runPhpunitCommand({
     }
     response = await runPlaygroundCommand("wordpress.phpunit", server, { code: bootstrapPhpCode(runtimeSpec, code, bootstrapArgs, undefined, resultFile) })
   } catch (error) {
+    // Capture before the runtime teardown can reset the command VFS.
+    const junitCapture = await persistPluginPhpunitJunitResult(server, junitFile, artifactRoot, processIdentity)
     await persistPluginPhpunitResult(server, resultFile, artifactRoot, processIdentity)
     await persistVfsDiagnosticFileToHost(server, resultFile, diagnosticHostFile, mounts)
     await captureCommandResultPaths(server, spec, artifactRoot)
     const completed = await readPluginPhpunitCompletedResult(server, resultFile) ?? parsePhpunitOutput(playgroundCommandDiagnosticText(error))
     if (completed) {
       await persistPluginPhpunitCompletedResult(artifactRoot, completed, processIdentity)
-      throw attachPlaygroundDiagnostics(completedPlaygroundCommandError("wordpress.phpunit", error), "wordpress.phpunit completed result", `completed ${completed.status}; total ${completed.total}; failed ${completed.failed}; skipped ${completed.skipped}; failures ${completed.failures}; errors ${completed.errors}`)
+      const completedError = attachPlaygroundDiagnostics(completedPlaygroundCommandError("wordpress.phpunit", error), "wordpress.phpunit completed result", `completed ${completed.status}; total ${completed.total}; failed ${completed.failed}; skipped ${completed.skipped}; failures ${completed.failures}; errors ${completed.errors}`)
+      const diagnostic = phpunitJunitCaptureDiagnostic(junitCapture)
+      throw diagnostic ? attachPlaygroundDiagnostics(completedError, "wordpress.phpunit JUnit artifact", diagnostic) : completedError
     }
     const structured = await readPluginPhpunitDiagnostic(server, resultFile)
     if (structured) {
@@ -1035,6 +1042,7 @@ export async function runPhpunitCommand({
     throw error
   }
 
+  const junitCapture = await persistPluginPhpunitJunitResult(server, junitFile, artifactRoot, processIdentity)
   await persistPluginPhpunitResult(server, resultFile, artifactRoot, processIdentity)
   await persistVfsDiagnosticFileToHost(server, resultFile, diagnosticHostFile, mounts)
   const structured = await readPluginPhpunitDiagnostic(server, resultFile)
@@ -1046,10 +1054,12 @@ export async function runPhpunitCommand({
   try {
     assertPlaygroundResponseOk("wordpress.phpunit", response)
   } catch (error) {
+    const diagnostic = phpunitJunitCaptureDiagnostic(junitCapture)
     if (structured) {
-      throw attachPlaygroundDiagnostics(error, "wordpress.phpunit structured diagnostics", structured)
+      const structuredError = attachPlaygroundDiagnostics(error, "wordpress.phpunit structured diagnostics", structured)
+      throw diagnostic ? attachPlaygroundDiagnostics(structuredError, "wordpress.phpunit JUnit artifact", diagnostic) : structuredError
     }
-    throw error
+    throw diagnostic ? attachPlaygroundDiagnostics(error, "wordpress.phpunit JUnit artifact", diagnostic) : error
   }
 
   if (discoveryOnly) {

tests/phpunit-runtime-failure-diagnostics.test.ts

@@ -32,13 +32,28 @@ await assert.rejects(
   (error: Error) => {
     assert.match(error.message, /wordpress\.phpunit failed with exit code 1/)
     assert.match(error.message, /wordpress\.phpunit structured diagnostics/)
+    assert.match(error.message, /wordpress\.phpunit JUnit artifact/)
     assert.match(error.message, /Bootstrap failed with token: \[redacted\]/)
     assert.match(error.message, /\[diagnostic truncated\]/)
     assert.doesNotMatch(error.message, new RegExp(secret))
     return true
   },
 )
 
+const clearedJunitFiles: string[] = []
+await assert.rejects(
+  () => runPhpunitCommand({
+    artifactRoot,
+    mounts: [],
+    runPlaygroundCommand: async () => ({ exitCode: 0, errors: "", text: "" }),
+    runtimeSpec: wordpressRuntimeSpec({ commands: ["wordpress.phpunit"] }),
+    server: { playground: { unlink: async (path: string) => { clearedJunitFiles.push(path) } } } as never,
+    spec: { command: "wordpress.phpunit", args: ["discovery-only=true", "code=<?php"] },
+  }),
+  /discovery-only cannot be combined/,
+)
+assert.deepEqual(clearedJunitFiles, ["/tmp/wp-codebox-phpunit-junit.xml"])
+
 const captured = await readFile(join(artifactRoot, "files", "phpunit", ".pg-test-result.txt"), "utf8")
 assert.match(captured, /Bootstrap failed with token: \[redacted\]/)
 assert.doesNotMatch(captured, new RegExp(secret))