Harden hosted run-scope lookup against HTML error pages

scripts/packed-core-release-acceptance.mjs

@@ -192,6 +192,15 @@ function json(response, status, body) {
   response.end(payload);
 }
 
+function htmlNotFound(response) {
+  const body = "<!DOCTYPE html><html><body>404: This page could not be found.</body></html>";
+  response.writeHead(404, {
+    "content-type": "text/html; charset=utf-8",
+    "content-length": Buffer.byteLength(body)
+  });
+  response.end(body);
+}
+
 function readBody(request) {
   return new Promise((resolveBody, reject) => {
     const chunks = [];
@@ -666,7 +675,7 @@ cases:
         return;
       }
       await readBody(request);
-      json(response, 404, { error: { code: "not_found" } });
+      htmlNotFound(response);
     }, async (mock) => {
       const env = {
         ...baseEnv,
@@ -755,7 +764,7 @@ cases:
         return;
       }
       await readBody(request);
-      json(response, 404, { error: { code: "not_found" } });
+      htmlNotFound(response);
     }, async (mock) => {
       const env = {
         ...baseEnv,
@@ -794,7 +803,7 @@ cases:
           return;
         }
         await readBody(request);
-        json(response, 404, { error: { code: "not_found" } });
+        htmlNotFound(response);
       }, async (mock) => {
         const env = {
           ...baseEnv,

scripts/packed-real-data-privacy.mjs

@@ -387,8 +387,12 @@ async function main() {
         response.end(payload);
         return;
       }
-      response.writeHead(404, { "content-type": "application/json" });
-      response.end(JSON.stringify({ error: "not found" }));
+      const missing = "<!DOCTYPE html><html><body>404: This page could not be found.</body></html>";
+      response.writeHead(404, {
+        "content-type": "text/html; charset=utf-8",
+        "content-length": Buffer.byteLength(missing)
+      });
+      response.end(missing);
     });
     await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen));
     const origin = `http://127.0.0.1:${server.address().port}`;

scripts/packed-report-fixture.mjs

@@ -80,6 +80,15 @@ function send(response, status, value) {
   response.end(body);
 }
 
+function sendHtmlNotFound(response) {
+  const body = "<!DOCTYPE html><html><body>404: This page could not be found.</body></html>";
+  response.writeHead(404, {
+    "content-type": "text/html; charset=utf-8",
+    "content-length": Buffer.byteLength(body)
+  });
+  response.end(body);
+}
+
 function cloneMutate(fixtures, name, origin, mutate) {
   const fixture = fixtureNamed(fixtures, name, origin);
   const body = JSON.parse(JSON.stringify(fixture.body));
@@ -213,7 +222,7 @@ function startFixtureServer(fixtures, producer) {
       send(response, fixture.status, fixture.body);
       return;
     }
-    send(response, 404, { error: { code: "NOT_FOUND", message: "missing packed report fixture route" } });
+    sendHtmlNotFound(response);
   });
   return { httpServer, requests, state };
 }

scripts/packed-saved-suite-release.mjs

@@ -297,6 +297,15 @@ function send(response, status, value) {
   response.end(body);
 }
 
+function sendHtmlNotFound(response) {
+  const body = "<!DOCTYPE html><html><body>404: This page could not be found.</body></html>";
+  response.writeHead(404, {
+    "content-type": "text/html; charset=utf-8",
+    "content-length": Buffer.byteLength(body)
+  });
+  response.end(body);
+}
+
 function rewriteOrigin(value, from, to) {
   if (typeof value === "string") return value.split(from).join(to);
   if (Array.isArray(value)) return value.map((item) => rewriteOrigin(item, from, to));
@@ -491,7 +500,7 @@ function startServer(billing, reports, producer) {
         );
         return;
       }
-      send(response, 404, { error: { code: "NOT_FOUND", message: `${request.method} ${url.pathname}` } });
+      sendHtmlNotFound(response);
     });
   });
   return { httpServer, requests, counts };

scripts/smoke-pack.mjs

@@ -475,8 +475,12 @@ async function assertPackedSelectionCompile(packedBin, consumerDirectory) {
         response.end(payload);
         return;
       }
-      response.writeHead(404, { "content-type": "application/json" });
-      response.end(JSON.stringify({ error: { code: "NOT_FOUND", message: "missing" } }));
+      const missing = "<!DOCTYPE html><html><body>404: This page could not be found.</body></html>";
+      response.writeHead(404, {
+        "content-type": "text/html; charset=utf-8",
+        "content-length": Buffer.byteLength(missing)
+      });
+      response.end(missing);
     });
   });
 

src/cloud/client.ts

@@ -719,10 +719,7 @@ export class CloudClient {
       if (!looksLikeExecutionScopeResponse(value)) return undefined;
       return parseExecutionScopeResponse(value);
     } catch (error) {
-      if (
-        error instanceof AwError &&
-        (error.details?.["http_status"] === 404 || error.code === "CLOUD_REQUEST_FAILED")
-      ) {
+      if (error instanceof AwError && error.details?.["http_status"] === 404) {
         return undefined;
       }
       throw error;
@@ -871,15 +868,20 @@ export class CloudClient {
       externalSignal?.removeEventListener("abort", onAbort);
     }
     let value: unknown;
+    let nonJsonErrorBody = false;
     try {
       value = text.length === 0 ? undefined : JSON.parse(text);
     } catch (error) {
-      throw new AwError({
-        code: "INVALID_CLOUD_RESPONSE",
-        category: "protocol",
-        message: "AugmentWorks returned an invalid JSON response.",
-        cause: error
-      });
+      if (response.ok || errorFamily === "billing") {
+        throw new AwError({
+          code: "INVALID_CLOUD_RESPONSE",
+          category: "protocol",
+          message: "AugmentWorks returned an invalid JSON response.",
+          cause: error
+        });
+      }
+      value = undefined;
+      nonJsonErrorBody = true;
     }
     if (value !== undefined) assertJsonLimits(value, "AugmentWorks response");
     if (response.status === 401 && allowAuthRefresh && this.#accessTokenProvider !== undefined) {
@@ -915,14 +917,16 @@ export class CloudClient {
           profileRecoveryUrl(this.apiUrl)
         );
       }
-      throw cloudHttpError(
+      const httpError = cloudHttpError(
         response.status,
         value,
         method,
         path,
         profileRecoveryUrl(this.apiUrl),
         response.headers.get("retry-after")
       );
+      if (!nonJsonErrorBody) throw httpError;
+      throw nameNonJsonRoute(httpError, method, path);
     }
     if (value === undefined) {
       throw new AwError({
@@ -1099,6 +1103,20 @@ function responseTooLarge(): AwError {
   });
 }
 
+function nameNonJsonRoute(error: AwError, method: "GET" | "POST", path: string): AwError {
+  if (error.message.includes(method) && error.message.includes(path)) return error;
+  return new AwError({
+    code: error.code,
+    category: error.category,
+    message: `${error.message} ${method} ${path}`,
+    retryable: error.retryable,
+    ...(error.operation === undefined ? {} : { operation: error.operation }),
+    ...(error.commandId === undefined ? {} : { commandId: error.commandId }),
+    ...(error.details === undefined ? {} : { details: error.details }),
+    cause: error
+  });
+}
+
 function cloudHttpError(
   status: number,
   value: unknown,

src/real-data/hosted.ts

@@ -25,29 +25,30 @@ import {
   assertBindingMatches
 } from "./scope-store.js";
 
-export async function requireHostedAuthorizedScope(
-  cloud: CloudClient,
-  ref: SuiteExecutionScopeRef,
+export async function assertHostedScopeUsable(
+  response: ExecutionScopeResponse,
   options: {
+    readonly cloud: CloudClient;
     readonly workspaceId: string;
     readonly resolved?: ResolvedConfig;
     readonly signal?: AbortSignal;
     readonly now?: number;
+    readonly revision?: number;
+    readonly scopeHash?: string;
   }
-): Promise<ExecutionScopeResponse> {
-  const capabilities = await cloud.getRealDataCapabilities(options.signal);
+): Promise<void> {
+  const capabilities = await options.cloud.getRealDataCapabilities(options.signal);
   assertHostedRealDataRelease(capabilities);
-  const response = await cloud.getExecutionScope(ref.scopeId, options.signal);
   if (response.scope.workspaceId !== options.workspaceId) {
     throw realDataError(
       "TARGET_SCOPE_MISMATCH",
       "The admitted execution scope belongs to a different workspace."
     );
   }
-  if (ref.revision !== undefined && response.scope.revision !== ref.revision) {
+  if (options.revision !== undefined && response.scope.revision !== options.revision) {
     throw realDataError("EXECUTION_SCOPE_STALE", "The admitted scope revision does not match the suite binding.");
   }
-  if (ref.scopeHash !== undefined && response.scope.scopeHash !== ref.scopeHash) {
+  if (options.scopeHash !== undefined && response.scope.scopeHash !== options.scopeHash) {
     throw realDataError("EXECUTION_SCOPE_STALE", "The admitted scope hash does not match the suite binding.");
   }
   if (response.availability === "revoked") {
@@ -76,6 +77,28 @@ export async function requireHostedAuthorizedScope(
   if (options.resolved !== undefined) {
     assertBoundaryMatchesConfig(response.targetBoundary, options.resolved);
   }
+}
+
+export async function requireHostedAuthorizedScope(
+  cloud: CloudClient,
+  ref: SuiteExecutionScopeRef,
+  options: {
+    readonly workspaceId: string;
+    readonly resolved?: ResolvedConfig;
+    readonly signal?: AbortSignal;
+    readonly now?: number;
+  }
+): Promise<ExecutionScopeResponse> {
+  const response = await cloud.getExecutionScope(ref.scopeId, options.signal);
+  await assertHostedScopeUsable(response, {
+    cloud,
+    workspaceId: options.workspaceId,
+    ...(options.resolved === undefined ? {} : { resolved: options.resolved }),
+    ...(options.signal === undefined ? {} : { signal: options.signal }),
+    ...(options.now === undefined ? {} : { now: options.now }),
+    ...(ref.revision === undefined ? {} : { revision: ref.revision }),
+    ...(ref.scopeHash === undefined ? {} : { scopeHash: ref.scopeHash })
+  });
   return response;
 }
 
@@ -241,11 +264,18 @@ export async function resolveDispatchPolicyForBinding(options: {
     return dispatchPolicyFromHostedScope(response);
   }
 
+  if (!packetRequiresHostedScope(options.binding.packet)) {
+    return undefined;
+  }
+
   const fromRun = await options.cloud.getRunExecutionScope(options.binding.run_id, options.signal);
   if (fromRun !== undefined) {
-    if (options.resolved !== undefined) {
-      assertBoundaryMatchesConfig(fromRun.targetBoundary, options.resolved);
-    }
+    await assertHostedScopeUsable(fromRun, {
+      cloud: options.cloud,
+      workspaceId: options.workspaceId,
+      ...(options.resolved === undefined ? {} : { resolved: options.resolved }),
+      ...(options.signal === undefined ? {} : { signal: options.signal })
+    });
     await persistHostedScopeForRun({
       runId: options.binding.run_id,
       configSha256: options.binding.config_sha256,
@@ -256,11 +286,8 @@ export async function resolveDispatchPolicyForBinding(options: {
     return dispatchPolicyFromHostedScope(fromRun);
   }
 
-  if (packetRequiresHostedScope(options.binding.packet)) {
-    throw realDataError(
-      "INVALID_EXECUTION_SCOPE",
-      "This authorized run has no persisted or server-admitted execution scope. Re-admit from the original suite, assessment, or manifest; do not reconstruct authority from YAML."
-    );
-  }
-  return undefined;
+  throw realDataError(
+    "INVALID_EXECUTION_SCOPE",
+    "This authorized run has no persisted or server-admitted execution scope. Re-admit from the original suite, assessment, or manifest; do not reconstruct authority from YAML."
+  );
 }

src/real-data/index.ts

@@ -105,6 +105,7 @@ export {
   localBinding
 } from "./scope-store.js";
 export {
+  assertHostedScopeUsable,
   requireHostedAuthorizedScope,
   dispatchPolicyFromHostedScope,
   persistHostedScopeForRun,

test/catalog/cli.test.ts

@@ -66,7 +66,12 @@ describe("catalog CLI", () => {
         send(response, 200, current?.response, current?.headers);
         return;
       }
-      send(response, 404, { error: { code: "not_found", message: "missing" } });
+      const missing = "<!DOCTYPE html><html><body>404: This page could not be found.</body></html>";
+      response.writeHead(404, {
+        "content-type": "text/html; charset=utf-8",
+        "content-length": Buffer.byteLength(missing)
+      });
+      response.end(missing);
     });
     const state = await mkdtemp(join(tmpdir(), "aw-catalog-"));
     temporaryDirectories.push(state);

test/cloud/request-errors.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { CloudClient } from "../../src/cloud/client.js";
+
+const HTML_NOT_FOUND = "<!DOCTYPE html><html><body>404: This page could not be found.</body></html>";
+
+function htmlResponse(status: number): Response {
+  return new Response(HTML_NOT_FOUND, {
+    status,
+    headers: { "content-type": "text/html; charset=utf-8" }
+  });
+}
+
+function client(fetchMock: typeof fetch): CloudClient {
+  return new CloudClient({
+    apiUrl: "http://127.0.0.1:8787",
+    accessToken: "project-token",
+    fetch: fetchMock
+  });
+}
+
+describe("cloud HTML error pages", () => {
+  it("keeps http_status on an HTML 404 instead of INVALID_CLOUD_RESPONSE", async () => {
+    const fetchMock = vi.fn<typeof fetch>(async () => htmlResponse(404));
+    const error = await client(fetchMock).getRunStatus("run-html").then(
+      () => {
+        throw new Error("expected HTML 404 to fail");
+      },
+      (caught: unknown) => caught
+    );
+    expect(error).toMatchObject({
+      code: "CLOUD_REQUEST_FAILED",
+      details: {
+        http_status: 404,
+        http_method: "GET",
+        http_path: "/v1/relay/runs/run-html"
+      }
+    });
+    expect(error).not.toMatchObject({ code: "INVALID_CLOUD_RESPONSE" });
+    expect((error as { message: string }).message).toContain("GET /v1/relay/runs/run-html");
+  });
+
+  it("still rejects an HTML 200 as INVALID_CLOUD_RESPONSE", async () => {
+    const fetchMock = vi.fn<typeof fetch>(async () => htmlResponse(200));
+    await expect(client(fetchMock).getRunStatus("run-html")).rejects.toMatchObject({
+      code: "INVALID_CLOUD_RESPONSE",
+      message: "AugmentWorks returned an invalid JSON response."
+    });
+  });
+
+  it("keeps a billing HTML error body on the existing invalid JSON path", async () => {
+    const fetchMock = vi.fn<typeof fetch>(async () => htmlResponse(502));
+    await expect(client(fetchMock).getBillingUsage()).rejects.toMatchObject({
+      code: "INVALID_CLOUD_RESPONSE",
+      message: "AugmentWorks returned an invalid JSON response."
+    });
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+});