diff --git a/docs/docs.json b/docs/docs.json index f2744d4f09..bfb8f71410 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -825,13 +825,15 @@ "pages": [ "reference/html-schema", "reference/color-grading", - "reference/audio-effects" + "reference/audio-effects", + "reference/cli-ledger" ] }, { "group": "Rendering paths", "pages": [ "guides/rendering", + "guides/offline-deterministic-renders", "deploy/overview", "deploy/cloud", "guides/deploy" diff --git a/docs/guides/offline-deterministic-renders.mdx b/docs/guides/offline-deterministic-renders.mdx new file mode 100644 index 0000000000..d3052fa1ee --- /dev/null +++ b/docs/guides/offline-deterministic-renders.mdx @@ -0,0 +1,100 @@ +--- +title: "Offline & deterministic renders" +sidebarTitle: "Offline renders" +description: "Make a composition render byte-identically with the network unplugged: inventory assets, vendor remotes, and gate CI on zero network references." +--- + +A HyperFrames render is a function of the composition and its assets — nothing +else. Two things quietly break that: + +- **Remote assets.** A CDN ` + + +${extraBody} +`, + ); + return dir; +} + +/** Silence console and reset spies for one test. */ +export function spyOnConsole(): void { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); +} diff --git a/packages/cli/src/commands/ledger.test.ts b/packages/cli/src/commands/ledger.test.ts new file mode 100644 index 0000000000..af0415c25a --- /dev/null +++ b/packages/cli/src/commands/ledger.test.ts @@ -0,0 +1,78 @@ +import { rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { consumeCommandResult } from "../utils/commandResult.js"; +import { + lastJsonOutput, + makeFixtureProject, + makeRunner, + spyOnConsole, +} from "./_offlineAssetsTestKit.js"; + +// withMeta just annotates the object; identity keeps the assertions simple. +vi.mock("../utils/updateCheck.js", () => ({ withMeta: (o: unknown) => o })); +// resolveProject reports invalid-dir failures to telemetry; keep tests silent. +vi.mock("../telemetry/events.js", () => ({ trackCommandFailure: () => {} })); + +import ledgerCommand from "./ledger.js"; + +const run = makeRunner(ledgerCommand); + +describe("ledger command", () => { + let dir: string; + + beforeEach(() => { + consumeCommandResult(); + spyOnConsole(); + dir = makeFixtureProject("hf-ledger-cmd-", ` `); + }); + + afterEach(() => { + vi.restoreAllMocks(); + consumeCommandResult(); + rmSync(dir, { recursive: true, force: true }); + }); + + it("--json reports the classified asset graph and exits 0", async () => { + await run({ dir, json: true, "strict-offline": false }); + expect(consumeCommandResult().exitCode).toBe(0); + + const output = lastJsonOutput(); + expect(output.ok).toBe(true); + expect(output.files).toEqual(["index.html"]); + expect(output.counts).toMatchObject({ total: 3, remote: 1, local: 1, missing: 1, data: 0 }); + expect(output.remoteUrls).toEqual([ + "https://cdn.jsdelivr.net/npm/gsap@3.12.5/dist/gsap.min.js", + ]); + }); + + it("--strict-offline exits 1 while a remote reference remains", async () => { + await run({ dir, json: true, "strict-offline": true }); + expect(consumeCommandResult().exitCode).toBe(1); + expect(lastJsonOutput().ok).toBe(false); + }); + + it("--strict-offline exits 0 once the project is fully local", async () => { + writeFileSync(join(dir, "index.html"), ``); + await run({ dir, json: true, "strict-offline": true }); + expect(consumeCommandResult().exitCode).toBe(0); + expect(lastJsonOutput().ok).toBe(true); + }); + + it("human-readable output prints counts and the vendor hint", async () => { + await run({ dir, json: false, "strict-offline": false }); + expect(consumeCommandResult().exitCode).toBe(0); + const printed = vi + .mocked(console.log) + .mock.calls.map((call) => call.join(" ")) + .join("\n"); + expect(printed).toContain("Asset ledger"); + expect(printed).toContain("hyperframes vendor"); + }); + + it("errors surface as JSON with exit 1", async () => { + await run({ dir: join(dir, "does-not-exist"), json: true, "strict-offline": false }); + expect(consumeCommandResult().exitCode).toBe(1); + expect(lastJsonOutput().ok).toBe(false); + }); +}); diff --git a/packages/cli/src/commands/ledger.ts b/packages/cli/src/commands/ledger.ts new file mode 100644 index 0000000000..8468289018 --- /dev/null +++ b/packages/cli/src/commands/ledger.ts @@ -0,0 +1,128 @@ +import { defineCommand } from "citty"; +import type { Example } from "./_examples.js"; +import type { AssetLedger } from "@hyperframes/core/asset-ledger"; +import { setCommandExitCode } from "../utils/commandResult.js"; +import { c } from "../ui/colors.js"; +import { resolveProject } from "../utils/project.js"; +import { withMeta } from "../utils/updateCheck.js"; + +export const examples: Example[] = [ + ["Inventory every declared asset", "hyperframes ledger"], + ["Machine-readable asset graph", "hyperframes ledger ./my-video --json"], + ["Fail CI when any remote reference remains", "hyperframes ledger --strict-offline"], +]; + +const STATUS_LABEL: Record string> = { + remote: (s) => c.warn(s), + missing: (s) => c.error(s), + local: (s) => c.success(s), + data: (s) => c.dim(s), +}; + +function printHumanLedger(ledger: AssetLedger, projectName: string): void { + console.log(`${c.accent("◆")} Asset ledger for ${c.accent(projectName)}`); + console.log( + ` ${ledger.files.length} HTML file${ledger.files.length === 1 ? "" : "s"} scanned, ` + + `${ledger.counts.total} asset reference${ledger.counts.total === 1 ? "" : "s"}`, + ); + console.log(); + console.log( + ` ${c.success(String(ledger.counts.local))} local ` + + `${c.warn(String(ledger.counts.remote))} remote ` + + `${c.dim(String(ledger.counts.data))} data ` + + `${c.error(String(ledger.counts.missing))} missing`, + ); + + const actionable = ledger.assets.filter( + (asset) => asset.status === "remote" || asset.status === "missing", + ); + if (actionable.length > 0) { + console.log(); + for (const asset of actionable) { + const paint = STATUS_LABEL[asset.status] ?? ((s: string) => s); + console.log(` ${paint(asset.status.padEnd(7))} ${asset.kind.padEnd(10)} ${asset.url}`); + console.log(` ${" ".repeat(7)} ${c.dim(`${asset.file} (${asset.via})`)}`); + } + } + + if (ledger.counts.remote > 0) { + console.log(); + console.log( + ` ${c.dim("Run")} hyperframes vendor ${c.dim("to download remote assets and rewrite references for offline, deterministic renders.")}`, + ); + } +} + +function printStrictViolation(remoteCount: number): void { + console.log(); + console.log( + c.error( + `✖ --strict-offline: ${remoteCount} remote reference${remoteCount === 1 ? "" : "s"} remain.`, + ), + ); +} + +export default defineCommand({ + meta: { + name: "ledger", + description: "Inventory every declared asset and classify it remote | local | data | missing", + }, + args: { + dir: { + type: "positional", + description: "Project directory", + required: false, + }, + json: { + type: "boolean", + description: "Output the asset ledger as JSON", + default: false, + }, + "strict-offline": { + type: "boolean", + description: "Exit non-zero when any remote asset reference remains", + default: false, + }, + }, + async run({ args }) { + const strictOffline = Boolean(args["strict-offline"]); + try { + const project = resolveProject(args.dir, { requireIndex: false }); + const { buildProjectAssetLedger } = await import("@hyperframes/core/asset-ledger"); + const ledger = buildProjectAssetLedger(project.dir); + const strictViolation = strictOffline && ledger.counts.remote > 0; + + if (args.json) { + console.log( + JSON.stringify( + withMeta({ + ok: !strictViolation, + strictOffline, + files: ledger.files, + counts: ledger.counts, + remoteUrls: ledger.remoteUrls, + assets: ledger.assets, + }), + null, + 2, + ), + ); + setCommandExitCode(strictViolation ? 1 : 0); + return; + } + + printHumanLedger(ledger, project.name); + if (strictViolation) printStrictViolation(ledger.counts.remote); + setCommandExitCode(strictViolation ? 1 : 0); + return; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + if (args.json) { + console.log(JSON.stringify(withMeta({ ok: false, error: message }), null, 2)); + } else { + console.error(message); + } + setCommandExitCode(1); + } + }, +}); diff --git a/packages/cli/src/commands/vendor.test.ts b/packages/cli/src/commands/vendor.test.ts new file mode 100644 index 0000000000..975ed3ed5c --- /dev/null +++ b/packages/cli/src/commands/vendor.test.ts @@ -0,0 +1,211 @@ +import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { consumeCommandResult } from "../utils/commandResult.js"; +import { + GSAP_URL, + lastJsonOutput, + makeFixtureProject, + makeRunner, + spyOnConsole, +} from "./_offlineAssetsTestKit.js"; + +// withMeta just annotates the object; identity keeps the assertions simple. +vi.mock("../utils/updateCheck.js", () => ({ withMeta: (o: unknown) => o })); +// resolveProject reports invalid-dir failures to telemetry; keep tests silent. +vi.mock("../telemetry/events.js", () => ({ trackCommandFailure: () => {} })); + +import vendorCommand, { + resolveVendorTarget, + rewriteHtmlReferences, + toFetchableUrl, + vendorFileName, +} from "./vendor.js"; + +const run = makeRunner(vendorCommand); + +function fetchResponse(body: string, contentType: string): Response { + return new Response(body, { status: 200, headers: { "content-type": contentType } }); +} + +describe("toFetchableUrl", () => { + it("allows http(s) and upgrades protocol-relative URLs to https", () => { + expect(toFetchableUrl(GSAP_URL)).toBe(GSAP_URL); + expect(toFetchableUrl("http://example.com/a.js")).toBe("http://example.com/a.js"); + expect(toFetchableUrl("//cdn.example.com/a.js")).toBe("https://cdn.example.com/a.js"); + }); + + it("rejects non-http(s) schemes", () => { + expect(toFetchableUrl("ftp://example.com/a.png")).toBeNull(); + expect(toFetchableUrl("file:///etc/passwd")).toBeNull(); + expect(toFetchableUrl("not a url")).toBeNull(); + }); +}); + +describe("vendorFileName", () => { + it("keeps the basename, appends a stable hash, and preserves the extension", () => { + const name = vendorFileName(GSAP_URL); + expect(name).toMatch(/^gsap\.min-[0-9a-f]{10}\.js$/); + expect(vendorFileName(GSAP_URL)).toBe(name); + }); + + it("derives the extension from content-type when the URL has none", () => { + expect(vendorFileName("https://fonts.example.com/inter", "font/woff2")).toMatch(/\.woff2$/); + }); +}); + +describe("resolveVendorTarget", () => { + it("resolves plain filenames under the vendor directory", () => { + const outDir = join("/tmp", "proj", "assets", "vendor"); + expect(resolveVendorTarget(outDir, "gsap.min-abc.js")).toBe(join(outDir, "gsap.min-abc.js")); + }); + + it("refuses filenames that would escape the vendor directory", () => { + const outDir = join("/tmp", "proj", "assets", "vendor"); + expect(() => resolveVendorTarget(outDir, "../../../etc/passwd")).toThrow( + /outside the vendor directory/, + ); + expect(() => resolveVendorTarget(outDir, "/etc/passwd")).toThrow( + /outside the vendor directory/, + ); + }); +}); + +describe("rewriteHtmlReferences", () => { + it("rewrites to a path relative to the referencing file", () => { + const html = ``; + const fromRoot = rewriteHtmlReferences(html, "index.html", [ + { rawUrl: GSAP_URL, vendorPath: "assets/vendor/gsap.min-abc.js" }, + ]); + expect(fromRoot.rewritten).toBe(1); + expect(fromRoot.html).toContain('src="assets/vendor/gsap.min-abc.js"'); + + const fromScene = rewriteHtmlReferences(html, "scenes/intro.html", [ + { rawUrl: GSAP_URL, vendorPath: "assets/vendor/gsap.min-abc.js" }, + ]); + expect(fromScene.html).toContain('src="../assets/vendor/gsap.min-abc.js"'); + }); + + it("replaces longer URLs first so prefixes cannot clobber", () => { + const html = ``; + const { html: out } = rewriteHtmlReferences(html, "index.html", [ + { rawUrl: "https://x.com/a.js", vendorPath: "v/a.js" }, + { rawUrl: "https://x.com/a.js.map", vendorPath: "v/a.js.map" }, + ]); + expect(out).toContain('src="v/a.js"'); + expect(out).toContain('src="v/a.js.map"'); + }); +}); + +describe("vendor command", () => { + let dir: string; + + beforeEach(() => { + consumeCommandResult(); + spyOnConsole(); + dir = makeFixtureProject("hf-vendor-cmd-"); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + consumeCommandResult(); + rmSync(dir, { recursive: true, force: true }); + }); + + it("downloads remotes, rewrites HTML, and writes the manifest", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => fetchResponse("/* gsap */", "text/javascript")), + ); + + await run({ dir, out: "assets/vendor", json: true, "strict-offline": true }); + expect(consumeCommandResult().exitCode).toBe(0); + + const output = lastJsonOutput(); + expect(output.ok).toBe(true); + expect(output.remainingRemoteUrls).toEqual([]); + expect(output.rewrittenReferences).toBe(1); + + const html = readFileSync(join(dir, "index.html"), "utf-8"); + expect(html).not.toContain("https://"); + expect(html).toMatch(/src="assets\/vendor\/gsap\.min-[0-9a-f]{10}\.js"/); + + const vendorFiles = readdirSync(join(dir, "assets", "vendor")); + expect(vendorFiles).toContain("vendor-manifest.json"); + expect(vendorFiles.some((f) => f.startsWith("gsap.min-"))).toBe(true); + + const manifest = JSON.parse( + readFileSync(join(dir, "assets", "vendor", "vendor-manifest.json"), "utf-8"), + ); + expect(manifest.assets).toHaveLength(1); + expect(manifest.assets[0]).toMatchObject({ url: GSAP_URL, bytes: 10 }); + expect(manifest.assets[0].sha256).toMatch(/^[0-9a-f]{64}$/); + }); + + it("--dry-run lists candidate downloads without touching the project", async () => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + await run({ dir, out: "assets/vendor", json: true, "dry-run": true }); + expect(consumeCommandResult().exitCode).toBe(0); + expect(fetchMock).not.toHaveBeenCalled(); + expect(lastJsonOutput().wouldDownload).toEqual([GSAP_URL]); + expect(existsSync(join(dir, "assets", "vendor"))).toBe(false); + expect(readFileSync(join(dir, "index.html"), "utf-8")).toContain(GSAP_URL); + }); + + it("reports failed downloads and exits 1 without rewriting their references", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("nope", { status: 404 })), + ); + + await run({ dir, out: "assets/vendor", json: true }); + expect(consumeCommandResult().exitCode).toBe(1); + + const output = lastJsonOutput(); + expect(output.ok).toBe(false); + expect(output.failed).toEqual([{ url: GSAP_URL, error: "HTTP 404" }]); + expect(readFileSync(join(dir, "index.html"), "utf-8")).toContain(GSAP_URL); + }); + + it("rejects downloads over the size cap without rewriting their references", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => fetchResponse("0123456789", "text/javascript")), + ); + + await run({ dir, out: "assets/vendor", json: true, "max-bytes": "4" }); + expect(consumeCommandResult().exitCode).toBe(1); + + const output = lastJsonOutput(); + expect(output.ok).toBe(false); + expect(output.failed).toEqual([ + { url: GSAP_URL, error: expect.stringMatching(/exceeds size cap/) }, + ]); + expect(readFileSync(join(dir, "index.html"), "utf-8")).toContain(GSAP_URL); + }); + + it("refuses an --out directory outside the project", async () => { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + await run({ dir, out: "../outside-vendor", json: true }); + expect(consumeCommandResult().exitCode).toBe(1); + expect(fetchMock).not.toHaveBeenCalled(); + expect(lastJsonOutput().error).toMatch(/inside the project directory/); + }); + + it("--strict-offline exits 1 when a non-vendorable remote (iframe) remains", async () => { + writeFileSync(join(dir, "index.html"), ``); + vi.stubGlobal("fetch", vi.fn()); + + await run({ dir, out: "assets/vendor", json: true, "strict-offline": true }); + expect(consumeCommandResult().exitCode).toBe(1); + + const output = lastJsonOutput(); + expect(output.ok).toBe(false); + expect(output.remainingRemoteUrls).toEqual(["https://example.com/embed"]); + }); +}); diff --git a/packages/cli/src/commands/vendor.ts b/packages/cli/src/commands/vendor.ts new file mode 100644 index 0000000000..8dfa1aff91 --- /dev/null +++ b/packages/cli/src/commands/vendor.ts @@ -0,0 +1,413 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, posix, resolve, sep } from "node:path"; +import { defineCommand } from "citty"; +import type { Example } from "./_examples.js"; +import type { LedgerAssetKind } from "@hyperframes/core/asset-ledger"; +import { setCommandExitCode } from "../utils/commandResult.js"; +import { c } from "../ui/colors.js"; +import { resolveProject } from "../utils/project.js"; +import { withMeta } from "../utils/updateCheck.js"; + +export const examples: Example[] = [ + ["Download remote assets and rewrite references", "hyperframes vendor"], + ["Vendor into a custom directory", "hyperframes vendor ./my-video --out assets/third-party"], + ["Preview what would be downloaded", "hyperframes vendor --dry-run"], + ["Fail unless the project ends up fully offline", "hyperframes vendor --strict-offline"], +]; + +/** Only these URL schemes are ever fetched. Everything else stays remote. */ +const ALLOWED_PROTOCOLS = new Set(["http:", "https:"]); + +/** Per-download size cap unless --max-bytes overrides it. */ +const DEFAULT_MAX_BYTES = 100 * 1024 * 1024; + +/** + * Kinds worth downloading. A remote iframe is a live page, not a static + * asset — vendoring cannot make it deterministic, so it is left in place + * (and still fails --strict-offline, which is the point). + */ +const VENDORABLE_KINDS = new Set([ + "script", + "stylesheet", + "font", + "image", + "audio", + "video", + "track", +]); + +const CONTENT_TYPE_EXT: Record = { + "text/javascript": ".js", + "application/javascript": ".js", + "text/css": ".css", + "font/woff2": ".woff2", + "font/woff": ".woff", + "font/ttf": ".ttf", + "font/otf": ".otf", + "image/png": ".png", + "image/jpeg": ".jpg", + "image/webp": ".webp", + "image/avif": ".avif", + "image/gif": ".gif", + "image/svg+xml": ".svg", + "audio/mpeg": ".mp3", + "audio/wav": ".wav", + "audio/ogg": ".ogg", + "video/mp4": ".mp4", + "video/webm": ".webm", + "text/vtt": ".vtt", + "application/json": ".json", +}; + +interface VendoredEntry { + url: string; + /** Project-root-relative path of the downloaded file (posix separators). */ + file: string; + bytes: number; + sha256: string; + contentType?: string; +} + +interface FailedDownload { + url: string; + error: string; +} + +/** Normalize a declared remote URL to something fetchable, or null. */ +export function toFetchableUrl(url: string): string | null { + // Protocol-relative URLs default to https — the only sane offline-prep choice. + const absolute = url.startsWith("//") ? `https:${url}` : url; + try { + const parsed = new URL(absolute); + return ALLOWED_PROTOCOLS.has(parsed.protocol) ? parsed.href : null; + } catch { + return null; + } +} + +/** Stable, collision-free local filename for one remote URL. */ +export function vendorFileName(url: string, contentType?: string): string { + const hash = createHash("sha256").update(url).digest("hex").slice(0, 10); + let base = "asset"; + let ext = ""; + try { + const pathname = new URL(url).pathname; + const last = pathname.split("/").filter(Boolean).pop() ?? ""; + const dot = last.lastIndexOf("."); + if (dot > 0) { + base = last.slice(0, dot); + ext = last.slice(dot); + } else if (last) { + base = last; + } + } catch { + /* keep defaults */ + } + if (!ext && contentType) { + const normalized = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; + ext = CONTENT_TYPE_EXT[normalized] ?? ""; + } + const safeBase = base.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^[.-]+/, "") || "asset"; + const safeExt = /^\.[a-zA-Z0-9]{1,8}$/.test(ext) ? ext.toLowerCase() : ""; + return `${safeBase}-${hash}${safeExt}`; +} + +/** + * Resolve a vendor file target and refuse anything that would escape the + * vendor directory (defense in depth — `vendorFileName` already sanitizes). + */ +export function resolveVendorTarget(outDir: string, fileName: string): string { + const target = resolve(outDir, fileName); + if (target !== outDir && !target.startsWith(outDir + sep)) { + throw new Error(`refusing to write outside the vendor directory: ${fileName}`); + } + return target; +} + +async function downloadAsset( + url: string, + timeoutMs: number, + maxBytes: number, +): Promise<{ body: Buffer; contentType?: string }> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { signal: controller.signal, redirect: "follow" }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const declared = Number(response.headers.get("content-length") ?? Number.NaN); + if (Number.isFinite(declared) && declared > maxBytes) { + throw new Error(`asset exceeds size cap (${declared} > ${maxBytes} bytes)`); + } + const body = Buffer.from(await response.arrayBuffer()); + if (body.length > maxBytes) { + throw new Error(`asset exceeds size cap (${body.length} > ${maxBytes} bytes)`); + } + const contentType = response.headers.get("content-type") ?? undefined; + return { body, ...(contentType !== undefined ? { contentType } : {}) }; + } finally { + clearTimeout(timer); + } +} + +/** + * Rewrite every occurrence of each vendored URL (its raw as-written form) in + * one HTML file to a path relative to that file. Text-level on purpose: it + * preserves the author's formatting and also rewrites URLs mentioned inside + * inline scripts/styles, which is exactly what an offline render needs. + */ +export function rewriteHtmlReferences( + html: string, + file: string, + replacements: Array<{ rawUrl: string; vendorPath: string }>, +): { html: string; rewritten: number } { + const fileDir = posix.dirname(file); + let out = html; + let rewritten = 0; + // Longest first so one URL that is a prefix of another can't clobber it. + const ordered = [...replacements].sort((a, b) => b.rawUrl.length - a.rawUrl.length); + for (const { rawUrl, vendorPath } of ordered) { + const relative = posix.relative(fileDir === "." ? "" : fileDir, vendorPath) || vendorPath; + if (!out.includes(rawUrl)) continue; + rewritten += out.split(rawUrl).length - 1; + out = out.split(rawUrl).join(relative); + } + return { html: out, rewritten }; +} + +function isVendoredEntry(value: unknown): value is VendoredEntry { + if (typeof value !== "object" || value === null) return false; + const entry = value as Record; // narrowed via the checks below + return ( + typeof entry.url === "string" && + typeof entry.file === "string" && + typeof entry.bytes === "number" && + typeof entry.sha256 === "string" + ); +} + +/** Entries from a previous manifest on disk, or empty when absent/malformed. */ +function readPreviousManifestEntries(manifestPath: string): VendoredEntry[] { + try { + const parsed: unknown = JSON.parse(readFileSync(manifestPath, "utf-8")); + if (typeof parsed !== "object" || parsed === null) return []; + const assets = (parsed as Record).assets; // narrowed below + if (!Array.isArray(assets)) return []; + return assets.filter(isVendoredEntry); + } catch { + return []; + } +} + +export default defineCommand({ + meta: { + name: "vendor", + description: "Download remote assets locally and rewrite references for offline renders", + }, + args: { + dir: { + type: "positional", + description: "Project directory", + required: false, + }, + out: { + type: "string", + alias: "o", + description: "Directory (project-relative) to download assets into", + default: "assets/vendor", + }, + json: { + type: "boolean", + description: "Output the vendoring report as JSON", + default: false, + }, + "dry-run": { + type: "boolean", + description: "List what would be downloaded without writing anything", + default: false, + }, + "strict-offline": { + type: "boolean", + description: "Exit non-zero when any remote reference remains after vendoring", + default: false, + }, + timeout: { + type: "string", + description: "Per-download timeout in ms (default: 30000)", + default: "30000", + }, + "max-bytes": { + type: "string", + description: "Per-download size cap in bytes (default: 104857600 = 100 MB)", + default: String(DEFAULT_MAX_BYTES), + }, + }, + // fallow-ignore-next-line complexity + async run({ args }) { + const strictOffline = Boolean(args["strict-offline"]); + const dryRun = Boolean(args["dry-run"]); + const timeoutMs = Number.parseInt(String(args.timeout), 10) || 30000; + const maxBytes = Number.parseInt(String(args["max-bytes"]), 10) || DEFAULT_MAX_BYTES; + try { + const project = resolveProject(args.dir, { requireIndex: false }); + const { buildProjectAssetLedger } = await import("@hyperframes/core/asset-ledger"); + const ledger = buildProjectAssetLedger(project.dir); + + const outRel = String(args.out ?? "assets/vendor").replace(/\\/g, "/"); + const projectRoot = resolve(project.dir); + const outDir = resolve(projectRoot, outRel); + if (outDir !== projectRoot && !outDir.startsWith(projectRoot + sep)) { + throw new Error(`--out must stay inside the project directory (got "${outRel}")`); + } + + // Unique fetchable URLs, keyed by DECODED url; remember raw forms per file. + const candidates = new Map(); + for (const asset of ledger.assets) { + if (asset.status !== "remote" || !VENDORABLE_KINDS.has(asset.kind)) continue; + if (!candidates.has(asset.url)) { + const fetchable = toFetchableUrl(asset.url); + if (fetchable) candidates.set(asset.url, fetchable); + } + } + + if (dryRun) { + const urls = [...candidates.keys()].sort(); + if (args.json) { + console.log( + JSON.stringify(withMeta({ ok: true, dryRun: true, wouldDownload: urls }), null, 2), + ); + } else { + console.log(`${c.accent("◆")} Would download ${urls.length} remote asset(s):`); + for (const url of urls) console.log(` ${url}`); + } + setCommandExitCode(0); + return; + } + + const vendored: VendoredEntry[] = []; + const failed: FailedDownload[] = []; + const vendorPathByUrl = new Map(); + + if (candidates.size > 0) mkdirSync(outDir, { recursive: true }); + for (const [declaredUrl, fetchUrl] of candidates) { + try { + const { body, contentType } = await downloadAsset(fetchUrl, timeoutMs, maxBytes); + const fileName = vendorFileName(fetchUrl, contentType); + const vendorPath = posix.join(outRel, fileName); + // Network→file is this command's entire purpose (download remote + // assets so renders run offline), so a CodeQL network-to-file-write + // finding here is expected. Guards: http(s)-only scheme allowlist + + // URL validation (toFetchableUrl), sanitized hash-suffixed filenames + // (vendorFileName), a traversal check pinning every write under the + // vendor directory (resolveVendorTarget, with outDir itself pinned + // under the project root), and a per-download size cap. + writeFileSync(resolveVendorTarget(outDir, fileName), body); + vendorPathByUrl.set(declaredUrl, vendorPath); + vendored.push({ + url: declaredUrl, + file: vendorPath, + bytes: body.length, + sha256: createHash("sha256").update(body).digest("hex"), + ...(contentType !== undefined ? { contentType } : {}), + }); + if (!args.json) console.log(`${c.success("↓")} ${declaredUrl} → ${vendorPath}`); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + failed.push({ url: declaredUrl, error: message }); + if (!args.json) console.error(`${c.error("✖")} ${declaredUrl} — ${message}`); + } + } + + // Rewrite each HTML file's references to the vendored copies. + let totalRewritten = 0; + for (const file of ledger.files) { + const replacements: Array<{ rawUrl: string; vendorPath: string }> = []; + const seenRaw = new Set(); + for (const asset of ledger.assets) { + if (asset.file !== file || asset.status !== "remote") continue; + const vendorPath = vendorPathByUrl.get(asset.url); + if (!vendorPath || seenRaw.has(asset.rawUrl)) continue; + seenRaw.add(asset.rawUrl); + replacements.push({ rawUrl: asset.rawUrl, vendorPath }); + } + if (replacements.length === 0) continue; + const absolute = join(project.dir, file); + const html = readFileSync(absolute, "utf-8"); + const { html: next, rewritten } = rewriteHtmlReferences(html, file, replacements); + if (rewritten > 0) { + writeFileSync(absolute, next); + totalRewritten += rewritten; + } + } + + // Persist a provenance manifest next to the downloads (merged over any + // previous run, keyed by URL, sorted for stable diffs). + if (vendored.length > 0) { + const manifestPath = join(outDir, "vendor-manifest.json"); + const byUrl = new Map(); + for (const entry of readPreviousManifestEntries(manifestPath)) byUrl.set(entry.url, entry); + for (const entry of vendored) byUrl.set(entry.url, entry); + const manifest = { + version: 1, + assets: [...byUrl.values()].sort((a, b) => a.url.localeCompare(b.url)), + }; + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + } + + const after = buildProjectAssetLedger(project.dir); + const remaining = after.remoteUrls; + const strictViolation = strictOffline && remaining.length > 0; + const ok = failed.length === 0 && !strictViolation; + + if (args.json) { + console.log( + JSON.stringify( + withMeta({ + ok, + strictOffline, + out: outRel, + downloaded: vendored, + failed, + rewrittenReferences: totalRewritten, + remainingRemoteUrls: remaining, + counts: after.counts, + }), + null, + 2, + ), + ); + setCommandExitCode(ok ? 0 : 1); + return; + } + + console.log(); + console.log( + `${c.accent("◆")} Vendored ${vendored.length} asset(s) into ${outRel}, ` + + `rewrote ${totalRewritten} reference(s).`, + ); + if (failed.length > 0) { + console.log(c.error(` ${failed.length} download(s) failed.`)); + } + if (remaining.length > 0) { + console.log(c.warn(` ${remaining.length} remote reference(s) remain:`)); + for (const url of remaining) console.log(` ${c.warn("•")} ${url}`); + } else { + console.log(c.success(" 0 remote references remain — project renders offline.")); + } + if (strictViolation) { + console.log(); + console.log(c.error("✖ --strict-offline: remote references remain.")); + } + setCommandExitCode(ok ? 0 : 1); + return; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + if (args.json) { + console.log(JSON.stringify(withMeta({ ok: false, error: message }), null, 2)); + } else { + console.error(message); + } + setCommandExitCode(1); + } + }, +}); diff --git a/packages/cli/src/help.ts b/packages/cli/src/help.ts index ea0c1a6833..7fad9d7608 100644 --- a/packages/cli/src/help.ts +++ b/packages/cli/src/help.ts @@ -33,6 +33,8 @@ const GROUPS: Group[] = [ title: "Project", commands: [ ["lint", "Validate a composition for common mistakes"], + ["ledger", "Inventory declared assets and classify them remote | local | data | missing"], + ["vendor", "Download remote assets locally and rewrite references for offline renders"], ["check", "Run lint, runtime validation, and layout inspection as one gate"], [ "validate", diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index beebf2669d..95751c5ec1 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -14,6 +14,13 @@ export default defineConfig({ find: /^@hyperframes\/core$/, replacement: resolve(__dirname, "../core/src/index.ts"), }, + // The ledger/vendor command tests import this node-only subpath; its + // "node" export condition points at an unbuilt dist in the test job, + // so resolve it to source like the bare entry above. + { + find: /^@hyperframes\/core\/asset-ledger$/, + replacement: resolve(__dirname, "../core/src/assets/ledger.ts"), + }, // Same reason the tsup build aliases this specifier to source: the CLI // bundles the producer rather than depending on it at runtime, so its // dist is not built for the test job. Without the alias, vite's import diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json index 8c135bcc87..e4402855de 100644 --- a/packages/core/package-subpaths.json +++ b/packages/core/package-subpaths.json @@ -32,6 +32,12 @@ "types": "./dist/beats/index.d.ts", "environments": ["browser", "bun", "node"] }, + "./asset-ledger": { + "source": "./src/assets/ledger.ts", + "runtime": "./dist/assets/ledger.js", + "types": "./dist/assets/ledger.d.ts", + "environments": ["bun", "node"] + }, "./html-attr-safety": { "source": "./src/utils/htmlAttrSafety.ts", "runtime": "./dist/utils/htmlAttrSafety.js", diff --git a/packages/core/package.json b/packages/core/package.json index b6875ee1b6..abb9740f9e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -46,6 +46,12 @@ "import": "./src/beats/index.ts", "types": "./src/beats/index.ts" }, + "./asset-ledger": { + "bun": "./src/assets/ledger.ts", + "node": "./dist/assets/ledger.js", + "import": "./src/assets/ledger.ts", + "types": "./src/assets/ledger.ts" + }, "./html-attr-safety": { "bun": "./src/utils/htmlAttrSafety.ts", "node": "./dist/utils/htmlAttrSafety.js", @@ -412,6 +418,10 @@ "import": "./dist/beats/index.js", "types": "./dist/beats/index.d.ts" }, + "./asset-ledger": { + "import": "./dist/assets/ledger.js", + "types": "./dist/assets/ledger.d.ts" + }, "./html-attr-safety": { "import": "./dist/utils/htmlAttrSafety.js", "types": "./dist/utils/htmlAttrSafety.d.ts" diff --git a/packages/core/src/assets/ledger.test.ts b/packages/core/src/assets/ledger.test.ts new file mode 100644 index 0000000000..4b1def83e5 --- /dev/null +++ b/packages/core/src/assets/ledger.test.ts @@ -0,0 +1,255 @@ +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + buildAssetLedger, + buildProjectAssetLedger, + classifyAssetUrl, + collectProjectHtmlFiles, + extractAssetRefs, +} from "./ledger"; + +// A representative composition: one CDN script (jsDelivr), local media, a +// data URI, a missing reference, and CSS-declared assets. +const FIXTURE_HTML = ` + + + + + + + + + +
+ + + + + + + + + + +
+ + +`; + +describe("classifyAssetUrl", () => { + it("classifies remote, data, and local candidates", () => { + expect(classifyAssetUrl("https://cdn.jsdelivr.net/npm/gsap@3/dist/gsap.min.js")).toBe("remote"); + expect(classifyAssetUrl("http://example.com/a.js")).toBe("remote"); + expect(classifyAssetUrl("//cdn.example.com/a.js")).toBe("remote"); + expect(classifyAssetUrl("data:image/png;base64,AAAA")).toBe("data"); + expect(classifyAssetUrl("assets/img/logo.png")).toBe("local-candidate"); + expect(classifyAssetUrl("/assets/img/logo.png")).toBe("local-candidate"); + }); + + it("skips fragments, runtime schemes, and templating placeholders", () => { + expect(classifyAssetUrl("")).toBe("skip"); + expect(classifyAssetUrl("#anchor")).toBe("skip"); + expect(classifyAssetUrl("blob:https://x/y")).toBe("skip"); + expect(classifyAssetUrl("javascript:void(0)")).toBe("skip"); + expect(classifyAssetUrl("about:blank")).toBe("skip"); + expect(classifyAssetUrl("{{ heroImage }}")).toBe("skip"); + expect(classifyAssetUrl("__POSTER__")).toBe("skip"); + }); + + it("marks unknown schemes as remote so --strict-offline surfaces them", () => { + expect(classifyAssetUrl("ftp://host/file.png")).toBe("remote"); + }); +}); + +describe("extractAssetRefs", () => { + const refs = extractAssetRefs(FIXTURE_HTML); + const find = (url: string) => refs.find((r) => r.url === url); + + it("finds remote script tags but not scripts inside string literals or comments", () => { + const script = find("https://cdn.jsdelivr.net/npm/gsap@3.12.5/dist/gsap.min.js"); + expect(script).toMatchObject({ kind: "script", via: "src" }); + expect(find("https://evil.example.com/x.png")).toBeUndefined(); + expect(find("https://commented-out.example.com/never.js")).toBeUndefined(); + }); + + it("classifies link tags by rel/as", () => { + expect(find("styles/theme.css")).toMatchObject({ kind: "stylesheet", via: "href" }); + expect(find("assets/fonts/Inter.woff2")).toMatchObject({ kind: "font", via: "href" }); + }); + + it("extracts CSS @import, @font-face urls, and background urls", () => { + expect(find("https://cdn.example.com/base.css")).toMatchObject({ + kind: "stylesheet", + via: "css-import", + }); + expect(find("assets/fonts/brand.woff2")).toMatchObject({ kind: "font", via: "css-url" }); + expect(find("assets/img/bg.png")).toMatchObject({ kind: "image", via: "css-url" }); + expect(find("assets/img/inline.png")).toMatchObject({ kind: "image", via: "style-attr" }); + }); + + it("extracts media elements including srcset, poster, and picture sources", () => { + expect(find("assets/video/intro.mp4")).toMatchObject({ kind: "video", via: "src" }); + expect(find("assets/img/poster.jpg")).toMatchObject({ kind: "image", via: "poster" }); + expect(find("assets/audio/bgm.mp3")).toMatchObject({ kind: "audio", via: "src" }); + expect(find("assets/img/logo@2x.png")).toMatchObject({ kind: "image", via: "srcset" }); + expect(find("assets/img/hero.avif")).toMatchObject({ kind: "image", via: "srcset" }); + expect(find("https://example.com/embed")).toMatchObject({ kind: "iframe", via: "src" }); + }); + + it("maps inside