diff --git a/opencut/host/editor-mount.tsx b/opencut/host/editor-mount.tsx index e9ce1df..5c010c8 100644 --- a/opencut/host/editor-mount.tsx +++ b/opencut/host/editor-mount.tsx @@ -77,6 +77,11 @@ import { type ArtifactFocusRequest, type ArtifactFocusResult, } from "./artifact-focus-contract"; +import { + heldMediaReplacements, + mediaMapSources, + type HeldArtifactMedia, +} from "./placeholder-media"; export type HostEditorMediaAsset = { artifact: { @@ -593,6 +598,72 @@ function rewriteProjectMediaIds({ }; } +/** The host asset list's files that the media bin holds. */ +function heldHostMedia({ + groupedAssets, + known, + map, +}: { + groupedAssets: ReturnType; + known: Set; + map: HostMediaMap; +}) { + const held: HeldArtifactMedia[] = []; + for (const [sourceKey, { aliases, asset }] of groupedAssets) { + const mapped = findMappedHostMedia({ aliases, map, sourceKey }); + if (!mapped || !known.has(mapped.mediaId)) continue; + held.push({ + artifactId: asset.artifact.artifactId, + hostAssetIds: [...aliases], + mediaId: mapped.mediaId, + sourceKey, + sourcePath: asset.artifact.sourcePath, + }); + } + return held; +} + +/** Point timeline elements whose media id the bin does not hold (an agent + * insert's `videofs-…` placeholder, or an id minted by another browser + * profile) at the bin's copy of the same file. */ +async function adoptHeldMedia({ + editor, + groupedAssets, + known, + map, + placeholderSources, + projectId, +}: { + editor: EditorCore; + groupedAssets: ReturnType; + known: Set; + map: HostMediaMap; + placeholderSources: ReadonlyMap; + projectId: string; +}) { + const activeProject = editor.project.getActiveOrNull(); + if (!activeProject || activeProject.metadata.id !== projectId) return false; + const replacements = heldMediaReplacements({ + heldMedia: heldHostMedia({ groupedAssets, known, map }), + knownMediaIds: known, + placeholderSources, + scenes: activeProject.scenes ?? [], + }); + if (!replacements.size) return false; + const rewritten = rewriteProjectMediaIds({ + project: activeProject, + replacements, + }); + if (!rewritten.changed) return false; + editor.scenes.setScenes({ + activeSceneId: rewritten.project.currentSceneId, + scenes: rewritten.project.scenes, + }); + editor.project.setActiveProject({ project: rewritten.project }); + await storageService.saveProject({ project: rewritten.project }); + return true; +} + async function reconcileDuplicateHostMedia({ editor, hostAssets, @@ -784,6 +855,10 @@ function useHostBridge( /** Revision of the last server doc this client synced against — sent as * baseRevision on pushes so a push can never clobber an unseen agent edit. */ const lastKnownRevisionRef = useRef(null); + const mediaAssetsRef = useRef(mediaAssets); + useEffect(() => { + mediaAssetsRef.current = mediaAssets; + }, [mediaAssets]); const mediaMapKey = opencutId ? `host-media-map-${opencutId}` : ""; // Media hydration: every host asset lands in the media bin exactly once. @@ -803,16 +878,14 @@ function useHostBridge( } const map = readHostMediaMap(mediaMapKey); const known = new Set(editor.media.getAssets().map((asset) => asset.id)); + // Read before imports overwrite entries: a map seeded from the + // server copy can hold placeholder ids that elements still use. + const placeholderSources = mediaMapSources(map); const groupedAssets = groupHostMediaAssets(mediaAssets); let mapChanged = false; for (const [sourceKey, { aliases, asset }] of groupedAssets) { if (cancelled) return; const mapped = findMappedHostMedia({ aliases, map, sourceKey }); - // Agent inserts may pre-seed the map server-side with a - // placeholder id; once the real asset is minted below, the - // placeholder is adopted by remapping timeline elements. - const seededMediaId = - mapped && !known.has(mapped.mediaId) ? mapped.mediaId : null; if (mapped && known.has(mapped.mediaId)) { if ( asset.artifact.contentHash && @@ -887,64 +960,24 @@ function useHostBridge( mapChanged = true; known.add(added.id); writeHostMediaMap(mediaMapKey, map); - const replacements = new Map(); - if (seededMediaId && seededMediaId !== added.id) { - replacements.set(seededMediaId, added.id); - } - // Elements carry their artifact identity; any element for - // THIS artifact whose media id is unknown here (seeded - // placeholder, or an id minted by another browser profile - // that has since left the map) is remapped to the fresh - // asset by that identity. - const activeForRemap = editor.project.getActiveOrNull(); - if (activeForRemap) { - for (const scene of activeForRemap.scenes ?? []) { - const tracks = scene.tracks as unknown as { - audio?: Array<{ elements?: Array> }>; - main?: { elements?: Array> }; - overlay?: Array<{ elements?: Array> }>; - }; - const allTrackElements = [ - ...(tracks?.main?.elements ?? []), - ...(tracks?.overlay ?? []).flatMap((t) => t.elements ?? []), - ...(tracks?.audio ?? []).flatMap((t) => t.elements ?? []), - ]; - for (const element of allTrackElements) { - const identity = element.videoFsArtifact as - | { artifactId?: string } - | undefined; - const mediaId = element.mediaId; - if ( - identity?.artifactId === asset.artifact.artifactId && - typeof mediaId === "string" && - mediaId !== added.id && - !known.has(mediaId) - ) { - replacements.set(mediaId, added.id); - } - } - } - if (replacements.size) { - const rewritten = rewriteProjectMediaIds({ - project: activeForRemap, - replacements, - }); - if (rewritten.changed) { - editor.scenes.setScenes({ - activeSceneId: rewritten.project.currentSceneId, - scenes: rewritten.project.scenes, - }); - editor.project.setActiveProject({ project: rewritten.project }); - await storageService.saveProject({ project: rewritten.project }); - } - } - } } } catch { /* skip failed asset; retried next mount */ } } if (!cancelled) { + // Every file the bin holds, not only those imported in this pass: + // an element can reference a file that was imported earlier. + await adoptHeldMedia({ + editor, + groupedAssets, + known, + map, + placeholderSources, + projectId: opencutId, + }).catch((caught) => + console.warn("[editor-sync] failed to adopt placeholder media", caught), + ); const didReconcile = await reconcileDuplicateHostMedia({ editor, hostAssets: mediaAssets, @@ -983,26 +1016,57 @@ function useHostBridge( doc.revision > lastAppliedRevisionRef.current && doc.project ) { - try { - // The wire doc is raw JSON — dates are STRINGS. saveProject - // calls toISOString() on them, so hydrate first or the apply - // throws and the agent's edit silently never reaches the - // editor (and the next push clobbers it). - await storageService.saveProject({ - project: hydrateWireProject(doc.project), - }); - lastPushedRef.current = JSON.stringify(doc.project); - await EditorCore.getInstance().project.loadProject({ id: opencutId }); - // Mark applied ONLY on success so a transient failure retries. - lastAppliedRevisionRef.current = doc.revision; - appliedAgentDoc = true; - dispatchEditorDocumentRevision(window, { - projectId: opencutId, - revision: doc.revision, - }); - } catch (caught) { - console.warn("[editor-sync] failed to apply agent doc", caught); - } + const revision: number = doc.revision; + const applyAgentDoc = async () => { + if (cancelled) return false; + let applied = false; + try { + // The wire doc is raw JSON — dates are STRINGS. saveProject + // calls toISOString() on them, so hydrate first or the apply + // throws and the agent's edit silently never reaches the + // editor (and the next push clobbers it). + await storageService.saveProject({ + project: hydrateWireProject(doc.project), + }); + lastPushedRef.current = JSON.stringify(doc.project); + await EditorCore.getInstance().project.loadProject({ id: opencutId }); + // Mark applied ONLY on success so a transient failure retries. + lastAppliedRevisionRef.current = revision; + applied = true; + dispatchEditorDocumentRevision(window, { + projectId: opencutId, + revision, + }); + // The edit can place a file the bin already holds under a + // placeholder id, and hydration does not run again for that + // file. Adopting it here lets the next tick push the + // repaired project. + const editor = EditorCore.getInstance(); + await adoptHeldMedia({ + editor, + groupedAssets: groupHostMediaAssets(mediaAssetsRef.current), + known: new Set(editor.media.getAssets().map((asset) => asset.id)), + map: readHostMediaMap(mediaMapKey), + placeholderSources: mediaMapSources(doc.mediaMap), + projectId: opencutId, + }).catch((caught) => + console.warn("[editor-sync] failed to adopt placeholder media", caught), + ); + } catch (caught) { + console.warn("[editor-sync] failed to apply agent doc", caught); + } + return applied; + }; + // Loading the project empties the media bin and refills it from + // storage. A hydration pass importing a file meanwhile can lose + // that file from the bin in memory while storage keeps it, and an + // element pointed at it then draws nothing. Applying the edit in + // the hydration queue keeps the two apart; it also runs after a + // pass that threw. + const queued = hydrationQueueRef.current.then(applyAgentDoc, applyAgentDoc); + hydrationQueueRef.current = queued.then(() => undefined); + appliedAgentDoc = await queued; + if (cancelled) return; } if (!appliedAgentDoc) { const loaded = await storageService.loadProject({ id: opencutId }); diff --git a/opencut/host/placeholder-media.ts b/opencut/host/placeholder-media.ts new file mode 100644 index 0000000..782bde8 --- /dev/null +++ b/opencut/host/placeholder-media.ts @@ -0,0 +1,140 @@ +import { artifactIdentityFromEditorValue } from "./editor-placement-contract"; + +/** A Canvas file the Editor's media bin holds. `sourcePath` is the version's + * project-relative file, or the record path of an unversioned artifact; + * `hostAssetIds` are the host asset ids resolving to the file (the artifact + * id itself names the artifact's current file). */ +export type HeldArtifactMedia = { + artifactId: string; + hostAssetIds: readonly string[]; + mediaId: string; + sourceKey: string; + sourcePath: string; +}; + +type UnknownRecord = Record; + +function record(value: unknown): UnknownRecord | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as UnknownRecord) + : null; +} + +/** The comparison the server applies to media source keys (pathname only, + * percent-decoded), so client and server agree on which file a key names. */ +export function mediaSourcePath(sourceKey: string | undefined) { + if (!sourceKey) return null; + try { + return decodeURIComponent( + new URL(sourceKey, "http://video-fs.local").pathname, + ); + } catch { + return null; + } +} + +function relativePath(value: string) { + return value.replaceAll("\\", "/").replace(/^\/+/, "").replace(/^(\.\/)+/, ""); +} + +/** Media id -> source key, for every entry of an edit document media map. */ +export function mediaMapSources(mediaMap: unknown) { + const sources = new Map(); + for (const value of Object.values(record(mediaMap) ?? {})) { + const entry = record(value); + if ( + typeof entry?.mediaId === "string" && + typeof entry.sourceKey === "string" + ) { + sources.set(entry.mediaId, entry.sourceKey); + } + } + return sources; +} + +function sceneElements(scenes: readonly unknown[]) { + const elements: UnknownRecord[] = []; + for (const rawScene of scenes) { + const tracks = record(record(rawScene)?.tracks); + const overlay = Array.isArray(tracks?.overlay) ? tracks.overlay : []; + const audio = Array.isArray(tracks?.audio) ? tracks.audio : []; + for (const track of [ + record(tracks?.main), + ...overlay.map(record), + ...audio.map(record), + ]) { + if (!track || !Array.isArray(track.elements)) continue; + for (const rawElement of track.elements) { + const element = record(rawElement); + if (element) elements.push(element); + } + } + } + return elements; +} + +function heldMediaForIdentity( + value: unknown, + heldMedia: readonly HeldArtifactMedia[], +) { + const identity = artifactIdentityFromEditorValue(value); + if (!identity) return null; + const sameArtifact = heldMedia.filter( + (media) => media.artifactId === identity.artifactId, + ); + if (identity.versionIndex !== null) { + // A version placement names its file; another version is not a stand-in. + const path = relativePath(identity.sourcePath); + return ( + sameArtifact.find((media) => relativePath(media.sourcePath) === path) ?? + null + ); + } + // A record placement shows the record's current file. + return ( + sameArtifact.find((media) => + media.hostAssetIds.includes(identity.artifactId), + ) ?? null + ); +} + +/** Agent inserts reference media through the edit document's media map. When + * that map has no entry for the exact file, the server writes a placeholder + * media id (`videofs-…`) on the element and records the file's source key + * under it (ensureArtifactEditorMedia). The Editor draws nothing for a media + * id its bin does not hold, so each such id maps to the bin id of the same + * file: the file its placeholder entry names (and none other, until the bin + * holds it), or, with no entry left, the file the element's artifact identity + * names. Ids the bin holds are never touched. */ +export function heldMediaReplacements({ + heldMedia, + knownMediaIds, + placeholderSources, + scenes, +}: { + heldMedia: readonly HeldArtifactMedia[]; + knownMediaIds: ReadonlySet; + placeholderSources: ReadonlyMap; + scenes: readonly unknown[]; +}) { + const usable = heldMedia.filter((media) => knownMediaIds.has(media.mediaId)); + const bySourcePath = new Map(); + for (const media of usable) { + const path = mediaSourcePath(media.sourceKey); + if (path && !bySourcePath.has(path)) bySourcePath.set(path, media); + } + const replacements = new Map(); + for (const element of sceneElements(scenes)) { + const mediaId = + typeof element.mediaId === "string" ? element.mediaId : null; + if (!mediaId || knownMediaIds.has(mediaId) || replacements.has(mediaId)) { + continue; + } + const placeholderPath = mediaSourcePath(placeholderSources.get(mediaId)); + const held = placeholderPath + ? bySourcePath.get(placeholderPath) + : heldMediaForIdentity(element.videoFsArtifact, usable); + if (held) replacements.set(mediaId, held.mediaId); + } + return replacements; +} diff --git a/tests/placeholder-media.test.ts b/tests/placeholder-media.test.ts new file mode 100644 index 0000000..a7c6b35 --- /dev/null +++ b/tests/placeholder-media.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from "vitest"; +import { + heldMediaReplacements, + mediaMapSources, + mediaSourcePath, + type HeldArtifactMedia, +} from "../opencut/host/placeholder-media"; + +const MEDIA = "/api/projects/project_one/media"; + +function recordIdentity(artifactId: string) { + return { + artifactId, + contentHash: "a".repeat(64), + entityRevision: 0, + path: `uploads/${artifactId}.md`, + version: null, + }; +} + +function versionIdentity(artifactId: string, index: number) { + return { + artifactId, + contentHash: "b".repeat(64), + entityRevision: index + 1, + path: `media/clips/${artifactId}.v${index + 1}.mp4`, + version: { index, sha256: "b".repeat(64), versionId: `v${index + 1}` }, + }; +} + +function scenes({ + audio = [], + main = [], + overlay = [], +}: { + audio?: unknown[]; + main?: unknown[]; + overlay?: unknown[]; +}) { + return [ + { + id: "scene_main", + tracks: { + audio: [{ elements: audio, id: "track_audio" }], + main: { elements: main, id: "track_main" }, + overlay: [{ elements: overlay, id: "track_overlay" }], + }, + }, + ]; +} + +const picture: HeldArtifactMedia = { + artifactId: "upload_picture", + hostAssetIds: ["upload_picture"], + mediaId: "bin-picture", + sourceKey: `${MEDIA}/uploads/upload_picture.png`, + sourcePath: "uploads/upload_picture.md", +}; + +function clipVersion(index: number, current: boolean): HeldArtifactMedia { + return { + artifactId: "clip_intro", + hostAssetIds: [current ? "clip_intro" : `clip_intro@v${index + 1}`], + mediaId: `bin-clip-v${index + 1}`, + sourceKey: `${MEDIA}/clips/clip_intro.v${index + 1}.mp4`, + sourcePath: `media/clips/clip_intro.v${index + 1}.mp4`, + }; +} + +describe("placeholder media adoption", () => { + it("points an agent placeholder at the bin's copy of the file its map entry names", () => { + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources: new Map([ + ["videofs-1", `${MEDIA}/uploads/upload_picture.png`], + ]), + scenes: scenes({ + main: [ + { + id: "element_agent", + mediaId: "videofs-1", + videoFsArtifact: recordIdentity("upload_picture"), + }, + ], + }), + }); + expect([...replacements]).toEqual([["videofs-1", "bin-picture"]]); + }); + + it("waits for the exact file a placeholder names instead of using another version", () => { + const element = { + id: "element_agent", + mediaId: "videofs-2", + videoFsArtifact: versionIdentity("clip_intro", 1), + }; + const placeholderSources = new Map([ + ["videofs-2", `${MEDIA}/clips/clip_intro.v2.mp4`], + ]); + const onlyFirstVersion = heldMediaReplacements({ + heldMedia: [clipVersion(0, false)], + knownMediaIds: new Set(["bin-clip-v1"]), + placeholderSources, + scenes: scenes({ main: [element] }), + }); + expect(onlyFirstVersion.size).toBe(0); + + const bothVersions = heldMediaReplacements({ + heldMedia: [clipVersion(0, false), clipVersion(1, true)], + knownMediaIds: new Set(["bin-clip-v1", "bin-clip-v2"]), + placeholderSources, + scenes: scenes({ main: [element] }), + }); + expect([...bothVersions]).toEqual([["videofs-2", "bin-clip-v2"]]); + }); + + it("adopts an id that left every map through the element's identity", () => { + const element = { + id: "element_agent", + mediaId: "minted-by-another-profile", + videoFsArtifact: recordIdentity("upload_picture"), + }; + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources: new Map(), + scenes: scenes({ main: [element] }), + }); + expect([...replacements]).toEqual([ + ["minted-by-another-profile", "bin-picture"], + ]); + }); + + it("gives a record placement only the record's current file", () => { + const element = { + id: "element_agent", + mediaId: "videofs-lost", + videoFsArtifact: recordIdentity("clip_intro"), + }; + const olderVersionOnly = heldMediaReplacements({ + heldMedia: [clipVersion(0, false)], + knownMediaIds: new Set(["bin-clip-v1"]), + placeholderSources: new Map(), + scenes: scenes({ main: [element] }), + }); + expect(olderVersionOnly.size).toBe(0); + + const withCurrent = heldMediaReplacements({ + heldMedia: [clipVersion(0, false), clipVersion(1, true)], + knownMediaIds: new Set(["bin-clip-v1", "bin-clip-v2"]), + placeholderSources: new Map(), + scenes: scenes({ main: [element] }), + }); + expect([...withCurrent]).toEqual([["videofs-lost", "bin-clip-v2"]]); + }); + + it("gives a version placement only its own version", () => { + const element = { + id: "element_agent", + mediaId: "videofs-lost", + videoFsArtifact: versionIdentity("clip_intro", 0), + }; + const replacements = heldMediaReplacements({ + heldMedia: [clipVersion(1, true), clipVersion(0, false)], + knownMediaIds: new Set(["bin-clip-v1", "bin-clip-v2"]), + placeholderSources: new Map(), + scenes: scenes({ main: [element] }), + }); + expect([...replacements]).toEqual([["videofs-lost", "bin-clip-v1"]]); + }); + + it("never touches ids the bin holds, elements without media, or elements it cannot place", () => { + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources: new Map(), + scenes: scenes({ + main: [ + { + id: "element_held", + mediaId: "bin-picture", + videoFsArtifact: recordIdentity("upload_picture"), + }, + { id: "element_text", type: "text" }, + { id: "element_by_hand", mediaId: "lost-without-identity" }, + { + id: "element_other_artifact", + mediaId: "videofs-other", + videoFsArtifact: recordIdentity("upload_other"), + }, + ], + }), + }); + expect(replacements.size).toBe(0); + }); + + it("does not use a file the bin no longer holds", () => { + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(), + placeholderSources: new Map([ + ["videofs-1", `${MEDIA}/uploads/upload_picture.png`], + ]), + scenes: scenes({ + main: [ + { + id: "element_agent", + mediaId: "videofs-1", + videoFsArtifact: recordIdentity("upload_picture"), + }, + ], + }), + }); + expect(replacements.size).toBe(0); + }); + + it("reads overlay and audio tracks and maps a shared placeholder once", () => { + const placeholderSources = new Map([ + ["videofs-1", `${MEDIA}/uploads/upload_picture.png`], + ]); + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources, + scenes: scenes({ + audio: [{ id: "element_audio", mediaId: "videofs-1" }], + overlay: [{ id: "element_overlay", mediaId: "videofs-1" }], + }), + }); + expect([...replacements]).toEqual([["videofs-1", "bin-picture"]]); + }); + + it("compares source keys the way the server does", () => { + expect(mediaSourcePath(`${MEDIA}/uploads/a%20b.png?v=3`)).toBe( + `${MEDIA}/uploads/a b.png`, + ); + expect(mediaSourcePath(undefined)).toBeNull(); + const replacements = heldMediaReplacements({ + heldMedia: [ + { ...picture, sourceKey: `${MEDIA}/uploads/upload%20picture.png?t=1` }, + ], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources: new Map([ + ["videofs-1", `${MEDIA}/uploads/upload picture.png`], + ]), + scenes: scenes({ main: [{ id: "element_agent", mediaId: "videofs-1" }] }), + }); + expect([...replacements]).toEqual([["videofs-1", "bin-picture"]]); + }); + + it("lists every media map entry that has a media id and a source key", () => { + const sources = mediaMapSources({ + upload_picture: { + mediaId: "videofs-1", + sourceKey: `${MEDIA}/uploads/upload_picture.png`, + }, + "videofs:upload_picture": { mediaId: "videofs-2" }, + broken: null, + }); + expect([...sources]).toEqual([ + ["videofs-1", `${MEDIA}/uploads/upload_picture.png`], + ]); + expect(mediaMapSources(undefined).size).toBe(0); + }); +}); diff --git a/tests/smoke/editor-agent-media.spec.ts b/tests/smoke/editor-agent-media.spec.ts new file mode 100644 index 0000000..11ea2c9 --- /dev/null +++ b/tests/smoke/editor-agent-media.spec.ts @@ -0,0 +1,432 @@ +import { deflateSync } from "node:zlib"; +import { + expect, + test, + type APIRequestContext, + type Page, +} from "@playwright/test"; + +// scripts/smoke-server.mjs starts the app with this tool token; an agent's +// MCP server reaches the same route with the project's own token. +const MCP_TOKEN = "smoke-only-token-not-a-production-credential"; +const TICKS_PER_SECOND = 120_000; +const PICTURE_COLOUR = [230, 40, 200] as const; + +type Upload = { id: string; kind: string; path: string }; +type EditorDoc = { + mediaMap?: Record; + project?: unknown; + revision?: number; +} | null; + +function crc32(bytes: Uint8Array) { + let crc = 0xffffffff; + for (const byte of bytes) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function pngChunk(type: string, data: Buffer) { + const body = Buffer.concat([Buffer.from(type, "ascii"), data]); + const chunk = Buffer.alloc(body.length + 8); + chunk.writeUInt32BE(data.length, 0); + body.copy(chunk, 4); + chunk.writeUInt32BE(crc32(body), body.length + 4); + return chunk; +} + +/** A picture of one colour, so the preview can be checked for that colour. */ +function solidPng(width: number, height: number, colour: readonly number[]) { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 2; + const row = Buffer.alloc(1 + width * 3); + for (let x = 0; x < width; x += 1) row.set(colour, 1 + x * 3); + const pixels = Buffer.concat(Array.from({ length: height }, () => row)); + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", header), + pngChunk("IDAT", deflateSync(pixels)), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +async function createProject(request: APIRequestContext, name: string) { + const created = await request.post("/api/projects", { data: { name } }); + expect(created.status()).toBe(200); + return ((await created.json()) as { project: { id: string } }).project; +} + +async function uploadPicture( + request: APIRequestContext, + projectId: string, + name: string, +) { + const response = await request.post( + `/api/projects/${projectId}/canvas/upload`, + { + multipart: { + files: { + buffer: solidPng(960, 540, PICTURE_COLOUR), + mimeType: "image/png", + name, + }, + }, + }, + ); + expect(response.status()).toBe(200); + const body = (await response.json()) as { uploaded: Upload[] }; + expect(body.uploaded).toHaveLength(1); + return body.uploaded[0]!; +} + +async function readEditorDoc( + request: APIRequestContext, + projectId: string, +): Promise { + const response = await request.get(`/api/projects/${projectId}/editor-doc`); + expect(response.status()).toBe(200); + return (await response.json()) as EditorDoc; +} + +function elementMediaId(project: unknown, elementId: string) { + type Track = { elements?: Array<{ id?: unknown; mediaId?: unknown }> }; + const scenes = + ( + project as { + scenes?: Array<{ + tracks?: { audio?: Track[]; main?: Track; overlay?: Track[] }; + }>; + } | null + )?.scenes ?? []; + for (const scene of scenes) { + const tracks = scene.tracks; + for (const track of [ + tracks?.main, + ...(tracks?.overlay ?? []), + ...(tracks?.audio ?? []), + ]) { + for (const element of track?.elements ?? []) { + if (element.id !== elementId) continue; + return typeof element.mediaId === "string" ? element.mediaId : null; + } + } + } + return null; +} + +/** What the Editor in the page works from: its own copy of the project and + * the ids of the media in its bin, as it stores them in the browser. */ +async function editorStorage(page: Page, hostProjectId: string) { + return page.evaluate(async (hostProjectId) => { + const projectMap = JSON.parse( + localStorage.getItem("host-project-map") ?? "{}", + ) as Record; + const opencutId = projectMap[hostProjectId]; + if (typeof opencutId !== "string") { + return { + binMediaIds: [] as string[], + hostMediaMap: {} as Record, + project: null as unknown, + }; + } + const existing = new Set( + (await indexedDB.databases()).map((database) => database.name), + ); + const settle = (request: IDBRequest) => + new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + // Opens only databases the app created, so reading never creates one. + const readStore = async (dbName: string, storeName: string, key?: string) => { + if (!existing.has(dbName)) return null; + const database = await settle(indexedDB.open(dbName)); + try { + if (!database.objectStoreNames.contains(storeName)) return null; + const store = database + .transaction(storeName, "readonly") + .objectStore(storeName); + return key === undefined + ? await settle(store.getAllKeys()) + : await settle(store.get(key)); + } finally { + database.close(); + } + }; + const keys = (await readStore( + `video-editor-media-${opencutId}`, + "media-metadata", + )) as IDBValidKey[] | null; + return { + binMediaIds: (keys ?? []).filter( + (key): key is string => typeof key === "string", + ), + hostMediaMap: JSON.parse( + localStorage.getItem(`host-media-map-${opencutId}`) ?? "{}", + ) as Record, + project: (await readStore("video-editor-projects", "projects", opencutId)) as unknown, + }; + }, hostProjectId); +} + +/** Share of the preview drawn in the test picture's colour. */ +async function previewColourShare(page: Page) { + const shot = await page + .getByRole("application", { name: "Preview canvas" }) + .screenshot(); + return page.evaluate( + async ({ base64, colour }) => { + const bytes = Uint8Array.from(atob(base64), (character) => + character.charCodeAt(0), + ); + const bitmap = await createImageBitmap( + new Blob([bytes], { type: "image/png" }), + ); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext("2d"); + if (!context) return 0; + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData(0, 0, bitmap.width, bitmap.height); + let matching = 0; + for (let index = 0; index < data.length; index += 4) { + if ( + Math.abs(data[index] - colour[0]) <= 40 && + Math.abs(data[index + 1] - colour[1]) <= 40 && + Math.abs(data[index + 2] - colour[2]) <= 40 + ) { + matching += 1; + } + } + return matching / (bitmap.width * bitmap.height); + }, + { base64: shot.toString("base64"), colour: PICTURE_COLOUR }, + ); +} + +async function showEditor(page: Page) { + await page.getByRole("tab", { name: "Editor", exact: true }).click(); + // The first visit compiles the Editor in dev mode. + await expect(page.locator(".opencut-scope")).toBeVisible({ timeout: 30_000 }); +} + +async function showCanvas(page: Page) { + await page.getByRole("tab", { name: "Canvas", exact: true }).click(); + await expect(page.locator(".opencut-scope")).toHaveCount(0); +} + +/** The edit document exists once the Editor has pushed its first state. */ +async function waitForEditDocument(request: APIRequestContext, projectId: string) { + await expect + .poll(async () => typeof (await readEditorDoc(request, projectId))?.revision, { + timeout: 60_000, + }) + .toBe("number"); +} + +/** The exact identity the Canvas resolves for an uploaded picture: what an + * agent's context hands it to place the file. */ +async function resolveIdentity( + request: APIRequestContext, + projectId: string, + upload: Upload, + title: string, +) { + const resolved = await request.post( + `/api/projects/${projectId}/canvas/context-identity`, + { + data: { + artifacts: [ + { + artifactId: upload.id, + kind: upload.kind, + sourcePath: upload.path, + title, + version: null, + }, + ], + mode: "resolve", + }, + }, + ); + expect(resolved.status()).toBe(200); + const [artifact] = ((await resolved.json()) as { artifacts: unknown[] }) + .artifacts; + return artifact; +} + +/** The insert an agent sends through its MCP server (editor_edit). Like an + * agent, it reads the edit document again and retries when an Editor push + * made its revision stale. */ +async function placeByAgent( + request: APIRequestContext, + projectId: string, + upload: Upload, + artifact: unknown, + elementId: string, +) { + for (let attempt = 1; ; attempt += 1) { + const doc = await readEditorDoc(request, projectId); + // The server writes a placeholder id only for a file its media map lacks. + // An open Editor sends its whole map with each push, so a push that lands + // between the import and this insert leaves nothing to test. + test.skip( + Object.keys(doc?.mediaMap ?? {}).includes(upload.id), + "the Editor pushed its media map, picture included, before the insert", + ); + const response = await request.post("/api/paper/tools", { + data: { + arguments: { + action: "insert", + actor: { id: "smoke-agent", type: "agent" }, + artifact, + baseEditorRevision: doc?.revision, + commandId: `insert-${elementId}`, + durationTicks: 4 * TICKS_PER_SECOND, + elementId, + idempotencyKey: `insert-${elementId}`, + origin: "mcp", + placement: { mode: "auto", trackType: "video" }, + projectId, + startTimeTicks: 0, + }, + tool: "editor_edit", + }, + headers: { authorization: `Bearer ${MCP_TOKEN}` }, + }); + const body = await response.text(); + if ( + response.status() === 409 && + body.includes("EDITOR_REVISION_CONFLICT") && + attempt < 3 + ) { + continue; + } + expect(response.status(), body).toBe(200); + break; + } + const placed = elementMediaId( + (await readEditorDoc(request, projectId))?.project, + elementId, + ); + expect(placed, "the insert carries the server's placeholder id").toMatch( + /^videofs-/, + ); +} + +/** The Editor draws a block from its own media bin: the block must end up + * pointing at media the bin holds, in the page and in the server copy the + * agent reads next, and the preview must show the picture. */ +async function expectEditorDrawsPlacement( + page: Page, + request: APIRequestContext, + projectId: string, + elementId: string, +) { + try { + await expect + .poll( + async () => { + const storage = await editorStorage(page, projectId); + const mediaId = elementMediaId(storage.project, elementId); + if (!mediaId) return "the block is not in the Editor's project"; + return storage.binMediaIds.includes(mediaId) + ? "held" + : `the block points at ${mediaId}, which the Editor's media bin does not hold`; + }, + { timeout: 30_000 }, + ) + .toBe("held"); + await expect + .poll( + async () => { + const storage = await editorStorage(page, projectId); + const doc = await readEditorDoc(request, projectId); + const mediaId = elementMediaId(doc?.project, elementId); + return mediaId && storage.binMediaIds.includes(mediaId) + ? "held" + : `the server copy points the block at ${mediaId}`; + }, + { timeout: 30_000 }, + ) + .toBe("held"); + await expect + .poll(() => previewColourShare(page), { timeout: 30_000 }) + .toBeGreaterThan(0.05); + } finally { + await page + .getByRole("application", { name: "Preview canvas" }) + .screenshot({ path: test.info().outputPath("preview.png") }) + .catch(() => undefined); + } +} + +test.describe("pictures an agent places on the Editor timeline", () => { + test.beforeEach(async ({ page }) => { + // The setup guide and the project tour dim the page, preview included. + await page.addInitScript(() => + localStorage.setItem( + "impractical-onboarding-v1", + JSON.stringify({ "studio-setup": true, "project-tour": true }), + ), + ); + }); + + test("show when the Editor already holds the picture", async ({ page, request }) => { + // The first visit compiles the project page and the Editor in dev mode. + test.slow(); + const project = await createProject(request, "Agent placement, picture in the bin"); + try { + await page.goto(`/projects/${project.id}`); + await showEditor(page); + await waitForEditDocument(request, project.id); + const upload = await uploadPicture(request, project.id, "in-the-bin.png"); + const artifact = await resolveIdentity(request, project.id, upload, "agent-picture-in-bin"); + // Insert as soon as the import lands, before the Editor's next push. + await expect + .poll( + async () => { + const storage = await editorStorage(page, project.id); + const mediaId = storage.hostMediaMap[upload.id]?.mediaId; + return Boolean(mediaId && storage.binMediaIds.includes(mediaId)); + }, + { + intervals: [100], + message: "the Editor imports the uploaded picture", + timeout: 60_000, + }, + ) + .toBe(true); + await placeByAgent(request, project.id, upload, artifact, "agent-picture-in-bin"); + await expectEditorDrawsPlacement(page, request, project.id, "agent-picture-in-bin"); + } finally { + await page.goto("/"); + expect((await request.delete(`/api/projects/${project.id}`)).status()).toBe(200); + } + }); + + test("show when the Editor opens after the agent placed the picture", async ({ page, request }) => { + test.slow(); + const project = await createProject(request, "Agent placement, Editor opened later"); + try { + await page.goto(`/projects/${project.id}`); + await showEditor(page); + await waitForEditDocument(request, project.id); + await showCanvas(page); + const upload = await uploadPicture(request, project.id, "placed-first.png"); + const artifact = await resolveIdentity(request, project.id, upload, "agent-picture-placed-first"); + await placeByAgent(request, project.id, upload, artifact, "agent-picture-placed-first"); + await showEditor(page); + await expectEditorDrawsPlacement(page, request, project.id, "agent-picture-placed-first"); + } finally { + await page.goto("/"); + expect((await request.delete(`/api/projects/${project.id}`)).status()).toBe(200); + } + }); +});