From ab9ff15a16750276df8099ed7b58aee8deb71fde Mon Sep 17 00:00:00 2001 From: anessbelbati <102622182+anessbelbati@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:58:20 +0100 Subject: [PATCH 1/3] Point agent-placed elements at the Editor's copy of their file When editor_edit inserts a Canvas file that the edit document's media map does not list yet, ensureArtifactEditorMedia writes a placeholder media id (videofs-) on the element and records the file's source key under it. The Editor draws nothing for a media id its bin does not hold, so the element is black in the preview and in an exported MP4, while the agent is told the insert succeeded. The placeholder was adopted only right after the browser imported that file for the first time, and only on the project loaded at that moment. Two common orders never reach that point: - the Editor already holds the file, so hydration skips it before any remap; - the Editor imports the file when it opens, before the sync tick loads the agent's edit, so the element is not there yet when the remap runs. The browser's next push then replaces the server's media map with its own, and the placeholder's entry is gone. Adoption now runs at the end of every hydration pass, for every host file the bin holds, and right after the sync tick loads an agent edit, in the queue that loads it. An element is pointed at the bin's copy of the file its placeholder entry names; with no entry left, at the file its artifact identity names: the exact version file for a version placement, the record's current file for a record placement. The old remap matched on the artifact id alone, so it could pick another version of the same artifact. tests/smoke/editor-agent-media.spec.ts places a picture through the editor_edit route in both orders and checks that the element ends up on media the bin holds, in the browser and in the server copy, and that the preview shows the picture. --- opencut/host/editor-mount.tsx | 162 +++++++---- opencut/host/placeholder-media.ts | 140 +++++++++ tests/placeholder-media.test.ts | 265 +++++++++++++++++ tests/smoke/editor-agent-media.spec.ts | 387 +++++++++++++++++++++++++ 4 files changed, 897 insertions(+), 57 deletions(-) create mode 100644 opencut/host/placeholder-media.ts create mode 100644 tests/placeholder-media.test.ts create mode 100644 tests/smoke/editor-agent-media.spec.ts diff --git a/opencut/host/editor-mount.tsx b/opencut/host/editor-mount.tsx index e9ce1df..647b3e9 100644 --- a/opencut/host/editor-mount.tsx +++ b/opencut/host/editor-mount.tsx @@ -77,6 +77,11 @@ import { type ArtifactFocusRequest, type ArtifactFocusResult, } from "./artifact-focus-contract"; +import { + heldMediaReplacements, + mediaMapSources, + type HeldArtifactMedia, +} from "./placeholder-media"; export type HostEditorMediaAsset = { artifact: { @@ -593,6 +598,72 @@ function rewriteProjectMediaIds({ }; } +/** The host asset list's files that the media bin holds. */ +function heldHostMedia({ + groupedAssets, + known, + map, +}: { + groupedAssets: ReturnType; + known: Set; + map: HostMediaMap; +}) { + const held: HeldArtifactMedia[] = []; + for (const [sourceKey, { aliases, asset }] of groupedAssets) { + const mapped = findMappedHostMedia({ aliases, map, sourceKey }); + if (!mapped || !known.has(mapped.mediaId)) continue; + held.push({ + artifactId: asset.artifact.artifactId, + hostAssetIds: [...aliases], + mediaId: mapped.mediaId, + sourceKey, + sourcePath: asset.artifact.sourcePath, + }); + } + return held; +} + +/** Point timeline elements whose media id the bin does not hold (an agent + * insert's `videofs-…` placeholder, or an id minted by another browser + * profile) at the bin's copy of the same file. */ +async function adoptHeldMedia({ + editor, + groupedAssets, + known, + map, + placeholderSources, + projectId, +}: { + editor: EditorCore; + groupedAssets: ReturnType; + known: Set; + map: HostMediaMap; + placeholderSources: ReadonlyMap; + projectId: string; +}) { + const activeProject = editor.project.getActiveOrNull(); + if (!activeProject || activeProject.metadata.id !== projectId) return false; + const replacements = heldMediaReplacements({ + heldMedia: heldHostMedia({ groupedAssets, known, map }), + knownMediaIds: known, + placeholderSources, + scenes: activeProject.scenes ?? [], + }); + if (!replacements.size) return false; + const rewritten = rewriteProjectMediaIds({ + project: activeProject, + replacements, + }); + if (!rewritten.changed) return false; + editor.scenes.setScenes({ + activeSceneId: rewritten.project.currentSceneId, + scenes: rewritten.project.scenes, + }); + editor.project.setActiveProject({ project: rewritten.project }); + await storageService.saveProject({ project: rewritten.project }); + return true; +} + async function reconcileDuplicateHostMedia({ editor, hostAssets, @@ -784,6 +855,10 @@ function useHostBridge( /** Revision of the last server doc this client synced against — sent as * baseRevision on pushes so a push can never clobber an unseen agent edit. */ const lastKnownRevisionRef = useRef(null); + const mediaAssetsRef = useRef(mediaAssets); + useEffect(() => { + mediaAssetsRef.current = mediaAssets; + }, [mediaAssets]); const mediaMapKey = opencutId ? `host-media-map-${opencutId}` : ""; // Media hydration: every host asset lands in the media bin exactly once. @@ -803,16 +878,14 @@ function useHostBridge( } const map = readHostMediaMap(mediaMapKey); const known = new Set(editor.media.getAssets().map((asset) => asset.id)); + // Read before imports overwrite entries: a map seeded from the + // server copy can hold placeholder ids that elements still use. + const placeholderSources = mediaMapSources(map); const groupedAssets = groupHostMediaAssets(mediaAssets); let mapChanged = false; for (const [sourceKey, { aliases, asset }] of groupedAssets) { if (cancelled) return; const mapped = findMappedHostMedia({ aliases, map, sourceKey }); - // Agent inserts may pre-seed the map server-side with a - // placeholder id; once the real asset is minted below, the - // placeholder is adopted by remapping timeline elements. - const seededMediaId = - mapped && !known.has(mapped.mediaId) ? mapped.mediaId : null; if (mapped && known.has(mapped.mediaId)) { if ( asset.artifact.contentHash && @@ -887,64 +960,24 @@ function useHostBridge( mapChanged = true; known.add(added.id); writeHostMediaMap(mediaMapKey, map); - const replacements = new Map(); - if (seededMediaId && seededMediaId !== added.id) { - replacements.set(seededMediaId, added.id); - } - // Elements carry their artifact identity; any element for - // THIS artifact whose media id is unknown here (seeded - // placeholder, or an id minted by another browser profile - // that has since left the map) is remapped to the fresh - // asset by that identity. - const activeForRemap = editor.project.getActiveOrNull(); - if (activeForRemap) { - for (const scene of activeForRemap.scenes ?? []) { - const tracks = scene.tracks as unknown as { - audio?: Array<{ elements?: Array> }>; - main?: { elements?: Array> }; - overlay?: Array<{ elements?: Array> }>; - }; - const allTrackElements = [ - ...(tracks?.main?.elements ?? []), - ...(tracks?.overlay ?? []).flatMap((t) => t.elements ?? []), - ...(tracks?.audio ?? []).flatMap((t) => t.elements ?? []), - ]; - for (const element of allTrackElements) { - const identity = element.videoFsArtifact as - | { artifactId?: string } - | undefined; - const mediaId = element.mediaId; - if ( - identity?.artifactId === asset.artifact.artifactId && - typeof mediaId === "string" && - mediaId !== added.id && - !known.has(mediaId) - ) { - replacements.set(mediaId, added.id); - } - } - } - if (replacements.size) { - const rewritten = rewriteProjectMediaIds({ - project: activeForRemap, - replacements, - }); - if (rewritten.changed) { - editor.scenes.setScenes({ - activeSceneId: rewritten.project.currentSceneId, - scenes: rewritten.project.scenes, - }); - editor.project.setActiveProject({ project: rewritten.project }); - await storageService.saveProject({ project: rewritten.project }); - } - } - } } } catch { /* skip failed asset; retried next mount */ } } if (!cancelled) { + // Every file the bin holds, not only those imported in this pass: + // an element can reference a file that was imported earlier. + await adoptHeldMedia({ + editor, + groupedAssets, + known, + map, + placeholderSources, + projectId: opencutId, + }).catch((caught) => + console.warn("[editor-sync] failed to adopt placeholder media", caught), + ); const didReconcile = await reconcileDuplicateHostMedia({ editor, hostAssets: mediaAssets, @@ -1000,6 +1033,21 @@ function useHostBridge( projectId: opencutId, revision: doc.revision, }); + // The edit can place a file the bin already holds under a + // placeholder id, and hydration does not run again for that + // file. Adopting it here, in the queue that applied the + // edit, lets the next tick push the repaired project. + const editor = EditorCore.getInstance(); + await adoptHeldMedia({ + editor, + groupedAssets: groupHostMediaAssets(mediaAssetsRef.current), + known: new Set(editor.media.getAssets().map((asset) => asset.id)), + map: readHostMediaMap(mediaMapKey), + placeholderSources: mediaMapSources(doc.mediaMap), + projectId: opencutId, + }).catch((caught) => + console.warn("[editor-sync] failed to adopt placeholder media", caught), + ); } catch (caught) { console.warn("[editor-sync] failed to apply agent doc", caught); } diff --git a/opencut/host/placeholder-media.ts b/opencut/host/placeholder-media.ts new file mode 100644 index 0000000..782bde8 --- /dev/null +++ b/opencut/host/placeholder-media.ts @@ -0,0 +1,140 @@ +import { artifactIdentityFromEditorValue } from "./editor-placement-contract"; + +/** A Canvas file the Editor's media bin holds. `sourcePath` is the version's + * project-relative file, or the record path of an unversioned artifact; + * `hostAssetIds` are the host asset ids resolving to the file (the artifact + * id itself names the artifact's current file). */ +export type HeldArtifactMedia = { + artifactId: string; + hostAssetIds: readonly string[]; + mediaId: string; + sourceKey: string; + sourcePath: string; +}; + +type UnknownRecord = Record; + +function record(value: unknown): UnknownRecord | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as UnknownRecord) + : null; +} + +/** The comparison the server applies to media source keys (pathname only, + * percent-decoded), so client and server agree on which file a key names. */ +export function mediaSourcePath(sourceKey: string | undefined) { + if (!sourceKey) return null; + try { + return decodeURIComponent( + new URL(sourceKey, "http://video-fs.local").pathname, + ); + } catch { + return null; + } +} + +function relativePath(value: string) { + return value.replaceAll("\\", "/").replace(/^\/+/, "").replace(/^(\.\/)+/, ""); +} + +/** Media id -> source key, for every entry of an edit document media map. */ +export function mediaMapSources(mediaMap: unknown) { + const sources = new Map(); + for (const value of Object.values(record(mediaMap) ?? {})) { + const entry = record(value); + if ( + typeof entry?.mediaId === "string" && + typeof entry.sourceKey === "string" + ) { + sources.set(entry.mediaId, entry.sourceKey); + } + } + return sources; +} + +function sceneElements(scenes: readonly unknown[]) { + const elements: UnknownRecord[] = []; + for (const rawScene of scenes) { + const tracks = record(record(rawScene)?.tracks); + const overlay = Array.isArray(tracks?.overlay) ? tracks.overlay : []; + const audio = Array.isArray(tracks?.audio) ? tracks.audio : []; + for (const track of [ + record(tracks?.main), + ...overlay.map(record), + ...audio.map(record), + ]) { + if (!track || !Array.isArray(track.elements)) continue; + for (const rawElement of track.elements) { + const element = record(rawElement); + if (element) elements.push(element); + } + } + } + return elements; +} + +function heldMediaForIdentity( + value: unknown, + heldMedia: readonly HeldArtifactMedia[], +) { + const identity = artifactIdentityFromEditorValue(value); + if (!identity) return null; + const sameArtifact = heldMedia.filter( + (media) => media.artifactId === identity.artifactId, + ); + if (identity.versionIndex !== null) { + // A version placement names its file; another version is not a stand-in. + const path = relativePath(identity.sourcePath); + return ( + sameArtifact.find((media) => relativePath(media.sourcePath) === path) ?? + null + ); + } + // A record placement shows the record's current file. + return ( + sameArtifact.find((media) => + media.hostAssetIds.includes(identity.artifactId), + ) ?? null + ); +} + +/** Agent inserts reference media through the edit document's media map. When + * that map has no entry for the exact file, the server writes a placeholder + * media id (`videofs-…`) on the element and records the file's source key + * under it (ensureArtifactEditorMedia). The Editor draws nothing for a media + * id its bin does not hold, so each such id maps to the bin id of the same + * file: the file its placeholder entry names (and none other, until the bin + * holds it), or, with no entry left, the file the element's artifact identity + * names. Ids the bin holds are never touched. */ +export function heldMediaReplacements({ + heldMedia, + knownMediaIds, + placeholderSources, + scenes, +}: { + heldMedia: readonly HeldArtifactMedia[]; + knownMediaIds: ReadonlySet; + placeholderSources: ReadonlyMap; + scenes: readonly unknown[]; +}) { + const usable = heldMedia.filter((media) => knownMediaIds.has(media.mediaId)); + const bySourcePath = new Map(); + for (const media of usable) { + const path = mediaSourcePath(media.sourceKey); + if (path && !bySourcePath.has(path)) bySourcePath.set(path, media); + } + const replacements = new Map(); + for (const element of sceneElements(scenes)) { + const mediaId = + typeof element.mediaId === "string" ? element.mediaId : null; + if (!mediaId || knownMediaIds.has(mediaId) || replacements.has(mediaId)) { + continue; + } + const placeholderPath = mediaSourcePath(placeholderSources.get(mediaId)); + const held = placeholderPath + ? bySourcePath.get(placeholderPath) + : heldMediaForIdentity(element.videoFsArtifact, usable); + if (held) replacements.set(mediaId, held.mediaId); + } + return replacements; +} diff --git a/tests/placeholder-media.test.ts b/tests/placeholder-media.test.ts new file mode 100644 index 0000000..a7c6b35 --- /dev/null +++ b/tests/placeholder-media.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from "vitest"; +import { + heldMediaReplacements, + mediaMapSources, + mediaSourcePath, + type HeldArtifactMedia, +} from "../opencut/host/placeholder-media"; + +const MEDIA = "/api/projects/project_one/media"; + +function recordIdentity(artifactId: string) { + return { + artifactId, + contentHash: "a".repeat(64), + entityRevision: 0, + path: `uploads/${artifactId}.md`, + version: null, + }; +} + +function versionIdentity(artifactId: string, index: number) { + return { + artifactId, + contentHash: "b".repeat(64), + entityRevision: index + 1, + path: `media/clips/${artifactId}.v${index + 1}.mp4`, + version: { index, sha256: "b".repeat(64), versionId: `v${index + 1}` }, + }; +} + +function scenes({ + audio = [], + main = [], + overlay = [], +}: { + audio?: unknown[]; + main?: unknown[]; + overlay?: unknown[]; +}) { + return [ + { + id: "scene_main", + tracks: { + audio: [{ elements: audio, id: "track_audio" }], + main: { elements: main, id: "track_main" }, + overlay: [{ elements: overlay, id: "track_overlay" }], + }, + }, + ]; +} + +const picture: HeldArtifactMedia = { + artifactId: "upload_picture", + hostAssetIds: ["upload_picture"], + mediaId: "bin-picture", + sourceKey: `${MEDIA}/uploads/upload_picture.png`, + sourcePath: "uploads/upload_picture.md", +}; + +function clipVersion(index: number, current: boolean): HeldArtifactMedia { + return { + artifactId: "clip_intro", + hostAssetIds: [current ? "clip_intro" : `clip_intro@v${index + 1}`], + mediaId: `bin-clip-v${index + 1}`, + sourceKey: `${MEDIA}/clips/clip_intro.v${index + 1}.mp4`, + sourcePath: `media/clips/clip_intro.v${index + 1}.mp4`, + }; +} + +describe("placeholder media adoption", () => { + it("points an agent placeholder at the bin's copy of the file its map entry names", () => { + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources: new Map([ + ["videofs-1", `${MEDIA}/uploads/upload_picture.png`], + ]), + scenes: scenes({ + main: [ + { + id: "element_agent", + mediaId: "videofs-1", + videoFsArtifact: recordIdentity("upload_picture"), + }, + ], + }), + }); + expect([...replacements]).toEqual([["videofs-1", "bin-picture"]]); + }); + + it("waits for the exact file a placeholder names instead of using another version", () => { + const element = { + id: "element_agent", + mediaId: "videofs-2", + videoFsArtifact: versionIdentity("clip_intro", 1), + }; + const placeholderSources = new Map([ + ["videofs-2", `${MEDIA}/clips/clip_intro.v2.mp4`], + ]); + const onlyFirstVersion = heldMediaReplacements({ + heldMedia: [clipVersion(0, false)], + knownMediaIds: new Set(["bin-clip-v1"]), + placeholderSources, + scenes: scenes({ main: [element] }), + }); + expect(onlyFirstVersion.size).toBe(0); + + const bothVersions = heldMediaReplacements({ + heldMedia: [clipVersion(0, false), clipVersion(1, true)], + knownMediaIds: new Set(["bin-clip-v1", "bin-clip-v2"]), + placeholderSources, + scenes: scenes({ main: [element] }), + }); + expect([...bothVersions]).toEqual([["videofs-2", "bin-clip-v2"]]); + }); + + it("adopts an id that left every map through the element's identity", () => { + const element = { + id: "element_agent", + mediaId: "minted-by-another-profile", + videoFsArtifact: recordIdentity("upload_picture"), + }; + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources: new Map(), + scenes: scenes({ main: [element] }), + }); + expect([...replacements]).toEqual([ + ["minted-by-another-profile", "bin-picture"], + ]); + }); + + it("gives a record placement only the record's current file", () => { + const element = { + id: "element_agent", + mediaId: "videofs-lost", + videoFsArtifact: recordIdentity("clip_intro"), + }; + const olderVersionOnly = heldMediaReplacements({ + heldMedia: [clipVersion(0, false)], + knownMediaIds: new Set(["bin-clip-v1"]), + placeholderSources: new Map(), + scenes: scenes({ main: [element] }), + }); + expect(olderVersionOnly.size).toBe(0); + + const withCurrent = heldMediaReplacements({ + heldMedia: [clipVersion(0, false), clipVersion(1, true)], + knownMediaIds: new Set(["bin-clip-v1", "bin-clip-v2"]), + placeholderSources: new Map(), + scenes: scenes({ main: [element] }), + }); + expect([...withCurrent]).toEqual([["videofs-lost", "bin-clip-v2"]]); + }); + + it("gives a version placement only its own version", () => { + const element = { + id: "element_agent", + mediaId: "videofs-lost", + videoFsArtifact: versionIdentity("clip_intro", 0), + }; + const replacements = heldMediaReplacements({ + heldMedia: [clipVersion(1, true), clipVersion(0, false)], + knownMediaIds: new Set(["bin-clip-v1", "bin-clip-v2"]), + placeholderSources: new Map(), + scenes: scenes({ main: [element] }), + }); + expect([...replacements]).toEqual([["videofs-lost", "bin-clip-v1"]]); + }); + + it("never touches ids the bin holds, elements without media, or elements it cannot place", () => { + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources: new Map(), + scenes: scenes({ + main: [ + { + id: "element_held", + mediaId: "bin-picture", + videoFsArtifact: recordIdentity("upload_picture"), + }, + { id: "element_text", type: "text" }, + { id: "element_by_hand", mediaId: "lost-without-identity" }, + { + id: "element_other_artifact", + mediaId: "videofs-other", + videoFsArtifact: recordIdentity("upload_other"), + }, + ], + }), + }); + expect(replacements.size).toBe(0); + }); + + it("does not use a file the bin no longer holds", () => { + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(), + placeholderSources: new Map([ + ["videofs-1", `${MEDIA}/uploads/upload_picture.png`], + ]), + scenes: scenes({ + main: [ + { + id: "element_agent", + mediaId: "videofs-1", + videoFsArtifact: recordIdentity("upload_picture"), + }, + ], + }), + }); + expect(replacements.size).toBe(0); + }); + + it("reads overlay and audio tracks and maps a shared placeholder once", () => { + const placeholderSources = new Map([ + ["videofs-1", `${MEDIA}/uploads/upload_picture.png`], + ]); + const replacements = heldMediaReplacements({ + heldMedia: [picture], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources, + scenes: scenes({ + audio: [{ id: "element_audio", mediaId: "videofs-1" }], + overlay: [{ id: "element_overlay", mediaId: "videofs-1" }], + }), + }); + expect([...replacements]).toEqual([["videofs-1", "bin-picture"]]); + }); + + it("compares source keys the way the server does", () => { + expect(mediaSourcePath(`${MEDIA}/uploads/a%20b.png?v=3`)).toBe( + `${MEDIA}/uploads/a b.png`, + ); + expect(mediaSourcePath(undefined)).toBeNull(); + const replacements = heldMediaReplacements({ + heldMedia: [ + { ...picture, sourceKey: `${MEDIA}/uploads/upload%20picture.png?t=1` }, + ], + knownMediaIds: new Set(["bin-picture"]), + placeholderSources: new Map([ + ["videofs-1", `${MEDIA}/uploads/upload picture.png`], + ]), + scenes: scenes({ main: [{ id: "element_agent", mediaId: "videofs-1" }] }), + }); + expect([...replacements]).toEqual([["videofs-1", "bin-picture"]]); + }); + + it("lists every media map entry that has a media id and a source key", () => { + const sources = mediaMapSources({ + upload_picture: { + mediaId: "videofs-1", + sourceKey: `${MEDIA}/uploads/upload_picture.png`, + }, + "videofs:upload_picture": { mediaId: "videofs-2" }, + broken: null, + }); + expect([...sources]).toEqual([ + ["videofs-1", `${MEDIA}/uploads/upload_picture.png`], + ]); + expect(mediaMapSources(undefined).size).toBe(0); + }); +}); diff --git a/tests/smoke/editor-agent-media.spec.ts b/tests/smoke/editor-agent-media.spec.ts new file mode 100644 index 0000000..51156a1 --- /dev/null +++ b/tests/smoke/editor-agent-media.spec.ts @@ -0,0 +1,387 @@ +import { deflateSync } from "node:zlib"; +import { + expect, + test, + type APIRequestContext, + type Page, +} from "@playwright/test"; + +// scripts/smoke-server.mjs starts the app with this tool token; an agent's +// MCP server reaches the same route with the project's own token. +const MCP_TOKEN = "smoke-only-token-not-a-production-credential"; +const TICKS_PER_SECOND = 120_000; +const PICTURE_COLOUR = [230, 40, 200] as const; + +type Upload = { id: string; kind: string; path: string }; +type EditorDoc = { + mediaMap?: Record; + project?: unknown; + revision?: number; +} | null; + +function crc32(bytes: Uint8Array) { + let crc = 0xffffffff; + for (const byte of bytes) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function pngChunk(type: string, data: Buffer) { + const body = Buffer.concat([Buffer.from(type, "ascii"), data]); + const chunk = Buffer.alloc(body.length + 8); + chunk.writeUInt32BE(data.length, 0); + body.copy(chunk, 4); + chunk.writeUInt32BE(crc32(body), body.length + 4); + return chunk; +} + +/** A picture of one colour, so the preview can be checked for that colour. */ +function solidPng(width: number, height: number, colour: readonly number[]) { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 2; + const row = Buffer.alloc(1 + width * 3); + for (let x = 0; x < width; x += 1) row.set(colour, 1 + x * 3); + const pixels = Buffer.concat(Array.from({ length: height }, () => row)); + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", header), + pngChunk("IDAT", deflateSync(pixels)), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +async function createProject(request: APIRequestContext, name: string) { + const created = await request.post("/api/projects", { data: { name } }); + expect(created.status()).toBe(200); + return ((await created.json()) as { project: { id: string } }).project; +} + +async function uploadPicture( + request: APIRequestContext, + projectId: string, + name: string, +) { + const response = await request.post( + `/api/projects/${projectId}/canvas/upload`, + { + multipart: { + files: { + buffer: solidPng(960, 540, PICTURE_COLOUR), + mimeType: "image/png", + name, + }, + }, + }, + ); + expect(response.status()).toBe(200); + const body = (await response.json()) as { uploaded: Upload[] }; + expect(body.uploaded).toHaveLength(1); + return body.uploaded[0]!; +} + +async function readEditorDoc( + request: APIRequestContext, + projectId: string, +): Promise { + const response = await request.get(`/api/projects/${projectId}/editor-doc`); + expect(response.status()).toBe(200); + return (await response.json()) as EditorDoc; +} + +function elementMediaId(project: unknown, elementId: string) { + type Track = { elements?: Array<{ id?: unknown; mediaId?: unknown }> }; + const scenes = + ( + project as { + scenes?: Array<{ + tracks?: { audio?: Track[]; main?: Track; overlay?: Track[] }; + }>; + } | null + )?.scenes ?? []; + for (const scene of scenes) { + const tracks = scene.tracks; + for (const track of [ + tracks?.main, + ...(tracks?.overlay ?? []), + ...(tracks?.audio ?? []), + ]) { + for (const element of track?.elements ?? []) { + if (element.id !== elementId) continue; + return typeof element.mediaId === "string" ? element.mediaId : null; + } + } + } + return null; +} + +/** What the Editor in the page works from: its own copy of the project and + * the ids of the media in its bin, as it stores them in the browser. */ +async function editorStorage(page: Page, hostProjectId: string) { + return page.evaluate(async (hostProjectId) => { + const projectMap = JSON.parse( + localStorage.getItem("host-project-map") ?? "{}", + ) as Record; + const opencutId = projectMap[hostProjectId]; + if (typeof opencutId !== "string") { + return { + binMediaIds: [] as string[], + hostMediaMap: {} as Record, + project: null as unknown, + }; + } + const existing = new Set( + (await indexedDB.databases()).map((database) => database.name), + ); + const settle = (request: IDBRequest) => + new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + // Opens only databases the app created, so reading never creates one. + const readStore = async (dbName: string, storeName: string, key?: string) => { + if (!existing.has(dbName)) return null; + const database = await settle(indexedDB.open(dbName)); + try { + if (!database.objectStoreNames.contains(storeName)) return null; + const store = database + .transaction(storeName, "readonly") + .objectStore(storeName); + return key === undefined + ? await settle(store.getAllKeys()) + : await settle(store.get(key)); + } finally { + database.close(); + } + }; + const keys = (await readStore( + `video-editor-media-${opencutId}`, + "media-metadata", + )) as IDBValidKey[] | null; + return { + binMediaIds: (keys ?? []).filter( + (key): key is string => typeof key === "string", + ), + hostMediaMap: JSON.parse( + localStorage.getItem(`host-media-map-${opencutId}`) ?? "{}", + ) as Record, + project: (await readStore("video-editor-projects", "projects", opencutId)) as unknown, + }; + }, hostProjectId); +} + +/** Share of the preview drawn in the test picture's colour. */ +async function previewColourShare(page: Page) { + const shot = await page + .getByRole("application", { name: "Preview canvas" }) + .screenshot(); + return page.evaluate( + async ({ base64, colour }) => { + const bytes = Uint8Array.from(atob(base64), (character) => + character.charCodeAt(0), + ); + const bitmap = await createImageBitmap( + new Blob([bytes], { type: "image/png" }), + ); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext("2d"); + if (!context) return 0; + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData(0, 0, bitmap.width, bitmap.height); + let matching = 0; + for (let index = 0; index < data.length; index += 4) { + if ( + Math.abs(data[index] - colour[0]) <= 40 && + Math.abs(data[index + 1] - colour[1]) <= 40 && + Math.abs(data[index + 2] - colour[2]) <= 40 + ) { + matching += 1; + } + } + return matching / (bitmap.width * bitmap.height); + }, + { base64: shot.toString("base64"), colour: PICTURE_COLOUR }, + ); +} + +async function showEditor(page: Page) { + await page.getByRole("tab", { name: "Editor", exact: true }).click(); + await expect(page.locator(".opencut-scope")).toBeVisible(); +} + +async function showCanvas(page: Page) { + await page.getByRole("tab", { name: "Canvas", exact: true }).click(); + await expect(page.locator(".opencut-scope")).toHaveCount(0); +} + +/** The edit document exists once the Editor has pushed its first state. */ +async function waitForEditDocument(request: APIRequestContext, projectId: string) { + await expect + .poll(async () => typeof (await readEditorDoc(request, projectId))?.revision, { + timeout: 60_000, + }) + .toBe("number"); +} + +/** The insert an agent sends through its MCP server (editor_edit), with the + * exact identity the Canvas resolves for the picture. */ +async function placeByAgent( + request: APIRequestContext, + projectId: string, + upload: Upload, + elementId: string, +) { + const resolved = await request.post( + `/api/projects/${projectId}/canvas/context-identity`, + { + data: { + artifacts: [ + { + artifactId: upload.id, + kind: upload.kind, + sourcePath: upload.path, + title: elementId, + version: null, + }, + ], + mode: "resolve", + }, + }, + ); + expect(resolved.status()).toBe(200); + const [artifact] = ((await resolved.json()) as { artifacts: unknown[] }) + .artifacts; + const doc = await readEditorDoc(request, projectId); + expect( + Object.keys(doc?.mediaMap ?? {}), + "the server's media map must not know the picture yet", + ).not.toContain(upload.id); + const response = await request.post("/api/paper/tools", { + data: { + arguments: { + action: "insert", + actor: { id: "smoke-agent", type: "agent" }, + artifact, + baseEditorRevision: doc?.revision, + commandId: `insert-${elementId}`, + durationTicks: 4 * TICKS_PER_SECOND, + elementId, + idempotencyKey: `insert-${elementId}`, + origin: "mcp", + placement: { mode: "auto", trackType: "video" }, + projectId, + startTimeTicks: 0, + }, + tool: "editor_edit", + }, + headers: { authorization: `Bearer ${MCP_TOKEN}` }, + }); + expect(response.status(), await response.text()).toBe(200); + const placed = elementMediaId( + (await readEditorDoc(request, projectId))?.project, + elementId, + ); + expect(placed, "the insert carries the server's placeholder id").toMatch( + /^videofs-/, + ); +} + +/** The Editor draws a block from its own media bin: the block must end up + * pointing at media the bin holds, in the page and in the server copy the + * agent reads next, and the preview must show the picture. */ +async function expectEditorDrawsPlacement( + page: Page, + request: APIRequestContext, + projectId: string, + elementId: string, +) { + try { + await expect + .poll( + async () => { + const storage = await editorStorage(page, projectId); + const mediaId = elementMediaId(storage.project, elementId); + if (!mediaId) return "the block is not in the Editor's project"; + return storage.binMediaIds.includes(mediaId) + ? "held" + : `the block points at ${mediaId}, which the Editor's media bin does not hold`; + }, + { timeout: 30_000 }, + ) + .toBe("held"); + await expect + .poll( + async () => { + const storage = await editorStorage(page, projectId); + const doc = await readEditorDoc(request, projectId); + const mediaId = elementMediaId(doc?.project, elementId); + return mediaId && storage.binMediaIds.includes(mediaId) + ? "held" + : `the server copy points the block at ${mediaId}`; + }, + { timeout: 30_000 }, + ) + .toBe("held"); + await expect + .poll(() => previewColourShare(page), { timeout: 30_000 }) + .toBeGreaterThan(0.05); + } finally { + await page + .getByRole("application", { name: "Preview canvas" }) + .screenshot({ path: test.info().outputPath("preview.png") }) + .catch(() => undefined); + } +} + +test.describe("pictures an agent places on the Editor timeline", () => { + test("show when the Editor already holds the picture", async ({ page, request }) => { + // The first visit compiles the project page and the Editor in dev mode. + test.slow(); + const project = await createProject(request, "Agent placement, picture in the bin"); + try { + await page.goto(`/projects/${project.id}`); + await showEditor(page); + await waitForEditDocument(request, project.id); + const upload = await uploadPicture(request, project.id, "in-the-bin.png"); + await expect + .poll( + async () => { + const storage = await editorStorage(page, project.id); + const mediaId = storage.hostMediaMap[upload.id]?.mediaId; + return Boolean(mediaId && storage.binMediaIds.includes(mediaId)); + }, + { timeout: 60_000, message: "the Editor imports the uploaded picture" }, + ) + .toBe(true); + await placeByAgent(request, project.id, upload, "agent-picture-in-bin"); + await expectEditorDrawsPlacement(page, request, project.id, "agent-picture-in-bin"); + } finally { + await page.goto("/"); + expect((await request.delete(`/api/projects/${project.id}`)).status()).toBe(200); + } + }); + + test("show when the Editor opens after the agent placed the picture", async ({ page, request }) => { + test.slow(); + const project = await createProject(request, "Agent placement, Editor opened later"); + try { + await page.goto(`/projects/${project.id}`); + await showEditor(page); + await waitForEditDocument(request, project.id); + await showCanvas(page); + const upload = await uploadPicture(request, project.id, "placed-first.png"); + await placeByAgent(request, project.id, upload, "agent-picture-placed-first"); + await showEditor(page); + await expectEditorDrawsPlacement(page, request, project.id, "agent-picture-placed-first"); + } finally { + await page.goto("/"); + expect((await request.delete(`/api/projects/${project.id}`)).status()).toBe(200); + } + }); +}); From 8cb7d4d4390b7c66b94e68e7033ffa6272a2404b Mon Sep 17 00:00:00 2001 From: anessbelbati <102622182+anessbelbati@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:17:40 +0100 Subject: [PATCH 2/3] Apply agent edits in the hydration queue Loading the project empties the Editor's media bin and refills it from storage. The sync tick loads the project to apply an agent edit, and its first run comes as the Editor opens, while hydration may be importing a file. When the two overlap, the import can drop out of the bin in memory while storage keeps it: the timeline points at a file the bin no longer holds, and the preview stays black until the next load. The browser test that opens the Editor after an agent placed a picture shows this on main too: the block is pointed at the bin's copy of the picture and the preview stays black (8 of 10 runs on Linux). The agent edit is now applied in the hydration queue, so a reload never overlaps an import. The browser test now turns the setup guide and the project tour off (the tour dims the preview), inserts as soon as the import lands, and skips when the Editor pushed its media map before the insert, since the server then writes no placeholder. --- opencut/host/editor-mount.tsx | 86 +++++++++++++++----------- tests/smoke/editor-agent-media.spec.ts | 55 ++++++++++++---- 2 files changed, 94 insertions(+), 47 deletions(-) diff --git a/opencut/host/editor-mount.tsx b/opencut/host/editor-mount.tsx index 647b3e9..5c010c8 100644 --- a/opencut/host/editor-mount.tsx +++ b/opencut/host/editor-mount.tsx @@ -1016,41 +1016,57 @@ function useHostBridge( doc.revision > lastAppliedRevisionRef.current && doc.project ) { - try { - // The wire doc is raw JSON — dates are STRINGS. saveProject - // calls toISOString() on them, so hydrate first or the apply - // throws and the agent's edit silently never reaches the - // editor (and the next push clobbers it). - await storageService.saveProject({ - project: hydrateWireProject(doc.project), - }); - lastPushedRef.current = JSON.stringify(doc.project); - await EditorCore.getInstance().project.loadProject({ id: opencutId }); - // Mark applied ONLY on success so a transient failure retries. - lastAppliedRevisionRef.current = doc.revision; - appliedAgentDoc = true; - dispatchEditorDocumentRevision(window, { - projectId: opencutId, - revision: doc.revision, - }); - // The edit can place a file the bin already holds under a - // placeholder id, and hydration does not run again for that - // file. Adopting it here, in the queue that applied the - // edit, lets the next tick push the repaired project. - const editor = EditorCore.getInstance(); - await adoptHeldMedia({ - editor, - groupedAssets: groupHostMediaAssets(mediaAssetsRef.current), - known: new Set(editor.media.getAssets().map((asset) => asset.id)), - map: readHostMediaMap(mediaMapKey), - placeholderSources: mediaMapSources(doc.mediaMap), - projectId: opencutId, - }).catch((caught) => - console.warn("[editor-sync] failed to adopt placeholder media", caught), - ); - } catch (caught) { - console.warn("[editor-sync] failed to apply agent doc", caught); - } + const revision: number = doc.revision; + const applyAgentDoc = async () => { + if (cancelled) return false; + let applied = false; + try { + // The wire doc is raw JSON — dates are STRINGS. saveProject + // calls toISOString() on them, so hydrate first or the apply + // throws and the agent's edit silently never reaches the + // editor (and the next push clobbers it). + await storageService.saveProject({ + project: hydrateWireProject(doc.project), + }); + lastPushedRef.current = JSON.stringify(doc.project); + await EditorCore.getInstance().project.loadProject({ id: opencutId }); + // Mark applied ONLY on success so a transient failure retries. + lastAppliedRevisionRef.current = revision; + applied = true; + dispatchEditorDocumentRevision(window, { + projectId: opencutId, + revision, + }); + // The edit can place a file the bin already holds under a + // placeholder id, and hydration does not run again for that + // file. Adopting it here lets the next tick push the + // repaired project. + const editor = EditorCore.getInstance(); + await adoptHeldMedia({ + editor, + groupedAssets: groupHostMediaAssets(mediaAssetsRef.current), + known: new Set(editor.media.getAssets().map((asset) => asset.id)), + map: readHostMediaMap(mediaMapKey), + placeholderSources: mediaMapSources(doc.mediaMap), + projectId: opencutId, + }).catch((caught) => + console.warn("[editor-sync] failed to adopt placeholder media", caught), + ); + } catch (caught) { + console.warn("[editor-sync] failed to apply agent doc", caught); + } + return applied; + }; + // Loading the project empties the media bin and refills it from + // storage. A hydration pass importing a file meanwhile can lose + // that file from the bin in memory while storage keeps it, and an + // element pointed at it then draws nothing. Applying the edit in + // the hydration queue keeps the two apart; it also runs after a + // pass that threw. + const queued = hydrationQueueRef.current.then(applyAgentDoc, applyAgentDoc); + hydrationQueueRef.current = queued.then(() => undefined); + appliedAgentDoc = await queued; + if (cancelled) return; } if (!appliedAgentDoc) { const loaded = await storageService.loadProject({ id: opencutId }); diff --git a/tests/smoke/editor-agent-media.spec.ts b/tests/smoke/editor-agent-media.spec.ts index 51156a1..349a475 100644 --- a/tests/smoke/editor-agent-media.spec.ts +++ b/tests/smoke/editor-agent-media.spec.ts @@ -229,13 +229,13 @@ async function waitForEditDocument(request: APIRequestContext, projectId: string .toBe("number"); } -/** The insert an agent sends through its MCP server (editor_edit), with the - * exact identity the Canvas resolves for the picture. */ -async function placeByAgent( +/** The exact identity the Canvas resolves for an uploaded picture: what an + * agent's context hands it to place the file. */ +async function resolveIdentity( request: APIRequestContext, projectId: string, upload: Upload, - elementId: string, + title: string, ) { const resolved = await request.post( `/api/projects/${projectId}/canvas/context-identity`, @@ -246,7 +246,7 @@ async function placeByAgent( artifactId: upload.id, kind: upload.kind, sourcePath: upload.path, - title: elementId, + title, version: null, }, ], @@ -257,11 +257,25 @@ async function placeByAgent( expect(resolved.status()).toBe(200); const [artifact] = ((await resolved.json()) as { artifacts: unknown[] }) .artifacts; + return artifact; +} + +/** The insert an agent sends through its MCP server (editor_edit). */ +async function placeByAgent( + request: APIRequestContext, + projectId: string, + upload: Upload, + artifact: unknown, + elementId: string, +) { const doc = await readEditorDoc(request, projectId); - expect( - Object.keys(doc?.mediaMap ?? {}), - "the server's media map must not know the picture yet", - ).not.toContain(upload.id); + // The server writes a placeholder id only for a file its media map lacks. + // An open Editor sends its whole map with each push, so a push that lands + // between the import and this insert leaves nothing to test. + test.skip( + Object.keys(doc?.mediaMap ?? {}).includes(upload.id), + "the Editor pushed its media map, picture included, before the insert", + ); const response = await request.post("/api/paper/tools", { data: { arguments: { @@ -340,6 +354,16 @@ async function expectEditorDrawsPlacement( } test.describe("pictures an agent places on the Editor timeline", () => { + test.beforeEach(async ({ page }) => { + // The setup guide and the project tour dim the page, preview included. + await page.addInitScript(() => + localStorage.setItem( + "impractical-onboarding-v1", + JSON.stringify({ "studio-setup": true, "project-tour": true }), + ), + ); + }); + test("show when the Editor already holds the picture", async ({ page, request }) => { // The first visit compiles the project page and the Editor in dev mode. test.slow(); @@ -349,6 +373,8 @@ test.describe("pictures an agent places on the Editor timeline", () => { await showEditor(page); await waitForEditDocument(request, project.id); const upload = await uploadPicture(request, project.id, "in-the-bin.png"); + const artifact = await resolveIdentity(request, project.id, upload, "agent-picture-in-bin"); + // Insert as soon as the import lands, before the Editor's next push. await expect .poll( async () => { @@ -356,10 +382,14 @@ test.describe("pictures an agent places on the Editor timeline", () => { const mediaId = storage.hostMediaMap[upload.id]?.mediaId; return Boolean(mediaId && storage.binMediaIds.includes(mediaId)); }, - { timeout: 60_000, message: "the Editor imports the uploaded picture" }, + { + intervals: [100], + message: "the Editor imports the uploaded picture", + timeout: 60_000, + }, ) .toBe(true); - await placeByAgent(request, project.id, upload, "agent-picture-in-bin"); + await placeByAgent(request, project.id, upload, artifact, "agent-picture-in-bin"); await expectEditorDrawsPlacement(page, request, project.id, "agent-picture-in-bin"); } finally { await page.goto("/"); @@ -376,7 +406,8 @@ test.describe("pictures an agent places on the Editor timeline", () => { await waitForEditDocument(request, project.id); await showCanvas(page); const upload = await uploadPicture(request, project.id, "placed-first.png"); - await placeByAgent(request, project.id, upload, "agent-picture-placed-first"); + const artifact = await resolveIdentity(request, project.id, upload, "agent-picture-placed-first"); + await placeByAgent(request, project.id, upload, artifact, "agent-picture-placed-first"); await showEditor(page); await expectEditorDrawsPlacement(page, request, project.id, "agent-picture-placed-first"); } finally { From 912741155bf14f9bee5672a8d4756b49437c9f35 Mon Sep 17 00:00:00 2001 From: anessbelbati <102622182+anessbelbati@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:26:36 +0100 Subject: [PATCH 3/3] Retry a stale insert in the browser test and wait for the Editor An Editor push between reading the edit document and the insert makes the insert's base revision stale. The test now reads the document again and retries, as the conflict answer tells an agent to; when that push carried the picture in its media map, the try is skipped, since the server then writes no placeholder. The Editor's first appearance gets 30 s: in a run of the whole suite it compiles on that visit. --- tests/smoke/editor-agent-media.spec.ts | 74 +++++++++++++++----------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/tests/smoke/editor-agent-media.spec.ts b/tests/smoke/editor-agent-media.spec.ts index 349a475..11ea2c9 100644 --- a/tests/smoke/editor-agent-media.spec.ts +++ b/tests/smoke/editor-agent-media.spec.ts @@ -212,7 +212,8 @@ async function previewColourShare(page: Page) { async function showEditor(page: Page) { await page.getByRole("tab", { name: "Editor", exact: true }).click(); - await expect(page.locator(".opencut-scope")).toBeVisible(); + // The first visit compiles the Editor in dev mode. + await expect(page.locator(".opencut-scope")).toBeVisible({ timeout: 30_000 }); } async function showCanvas(page: Page) { @@ -260,7 +261,9 @@ async function resolveIdentity( return artifact; } -/** The insert an agent sends through its MCP server (editor_edit). */ +/** The insert an agent sends through its MCP server (editor_edit). Like an + * agent, it reads the edit document again and retries when an Editor push + * made its revision stale. */ async function placeByAgent( request: APIRequestContext, projectId: string, @@ -268,35 +271,46 @@ async function placeByAgent( artifact: unknown, elementId: string, ) { - const doc = await readEditorDoc(request, projectId); - // The server writes a placeholder id only for a file its media map lacks. - // An open Editor sends its whole map with each push, so a push that lands - // between the import and this insert leaves nothing to test. - test.skip( - Object.keys(doc?.mediaMap ?? {}).includes(upload.id), - "the Editor pushed its media map, picture included, before the insert", - ); - const response = await request.post("/api/paper/tools", { - data: { - arguments: { - action: "insert", - actor: { id: "smoke-agent", type: "agent" }, - artifact, - baseEditorRevision: doc?.revision, - commandId: `insert-${elementId}`, - durationTicks: 4 * TICKS_PER_SECOND, - elementId, - idempotencyKey: `insert-${elementId}`, - origin: "mcp", - placement: { mode: "auto", trackType: "video" }, - projectId, - startTimeTicks: 0, + for (let attempt = 1; ; attempt += 1) { + const doc = await readEditorDoc(request, projectId); + // The server writes a placeholder id only for a file its media map lacks. + // An open Editor sends its whole map with each push, so a push that lands + // between the import and this insert leaves nothing to test. + test.skip( + Object.keys(doc?.mediaMap ?? {}).includes(upload.id), + "the Editor pushed its media map, picture included, before the insert", + ); + const response = await request.post("/api/paper/tools", { + data: { + arguments: { + action: "insert", + actor: { id: "smoke-agent", type: "agent" }, + artifact, + baseEditorRevision: doc?.revision, + commandId: `insert-${elementId}`, + durationTicks: 4 * TICKS_PER_SECOND, + elementId, + idempotencyKey: `insert-${elementId}`, + origin: "mcp", + placement: { mode: "auto", trackType: "video" }, + projectId, + startTimeTicks: 0, + }, + tool: "editor_edit", }, - tool: "editor_edit", - }, - headers: { authorization: `Bearer ${MCP_TOKEN}` }, - }); - expect(response.status(), await response.text()).toBe(200); + headers: { authorization: `Bearer ${MCP_TOKEN}` }, + }); + const body = await response.text(); + if ( + response.status() === 409 && + body.includes("EDITOR_REVISION_CONFLICT") && + attempt < 3 + ) { + continue; + } + expect(response.status(), body).toBe(200); + break; + } const placed = elementMediaId( (await readEditorDoc(request, projectId))?.project, elementId,