feat(runtime): record scenes in desktop electron apps

.github/workflows/ci.yml

@@ -31,17 +31,43 @@ jobs:
       - run: pnpm lint
       - run: pnpm format:check
       - run: pnpm typecheck
-      - run: pnpm test:coverage
-      # The desktop app, built and launched under a virtual display. Ubuntu 24 restricts the user
-      # namespaces Chromium's sandbox needs: allowed back, so the app runs sandboxed as shipped.
+      # Ubuntu 24 restricts the user namespaces Chromium's sandbox needs: allowed back (Electron runs
+      # sandboxed as shipped). The tests run under a virtual display: the Electron target's tests
+      # launch a fixture desktop app.
+      - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+      - run: xvfb-run -a pnpm test:coverage
+        env:
+          # Slow shutdown phases of a desktop app's launch, and its failures' detail, said (the
+          # Electron target's evidence).
+          KIFRAME_ELECTRON_TIMING: "1"
+          KIFRAME_ELECTRON_DEBUG: "1"
+      # The desktop app, built and launched under a virtual display.
       - name: Desktop end-to-end
-        run: |
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
-          xvfb-run -a pnpm --filter @kiframe/desktop e2e
+        run: xvfb-run -a pnpm --filter @kiframe/desktop e2e
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage/lcov.info
           fail_ci_if_error: false
+
+  # Desktop apps run on macOS only (Seatbelt, codesign, plutil): their suite, where it holds.
+  electron-macos:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      # A desktop app's takes are encoded with ffmpeg (as on Linux).
+      - run: brew install ffmpeg
+      # The studio's desktop tests drive a web page beside the app too.
+      - run: pnpm --filter @kiframe/runtime exec playwright install chromium
+      - run: pnpm exec vitest run --project electron
+        env:
+          KIFRAME_ELECTRON_TIMING: "1"
+          KIFRAME_ELECTRON_DEBUG: "1"

apps/desktop/e2e/desktop-app.e2e.ts

@@ -0,0 +1,149 @@
+// "Add desktop app…" end to end (macOS: Seatbelt, codesign): the built app adds a real bundle (the
+// repo's Electron, the runtime's fixture as its app, re-signed ad hoc) picked through main's
+// dialog (stubbed), inspected, tried confined, written to the project and approved. Its work area
+// is a test's own (never ~/.kiframe).
+import { execFileSync } from "node:child_process"
+import { cpSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { createRequire } from "node:module"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { createProject } from "@kiframe/project"
+import { _electron as electron, type ElectronApplication, type Page } from "playwright"
+import { afterAll, beforeAll, describe, expect, it } from "vitest"
+
+const appDir = join(import.meta.dirname, "..")
+const mac = process.platform === "darwin"
+let app: ElectronApplication | undefined
+let page: Page
+let root: string
+let bundle: string
+let work: string
+
+beforeAll(async () => {
+  if (!mac) return
+  root = mkdtempSync(join(tmpdir(), "kiframe-e2e-desktop-"))
+  // A real bundle with no hook in it: Electron cloned, the fixture as its app, re-signed.
+  const electronExe = createRequire(join(appDir, "package.json"))("electron") as string
+  bundle = join(root, "Notes.app")
+  execFileSync("cp", ["-cR", electronExe.slice(0, electronExe.indexOf(".app/") + 4), bundle])
+  cpSync(
+    join(appDir, "../../packages/runtime/test/fixtures/electron-app"),
+    join(bundle, "Contents/Resources/app"),
+    { recursive: true },
+  )
+  const plist = join(bundle, "Contents/Info.plist")
+  execFileSync("plutil", [
+    "-replace",
+    "CFBundleIdentifier",
+    "-string",
+    "com.kiframe.e2e-notes",
+    plist,
+  ])
+  for (const key of ["CFBundleName", "CFBundleDisplayName"]) {
+    execFileSync("plutil", ["-replace", key, "-string", "Notes", plist])
+  }
+  execFileSync("codesign", ["-s", "-", "-f", "--deep", bundle], { stdio: "ignore" })
+  work = join(root, "work")
+  const profile = join(root, "profile")
+  const env = Object.fromEntries(
+    Object.entries(process.env).filter(
+      (e): e is [string, string] =>
+        e[1] !== undefined && e[0] !== "ELECTRON_RENDERER_URL" && e[0] !== "ELECTRON_RUN_AS_NODE",
+    ),
+  )
+  env.KIFRAME_TEST_KEYCHAIN = "memory"
+  env.KIFRAME_WORK_DIR = work
+  // Kif, scripted: a step in the desktop app once it's added (launched from its approval).
+  const model = join(root, "model.json")
+  writeFileSync(
+    model,
+    JSON.stringify([
+      {
+        kind: "tool_calls",
+        calls: [
+          {
+            id: "d1",
+            name: "run_step",
+            arguments: JSON.stringify({
+              scene: "tour",
+              start_app: "notes",
+              step: {
+                id: "add",
+                action: "click",
+                target: { by: "role", role: "button", name: "Add note" },
+              },
+            }),
+          },
+        ],
+      },
+      { kind: "text", text: "Added a note in Notes." },
+    ]),
+  )
+  env.KIFRAME_TEST_MODEL = model
+  app = await electron.launch({ args: [appDir, `--user-data-dir=${profile}`], cwd: appDir, env })
+  page = await app.firstWindow()
+  await page.getByLabel("OpenRouter API key").fill("sk-or-test-not-a-real-key")
+  await page.getByRole("button", { name: "Save key" }).click()
+}, 120_000)
+
+afterAll(async () => {
+  await app?.close()
+  if (root !== undefined) rmSync(root, { recursive: true, force: true })
+})
+
+describe.runIf(mac)("adding a desktop app", () => {
+  it("picks it, tries it confined, adds it to the project and approves it on this Mac", async () => {
+    const dir = join(root, "demo.kiframe")
+    createProject(dir, { id: "p1", name: "Desktop demo", url: "https://app.example" })
+    await app!.evaluate(({ dialog }, picked) => {
+      dialog.showOpenDialog = () => Promise.resolve({ canceled: false, filePaths: [picked] })
+    }, dir)
+    await page.getByRole("button", { name: "Open a project…" }).click()
+    await page.getByRole("button", { name: /app\.example/ }).click()
+    const panel = page.getByRole("dialog", { name: "Apps" })
+    // The app picker answers with the bundle (main's dialog, stubbed).
+    await app!.evaluate(({ dialog }, picked) => {
+      dialog.showOpenDialog = () => Promise.resolve({ canceled: false, filePaths: [picked] })
+    }, bundle)
+    await panel.getByRole("button", { name: "Add desktop app…" }).click()
+    const adding = panel.getByRole("region", { name: "Adding Notes" })
+    await expect.poll(() => adding.textContent()).toMatch(/not signed by a developer: pinned/)
+    await adding.getByRole("button", { name: "Check" }).click()
+    await expect
+      .poll(() => adding.getByRole("status").textContent(), { timeout: 40_000 })
+      .toMatch(/runs confined/)
+    await adding.getByRole("button", { name: "Add Notes" }).click()
+    await expect
+      .poll(() => panel.getByRole("list", { name: "The project's apps" }).textContent(), {
+        timeout: 10_000,
+      })
+      .toMatch(/notes.*com\.kiframe\.e2e-notes.*Ready on this Mac/s)
+    const written = JSON.parse(readFileSync(join(dir, "project.json"), "utf8")) as {
+      apps: Record<string, unknown>
+    }
+    expect(written.apps["notes"]).toMatchObject({
+      kind: "electron",
+      bundleId: "com.kiframe.e2e-notes",
+    })
+    // The trial's sandbox gone; nothing of it in the user's own work area.
+    expect(readdirSync(join(work, "sandboxes")).filter((n) => !n.endsWith(".canary"))).toEqual([])
+  }, 120_000)
+
+  it("lets Kif work in it: launched from its approval, confined, closed with the project", async () => {
+    // The Apps panel closed (the chat beside it).
+    await page.keyboard.press("Escape")
+    const box = page.getByLabel("Message Kif")
+    await box.fill("Add a note in Notes")
+    await box.press("Enter")
+    await expect
+      .poll(() => page.getByText("Added a note in Notes.").count(), { timeout: 60_000 })
+      .toBeGreaterThan(0)
+    // Its live app: one confined launch in the test's work area.
+    const live = () => readdirSync(join(work, "sandboxes")).filter((n) => !n.endsWith(".canary"))
+    expect(live()).toHaveLength(1)
+    // The project closed: Kif's app ends with it.
+    await page.getByRole("button", { name: /Desktop demo/ }).click()
+    await page.getByRole("menuitem", { name: "Close project" }).click()
+    await expect.poll(() => live().length, { timeout: 20_000 }).toBe(0)
+  }, 120_000)
+})

apps/desktop/src/main/agent.ts

@@ -12,6 +12,7 @@ import {
   notesBlock,
   runNotes,
   Studio,
+  type StudioOptions,
   studioTools,
   systemPrompt,
 } from "@kiframe/studio"
@@ -99,6 +100,8 @@ export interface AgentHostOptions {
   projectChanged: () => void
   /** Where the files the agent replaces or deletes are kept first (none: not kept). */
   versions?: FileVersions
+  /** Launches the project's desktop apps (main's launcher, from their approvals). None: none. */
+  launchDesktop?: StudioOptions["launchDesktop"]
 }
 
 /** A message as sent: its text as the user wrote it, as the history keeps it, its files. */
@@ -515,6 +518,9 @@ export class AgentHost {
       takes,
       browser,
       ...(this.#options.afterRecord !== undefined && { afterRecord: this.#options.afterRecord }),
+      ...(this.#options.launchDesktop !== undefined && {
+        launchDesktop: this.#options.launchDesktop,
+      }),
       requestUser: (request, signal) => this.#ask(request, signal),
       // What the agent read of the project's files: this host's (outlives a studio remade).
       fileReads: this.#fileReads,