Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/desktop/src/ipc/channels.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ export const CHECK_SYSTEM_PERMISSION_CHANNEL = "desktop:check-system-permission"

export const PREVIEW_RECORDING_INPUT_CHANNEL = "desktop:preview-recording-input";

export const PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL = "desktop:preview-annotation-voice-event";
export const PREVIEW_ANNOTATION_VOICE_STATE_CHANNEL = "desktop:preview-annotation-voice-state";

export const RECEIVE_PROVIDER_AUTH_CALLBACK_CHANNEL = "desktop:receive-provider-auth-callback";
export const CANCEL_PROVIDER_AUTH_CALLBACK_CHANNEL = "desktop:cancel-provider-auth-callback";
export const TAKE_LEGACY_LOCAL_STORAGE_CHANNEL = "desktop:take-legacy-local-storage";
Expand Down
19 changes: 16 additions & 3 deletions apps/desktop/src/ipc/methods/preview.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import {
DesktopPreviewAnnotationSendEnabledInputSchema,
DesktopPreviewAnnotationThemeInputSchema,
DesktopPreviewAnnotationVoiceStateSchema,
DesktopPreviewPickElementInputSchema,
DesktopPreviewArtifactInputSchema,
DesktopPreviewConfigInputSchema,
DesktopPreviewNavigateInputSchema,
Expand Down Expand Up @@ -363,11 +365,21 @@ const setAnnotationTheme = DesktopIpc.makeIpcMethod({

const pickElement = DesktopIpc.makeIpcMethod({
channel: IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL,
payload: DesktopPreviewTabInputSchema,
payload: DesktopPreviewPickElementInputSchema,
result: Schema.NullOr(PreviewAnnotationSubmissionResultSchema),
handler: Effect.fn("desktop.ipc.preview.pickElement")(function* ({ tabId }) {
handler: Effect.fn("desktop.ipc.preview.pickElement")(function* ({ tabId, voice }) {
const manager = yield* PreviewManager.PreviewManager;
return yield* manager.pickElement(tabId, voice);
}),
});

const updateAnnotationVoice = DesktopIpc.makeIpcMethod({
channel: IpcChannels.PREVIEW_ANNOTATION_VOICE_STATE_CHANNEL,
payload: DesktopPreviewAnnotationVoiceStateSchema,
result: Schema.Void,
handler: Effect.fn("desktop.ipc.preview.updateAnnotationVoice")(function* (state) {
const manager = yield* PreviewManager.PreviewManager;
return yield* manager.pickElement(tabId);
yield* manager.updateAnnotationVoice(state);
}),
});

Expand Down Expand Up @@ -428,6 +440,7 @@ const saveRecording = DesktopIpc.makeIpcMethod({

export const methods = [
setForwardedShortcuts,
updateAnnotationVoice,
createTab,
closeTab,
registerWebview,
Expand Down
16 changes: 15 additions & 1 deletion apps/desktop/src/preload.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type {
DesktopBridge,
DesktopPreviewAnnotationVoiceEvent,
DesktopPreviewOpenLinkEvent,
DesktopPreviewPointerEvent,
DesktopPreviewRecordingInputEvent,
Expand Down Expand Up @@ -368,7 +369,20 @@ contextBridge.exposeInMainWorld("desktopBridge", {
tabId,
enabled,
}),
pickElement: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, { tabId }),
annotationVoice: {
onEvent: (listener) => {
const onEvent = (_event: Electron.IpcRendererEvent, tabId: string, value: unknown) => {
if (typeof tabId !== "string" || typeof value !== "object" || value === null) return;
listener(tabId, value as DesktopPreviewAnnotationVoiceEvent);
};
ipcRenderer.on(IpcChannels.PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL, onEvent);
return () => ipcRenderer.off(IpcChannels.PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL, onEvent);
},
update: (state) =>
ipcRenderer.invoke(IpcChannels.PREVIEW_ANNOTATION_VOICE_STATE_CHANNEL, state),
},
pickElement: (tabId, voice) =>
ipcRenderer.invoke(IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, { tabId, voice }),
cancelPickElement: (tabId) =>
ipcRenderer.invoke(IpcChannels.PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL, { tabId }),
captureScreenshot: (tabId) =>
Expand Down
6 changes: 6 additions & 0 deletions apps/desktop/src/preview/Annotation.css
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
--t3-popover-foreground: oklch(0.269 0 0);
--t3-primary: oklch(0.488 0.217 264);
--t3-primary-foreground: white;
--t3-destructive: oklch(0.577 0.245 27.325);
--t3-muted: rgb(0 0 0 / 4%);
--t3-muted-foreground: oklch(0.556 0 0);
--t3-accent: rgb(0 0 0 / 4%);
Expand Down Expand Up @@ -66,3 +67,8 @@ textarea:focus-visible {
outline: 2px solid color-mix(in srgb, var(--t3-ring) 72%, transparent);
outline-offset: -2px;
}

/* Utility display classes must not override the native hidden state. */
[hidden] {
display: none !important;
}
2 changes: 1 addition & 1 deletion apps/desktop/src/preview/AnnotationStyles.generated.ts

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions apps/desktop/src/preview/GuestProtocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,5 @@ export const RECORDING_CONTROLLER_CHANNEL = "preview:recording-controller";
export const PASSKEY_CREATE_CHANNEL = "preview:passkey-create";
export const PASSKEY_GET_CHANNEL = "preview:passkey-get";
export const PASSKEY_BRIDGE_ARGUMENT = "--t3code-preview-passkey-bridge";
export const ANNOTATION_VOICE_EVENT_CHANNEL = "preview:annotation-voice-event";
export const ANNOTATION_VOICE_STATE_CHANNEL = "preview:annotation-voice-state";
64 changes: 64 additions & 0 deletions apps/desktop/src/preview/Manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2443,6 +2443,70 @@ describe("PreviewManager", () => {
),
);

effectIt.effect("delivers annotation voice state only to the pick session that owns it", () =>
withManager((manager) =>
Effect.gen(function* () {
const firstReady = yield* Deferred.make<void>();
const secondReady = yield* Deferred.make<void>();
let registrations = 0;
const guest = {
...makeTestPreviewWebContents(async () => ({
toPNG: () => Buffer.from("unused"),
toJPEG: () => Buffer.from("unused"),
getSize: () => ({ width: 1, height: 1 }),
})),
isFocused: () => true,
focus: vi.fn(),
send: (channel: string, ...args: unknown[]) => {
webviewSend(channel, ...args);
if (channel === "preview:start-pick")
Deferred.doneUnsafe(registrations++ === 0 ? firstReady : secondReady, Effect.void);
},
once: vi.fn(),
ipc: { on: vi.fn(), off: vi.fn(), removeListener: vi.fn() },
};
fromId.mockReturnValue(guest as never);
yield* manager.createTab("tab_1");
yield* manager.registerWebview("tab_1", 42);
const first = yield* manager
.pickElement("tab_1", { sessionId: "first", shortcuts: [] })
.pipe(Effect.forkChild);
yield* Deferred.await(firstReady);
const state = {
tabId: "tab_1",
sessionId: "first",
available: true,
phase: "recording" as const,
status: "Recording",
errorAction: null,
preview: null,
freezesEditor: true,
blocksSubmission: true,
level: 0,
shortcutLabel: "Ctrl+D",
draft: { text: "Tighten spacing", cursor: 15 },
};
yield* manager.updateAnnotationVoice(state);
expect(webviewSend).toHaveBeenCalledWith("preview:annotation-voice-state", state);
const second = yield* manager
.pickElement("tab_1", { sessionId: "second", shortcuts: [] })
.pipe(Effect.forkChild);
yield* Deferred.await(secondReady);
expect(yield* Fiber.join(first)).toBeNull();
webviewSend.mockClear();
yield* manager.updateAnnotationVoice(state);
expect(webviewSend).not.toHaveBeenCalled();
yield* manager.updateAnnotationVoice({ ...state, sessionId: "second" });
expect(webviewSend).toHaveBeenCalledOnce();
yield* manager.cancelPickElement("tab_1");
expect(yield* Fiber.join(second)).toBeNull();
webviewSend.mockClear();
yield* manager.updateAnnotationVoice({ ...state, sessionId: "second" });
expect(webviewSend).not.toHaveBeenCalled();
}),
),
);

effectIt.effect("keeps annotation send grants isolated and updates an active pick", () =>
withManager((manager) =>
Effect.gen(function* () {
Expand Down
72 changes: 68 additions & 4 deletions apps/desktop/src/preview/Manager.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import * as NodeZlib from "node:zlib";
import {
DesktopPreviewAnnotationVoiceEventSchema,
DesktopPreviewRecordingInputSchema,
DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER,
} from "@t3tools/contracts";
import type {
DesktopPreviewAnnotationTheme,
DesktopPreviewAnnotationVoiceConfig,
DesktopPreviewAnnotationVoiceState,
DesktopPreviewColorScheme,
DesktopPreviewFavicon,
DesktopPreviewOpenLinkEvent,
Expand Down Expand Up @@ -56,13 +59,19 @@ import * as SynchronizedRef from "effect/SynchronizedRef";

import * as DesktopEnvironment from "../app/DesktopEnvironment.ts";
import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts";
import { MENU_ACTION_CHANNEL, PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL } from "../ipc/channels.ts";
import {
MENU_ACTION_CHANNEL,
PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL,
PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL,
} from "../ipc/channels.ts";
import * as DesktopBrowserHost from "./DesktopBrowserHost.ts";
import * as PreviewPasskeys from "./Passkeys.ts";
import * as BrowserSession from "./BrowserSession.ts";
import {
ANNOTATION_CAPTURED_CHANNEL,
ANNOTATION_SEND_ENABLED_CHANNEL,
ANNOTATION_VOICE_EVENT_CHANNEL,
ANNOTATION_VOICE_STATE_CHANNEL,
ANNOTATION_THEME_CHANNEL,
CANCEL_PICK_CHANNEL,
ELEMENT_PICKED_CHANNEL,
Expand Down Expand Up @@ -396,9 +405,14 @@ interface PendingRecording {
}

interface PickSession {
readonly voiceSessionId?: string | undefined;
readonly cancel: Effect.Effect<void>;
}

const decodeAnnotationVoiceEvent = Schema.decodeUnknownOption(
DesktopPreviewAnnotationVoiceEventSchema,
);

interface BrowserControlSession {
readonly webContentsId: number;
// Retain the attached Debugger to prevent GC crashes (electron/electron#53376); wc.debugger throws after destruction.
Expand Down Expand Up @@ -2074,16 +2088,35 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
});
});

const pickElement = Effect.fn("PreviewManager.pickElement")(function* (tabId: string) {
// Voice state reaches a guest only while its pick session, by id, still owns the tab.
const updateAnnotationVoice = Effect.fn("PreviewManager.updateAnnotationVoice")(function* (
state: DesktopPreviewAnnotationVoiceState,
) {
const sessions = yield* Ref.get(pickSessionsRef);
if (sessions.get(state.tabId)?.voiceSessionId !== state.sessionId) return;
const wc = yield* requireWebContents(state.tabId);
yield* attempt({ operation: "annotationVoice.update", tabId: state.tabId }, () =>
wc.send(ANNOTATION_VOICE_STATE_CHANNEL, state),
);
});

const pickElement = Effect.fn("PreviewManager.pickElement")(function* (
tabId: string,
voice?: DesktopPreviewAnnotationVoiceConfig,
) {
const wc = yield* requireWebContents(tabId);
yield* cancelPickElement(tabId);
const annotationTheme = yield* Ref.get(annotationThemeRef);
return yield* Effect.callback<PreviewAnnotationSubmissionResult | null, PreviewManagerError>(
(resume) => {
const session: PickSession = { cancel: Effect.suspend(() => cancelPickSession()) };
const session: PickSession = {
voiceSessionId: voice?.sessionId,
cancel: Effect.suspend(() => cancelPickSession()),
};
const cleanup = Effect.fn("PreviewManager.cleanupPickElement")(function* () {
yield* attempt({ operation: "pickElement.cleanup", tabId, webContentsId: wc.id }, () => {
wc.ipc.removeListener(ELEMENT_PICKED_CHANNEL, onMessage);
wc.ipc.removeListener(ANNOTATION_VOICE_EVENT_CHANNEL, onVoiceEvent);
wc.off("destroyed", onDestroyed);
wc.off("did-start-navigation", onNavigated);
}).pipe(Effect.ignore);
Expand Down Expand Up @@ -2191,6 +2224,25 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
),
);
};
// Only this guest's listener sees these; the renderer still checks the session id.
const onVoiceEvent = (_event: Electron.IpcMainEvent, value: unknown) => {
const decoded = decodeAnnotationVoiceEvent(value);
if (settled || !voice || Option.isNone(decoded)) return;
if (decoded.value.sessionId !== voice.sessionId) return;
runFork(
Effect.gen(function* () {
const window = yield* Ref.get(mainWindowRef);
if (Option.isNone(window) || window.value.isDestroyed()) return;
yield* attempt({ operation: "annotationVoice.forward", tabId }, () =>
window.value.webContents.send(
PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL,
tabId,
decoded.value,
),
).pipe(Effect.ignore);
}),
);
};
const onDestroyed = () => settle(null);
const onNavigated = (
event: Electron.Event<Electron.WebContentsDidStartNavigationEventParams>,
Expand All @@ -2210,10 +2262,16 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
if (settled) return;
yield* attempt({ operation: "pickElement.register", tabId, webContentsId: wc.id }, () => {
wc.ipc.on(ELEMENT_PICKED_CHANNEL, onMessage);
if (voice) wc.ipc.on(ANNOTATION_VOICE_EVENT_CHANNEL, onVoiceEvent);
wc.once("destroyed", onDestroyed);
wc.on("did-start-navigation", onNavigated);
if (!wc.isFocused()) wc.focus();
wc.send(START_PICK_CHANNEL, annotationTheme, annotationSendEnabled.get(tabId) === true);
wc.send(
START_PICK_CHANNEL,
annotationTheme,
annotationSendEnabled.get(tabId) === true,
voice,
);
});
});
runFork(
Expand Down Expand Up @@ -3312,6 +3370,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
openPictureInPicture,
openDevTools,
pickElement,
updateAnnotationVoice,
prepareWebview,
reapplyZoom,
refresh,
Expand Down Expand Up @@ -3552,7 +3611,11 @@ export class PreviewManager extends Context.Service<
) => Effect.Effect<void, PreviewManagerError>;
readonly pickElement: (
tabId: string,
voice?: DesktopPreviewAnnotationVoiceConfig,
) => Effect.Effect<PreviewAnnotationSubmissionResult | null, PreviewManagerError>;
readonly updateAnnotationVoice: (
state: DesktopPreviewAnnotationVoiceState,
) => Effect.Effect<void, PreviewManagerError>;
readonly cancelPickElement: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
readonly captureScreenshot: (
tabId: string,
Expand Down Expand Up @@ -3703,6 +3766,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
setAnnotationTheme: operations.setAnnotationTheme,
setAnnotationSendEnabled: operations.setAnnotationSendEnabled,
pickElement: operations.pickElement,
updateAnnotationVoice: operations.updateAnnotationVoice,
cancelPickElement: operations.cancelPickElement,
captureScreenshot: operations.captureScreenshot,
revealArtifact: operations.revealArtifact,
Expand Down
Loading
Loading