@devries-cameron20 / t3code Mirror · read-only / commits / ff17cd307f
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 13:30 UTC

fix(desktop): agent snapshots keep full detail in narrow and mini previews (#36944)

Julius Marminge committed Nov 14, 2026 at 14:06 UTC ff17cd307fca939329e5f2e35eb8b0f250ac644c
11 files changed +256 -82
apps/desktop/src/ipc/channels.ts
+1
@@ -101,6 +101,7 @@ export const PREVIEW_SET_AUDIO_MUTED_CHANNEL = "desktop:preview-set-audio-muted"
101 export const PREVIEW_FIND_IN_PAGE_CHANNEL = "desktop:preview-find-in-page";
102 export const PREVIEW_STOP_FIND_IN_PAGE_CHANNEL = "desktop:preview-stop-find-in-page";
103 export const PREVIEW_SET_SHOWN_CHANNEL = "desktop:preview-set-shown";
104 +export const PREVIEW_SET_CAPTURE_DRAWN_CHANNEL = "desktop:preview-set-capture-drawn";
105 export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools";
106 export const PREVIEW_CLEAR_COOKIES_CHANNEL = "desktop:preview-clear-cookies";
107 export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache";
apps/desktop/src/ipc/methods/preview.ts
+11
@@ -13,6 +13,7 @@ import {
13 DesktopPreviewScreenshotArtifactSchema,
14 DesktopPreviewSetAudioMutedInputSchema,
15 DesktopPreviewSetShownInputSchema,
16 + DesktopPreviewSetCaptureDrawnInputSchema,
17 DesktopPreviewSetColorSchemeInputSchema,
18 DesktopPreviewSetZoomFactorInputSchema,
19 BrowserImportResult,
@@ -234,6 +235,15 @@ const setShown = DesktopIpc.makeIpcMethod({
235 yield* manager.setShown(tabId, shown);
236 }),
237 });
238 +const setCaptureDrawn = DesktopIpc.makeIpcMethod({
239 + channel: IpcChannels.PREVIEW_SET_CAPTURE_DRAWN_CHANNEL,
240 + payload: DesktopPreviewSetCaptureDrawnInputSchema,
241 + result: Schema.Void,
242 + handler: Effect.fn("desktop.ipc.preview.setCaptureDrawn")(function* ({ tabId, drawn }) {
243 + const manager = yield* PreviewManager.PreviewManager;
244 + yield* manager.setCaptureDrawn(tabId, drawn);
245 + }),
246 +});
247 const openDevTools = tabMethod(
248 IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL,
249 "desktop.ipc.preview.openDevTools",
@@ -488,6 +498,7 @@ export const methods = [
498 setZoomFactor,
499 setAudioMuted,
500 setShown,
501 + setCaptureDrawn,
502 openDevTools,
503 clearCookies,
504 clearCache,
apps/desktop/src/preload.ts
+2
@@ -414,6 +414,8 @@ contextBridge.exposeInMainWorld("desktopBridge", {
414 ipcRenderer.invoke(IpcChannels.PREVIEW_STOP_FIND_IN_PAGE_CHANNEL, { tabId }),
415 setShown: (tabId, shown) =>
416 ipcRenderer.invoke(IpcChannels.PREVIEW_SET_SHOWN_CHANNEL, { tabId, shown }),
417 + setCaptureDrawn: (tabId, drawn) =>
418 + ipcRenderer.invoke(IpcChannels.PREVIEW_SET_CAPTURE_DRAWN_CHANNEL, { tabId, drawn }),
419 openDevTools: (tabId) =>
420 ipcRenderer.invoke(IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, { tabId }),
421 listBrowserImportSources: () => ipcRenderer.invoke(IpcChannels.PREVIEW_IMPORT_SOURCES_CHANNEL),
apps/desktop/src/preview/DesktopBrowserHost.test.ts
+72 -38
@@ -26,6 +26,11 @@ const decodeCdpMessage = Schema.decodeUnknownSync(
26 ),
27 );
28 const key = { threadId: "thread-1", tabId: "tab-1" };
29 +
30 +/** A host whose client already draws `key` at full detail, so its screenshots go out at once. */
31 +const makeDrawnHost = Effect.tap(DesktopBrowserHost.make, (host) =>
32 + Effect.sync(() => host.setCaptureDrawn(key, true)),
33 +);
34 /** Sends a page command whose id the fake debugger records as it receives it. */
35 const command = (
36 host: DesktopBrowserHost.DesktopBrowserHost["Service"],
@@ -172,7 +177,7 @@ const watchPainting = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) =
177 describe("DesktopBrowserHost", () => {
178 it.effect("announces tabs already attached to a backend that starts later", () =>
179 Effect.gen(function* () {
175 - const host = yield* DesktopBrowserHost.make;
180 + const host = yield* makeDrawnHost;
181 host.attach(key, makeDebuggee().tab);
182
183 expect(yield* takeEvents(host, 1)).toEqual([{ type: "attached", ...key }]);
@@ -182,7 +187,7 @@ describe("DesktopBrowserHost", () => {
187
188 it.effect("announces a guest still loading its first page only once that page is up", () =>
189 Effect.gen(function* () {
185 - const host = yield* DesktopBrowserHost.make;
190 + const host = yield* makeDrawnHost;
191 const loading = makeDebuggee({ loading: true });
192 const other = { ...key, tabId: "tab-2" };
193 const reader = yield* takeEvents(host, 3).pipe(Effect.forkScoped);
@@ -205,7 +210,7 @@ describe("DesktopBrowserHost", () => {
210 "keeps a loading guest unannounced across a replaced navigation and a new backend",
211 () =>
212 Effect.gen(function* () {
208 - const host = yield* DesktopBrowserHost.make;
213 + const host = yield* makeDrawnHost;
214 const loading = makeDebuggee({ loading: true });
215 const other = { ...key, tabId: "tab-2" };
216 host.attach(key, loading.tab);
@@ -227,7 +232,7 @@ describe("DesktopBrowserHost", () => {
232 "announces a first page that never comes up, but never a tab closed or destroyed first",
233 () =>
234 Effect.gen(function* () {
230 - const host = yield* DesktopBrowserHost.make;
235 + const host = yield* makeDrawnHost;
236 const closed = { ...key, tabId: "tab-2" };
237 const destroyed = { ...key, tabId: "tab-3" };
238 const destroyedGuest = makeDebuggee({ loading: true });
@@ -247,7 +252,7 @@ describe("DesktopBrowserHost", () => {
252
253 it.effect("re-sends shown to a backend that starts later and when a tab attaches again", () =>
254 Effect.gen(function* () {
250 - const host = yield* DesktopBrowserHost.make;
255 + const host = yield* makeDrawnHost;
256 host.attach(key, makeDebuggee().tab);
257 host.setShown(key, true);
258 const reader = yield* takeEvents(host, 5).pipe(Effect.forkScoped);
@@ -268,7 +273,7 @@ describe("DesktopBrowserHost", () => {
273
274 it.effect("replays an explicit hidden report before announcing another tab", () =>
275 Effect.gen(function* () {
271 - const host = yield* DesktopBrowserHost.make;
276 + const host = yield* makeDrawnHost;
277 const other = { ...key, tabId: "tab-2" };
278 host.setShown(key, false);
279 host.attach(key, makeDebuggee().tab);
@@ -285,7 +290,7 @@ describe("DesktopBrowserHost", () => {
290
291 it.effect("drops replies from a relay the server released", () =>
292 Effect.gen(function* () {
288 - const host = yield* DesktopBrowserHost.make;
293 + const host = yield* makeDrawnHost;
294 const debuggee = makeDebuggee();
295 host.attach(key, debuggee.tab);
296 const reader = yield* takeEvents(host, 2).pipe(Effect.forkScoped);
@@ -313,7 +318,7 @@ describe("DesktopBrowserHost", () => {
318
319 it.effect("sends a page one screenshot at a time", () =>
320 Effect.gen(function* () {
316 - const host = yield* DesktopBrowserHost.make;
321 + const host = yield* makeDrawnHost;
322 const debuggee = makeDebuggee();
323 host.attach(key, debuggee.tab);
324 const replies = yield* watchReplies(host);
@@ -330,9 +335,38 @@ describe("DesktopBrowserHost", () => {
335 }),
336 );
337
338 + it.effect(
339 + "sends a screenshot once the client draws the guest at full detail, or soon after",
340 + () =>
341 + Effect.gen(function* () {
342 + const host = yield* DesktopBrowserHost.make;
343 + const debuggee = makeDebuggee();
344 + host.attach(key, debuggee.tab);
345 + const replies = yield* watchReplies(host);
346 +
347 + yield* command(host, 1, "Page.captureScreenshot");
348 + yield* settle;
349 + expect(debuggee.sent).toEqual([]);
350 + host.setCaptureDrawn(key, true);
351 + yield* settle;
352 + expect(debuggee.sent).toEqual([1]);
353 + debuggee.releaseFirst();
354 + expect(yield* Queue.take(replies)).toBe(1);
355 +
356 + // A client that never draws, such as a hidden window, only delays the capture.
357 + host.setCaptureDrawn(key, false);
358 + yield* command(host, 2, "Page.captureScreenshot");
359 + yield* settle;
360 + expect(debuggee.sent).toEqual([1]);
361 + yield* TestClock.adjust("1 second");
362 + yield* settle;
363 + expect(debuggee.sent).toEqual([1, 2]);
364 + }).pipe(Effect.scoped),
365 + );
366 +
367 it.effect("stops waiting on a screenshot the page navigated away from", () =>
368 Effect.gen(function* () {
335 - const host = yield* DesktopBrowserHost.make;
369 + const host = yield* makeDrawnHost;
370 const debuggee = makeDebuggee();
371 host.attach(key, debuggee.tab);
372 const replies = yield* watchReplies(host);
@@ -357,7 +391,7 @@ describe("DesktopBrowserHost", () => {
391
392 it.effect("drops screenshots a released relay left waiting", () =>
393 Effect.gen(function* () {
360 - const host = yield* DesktopBrowserHost.make;
394 + const host = yield* makeDrawnHost;
395 const debuggee = makeDebuggee();
396 host.attach(key, debuggee.tab);
397
@@ -377,7 +411,7 @@ describe("DesktopBrowserHost", () => {
411
412 it.effect("emulates focus while the server drives a guest, then lets a hidden one throttle", () =>
413 Effect.gen(function* () {
380 - const host = yield* DesktopBrowserHost.make;
414 + const host = yield* makeDrawnHost;
415 const debuggee = makeDebuggee();
416 host.attach(key, debuggee.tab);
417
@@ -408,7 +442,7 @@ describe("DesktopBrowserHost", () => {
442
443 it.effect("keeps a guest focused while a command it is running is still pending", () =>
444 Effect.gen(function* () {
411 - const host = yield* DesktopBrowserHost.make;
445 + const host = yield* makeDrawnHost;
446 const debuggee = makeDebuggee();
447 host.attach(key, debuggee.tab);
448 const replies = yield* watchReplies(host);
@@ -429,7 +463,7 @@ describe("DesktopBrowserHost", () => {
463
464 it.effect("lets a guest throttle once Chromium can no longer answer its pending command", () =>
465 Effect.gen(function* () {
432 - const host = yield* DesktopBrowserHost.make;
466 + const host = yield* makeDrawnHost;
467 const debuggee = makeDebuggee();
468 host.attach(key, debuggee.tab);
469
@@ -451,7 +485,7 @@ describe("DesktopBrowserHost", () => {
485
486 it.effect("keeps a guest focused for the screenshot a navigation lets start", () =>
487 Effect.gen(function* () {
454 - const host = yield* DesktopBrowserHost.make;
488 + const host = yield* makeDrawnHost;
489 const debuggee = makeDebuggee();
490 host.attach(key, debuggee.tab);
491
@@ -467,7 +501,7 @@ describe("DesktopBrowserHost", () => {
501
502 it.effect("never sends a screenshot a navigation superseded while focus was changing", () =>
503 Effect.gen(function* () {
470 - const host = yield* DesktopBrowserHost.make;
504 + const host = yield* makeDrawnHost;
505 const debuggee = makeDebuggee({ deferFocus: true });
506 host.attach(key, debuggee.tab);
507
@@ -484,7 +518,7 @@ describe("DesktopBrowserHost", () => {
518
519 it.effect("keeps a guest focused when its turn starts after the agent's first command", () =>
520 Effect.gen(function* () {
487 - const host = yield* DesktopBrowserHost.make;
521 + const host = yield* makeDrawnHost;
522 const debuggee = makeDebuggee();
523 host.attach(key, debuggee.tab);
524 const turn = (active: boolean) =>
@@ -506,7 +540,7 @@ describe("DesktopBrowserHost", () => {
540
541 it.effect("lets a guest throttle through a turn that only sent it housekeeping", () =>
542 Effect.gen(function* () {
509 - const host = yield* DesktopBrowserHost.make;
543 + const host = yield* makeDrawnHost;
544 const debuggee = makeDebuggee();
545 host.attach(key, debuggee.tab);
546 yield* host.handleCommandLine(
@@ -535,7 +569,7 @@ describe("DesktopBrowserHost", () => {
569
570 it.effect("keeps a guest the agent used focused through its turn's pauses", () =>
571 Effect.gen(function* () {
538 - const host = yield* DesktopBrowserHost.make;
572 + const host = yield* makeDrawnHost;
573 const debuggee = makeDebuggee();
574 host.attach(key, debuggee.tab);
575 const turn = (active: boolean) =>
@@ -563,7 +597,7 @@ describe("DesktopBrowserHost", () => {
597 "ends a command's hold with the turn that sent it, even if Chromium never answers",
598 () =>
599 Effect.gen(function* () {
566 - const host = yield* DesktopBrowserHost.make;
600 + const host = yield* makeDrawnHost;
601 const debuggee = makeDebuggee();
602 host.attach(key, debuggee.tab);
603 yield* host.handleCommandLine(
@@ -584,7 +618,7 @@ describe("DesktopBrowserHost", () => {
618
619 it.effect("ends the agent's own session's holds with its turn, but not a viewer's stream", () =>
620 Effect.gen(function* () {
587 - const host = yield* DesktopBrowserHost.make;
621 + const host = yield* makeDrawnHost;
622 const debuggee = makeDebuggee();
623 host.attach(key, debuggee.tab);
624 const replies = yield* watchReplies(host);
@@ -617,7 +651,7 @@ describe("DesktopBrowserHost", () => {
651
652 it.effect("paints for a screenshot queued behind one its viewer abandoned mid-capture", () =>
653 Effect.gen(function* () {
620 - const host = yield* DesktopBrowserHost.make;
654 + const host = yield* makeDrawnHost;
655 const debuggee = makeDebuggee();
656 host.attach(key, debuggee.tab);
657 const replies = yield* watchReplies(host);
@@ -655,7 +689,7 @@ describe("DesktopBrowserHost", () => {
689
690 it.effect("stops painting for a queued screenshot once its session detaches", () =>
691 Effect.gen(function* () {
658 - const host = yield* DesktopBrowserHost.make;
692 + const host = yield* makeDrawnHost;
693 const debuggee = makeDebuggee();
694 host.attach(key, debuggee.tab);
695 const replies = yield* watchReplies(host);
@@ -694,7 +728,7 @@ describe("DesktopBrowserHost", () => {
728
729 it.effect("never sends a screenshot whose session detached while focus was changing", () =>
730 Effect.gen(function* () {
697 - const host = yield* DesktopBrowserHost.make;
731 + const host = yield* makeDrawnHost;
732 const debuggee = makeDebuggee({ deferFocus: true });
733 host.attach(key, debuggee.tab);
734 const replies = yield* watchReplies(host);
@@ -718,7 +752,7 @@ describe("DesktopBrowserHost", () => {
752
753 it.effect("holds no focus for a screenshot still queued when the agent's turn ends", () =>
754 Effect.gen(function* () {
721 - const host = yield* DesktopBrowserHost.make;
755 + const host = yield* makeDrawnHost;
756 const debuggee = makeDebuggee();
757 host.attach(key, debuggee.tab);
758 const painting = yield* watchPainting(host);
@@ -748,7 +782,7 @@ describe("DesktopBrowserHost", () => {
782
783 it.effect("lets a guest throttle when Chromium refuses a viewer's screencast", () =>
784 Effect.gen(function* () {
751 - const host = yield* DesktopBrowserHost.make;
785 + const host = yield* makeDrawnHost;
786 const debuggee = makeDebuggee({ refuse: (method) => method === "Page.startScreencast" });
787 host.attach(key, debuggee.tab);
788 const replies = yield* watchReplies(host);
@@ -769,7 +803,7 @@ describe("DesktopBrowserHost", () => {
803
804 it.effect("lets a guest throttle after a person's input through a viewer that then closed", () =>
805 Effect.gen(function* () {
772 - const host = yield* DesktopBrowserHost.make;
806 + const host = yield* makeDrawnHost;
807 const debuggee = makeDebuggee();
808 host.attach(key, debuggee.tab);
809 const replies = yield* watchReplies(host);
@@ -798,7 +832,7 @@ describe("DesktopBrowserHost", () => {
832
833 it.effect("keeps a page's screencast when a child session's screencast is refused", () =>
834 Effect.gen(function* () {
801 - const host = yield* DesktopBrowserHost.make;
835 + const host = yield* makeDrawnHost;
836 const debuggee = makeDebuggee({
837 refuse: (method, sessionId) => method === "Page.startScreencast" && sessionId !== undefined,
838 });
@@ -820,7 +854,7 @@ describe("DesktopBrowserHost", () => {
854
855 it.effect("ignores a screencast that arrives on a session after it detached", () =>
856 Effect.gen(function* () {
823 - const host = yield* DesktopBrowserHost.make;
857 + const host = yield* makeDrawnHost;
858 const debuggee = makeDebuggee();
859 host.attach(key, debuggee.tab);
860 const replies = yield* watchReplies(host);
@@ -846,7 +880,7 @@ describe("DesktopBrowserHost", () => {
880
881 it.effect("lets a guest throttle once the session waiting on its command detaches", () =>
882 Effect.gen(function* () {
849 - const host = yield* DesktopBrowserHost.make;
883 + const host = yield* makeDrawnHost;
884 const debuggee = makeDebuggee();
885 host.attach(key, debuggee.tab);
886 const replies = yield* watchReplies(host);
@@ -873,7 +907,7 @@ describe("DesktopBrowserHost", () => {
907
908 it.effect("never sends a screenshot queued by a session that has since detached", () =>
909 Effect.gen(function* () {
876 - const host = yield* DesktopBrowserHost.make;
910 + const host = yield* makeDrawnHost;
911 const debuggee = makeDebuggee();
912 host.attach(key, debuggee.tab);
913 const replies = yield* watchReplies(host);
@@ -904,7 +938,7 @@ describe("DesktopBrowserHost", () => {
938
939 it.effect("keeps a streamed guest focused until its screencast ends", () =>
940 Effect.gen(function* () {
907 - const host = yield* DesktopBrowserHost.make;
941 + const host = yield* makeDrawnHost;
942 const debuggee = makeDebuggee();
943 host.attach(key, debuggee.tab);
944
@@ -933,7 +967,7 @@ describe("DesktopBrowserHost", () => {
967
968 it.effect("keeps a guest focused while any of its screencasts runs", () =>
969 Effect.gen(function* () {
936 - const host = yield* DesktopBrowserHost.make;
970 + const host = yield* makeDrawnHost;
971 const debuggee = makeDebuggee();
972 host.attach(key, debuggee.tab);
973 const replies = yield* watchReplies(host);
@@ -965,7 +999,7 @@ describe("DesktopBrowserHost", () => {
999
1000 it.effect("paints a hidden guest only while the screenshot it is taking is pending", () =>
1001 Effect.gen(function* () {
968 - const host = yield* DesktopBrowserHost.make;
1002 + const host = yield* makeDrawnHost;
1003 const debuggee = makeDebuggee();
1004 host.attach(key, debuggee.tab);
1005 const painting = yield* watchPainting(host);
@@ -991,7 +1025,7 @@ describe("DesktopBrowserHost", () => {
1025
1026 it.effect("stops painting for a screenshot Chromium never answers once its document goes", () =>
1027 Effect.gen(function* () {
994 - const host = yield* DesktopBrowserHost.make;
1028 + const host = yield* makeDrawnHost;
1029 const debuggee = makeDebuggee();
1030 host.attach(key, debuggee.tab);
1031 const painting = yield* watchPainting(host);
@@ -1036,7 +1070,7 @@ describe("DesktopBrowserHost", () => {
1070
1071 it.effect("paints a streamed guest until its last screencast ends", () =>
1072 Effect.gen(function* () {
1039 - const host = yield* DesktopBrowserHost.make;
1073 + const host = yield* makeDrawnHost;
1074 const debuggee = makeDebuggee();
1075 host.attach(key, debuggee.tab);
1076 const replies = yield* watchReplies(host);
@@ -1243,7 +1277,7 @@ describe("DesktopBrowserHost", () => {
1277
1278 it.effect("saves a server tab's download under its CDP guid where the server asked", () =>
1279 Effect.gen(function* () {
1246 - const host = yield* DesktopBrowserHost.make;
1280 + const host = yield* makeDrawnHost;
1281 const debuggee = makeDebuggee();
1282 host.attach(key, debuggee.tab);
1283 const replies = yield* watchReplies(host);
@@ -1278,7 +1312,7 @@ describe("DesktopBrowserHost", () => {
1312
1313 it.effect("tells a download the person clicked from one the agent's input started", () =>
1314 Effect.gen(function* () {
1281 - const host = yield* DesktopBrowserHost.make;
1315 + const host = yield* makeDrawnHost;
1316 const debuggee = makeDebuggee();
1317 host.attach(key, debuggee.tab);
1318
@@ -1342,7 +1376,7 @@ describe("DesktopBrowserHost", () => {
1376
1377 it.effect("detaches only the tab whose CDP event is too large to relay", () =>
1378 Effect.gen(function* () {
1345 - const host = yield* DesktopBrowserHost.make;
1379 + const host = yield* makeDrawnHost;
1380 const tabs = [makeDebuggee({ deferFocus: true }), makeDebuggee()];
1381 const send = vi.spyOn(tabs[0]!.tab.debugger, "sendCommand");
1382 const keys = [key, { threadId: "thread-2", tabId: "tab-2" }];
apps/desktop/src/preview/DesktopBrowserHost.ts
+47 -6
@@ -30,6 +30,8 @@ const AGENT_INPUT_WINDOW_MS = 5_000;
30 /** A viewer's pointer travelling over the stream activates nothing on the page. */
31 const POINTER_TRAVEL = new Set<unknown>(["mouseMoved", "mouseWheel"]);
32 const FOCUS_IDLE = "5 seconds";
33 +/** How long a screenshot waits for the client to draw its guest at full detail. */
34 +const CAPTURE_DRAW_WAIT = "1 second";
35 /** Well inside the server's 10 s DESKTOP_ATTACH_TIMEOUT, for a first page that never reports it is up. */
36 const FIRST_PAGE_TIMEOUT = "5 seconds";
37 const ERR_ABORTED = -3;
@@ -101,6 +103,8 @@ export class DesktopBrowserHost extends Context.Service<
103 readonly detach: (key: DesktopBrowserTabKey) => void;
104 readonly loadFailed: (key: DesktopBrowserTabKey, failure: PreviewAutomationLoadError) => void;
105 readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void;
106 + /** Whether the client draws the guest at full detail while it is captured. */
107 + readonly setCaptureDrawn: (key: DesktopBrowserTabKey, drawn: boolean | undefined) => void;
108 readonly humanStartedDownload: (source: Electron.WebContents) => boolean;
109 /** Whether what the page does now follows the agent's input, not a viewer's. */
110 readonly agentLedPage: (source: Electron.WebContents) => boolean;
@@ -138,6 +142,8 @@ export const make = Effect.gen(function* () {
142 // Kept apart from attachment: the renderer can show a webview before its debugger attaches,
143 // and the server forgets `shown` on detach, for example while DevTools holds the debugger.
144 const shownTabs = new Map<string, boolean>();
145 + const drawnTabs = new Set<string>();
146 + const drawWaiters = new Map<string, Set<() => void>>();
147 const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
148
149 const nextScreenshot = (tab: AttachedTab) => {
@@ -266,12 +272,32 @@ export const make = Effect.gen(function* () {
272 });
273 };
274
275 + /**
276 + * A scaled guest rasters at its on-screen size until the client draws it at full detail,
277 + * and a screenshot taken before that copies the blurred frame. A client that cannot draw
278 + * (a hidden window, say) only delays the capture.
279 + */
280 + const whenDrawn = (tab: AttachedTab) =>
281 + new Promise<void>((resolve) => {
282 + const id = keyOf(tab.key);
283 + const waiters = drawWaiters.get(id) ?? new Set();
284 + drawWaiters.set(id, waiters);
285 + const done = () => {
286 + waiters.delete(done);
287 + if (waiters.size === 0 && drawWaiters.get(id) === waiters) drawWaiters.delete(id);
288 + resolve();
289 + };
290 + waiters.add(done);
291 + runFork(Effect.sleep(CAPTURE_DRAW_WAIT).pipe(Effect.andThen(Effect.sync(done))));
292 + });
293 +
294 /** Chromium answers a capture, and frees its wake lock, only once the guest paints a frame. */
295 const paintUntilAnswered = <A>(tab: AttachedTab, send: () => Promise<A>) => {
296 const sent = {};
297 tab.capturing.add(sent);
298 syncPainting(tab);
274 - return send().finally(() => {
299 + // Sent at once when already drawn, so it keeps its place among the commands after it.
300 + return (drawnTabs.has(keyOf(tab.key)) ? send() : whenDrawn(tab).then(send)).finally(() => {
301 tab.capturing.delete(sent);
302 syncPainting(tab);
303 });
@@ -368,12 +394,15 @@ export const make = Effect.gen(function* () {
394 // Waits for focus emulation; insertText and other non-CDP input would not queue behind it.
395 const forward = (current?: () => boolean) => {
396 const change = tab.focusChange;
371 - const run = () =>
372 - tab.relay !== relay || (current !== undefined && !current())
397 + const stale = () => tab.relay !== relay || (current !== undefined && !current());
398 + const sendUnlessStale = () =>
399 + stale()
400 ? Promise.reject(new Error("The page this screenshot was for has moved on."))
374 - : current === undefined
375 - ? send(method, params, sessionId)
376 - : paintUntilAnswered(tab, () => send(method, params, sessionId));
401 + : send(method, params, sessionId);
402 + const run = () =>
403 + current === undefined || stale()
404 + ? sendUnlessStale()
405 + : paintUntilAnswered(tab, sendUnlessStale);
406 return change === null ? run() : change.then(run);
407 };
408 return method === "Page.captureScreenshot"
@@ -587,6 +616,17 @@ export const make = Effect.gen(function* () {
616 if (tabs.get(id)?.stopWaiting === null) emit({ type: "shown", ...key, shown });
617 };
618
619 + const setCaptureDrawn = (key: DesktopBrowserTabKey, drawn: boolean | undefined) => {
620 + const id = keyOf(key);
621 + if (!drawn) {
622 + drawnTabs.delete(id);
623 + return;
624 + }
625 + drawnTabs.add(id);
626 + for (const done of drawWaiters.get(id) ?? []) done();
627 + drawWaiters.delete(id);
628 + };
629 +
630 const loadFailed: DesktopBrowserHost["Service"]["loadFailed"] = (key, failure) => {
631 if (tabs.has(keyOf(key))) emit({ type: "loadFailed", ...key, ...failure });
632 };
@@ -663,6 +703,7 @@ export const make = Effect.gen(function* () {
703 detach,
704 loadFailed,
705 setShown,
706 + setCaptureDrawn,
707 placeDownload,
708 humanStartedDownload,
709 agentLedPage,
apps/desktop/src/preview/Manager.test.ts
+2
@@ -684,6 +684,7 @@ describe("PreviewManager", () => {
684 attach: () => {},
685 detach: () => {},
686 setShown: () => {},
687 + setCaptureDrawn: () => {},
688 loadFailed: () => {},
689 humanStartedDownload: () => true,
690 agentLedPage: () => false,
@@ -2462,6 +2463,7 @@ describe("PreviewManager", () => {
2463 }),
2464 );
2465 expect(yield* Queue.take(capturing)).toBe(true);
2466 + yield* manager.setCaptureDrawn("tab_1", true);
2467 screenshot.resolve({ data: "" });
2468 expect(yield* Queue.take(capturing)).toBe(false);
2469 }).pipe(Effect.provide(managerLayer())),
apps/desktop/src/preview/Manager.ts
+16 -1
@@ -1827,7 +1827,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
1827 });
1828 if (Option.isNone(tab)) return;
1829 const closedTab = tab.value;
1830 - if (closedTab.serverTab) browserHost.setShown(closedTab.serverTab, undefined);
1830 + if (closedTab.serverTab) {
1831 + browserHost.setShown(closedTab.serverTab, undefined);
1832 + browserHost.setCaptureDrawn(closedTab.serverTab, undefined);
1833 + }
1834 if (closedTab.webContentsId != null) {
1835 yield* Effect.all(
1836 [
@@ -2635,12 +2638,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
2638 if (serverTab) browserHost.setShown(serverTab, shown);
2639 });
2640
2641 + const setCaptureDrawn = Effect.fn("PreviewManager.setCaptureDrawn")(function* (
2642 + tabId: string,
2643 + drawn: boolean,
2644 + ) {
2645 + const serverTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId)?.serverTab;
2646 + if (serverTab) browserHost.setCaptureDrawn(serverTab, drawn);
2647 + });
2648 +
2649 // A window reload or guest crash destroys the guest without the renderer reporting it hidden.
2650 // The renderer reports again once a new guest registers.
2651 const hideDestroyedGuest = Effect.fnUntraced(function* (webContentsId: number) {
2652 for (const tab of (yield* SynchronizedRef.get(tabsRef)).values()) {
2653 if (tab.webContentsId === webContentsId && tab.serverTab) {
2654 browserHost.setShown(tab.serverTab, false);
2655 + browserHost.setCaptureDrawn(tab.serverTab, false);
2656 }
2657 }
2658 });
@@ -3716,6 +3728,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
3728 setAnnotationSendEnabled,
3729 setAudioMuted,
3730 setShown,
3731 + setCaptureDrawn,
3732 setColorScheme,
3733 setMainWindow,
3734 setForwardedShortcuts: (shortcuts: ReadonlyArray<PreviewForwardedShortcut>) =>
@@ -3932,6 +3945,7 @@ export class PreviewManager extends Context.Service<
3945 audioMuted: boolean,
3946 ) => Effect.Effect<void, PreviewManagerError>;
3947 readonly setShown: (tabId: string, shown: boolean) => Effect.Effect<void>;
3948 + readonly setCaptureDrawn: (tabId: string, drawn: boolean) => Effect.Effect<void>;
3949 readonly openDevTools: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
3950 readonly clearCookies: (
3951 partitions?: ReadonlyArray<string>,
@@ -4094,6 +4108,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
4108 setZoomFactor: operations.setZoomFactor,
4109 setAudioMuted: operations.setAudioMuted,
4110 setShown: operations.setShown,
4111 + setCaptureDrawn: operations.setCaptureDrawn,
4112 openDevTools: operations.openDevTools,
4113 clearCookies: Effect.fn("PreviewManager.clearCookies")(function* (partitions) {
4114 yield* browserSession
apps/web/src/browser/HostedBrowserWebview.tsx
+65 -37
@@ -20,6 +20,7 @@ import { useActiveBrowserRecordingTabIds } from "./browserRecording";
20 import {
21 browserViewportSettingKey,
22 resolveBrowserViewportLayout,
23 + resolveBrowserViewportTransform,
24 resolveFittedBrowserViewport,
25 } from "./browserViewportLayout";
26 import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar";
@@ -313,6 +314,23 @@ export function HostedBrowserWebview(props: {
314 return () => withDesktopTab(runtimeTabId, () => bridge.setShown(runtimeTabId, false));
315 }, [clientSettingsHydrated, runtimeTabId, serverDriven, shown]);
316
317 + // Captures copy the guest's raster, which a scaled guest only keeps at full detail on request.
318 + const fullDetail = backgroundActivity || pictureInPicture || recordingActive;
319 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the desktop host once the guest is drawn at full detail, so its screenshot waits for that frame
320 + useEffect(() => {
321 + const bridge = window.desktopBridge?.preview;
322 + if (!clientSettingsHydrated || !bridge || !serverDriven || !fullDetail) return;
323 + let frame = requestAnimationFrame(() => {
324 + frame = requestAnimationFrame(() =>
325 + withDesktopTab(runtimeTabId, () => bridge.setCaptureDrawn(runtimeTabId, true)),
326 + );
327 + });
328 + return () => {
329 + cancelAnimationFrame(frame);
330 + withDesktopTab(runtimeTabId, () => bridge.setCaptureDrawn(runtimeTabId, false));
331 + };
332 + }, [clientSettingsHydrated, fullDetail, runtimeTabId, serverDriven]);
333 +
334 if (!clientSettingsHydrated || !config) return null;
335
336 // This fixed wrapper sits outside the right panel shell, so it takes thumb presses itself.
@@ -322,6 +340,7 @@ export function HostedBrowserWebview(props: {
340 };
341
342 const renderingActive = active || backgroundActivity || pictureInPicture || recordingActive;
343 + const viewportTransform = resolveBrowserViewportTransform(layout.viewportScale, fullDetail);
344 const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({
345 active,
346 renderingActive,
@@ -354,47 +373,56 @@ export function HostedBrowserWebview(props: {
373 onChange={commitViewportChange}
374 />
375 ) : null}
357 - <webview
358 - key={webviewGeneration}
359 - ref={setWebviewRef}
360 - // Electron needs allowpopups at attachment. React drops boolean values for unknown attributes.
361 - {...({ allowpopups: "true" } as unknown as { readonly allowpopups?: boolean })}
362 - src={webviewGeneration === 0 ? initialSrc : recoverySrc}
363 - partition={config.partition}
364 - webpreferences={config.webPreferences}
365 - {...(config.preloadUrl ? { preload: config.preloadUrl } : {})}
366 - data-preview-tab={runtimeTabId}
367 - data-preview-server-tab={tabId}
368 - data-preview-viewport-mode={effectiveViewport._tag}
369 - data-preview-viewport-key={browserViewportSettingKey(effectiveViewport)}
370 - data-preview-css-width={
371 - fittedSourceViewport
372 - ? fittedSourceViewport.width
373 - : effectiveViewport._tag === "fill"
374 - ? Math.max(1, Math.round(layout.viewportWidth / layoutZoomFactor))
375 - : effectiveViewport.width
376 - }
377 - data-preview-css-height={
378 - fittedSourceViewport
379 - ? fittedSourceViewport.height
380 - : effectiveViewport._tag === "fill"
381 - ? Math.max(1, Math.round(layout.viewportHeight / layoutZoomFactor))
382 - : effectiveViewport.height
383 - }
384 - aria-hidden={active ? undefined : true}
385 - className={cn(
386 - "absolute flex overflow-hidden bg-white",
387 - active && !layout.fillsPanel && "ring-1 ring-border/70 shadow-sm",
388 - )}
376 + <div
377 + className="absolute"
378 style={{
379 left: layout.viewportX,
380 top: layout.viewportY,
392 - width: layout.viewportWidth / layout.viewportScale,
393 - height: layout.viewportHeight / layout.viewportScale,
394 - transform: layout.viewportScale < 1 ? `scale(${layout.viewportScale})` : undefined,
395 - transformOrigin: "top left",
381 + width: layout.viewportWidth,
382 + height: layout.viewportHeight,
383 + transformStyle: viewportTransform && fullDetail ? "preserve-3d" : undefined,
384 }}
397 - />
385 + >
386 + <webview
387 + key={webviewGeneration}
388 + ref={setWebviewRef}
389 + // Electron needs allowpopups at attachment. React drops boolean values for unknown attributes.
390 + {...({ allowpopups: "true" } as unknown as { readonly allowpopups?: boolean })}
391 + src={webviewGeneration === 0 ? initialSrc : recoverySrc}
392 + partition={config.partition}
393 + webpreferences={config.webPreferences}
394 + {...(config.preloadUrl ? { preload: config.preloadUrl } : {})}
395 + data-preview-tab={runtimeTabId}
396 + data-preview-server-tab={tabId}
397 + data-preview-viewport-mode={effectiveViewport._tag}
398 + data-preview-viewport-key={browserViewportSettingKey(effectiveViewport)}
399 + data-preview-css-width={
400 + fittedSourceViewport
401 + ? fittedSourceViewport.width
402 + : effectiveViewport._tag === "fill"
403 + ? Math.max(1, Math.round(layout.viewportWidth / layoutZoomFactor))
404 + : effectiveViewport.width
405 + }
406 + data-preview-css-height={
407 + fittedSourceViewport
408 + ? fittedSourceViewport.height
409 + : effectiveViewport._tag === "fill"
410 + ? Math.max(1, Math.round(layout.viewportHeight / layoutZoomFactor))
411 + : effectiveViewport.height
412 + }
413 + aria-hidden={active ? undefined : true}
414 + className={cn(
415 + "absolute top-0 left-0 flex overflow-hidden bg-white",
416 + active && !layout.fillsPanel && "ring-1 ring-border/70 shadow-sm",
417 + )}
418 + style={{
419 + width: layout.viewportWidth / layout.viewportScale,
420 + height: layout.viewportHeight / layout.viewportScale,
421 + transform: viewportTransform,
422 + transformOrigin: "top left",
423 + }}
424 + />
425 + </div>
426 {active && effectiveViewport._tag !== "fill" && !fittedSourceViewport ? (
427 <>
428 <BrowserViewportResizeHandles
apps/web/src/browser/browserViewportLayout.test.ts
+17
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test";
2
3 import {
4 resizeBrowserViewportFromRail,
5 + resolveBrowserViewportTransform,
6 resolveFittedBrowserViewport,
7 } from "./browserViewportLayout";
8
@@ -43,3 +44,19 @@ describe("resolveBrowserViewportLayout", () => {
44 ).toEqual({ width: 460, height: 409 });
45 });
46 });
47 +
48 +describe("resolveBrowserViewportTransform", () => {
49 + it("keeps a full-detail guest at raster scale 1 while it draws at the fitted scale", () => {
50 + expect(resolveBrowserViewportTransform(1, true)).toBeUndefined();
51 + expect(resolveBrowserViewportTransform(0.6, false)).toBe("scale(0.6)");
52 + const values = resolveBrowserViewportTransform(0.6, true)
53 + ?.match(/^matrix3d\((.*)\)$/)?.[1]
54 + ?.split(",")
55 + .map(Number);
56 + // Column-major: the x and y columns keep their 0.6 screen terms and reach length 1 through z.
57 + expect(values?.slice(0, 2)).toEqual([0.6, 0]);
58 + expect(values?.slice(4, 6)).toEqual([0, 0.6]);
59 + expect(Math.hypot(...(values?.slice(0, 3) ?? []))).toBeCloseTo(1);
60 + expect(Math.hypot(...(values?.slice(4, 7) ?? []))).toBeCloseTo(1);
61 + });
62 +});
apps/web/src/browser/browserViewportLayout.ts
+16
@@ -125,6 +125,22 @@ export function resolveBrowserViewportLayout(
125 };
126 }
127
128 +/**
129 + * Chromium rasters a CSS-scaled guest at its on-screen scale, so captures of it lose detail.
130 + * Its raster scale also counts x/y-to-z terms, which a `preserve-3d` parent keeps and the
131 + * screen drops, so `fullDetail` draws like `scale()` but keeps full-resolution frames.
132 + */
133 +export function resolveBrowserViewportTransform(
134 + scale: number,
135 + fullDetail: boolean,
136 +): string | undefined {
137 + if (scale >= 1) return undefined;
138 + if (!fullDetail) return `scale(${scale})`;
139 + // Positive depth keeps the guest in front of its parent for hit testing.
140 + const depth = Math.sqrt(1 - scale * scale);
141 + return `matrix3d(${scale}, 0, ${depth}, 0, 0, ${scale}, ${depth}, 0, 0, 0, 1, 0, 0, 0, 0, 1)`;
142 +}
143 +
144 export function resolveBrowserDeviceViewportLayout(
145 container: { readonly width: number; readonly height: number },
146 setting: Exclude<PreviewViewportSetting, { readonly _tag: "fill" }>,
packages/contracts/src/ipc.ts
+7
@@ -1021,6 +1021,11 @@ export const DesktopPreviewSetShownInputSchema = Schema.Struct({
1021 shown: Schema.Boolean,
1022 });
1023
1024 +export const DesktopPreviewSetCaptureDrawnInputSchema = Schema.Struct({
1025 + tabId: DesktopPreviewTabIdSchema,
1026 + drawn: Schema.Boolean,
1027 +});
1028 +
1029 export const DesktopPreviewAnnotationThemeInputSchema = Schema.Struct({
1030 theme: DesktopPreviewAnnotationThemeSchema,
1031 });
@@ -1287,6 +1292,8 @@ export interface DesktopPreviewBridge {
1292 stopFindInPage: (tabId: string) => Promise<void>;
1293 /** Tells the tab's server whether its webview is on screen. */
1294 setShown: (tabId: string, shown: boolean) => Promise<void>;
1295 + /** Tells the tab's server whether its webview is drawn at full detail for a capture. */
1296 + setCaptureDrawn: (tabId: string, drawn: boolean) => Promise<void>;
1297 openDevTools: (tabId: string) => Promise<void>;
1298 clearCookies: (environmentId: EnvironmentId, profileId?: string) => Promise<void>;
1299 clearCache: (environmentId: EnvironmentId, profileId?: string) => Promise<void>;