@devries-cameron20 / t3code Mirror · read-only / commits / fd71ddc223
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 17:00 UTC

fix(desktop): hidden preview tabs throttle instead of running at full speed (#25184)

Focus emulation is now held only while a relay command is pending, the agent used the tab during its turn, a screencast is live, or a held screenshot waits its turn. Hidden guests throttle again. Takes over #24806. Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 25, 2026 at 19:30 UTC fd71ddc223091093d172a431f4080772fa41c2a9
19 files changed +1479 -56
apps/desktop/src/preview/CdpRelay.test.ts
+27 -4
@@ -65,8 +65,8 @@ describe("CDP relay", () => {
65 ]);
66 pending[0]!();
67 await settle();
68 - expect(send).toHaveBeenNthCalledWith(1, "Page.captureScreenshot", {}, undefined);
69 - expect(send).toHaveBeenNthCalledWith(2, "DOM.enable", {}, "iframe-session");
68 + expect(send).toHaveBeenNthCalledWith(1, "Page.captureScreenshot", {}, undefined, undefined);
69 + expect(send).toHaveBeenNthCalledWith(2, "DOM.enable", {}, "iframe-session", undefined);
70 expect(written.at(-1)).toEqual({
71 id: 5,
72 result: { method: "Page.captureScreenshot" },
@@ -92,7 +92,30 @@ describe("CDP relay", () => {
92 }),
93 );
94 await settle();
95 - expect(send.mock.calls).toEqual([["Runtime.runIfWaitingForDebugger", {}, "iframe-session"]]);
95 + expect(send.mock.calls).toEqual([
96 + ["Runtime.runIfWaitingForDebugger", {}, "iframe-session", undefined],
97 + ]);
98 + });
99 +
100 + // Playwright enables focus emulation on every page it connects to, and Chromium then
101 + // counts the guest as captured: a preview the panel hides keeps its timers unthrottled.
102 + it("acknowledges Playwright's focus emulation without forcing it onto the guest", async () => {
103 + const send = vi.fn(async () => ({}));
104 + const written: Array<Record<string, unknown>> = [];
105 + const relay = createCdpRelayConnection(makeTarget(send), (raw) =>
106 + written.push(JSON.parse(raw)),
107 + );
108 + relay.receive(
109 + JSON.stringify({
110 + id: 1,
111 + method: "Emulation.setFocusEmulationEnabled",
112 + params: { enabled: true },
113 + sessionId: "t3-preview-page",
114 + }),
115 + );
116 + await settle();
117 + expect(written).toEqual([{ id: 1, result: {}, sessionId: "t3-preview-page" }]);
118 + expect(send).not.toHaveBeenCalled();
119 });
120
121 it("routes tab events to the page session once the page is announced", async () => {
@@ -147,7 +170,7 @@ describe("CDP relay extra sessions", () => {
170 expect(written.find((m) => m["id"] === 4)).toHaveProperty("error");
171 relay.receive(JSON.stringify({ id: 5, method: "Page.startScreencast", sessionId: extra }));
172 await settle();
150 - expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined);
173 + expect(send).toHaveBeenLastCalledWith("Page.startScreencast", {}, undefined, extra);
174 written.length = 0;
175 relay.event("Page.screencastFrame", { data: "x" }, undefined);
176 expect(written.map((m) => m["sessionId"])).toEqual(["t3-preview-page", extra]);
apps/desktop/src/preview/CdpRelay.ts
+11 -2
@@ -5,7 +5,10 @@ export interface CdpRelayTarget {
5 method: string,
6 params: Record<string, unknown>,
7 sessionId: string | undefined,
8 + /** The extra session that sent it, such as a viewer's; the main connection has none. */
9 + owner?: string,
10 ) => Promise<unknown>;
11 + readonly sessionDetached?: (owner: string) => void;
12 /** The page's own CDP target id; Playwright keys the main frame by it. */
13 readonly targetId: () => Promise<string>;
14 readonly url: () => string;
@@ -111,8 +114,10 @@ export function createCdpRelayConnection(
114 }
115 case "Target.detachFromTarget": {
116 const sessionId = command.params?.["sessionId"];
114 - if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID)
117 + if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) {
118 sessions.delete(sessionId);
119 + target.sessionDetached?.(sessionId);
120 + }
121 return {};
122 }
123 case "Target.setAutoAttach": {
@@ -133,11 +138,15 @@ export function createCdpRelayConnection(
138
139 const pageCommand = (command: CdpCommand): Promise<unknown> => {
140 const child = sessions.get(command.sessionId!) === "page" ? undefined : command.sessionId;
141 + const owner =
142 + child === undefined && command.sessionId !== PAGE_SESSION_ID ? command.sessionId : undefined;
143 // Child frames/workers pause on auto-attach until resumed; only the main page can skip this command.
144 if (command.method === "Runtime.runIfWaitingForDebugger" && child === undefined) {
145 return Promise.resolve({});
146 }
140 - return target.send(command.method, command.params ?? {}, child);
147 + // Playwright asks for this on every page; the host emulates focus only while it is in use.
148 + if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({});
149 + return target.send(command.method, command.params ?? {}, child, owner);
150 };
151
152 return {
apps/desktop/src/preview/DesktopBrowserHost.test.ts
+723 -3
@@ -6,6 +6,7 @@ import * as Fiber from "effect/Fiber";
6 import * as Queue from "effect/Queue";
7 import * as Schema from "effect/Schema";
8 import * as Stream from "effect/Stream";
9 +import * as TestClock from "effect/testing/TestClock";
10 import * as NodeEvents from "node:events";
11
12 import * as DesktopBrowserHost from "./DesktopBrowserHost.ts";
@@ -30,18 +31,39 @@ const command = (
31 }),
32 );
33
33 -const makeDebuggee = () => {
34 +const makeDebuggee = (
35 + options: {
36 + readonly deferFocus?: boolean;
37 + readonly refuse?: (method: string, sessionId: string | undefined) => boolean;
38 + } = {},
39 +) => {
40 const emitter = new NodeEvents.EventEmitter();
41 const pending: Array<() => void> = [];
42 + const focusChanges: Array<() => void> = [];
43 const sent: Array<unknown> = [];
44 const sendWaiters: Array<() => void> = [];
45 + const focusEmulation: Array<boolean> = [];
46 const debuggee = Object.assign(emitter, {
39 - sendCommand: (method: string, params?: { id?: number }) =>
40 - new Promise((resolve) => {
47 + sendCommand: (
48 + method: string,
49 + params?: { id?: number; enabled?: boolean },
50 + sessionId?: string,
51 + ) =>
52 + new Promise((resolve, reject) => {
53 + if (options.refuse?.(method, sessionId)) {
54 + reject(new Error("Refused"));
55 + return;
56 + }
57 if (method === "Target.getTargetInfo") {
58 resolve({ targetInfo: { targetId: "GUEST" } });
59 return;
60 }
61 + if (method === "Emulation.setFocusEmulationEnabled") {
62 + focusEmulation.push(params?.enabled === true);
63 + if (options.deferFocus) focusChanges.push(() => resolve({}));
64 + else resolve({});
65 + return;
66 + }
67 sent.push(params?.id);
68 pending.push(() => resolve({ method }));
69 sendWaiters.shift()?.();
@@ -61,12 +83,18 @@ const makeDebuggee = () => {
83 navigate: () => webContents.emit("did-navigate"),
84 /** Ids of the commands the debugger received, in order. */
85 sent,
86 + focusEmulation,
87 nextSend: () => new Promise<void>((resolve) => sendWaiters.push(resolve)),
88 release: () => pending.splice(0).forEach((resolve) => resolve()),
89 releaseFirst: () => pending.shift()?.(),
90 + settleFocus: () => focusChanges.splice(0).forEach((resolve) => resolve()),
91 };
92 };
93
94 +/** Commands wait behind a focus emulation change; it settles within a macrotask. */
95 +// oxlint-disable-next-line t3code/no-real-time-wait-in-tests -- a zero-delay macrotask boundary, not a wait for time
96 +const settle = Effect.promise(() => new Promise<void>((resolve) => setImmediate(resolve)));
97 +
98 const takeEvents = (host: DesktopBrowserHost.DesktopBrowserHost["Service"], count: number) =>
99 host.events.pipe(
100 Stream.take(count),
@@ -93,6 +121,18 @@ const watchReplies = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) =>
121 return replies;
122 });
123
124 +/** Whether the client should paint the guest, each time that changes. */
125 +const watchPainting = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) =>
126 + Effect.gen(function* () {
127 + const painting: Array<boolean> = [];
128 + yield* host.painting.pipe(
129 + Stream.runForEach(({ active }) => Effect.sync(() => painting.push(active))),
130 + Effect.forkScoped,
131 + );
132 + yield* Effect.yieldNow;
133 + return painting;
134 + });
135 +
136 describe("DesktopBrowserHost", () => {
137 it.effect("announces tabs already attached to a backend that starts later", () =>
138 Effect.gen(function* () {
@@ -123,6 +163,7 @@ describe("DesktopBrowserHost", () => {
163 yield* command(1, "Page.captureScreenshot");
164 yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
165 yield* command(2, "DOM.enable");
166 + yield* settle;
167 debuggee.release();
168 const [, reply] = yield* Fiber.join(reader);
169
@@ -141,6 +182,7 @@ describe("DesktopBrowserHost", () => {
182 yield* command(host, 1, "Page.captureScreenshot");
183 yield* command(host, 2, "Page.captureScreenshot");
184 yield* command(host, 3, "DOM.enable");
185 + yield* settle;
186 expect(debuggee.sent).toEqual([1, 3]);
187
188 debuggee.releaseFirst();
@@ -159,6 +201,7 @@ describe("DesktopBrowserHost", () => {
201 yield* command(host, 1, "Page.captureScreenshot");
202 yield* command(host, 2, "Page.captureScreenshot");
203 yield* command(host, 3, "Page.captureScreenshot");
204 + yield* settle;
205 debuggee.navigate();
206 expect(debuggee.sent).toEqual([1, 2]);
207
@@ -181,6 +224,7 @@ describe("DesktopBrowserHost", () => {
224
225 yield* command(host, 1, "Page.captureScreenshot");
226 yield* command(host, 2, "Page.captureScreenshot");
227 + yield* settle;
228 yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
229 yield* command(host, 3, "Page.captureScreenshot");
230 expect(debuggee.sent).toEqual([1]);
@@ -192,6 +236,682 @@ describe("DesktopBrowserHost", () => {
236 }),
237 );
238
239 + it.effect("emulates focus while the server drives a guest, then lets a hidden one throttle", () =>
240 + Effect.gen(function* () {
241 + const host = yield* DesktopBrowserHost.make;
242 + const debuggee = makeDebuggee();
243 + host.attach(key, debuggee.tab);
244 +
245 + yield* command(host, 1, "DOM.enable");
246 + yield* settle;
247 + expect(debuggee.focusEmulation).toEqual([true]);
248 + expect(debuggee.sent).toEqual([1]);
249 + debuggee.release();
250 + yield* settle;
251 +
252 + yield* TestClock.adjust("4 seconds");
253 + yield* command(host, 2, "Input.dispatchMouseEvent");
254 + yield* settle;
255 + debuggee.release();
256 + yield* settle;
257 + yield* TestClock.adjust("4 seconds");
258 + expect(debuggee.focusEmulation).toEqual([true]);
259 +
260 + yield* TestClock.adjust("10 seconds");
261 + expect(debuggee.focusEmulation).toEqual([true, false]);
262 +
263 + yield* command(host, 3, "Page.captureScreenshot");
264 + yield* settle;
265 + expect(debuggee.focusEmulation).toEqual([true, false, true]);
266 + expect(debuggee.sent).toEqual([1, 2, 3]);
267 + }),
268 + );
269 +
270 + it.effect("keeps a guest focused while a command it is running is still pending", () =>
271 + Effect.gen(function* () {
272 + const host = yield* DesktopBrowserHost.make;
273 + const debuggee = makeDebuggee();
274 + host.attach(key, debuggee.tab);
275 + const replies = yield* watchReplies(host);
276 +
277 + // An evaluation awaiting the page's next frame, which a throttled guest never paints.
278 + yield* command(host, 1, "Runtime.evaluate");
279 + yield* settle;
280 + expect(debuggee.sent).toEqual([1]);
281 + yield* TestClock.adjust("1 minute");
282 + expect(debuggee.focusEmulation).toEqual([true]);
283 +
284 + debuggee.release();
285 + expect(yield* Queue.take(replies)).toBe(1);
286 + yield* TestClock.adjust("1 minute");
287 + expect(debuggee.focusEmulation).toEqual([true, false]);
288 + }).pipe(Effect.scoped),
289 + );
290 +
291 + it.effect("lets a guest throttle once Chromium can no longer answer its pending command", () =>
292 + Effect.gen(function* () {
293 + const host = yield* DesktopBrowserHost.make;
294 + const debuggee = makeDebuggee();
295 + host.attach(key, debuggee.tab);
296 +
297 + yield* command(host, 1, "Runtime.evaluate");
298 + yield* settle;
299 + yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
300 + yield* TestClock.adjust("1 minute");
301 + expect(debuggee.focusEmulation).toEqual([true, false]);
302 +
303 + // A command the old document never answers.
304 + yield* command(host, 2, "Page.captureScreenshot");
305 + yield* settle;
306 + expect(debuggee.sent).toEqual([1, 2]);
307 + debuggee.navigate();
308 + yield* TestClock.adjust("1 minute");
309 + expect(debuggee.focusEmulation).toEqual([true, false, true, false]);
310 + }),
311 + );
312 +
313 + it.effect("keeps a guest focused for the screenshot a navigation lets start", () =>
314 + Effect.gen(function* () {
315 + const host = yield* DesktopBrowserHost.make;
316 + const debuggee = makeDebuggee();
317 + host.attach(key, debuggee.tab);
318 +
319 + yield* command(host, 1, "Page.captureScreenshot");
320 + yield* command(host, 2, "Page.captureScreenshot");
321 + yield* settle;
322 + debuggee.navigate();
323 + expect(debuggee.sent).toEqual([1, 2]);
324 + yield* TestClock.adjust("1 minute");
325 + expect(debuggee.focusEmulation).toEqual([true]);
326 + }),
327 + );
328 +
329 + it.effect("never sends a screenshot a navigation superseded while focus was changing", () =>
330 + Effect.gen(function* () {
331 + const host = yield* DesktopBrowserHost.make;
332 + const debuggee = makeDebuggee({ deferFocus: true });
333 + host.attach(key, debuggee.tab);
334 +
335 + yield* command(host, 1, "Page.captureScreenshot");
336 + yield* command(host, 2, "Page.captureScreenshot");
337 + yield* settle;
338 + debuggee.navigate();
339 + debuggee.settleFocus();
340 + yield* settle;
341 + // Two captures in flight would undo the one-at-a-time guard.
342 + expect(debuggee.sent).toEqual([2]);
343 + }),
344 + );
345 +
346 + it.effect("keeps a guest focused when its turn starts after the agent's first command", () =>
347 + Effect.gen(function* () {
348 + const host = yield* DesktopBrowserHost.make;
349 + const debuggee = makeDebuggee();
350 + host.attach(key, debuggee.tab);
351 + const turn = (active: boolean) =>
352 + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active }));
353 +
354 + yield* command(host, 1, "Input.dispatchKeyEvent");
355 + yield* settle;
356 + yield* turn(true);
357 + debuggee.release();
358 + yield* settle;
359 + yield* TestClock.adjust("1 minute");
360 + expect(debuggee.focusEmulation).toEqual([true]);
361 +
362 + yield* turn(false);
363 + yield* TestClock.adjust("1 minute");
364 + expect(debuggee.focusEmulation).toEqual([true, false]);
365 + }),
366 + );
367 +
368 + it.effect("lets a guest throttle through a turn that only sent it housekeeping", () =>
369 + Effect.gen(function* () {
370 + const host = yield* DesktopBrowserHost.make;
371 + const debuggee = makeDebuggee();
372 + host.attach(key, debuggee.tab);
373 + yield* host.handleCommandLine(
374 + encodeJson({ type: "turn", threadId: key.threadId, active: true }),
375 + );
376 +
377 + // Playwright re-enabling a domain after a reload is not the agent using the tab.
378 + yield* command(host, 1, "Runtime.enable");
379 + yield* settle;
380 + debuggee.release();
381 + yield* settle;
382 + yield* TestClock.adjust("1 minute");
383 + expect(debuggee.focusEmulation).toEqual([true, false]);
384 +
385 + yield* command(host, 2, "Runtime.evaluate");
386 + yield* settle;
387 + debuggee.release();
388 + yield* settle;
389 + yield* TestClock.adjust("1 minute");
390 + expect(debuggee.focusEmulation).toEqual([true, false, true]);
391 + }),
392 + );
393 +
394 + it.effect("keeps a guest the agent used focused through its turn's pauses", () =>
395 + Effect.gen(function* () {
396 + const host = yield* DesktopBrowserHost.make;
397 + const debuggee = makeDebuggee();
398 + host.attach(key, debuggee.tab);
399 + const turn = (active: boolean) =>
400 + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active }));
401 +
402 + yield* turn(true);
403 + yield* TestClock.adjust("1 minute");
404 + expect(debuggee.focusEmulation).toEqual([]);
405 +
406 + yield* command(host, 1, "Input.dispatchKeyEvent");
407 + yield* settle;
408 + debuggee.release();
409 + yield* settle;
410 + // The model thinks between steps; a focus-sensitive menu must not blur meanwhile.
411 + yield* TestClock.adjust("1 minute");
412 + expect(debuggee.focusEmulation).toEqual([true]);
413 +
414 + yield* turn(false);
415 + yield* TestClock.adjust("1 minute");
416 + expect(debuggee.focusEmulation).toEqual([true, false]);
417 + }),
418 + );
419 +
420 + it.effect(
421 + "ends a command's hold with the turn that sent it, even if Chromium never answers",
422 + () =>
423 + Effect.gen(function* () {
424 + const host = yield* DesktopBrowserHost.make;
425 + const debuggee = makeDebuggee();
426 + host.attach(key, debuggee.tab);
427 + yield* host.handleCommandLine(
428 + encodeJson({ type: "turn", threadId: key.threadId, active: true }),
429 + );
430 +
431 + // A screenshot of a hidden guest the server already gave up on.
432 + yield* command(host, 1, "Page.captureScreenshot");
433 + yield* settle;
434 + expect(debuggee.sent).toEqual([1]);
435 + yield* host.handleCommandLine(
436 + encodeJson({ type: "turn", threadId: key.threadId, active: false }),
437 + );
438 + yield* TestClock.adjust("1 minute");
439 + expect(debuggee.focusEmulation).toEqual([true, false]);
440 + }),
441 + );
442 +
443 + it.effect("ends the agent's own session's holds with its turn, but not a viewer's stream", () =>
444 + Effect.gen(function* () {
445 + const host = yield* DesktopBrowserHost.make;
446 + const debuggee = makeDebuggee();
447 + host.attach(key, debuggee.tab);
448 + const replies = yield* watchReplies(host);
449 + const raw = (message: Record<string, unknown>) =>
450 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
451 + const turn = (active: boolean) =>
452 + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active }));
453 +
454 + // The server's own CDP session, evaluating a promise that never settles.
455 + yield* turn(true);
456 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
457 + expect(yield* Queue.take(replies)).toBe(1);
458 + yield* raw({ id: 2, method: "Runtime.evaluate", sessionId: "t3-preview-page-1" });
459 + yield* settle;
460 + yield* turn(false);
461 + yield* TestClock.adjust("1 minute");
462 + expect(debuggee.focusEmulation).toEqual([true, false]);
463 +
464 + // A viewer streaming the guest keeps it focused through a turn's end.
465 + yield* turn(true);
466 + yield* raw({ id: 3, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
467 + expect(yield* Queue.take(replies)).toBe(3);
468 + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" });
469 + yield* settle;
470 + yield* turn(false);
471 + yield* TestClock.adjust("1 minute");
472 + expect(debuggee.focusEmulation).toEqual([true, false, true]);
473 + }).pipe(Effect.scoped),
474 + );
475 +
476 + it.effect("paints for a screenshot queued behind one its viewer abandoned mid-capture", () =>
477 + Effect.gen(function* () {
478 + const host = yield* DesktopBrowserHost.make;
479 + const debuggee = makeDebuggee();
480 + host.attach(key, debuggee.tab);
481 + const replies = yield* watchReplies(host);
482 + const painting = yield* watchPainting(host);
483 + const raw = (message: Record<string, unknown>) =>
484 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
485 +
486 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
487 + expect(yield* Queue.take(replies)).toBe(1);
488 + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" });
489 + yield* settle;
490 + yield* raw({
491 + id: 3,
492 + method: "Target.detachFromTarget",
493 + params: { sessionId: "t3-preview-page-1" },
494 + });
495 + expect(yield* Queue.take(replies)).toBe(3);
496 + expect(painting).toEqual([true, false]);
497 +
498 + // The agent's screenshot waits its turn; painting lets the abandoned one finish first.
499 + yield* command(host, 4, "Page.captureScreenshot");
500 + yield* settle;
501 + expect(painting).toEqual([true, false, true]);
502 + debuggee.release();
503 + expect(yield* Queue.take(replies)).toBe(2);
504 + yield* settle;
505 + expect(debuggee.sent).toEqual([undefined, 4]);
506 + debuggee.release();
507 + expect(yield* Queue.take(replies)).toBe(4);
508 + yield* settle;
509 + expect(painting).toEqual([true, false, true, false]);
510 + }).pipe(Effect.scoped),
511 + );
512 +
513 + it.effect("stops painting for a queued screenshot once its session detaches", () =>
514 + Effect.gen(function* () {
515 + const host = yield* DesktopBrowserHost.make;
516 + const debuggee = makeDebuggee();
517 + host.attach(key, debuggee.tab);
518 + const replies = yield* watchReplies(host);
519 + const painting = yield* watchPainting(host);
520 + const raw = (message: Record<string, unknown>) =>
521 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
522 + const turn = (active: boolean) =>
523 + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active }));
524 +
525 + // The agent's screenshot is abandoned at turn end; a viewer's waits behind it, then leaves.
526 + yield* turn(true);
527 + yield* command(host, 1, "Page.captureScreenshot");
528 + yield* settle;
529 + yield* turn(false);
530 + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
531 + expect(yield* Queue.take(replies)).toBe(2);
532 + yield* raw({ id: 3, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" });
533 + yield* settle;
534 + expect(painting).toEqual([true, false, true]);
535 + yield* raw({
536 + id: 4,
537 + method: "Target.detachFromTarget",
538 + params: { sessionId: "t3-preview-page-1" },
539 + });
540 + expect(yield* Queue.take(replies)).toBe(4);
541 + expect(painting).toEqual([true, false, true, false]);
542 + yield* TestClock.adjust("1 minute");
543 + expect(debuggee.focusEmulation).toEqual([true, false]);
544 + }).pipe(Effect.scoped),
545 + );
546 +
547 + it.effect("never sends a screenshot whose session detached while focus was changing", () =>
548 + Effect.gen(function* () {
549 + const host = yield* DesktopBrowserHost.make;
550 + const debuggee = makeDebuggee({ deferFocus: true });
551 + host.attach(key, debuggee.tab);
552 + const replies = yield* watchReplies(host);
553 + const raw = (message: Record<string, unknown>) =>
554 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
555 +
556 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
557 + expect(yield* Queue.take(replies)).toBe(1);
558 + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" });
559 + yield* raw({
560 + id: 3,
561 + method: "Target.detachFromTarget",
562 + params: { sessionId: "t3-preview-page-1" },
563 + });
564 + expect(yield* Queue.take(replies)).toBe(3);
565 + debuggee.settleFocus();
566 + yield* settle;
567 + expect(debuggee.sent).toEqual([]);
568 + }).pipe(Effect.scoped),
569 + );
570 +
571 + it.effect("stops painting for a screenshot still queued when the agent's turn ends", () =>
572 + Effect.gen(function* () {
573 + const host = yield* DesktopBrowserHost.make;
574 + const debuggee = makeDebuggee();
575 + host.attach(key, debuggee.tab);
576 + const painting = yield* watchPainting(host);
577 + const turn = (active: boolean) =>
578 + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active }));
579 +
580 + yield* turn(true);
581 + yield* command(host, 1, "Page.captureScreenshot");
582 + yield* command(host, 2, "Page.captureScreenshot");
583 + yield* settle;
584 + yield* turn(false);
585 + yield* settle;
586 + expect(painting).toEqual([true, false]);
587 +
588 + // The first answers late; the second still goes in turn, but holds nothing.
589 + debuggee.releaseFirst();
590 + yield* settle;
591 + expect(debuggee.sent).toEqual([1, 2]);
592 + expect(painting).toEqual([true, false]);
593 + yield* TestClock.adjust("1 minute");
594 + expect(debuggee.focusEmulation).toEqual([true, false]);
595 + }).pipe(Effect.scoped),
596 + );
597 +
598 + it.effect("lets a guest throttle when Chromium refuses a viewer's screencast", () =>
599 + Effect.gen(function* () {
600 + const host = yield* DesktopBrowserHost.make;
601 + const debuggee = makeDebuggee({ refuse: (method) => method === "Page.startScreencast" });
602 + host.attach(key, debuggee.tab);
603 + const replies = yield* watchReplies(host);
604 + const painting = yield* watchPainting(host);
605 + const raw = (message: Record<string, unknown>) =>
606 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
607 +
608 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
609 + expect(yield* Queue.take(replies)).toBe(1);
610 + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "t3-preview-page-1" });
611 + expect(yield* Queue.take(replies)).toBe(2);
612 + yield* settle;
613 + expect(painting).toEqual([true, false]);
614 + yield* TestClock.adjust("1 minute");
615 + expect(debuggee.focusEmulation).toEqual([true, false]);
616 + }).pipe(Effect.scoped),
617 + );
618 +
619 + it.effect("lets a guest throttle after a person's input through a viewer that then closed", () =>
620 + Effect.gen(function* () {
621 + const host = yield* DesktopBrowserHost.make;
622 + const debuggee = makeDebuggee();
623 + host.attach(key, debuggee.tab);
624 + const replies = yield* watchReplies(host);
625 + const raw = (message: Record<string, unknown>) =>
626 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
627 +
628 + yield* host.handleCommandLine(
629 + encodeJson({ type: "turn", threadId: key.threadId, active: true }),
630 + );
631 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
632 + expect(yield* Queue.take(replies)).toBe(1);
633 + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "t3-preview-page-1" });
634 + yield* raw({ id: 3, method: "Input.dispatchMouseEvent", sessionId: "t3-preview-page-1" });
635 + yield* settle;
636 + debuggee.release();
637 + yield* raw({
638 + id: 4,
639 + method: "Target.detachFromTarget",
640 + params: { sessionId: "t3-preview-page-1" },
641 + });
642 + yield* settle;
643 + yield* TestClock.adjust("1 minute");
644 + expect(debuggee.focusEmulation).toEqual([true, false]);
645 + }).pipe(Effect.scoped),
646 + );
647 +
648 + it.effect("keeps a page's screencast when a child session's screencast is refused", () =>
649 + Effect.gen(function* () {
650 + const host = yield* DesktopBrowserHost.make;
651 + const debuggee = makeDebuggee({
652 + refuse: (method, sessionId) => method === "Page.startScreencast" && sessionId !== undefined,
653 + });
654 + host.attach(key, debuggee.tab);
655 + const replies = yield* watchReplies(host);
656 + const raw = (message: Record<string, unknown>) =>
657 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
658 +
659 + yield* command(host, 1, "Page.startScreencast");
660 + yield* settle;
661 + debuggee.release();
662 + expect(yield* Queue.take(replies)).toBe(1);
663 + yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "child-frame" });
664 + expect(yield* Queue.take(replies)).toBe(2);
665 + yield* TestClock.adjust("1 minute");
666 + expect(debuggee.focusEmulation).toEqual([true]);
667 + }).pipe(Effect.scoped),
668 + );
669 +
670 + it.effect("ignores a screencast that arrives on a session after it detached", () =>
671 + Effect.gen(function* () {
672 + const host = yield* DesktopBrowserHost.make;
673 + const debuggee = makeDebuggee();
674 + host.attach(key, debuggee.tab);
675 + const replies = yield* watchReplies(host);
676 + const raw = (message: Record<string, unknown>) =>
677 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
678 +
679 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
680 + expect(yield* Queue.take(replies)).toBe(1);
681 + yield* raw({
682 + id: 2,
683 + method: "Target.detachFromTarget",
684 + params: { sessionId: "t3-preview-page-1" },
685 + });
686 + expect(yield* Queue.take(replies)).toBe(2);
687 + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" });
688 + yield* settle;
689 + debuggee.release();
690 + yield* settle;
691 + yield* TestClock.adjust("1 minute");
692 + expect(debuggee.focusEmulation).toEqual([true, false]);
693 + }).pipe(Effect.scoped),
694 + );
695 +
696 + it.effect("lets a guest throttle once the session waiting on its command detaches", () =>
697 + Effect.gen(function* () {
698 + const host = yield* DesktopBrowserHost.make;
699 + const debuggee = makeDebuggee();
700 + host.attach(key, debuggee.tab);
701 + const replies = yield* watchReplies(host);
702 + const raw = (message: Record<string, unknown>) =>
703 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
704 +
705 + // A viewer's own CDP session, closed while its still of a hidden guest is pending.
706 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
707 + expect(yield* Queue.take(replies)).toBe(1);
708 + yield* raw({ id: 2, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" });
709 + yield* settle;
710 + yield* TestClock.adjust("1 minute");
711 + expect(debuggee.focusEmulation).toEqual([true]);
712 +
713 + yield* raw({
714 + id: 3,
715 + method: "Target.detachFromTarget",
716 + params: { sessionId: "t3-preview-page-1" },
717 + });
718 + yield* TestClock.adjust("1 minute");
719 + expect(debuggee.focusEmulation).toEqual([true, false]);
720 + }).pipe(Effect.scoped),
721 + );
722 +
723 + it.effect("never sends a screenshot queued by a session that has since detached", () =>
724 + Effect.gen(function* () {
725 + const host = yield* DesktopBrowserHost.make;
726 + const debuggee = makeDebuggee();
727 + host.attach(key, debuggee.tab);
728 + const replies = yield* watchReplies(host);
729 + const raw = (message: Record<string, unknown>) =>
730 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
731 +
732 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
733 + expect(yield* Queue.take(replies)).toBe(1);
734 + yield* command(host, 2, "Page.captureScreenshot");
735 + yield* raw({ id: 3, method: "Page.captureScreenshot", sessionId: "t3-preview-page-1" });
736 + yield* raw({
737 + id: 4,
738 + method: "Target.detachFromTarget",
739 + params: { sessionId: "t3-preview-page-1" },
740 + });
741 + yield* settle;
742 + debuggee.release();
743 + expect(yield* Queue.take(replies)).toBe(4);
744 + // The dropped screenshot's error reply leaves as the first frees its turn.
745 + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([3, 2]);
746 + yield* settle;
747 + expect(debuggee.sent).toEqual([2]);
748 +
749 + yield* TestClock.adjust("1 minute");
750 + expect(debuggee.focusEmulation).toEqual([true, false]);
751 + }).pipe(Effect.scoped),
752 + );
753 +
754 + it.effect("keeps a streamed guest focused until its screencast ends", () =>
755 + Effect.gen(function* () {
756 + const host = yield* DesktopBrowserHost.make;
757 + const debuggee = makeDebuggee();
758 + host.attach(key, debuggee.tab);
759 +
760 + yield* command(host, 1, "Page.startScreencast");
761 + yield* TestClock.adjust("1 minute");
762 + expect(debuggee.focusEmulation).toEqual([true]);
763 +
764 + yield* command(host, 2, "Page.stopScreencast");
765 + yield* settle;
766 + debuggee.release();
767 + yield* settle;
768 + yield* TestClock.adjust("1 minute");
769 + expect(debuggee.focusEmulation).toEqual([true, false]);
770 +
771 + // A viewer whose connection went away without stopping its screencast.
772 + yield* command(host, 3, "Page.startScreencast");
773 + yield* settle;
774 + debuggee.release();
775 + yield* settle;
776 + yield* TestClock.adjust("1 minute");
777 + yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
778 + yield* TestClock.adjust("1 minute");
779 + expect(debuggee.focusEmulation).toEqual([true, false, true, false]);
780 + }),
781 + );
782 +
783 + it.effect("keeps a guest focused while any of its screencasts runs", () =>
784 + Effect.gen(function* () {
785 + const host = yield* DesktopBrowserHost.make;
786 + const debuggee = makeDebuggee();
787 + host.attach(key, debuggee.tab);
788 + const replies = yield* watchReplies(host);
789 + const raw = (message: Record<string, unknown>) =>
790 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
791 +
792 + // Two viewers, or a viewer and a recording, each on its own session.
793 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
794 + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
795 + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]);
796 + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" });
797 + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" });
798 + yield* raw({ id: 5, method: "Page.stopScreencast", sessionId: "t3-preview-page-1" });
799 + yield* settle;
800 + debuggee.release();
801 + yield* settle;
802 + yield* TestClock.adjust("1 minute");
803 + expect(debuggee.focusEmulation).toEqual([true]);
804 +
805 + yield* raw({
806 + id: 6,
807 + method: "Target.detachFromTarget",
808 + params: { sessionId: "t3-preview-page-2" },
809 + });
810 + yield* TestClock.adjust("1 minute");
811 + expect(debuggee.focusEmulation).toEqual([true, false]);
812 + }).pipe(Effect.scoped),
813 + );
814 +
815 + it.effect("paints a hidden guest only while the screenshot it is taking is pending", () =>
816 + Effect.gen(function* () {
817 + const host = yield* DesktopBrowserHost.make;
818 + const debuggee = makeDebuggee();
819 + host.attach(key, debuggee.tab);
820 + const painting = yield* watchPainting(host);
821 +
822 + yield* command(host, 1, "DOM.enable");
823 + yield* settle;
824 + expect(painting).toEqual([]);
825 +
826 + yield* command(host, 2, "Page.captureScreenshot");
827 + yield* command(host, 3, "Page.captureScreenshot");
828 + yield* settle;
829 + expect(painting).toEqual([true]);
830 +
831 + // The first answers once the guest paints; the queued one then takes its turn.
832 + debuggee.release();
833 + yield* settle;
834 + expect(debuggee.sent).toEqual([1, 2, 3]);
835 + debuggee.release();
836 + yield* settle;
837 + expect(painting).toEqual([true, false]);
838 + }).pipe(Effect.scoped),
839 + );
840 +
841 + it.effect("stops painting for a screenshot Chromium never answers", () =>
842 + Effect.gen(function* () {
843 + const host = yield* DesktopBrowserHost.make;
844 + const debuggee = makeDebuggee();
845 + host.attach(key, debuggee.tab);
846 + const painting = yield* watchPainting(host);
847 + const turn = (active: boolean) =>
848 + host.handleCommandLine(encodeJson({ type: "turn", threadId: key.threadId, active }));
849 +
850 + // The server gave up on it, and the agent's turn ends.
851 + yield* turn(true);
852 + yield* command(host, 1, "Page.captureScreenshot");
853 + yield* settle;
854 + yield* turn(false);
855 + yield* settle;
856 + expect(painting).toEqual([true, false]);
857 +
858 + // The page loads a new document, which also frees the screenshot turn.
859 + debuggee.navigate();
860 + yield* command(host, 2, "Page.captureScreenshot");
861 + yield* settle;
862 + debuggee.navigate();
863 + yield* settle;
864 + expect(painting).toEqual([true, false, true, false]);
865 +
866 + // The server releases the relay.
867 + yield* command(host, 3, "Page.captureScreenshot");
868 + yield* settle;
869 + yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
870 + yield* settle;
871 + expect(painting).toEqual([true, false, true, false, true, false]);
872 +
873 + // The guest detaches; a replacement starts with no screenshot pending.
874 + const replacement = makeDebuggee();
875 + host.attach(key, replacement.tab);
876 + yield* command(host, 4, "Page.captureScreenshot");
877 + yield* settle;
878 + host.detach(key);
879 + yield* settle;
880 + expect(painting).toEqual([true, false, true, false, true, false, true, false]);
881 + }).pipe(Effect.scoped),
882 + );
883 +
884 + it.effect("paints a streamed guest until its last screencast ends", () =>
885 + Effect.gen(function* () {
886 + const host = yield* DesktopBrowserHost.make;
887 + const debuggee = makeDebuggee();
888 + host.attach(key, debuggee.tab);
889 + const replies = yield* watchReplies(host);
890 + const painting = yield* watchPainting(host);
891 + const raw = (message: Record<string, unknown>) =>
892 + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) }));
893 +
894 + yield* raw({ id: 1, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
895 + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } });
896 + expect([yield* Queue.take(replies), yield* Queue.take(replies)]).toEqual([1, 2]);
897 + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" });
898 + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-2" });
899 + yield* raw({ id: 5, method: "Page.stopScreencast", sessionId: "t3-preview-page-1" });
900 + yield* settle;
901 + debuggee.release();
902 + yield* settle;
903 + expect(painting).toEqual([true]);
904 +
905 + yield* raw({
906 + id: 6,
907 + method: "Target.detachFromTarget",
908 + params: { sessionId: "t3-preview-page-2" },
909 + });
910 + yield* settle;
911 + expect(painting).toEqual([true, false]);
912 + }).pipe(Effect.scoped),
913 + );
914 +
915 it.effect("saves a server tab's download under its CDP guid where the server asked", () =>
916 Effect.gen(function* () {
917 const host = yield* DesktopBrowserHost.make;
apps/desktop/src/preview/DesktopBrowserHost.ts
+271 -28
@@ -22,7 +22,10 @@ const decodeCommand = Schema.decodeUnknownOption(Schema.fromJsonString(DesktopBr
22 const lineEncoder = new TextEncoder();
23 const AGENT_INPUT_COMMAND =
24 /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/;
25 +/** The same commands as above, by method, for the relay's sends. */
26 +const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/;
27 const AGENT_DOWNLOAD_WINDOW_MS = 5_000;
28 +const FOCUS_IDLE = "5 seconds";
29
30 export interface DesktopBrowserTabKey {
31 readonly threadId: string;
@@ -41,10 +44,33 @@ interface AttachedTab {
44 readonly debuggee: DesktopBrowserTabDebugger;
45 relay: CdpRelayConnection | null;
46 screenshotTurn: object | null;
44 - readonly waitingScreenshots: Array<() => void>;
47 + /** Each holds focus and painting while `held` and whoever queued it still waits on it. */
48 + readonly waitingScreenshots: Array<{
49 + readonly start: () => void;
50 + readonly live: () => boolean;
51 + held: boolean;
52 + }>;
53 downloadDirectory: string | null;
54 pendingDownloadGuid: string | null;
55 agentInputAt: number;
56 + focusEmulated: boolean;
57 + focusChange: Promise<void> | null;
58 + focusUses: number;
59 + /** Each pending command, by the extra session waiting on it; the main connection's have none. */
60 + readonly focusPending: Map<
61 + object,
62 + { readonly owner: string | undefined; readonly paints: boolean }
63 + >;
64 + /** Whether the client keeps the guest painting, as published on `painting`. */
65 + painting: boolean;
66 + /**
67 + * Driven by the agent since its thread's last turn ended or the guest last throttled. A
68 + * turn's start can reach us after its first command, so this holds focus once it does.
69 + */
70 + agentUsed: boolean;
71 + focusReleasing: boolean;
72 + /** The sessions streaming the guest; one stopping leaves the others' screencasts running. */
73 + readonly screencasts: Set<string | undefined>;
74 readonly onMessage: (
75 event: Electron.Event,
76 method: string,
@@ -69,6 +95,11 @@ export class DesktopBrowserHost extends Context.Service<
95 readonly x: number;
96 readonly y: number;
97 }>;
98 + /** Whether a guest needs its frames painted while the panel hides it. */
99 + readonly painting: Stream.Stream<{
100 + readonly key: DesktopBrowserTabKey;
101 + readonly active: boolean;
102 + }>;
103 }
104 >()("@t3tools/desktop/preview/DesktopBrowserHost") {}
105
@@ -80,44 +111,192 @@ export const make = Effect.gen(function* () {
111 readonly x: number;
112 readonly y: number;
113 }>(16);
114 + const painting = yield* PubSub.unbounded<{
115 + readonly key: DesktopBrowserTabKey;
116 + readonly active: boolean;
117 + }>();
118 const runFork = Effect.runForkWith(yield* Effect.context<never>());
119 const tabs = new Map<string, AttachedTab>();
120 + /** Threads whose agent turn runs, by the server's account. */
121 + const agentTurns = new Set<string>();
122 const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
123
124 const nextScreenshot = (tab: AttachedTab) => {
125 tab.screenshotTurn = null;
89 - tab.waitingScreenshots.shift()?.();
126 + tab.waitingScreenshots.shift()?.start();
127 + releaseFocusWhenIdle(tab);
128 + syncPainting(tab);
129 };
130
131 /**
132 * Chromium below 157 crashes the whole app when a guest's renderer exits with a
133 * screenshot copy in flight (crbug.com/567321049). Overlapping screenshots make that
134 * far likelier, and one the server gave up on stays pending in Chromium, so each guest
96 - * gets one at a time. One queued by a relay released since then is dropped unsent.
135 + * gets one at a time. One whose relay or session has since gone is dropped unsent.
136 */
137 const takeScreenshot = (
138 tab: AttachedTab,
139 relay: CdpRelayConnection,
101 - capture: () => Promise<unknown>,
140 + ownerLive: () => boolean,
141 + /** Sends only while `current` holds; holds focus and painting only while `held`. */
142 + capture: (current: () => boolean, held: boolean) => Promise<unknown>,
143 ) =>
144 new Promise<unknown>((resolve, reject) => {
104 - const start = () => {
105 - if (tab.relay !== relay) {
106 - reject(new Error("The preview connection that asked for this screenshot closed."));
107 - nextScreenshot(tab);
145 + const live = () => tab.relay === relay && ownerLive();
146 + const waiting = {
147 + start: () => {
148 + if (!live()) {
149 + reject(new Error("The preview connection that asked for this screenshot closed."));
150 + nextScreenshot(tab);
151 + return;
152 + }
153 + const turn = {};
154 + tab.screenshotTurn = turn;
155 + capture(() => tab.screenshotTurn === turn && live(), waiting.held)
156 + .finally(() => {
157 + if (tab.screenshotTurn === turn) nextScreenshot(tab);
158 + })
159 + .then(resolve, reject);
160 + },
161 + live,
162 + held: true,
163 + };
164 + if (tab.screenshotTurn === null) waiting.start();
165 + else {
166 + // The one ahead may be abandoned mid-capture; it needs frames to finish and free its turn.
167 + tab.waitingScreenshots.push(waiting);
168 + setFocusEmulation(tab, true);
169 + syncPainting(tab);
170 + }
171 + });
172 +
173 + /**
174 + * Chromium counts a focus-emulated guest as captured, so a guest the panel hides would
175 + * never throttle. Without it, a hidden guest paints no frames and focus() fires no focus
176 + * events, so the guest is emulated focused while the server drives it or streams it.
177 + */
178 + const setFocusEmulation = (tab: AttachedTab, enabled: boolean) => {
179 + if (tab.focusEmulated === enabled) return;
180 + tab.focusEmulated = enabled;
181 + // A guest that refuses it, or a debugger already gone, behaves as unfocused.
182 + const change: Promise<void> = tab.debuggee.debugger
183 + .sendCommand("Emulation.setFocusEmulationEnabled", { enabled })
184 + .then(
185 + () => undefined,
186 + () => undefined,
187 + )
188 + .finally(() => {
189 + if (tab.focusChange === change) tab.focusChange = null;
190 + });
191 + tab.focusChange = change;
192 + };
193 +
194 + const waitingHolds = (tab: AttachedTab) =>
195 + tab.waitingScreenshots.some((waiting) => waiting.held && waiting.live());
196 +
197 + const releaseFocusWhenIdle = (tab: AttachedTab) => {
198 + if (tab.focusReleasing) return;
199 + tab.focusReleasing = true;
200 + runFork(
201 + Effect.gen(function* () {
202 + let uses: number;
203 + do {
204 + uses = tab.focusUses;
205 + yield* Effect.sleep(FOCUS_IDLE);
206 + } while (uses !== tab.focusUses);
207 + tab.focusReleasing = false;
208 + // The last pending command to settle starts another idle wait.
209 + if (tab.focusPending.size > 0 || tab.screencasts.size > 0 || waitingHolds(tab)) {
210 return;
211 }
110 - const turn = {};
111 - tab.screenshotTurn = turn;
112 - capture()
113 - .finally(() => {
114 - if (tab.screenshotTurn === turn) nextScreenshot(tab);
115 - })
116 - .then(resolve, reject);
117 - };
118 - if (tab.screenshotTurn === null) start();
119 - else tab.waitingScreenshots.push(start);
212 + // The agent pauses to think between steps; its tab stays focused until the turn ends.
213 + if (tab.agentUsed && agentTurns.has(tab.key.threadId)) return;
214 + tab.agentUsed = false;
215 + if (tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false);
216 + }),
217 + );
218 + };
219 +
220 + /**
221 + * Electron composites no offscreen guest, so its screenshot or screencast waits for a frame
222 + * that never comes. The client paints the guest behind the app while one is pending; the
223 + * capture then finishes on its own, and the guest goes back offscreen once none is.
224 + */
225 + const syncPainting = (tab: AttachedTab) => {
226 + const active =
227 + tabs.get(keyOf(tab.key)) === tab &&
228 + (tab.screencasts.size > 0 ||
229 + waitingHolds(tab) ||
230 + [...tab.focusPending.values()].some((hold) => hold.paints));
231 + if (tab.painting === active) return;
232 + tab.painting = active;
233 + runFork(PubSub.publish(painting, { key: tab.key, active }));
234 + };
235 +
236 + /** A command may wait on the page's next frame, so the guest stays focused until it settles. */
237 + const holdFocus = <A>(
238 + tab: AttachedTab,
239 + method: string,
240 + owner: string | undefined,
241 + work: () => Promise<A>,
242 + ) => {
243 + const pending = {};
244 + tab.focusPending.set(pending, { owner, paints: method === "Page.captureScreenshot" });
245 + tab.focusUses += 1;
246 + setFocusEmulation(tab, true);
247 + releaseFocusWhenIdle(tab);
248 + syncPainting(tab);
249 + return work().finally(() => {
250 + if (!tab.focusPending.delete(pending)) return;
251 + tab.focusUses += 1;
252 + releaseFocusWhenIdle(tab);
253 + syncPainting(tab);
254 });
255 + };
256 +
257 + /**
258 + * Chromium can leave a command unanswered, so the screenshot guard's release points
259 + * also end every hold: a released relay has no one waiting, and a new document answers
260 + * nothing pending from the old one. A released relay's screencast has no viewer to stop it.
261 + */
262 + const dropFocusHolds = (tab: AttachedTab) => {
263 + tab.focusPending.clear();
264 + releaseFocusWhenIdle(tab);
265 + syncPainting(tab);
266 + };
267 +
268 + const dropRelay = (tab: AttachedTab) => {
269 + tab.relay = null;
270 + tab.screencasts.clear();
271 + tab.agentUsed = false;
272 + dropFocusHolds(tab);
273 + };
274 +
275 + /** A detached session, such as a closed viewer, waits on nothing it sent. */
276 + const dropSessionHolds = (tab: AttachedTab, owner: string) => {
277 + tab.screencasts.delete(owner);
278 + for (const [pending, hold] of tab.focusPending) {
279 + if (hold.owner === owner) tab.focusPending.delete(pending);
280 + }
281 + releaseFocusWhenIdle(tab);
282 + syncPainting(tab);
283 + };
284 +
285 + /**
286 + * A command the server stopped waiting on, such as a timed-out screenshot or evaluation,
287 + * may never settle; the turn's end ends every hold. The agent also sends on its own extra
288 + * session, so this cannot spare other sessions; a viewer's screencast still holds.
289 + */
290 + const endAgentTurn = (threadId: string) => {
291 + agentTurns.delete(threadId);
292 + for (const tab of tabs.values()) {
293 + if (tab.key.threadId !== threadId) continue;
294 + tab.agentUsed = false;
295 + // Still sent in turn, so the queue keeps moving, but no longer kept painting.
296 + for (const waiting of tab.waitingScreenshots) waiting.held = false;
297 + dropFocusHolds(tab);
298 + }
299 + };
300
301 const relayFor = (tab: AttachedTab) => {
302 if (tab.relay) return tab.relay;
@@ -126,12 +305,57 @@ export const make = Effect.gen(function* () {
305 sessionId === undefined
306 ? sendPageCommand(webContents, debuggee, method, params)
307 : debuggee.sendCommand(method, params, sessionId);
308 + const detachedOwners = new Set<string>();
309 + /** Viewers and recordings stream; the agent's own sessions never do. */
310 + const streamingOwners = new Set<string>();
311 const relay: CdpRelayConnection = createCdpRelayConnection(
312 {
131 - send: (method, params, sessionId) =>
132 - method === "Page.captureScreenshot"
133 - ? takeScreenshot(tab, relay, () => send(method, params, sessionId))
134 - : send(method, params, sessionId),
313 + send: (method, params, sessionId, owner) => {
314 + // A child session's screencast is not the page's, such as a late one on a detached session.
315 + if (sessionId === undefined && method === "Page.startScreencast") {
316 + tab.screencasts.add(owner);
317 + if (owner !== undefined) streamingOwners.add(owner);
318 + } else if (sessionId === undefined && method === "Page.stopScreencast") {
319 + tab.screencasts.delete(owner);
320 + }
321 + // A person's input through a viewer, or Playwright's own housekeeping, is not the agent's.
322 + if (
323 + AGENT_INPUT_METHOD.test(method) &&
324 + (owner === undefined || !streamingOwners.has(owner))
325 + ) {
326 + tab.agentUsed = true;
327 + }
328 + // Waits for focus emulation; insertText and other non-CDP input would not queue behind it.
329 + const forward = (current?: () => boolean) => {
330 + const change = tab.focusChange;
331 + const run = () =>
332 + current === undefined || current()
333 + ? send(method, params, sessionId)
334 + : Promise.reject(new Error("The page this screenshot was for has moved on."));
335 + return change === null ? run() : change.then(run);
336 + };
337 + return method === "Page.captureScreenshot"
338 + ? takeScreenshot(
339 + tab,
340 + relay,
341 + () => owner === undefined || !detachedOwners.has(owner),
342 + (current, held) =>
343 + held ? holdFocus(tab, method, owner, () => forward(current)) : forward(current),
344 + )
345 + : method === "Page.startScreencast"
346 + ? holdFocus(tab, method, owner, forward).catch((cause: unknown) => {
347 + // A viewer can outlive a refused stream; nothing is streaming for it.
348 + if (sessionId === undefined && tab.relay === relay) tab.screencasts.delete(owner);
349 + releaseFocusWhenIdle(tab);
350 + syncPainting(tab);
351 + throw cause;
352 + })
353 + : holdFocus(tab, method, owner, forward);
354 + },
355 + sessionDetached: (owner) => {
356 + detachedOwners.add(owner);
357 + dropSessionHolds(tab, owner);
358 + },
359 targetId: () =>
360 debuggee
361 .sendCommand("Target.getTargetInfo")
@@ -172,6 +396,7 @@ export const make = Effect.gen(function* () {
396 tab.relay = null;
397 tab.debuggee.debugger.off("message", tab.onMessage);
398 tab.debuggee.webContents.off("did-navigate", tab.onNewDocument);
399 + syncPainting(tab);
400 emit({ type: "detached", ...key });
401 };
402
@@ -188,6 +413,14 @@ export const make = Effect.gen(function* () {
413 downloadDirectory: null,
414 pendingDownloadGuid: null,
415 agentInputAt: Number.NEGATIVE_INFINITY,
416 + focusEmulated: false,
417 + focusChange: null,
418 + focusUses: 0,
419 + focusPending: new Map(),
420 + painting: false,
421 + agentUsed: false,
422 + focusReleasing: false,
423 + screencasts: new Set(),
424 onMessage: (_event, method, params, sessionId) => {
425 if (method === "Browser.downloadWillBegin") {
426 const guid = (params as { guid?: unknown } | undefined)?.guid;
@@ -198,6 +431,8 @@ export const make = Effect.gen(function* () {
431 // A screenshot pending when the page loads a new document, a reload included, can
432 // stay unanswered. A crashed page lands here once it reloads, well after its renderer died.
433 onNewDocument: () => {
434 + // Before the next screenshot starts, so its own hold survives.
435 + dropFocusHolds(tab);
436 if (tab.screenshotTurn !== null) nextScreenshot(tab);
437 },
438 };
@@ -211,6 +446,11 @@ export const make = Effect.gen(function* () {
446 Effect.sync(() => {
447 const command = decodeCommand(line);
448 if (Option.isNone(command)) return;
449 + if (command.value.type === "turn") {
450 + if (command.value.active) agentTurns.add(command.value.threadId);
451 + else endAgentTurn(command.value.threadId);
452 + return;
453 + }
454 const tab = tabs.get(keyOf(command.value));
455 if (!tab) return;
456 if (command.value.type === "pointer") {
@@ -219,7 +459,7 @@ export const make = Effect.gen(function* () {
459 return;
460 }
461 if (command.value.type === "release") {
222 - tab.relay = null;
462 + dropRelay(tab);
463 return;
464 }
465 if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now();
@@ -231,19 +471,22 @@ export const make = Effect.gen(function* () {
471 return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS;
472 };
473
234 - const announceAll = Effect.suspend(() =>
235 - Effect.forEach(
474 + const announceAll = Effect.suspend(() => {
475 + // A new backend: the old one's turns ended without telling us.
476 + agentTurns.clear();
477 + return Effect.forEach(
478 [...tabs.values()],
479 (tab) => {
238 - tab.relay = null;
480 + dropRelay(tab);
481 return PubSub.publish(outbox, { type: "attached", ...tab.key });
482 },
483 { discard: true },
242 - ),
243 - );
484 + );
485 + });
486
487 return DesktopBrowserHost.of({
488 pointers: Stream.fromPubSub(pointers),
489 + painting: Stream.fromPubSub(painting),
490 // Subscribes before announcing, so no attach falls between the two.
491 events: Stream.unwrap(
492 Effect.gen(function* () {
apps/desktop/src/preview/Manager.test.ts
+59
@@ -1713,6 +1713,65 @@ describe("PreviewManager", () => {
1713 }).pipe(Effect.provide(managerLayer()), Effect.scoped),
1714 );
1715
1716 + effectIt.effect(
1717 + "asks the client to paint a navigated server tab while the agent captures it",
1718 + () =>
1719 + Effect.gen(function* () {
1720 + const host = yield* DesktopBrowserHost.DesktopBrowserHost;
1721 + const serverTab = { threadId: "thread-1", tabId: "server-tab-1" };
1722 + const contents = Object.assign(
1723 + makeTestPreviewWebContents(async () => {
1724 + throw new Error("unused");
1725 + }, 42),
1726 + {
1727 + isDevToolsOpened: () => false,
1728 + getUserAgent: () => "Electron",
1729 + loadURL: vi.fn(async () => undefined),
1730 + },
1731 + );
1732 + const screenshot = Promise.withResolvers<unknown>();
1733 + contents.debugger.sendCommand = vi.fn((method: string) =>
1734 + method === "Page.captureScreenshot" ? screenshot.promise : Promise.resolve(undefined),
1735 + ) as unknown as typeof contents.debugger.sendCommand;
1736 + fromId.mockReturnValue(contents);
1737 + const manager = yield* PreviewManager.PreviewManager;
1738 + const capturing = yield* Queue.unbounded<boolean>();
1739 + yield* manager.subscribeStateChanges((_tabId, state) =>
1740 + Queue.offer(capturing, state.capturing).pipe(Effect.asVoid),
1741 + );
1742 + const attached = yield* Deferred.make<void>();
1743 + yield* host.events.pipe(
1744 + Stream.runForEach((line) =>
1745 + new TextDecoder().decode(line).includes('"type":"attached"')
1746 + ? Deferred.succeed(attached, undefined)
1747 + : Effect.void,
1748 + ),
1749 + Effect.forkScoped,
1750 + );
1751 + yield* manager.createTab("tab_1", { serverTab });
1752 + yield* manager.registerWebview("tab_1", 42);
1753 + yield* Deferred.await(attached);
1754 + // Navigating rebuilds the tab's state, which must keep the server tab it answers for.
1755 + yield* manager.navigate("tab_1", "https://example.com/next");
1756 + yield* Queue.clear(capturing);
1757 +
1758 + yield* host.handleCommandLine(
1759 + JSON.stringify({
1760 + type: "cdp",
1761 + ...serverTab,
1762 + message: JSON.stringify({
1763 + id: 1,
1764 + method: "Page.captureScreenshot",
1765 + sessionId: "t3-preview-page",
1766 + }),
1767 + }),
1768 + );
1769 + expect(yield* Queue.take(capturing)).toBe(true);
1770 + screenshot.resolve({ data: "" });
1771 + expect(yield* Queue.take(capturing)).toBe(false);
1772 + }).pipe(Effect.provide(managerLayer()), Effect.scoped),
1773 + );
1774 +
1775 effectIt.effect("discards a screenshot that resolves after its guest is replaced", () =>
1776 withManager((manager) =>
1777 Effect.gen(function* () {
apps/desktop/src/preview/Manager.ts
+33 -9
@@ -102,6 +102,7 @@ export interface PreviewTabState {
102 audioMuted: boolean;
103 audible: boolean;
104 controller: "human" | "agent" | "none";
105 + capturing: boolean;
106 favicon?: DesktopPreviewFavicon;
107 serverTab?: { readonly threadId: string; readonly tabId: string };
108 updatedAt: string;
@@ -1652,6 +1653,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
1653 audioMuted: false,
1654 audible: false,
1655 controller: "none",
1656 + capturing: false,
1657 updatedAt,
1658 };
1659 return [
@@ -1725,6 +1727,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
1727 audioMuted: false,
1728 audible: false,
1729 controller: "none",
1730 + capturing: false,
1731 updatedAt,
1732 };
1733 yield* emit(tabId, closed);
@@ -1946,7 +1949,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
1949 audioMuted: current?.audioMuted ?? false,
1950 audible: current?.audible ?? false,
1951 controller: current?.controller ?? "none",
1952 + capturing: current?.capturing ?? false,
1953 ...(current?.favicon ? { favicon: current.favicon } : {}),
1954 + ...(current?.serverTab ? { serverTab: current.serverTab } : {}),
1955 updatedAt,
1956 };
1957 return [
@@ -2299,7 +2304,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
2304 return;
2305 }
2306 if (afterAttach.serverTab) {
2302 - yield* listenForAgentPointers;
2307 + yield* listenToBrowserHost;
2308 browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger });
2309 }
2310 if (afterAttach.colorScheme !== "system") {
@@ -3239,14 +3244,33 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
3244 { discard: true },
3245 );
3246 });
3242 - let pointersStarted = false;
3243 - const listenForAgentPointers = Effect.suspend(() => {
3244 - if (pointersStarted) return Effect.void;
3245 - pointersStarted = true;
3246 - return browserHost.pointers.pipe(
3247 - Stream.runForEach(emitAgentPointer),
3248 - Effect.forkIn(parentScope),
3249 - Effect.asVoid,
3247 + const updateServerTabCapturing = Effect.fn("PreviewManager.updateServerTabCapturing")(
3248 + function* (painting: {
3249 + readonly key: { readonly threadId: string; readonly tabId: string };
3250 + readonly active: boolean;
3251 + }) {
3252 + const tab = [...(yield* SynchronizedRef.get(tabsRef)).values()].find(
3253 + (candidate) =>
3254 + candidate.serverTab?.threadId === painting.key.threadId &&
3255 + candidate.serverTab.tabId === painting.key.tabId,
3256 + );
3257 + if (tab && tab.capturing !== painting.active) {
3258 + yield* update(tab.tabId, { capturing: painting.active });
3259 + }
3260 + },
3261 + );
3262 + let browserHostStarted = false;
3263 + // Subscribes before the attach that follows, so no event falls between the two.
3264 + const listenToBrowserHost = Effect.suspend(() => {
3265 + if (browserHostStarted) return Effect.void;
3266 + browserHostStarted = true;
3267 + const listen = Effect.forkIn(parentScope, { startImmediately: true });
3268 + return Effect.all(
3269 + [
3270 + browserHost.pointers.pipe(Stream.runForEach(emitAgentPointer), listen),
3271 + browserHost.painting.pipe(Stream.runForEach(updateServerTabCapturing), listen),
3272 + ],
3273 + { discard: true },
3274 );
3275 });
3276
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+161 -3
@@ -46,6 +46,7 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"
46 import * as ProjectionStore from "./ProjectionStore.ts";
47 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
48 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
49 +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
50 import * as RunExecutionService from "./RunExecutionService.ts";
51 import * as RunFinalizationService from "./RunFinalizationService.ts";
52 import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
@@ -65,6 +66,7 @@ const layerRunExecutionTest = RunExecutionService.layer.pipe(
66 Layer.provide(
67 Layer.mergeAll(
68 McpAppModelContext.layerEmpty,
69 + PreviewAgentActivity.layer,
70 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
71 layerTurnItemPositionsTest,
72 Layer.mock(EventSink.EventSinkV2)({}),
@@ -878,6 +880,7 @@ it.effect("passes the thread's MCP app context to the provider under a safe key"
880 RunExecutionService.layer.pipe(
881 Layer.provide(
882 Layer.mergeAll(
883 + PreviewAgentActivity.layer,
884 Layer.mock(McpAppModelContext.McpAppModelContext)({
885 forThread: () =>
886 Effect.succeed([
@@ -919,6 +922,7 @@ it.effect("fails the run when its ownership check cannot be read before calling
922 Layer.provide(
923 Layer.mergeAll(
924 McpAppModelContext.layerEmpty,
925 + PreviewAgentActivity.layer,
926 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
927 layerTurnItemPositionsTest,
928 Layer.mock(EventSink.EventSinkV2)({
@@ -1120,6 +1124,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () =>
1124 Layer.provide(
1125 Layer.mergeAll(
1126 McpAppModelContext.layerEmpty,
1127 + PreviewAgentActivity.layer,
1128 Layer.mock(CheckpointService.CheckpointServiceV2)({
1129 captureBaseline: () =>
1130 Effect.fail(
@@ -1239,6 +1244,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co
1244 Layer.provide(
1245 Layer.mergeAll(
1246 McpAppModelContext.layerEmpty,
1247 + PreviewAgentActivity.layer,
1248 Layer.mock(CheckpointService.CheckpointServiceV2)({
1249 captureBaseline: () =>
1250 scenario === "start-guard" ? Effect.void : Effect.die("not reached"),
@@ -1410,6 +1416,7 @@ it.effect("ingests new member threads of an owned child after the root turn term
1416 Layer.provide(
1417 Layer.mergeAll(
1418 McpAppModelContext.layerEmpty,
1419 + PreviewAgentActivity.layer,
1420 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1421 layerTurnItemPositionsTest,
1422 Layer.mock(EventSink.EventSinkV2)({
@@ -1658,6 +1665,7 @@ it.effect("ends a nested native subagent and its thread when the root run is int
1665 Layer.provide(
1666 Layer.mergeAll(
1667 McpAppModelContext.layerEmpty,
1668 + PreviewAgentActivity.layer,
1669 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1670 layerTurnItemPositionsTest,
1671 Layer.mock(EventSink.EventSinkV2)({
@@ -1851,6 +1859,46 @@ it.effect.each([null, "run:nested-sdk"] as const)(
1859 }),
1860 );
1861
1862 +// The handover event is not this run's, yet it is what settles the run's agent turns.
1863 +it.effect(
1864 + "ends preview activity when a transferred child settles a run kept open by background work",
1865 + () =>
1866 + Effect.gen(function* () {
1867 + const active = yield* Deferred.make<boolean>();
1868 + yield* runBackgroundItemScenario(
1869 + "child-transfer-preview",
1870 + (ids) => {
1871 + const running = subagentEvent(ids, "running");
1872 + if (running.type !== "subagent.updated") throw new Error("Missing subagent");
1873 + const linked = {
1874 + ...running,
1875 + subagent: {
1876 + ...running.subagent,
1877 + origin: "provider_native" as const,
1878 + childThreadId: ids.childThreadId,
1879 + providerThreadId: null,
1880 + },
1881 + };
1882 + return [
1883 + childThreadCreatedEvent(ids),
1884 + linked,
1885 + backgroundTurnItemEvent(ids, "command_execution", "running", 1),
1886 + rootTerminalEvent(ids, "completed"),
1887 + { ...linked, subagent: { ...linked.subagent, runId: RunId.make("run:replacement") } },
1888 + ];
1889 + },
1890 + {
1891 + keepEventStreamOpen: true,
1892 + afterEvents: (activity) =>
1893 + activity
1894 + .active(backgroundScenarioIds("child-transfer-preview").threadId)
1895 + .pipe(Effect.flatMap((value) => Deferred.succeed(active, value))),
1896 + },
1897 + ).pipe(Effect.forkScoped);
1898 + assert.isFalse(yield* Deferred.await(active));
1899 + }).pipe(Effect.scoped),
1900 +);
1901 +
1902 it.effect("keeps ingesting a failed root's background item when its work continues", () =>
1903 Effect.gen(function* () {
1904 const observed = yield* runBackgroundItemScenario("bg-failed-continues", (ids) => [
@@ -1960,6 +2008,7 @@ it.effect(
2008 Layer.provide(
2009 Layer.mergeAll(
2010 McpAppModelContext.layerEmpty,
2011 + PreviewAgentActivity.layer,
2012 Layer.mock(CheckpointService.CheckpointServiceV2)({
2013 captureBaseline: () => Effect.void,
2014 }),
@@ -2769,6 +2818,95 @@ it.effect("does not interrupt a provider when ingestion fails before the root tu
2818 }),
2819 );
2820
2821 +// The server browser keeps an agent's tab focused for exactly this span.
2822 +it.effect.each(["completed", "failed"] as const)(
2823 + "reports the thread executing until its %s run's event consumer ends",
2824 + (outcome) =>
2825 + Effect.gen(function* () {
2826 + const gate = yield* Deferred.make<void>();
2827 + const { previewActivityEnded } = yield* captureRootRunTermination({
2828 + key: `preview-activity:${outcome}`,
2829 + shouldFinalizeRun: () => Effect.succeed(true),
2830 + events: (ids) =>
2831 + Stream.fromEffect(Deferred.await(gate)).pipe(
2832 + Stream.flatMap(() =>
2833 + outcome === "completed"
2834 + ? Stream.make(rootTerminalEvent(ids, "completed"))
2835 + : Stream.fail(
2836 + new ProviderAdapter.ProviderAdapterEventStreamError({
2837 + driver,
2838 + providerSessionId: ProviderSessionId.make("session:exited"),
2839 + cause: "provider process exited",
2840 + }),
2841 + ),
2842 + ),
2843 + ),
2844 + afterStart: () =>
2845 + Effect.gen(function* () {
2846 + const activity = yield* PreviewAgentActivity.PreviewAgentActivity;
2847 + assert.isTrue(
2848 + yield* activity.active(ThreadId.make(`thread:preview-activity:${outcome}`)),
2849 + );
2850 + yield* Deferred.succeed(gate, undefined);
2851 + }),
2852 + });
2853 + assert.isTrue(previewActivityEnded);
2854 + }),
2855 +);
2856 +
2857 +// Background commands keep the consumer alive for the whole session; only the agent's own
2858 +// turn and the children it drives should hold the browser tab focused.
2859 +it.effect.each([
2860 + { name: "ends once the root turn finishes beside background work", openSubagent: false },
2861 + { name: "stays while a run-owned subagent continues", openSubagent: true },
2862 +])("preview activity $name", ({ openSubagent }) =>
2863 + Effect.gen(function* () {
2864 + const key = `preview-activity:background:${openSubagent}`;
2865 + const consumerWaiting = yield* Deferred.make<void>();
2866 + const release = yield* Deferred.make<void>();
2867 + const providerInstanceId = ProviderInstanceId.make("codex");
2868 + yield* captureRootRunTermination({
2869 + key,
2870 + shouldFinalizeRun: () => Effect.succeed(true),
2871 + hasPendingBackgroundWorkForThread: () => Effect.succeed(true),
2872 + events: (ids) =>
2873 + Stream.concat(
2874 + Stream.fromIterable([
2875 + ...(openSubagent
2876 + ? [
2877 + {
2878 + type: "subagent.updated",
2879 + driver,
2880 + subagent: makeRunOwnedSubagentFixture({
2881 + ids,
2882 + providerInstanceId,
2883 + childThreadId: ids.childThreadId,
2884 + driver,
2885 + status: "running",
2886 + }),
2887 + } as const,
2888 + ]
2889 + : []),
2890 + rootTerminalEvent(ids, "completed"),
2891 + ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>),
2892 + Stream.fromEffect(
2893 + Deferred.succeed(consumerWaiting, undefined).pipe(
2894 + Effect.andThen(Deferred.await(release)),
2895 + ),
2896 + ).pipe(Stream.drain),
2897 + ),
2898 + afterStart: (closed) =>
2899 + Effect.gen(function* () {
2900 + yield* Deferred.await(consumerWaiting);
2901 + assert.isFalse(yield* Deferred.isDone(closed));
2902 + const activity = yield* PreviewAgentActivity.PreviewAgentActivity;
2903 + assert.equal(yield* activity.active(ThreadId.make(`thread:${key}`)), openSubagent);
2904 + yield* Deferred.succeed(release, undefined);
2905 + }),
2906 + });
2907 + }),
2908 +);
2909 +
2910 it.effect("fails the run when its event stream ends without a root terminal", () =>
2911 Effect.gen(function* () {
2912 const { observed, finalEvents } = yield* captureRootRunTermination({
@@ -2838,6 +2976,7 @@ function captureRootRunTermination(input: {
2976 readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
2977 readonly rejectTerminalWrite?: boolean;
2978 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
2979 + readonly hasPendingBackgroundWorkForThread?: ProviderAdapter.ProviderAdapterV2SessionRuntime["hasPendingBackgroundWorkForThread"];
2980 readonly seedOpenSubagent?: boolean;
2981 readonly events?: (
2982 ids: BackgroundScenarioIds,
@@ -2859,7 +2998,9 @@ function captureRootRunTermination(input: {
2998 readonly beforeTerminalWrite?: (
2999 request: Parameters<EventSink.EventSinkV2["Service"]["writeIfRunCurrent"]>[0],
3000 ) => Effect.Effect<boolean, EventSink.EventSinkV2Error>;
2862 - readonly afterStart?: (closed: Deferred.Deferred<void>) => Effect.Effect<void>;
3001 + readonly afterStart?: (
3002 + closed: Deferred.Deferred<void>,
3003 + ) => Effect.Effect<void, never, PreviewAgentActivity.PreviewAgentActivity>;
3004 }) {
3005 return Effect.gen(function* () {
3006 const ids = backgroundScenarioIds(input.key);
@@ -2946,9 +3087,10 @@ function captureRootRunTermination(input: {
3087 }),
3088 ),
3089 ),
3090 + Layer.provideMerge(PreviewAgentActivity.layer),
3091 );
3092
2951 - yield* Effect.gen(function* () {
3093 + const previewActivityEnded = yield* Effect.gen(function* () {
3094 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
3095 const start = runExecution.startRootRun({
3096 commandId: CommandId.make(`command:${input.key}`),
@@ -2989,6 +3131,7 @@ function captureRootRunTermination(input: {
3131 }),
3132 startTurn: input.startTurn ?? (() => Effect.void),
3133 interruptTurn: input.interruptTurn ?? (() => Effect.void),
3134 + hasPendingBackgroundWorkForThread: input.hasPendingBackgroundWorkForThread,
3135 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
3136 run: {
3137 id: ids.runId,
@@ -3050,9 +3193,12 @@ function captureRootRunTermination(input: {
3193 );
3194 yield* input.afterStart?.(ingestionDone) ?? Effect.void;
3195 yield* Deferred.await(ingestionDone);
3196 + const activity = yield* PreviewAgentActivity.PreviewAgentActivity;
3197 + return !(yield* activity.active(ids.threadId));
3198 }).pipe(Effect.provide(layerTest));
3199
3200 return {
3201 + previewActivityEnded,
3202 written: yield* Ref.get(writtenItems),
3203 finalEvents: yield* Ref.get(finalEvents),
3204 observed: yield* Ref.get(observed),
@@ -3387,6 +3533,9 @@ function runBackgroundItemScenario(
3533 >;
3534 readonly onSubscribe?: Effect.Effect<void>;
3535 readonly onEventsConsumed?: Effect.Effect<void>;
3536 + readonly afterEvents?: (
3537 + activity: PreviewAgentActivity.PreviewAgentActivity["Service"],
3538 + ) => Effect.Effect<void>;
3539 readonly onFinalized?: (
3540 events: ReadonlyArray<OrchestrationV2DomainEvent>,
3541 ) => Effect.Effect<void>;
@@ -3397,10 +3546,14 @@ function runBackgroundItemScenario(
3546 const providerInstanceId = ProviderInstanceId.make("codex");
3547 const observed = yield* Ref.make<ReadonlyArray<string>>([]);
3548 const ingestionDone = yield* Deferred.make<void>();
3549 + const previewActivity = yield* PreviewAgentActivity.PreviewAgentActivity.pipe(
3550 + Effect.provide(PreviewAgentActivity.layer),
3551 + );
3552 const layerTest = RunExecutionService.layer.pipe(
3553 Layer.provide(
3554 Layer.mergeAll(
3555 McpAppModelContext.layerEmpty,
3556 + Layer.succeed(PreviewAgentActivity.PreviewAgentActivity, previewActivity),
3557 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
3558 layerTurnItemPositionsTest,
3559 Layer.mock(EventSink.EventSinkV2)({
@@ -3456,7 +3609,12 @@ function runBackgroundItemScenario(
3609 yield* options?.onSubscribe ?? Effect.void;
3610 const events = Stream.fromIterable(makeEvents(ids)).pipe(
3611 Stream.concat(
3459 - Stream.fromEffect(options?.onEventsConsumed ?? Effect.void).pipe(Stream.drain),
3612 + Stream.fromEffect(
3613 + Effect.andThen(
3614 + options?.onEventsConsumed ?? Effect.void,
3615 + options?.afterEvents?.(previewActivity) ?? Effect.void,
3616 + ),
3617 + ).pipe(Stream.drain),
3618 ),
3619 );
3620 return {
apps/server/src/orchestration-v2/RunExecutionService.ts
+26 -2
@@ -39,6 +39,7 @@ import * as Stream from "effect/Stream";
39
40 import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
41 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
42 +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
43 import * as ServerSettings from "../serverSettings.ts";
44 import * as CheckpointService from "./CheckpointService.ts";
45 import * as EventSink from "./EventSink.ts";
@@ -595,6 +596,7 @@ export const layer: Layer.Layer<
596 | ProviderEventIngestor.ProviderEventIngestorV2
597 | ServerSettings.ServerSettingsService
598 | McpAppModelContext.McpAppModelContext
599 + | PreviewAgentActivity.PreviewAgentActivity
600 | TurnItemPositionStore.TurnItemPositionStoreV2
601 > = Layer.effect(
602 RunExecutionServiceV2,
@@ -606,6 +608,7 @@ export const layer: Layer.Layer<
608 const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
609 const serverSettings = yield* ServerSettings.ServerSettingsService;
610 const mcpAppModelContext = yield* McpAppModelContext.McpAppModelContext;
611 + const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity;
612 const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2;
613 const finalizationObserver = yield* RunFinalizationService.RunFinalizationObserver;
614
@@ -1220,7 +1223,7 @@ export const layer: Layer.Layer<
1223 }
1224 }
1225 });
1223 - const shouldStopProviderEventIngestion = Effect.gen(function* () {
1226 + const agentTurnsSettled = Effect.gen(function* () {
1227 if (!(yield* Ref.get(rootTerminalSeen))) {
1228 return false;
1229 }
@@ -1237,9 +1240,20 @@ export const layer: Layer.Layer<
1240 return false;
1241 }
1242 const childSubagents = yield* Ref.get(activeChildSubagents);
1240 - if (childSubagents.size > 0) {
1243 + return childSubagents.size === 0;
1244 + });
1245 + const shouldStopProviderEventIngestion = Effect.gen(function* () {
1246 + if (!(yield* agentTurnsSettled)) {
1247 return false;
1248 }
1249 + const terminal = yield* Ref.get(terminalEvent);
1250 + if (
1251 + terminal !== null &&
1252 + terminal.status !== "completed" &&
1253 + !childTurnsContinue(terminal)
1254 + ) {
1255 + return true;
1256 + }
1257 // Newly owned items depend on adapters emitting a non-terminal event before the root
1258 // terminal. Owner loss (a newer run claimed lastRunOrdinal) must not close this stream
1259 // while these sets are non-empty: turn_item.updated writes are not ownership-gated, so
@@ -1269,6 +1283,9 @@ export const layer: Layer.Layer<
1283 // Held text writes nothing, so its item would take a position only
1284 // when finally written, behind every item that began after it.
1285 const reservedHeldTurnItemIds = new Set<TurnItemId>();
1286 + // Ends once the root turn and the children it drives settle. Background commands such
1287 + // as a dev server keep this consumer alive afterwards and must not pin preview focus.
1288 + const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId);
1289 const providerEventFiber = yield* eventSubscription.events.pipe(
1290 Stream.mapEffect((event) =>
1291 Effect.gen(function* () {
@@ -1444,6 +1461,12 @@ export const layer: Layer.Layer<
1461 yield* trackChildLifecycle(event, deliveredEvent !== null);
1462 }),
1463 ),
1464 + // Also after events this run does not accept, such as a child handed to another run.
1465 + Stream.tap(() =>
1466 + Effect.gen(function* () {
1467 + if (yield* agentTurnsSettled) yield* endPreviewActivity;
1468 + }),
1469 + ),
1470 Stream.takeUntilEffect(() => shouldStopProviderEventIngestion),
1471 Stream.runDrain,
1472 Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })),
@@ -1574,6 +1597,7 @@ export const layer: Layer.Layer<
1597 );
1598 }),
1599 ),
1600 + Effect.ensuring(endPreviewActivity),
1601 Effect.ensuring(eventSubscription.close),
1602 Effect.forkDetach,
1603 );
apps/server/src/orchestration-v2/runtimeLayer.ts
+2
@@ -6,6 +6,7 @@ import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
6 import * as OrchestrationCommandReceipts from "../persistence/OrchestrationCommandReceipts.ts";
7 import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts";
8 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
9 +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
10 import * as McpAppRequests from "../mcpApps/McpAppRequests.ts";
11 import * as ProviderSessionRuntime from "../persistence/ProviderSessionRuntime.ts";
12 import * as TextGeneration from "../textGeneration/TextGeneration.ts";
@@ -153,6 +154,7 @@ const layerRunExecutionServiceProvided = RunExecutionService.layer.pipe(
154 Layer.provide(
155 Layer.mergeAll(
156 McpAppModelContext.layer,
157 + PreviewAgentActivity.layer,
158 layerCheckpointServiceProvided,
159 layerEventSinkProvided,
160 IdAllocator.layer,
apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
+2
@@ -48,6 +48,7 @@ import * as ProviderTurnControlService from "../ProviderTurnControlService.ts";
48 import * as ProviderTurnStartService from "../ProviderTurnStartService.ts";
49 import * as ProviderTurnStartServiceTestkit from "../ProviderTurnStartService.testkit.ts";
50 import * as McpAppModelContext from "../../mcpApps/McpAppModelContext.ts";
51 +import * as PreviewAgentActivity from "../../preview/PreviewAgentActivity.ts";
52 import * as RunExecutionService from "../RunExecutionService.ts";
53 import * as RunFinalizationService from "../RunFinalizationService.ts";
54 import * as ThreadTitleRegenerationService from "../ThreadTitleRegenerationService.ts";
@@ -378,6 +379,7 @@ export function layerWithRegistry<Error>(
379 Layer.provide(
380 Layer.mergeAll(
381 McpAppModelContext.layerEmpty,
382 + PreviewAgentActivity.layer,
383 layerCheckpointServiceProvided,
384 layerEventSinkProvided,
385 IdAllocator.layer,
apps/server/src/preview/DesktopBrowserChannel.ts
+4
@@ -49,6 +49,8 @@ export class DesktopBrowserChannel extends Context.Service<
49 key: DesktopTabKey,
50 pointer: { readonly phase: "move" | "click"; readonly x: number; readonly y: number },
51 ) => Effect.Effect<void>;
52 + /** Never fails. A failed write means a broken channel; the next backend starts with no turns. */
53 + readonly turn: (threadId: string, active: boolean) => Effect.Effect<void>;
54 }
55 >()("t3/preview/DesktopBrowserChannel") {}
56
@@ -70,6 +72,7 @@ const make = Effect.gen(function* () {
72 isAttached: () => Effect.succeed(false),
73 endpoint: () => Effect.die("No desktop app is attached to this server."),
74 pointer: () => Effect.void,
75 + turn: () => Effect.void,
76 });
77 }
78
@@ -207,6 +210,7 @@ const make = Effect.gen(function* () {
210 isAttached: (key) => Effect.sync(() => attachedTabs.has(keyOf(key))),
211 endpoint,
212 pointer: (key, pointer) => command({ type: "pointer", ...key, ...pointer }),
213 + turn: (threadId, active) => command({ type: "turn", threadId, active }),
214 });
215 });
216
apps/server/src/preview/PreviewAgentActivity.ts new
+58
@@ -0,0 +1,58 @@
1 +import type { ThreadId } from "@t3tools/contracts";
2 +import * as Context from "effect/Context";
3 +import * as Effect from "effect/Effect";
4 +import * as Layer from "effect/Layer";
5 +import * as PubSub from "effect/PubSub";
6 +import type * as Scope from "effect/Scope";
7 +
8 +export interface PreviewAgentTurnChange {
9 + readonly threadId: ThreadId;
10 + readonly active: boolean;
11 +}
12 +
13 +/**
14 + * Whether an agent turn runs on a thread, model thinking included, so its preview tab
15 + * stays focused between tool calls. Background commands outlive the turn and do not count.
16 + */
17 +export class PreviewAgentActivity extends Context.Service<
18 + PreviewAgentActivity,
19 + {
20 + /** Marks the thread active until the returned effect runs; it runs at most once. */
21 + readonly begin: (threadId: ThreadId) => Effect.Effect<Effect.Effect<void>>;
22 + readonly active: (threadId: ThreadId) => Effect.Effect<boolean>;
23 + /** Each thread's transitions between no turn and some turn, from subscription on. */
24 + readonly changes: Effect.Effect<
25 + PubSub.Subscription<PreviewAgentTurnChange>,
26 + never,
27 + Scope.Scope
28 + >;
29 + }
30 +>()("t3/preview/PreviewAgentActivity") {}
31 +
32 +const make = Effect.gen(function* () {
33 + const changes = yield* PubSub.unbounded<PreviewAgentTurnChange>();
34 + // A token per turn, so overlapping turns on one thread end only when the last does.
35 + const turns = new Map<ThreadId, Set<symbol>>();
36 +
37 + const begin = Effect.fn("PreviewAgentActivity.begin")(function* (threadId: ThreadId) {
38 + const existing = turns.get(threadId);
39 + const tokens = existing ?? new Set<symbol>();
40 + if (existing === undefined) turns.set(threadId, tokens);
41 + const token = Symbol(threadId);
42 + tokens.add(token);
43 + if (existing === undefined) yield* PubSub.publish(changes, { threadId, active: true });
44 + return Effect.suspend(() => {
45 + if (!tokens.delete(token) || tokens.size > 0) return Effect.void;
46 + if (turns.get(threadId) === tokens) turns.delete(threadId);
47 + return PubSub.publish(changes, { threadId, active: false }).pipe(Effect.asVoid);
48 + });
49 + });
50 +
51 + return PreviewAgentActivity.of({
52 + begin,
53 + active: (threadId) => Effect.sync(() => turns.has(threadId)),
54 + changes: PubSub.subscribe(changes),
55 + });
56 +});
57 +
58 +export const layer = Layer.effect(PreviewAgentActivity, make);
apps/server/src/preview/ServerBrowser.test.ts
+26
@@ -32,6 +32,7 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
32 import * as Broker from "../mcp/PreviewAutomationBroker.ts";
33 import * as DesktopChannel from "./DesktopBrowserChannel.ts";
34 import * as Manager from "./Manager.ts";
35 +import * as PreviewAgentActivity from "./PreviewAgentActivity.ts";
36 import * as ServerBrowser from "./ServerBrowser.ts";
37 import * as PreviewBrowser from "./PreviewBrowser.ts";
38
@@ -275,6 +276,7 @@ const desktopRenders = (tabId: string) => {
276 return desktopTabs.has(tabId);
277 };
278 const releasedDesktopTabs: Array<string> = [];
279 +const desktopTurns = new NodeEvents.EventEmitter();
280 const desktopConnections: Array<{ endpoint: string; context: ReturnType<typeof makeContext> }> = [];
281 const testThread = {
282 threadId: ThreadId.make("browser-test-thread"),
@@ -335,7 +337,9 @@ const dependencies = Layer.mergeAll(
337 Effect.sync(() => releasedDesktopTabs.push(key.tabId)),
338 ),
339 pointer: () => Effect.void,
340 + turn: (threadId, active) => Effect.sync(() => desktopTurns.emit("turn", threadId, active)),
341 }),
342 + PreviewAgentActivity.layer,
343 ).pipe(
344 Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-server-browser-" })),
345 Layer.provideMerge(NodeServices.layer),
@@ -378,6 +382,28 @@ beforeEach(() => {
382 desktopConnections.length = 0;
383 });
384
385 +it.live("tells the desktop once per thread when its agent turns start and end", () =>
386 + Effect.gen(function* () {
387 + yield* ServerBrowser.ServerBrowser;
388 + const activity = yield* PreviewAgentActivity.PreviewAgentActivity;
389 + const threadId = scope.thread.threadId;
390 + const turns = yield* Queue.unbounded<readonly [string, boolean]>();
391 + const record = (id: string, active: boolean) => Queue.offerUnsafe(turns, [id, active]);
392 + desktopTurns.on("turn", record);
393 + yield* Effect.addFinalizer(() => Effect.sync(() => desktopTurns.off("turn", record)));
394 + const endFirst = yield* activity.begin(threadId);
395 + // An overlapping turn on the same thread neither starts nor ends focus on its own.
396 + const endSecond = yield* activity.begin(threadId);
397 + expect(yield* Queue.take(turns)).toEqual([threadId, true]);
398 + yield* endFirst;
399 + yield* endFirst;
400 + expect(yield* activity.active(threadId)).toBe(true);
401 + yield* endSecond;
402 + expect(yield* Queue.take(turns)).toEqual([threadId, false]);
403 + expect(yield* Queue.size(turns)).toBe(0);
404 + }).pipe(Effect.scoped, Effect.provide(layer)),
405 +);
406 +
407 it.live("preview_open reports its navigation timeout instead of the request budget", () =>
408 Effect.scoped(
409 Effect.gen(function* () {
apps/server/src/preview/ServerBrowser.ts
+7
@@ -77,6 +77,7 @@ import { resolveRootCliCommand } from "../cli/invocation.ts";
77 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
78 import * as PreviewAutomationBroker from "../mcp/PreviewAutomationBroker.ts";
79 import * as DesktopBrowserChannel from "./DesktopBrowserChannel.ts";
80 +import * as PreviewAgentActivity from "./PreviewAgentActivity.ts";
81 import * as PreviewManager from "./Manager.ts";
82 import * as ServerBrowserPage from "./ServerBrowserPage.ts";
83 import * as PreviewBrowser from "./PreviewBrowser.ts";
@@ -454,6 +455,7 @@ const make = Effect.gen(function* () {
455 const environment = yield* ServerEnvironment.ServerEnvironment;
456 const previewBrowser = yield* PreviewBrowser.PreviewBrowser;
457 const desktopChannel = yield* DesktopBrowserChannel.DesktopBrowserChannel;
458 + const agentTurns = yield* (yield* PreviewAgentActivity.PreviewAgentActivity).changes;
459 const runFork = Effect.runForkWith(yield* Effect.context<never>());
460 const launchServices = yield* Effect.context<ChildProcessSpawner.ChildProcessSpawner>();
461 const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND);
@@ -2458,6 +2460,11 @@ const make = Effect.gen(function* () {
2460 ),
2461 Effect.forkScoped,
2462 );
2463 + // The desktop keeps the tabs an agent uses focused until its turn ends, not just per command.
2464 + yield* Stream.fromSubscription(agentTurns).pipe(
2465 + Stream.runForEach((change) => desktopChannel.turn(change.threadId, change.active)),
2466 + Effect.forkScoped,
2467 + );
2468 // Reconnect returned pages immediately so all clients keep receiving URL and title changes.
2469 yield* desktopChannel.attached.pipe(
2470 Stream.runForEach((key) =>
apps/server/src/server.ts
+7 -1
@@ -74,6 +74,7 @@ import * as DeviceHubProxy from "./device/DeviceHubProxy.ts";
74 import * as PreviewManager from "./preview/Manager.ts";
75 import * as PortScanner from "./preview/PortScanner.ts";
76 import * as ServerBrowser from "./preview/ServerBrowser.ts";
77 +import * as PreviewAgentActivity from "./preview/PreviewAgentActivity.ts";
78 import * as DesktopBrowserChannel from "./preview/DesktopBrowserChannel.ts";
79 import * as ServerBrowserStream from "./preview/ServerBrowserStream.ts";
80 import * as PreviewBrowser from "./preview/PreviewBrowser.ts";
@@ -732,7 +733,12 @@ const layerMakeRoutes = Layer.mergeAll(
733 ),
734 // Share the service so HTTP/WS clients single-flight reads and invalidate each other's cached patches.
735 Layer.provide(layerPullRequestService),
735 - Layer.provide(ServerBrowser.layer.pipe(Layer.provide(DesktopBrowserChannel.layer))),
736 + // The same activity instance RunExecutionService reports to, by layer memoization.
737 + Layer.provide(
738 + ServerBrowser.layer.pipe(
739 + Layer.provide(Layer.merge(DesktopBrowserChannel.layer, PreviewAgentActivity.layer)),
740 + ),
741 + ),
742 Layer.provide(PreviewBrowser.layer),
743 Layer.provide(PreviewAutomationBroker.layer),
744 Layer.provide(ServerSelfUpdate.layer.pipe(Layer.provide(layerDesktopAppUpdate))),
apps/web/src/components/preview/usePreviewBridge.test.ts
+24 -1
@@ -1,7 +1,9 @@
1 import type { DesktopPreviewTabState } from "@t3tools/contracts";
2 import { describe, expect, it } from "vite-plus/test";
3
4 -import { projectDesktopState } from "./usePreviewBridge";
4 +import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore";
5 +
6 +import { createCaptureActivity, projectDesktopState } from "./usePreviewBridge";
7
8 const favicon = {
9 dataUrl: "data:image/png;base64,AAAA",
@@ -22,6 +24,7 @@ function state(navStatus: DesktopPreviewTabState["navStatus"]): DesktopPreviewTa
24 audioMuted: false,
25 audible: false,
26 controller: "none",
27 + capturing: false,
28 favicon,
29 updatedAt: "2026-08-09T00:00:00.000Z",
30 };
@@ -48,3 +51,23 @@ describe("projectDesktopState", () => {
51 expect(projectDesktopState(state({ kind: "Idle" })).favicon).toBeNull();
52 });
53 });
54 +
55 +describe("createCaptureActivity", () => {
56 + it("keeps the guest painting while the host captures it, alongside other activity", () => {
57 + const activity = () => useBrowserSurfaceStore.getState().activityByTabId["tab-1"];
58 + const recording = useBrowserSurfaceStore.getState().acquireActivity("tab-1");
59 + const capture = createCaptureActivity("tab-1");
60 +
61 + capture.update(true);
62 + capture.update(true);
63 + expect(activity()).toBe(2);
64 + capture.update(false);
65 + expect(activity()).toBe(1);
66 + capture.update(true);
67 + capture.release();
68 + capture.release();
69 + expect(activity()).toBe(1);
70 + recording();
71 + expect(activity()).toBeUndefined();
72 + });
73 +});
apps/web/src/components/preview/usePreviewBridge.ts
+30 -3
@@ -8,6 +8,7 @@ import { useEffect, useEffectEvent, useRef } from "react";
8
9 import { recordFaviconForThread } from "~/browserFaviconStore";
10 import { useBrowserPointerStore } from "~/browser/browserPointerStore";
11 +import { acquireBrowserSurfaceActivity } from "~/browser/browserSurfaceStore";
12 import { applyPreviewDesktopState, type DesktopPreviewOverlay } from "~/previewStateStore";
13 import { previewEnvironment } from "~/state/preview";
14 import { useAtomCommand } from "~/state/use-atom-command";
@@ -22,6 +23,22 @@ function originOf(url: string): string | null {
23 }
24 }
25
26 +/**
27 + * Keeps a hidden guest painting behind the app while the desktop host captures it; Electron
28 + * composites no offscreen guest, so its screenshot or stream would otherwise never finish.
29 + */
30 +export function createCaptureActivity(tabId: string) {
31 + let release: (() => void) | null = null;
32 + const update = (capturing: boolean) => {
33 + if (capturing && !release) release = acquireBrowserSurfaceActivity(tabId);
34 + else if (!capturing && release) {
35 + release();
36 + release = null;
37 + }
38 + };
39 + return { update, release: () => update(false) };
40 +}
41 +
42 /** ElectronBrowserHost owns webview lifetime. */
43 export function usePreviewBridge(input: {
44 threadRef: ScopedThreadRef;
@@ -38,6 +55,7 @@ export function usePreviewBridge(input: {
55 const lastReportedUrl = useRef<string | null>(null);
56 const lastReportedKind = useRef<DesktopPreviewTabState["navStatus"]["kind"] | null>(null);
57 const lastDesktopNavStatus = useRef<DesktopPreviewTabState["navStatus"] | null>(null);
58 + const captureActivity = useRef<ReturnType<typeof createCaptureActivity> | null>(null);
59 const handleStateChange = useEffectEvent(
60 (changedTabId: string, state: DesktopPreviewTabState): void => {
61 if (changedTabId !== runtimeTabId) return;
@@ -45,6 +63,7 @@ export function usePreviewBridge(input: {
63 clearBrowserPointer(runtimeTabId);
64 }
65 lastDesktopNavStatus.current = state.navStatus;
66 + captureActivity.current?.update(state.capturing);
67 applyPreviewDesktopState(stableThreadRef, tabId, projectDesktopState(state));
68 if (state.favicon) {
69 recordFaviconForThread(stableThreadRef, state.favicon);
@@ -70,11 +89,19 @@ export function usePreviewBridge(input: {
89 lastReportedUrl.current = null;
90 lastReportedKind.current = null;
91 lastDesktopNavStatus.current = null;
73 - return previewBridge.onStateChange(handleStateChange);
92 + const activity = createCaptureActivity(runtimeTabId);
93 + captureActivity.current = activity;
94 + const unsubscribe = previewBridge.onStateChange(handleStateChange);
95 + return () => {
96 + unsubscribe();
97 + activity.release();
98 + if (captureActivity.current === activity) captureActivity.current = null;
99 + };
100 }, [
75 - // oxlint-disable-next-line react/exhaustive-effect-dependencies -- A different tab or thread starts its status reports and pointer tracking from scratch.
101 runtimeTabId,
77 - stableThreadRef,
102 + // oxlint-disable-next-line react/exhaustive-effect-dependencies -- A different tab or thread starts its status reports and pointer tracking from scratch.
103 + environmentId,
104 + threadId,
105 tabId,
106 ]);
107 }
packages/contracts/src/desktopBrowser.ts
+6
@@ -25,5 +25,11 @@ export const DesktopBrowserCommand = Schema.Union([
25 x: Schema.Finite,
26 y: Schema.Finite,
27 }),
28 + /** Whether an agent turn runs on the thread, so the tabs it uses stay focused between steps. */
29 + Schema.Struct({
30 + type: Schema.Literal("turn"),
31 + threadId: TrimmedNonEmptyString,
32 + active: Schema.Boolean,
33 + }),
34 ]);
35 export type DesktopBrowserCommand = typeof DesktopBrowserCommand.Type;
packages/contracts/src/ipc.ts
+2
@@ -590,6 +590,8 @@ export interface DesktopPreviewTabState {
590 /** Chromium reports playback even when audioMuted is true. */
591 audible: boolean;
592 controller: "human" | "agent" | "none";
593 + /** A screenshot or stream of the guest is pending, so the client keeps it painting while hidden. */
594 + capturing: boolean;
595 favicon?: DesktopPreviewFavicon;
596 updatedAt: string;
597 }