@devries-cameron20 / t3code Mirror · read-only / commits / 2763bbeb73
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 15:45 UTC

fix(server): preview snapshots preserve blurred backdrops (#24618)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 25, 2026 at 23:50 UTC 2763bbeb739e64c181d7800ddb9c1459f678dea4
4 files changed +154 -48
apps/server/src/preview/ServerBrowser.test.ts
+2
@@ -96,6 +96,8 @@ function makeSession(frame?: ReturnType<typeof makeFrame>) {
96 // A snapshot sizes its screenshot from the PNG header; viewers get JPEG frames as-is.
97 return { data: (input as { format: string }).format === "png" ? PNG_HEADER : "ZnJhbWU=" };
98 }
99 + // Snapshots shrink their screenshot in the page; the header stands in for the result.
100 + if (method === "Runtime.callFunctionOn") return { result: { value: PNG_HEADER } };
101 if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } };
102 if (method === "Page.createIsolatedWorld") {
103 return {
apps/server/src/preview/ServerBrowser.ts
+2 -9
@@ -1467,10 +1467,9 @@ const make = Effect.gen(function* () {
1467 try {
1468 await encoder.evaluate(ServerBrowserPage.recordingEncoderScript(recordingFrameRate));
1469 // Seed idle pages before returning so an immediate stop has a frame to encode.
1470 - const firstFrame = await ServerBrowserPage.captureViewport(tab.page, tab.cdp, {
1470 + const firstFrame = await ServerBrowserPage.captureViewport(tab.cdp, {
1471 format: "jpeg",
1472 quality: RECORDING_SCREENCAST.quality,
1473 - scale: 1,
1473 });
1474 await encoder.evaluate(
1475 ([data, width]) =>
@@ -1522,7 +1521,6 @@ const make = Effect.gen(function* () {
1521 return started;
1522 };
1523
1525 - // Scaled captures repaint every screencast; pause them to avoid leaking that frame.
1524 const withScreencastsPaused = <A>(tab: ServerTab, capture: () => Promise<A>): Promise<A> =>
1525 withCaptureLock(tab, async () => {
1526 tab.capturing += 1;
@@ -2333,14 +2331,9 @@ const make = Effect.gen(function* () {
2331 }
2332 broadcastControl(tab);
2333 pushFileChooser(tab);
2336 - // Full scale: a scaled capture would flash in every other viewer.
2334 const pushStill = async () => {
2335 const data = await withCaptureLock(tab, () =>
2339 - ServerBrowserPage.captureViewport(tab.page, session, {
2340 - format: "jpeg",
2341 - quality,
2342 - scale: 1,
2343 - }),
2336 + ServerBrowserPage.captureViewport(session, { format: "jpeg", quality }),
2337 ).catch(() => null);
2338 if (data) viewer.push({ _tag: "frame", data: Buffer.from(data, "base64"), ack: Effect.void });
2339 };
apps/server/src/preview/ServerBrowserPage.test.ts
+103
@@ -245,6 +245,109 @@ describe("server browser element refs", () => {
245 });
246 });
247
248 +describe("server browser snapshot screenshot", () => {
249 + let browser: Browser;
250 + let context: BrowserContext;
251 +
252 + beforeAll(async () => {
253 + // ServerBrowserContexts launches with these, and the capture scale depends on them.
254 + browser = await chromium.launch({
255 + headless: true,
256 + args: ["--disable-gpu", "--force-device-scale-factor=2"],
257 + });
258 + });
259 + afterAll(async () => {
260 + await browser?.close();
261 + });
262 + beforeEach(async () => {
263 + context = await browser.newContext({
264 + viewport: { width: 1280, height: 800 },
265 + deviceScaleFactor: 2,
266 + });
267 + });
268 + afterEach(async () => {
269 + await context.close();
270 + });
271 +
272 + // Blue on the left half and red on the right, so a mirrored left half shows up on the right.
273 + const halves = (overlay: string) => `
274 + <style>
275 + body { margin: 0; background: white; }
276 + .half { position: absolute; top: 0; width: 640px; height: 800px; }
277 + .overlay { position: fixed; inset: 0; ${overlay} }
278 + </style>
279 + <div class="half" style="left: 0; background: #2563eb"></div>
280 + <div class="half" style="left: 640px; background: #ef4444"></div>
281 + <div class="overlay"></div>`;
282 +
283 + const screenshotOf = async (html: string) => {
284 + const page = await context.newPage();
285 + await page.setContent(html);
286 + const { screenshot } = await ServerBrowserPage.snapshot({
287 + page,
288 + cdp: await context.newCDPSession(page),
289 + renderScale: 2,
290 + consoleEntries: [],
291 + networkEntries: [],
292 + actionTimeline: [],
293 + });
294 + // The server has no DOM types, so the decoder runs as page source.
295 + const points = [
296 + [0.15, 0.75],
297 + [0.85, 0.25],
298 + [0.85, 0.75],
299 + ];
300 + const pixels = (await page.evaluate(`(async () => {
301 + const bytes = Uint8Array.from(atob(${JSON.stringify(screenshot.data)}), (char) => char.charCodeAt(0));
302 + const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/png" }));
303 + const canvas = document.createElement("canvas");
304 + canvas.width = bitmap.width;
305 + canvas.height = bitmap.height;
306 + const context = canvas.getContext("2d");
307 + context.drawImage(bitmap, 0, 0);
308 + return ${JSON.stringify(points)}.map(([x, y]) =>
309 + Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data),
310 + );
311 + })()`)) as Array<[number, number, number, number]>;
312 + return { screenshot, pixels };
313 + };
314 + const colorOf = ([red, green, blue]: readonly [number, number, number, number]) =>
315 + blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`;
316 +
317 + it("captures a blurred backdrop-filter page as rendered, not mirrored", async () => {
318 + const { screenshot, pixels } = await screenshotOf(halves("backdrop-filter: blur(4px);"));
319 +
320 + expect({
321 + size: [screenshot.width, screenshot.height],
322 + colors: pixels.map(colorOf),
323 + }).toEqual({ size: [1280, 800], colors: ["blue", "red", "red"] });
324 + });
325 +
326 + it("captures a plain page at the shrunk size with its content in place", async () => {
327 + const { screenshot, pixels } = await screenshotOf(halves(""));
328 +
329 + expect({
330 + size: [screenshot.width, screenshot.height],
331 + colors: pixels.map(colorOf),
332 + }).toEqual({ size: [1280, 800], colors: ["blue", "red", "red"] });
333 + });
334 +
335 + it("shrinks the screenshot of an XML document, whose createElement makes no canvas", async () => {
336 + const page = await context.newPage();
337 + await page.goto("data:text/xml,<a/>");
338 + const { screenshot } = await ServerBrowserPage.snapshot({
339 + page,
340 + cdp: await context.newCDPSession(page),
341 + renderScale: 2,
342 + consoleEntries: [],
343 + networkEntries: [],
344 + actionTimeline: [],
345 + });
346 +
347 + expect([screenshot.width, screenshot.height]).toEqual([1280, 800]);
348 + });
349 +});
350 +
351 describe("server browser drag", () => {
352 it("fails the drag, not the server, when it times out while the cursor is still moving", async () => {
353 const timedOut = new Error("locator.dragTo: Timeout 30000ms exceeded.");
apps/server/src/preview/ServerBrowserPage.ts
+47 -39
@@ -410,8 +410,6 @@ const pngSize = (base64: string) => {
410 return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) };
411 };
412
413 -type Viewport = { readonly width: number; readonly height: number };
414 -
413 /**
414 * A page's display ratio and CSS viewport, read in an isolated world since page script can
415 * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom.
@@ -438,13 +436,7 @@ const readPageMetrics = async (cdp: CDPSession) => {
436 readonly width: number;
437 readonly height: number;
438 };
441 - return {
442 - ratio,
443 - width,
444 - height,
445 - zoom,
446 - offset: { x: cssVisualViewport.pageX * zoom, y: cssVisualViewport.pageY * zoom },
447 - };
439 + return { ratio, width, height, zoom };
440 }
441 }
442 throw new Error(
@@ -452,41 +444,56 @@ const readPageMetrics = async (cdp: CDPSession) => {
444 );
445 };
446
455 -// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets.
447 +// Never pass a clip scale: Chromium mirrors the top-left of the page into a `backdrop-filter`
448 +// in any capture it shrinks.
449 export const captureViewport = async (
457 - page: Page,
450 cdp: CDPSession,
459 - options: {
460 - readonly format: "png" | "jpeg";
461 - readonly quality?: number;
462 - readonly scale: number;
463 - readonly viewport?: Viewport;
464 - readonly offset?: { readonly x: number; readonly y: number } | undefined;
465 - },
451 + options: { readonly format: "png" | "jpeg"; readonly quality?: number },
452 ) => {
467 - let clip;
468 - if (options.scale < 1) {
469 - const viewport = options.viewport ?? page.viewportSize() ?? { width: 1280, height: 800 };
470 - let offset = options.offset;
471 - if (!offset) {
472 - const { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics");
473 - const zoom = cssVisualViewport.zoom ?? 1;
474 - offset = { x: cssVisualViewport.pageX * zoom, y: cssVisualViewport.pageY * zoom };
475 - }
476 - clip = {
477 - ...offset,
478 - ...viewport,
479 - scale: options.scale,
480 - };
481 - }
453 const { data } = await cdp.send("Page.captureScreenshot", {
454 format: options.format,
455 ...(options.quality === undefined ? {} : { quality: options.quality }),
485 - ...(clip ? { clip } : {}),
456 });
457 return data;
458 };
459
460 +/** Runs in the page's own isolated world, as the server has no image library. */
461 +const SHRINK_PNG = `async function (png, width, height) {
462 + const bytes = Uint8Array.from(atob(png), (char) => char.charCodeAt(0));
463 + const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/png" }));
464 + // Namespaced, as an XML document's createElement makes a plain element with no getContext.
465 + const canvas = document.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
466 + canvas.width = width;
467 + canvas.height = height;
468 + const context = canvas.getContext("2d");
469 + context.imageSmoothingQuality = "high";
470 + context.drawImage(bitmap, 0, 0, width, height);
471 + bitmap.close();
472 + return canvas.toDataURL("image/png").slice("data:image/png;base64,".length);
473 +}`;
474 +
475 +const shrinkPng = async (cdp: CDPSession, png: string, scale: number) => {
476 + const size = pngSize(png);
477 + const { frameTree } = await cdp.send("Page.getFrameTree");
478 + const { executionContextId } = await cdp.send("Page.createIsolatedWorld", {
479 + frameId: frameTree.frame.id,
480 + worldName: "t3-preview-screenshot",
481 + });
482 + const { result, exceptionDetails } = await cdp.send("Runtime.callFunctionOn", {
483 + functionDeclaration: SHRINK_PNG,
484 + executionContextId,
485 + arguments: [
486 + { value: png },
487 + { value: Math.round(size.width * scale) },
488 + { value: Math.round(size.height * scale) },
489 + ],
490 + awaitPromise: true,
491 + returnByValue: true,
492 + });
493 + if (exceptionDetails) throw new Error("The snapshot could not shrink its screenshot.");
494 + return result.value as string;
495 +};
496 +
497 type SnapshotInput = {
498 readonly page: Page;
499 readonly cdp: CDPSession;
@@ -497,17 +504,16 @@ type SnapshotInput = {
504 readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"];
505 };
506
500 -/** The scale a page renders at and its viewport in device-independent pixels, a clip's units. */
507 +/** The scale a page renders at and its viewport in device-independent pixels. */
508 const renderingOf = async (input: SnapshotInput) => {
509 if (input.renderScale !== "display") {
510 const viewport = input.page.viewportSize() ?? { width: 1280, height: 800 };
504 - return { renderScale: input.renderScale, viewport, offset: undefined };
511 + return { renderScale: input.renderScale, viewport };
512 }
513 // The display ratio a desktop page reports includes its page zoom, and its sizes are CSS pixels.
514 const page = await readPageMetrics(input.cdp);
515 return {
516 renderScale: page.ratio / page.zoom,
510 - offset: page.offset,
517 viewport: {
518 width: Math.round(page.width * page.zoom),
519 height: Math.round(page.height * page.zoom),
@@ -519,7 +525,7 @@ const capture = async (
525 input: SnapshotInput,
526 deadline: number,
527 ): Promise<PreviewAutomationSnapshot> => {
522 - const { renderScale, viewport, offset } = await renderingOf(input);
528 + const { renderScale, viewport } = await renderingOf(input);
529 // A late metrics reply must not begin a capture after viewers have resumed.
530 if (Date.now() >= deadline) throw tooBusy();
531 const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * renderScale));
@@ -536,7 +542,9 @@ const capture = async (
542 "url" | "title" | "loading" | "visibleText" | "interactiveElements"
543 >
544 >,
539 - captureViewport(input.page, input.cdp, { format: "png", scale, viewport, offset }),
545 + captureViewport(input.cdp, { format: "png" }).then((png) =>
546 + scale < 1 ? shrinkPng(input.cdp, png, scale) : png,
547 + ),
548 ]);
549 void pageAndScreenshot.catch(constVoid);
550 // Leaves time for the second read; a frame that never loads holds the tree until then.