@devries-cameron20 / t3code Mirror · read-only / commits / 5c85e93f0f
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 13:30 UTC

perf(server): text-only browser snapshots skip the PNG encode (66 ms → 20 ms median) (#30305)

Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Andrei committed Nov 15, 2026 at 07:13 UTC 5c85e93f0f25ad5d3a24053f17ad5211e9aea311
8 files changed +165 -45
apps/server/src/mcp/McpHttpServer.test.ts
+23 -3
@@ -273,6 +273,22 @@ it.effect.each([
273 ),
274 );
275
276 +it.effect("rejects a metadata-only response when the screenshot is saved", () =>
277 + Effect.scoped(
278 + Effect.gen(function* () {
279 + yield* serveSnapshots("mcp-missing-pixels", {
280 + ...snapshotResult,
281 + screenshot: { width: 10, height: 5 },
282 + });
283 + const result = yield* callSnapshot({ save: true });
284 + expect(result.isError).toBe(true);
285 + expect(result.content).toEqual([
286 + { type: "text", text: expect.stringContaining("did not contain requested PNG pixels") },
287 + ]);
288 + }),
289 + ).pipe(Effect.provide(layerTest)),
290 +);
291 +
292 const layerRecordingTools = (calls: Array<unknown>) => {
293 const RecordingToolkit = Toolkit.make(
294 OrchestratorToolkit.tools.delegate_task,
@@ -418,7 +434,9 @@ it.effect.each([
434 tabId: alternateTabId,
435 threadId,
436 });
421 - expect(event.request.input).toEqual({});
437 + expect(event.request.input).toEqual(
438 + images && requests !== 7 ? {} : { pixelsRequired: false },
439 + );
440 return broker.respond({
441 clientId: "mcp-image-option-client",
442 connectionId: event.connectionId,
@@ -427,7 +445,8 @@ it.effect.each([
445 result: {
446 ...page,
447 title: `Snapshot ${requests}`,
430 - screenshot: { ...screenshot, data: png },
448 + screenshot:
449 + images || requests === 7 ? { ...screenshot, data: png } : { width: 1, height: 1 },
450 },
451 });
452 }).pipe(Effect.forkScoped);
@@ -545,11 +564,12 @@ it.effect("names the screenshot destination when saving it fails", () =>
564 Effect.gen(function* () {
565 const config = yield* ServerConfig.ServerConfig;
566 yield* (yield* FileSystem.FileSystem).writeFileString(config.browserArtifactsDir, "occupied");
548 - yield* serveSnapshots("mcp-save-failure-client", snapshotResult);
567 + const inputs = yield* serveSnapshots("mcp-save-failure-client", snapshotResult);
568
569 const snapshot = yield* callSnapshot({ tabId: alternateTabId, save: true });
570 const text = snapshot.content[0]?.type === "text" ? snapshot.content[0].text : "";
571
572 + expect(inputs).toEqual([{ fullScreenshot: true }]);
573 expect(snapshot.isError).toBe(true);
574 expect(text).toMatch(/^Preview snapshot failed: Could not save preview screenshot to /);
575 expect(text).toContain(config.browserArtifactsDir);
apps/server/src/mcp/McpHttpServer.ts
+21 -9
@@ -574,7 +574,9 @@ const previewSnapshotFailure = Effect.fn("previewSnapshotFailure")(function <E>(
574 : "PreviewSnapshotError";
575 // Preview errors contain server-authored recovery guidance, never page output.
576 const message =
577 - isPreviewAutomationError(firstFailure) || isPreviewScreenshotSaveError(firstFailure)
577 + isPreviewAutomationError(firstFailure) ||
578 + isPreviewScreenshotSaveError(firstFailure) ||
579 + isOrchestratorMcpFailure(firstFailure)
580 ? firstFailure.message
581 : parameterValidationMessage(firstFailure);
582 const result = new McpSchema.CallToolResult({
@@ -641,17 +643,28 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
643 const snapshot = encodedResult as SnapshotMetadata & {
644 readonly url: string;
645 readonly screenshot: {
644 - readonly mimeType: "image/png";
645 - readonly data: string;
646 + readonly mimeType?: "image/png";
647 + readonly data?: string;
648 readonly width: number;
649 readonly height: number;
650 };
651 readonly fullScreenshot?: { readonly data: string };
652 };
653 const { screenshot, fullScreenshot, ...page } = snapshot;
652 - const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
654 + const includeImage = payload?.includeImage === true;
655 + const needsPixels = payload?.save === true || includeImage;
656 + if (needsPixels && (screenshot.mimeType !== "image/png" || !screenshot.data)) {
657 + return yield* new OrchestratorMcpFailure({
658 + code: "invalid_request",
659 + message: "Preview snapshot did not contain requested PNG pixels.",
660 + });
661 + }
662 + const png =
663 + needsPixels && screenshot.data
664 + ? new Uint8Array(Buffer.from(screenshot.data, "base64"))
665 + : undefined;
666 const screenshotPath =
654 - payload?.save === true
667 + payload?.save === true && png
668 ? yield* saveScreenshot(
669 snapshot.url,
670 fullScreenshot
@@ -660,7 +673,6 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
673 )
674 : undefined;
675 // Providers replay images on later requests, and some reject them; include images only when requested.
663 - const includeImage = payload?.includeImage === true;
676 if (screenshotPath !== undefined && !includeImage) {
677 const saved = {
678 url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
@@ -675,7 +687,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
687 const metadata = {
688 ...page,
689 screenshot: {
678 - mimeType: screenshot.mimeType,
690 + mimeType: "image/png" as const,
691 width: screenshot.width,
692 height: screenshot.height,
693 },
@@ -704,8 +716,8 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
716 text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`,
717 },
718 ]),
707 - ...(includeImage
708 - ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]
719 + ...(includeImage && png
720 + ? [{ type: "image" as const, data: png, mimeType: "image/png" as const }]
721 : []),
722 ],
723 });
apps/server/src/mcp/toolkits/preview/handlers.ts
+10 -5
@@ -18,6 +18,7 @@ import {
18 type PreviewAutomationResizeResult,
19 type PreviewAutomationSetColorSchemeResult,
20 type PreviewAutomationSnapshot,
21 + type PreviewAutomationSnapshotMetadata,
22 type PreviewAutomationStatus,
23 type PreviewTabId,
24 } from "@t3tools/contracts";
@@ -221,12 +222,16 @@ const handlers = {
222 invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
223 ),
224 preview_snapshot: McpToolAccess.readsAsCaller((input) => {
224 - const { includeImage: _includeImage, save, ...operationInput } = input ?? {};
225 + const { includeImage, save, ...operationInput } = input ?? {};
226 // The saved file is for people, so it keeps the resolution the agent's image is capped below.
226 - return invokeTargeted<PreviewAutomationSnapshot>("snapshot", {
227 - ...operationInput,
228 - ...(save === true ? { fullScreenshot: true } : {}),
229 - });
227 + return invokeTargeted<PreviewAutomationSnapshot | PreviewAutomationSnapshotMetadata>(
228 + "snapshot",
229 + {
230 + ...operationInput,
231 + ...(save === true ? { fullScreenshot: true } : {}),
232 + ...(includeImage === true || save === true ? {} : { pixelsRequired: false }),
233 + },
234 + );
235 }),
236 preview_act: McpToolAccess.actsAsCaller(({ action, ...input }) =>
237 decodeActInput[action](input).pipe(
apps/server/src/mcp/toolkits/preview/tools.ts
+2 -1
@@ -16,6 +16,7 @@ import {
16 PreviewAutomationSetColorSchemeInput,
17 PreviewAutomationSetColorSchemeResult,
18 PreviewAutomationSnapshot,
19 + PreviewAutomationSnapshotMetadata,
20 PreviewAutomationStatus,
21 PreviewAutomationTabTargetInput,
22 PreviewAutomationWaitForInput,
@@ -151,7 +152,7 @@ export const PreviewSnapshotTool = safeBrowserTool(
152 }),
153 ),
154 }),
154 - success: PreviewAutomationSnapshot,
155 + success: Schema.Union([PreviewAutomationSnapshot, PreviewAutomationSnapshotMetadata]),
156 failure: PreviewToolFailure,
157 dependencies,
158 }).annotate(Tool.Title, "Inspect browser page"),
apps/server/src/preview/ServerBrowser.ts
+9 -5
@@ -2830,12 +2830,16 @@ const make = Effect.gen(function* () {
2830 return { tabId: tab.tabId, colorScheme };
2831 }
2832 case "snapshot": {
2833 + const snapshotInput = input as PreviewAutomationSnapshotInput | undefined;
2834 + const rendering = {
2835 + ...tab,
2836 + renderScale: tab.desktop ? "display" : RENDER_SCALE,
2837 + fullScreenshot: snapshotInput?.fullScreenshot,
2838 + } as const;
2839 const snapshot = await withScreencastsPaused(tab, () =>
2834 - ServerBrowserPage.snapshot({
2835 - ...tab,
2836 - renderScale: tab.desktop ? "display" : RENDER_SCALE,
2837 - fullScreenshot: (input as PreviewAutomationSnapshotInput | undefined)?.fullScreenshot,
2838 - }),
2840 + snapshotInput?.pixelsRequired === false
2841 + ? ServerBrowserPage.snapshotMetadata(rendering)
2842 + : ServerBrowserPage.snapshot(rendering),
2843 );
2844 const loadError = loadErrorOf(tab);
2845 return loadError ? { ...snapshot, url: loadError.url, loadError } : snapshot;
apps/server/src/preview/ServerBrowserPage.test.ts
+31 -5
@@ -23,6 +23,24 @@ import { SessionControl } from "./SessionControl.ts";
23
24 const decodeEvaluateInput = Schema.decodeUnknownSync(PreviewAutomationEvaluateInput);
25
26 +describe("JPEG screenshot dimensions", () => {
27 + const encode = (bytes: number[]) => Buffer.from(bytes).toString("base64");
28 + const frame = [0xff, 0xd8, 0xff, 0xc0, 0, 7, 8, 0, 24, 0, 32];
29 +
30 + it("reads valid dimensions and rejects invalid or incomplete frames", () => {
31 + expect(ServerBrowserPage.jpegSize(encode(frame))).toEqual({ width: 32, height: 24 });
32 + for (const bytes of [
33 + [],
34 + [0xff, 0xd8, 0xff, 0xe1, 0, 20, 1],
35 + frame.slice(0, -1),
36 + [...frame.slice(0, 7), 0, 0, 0, 32],
37 + [...frame.slice(0, 9), 0, 0],
38 + ]) {
39 + expect(() => ServerBrowserPage.jpegSize(encode(bytes))).toThrow("Invalid JPEG screenshot.");
40 + }
41 + });
42 +});
43 +
44 describe("snapshot deadline diagnostics", () => {
45 const never = () => new Promise<never>(() => {});
46 const stalledSnapshot = async (options: {
@@ -591,7 +609,7 @@ describe("server browser snapshot screenshot", () => {
609 const screenshotOf = async (html: string) => {
610 const page = await context.newPage();
611 await page.setContent(html);
594 - const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({
612 + const input = {
613 page,
614 cdp: await context.newCDPSession(page),
615 renderScale: 2,
@@ -599,7 +617,8 @@ describe("server browser snapshot screenshot", () => {
617 consoleEntries: [],
618 networkEntries: [],
619 actionTimeline: [],
602 - });
620 + };
621 + const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot(input);
622 // The server has no DOM types, so the decoder runs as page source.
623 const points = [
624 [0.15, 0.75],
@@ -618,7 +637,7 @@ describe("server browser snapshot screenshot", () => {
637 Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data),
638 );
639 })()`)) as Array<[number, number, number, number]>;
621 - return { screenshot, fullScreenshot, pixels };
640 + return { screenshot, fullScreenshot, pixels, input };
641 };
642 const colorOf = ([red, green, blue]: readonly [number, number, number, number]) =>
643 blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`;
@@ -633,13 +652,20 @@ describe("server browser snapshot screenshot", () => {
652 });
653
654 it("captures a plain page at the shrunk size with its content in place", async () => {
636 - const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves(""));
655 + const { screenshot, fullScreenshot, pixels, input } = await screenshotOf(halves(""));
656 + const metadata = await ServerBrowserPage.snapshotMetadata(input);
657
658 expect({
659 size: [screenshot.width, screenshot.height],
660 fullSize: [fullScreenshot?.width, fullScreenshot?.height],
661 + metadataSize: metadata.screenshot,
662 colors: pixels.map(colorOf),
642 - }).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] });
663 + }).toEqual({
664 + size: [1280, 800],
665 + fullSize: [2560, 1600],
666 + metadataSize: { width: 1280, height: 800 },
667 + colors: ["blue", "red", "red"],
668 + });
669 });
670
671 it("records the destination of a finite entrance animation without waiting on infinite ones", async () => {
apps/server/src/preview/ServerBrowserPage.ts
+61 -17
@@ -14,6 +14,7 @@ import {
14 type PreviewAutomationSelectInput,
15 type PreviewAutomationSelectResult,
16 type PreviewAutomationSnapshot,
17 + type PreviewAutomationSnapshotMetadata,
18 type PreviewAutomationSnapshotStage,
19 type PreviewAutomationTypeInput,
20 type PreviewAutomationUploadInput,
@@ -483,6 +484,37 @@ const pngSize = (base64: string) => {
484 return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) };
485 };
486
487 +export const jpegSize = (base64: string) => {
488 + const bytes = Buffer.from(base64, "base64");
489 + if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8)
490 + throw new Error("Invalid JPEG screenshot.");
491 + let offset = 2;
492 + while (offset < bytes.length) {
493 + if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot.");
494 + while (bytes[offset] === 0xff) offset++;
495 + const marker = bytes[offset++];
496 + if (marker === undefined || marker === 0xd9 || marker === 0xda) break;
497 + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
498 + if (offset + 2 > bytes.length) break;
499 + const length = bytes.readUInt16BE(offset);
500 + if (length < 2 || offset + length > bytes.length) break;
501 + if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) {
502 + if (length < 7) break;
503 + const height = bytes.readUInt16BE(offset + 3);
504 + const width = bytes.readUInt16BE(offset + 5);
505 + if (width && height) return { width, height };
506 + break;
507 + }
508 + offset += length;
509 + }
510 + throw new Error("Invalid JPEG screenshot.");
511 +};
512 +
513 +const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) =>
514 + scale < 1
515 + ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) }
516 + : size;
517 +
518 /**
519 * A page's display ratio and CSS viewport, read in an isolated world since page script can
520 * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom.
@@ -634,7 +666,8 @@ const capture = async (
666 input: SnapshotInput,
667 deadline: number,
668 stages: SnapshotStages,
637 -): Promise<PreviewAutomationSnapshot> => {
669 + metadataOnly: boolean,
670 +) => {
671 const { running } = stages;
672 const track = async <A>(stage: SnapshotStage, operation: () => Promise<A>) => {
673 running.set(stage, (running.get(stage) ?? 0) + 1);
@@ -672,15 +705,21 @@ const capture = async (
705 >
706 >,
707 ),
708 + // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG.
709 track("screenshot", () =>
676 - captureViewport(input.cdp, { format: "png" }).then(async (full) =>
677 - scale < 1
678 - ? {
679 - data: await shrinkPng(input.cdp, full, scale),
680 - full: input.fullScreenshot ? full : undefined,
681 - }
682 - : { data: full, full: undefined },
683 - ),
710 + metadataOnly
711 + ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }).then((data) => ({
712 + data,
713 + full: undefined,
714 + }))
715 + : captureViewport(input.cdp, { format: "png" }).then(async (full) =>
716 + scale < 1
717 + ? {
718 + data: await shrinkPng(input.cdp, full, scale),
719 + full: input.fullScreenshot ? full : undefined,
720 + }
721 + : { data: full, full: undefined },
722 + ),
723 ),
724 ]);
725 void pageAndScreenshot.catch(constVoid);
@@ -731,11 +770,10 @@ const capture = async (
770 consoleEntries: [...input.consoleEntries],
771 networkEntries: [...input.networkEntries],
772 actionTimeline: [...input.actionTimeline],
734 - screenshot: {
735 - mimeType: "image/png",
736 - data,
737 - ...pngSize(data),
738 - },
773 + // Reports the size the PNG path would have shrunk to.
774 + screenshot: metadataOnly
775 + ? scaledSize(jpegSize(data), scale)
776 + : { mimeType: "image/png" as const, data, ...pngSize(data) },
777 ...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}),
778 };
779 };
@@ -743,7 +781,7 @@ const capture = async (
781 /** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */
782 const snapshotTurns = new WeakMap<Page, number>();
783
746 -export const snapshot = async (input: SnapshotInput): Promise<PreviewAutomationSnapshot> => {
784 +const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => {
785 const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS;
786 const stages: SnapshotStages = { running: new Map(), timedOut: new Set() };
787 let timer: ReturnType<typeof setTimeout> | undefined;
@@ -753,11 +791,11 @@ export const snapshot = async (input: SnapshotInput): Promise<PreviewAutomationS
791 const turn = (snapshotTurns.get(input.page) ?? 0) + 1;
792 snapshotTurns.set(input.page, turn);
793 // Retry only before the deadline and the next snapshot, whose refs a later retry would revoke.
756 - const captured = capture(input, deadline, stages).catch((cause: unknown) =>
794 + const captured = capture(input, deadline, stages, metadataOnly).catch((cause: unknown) =>
795 Date.now() < deadline &&
796 snapshotTurns.get(input.page) === turn &&
797 !(cause instanceof Error && cause.message === VALUE_TOO_LONG)
760 - ? capture(input, deadline, stages)
798 + ? capture(input, deadline, stages, metadataOnly)
799 : Promise.reject(cause),
800 );
801 try {
@@ -769,6 +807,12 @@ export const snapshot = async (input: SnapshotInput): Promise<PreviewAutomationS
807 }
808 };
809
810 +export const snapshot = (input: SnapshotInput) =>
811 + runSnapshot(input, false) as Promise<PreviewAutomationSnapshot>;
812 +
813 +export const snapshotMetadata = (input: SnapshotInput) =>
814 + runSnapshot(input, true) as Promise<PreviewAutomationSnapshotMetadata>;
815 +
816 /** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */
817 export const click = async (
818 page: Page,
packages/contracts/src/previewAutomation.ts
+8
@@ -569,11 +569,19 @@ export const PreviewAutomationSnapshot = Schema.Struct({
569 });
570 export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type;
571
572 +// pixelsRequired is false when the caller discards the screenshot; hosts that ignore it still return a PNG.
573 export const PreviewAutomationSnapshotInput = Schema.Struct({
574 fullScreenshot: Schema.optional(Schema.Boolean),
575 + pixelsRequired: Schema.optional(Schema.Boolean),
576 });
577 export type PreviewAutomationSnapshotInput = typeof PreviewAutomationSnapshotInput.Type;
578
579 +export const PreviewAutomationSnapshotMetadata = Schema.Struct({
580 + ...PreviewAutomationSnapshot.fields,
581 + screenshot: Schema.Struct({ width: Schema.Int, height: Schema.Int }),
582 +});
583 +export type PreviewAutomationSnapshotMetadata = typeof PreviewAutomationSnapshotMetadata.Type;
584 +
585 export const PreviewAutomationRecordingStatus = Schema.Struct({
586 tabId: PreviewTabId,
587 recording: Schema.Boolean,