The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 13:30 UTC
perf(server): text-only browser snapshots skip the PNG encode (66 ms → 20 ms median) (#30305)
Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Andrei committed
Nov 15, 2026 at 07:13 UTC
5c85e93f0f25ad5d3a24053f17ad5211e9aea311
8 files changed
+165
-45
apps/server/src/mcp/McpHttpServer.test.ts
+23
-3
@@ -273,6 +273,22 @@ it.effect.each([
273
),
274
);
275
276
+it.effect("rejects a metadata-only response when the screenshot is saved", () =>
277
+ Effect.scoped(
278
+ Effect.gen(function* () {
279
+ yield* serveSnapshots("mcp-missing-pixels", {
280
+ ...snapshotResult,
281
+ screenshot: { width: 10, height: 5 },
282
+ });
283
+ const result = yield* callSnapshot({ save: true });
284
+ expect(result.isError).toBe(true);
285
+ expect(result.content).toEqual([
286
+ { type: "text", text: expect.stringContaining("did not contain requested PNG pixels") },
287
+ ]);
288
+ }),
289
+ ).pipe(Effect.provide(layerTest)),
290
+);
291
+
292
const layerRecordingTools = (calls: Array<unknown>) => {
293
const RecordingToolkit = Toolkit.make(
294
OrchestratorToolkit.tools.delegate_task,
@@ -418,7 +434,9 @@ it.effect.each([
434
tabId: alternateTabId,
435
threadId,
436
});
421
- expect(event.request.input).toEqual({});
437
+ expect(event.request.input).toEqual(
438
+ images && requests !== 7 ? {} : { pixelsRequired: false },
439
+ );
440
return broker.respond({
441
clientId: "mcp-image-option-client",
442
connectionId: event.connectionId,
@@ -427,7 +445,8 @@ it.effect.each([
445
result: {
446
...page,
447
title: `Snapshot ${requests}`,
430
- screenshot: { ...screenshot, data: png },
448
+ screenshot:
449
+ images || requests === 7 ? { ...screenshot, data: png } : { width: 1, height: 1 },
450
},
451
});
452
}).pipe(Effect.forkScoped);
@@ -545,11 +564,12 @@ it.effect("names the screenshot destination when saving it fails", () =>
564
Effect.gen(function* () {
565
const config = yield* ServerConfig.ServerConfig;
566
yield* (yield* FileSystem.FileSystem).writeFileString(config.browserArtifactsDir, "occupied");
548
- yield* serveSnapshots("mcp-save-failure-client", snapshotResult);
567
+ const inputs = yield* serveSnapshots("mcp-save-failure-client", snapshotResult);
568
569
const snapshot = yield* callSnapshot({ tabId: alternateTabId, save: true });
570
const text = snapshot.content[0]?.type === "text" ? snapshot.content[0].text : "";
571
572
+ expect(inputs).toEqual([{ fullScreenshot: true }]);
573
expect(snapshot.isError).toBe(true);
574
expect(text).toMatch(/^Preview snapshot failed: Could not save preview screenshot to /);
575
expect(text).toContain(config.browserArtifactsDir);
apps/server/src/mcp/McpHttpServer.ts
+21
-9
@@ -574,7 +574,9 @@ const previewSnapshotFailure = Effect.fn("previewSnapshotFailure")(function <E>(
574
: "PreviewSnapshotError";
575
// Preview errors contain server-authored recovery guidance, never page output.
576
const message =
577
- isPreviewAutomationError(firstFailure) || isPreviewScreenshotSaveError(firstFailure)
577
+ isPreviewAutomationError(firstFailure) ||
578
+ isPreviewScreenshotSaveError(firstFailure) ||
579
+ isOrchestratorMcpFailure(firstFailure)
580
? firstFailure.message
581
: parameterValidationMessage(firstFailure);
582
const result = new McpSchema.CallToolResult({
@@ -641,17 +643,28 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
643
const snapshot = encodedResult as SnapshotMetadata & {
644
readonly url: string;
645
readonly screenshot: {
644
- readonly mimeType: "image/png";
645
- readonly data: string;
646
+ readonly mimeType?: "image/png";
647
+ readonly data?: string;
648
readonly width: number;
649
readonly height: number;
650
};
651
readonly fullScreenshot?: { readonly data: string };
652
};
653
const { screenshot, fullScreenshot, ...page } = snapshot;
652
- const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
654
+ const includeImage = payload?.includeImage === true;
655
+ const needsPixels = payload?.save === true || includeImage;
656
+ if (needsPixels && (screenshot.mimeType !== "image/png" || !screenshot.data)) {
657
+ return yield* new OrchestratorMcpFailure({
658
+ code: "invalid_request",
659
+ message: "Preview snapshot did not contain requested PNG pixels.",
660
+ });
661
+ }
662
+ const png =
663
+ needsPixels && screenshot.data
664
+ ? new Uint8Array(Buffer.from(screenshot.data, "base64"))
665
+ : undefined;
666
const screenshotPath =
654
- payload?.save === true
667
+ payload?.save === true && png
668
? yield* saveScreenshot(
669
snapshot.url,
670
fullScreenshot
@@ -660,7 +673,6 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
673
)
674
: undefined;
675
// Providers replay images on later requests, and some reject them; include images only when requested.
663
- const includeImage = payload?.includeImage === true;
676
if (screenshotPath !== undefined && !includeImage) {
677
const saved = {
678
url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
@@ -675,7 +687,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
687
const metadata = {
688
...page,
689
screenshot: {
678
- mimeType: screenshot.mimeType,
690
+ mimeType: "image/png" as const,
691
width: screenshot.width,
692
height: screenshot.height,
693
},
@@ -704,8 +716,8 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
716
text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`,
717
},
718
]),
707
- ...(includeImage
708
- ? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]
719
+ ...(includeImage && png
720
+ ? [{ type: "image" as const, data: png, mimeType: "image/png" as const }]
721
: []),
722
],
723
});
apps/server/src/mcp/toolkits/preview/handlers.ts
+10
-5
@@ -18,6 +18,7 @@ import {
18
type PreviewAutomationResizeResult,
19
type PreviewAutomationSetColorSchemeResult,
20
type PreviewAutomationSnapshot,
21
+ type PreviewAutomationSnapshotMetadata,
22
type PreviewAutomationStatus,
23
type PreviewTabId,
24
} from "@t3tools/contracts";
@@ -221,12 +222,16 @@ const handlers = {
222
invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
223
),
224
preview_snapshot: McpToolAccess.readsAsCaller((input) => {
224
- const { includeImage: _includeImage, save, ...operationInput } = input ?? {};
225
+ const { includeImage, save, ...operationInput } = input ?? {};
226
// The saved file is for people, so it keeps the resolution the agent's image is capped below.
226
- return invokeTargeted<PreviewAutomationSnapshot>("snapshot", {
227
- ...operationInput,
228
- ...(save === true ? { fullScreenshot: true } : {}),
229
- });
227
+ return invokeTargeted<PreviewAutomationSnapshot | PreviewAutomationSnapshotMetadata>(
228
+ "snapshot",
229
+ {
230
+ ...operationInput,
231
+ ...(save === true ? { fullScreenshot: true } : {}),
232
+ ...(includeImage === true || save === true ? {} : { pixelsRequired: false }),
233
+ },
234
+ );
235
}),
236
preview_act: McpToolAccess.actsAsCaller(({ action, ...input }) =>
237
decodeActInput[action](input).pipe(
apps/server/src/mcp/toolkits/preview/tools.ts
+2
-1
@@ -16,6 +16,7 @@ import {
16
PreviewAutomationSetColorSchemeInput,
17
PreviewAutomationSetColorSchemeResult,
18
PreviewAutomationSnapshot,
19
+ PreviewAutomationSnapshotMetadata,
20
PreviewAutomationStatus,
21
PreviewAutomationTabTargetInput,
22
PreviewAutomationWaitForInput,
@@ -151,7 +152,7 @@ export const PreviewSnapshotTool = safeBrowserTool(
152
}),
153
),
154
}),
154
- success: PreviewAutomationSnapshot,
155
+ success: Schema.Union([PreviewAutomationSnapshot, PreviewAutomationSnapshotMetadata]),
156
failure: PreviewToolFailure,
157
dependencies,
158
}).annotate(Tool.Title, "Inspect browser page"),
apps/server/src/preview/ServerBrowser.ts
+9
-5
@@ -2830,12 +2830,16 @@ const make = Effect.gen(function* () {
2830
return { tabId: tab.tabId, colorScheme };
2831
}
2832
case "snapshot": {
2833
+ const snapshotInput = input as PreviewAutomationSnapshotInput | undefined;
2834
+ const rendering = {
2835
+ ...tab,
2836
+ renderScale: tab.desktop ? "display" : RENDER_SCALE,
2837
+ fullScreenshot: snapshotInput?.fullScreenshot,
2838
+ } as const;
2839
const snapshot = await withScreencastsPaused(tab, () =>
2834
- ServerBrowserPage.snapshot({
2835
- ...tab,
2836
- renderScale: tab.desktop ? "display" : RENDER_SCALE,
2837
- fullScreenshot: (input as PreviewAutomationSnapshotInput | undefined)?.fullScreenshot,
2838
- }),
2840
+ snapshotInput?.pixelsRequired === false
2841
+ ? ServerBrowserPage.snapshotMetadata(rendering)
2842
+ : ServerBrowserPage.snapshot(rendering),
2843
);
2844
const loadError = loadErrorOf(tab);
2845
return loadError ? { ...snapshot, url: loadError.url, loadError } : snapshot;
apps/server/src/preview/ServerBrowserPage.test.ts
+31
-5
@@ -23,6 +23,24 @@ import { SessionControl } from "./SessionControl.ts";
23
24
const decodeEvaluateInput = Schema.decodeUnknownSync(PreviewAutomationEvaluateInput);
25
26
+describe("JPEG screenshot dimensions", () => {
27
+ const encode = (bytes: number[]) => Buffer.from(bytes).toString("base64");
28
+ const frame = [0xff, 0xd8, 0xff, 0xc0, 0, 7, 8, 0, 24, 0, 32];
29
+
30
+ it("reads valid dimensions and rejects invalid or incomplete frames", () => {
31
+ expect(ServerBrowserPage.jpegSize(encode(frame))).toEqual({ width: 32, height: 24 });
32
+ for (const bytes of [
33
+ [],
34
+ [0xff, 0xd8, 0xff, 0xe1, 0, 20, 1],
35
+ frame.slice(0, -1),
36
+ [...frame.slice(0, 7), 0, 0, 0, 32],
37
+ [...frame.slice(0, 9), 0, 0],
38
+ ]) {
39
+ expect(() => ServerBrowserPage.jpegSize(encode(bytes))).toThrow("Invalid JPEG screenshot.");
40
+ }
41
+ });
42
+});
43
+
44
describe("snapshot deadline diagnostics", () => {
45
const never = () => new Promise<never>(() => {});
46
const stalledSnapshot = async (options: {
@@ -591,7 +609,7 @@ describe("server browser snapshot screenshot", () => {
609
const screenshotOf = async (html: string) => {
610
const page = await context.newPage();
611
await page.setContent(html);
594
- const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({
612
+ const input = {
613
page,
614
cdp: await context.newCDPSession(page),
615
renderScale: 2,
@@ -599,7 +617,8 @@ describe("server browser snapshot screenshot", () => {
617
consoleEntries: [],
618
networkEntries: [],
619
actionTimeline: [],
602
- });
620
+ };
621
+ const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot(input);
622
// The server has no DOM types, so the decoder runs as page source.
623
const points = [
624
[0.15, 0.75],
@@ -618,7 +637,7 @@ describe("server browser snapshot screenshot", () => {
637
Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data),
638
);
639
})()`)) as Array<[number, number, number, number]>;
621
- return { screenshot, fullScreenshot, pixels };
640
+ return { screenshot, fullScreenshot, pixels, input };
641
};
642
const colorOf = ([red, green, blue]: readonly [number, number, number, number]) =>
643
blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`;
@@ -633,13 +652,20 @@ describe("server browser snapshot screenshot", () => {
652
});
653
654
it("captures a plain page at the shrunk size with its content in place", async () => {
636
- const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves(""));
655
+ const { screenshot, fullScreenshot, pixels, input } = await screenshotOf(halves(""));
656
+ const metadata = await ServerBrowserPage.snapshotMetadata(input);
657
658
expect({
659
size: [screenshot.width, screenshot.height],
660
fullSize: [fullScreenshot?.width, fullScreenshot?.height],
661
+ metadataSize: metadata.screenshot,
662
colors: pixels.map(colorOf),
642
- }).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] });
663
+ }).toEqual({
664
+ size: [1280, 800],
665
+ fullSize: [2560, 1600],
666
+ metadataSize: { width: 1280, height: 800 },
667
+ colors: ["blue", "red", "red"],
668
+ });
669
});
670
671
it("records the destination of a finite entrance animation without waiting on infinite ones", async () => {
apps/server/src/preview/ServerBrowserPage.ts
+61
-17
@@ -14,6 +14,7 @@ import {
14
type PreviewAutomationSelectInput,
15
type PreviewAutomationSelectResult,
16
type PreviewAutomationSnapshot,
17
+ type PreviewAutomationSnapshotMetadata,
18
type PreviewAutomationSnapshotStage,
19
type PreviewAutomationTypeInput,
20
type PreviewAutomationUploadInput,
@@ -483,6 +484,37 @@ const pngSize = (base64: string) => {
484
return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) };
485
};
486
487
+export const jpegSize = (base64: string) => {
488
+ const bytes = Buffer.from(base64, "base64");
489
+ if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8)
490
+ throw new Error("Invalid JPEG screenshot.");
491
+ let offset = 2;
492
+ while (offset < bytes.length) {
493
+ if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot.");
494
+ while (bytes[offset] === 0xff) offset++;
495
+ const marker = bytes[offset++];
496
+ if (marker === undefined || marker === 0xd9 || marker === 0xda) break;
497
+ if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
498
+ if (offset + 2 > bytes.length) break;
499
+ const length = bytes.readUInt16BE(offset);
500
+ if (length < 2 || offset + length > bytes.length) break;
501
+ if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) {
502
+ if (length < 7) break;
503
+ const height = bytes.readUInt16BE(offset + 3);
504
+ const width = bytes.readUInt16BE(offset + 5);
505
+ if (width && height) return { width, height };
506
+ break;
507
+ }
508
+ offset += length;
509
+ }
510
+ throw new Error("Invalid JPEG screenshot.");
511
+};
512
+
513
+const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) =>
514
+ scale < 1
515
+ ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) }
516
+ : size;
517
+
518
/**
519
* A page's display ratio and CSS viewport, read in an isolated world since page script can
520
* reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom.
@@ -634,7 +666,8 @@ const capture = async (
666
input: SnapshotInput,
667
deadline: number,
668
stages: SnapshotStages,
637
-): Promise<PreviewAutomationSnapshot> => {
669
+ metadataOnly: boolean,
670
+) => {
671
const { running } = stages;
672
const track = async <A>(stage: SnapshotStage, operation: () => Promise<A>) => {
673
running.set(stage, (running.get(stage) ?? 0) + 1);
@@ -672,15 +705,21 @@ const capture = async (
705
>
706
>,
707
),
708
+ // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG.
709
track("screenshot", () =>
676
- captureViewport(input.cdp, { format: "png" }).then(async (full) =>
677
- scale < 1
678
- ? {
679
- data: await shrinkPng(input.cdp, full, scale),
680
- full: input.fullScreenshot ? full : undefined,
681
- }
682
- : { data: full, full: undefined },
683
- ),
710
+ metadataOnly
711
+ ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }).then((data) => ({
712
+ data,
713
+ full: undefined,
714
+ }))
715
+ : captureViewport(input.cdp, { format: "png" }).then(async (full) =>
716
+ scale < 1
717
+ ? {
718
+ data: await shrinkPng(input.cdp, full, scale),
719
+ full: input.fullScreenshot ? full : undefined,
720
+ }
721
+ : { data: full, full: undefined },
722
+ ),
723
),
724
]);
725
void pageAndScreenshot.catch(constVoid);
@@ -731,11 +770,10 @@ const capture = async (
770
consoleEntries: [...input.consoleEntries],
771
networkEntries: [...input.networkEntries],
772
actionTimeline: [...input.actionTimeline],
734
- screenshot: {
735
- mimeType: "image/png",
736
- data,
737
- ...pngSize(data),
738
- },
773
+ // Reports the size the PNG path would have shrunk to.
774
+ screenshot: metadataOnly
775
+ ? scaledSize(jpegSize(data), scale)
776
+ : { mimeType: "image/png" as const, data, ...pngSize(data) },
777
...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}),
778
};
779
};
@@ -743,7 +781,7 @@ const capture = async (
781
/** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */
782
const snapshotTurns = new WeakMap<Page, number>();
783
746
-export const snapshot = async (input: SnapshotInput): Promise<PreviewAutomationSnapshot> => {
784
+const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => {
785
const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS;
786
const stages: SnapshotStages = { running: new Map(), timedOut: new Set() };
787
let timer: ReturnType<typeof setTimeout> | undefined;
@@ -753,11 +791,11 @@ export const snapshot = async (input: SnapshotInput): Promise<PreviewAutomationS
791
const turn = (snapshotTurns.get(input.page) ?? 0) + 1;
792
snapshotTurns.set(input.page, turn);
793
// Retry only before the deadline and the next snapshot, whose refs a later retry would revoke.
756
- const captured = capture(input, deadline, stages).catch((cause: unknown) =>
794
+ const captured = capture(input, deadline, stages, metadataOnly).catch((cause: unknown) =>
795
Date.now() < deadline &&
796
snapshotTurns.get(input.page) === turn &&
797
!(cause instanceof Error && cause.message === VALUE_TOO_LONG)
760
- ? capture(input, deadline, stages)
798
+ ? capture(input, deadline, stages, metadataOnly)
799
: Promise.reject(cause),
800
);
801
try {
@@ -769,6 +807,12 @@ export const snapshot = async (input: SnapshotInput): Promise<PreviewAutomationS
807
}
808
};
809
810
+export const snapshot = (input: SnapshotInput) =>
811
+ runSnapshot(input, false) as Promise<PreviewAutomationSnapshot>;
812
+
813
+export const snapshotMetadata = (input: SnapshotInput) =>
814
+ runSnapshot(input, true) as Promise<PreviewAutomationSnapshotMetadata>;
815
+
816
/** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */
817
export const click = async (
818
page: Page,
packages/contracts/src/previewAutomation.ts
+8
@@ -569,11 +569,19 @@ export const PreviewAutomationSnapshot = Schema.Struct({
569
});
570
export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type;
571
572
+// pixelsRequired is false when the caller discards the screenshot; hosts that ignore it still return a PNG.
573
export const PreviewAutomationSnapshotInput = Schema.Struct({
574
fullScreenshot: Schema.optional(Schema.Boolean),
575
+ pixelsRequired: Schema.optional(Schema.Boolean),
576
});
577
export type PreviewAutomationSnapshotInput = typeof PreviewAutomationSnapshotInput.Type;
578
579
+export const PreviewAutomationSnapshotMetadata = Schema.Struct({
580
+ ...PreviewAutomationSnapshot.fields,
581
+ screenshot: Schema.Struct({ width: Schema.Int, height: Schema.Int }),
582
+});
583
+export type PreviewAutomationSnapshotMetadata = typeof PreviewAutomationSnapshotMetadata.Type;
584
+
585
export const PreviewAutomationRecordingStatus = Schema.Struct({
586
tabId: PreviewTabId,
587
recording: Schema.Boolean,