The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 18:15 UTC
fix(server): agent recordings honor the browser frame rate setting (#23895)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 23, 2026 at 20:15 UTC
70bd96282cbb46d52a9430c3baf5d16f0369048c
6 files changed
+112
-12
apps/server/src/preview/ServerBrowser.test.ts
+77
-1
@@ -1,4 +1,5 @@
1
import * as NodeEvents from "node:events";
2
+import * as NodeVM from "node:vm";
3
// @effect-diagnostics-next-line nodeBuiltinImport:off - Playwright download mocks perform real file operations outside the Effect runtime.
4
import * as NodeFSP from "node:fs/promises";
5
@@ -6,6 +7,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
7
import * as HostProcess from "@t3tools/shared/HostProcess";
8
import { it } from "@effect/vitest";
9
import {
10
+ type BrowserRecordingFrameRate,
11
EnvironmentId,
12
PreviewTabId,
13
ProviderInstanceId,
@@ -49,7 +51,7 @@ vi.mock("./ServerBrowserContexts.ts", () => ({
51
return context as unknown as BrowserContext;
52
}
53
async scratchPage() {
52
- return makeContext().page as unknown as Page;
54
+ return makeEncoderPage() as unknown as Page;
55
}
56
async connectDesktopPage(endpoint: string) {
57
const context = makeContext();
@@ -212,7 +214,50 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
214
return context;
215
}
216
217
+// Runs the real encoder script against fake canvas and MediaRecorder globals.
218
+function makeEncoderPage() {
219
+ const canvas = {
220
+ width: 0,
221
+ height: 0,
222
+ getContext: () => ({ drawImage() {}, fillRect() {} }),
223
+ captureStream: (frameRate?: number) => ({ frameRate, getTracks: () => [] }),
224
+ };
225
+ class FakeMediaRecorder {
226
+ static isTypeSupported = (type: string) => type === "video/webm";
227
+ onstart: (() => void) | null = null;
228
+ constructor(stream: { frameRate?: number }, options: { videoBitsPerSecond: number }) {
229
+ encoderStreams.push({
230
+ frameRate: stream.frameRate,
231
+ bitsPerSecond: options.videoBitsPerSecond,
232
+ });
233
+ }
234
+ start() {
235
+ this.onstart?.();
236
+ }
237
+ }
238
+ const sandbox: Record<string, unknown> = {
239
+ document: { createElement: () => canvas },
240
+ Path2D: function Path2D() {},
241
+ MediaRecorder: FakeMediaRecorder,
242
+ createImageBitmap: async () => ({ width: 1280, height: 800, close() {} }),
243
+ atob,
244
+ Blob,
245
+ performance,
246
+ };
247
+ sandbox.window = sandbox;
248
+ const context = NodeVM.createContext(sandbox);
249
+ return {
250
+ ...makeContext().page,
251
+ evaluate: vi.fn(async (script: string | ((arg: unknown) => unknown), arg?: unknown) =>
252
+ typeof script === "string"
253
+ ? NodeVM.runInContext(script, context)
254
+ : NodeVM.runInContext(`(${script.toString()})`, context)(arg),
255
+ ),
256
+ };
257
+}
258
+
259
const contexts: ReturnType<typeof makeContext>[] = [];
260
+const encoderStreams: Array<{ frameRate: number | undefined; bitsPerSecond: number }> = [];
261
const contextRequests: Array<{ profileId: string; isolated: boolean }> = [];
262
let contextGate: PromiseWithResolvers<void> | null = null;
263
let pageCopy = "";
@@ -320,6 +365,7 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({
365
366
beforeEach(() => {
367
contexts.length = 0;
368
+ encoderStreams.length = 0;
369
contextRequests.length = 0;
370
contextGate = null;
371
contextFailure = null;
@@ -418,6 +464,36 @@ it.live.each([
464
).pipe(Effect.provide(layer)),
465
);
466
467
+// The fake encoder decodes every frame at 1280x800, so 30 fps hits the bitrate floor.
468
+it.live.each([
469
+ { reports: [], frameRate: 30, bitsPerSecond: 2.5e6 },
470
+ { reports: [30], frameRate: 30, bitsPerSecond: 2.5e6 },
471
+ { reports: [60], frameRate: 60, bitsPerSecond: 1280 * 800 * 60 * 0.05 },
472
+ { reports: [60, undefined], frameRate: 60, bitsPerSecond: 1280 * 800 * 60 * 0.05 },
473
+ { reports: [60, 30], frameRate: 30, bitsPerSecond: 2.5e6 },
474
+] satisfies ReadonlyArray<{
475
+ reports: ReadonlyArray<BrowserRecordingFrameRate | undefined>;
476
+ frameRate: number;
477
+ bitsPerSecond: number;
478
+}>)(
479
+ "agent recordings encode at the last reported frame rate ($reports)",
480
+ ({ reports, frameRate, bitsPerSecond }) =>
481
+ Effect.scoped(
482
+ Effect.gen(function* () {
483
+ const { browser, broker, tabId } = yield* ready;
484
+ for (const recordingFrameRate of reports) {
485
+ yield* browser.reportProfiles({
486
+ profiles: [],
487
+ defaultProfileId: "default",
488
+ ...(recordingFrameRate === undefined ? {} : { recordingFrameRate }),
489
+ });
490
+ }
491
+ yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} });
492
+ expect(encoderStreams).toEqual([{ frameRate, bitsPerSecond }]);
493
+ }),
494
+ ).pipe(Effect.provide(layer)),
495
+);
496
+
497
it.live("enforces provider ownership and explicit targets when a session has multiple tabs", () =>
498
Effect.scoped(
499
Effect.gen(function* () {
apps/server/src/preview/ServerBrowser.ts
+6
-1
@@ -4,6 +4,7 @@
4
import {
5
BUILT_IN_BROWSER_PROFILES,
6
DEFAULT_BROWSER_PROFILE_ID,
7
+ DEFAULT_BROWSER_RECORDING_FRAME_RATE,
8
FILL_PREVIEW_VIEWPORT,
9
findBrowserProfile,
10
INCOGNITO_BROWSER_PROFILE_ID,
@@ -1313,9 +1314,13 @@ const make = Effect.gen(function* () {
1314
readonly profiles: ReadonlyArray<BrowserProfile>;
1315
readonly defaultProfileId: string;
1316
} | null = null;
1317
+ // Agent recordings encode here, so they need the client's Browser recording frame rate. The
1318
+ // last report outlives the client, which lets agents record while no client is connected.
1319
+ let recordingFrameRate = DEFAULT_BROWSER_RECORDING_FRAME_RATE;
1320
1321
const reportProfiles: ServerBrowser["Service"]["reportProfiles"] = (input) =>
1322
Effect.sync(() => {
1323
+ if (input.recordingFrameRate !== undefined) recordingFrameRate = input.recordingFrameRate;
1324
const profiles = resolveBrowserProfiles(input.profiles);
1325
reportedProfiles = {
1326
profiles,
@@ -1472,7 +1477,7 @@ const make = Effect.gen(function* () {
1477
const encoder = await contexts.scratchPage();
1478
let session: CDPSession | null = null;
1479
try {
1475
- await encoder.evaluate(ServerBrowserPage.RECORDING_ENCODER_SCRIPT);
1480
+ await encoder.evaluate(ServerBrowserPage.recordingEncoderScript(recordingFrameRate));
1481
// Seed idle pages before returning so an immediate stop has a frame to encode.
1482
const firstFrame = await ServerBrowserPage.captureViewport(tab.page, tab.cdp, {
1483
format: "jpeg",
apps/server/src/preview/ServerBrowserPage.ts
+4
-3
@@ -1,6 +1,7 @@
1
// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off - Playwright callbacks run outside the Effect runtime.
2
3
import {
4
+ type BrowserRecordingFrameRate,
5
PROVIDER_SEND_TURN_MAX_FILE_BYTES,
6
type PreviewAutomationClickInput,
7
type PreviewAutomationConsoleEntry,
@@ -922,7 +923,7 @@ export const parseNetError = (errorText: string) => {
923
return { description, code: NET_ERROR_CODES[description] ?? -2 };
924
};
925
925
-export const RECORDING_ENCODER_SCRIPT = `(() => {
926
+export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) => `(() => {
927
const canvas = document.createElement("canvas");
928
const context = canvas.getContext("2d");
929
const chunks = [];
@@ -991,8 +992,8 @@ export const RECORDING_ENCODER_SCRIPT = `(() => {
992
canvas.width = bitmap.width;
993
canvas.height = bitmap.height;
994
const mimeType = ["video/mp4;codecs=avc1.640033", "video/webm;codecs=vp9", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type));
994
- const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * 30 * 0.05));
995
- recorder = new MediaRecorder(canvas.captureStream(30), { mimeType, videoBitsPerSecond: bitsPerSecond });
995
+ const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * ${frameRate} * 0.05));
996
+ recorder = new MediaRecorder(canvas.captureStream(${frameRate}), { mimeType, videoBitsPerSecond: bitsPerSecond });
997
started = new Promise((resolve, reject) => {
998
recorder.onstart = resolve;
999
recorder.onerror = (event) => reject(event.error);
apps/web/src/browser/BrowserProfileReporter.tsx
+11
-3
@@ -1,4 +1,8 @@
1
-import { AuthPreviewOperateScope, type BrowserProfile } from "@t3tools/contracts";
1
+import {
2
+ AuthPreviewOperateScope,
3
+ type BrowserProfile,
4
+ type BrowserRecordingFrameRate,
5
+} from "@t3tools/contracts";
6
import { useEffect } from "react";
7
8
import { useClientSettings, useClientSettingsHydrated } from "~/hooks/useSettings";
@@ -12,11 +16,15 @@ const selectProfiles = (settings: { readonly browserProfiles: ReadonlyArray<Brow
16
settings.browserProfiles;
17
const selectDefaultProfileId = (settings: { readonly browserDefaultProfileId: string }) =>
18
settings.browserDefaultProfileId;
19
+const selectRecordingFrameRate = (settings: {
20
+ readonly browserRecordingFrameRate: BrowserRecordingFrameRate;
21
+}) => settings.browserRecordingFrameRate;
22
23
export function BrowserProfileReporter() {
24
const hydrated = useClientSettingsHydrated();
25
const profiles = useClientSettings(selectProfiles);
26
const defaultProfileId = useClientSettings(selectDefaultProfileId);
27
+ const recordingFrameRate = useClientSettings(selectRecordingFrameRate);
28
const { environments } = useEnvironments();
29
const serverConfigs = useServerConfigs();
30
const operable = useEnvironmentsWithScope(environments, AuthPreviewOperateScope);
@@ -26,9 +34,9 @@ export function BrowserProfileReporter() {
34
if (!hydrated) return;
35
for (const [environmentId, config] of serverConfigs) {
36
if (!config.environment.capabilities.serverBrowser || !operable.has(environmentId)) continue;
29
- void report({ environmentId, input: { profiles, defaultProfileId } });
37
+ void report({ environmentId, input: { profiles, defaultProfileId, recordingFrameRate } });
38
}
31
- }, [defaultProfileId, hydrated, operable, profiles, report, serverConfigs]);
39
+ }, [defaultProfileId, hydrated, operable, profiles, recordingFrameRate, report, serverConfigs]);
40
41
return null;
42
}
packages/contracts/src/preview.ts
+7
@@ -241,9 +241,16 @@ export const PreviewCloseInput = Schema.Struct({
241
});
242
export type PreviewCloseInput = typeof PreviewCloseInput.Type;
243
244
+export const BROWSER_RECORDING_FRAME_RATES = [30, 60] as const;
245
+export const BrowserRecordingFrameRate = Schema.Literals(BROWSER_RECORDING_FRAME_RATES);
246
+export type BrowserRecordingFrameRate = typeof BrowserRecordingFrameRate.Type;
247
+export const DEFAULT_BROWSER_RECORDING_FRAME_RATE: BrowserRecordingFrameRate = 30;
248
+
249
export const PreviewReportProfilesInput = Schema.Struct({
250
profiles: Schema.Array(BrowserProfile).check(Schema.isMaxLength(BROWSER_PROFILE_MAX_COUNT)),
251
defaultProfileId: BrowserProfileId,
252
+ /** Optional so older clients keep reporting; omitting it keeps the last reported rate. */
253
+ recordingFrameRate: Schema.optional(BrowserRecordingFrameRate),
254
});
255
export type PreviewReportProfilesInput = typeof PreviewReportProfilesInput.Type;
256
packages/contracts/src/settings.ts
+7
-4
@@ -33,6 +33,8 @@ import { ProjectScript, type ProjectMutation } from "./project.ts";
33
import { DEFAULT_RUNTIME_MODE, RuntimeMode } from "./providerPolicy.ts";
34
import { BrowserProfile, BrowserProfileId, DEFAULT_BROWSER_PROFILE_ID } from "./browserProfile.ts";
35
import {
36
+ BrowserRecordingFrameRate,
37
+ DEFAULT_BROWSER_RECORDING_FRAME_RATE,
38
DEFAULT_PREVIEW_APPEARANCE,
39
DEFAULT_PREVIEW_ZOOM_FACTOR,
40
FILL_PREVIEW_VIEWPORT,
@@ -257,10 +259,11 @@ export type DefaultThemePreference = typeof DefaultThemePreference.Type;
259
260
export const DEFAULT_BROWSER_VIEWPORT: PreviewViewportSetting = FILL_PREVIEW_VIEWPORT;
261
export const DEFAULT_BROWSER_AUTO_SHOW_FLOATING_PREVIEW = true;
260
-export const BROWSER_RECORDING_FRAME_RATES = [30, 60] as const;
261
-export const BrowserRecordingFrameRate = Schema.Literals(BROWSER_RECORDING_FRAME_RATES);
262
-export type BrowserRecordingFrameRate = typeof BrowserRecordingFrameRate.Type;
263
-export const DEFAULT_BROWSER_RECORDING_FRAME_RATE: BrowserRecordingFrameRate = 30;
262
+export {
263
+ BROWSER_RECORDING_FRAME_RATES,
264
+ BrowserRecordingFrameRate,
265
+ DEFAULT_BROWSER_RECORDING_FRAME_RATE,
266
+} from "./preview.ts";
267
export const BrowserLinkTarget = Schema.Literals(["system", "app"]);
268
export type BrowserLinkTarget = typeof BrowserLinkTarget.Type;
269
export const DEFAULT_BROWSER_LINK_TARGET: BrowserLinkTarget = "system";