@devries-cameron20 / t3code Mirror · read-only / commits / 70bd96282c
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 18:15 UTC

fix(server): agent recordings honor the browser frame rate setting (#23895)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 23, 2026 at 20:15 UTC 70bd96282cbb46d52a9430c3baf5d16f0369048c
6 files changed +112 -12
apps/server/src/preview/ServerBrowser.test.ts
+77 -1
@@ -1,4 +1,5 @@
1 import * as NodeEvents from "node:events";
2 +import * as NodeVM from "node:vm";
3 // @effect-diagnostics-next-line nodeBuiltinImport:off - Playwright download mocks perform real file operations outside the Effect runtime.
4 import * as NodeFSP from "node:fs/promises";
5
@@ -6,6 +7,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
7 import * as HostProcess from "@t3tools/shared/HostProcess";
8 import { it } from "@effect/vitest";
9 import {
10 + type BrowserRecordingFrameRate,
11 EnvironmentId,
12 PreviewTabId,
13 ProviderInstanceId,
@@ -49,7 +51,7 @@ vi.mock("./ServerBrowserContexts.ts", () => ({
51 return context as unknown as BrowserContext;
52 }
53 async scratchPage() {
52 - return makeContext().page as unknown as Page;
54 + return makeEncoderPage() as unknown as Page;
55 }
56 async connectDesktopPage(endpoint: string) {
57 const context = makeContext();
@@ -212,7 +214,50 @@ function makeContext(onClose?: (context: BrowserContext) => void) {
214 return context;
215 }
216
217 +// Runs the real encoder script against fake canvas and MediaRecorder globals.
218 +function makeEncoderPage() {
219 + const canvas = {
220 + width: 0,
221 + height: 0,
222 + getContext: () => ({ drawImage() {}, fillRect() {} }),
223 + captureStream: (frameRate?: number) => ({ frameRate, getTracks: () => [] }),
224 + };
225 + class FakeMediaRecorder {
226 + static isTypeSupported = (type: string) => type === "video/webm";
227 + onstart: (() => void) | null = null;
228 + constructor(stream: { frameRate?: number }, options: { videoBitsPerSecond: number }) {
229 + encoderStreams.push({
230 + frameRate: stream.frameRate,
231 + bitsPerSecond: options.videoBitsPerSecond,
232 + });
233 + }
234 + start() {
235 + this.onstart?.();
236 + }
237 + }
238 + const sandbox: Record<string, unknown> = {
239 + document: { createElement: () => canvas },
240 + Path2D: function Path2D() {},
241 + MediaRecorder: FakeMediaRecorder,
242 + createImageBitmap: async () => ({ width: 1280, height: 800, close() {} }),
243 + atob,
244 + Blob,
245 + performance,
246 + };
247 + sandbox.window = sandbox;
248 + const context = NodeVM.createContext(sandbox);
249 + return {
250 + ...makeContext().page,
251 + evaluate: vi.fn(async (script: string | ((arg: unknown) => unknown), arg?: unknown) =>
252 + typeof script === "string"
253 + ? NodeVM.runInContext(script, context)
254 + : NodeVM.runInContext(`(${script.toString()})`, context)(arg),
255 + ),
256 + };
257 +}
258 +
259 const contexts: ReturnType<typeof makeContext>[] = [];
260 +const encoderStreams: Array<{ frameRate: number | undefined; bitsPerSecond: number }> = [];
261 const contextRequests: Array<{ profileId: string; isolated: boolean }> = [];
262 let contextGate: PromiseWithResolvers<void> | null = null;
263 let pageCopy = "";
@@ -320,6 +365,7 @@ const viewerInput = (tabId: string, canOperate: boolean) => ({
365
366 beforeEach(() => {
367 contexts.length = 0;
368 + encoderStreams.length = 0;
369 contextRequests.length = 0;
370 contextGate = null;
371 contextFailure = null;
@@ -418,6 +464,36 @@ it.live.each([
464 ).pipe(Effect.provide(layer)),
465 );
466
467 +// The fake encoder decodes every frame at 1280x800, so 30 fps hits the bitrate floor.
468 +it.live.each([
469 + { reports: [], frameRate: 30, bitsPerSecond: 2.5e6 },
470 + { reports: [30], frameRate: 30, bitsPerSecond: 2.5e6 },
471 + { reports: [60], frameRate: 60, bitsPerSecond: 1280 * 800 * 60 * 0.05 },
472 + { reports: [60, undefined], frameRate: 60, bitsPerSecond: 1280 * 800 * 60 * 0.05 },
473 + { reports: [60, 30], frameRate: 30, bitsPerSecond: 2.5e6 },
474 +] satisfies ReadonlyArray<{
475 + reports: ReadonlyArray<BrowserRecordingFrameRate | undefined>;
476 + frameRate: number;
477 + bitsPerSecond: number;
478 +}>)(
479 + "agent recordings encode at the last reported frame rate ($reports)",
480 + ({ reports, frameRate, bitsPerSecond }) =>
481 + Effect.scoped(
482 + Effect.gen(function* () {
483 + const { browser, broker, tabId } = yield* ready;
484 + for (const recordingFrameRate of reports) {
485 + yield* browser.reportProfiles({
486 + profiles: [],
487 + defaultProfileId: "default",
488 + ...(recordingFrameRate === undefined ? {} : { recordingFrameRate }),
489 + });
490 + }
491 + yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} });
492 + expect(encoderStreams).toEqual([{ frameRate, bitsPerSecond }]);
493 + }),
494 + ).pipe(Effect.provide(layer)),
495 +);
496 +
497 it.live("enforces provider ownership and explicit targets when a session has multiple tabs", () =>
498 Effect.scoped(
499 Effect.gen(function* () {
apps/server/src/preview/ServerBrowser.ts
+6 -1
@@ -4,6 +4,7 @@
4 import {
5 BUILT_IN_BROWSER_PROFILES,
6 DEFAULT_BROWSER_PROFILE_ID,
7 + DEFAULT_BROWSER_RECORDING_FRAME_RATE,
8 FILL_PREVIEW_VIEWPORT,
9 findBrowserProfile,
10 INCOGNITO_BROWSER_PROFILE_ID,
@@ -1313,9 +1314,13 @@ const make = Effect.gen(function* () {
1314 readonly profiles: ReadonlyArray<BrowserProfile>;
1315 readonly defaultProfileId: string;
1316 } | null = null;
1317 + // Agent recordings encode here, so they need the client's Browser recording frame rate. The
1318 + // last report outlives the client, which lets agents record while no client is connected.
1319 + let recordingFrameRate = DEFAULT_BROWSER_RECORDING_FRAME_RATE;
1320
1321 const reportProfiles: ServerBrowser["Service"]["reportProfiles"] = (input) =>
1322 Effect.sync(() => {
1323 + if (input.recordingFrameRate !== undefined) recordingFrameRate = input.recordingFrameRate;
1324 const profiles = resolveBrowserProfiles(input.profiles);
1325 reportedProfiles = {
1326 profiles,
@@ -1472,7 +1477,7 @@ const make = Effect.gen(function* () {
1477 const encoder = await contexts.scratchPage();
1478 let session: CDPSession | null = null;
1479 try {
1475 - await encoder.evaluate(ServerBrowserPage.RECORDING_ENCODER_SCRIPT);
1480 + await encoder.evaluate(ServerBrowserPage.recordingEncoderScript(recordingFrameRate));
1481 // Seed idle pages before returning so an immediate stop has a frame to encode.
1482 const firstFrame = await ServerBrowserPage.captureViewport(tab.page, tab.cdp, {
1483 format: "jpeg",
apps/server/src/preview/ServerBrowserPage.ts
+4 -3
@@ -1,6 +1,7 @@
1 // @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off - Playwright callbacks run outside the Effect runtime.
2
3 import {
4 + type BrowserRecordingFrameRate,
5 PROVIDER_SEND_TURN_MAX_FILE_BYTES,
6 type PreviewAutomationClickInput,
7 type PreviewAutomationConsoleEntry,
@@ -922,7 +923,7 @@ export const parseNetError = (errorText: string) => {
923 return { description, code: NET_ERROR_CODES[description] ?? -2 };
924 };
925
925 -export const RECORDING_ENCODER_SCRIPT = `(() => {
926 +export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) => `(() => {
927 const canvas = document.createElement("canvas");
928 const context = canvas.getContext("2d");
929 const chunks = [];
@@ -991,8 +992,8 @@ export const RECORDING_ENCODER_SCRIPT = `(() => {
992 canvas.width = bitmap.width;
993 canvas.height = bitmap.height;
994 const mimeType = ["video/mp4;codecs=avc1.640033", "video/webm;codecs=vp9", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type));
994 - const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * 30 * 0.05));
995 - recorder = new MediaRecorder(canvas.captureStream(30), { mimeType, videoBitsPerSecond: bitsPerSecond });
995 + const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * ${frameRate} * 0.05));
996 + recorder = new MediaRecorder(canvas.captureStream(${frameRate}), { mimeType, videoBitsPerSecond: bitsPerSecond });
997 started = new Promise((resolve, reject) => {
998 recorder.onstart = resolve;
999 recorder.onerror = (event) => reject(event.error);
apps/web/src/browser/BrowserProfileReporter.tsx
+11 -3
@@ -1,4 +1,8 @@
1 -import { AuthPreviewOperateScope, type BrowserProfile } from "@t3tools/contracts";
1 +import {
2 + AuthPreviewOperateScope,
3 + type BrowserProfile,
4 + type BrowserRecordingFrameRate,
5 +} from "@t3tools/contracts";
6 import { useEffect } from "react";
7
8 import { useClientSettings, useClientSettingsHydrated } from "~/hooks/useSettings";
@@ -12,11 +16,15 @@ const selectProfiles = (settings: { readonly browserProfiles: ReadonlyArray<Brow
16 settings.browserProfiles;
17 const selectDefaultProfileId = (settings: { readonly browserDefaultProfileId: string }) =>
18 settings.browserDefaultProfileId;
19 +const selectRecordingFrameRate = (settings: {
20 + readonly browserRecordingFrameRate: BrowserRecordingFrameRate;
21 +}) => settings.browserRecordingFrameRate;
22
23 export function BrowserProfileReporter() {
24 const hydrated = useClientSettingsHydrated();
25 const profiles = useClientSettings(selectProfiles);
26 const defaultProfileId = useClientSettings(selectDefaultProfileId);
27 + const recordingFrameRate = useClientSettings(selectRecordingFrameRate);
28 const { environments } = useEnvironments();
29 const serverConfigs = useServerConfigs();
30 const operable = useEnvironmentsWithScope(environments, AuthPreviewOperateScope);
@@ -26,9 +34,9 @@ export function BrowserProfileReporter() {
34 if (!hydrated) return;
35 for (const [environmentId, config] of serverConfigs) {
36 if (!config.environment.capabilities.serverBrowser || !operable.has(environmentId)) continue;
29 - void report({ environmentId, input: { profiles, defaultProfileId } });
37 + void report({ environmentId, input: { profiles, defaultProfileId, recordingFrameRate } });
38 }
31 - }, [defaultProfileId, hydrated, operable, profiles, report, serverConfigs]);
39 + }, [defaultProfileId, hydrated, operable, profiles, recordingFrameRate, report, serverConfigs]);
40
41 return null;
42 }
packages/contracts/src/preview.ts
+7
@@ -241,9 +241,16 @@ export const PreviewCloseInput = Schema.Struct({
241 });
242 export type PreviewCloseInput = typeof PreviewCloseInput.Type;
243
244 +export const BROWSER_RECORDING_FRAME_RATES = [30, 60] as const;
245 +export const BrowserRecordingFrameRate = Schema.Literals(BROWSER_RECORDING_FRAME_RATES);
246 +export type BrowserRecordingFrameRate = typeof BrowserRecordingFrameRate.Type;
247 +export const DEFAULT_BROWSER_RECORDING_FRAME_RATE: BrowserRecordingFrameRate = 30;
248 +
249 export const PreviewReportProfilesInput = Schema.Struct({
250 profiles: Schema.Array(BrowserProfile).check(Schema.isMaxLength(BROWSER_PROFILE_MAX_COUNT)),
251 defaultProfileId: BrowserProfileId,
252 + /** Optional so older clients keep reporting; omitting it keeps the last reported rate. */
253 + recordingFrameRate: Schema.optional(BrowserRecordingFrameRate),
254 });
255 export type PreviewReportProfilesInput = typeof PreviewReportProfilesInput.Type;
256
packages/contracts/src/settings.ts
+7 -4
@@ -33,6 +33,8 @@ import { ProjectScript, type ProjectMutation } from "./project.ts";
33 import { DEFAULT_RUNTIME_MODE, RuntimeMode } from "./providerPolicy.ts";
34 import { BrowserProfile, BrowserProfileId, DEFAULT_BROWSER_PROFILE_ID } from "./browserProfile.ts";
35 import {
36 + BrowserRecordingFrameRate,
37 + DEFAULT_BROWSER_RECORDING_FRAME_RATE,
38 DEFAULT_PREVIEW_APPEARANCE,
39 DEFAULT_PREVIEW_ZOOM_FACTOR,
40 FILL_PREVIEW_VIEWPORT,
@@ -257,10 +259,11 @@ export type DefaultThemePreference = typeof DefaultThemePreference.Type;
259
260 export const DEFAULT_BROWSER_VIEWPORT: PreviewViewportSetting = FILL_PREVIEW_VIEWPORT;
261 export const DEFAULT_BROWSER_AUTO_SHOW_FLOATING_PREVIEW = true;
260 -export const BROWSER_RECORDING_FRAME_RATES = [30, 60] as const;
261 -export const BrowserRecordingFrameRate = Schema.Literals(BROWSER_RECORDING_FRAME_RATES);
262 -export type BrowserRecordingFrameRate = typeof BrowserRecordingFrameRate.Type;
263 -export const DEFAULT_BROWSER_RECORDING_FRAME_RATE: BrowserRecordingFrameRate = 30;
262 +export {
263 + BROWSER_RECORDING_FRAME_RATES,
264 + BrowserRecordingFrameRate,
265 + DEFAULT_BROWSER_RECORDING_FRAME_RATE,
266 +} from "./preview.ts";
267 export const BrowserLinkTarget = Schema.Literals(["system", "app"]);
268 export type BrowserLinkTarget = typeof BrowserLinkTarget.Type;
269 export const DEFAULT_BROWSER_LINK_TARGET: BrowserLinkTarget = "system";