@devries-cameron20 / t3code-1 / commits / d6c2b09c52
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

feat(server): html tools take a file path, so agents stop retyping pages (#32833)

Agents can pass `path` to an absolute .html file instead of retyping the page as inline `html`. The server reads the file only for full-access threads, with the existing disk-read guards, and snapshots it at publish. ACP render recovery matches path calls by path. The mcp-tool-use eval grades pages passed by path. Takes over #32817. Co-authored-by: Stoli Geyer <stoli@tractorbeam.ai> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Nov 7, 2026 at 21:26 UTC d6c2b09c52e96a71e22124a734190cc0d14ca25f
12 files changed +608 -85
apps/server/src/htmlRender/HtmlRender.test.ts
+114
@@ -4,6 +4,7 @@ import { assert, describe, expect, it } from "@effect/vitest";
4 import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";
5 import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
6 import * as HostProcess from "@t3tools/shared/HostProcess";
7 +import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
8 import * as Deferred from "effect/Deferred";
9 import * as Effect from "effect/Effect";
10 import * as Exit from "effect/Exit";
@@ -259,6 +260,119 @@ describe("HtmlRender", () => {
260 }).pipe(Effect.provide(layerTest)),
261 );
262
263 + it.effect("publishes a page from an .html file as a snapshot that path calls recover", () =>
264 + Effect.gen(function* () {
265 + const fileSystem = yield* FileSystem.FileSystem;
266 + const path = yield* Path.Path;
267 + const config = yield* ServerConfig.ServerConfig;
268 + const htmlRender = yield* HtmlRender.HtmlRender;
269 + const mcpSessions = yield* McpProviderSessions.McpProviderSessions;
270 + const threadId = ThreadId.make("thread-html-path");
271 + const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-html-page-" });
272 + const file = path.join(directory, "report.html");
273 + yield* fileSystem.writeFileString(file, "<!doctype html><p>First draft</p>");
274 + const page = { path: file, title: "Report", height: 300 };
275 +
276 + const first = yield* htmlRender.publish({ threadId, ...page });
277 + yield* fileSystem.writeFileString(file, "<!doctype html><p>Second draft</p>");
278 + const second = yield* htmlRender.publish({ threadId, ...page });
279 + const inline = { html: "<!doctype html><p>First draft</p>", title: "Report", height: 300 };
280 + const inlineReference = yield* htmlRender.publish({ threadId, ...inline });
281 +
282 + const stored = yield* fileSystem.readFileString(
283 + resolveAttachmentPathById({
284 + attachmentsDir: config.attachmentsDir,
285 + attachmentId: first.attachmentId,
286 + }) ?? "",
287 + );
288 + expect(stored).toContain("<p>First draft</p>");
289 + expect(yield* mcpSessions.takeHtmlRender(threadId, { ...page, path: `${file}x` })).toBe(
290 + undefined,
291 + );
292 + for (const html of [inline.html, false]) {
293 + expect(yield* mcpSessions.takeHtmlRender(threadId, { ...page, html })).toBe(undefined);
294 + }
295 + expect(yield* mcpSessions.takeHtmlRender(threadId, { ...inline, path: null })).toEqual(
296 + inlineReference,
297 + );
298 + expect(yield* mcpSessions.takeHtmlRender(threadId, { ...page, html: null })).toEqual(first);
299 + expect(yield* mcpSessions.takeHtmlRender(threadId, page)).toEqual(second);
300 + expect(yield* mcpSessions.takeHtmlRender(threadId, page)).toBe(undefined);
301 + }).pipe(Effect.provide(layerTest)),
302 + );
303 +
304 + it.effect("refuses anything but a regular, singly linked .html file of at most 25 MiB", () =>
305 + Effect.gen(function* () {
306 + const fileSystem = yield* FileSystem.FileSystem;
307 + const path = yield* Path.Path;
308 + const config = yield* ServerConfig.ServerConfig;
309 + const htmlRender = yield* HtmlRender.HtmlRender;
310 + const threadId = ThreadId.make("thread-html-path-guard");
311 + const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-html-page-" });
312 + const secret = path.join(directory, "id_rsa");
313 + yield* fileSystem.writeFileString(secret, "PRIVATE KEY");
314 + const page = path.join(directory, "page.html");
315 + yield* fileSystem.writeFileString(page, "<!doctype html><p>linked</p>");
316 + const hardLinked = path.join(directory, "linked.html");
317 + yield* fileSystem.link(page, hardLinked);
318 + // `mv .env renamed.html` keeps one link and an .html name; only its bytes give it away.
319 + const renamed = path.join(directory, "renamed.html");
320 + yield* fileSystem.writeFileString(renamed, "API_KEY=abc123\n");
321 + const folder = path.join(directory, "folder.html");
322 + yield* fileSystem.makeDirectory(folder);
323 + const internal = path.join(config.secretsDir, "page.html");
324 + yield* fileSystem.makeDirectory(config.secretsDir, { recursive: true });
325 + yield* fileSystem.writeFileString(internal, "<!doctype html><p>internal</p>");
326 +
327 + for (const file of [
328 + "page.html",
329 + "//attacker/share/page.html",
330 + path.join(directory, "missing.html"),
331 + secret,
332 + hardLinked,
333 + renamed,
334 + folder,
335 + internal,
336 + ]) {
337 + const error = yield* htmlRender
338 + .publish({ threadId, path: file, title: "X", height: 200 })
339 + .pipe(Effect.flip);
340 + expect(error, file).toBeInstanceOf(HtmlRender.HtmlRenderFileError);
341 + }
342 + const huge = path.join(directory, "huge.html");
343 + yield* fileSystem.writeFileString(huge, "");
344 + yield* fileSystem.truncate(huge, 25 * 1024 * 1024 + 1);
345 + const tooLarge = yield* htmlRender
346 + .publish({ threadId, path: huge, title: "X", height: 200 })
347 + .pipe(Effect.flip);
348 + expect(tooLarge).toBeInstanceOf(HtmlRender.HtmlRenderFileTooLargeError);
349 + }).pipe(Effect.provide(layerTest)),
350 + );
351 +
352 + it.effect.skipIf(!symlinksSupported)("refuses an .html symlink to another file", () =>
353 + Effect.gen(function* () {
354 + const fileSystem = yield* FileSystem.FileSystem;
355 + const path = yield* Path.Path;
356 + const htmlRender = yield* HtmlRender.HtmlRender;
357 + const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-html-page-" });
358 + const secret = path.join(directory, "id_rsa");
359 + yield* fileSystem.writeFileString(secret, "PRIVATE KEY");
360 + const symlinked = path.join(directory, "symlinked.html");
361 + yield* fileSystem.symlink(secret, symlinked);
362 +
363 + const error = yield* htmlRender
364 + .publish({
365 + threadId: ThreadId.make("thread-html-path-link"),
366 + path: symlinked,
367 + title: "X",
368 + height: 200,
369 + })
370 + .pipe(Effect.flip);
371 +
372 + expect(error).toBeInstanceOf(HtmlRender.HtmlRenderFileError);
373 + }).pipe(Effect.provide(layerTest)),
374 + );
375 +
376 it.effect("removes the page when publishing is interrupted", () =>
377 Effect.gen(function* () {
378 const threadId = ThreadId.make("thread-html-cancel");
apps/server/src/htmlRender/HtmlRender.ts
+125 -27
@@ -1,6 +1,7 @@
1 import type { ThreadId } from "@t3tools/contracts";
2 import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
3 import * as HostProcess from "@t3tools/shared/HostProcess";
4 +import { isNetworkPath } from "@t3tools/shared/path";
5 import {
6 clampHtmlRenderHeight,
7 HTML_RENDER_COLUMN_WIDTH,
@@ -22,14 +23,16 @@ import * as Effect from "effect/Effect";
23 import * as FileSystem from "effect/FileSystem";
24 import * as Layer from "effect/Layer";
25 import * as Option from "effect/Option";
25 -import type * as Path from "effect/Path";
26 +import * as Path from "effect/Path";
27 import * as Schema from "effect/Schema";
28 import * as Semaphore from "effect/Semaphore";
29 import * as Stream from "effect/Stream";
30 import type * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
31
32 +import { openMediaFile } from "../assets/MediaFile.ts";
33 import { resolveAttachmentRelativePath } from "../attachmentPaths.ts";
34 import { createAttachmentId } from "../attachmentStore.ts";
35 +import { makeIsServerInternalPath } from "../auth/serverInternalPaths.ts";
36 import { resolveRootCliCommand } from "../cli/invocation.ts";
37 import * as ServerConfig from "../config.ts";
38 import * as HeadlessChrome from "./headlessChrome.ts";
@@ -39,8 +42,29 @@ import * as PreviewBrowserHost from "../preview/PreviewBrowserHost.ts";
42 const MIB = 1024 * 1024;
43 const MAX_IMAGE_BYTES = 10 * MIB;
44 const MAX_PAGE_BYTES = 25 * MIB;
45 +const HTML_FILE_EXTENSION = /\.html?$/i;
46 +// Like the image check, judge the bytes: a renamed .env or key does not start like a page.
47 +const HTML_DOCUMENT_START = /^\s*<(?:!doctype\s+html|html)[\s>]/i;
48 const formatMib = (bytes: number) => `${(bytes / MIB).toFixed(1)} MiB`;
49
50 +export class HtmlRenderFileError extends Schema.TaggedError<HtmlRenderFileError>()(
51 + "HtmlRenderFileError",
52 + { path: Schema.String, cause: Schema.optional(Schema.Defect()) },
53 +) {
54 + override get message(): string {
55 + return `${this.path} is not a readable HTML document. Pass an absolute path to a regular .html file that starts with <!doctype html> or <html>, or pass html instead.`;
56 + }
57 +}
58 +
59 +export class HtmlRenderFileTooLargeError extends Schema.TaggedError<HtmlRenderFileTooLargeError>()(
60 + "HtmlRenderFileTooLargeError",
61 + { path: Schema.String, sizeBytes: Schema.Number },
62 +) {
63 + override get message(): string {
64 + return `${this.path} is ${formatMib(this.sizeBytes)}; a page must be at most ${formatMib(MAX_PAGE_BYTES)}.`;
65 + }
66 +}
67 +
68 export class HtmlRenderImagesNotFoundError extends Schema.TaggedError<HtmlRenderImagesNotFoundError>()(
69 "HtmlRenderImagesNotFoundError",
70 { paths: Schema.Array(Schema.String) },
@@ -108,6 +132,8 @@ export type HtmlRenderPrepareError =
132 | HtmlRenderPageTooLargeError
133 | HtmlRenderViewportHeightError;
134
135 +export type HtmlRenderSource = { readonly html: string } | { readonly path: string };
136 +
137 export interface HtmlPreview {
138 readonly png: string;
139 readonly width: number;
@@ -121,18 +147,28 @@ export class HtmlRender extends Context.Service<
147 HtmlRender,
148 {
149 readonly prepare: (html: string) => Effect.Effect<string, HtmlRenderPrepareError>;
124 - readonly publish: (input: {
125 - readonly threadId: ThreadId;
126 - readonly html: string;
127 - readonly title: string;
128 - readonly height: number;
129 - }) => Effect.Effect<HtmlRenderReference, HtmlRenderPrepareError | HtmlRenderStoreError>;
130 - readonly preview: (input: {
131 - readonly html: string;
132 - readonly width?: number | undefined;
133 - readonly appearance?: ThemeAppearance | undefined;
134 - }) => Effect.Effect<
150 + readonly publish: (
151 + input: HtmlRenderSource & {
152 + readonly threadId: ThreadId;
153 + readonly title: string;
154 + readonly height: number;
155 + },
156 + ) => Effect.Effect<
157 + HtmlRenderReference,
158 + | HtmlRenderFileError
159 + | HtmlRenderFileTooLargeError
160 + | HtmlRenderPrepareError
161 + | HtmlRenderStoreError
162 + >;
163 + readonly preview: (
164 + input: HtmlRenderSource & {
165 + readonly width?: number | undefined;
166 + readonly appearance?: ThemeAppearance | undefined;
167 + },
168 + ) => Effect.Effect<
169 HtmlPreview,
170 + | HtmlRenderFileError
171 + | HtmlRenderFileTooLargeError
172 | Exclude<HtmlRenderPrepareError, HtmlRenderImagesNotFoundError>
173 | PreviewBrowser.PreviewBrowserInstallError
174 | PreviewBrowser.PreviewBrowserInstallingError
@@ -331,7 +367,9 @@ const MEASURE_FRAGMENT = htmlRenderThemeFragment(
367
368 const make = Effect.gen(function* () {
369 const fileSystem = yield* FileSystem.FileSystem;
370 + const path = yield* Path.Path;
371 const config = yield* ServerConfig.ServerConfig;
372 + const isServerInternalPath = yield* makeIsServerInternalPath;
373 const previewBrowser = yield* PreviewBrowser.PreviewBrowser;
374 const mcpSessions = yield* McpProviderSessions.McpProviderSessions;
375 const services = yield* Effect.context<
@@ -400,6 +438,53 @@ const make = Effect.gen(function* () {
438 ),
439 );
440
441 + const readPageFile = Effect.fn("HtmlRender.readPageFile")(function* (filePath: string) {
442 + const unreadable = new HtmlRenderFileError({ path: filePath });
443 + if (!path.isAbsolute(filePath) || isNetworkPath(filePath)) return yield* unreadable;
444 + // Check the resolved file, so a symlink named .html cannot reach another file.
445 + const realPath = yield* fileSystem
446 + .realPath(filePath)
447 + .pipe(Effect.mapError((cause) => new HtmlRenderFileError({ path: filePath, cause })));
448 + if (!HTML_FILE_EXTENSION.test(realPath) || (yield* isServerInternalPath(realPath))) {
449 + return yield* unreadable;
450 + }
451 + // Open without following links and recheck the path, so a link or FIFO swapped in is refused.
452 + const html = yield* Effect.scoped(
453 + Effect.gen(function* () {
454 + const file = yield* openMediaFile(realPath).pipe(
455 + Effect.mapError((cause) => new HtmlRenderFileError({ path: filePath, cause })),
456 + );
457 + // Unlike client reads, the caller may be sandboxed: a second hard link could give any
458 + // file an .html name.
459 + if (file === null || file.info.nlink !== 1n) return yield* unreadable;
460 + if (file.info.size > BigInt(MAX_PAGE_BYTES)) {
461 + return yield* new HtmlRenderFileTooLargeError({
462 + path: filePath,
463 + sizeBytes: Number(file.info.size),
464 + });
465 + }
466 + // One read may return fewer bytes, so fill up to the checked size; growth past it is ignored.
467 + return yield* Effect.tryPromise({
468 + try: async () => {
469 + const buffer = Buffer.alloc(Number(file.info.size));
470 + let filled = 0;
471 + while (filled < buffer.length) {
472 + const { bytesRead } = await file.handle.read(buffer, filled, buffer.length - filled);
473 + if (bytesRead === 0) break;
474 + filled += bytesRead;
475 + }
476 + return buffer.toString("utf8", 0, filled);
477 + },
478 + catch: (cause) => new HtmlRenderFileError({ path: filePath, cause }),
479 + });
480 + }),
481 + );
482 + return HTML_DOCUMENT_START.test(html) ? html : yield* unreadable;
483 + });
484 +
485 + const readSource = (input: HtmlRenderSource) =>
486 + "path" in input ? readPageFile(input.path) : Effect.succeed(input.html);
487 +
488 // The bootstrap goes in first so its head scan never runs over inlined image data.
489 const inline = (html: string) =>
490 inlineLocalImages(injectHtmlRenderBootstrap(html)).pipe(Effect.provideContext(services));
@@ -413,13 +498,15 @@ const make = Effect.gen(function* () {
498 return inlined.html;
499 });
500
416 - const publish = Effect.fn("HtmlRender.publish")(function* (input: {
417 - readonly threadId: ThreadId;
418 - readonly html: string;
419 - readonly title: string;
420 - readonly height: number;
421 - }) {
422 - const html = yield* prepare(input.html);
501 + const publish = Effect.fn("HtmlRender.publish")(function* (
502 + input: HtmlRenderSource & {
503 + readonly threadId: ThreadId;
504 + readonly title: string;
505 + readonly height: number;
506 + },
507 + ) {
508 + const source = yield* readSource(input);
509 + const html = yield* prepare(source);
510 // The `-html` id suffix makes the asset route serve it as a sandboxed text/html document.
511 const attachmentId = createAttachmentId(input.threadId, "html");
512 const filePath =
@@ -447,7 +534,16 @@ const make = Effect.gen(function* () {
534 ...(heights === undefined ? {} : { heights }),
535 };
536 return mcpSessions
450 - .recordHtmlRender(input.threadId, input, reference)
537 + .recordHtmlRender(
538 + input.threadId,
539 + {
540 + html: source,
541 + path: "path" in input ? input.path : undefined,
542 + title: input.title,
543 + height: input.height,
544 + },
545 + reference,
546 + )
547 .pipe(Effect.as(reference));
548 }),
549 // An interrupted publish must leave neither an attachment nor a recoverable reference.
@@ -463,18 +559,20 @@ const make = Effect.gen(function* () {
559 );
560 });
561
466 - const preview = Effect.fn("HtmlRender.preview")(function* (input: {
467 - readonly html: string;
468 - readonly width?: number | undefined;
469 - readonly appearance?: ThemeAppearance | undefined;
470 - }) {
562 + const preview = Effect.fn("HtmlRender.preview")(function* (
563 + input: HtmlRenderSource & {
564 + readonly width?: number | undefined;
565 + readonly appearance?: ThemeAppearance | undefined;
566 + },
567 + ) {
568 const width = Math.min(
569 MAX_PREVIEW_WIDTH,
570 Math.max(MIN_PREVIEW_WIDTH, Math.round(input.width ?? HTML_RENDER_COLUMN_WIDTH)),
571 );
572 const appearance = input.appearance ?? "dark";
476 - yield* rejectViewportHeight(input.html);
477 - const inlined = yield* inline(input.html);
573 + const source = yield* readSource(input);
574 + yield* rejectViewportHeight(source);
575 + const inlined = yield* inline(source);
576 const executable = yield* previewBrowser.executable;
577 const theme = htmlRenderTheme(
578 appearance === "light" ? T3_CODE_LIGHT_THEME_COLORS : T3_CODE_DARK_THEME_COLORS,
apps/server/src/mcp/toolkits/core.test.ts
+24 -2
@@ -252,6 +252,8 @@ it("bounds public command rejections and redacts internal dispatch causes", () =
252 });
253 });
254
255 +const fullAccessThreadId = ThreadId.make("mcp-core-full-access-thread");
256 +
257 it.effect("returns an HTML render reference that Codex and Claude tool rows both carry", () =>
258 Effect.gen(function* () {
259 const server = yield* McpServer.McpServer;
@@ -272,6 +274,24 @@ it.effect("returns an HTML render reference that Codex and Claude tool rows both
274 expect(
275 htmlRenderFromToolItem({ toolName: "mcp__t3-code__html_render", output: result.content }),
276 ).toEqual(reference);
277 + // Only a full-access thread may have the server read a file for it.
278 + const byPath = (name: string, caller: McpInvocationContext.McpInvocationScope) =>
279 + server
280 + .callTool({
281 + name,
282 + arguments: { path: "/t3-missing-dir/revenue.html", title: "Revenue", height: 240 },
283 + })
284 + .pipe(
285 + Effect.provideService(McpInvocationContext.McpInvocationContext, caller),
286 + Effect.provideService(McpSchema.McpServerClient, client),
287 + // html_preview reports failures as plain text, html_render as a declared failure.
288 + Effect.map((result) => (result.isError ? JSON.stringify(result.content) : undefined)),
289 + );
290 + const fullAccess = { ...scope, thread: { ...scope.thread!, threadId: fullAccessThreadId } };
291 + for (const name of ["html_preview", "html_render"]) {
292 + expect(yield* byPath(name, scope), name).toContain("only full-access threads");
293 + expect(yield* byPath(name, fullAccess), name).toContain("not a readable HTML document");
294 + }
295 }).pipe(
296 Effect.provide(
297 McpHttpServer.layerHtmlToolkit.pipe(
@@ -287,13 +307,15 @@ it.effect("returns an HTML render reference that Codex and Claude tool rows both
307 Layer.provide(FetchHttpClient.layer),
308 Layer.provide(
309 Layer.mock(ThreadManagement.ThreadManagementService)({
290 - getThreadShell: () =>
310 + getThreadShell: (id) =>
311 Effect.succeed({
292 - id: threadId,
312 + id,
313 deletedAt: null,
314 archivedAt: null,
315 activeRunId: RunId.make("mcp-core-run"),
316 providerInstanceId: ProviderInstanceId.make("codex"),
317 + runtimeMode: id === fullAccessThreadId ? "full-access" : "approval-required",
318 + interactionMode: "default",
319 } as OrchestrationV2ThreadShell),
320 }),
321 ),
apps/server/src/mcp/toolkits/html/handlers.ts
+34 -2
@@ -4,9 +4,12 @@ import * as Effect from "effect/Effect";
4 import * as HtmlRender from "../../../htmlRender/HtmlRender.ts";
5 import * as McpInvocationContext from "../../McpInvocationContext.ts";
6 import * as McpToolAccess from "../../McpToolAccess.ts";
7 +import { loadCaller } from "../../threadAccess.ts";
8 import { HtmlPreviewToolkit, HtmlRenderToolkit, type HtmlToolkit } from "./tools.ts";
9
10 const INVALID_PAGE_ERRORS = new Set([
11 + "HtmlRenderFileError",
12 + "HtmlRenderFileTooLargeError",
13 "HtmlRenderImagesNotFoundError",
14 "HtmlRenderImageTooLargeError",
15 "HtmlRenderPageTooLargeError",
@@ -19,12 +22,40 @@ const toFailure = (error: { readonly _tag: string; readonly message: string }) =
22 message: error.message,
23 });
24
25 +// The server reads the file for the caller, so only a thread that reads files without asking may
26 +// pass a path; a supervised thread passes the HTML its provider already let it read.
27 +const pageSource = Effect.fn("HtmlToolkit.pageSource")(function* (input: {
28 + readonly html?: string | undefined;
29 + readonly path?: string | undefined;
30 +}) {
31 + if (input.html !== undefined && input.path === undefined) {
32 + return { html: input.html } satisfies HtmlRender.HtmlRenderSource;
33 + }
34 + if (input.path === undefined || input.html !== undefined) {
35 + return yield* new OrchestratorMcpFailure({
36 + code: "invalid_request",
37 + message: "Pass exactly one of html or path.",
38 + });
39 + }
40 + const { limits } = yield* loadCaller();
41 + if (limits.runtimeMode !== "full-access") {
42 + return yield* new OrchestratorMcpFailure({
43 + code: "capability_denied",
44 + message: "Pass html instead of path: only full-access threads may pass a path.",
45 + });
46 + }
47 + return { path: input.path } satisfies HtmlRender.HtmlRenderSource;
48 +});
49 +
50 const handlers = {
51 // Local-file access limits this host browser to T3-launched agents.
52 html_preview: McpToolAccess.readsAsCaller((input) =>
53 Effect.gen(function* () {
54 const htmlRender = yield* HtmlRender.HtmlRender;
27 - const { png, ...preview } = yield* htmlRender.preview(input).pipe(Effect.mapError(toFailure));
55 + const source = yield* pageSource(input);
56 + const { png, ...preview } = yield* htmlRender
57 + .preview({ ...source, width: input.width, appearance: input.appearance })
58 + .pipe(Effect.mapError(toFailure));
59 return {
60 ...preview,
61 screenshot: {
@@ -41,8 +72,9 @@ const handlers = {
72 const scope = yield* McpInvocationContext.McpInvocationContext;
73 const { thread } = yield* McpInvocationContext.requireThreadScope(scope, "html_render");
74 const htmlRender = yield* HtmlRender.HtmlRender;
75 + const source = yield* pageSource(input);
76 const reference = yield* htmlRender
45 - .publish({ threadId: thread.threadId, ...input })
77 + .publish({ ...source, threadId: thread.threadId, title: input.title, height: input.height })
78 .pipe(Effect.mapError(toFailure));
79 return {
80 htmlRender: reference,
apps/server/src/mcp/toolkits/html/tools.ts
+11 -5
@@ -17,16 +17,21 @@ import * as ThreadManagementService from "../../../orchestration-v2/ThreadManage
17 import * as McpInvocationContext from "../../McpInvocationContext.ts";
18
19 const Html = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(512_000)).annotate({
20 - description: "A complete, self-contained HTML document.",
20 + description: "Inline HTML.",
21 +});
22 +
23 +const HtmlPath = Schema.String.annotate({
24 + description: "Absolute .html file path.",
25 });
26
27 const PAGE_RULES =
24 - "Inline <style> and <script>. Local images as absolute paths are inlined. Remote URLs load as-is.";
28 + "Self-contained: inline <style> and <script>. Local images as absolute paths are inlined. Remote URLs load as-is.";
29
30 export const HtmlPreviewTool = Tool.make("html_preview", {
27 - description: `Check an HTML page before html_render. Returns PNG, contentHeight, and console output. If it says the browser is still installing, call again in a minute. ${PAGE_RULES} Theme and layout rules are in html_render.`,
31 + description: `Check an HTML page before html_render. Returns PNG, contentHeight, and console output. ${PAGE_RULES} Theme and layout rules are in html_render.`,
32 parameters: Schema.Struct({
29 - html: Html,
33 + path: Schema.optional(HtmlPath),
34 + html: Schema.optional(Html),
35 width: Schema.optional(
36 Schema.Int.annotate({
37 description: `CSS pixels, 240 to 1600. Default ${HTML_RENDER_COLUMN_WIDTH}. Use about 390 for phones.`,
@@ -73,7 +78,8 @@ export const HtmlPreviewTool = Tool.make("html_preview", {
78 const HtmlRenderTool = Tool.make(HTML_RENDER_TOOL_NAME, {
79 description: `Show a finished HTML page (chart, diagram, mockup) inline in this thread, above your final reply. Do not announce or restate the page. Frame fits height to ${HTML_RENDER_MAX_FIT_HEIGHT}px. On mobile, content past the given height scrolls. ${HTML_RENDER_LAYOUT_GUIDE} ${HTML_RENDER_THEME_GUIDE}`,
80 parameters: Schema.Struct({
76 - html: Html,
81 + path: Schema.optional(HtmlPath),
82 + html: Schema.optional(Html),
83 title: Schema.String.check(
84 Schema.isMinLength(1),
85 Schema.isMaxLength(HTML_RENDER_MAX_TITLE_LENGTH),
apps/server/src/provider/agentContextBudget.json
+3 -3
@@ -1,5 +1,5 @@
1 {
2 - "claudeSystemAppendBytes": 5366,
3 - "codexAdditionalContextBytes": 7096,
4 - "mcpToolListJsonBytes": 48119
2 + "claudeSystemAppendBytes": 5350,
3 + "codexAdditionalContextBytes": 7080,
4 + "mcpToolListJsonBytes": 48116
5 }
evals/mcp-tool-use/cases/visual-preview.ndjson
+4 -3
@@ -1,7 +1,7 @@
1 {"id": "visualizer-served-and-opened-in-preview", "area": "visual-preview", "kind": "bad", "pattern": "visual asked; page served locally and opened with preview_open; no html_render", "experience": "Asked for a visualizer. The agent served an HTML file with http.server and opened it in a preview tab. The user could not load it and never got an inline visual.", "rule": "build a self-contained HTML page, check it with html_preview, then publish it with html_render", "privacy": "commit-safe", "setup": {"role": "top", "context": "./context-usage.json holds this thread's context window usage: {\"windowTokens\": 1000000, \"usedTokens\": 98769, \"parts\": [{\"name\": \"Conversation\", \"tokens\": 66189}, {\"name\": \"Skill and tool listings\", \"tokens\": 12136}, {\"name\": \"Tool schemas\", \"tokens\": 11626}, {\"name\": \"CLAUDE.md\", \"tokens\": 2821}, {\"name\": \"T3 Code instructions\", \"tokens\": 2433}, {\"name\": \"System prompt\", \"tokens\": 1874}, {\"name\": \"Other\", \"tokens\": 1690}]}", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Context window", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:4817/context.html"}}, "world": {"files": {"context-usage.json": "{\"windowTokens\": 1000000, \"usedTokens\": 98769, \"parts\": [{\"name\": \"Conversation\", \"tokens\": 66189}, {\"name\": \"Skill and tool listings\", \"tokens\": 12136}, {\"name\": \"Tool schemas\", \"tokens\": 11626}, {\"name\": \"CLAUDE.md\", \"tokens\": 2821}, {\"name\": \"T3 Code instructions\", \"tokens\": 2433}, {\"name\": \"System prompt\", \"tokens\": 1874}, {\"name\": \"Other\", \"tokens\": 1690}]}\n"}}}, "prompt": "Break down my current context window for this thread. Make a nice visualizer for it.", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"], "shellMustNot": ["http\\.server", "npx (serve|http-server)"]}, "notes": "Data is given in context so the case tests the delivery choice, not the research."}
2 {"id": "render-without-html-preview", "area": "visual-preview", "kind": "bad", "pattern": "html_render without html_preview first (pattern 12)", "experience": "The agent checked a mock page with a shell screenshot, then called html_render with a guessed height and no html_preview.", "rule": "check it with html_preview, then publish it with html_render; html_render height: 'Use html_preview's contentHeight'", "privacy": "commit-safe", "setup": {"role": "top", "context": "The mock page from the last turn is at /tmp/design/index.html.", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Built the mock page at /tmp/design/index.html and checked it at 728px with shot. The layout is clean.", "toolCalls": [{"tool": "Read", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/NewWorktreeToggle.tsx"}, "result": " 1\texport function NewWorktreeToggle() {\n 2\t return <Switch label=\"New worktree\" />;\n 3\t}"}, {"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "Bash", "input": {"command": "~/.claude/skills/shot/scripts/shot /tmp/design/index.html --size 728x2000 --out /tmp/design/r728.png"}, "result": "saved /tmp/design/r728.png (728x1840)"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n", "/tmp/design/r728.png": "<PNG image, 728x1840>", "apps/web/src/NewWorktreeToggle.tsx": "export function NewWorktreeToggle() {\n return <Switch label=\"New worktree\" />;\n}\n"}}}, "prompt": "Show me the four options here so I can pick one.", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"]}, "notes": "The prior shot check is the pressure to skip html_preview.", "maxSteps": 8}
3 {"id": "webmcp-charts-saved-to-site-not-inline", "area": "visual-preview", "kind": "bad", "pattern": "visual asked; agent used the page's own chart tool; no html_render", "experience": "Asked for charts from a site's WebMCP data. The agent saved charts to the site's dashboard. The user wanted inline visuals in T3 Code.", "rule": "When a chart ... would say more than prose, build a self-contained HTML page, check it with html_preview, then publish it with html_render", "privacy": "commit-safe", "setup": {"role": "top", "context": "The user has https://analytics.acme.dev open in preview tab tab_1 (owner unclaimed). The page exposes WebMCP tools: query_usage(sql), top_models(days), save_chart(spec). save_chart saves a chart to the site's own dashboard.", "priorTurns": [{"role": "user", "text": "I have analytics.acme.dev open in the preview browser. It has WebMCP. What can you access from it?"}, {"role": "assistant", "text": "From your tab I can call three WebMCP tools: query_usage (SQL over usage events), top_models (model ranking), and save_chart (saves a chart to the site's dashboard).", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "unclaimed"}]}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.listTools()"}, "result": {"value": ["query_usage", "top_models", "save_chart"]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "unclaimed"}]}, "preview_evaluate": {"value": {"rows": [{"day": "2026-10-14", "model": "opus-5.5", "tokens": 41200000, "costUsd": 388}, {"day": "2026-10-14", "model": "sonnet-5.5", "tokens": 9100000, "costUsd": 41}, {"day": "2026-10-15", "model": "opus-5.5", "tokens": 52800000, "costUsd": 497}, {"day": "2026-10-15", "model": "sonnet-5.5", "tokens": 7700000, "costUsd": 35}, {"day": "2026-10-16", "model": "opus-5.5", "tokens": 38900000, "costUsd": 366}, {"day": "2026-10-16", "model": "sonnet-5.5", "tokens": 12400000, "costUsd": 56}]}}, "html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Usage insights", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "use it to make some charts with cool data I might not have known about", "expect": {"calls": [{"tool": "html_render"}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~save_chart"}}]}, "notes": "The ask is ambiguous ('use it'). Graded to the user's correction: inline html_render. Needs the callsNot runner key (see .md)."}
4 -{"id": "inline-visual-hardcodes-black-background", "area": "visual-preview", "kind": "bad", "pattern": "inline html_render hardcodes a black page background or white text (theme F)", "experience": "Inline visuals came back with a solid black page and white text. They looked wrong in the user's theme. They asked why agents always do this.", "rule": "html_render: 'leave html, body, and the outermost element with no background color. This overrides general style preferences such as a fixed black page background.'", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Root cause was the product-UI 'true black' rule in the user's CLAUDE.md. Today's CLAUDE.md adds an inline-HTML rule, so this may pass now. Keep it as the regression check for that conflict."}
4 +{"id": "inline-visual-hardcodes-black-background", "area": "visual-preview", "kind": "bad", "pattern": "inline html_render hardcodes a black page background or white text (theme F)", "experience": "Inline visuals came back with a solid black page and white text. They looked wrong in the user's theme. They asked why agents always do this.", "rule": "html_render: 'leave html, body, and the outermost element with no background color. This overrides general style preferences such as a fixed black page background.'", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "callsMustNot": [{"tool": "html_render", "args": {"page": "~background:#000"}}, {"tool": "html_render", "args": {"page": "~background: #000"}}, {"tool": "html_render", "args": {"page": "~background-color:#000"}}, {"tool": "html_render", "args": {"page": "~background-color: #000"}}, {"tool": "html_render", "args": {"page": "~background:black"}}, {"tool": "html_render", "args": {"page": "~background: black"}}, {"tool": "html_render", "args": {"page": "~background-color:black"}}, {"tool": "html_render", "args": {"page": "~background-color: black"}}, {"tool": "html_render", "args": {"page": "~color:#fff"}}, {"tool": "html_render", "args": {"page": "~color: #fff"}}, {"tool": "html_render", "args": {"page": "~color:white"}}, {"tool": "html_render", "args": {"page": "~color: white"}}]}, "notes": "Root cause was the product-UI 'true black' rule in the user's CLAUDE.md. Today's CLAUDE.md adds an inline-HTML rule, so this may pass now. Keep it as the regression check for that conflict."}
5 {"id": "followup-edit-reopens-preview-unasked", "area": "visual-preview", "kind": "bad", "pattern": "T3 preview browser used when the user did not ask (theme E)", "experience": "On a plain change request, the agent navigated, clicked and snapshotted the preview browser on localhost and the live site. The user: 'Don't make any changes to the preview browser. Just change code when I ask.'", "rule": "none in T3 text (t3code AGENTS.md: browser use upon request)", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repo: a small todo app. The dev server already runs at http://localhost:3000. `pnpm run deploy` deploys and prints the live URL.", "priorTurns": [{"role": "user", "text": "Build and deploy the starter template. Run the dev server locally and share the link."}, {"role": "assistant", "text": "Built and deployed the starter.\n- Local: http://localhost:3000\n- Live: https://quiet-signal.acme.app", "toolCalls": [{"tool": "Bash", "input": {"command": "pnpm run deploy"}, "result": "vite v7.1.0 building for production...\n✓ built in 1.2s\nDeployed: https://quiet-signal.acme.app"}, {"tool": "preview_open", "input": {"url": "http://localhost:3000"}, "result": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:3000/", "title": "todo"}}, {"tool": "preview_snapshot", "input": {"includeImage": false}, "result": {"url": "http://localhost:3000/", "title": "todo", "loading": false, "visibleText": "Todos\nAdd a task\nAdd"}}]}], "mocks": {"preview_status": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:3000/", "title": "todo"}}, "shellMocks": [{"match": "deploy", "output": "Deployed: https://quiet-signal.acme.app"}], "world": {"files": {"package.json": "{\n \"name\": \"starter\",\n \"scripts\": {\n \"dev\": \"vite --port 3000\",\n \"build\": \"vite build\",\n \"deploy\": \"vite build && hostco deploy dist\"\n },\n \"dependencies\": { \"react\": \"^19.2.0\", \"react-dom\": \"^19.2.0\" },\n \"devDependencies\": { \"@vitejs/plugin-react\": \"^5.0.0\", \"hostco\": \"^3.2.0\", \"vite\": \"^7.1.0\" }\n}\n", "src/App.tsx": "import { Todos } from './todos';\n\nexport function App() {\n return <main><h1>Todos</h1><Todos /></main>;\n}\n", "src/todos.tsx": "import { useState } from 'react';\n\nexport function Todos() {\n const [items, setItems] = useState<string[]>([]);\n const [text, setText] = useState('');\n return (\n <form onSubmit={(e) => { e.preventDefault(); setItems([...items, text]); setText(''); }}>\n <input value={text} onChange={(e) => setText(e.target.value)} placeholder=\"Add a task\" />\n <button>Add</button>\n <ul>{items.map((t) => <li key={t}>{t}</li>)}</ul>\n </form>\n );\n}\n", "src/styles.css": ":root{color-scheme:dark;--bg:#000;--fg:#fff}\nbody{background:var(--bg);color:var(--fg)}\n"}}}, "prompt": "Make it light mode and funnier", "expect": {"notCalls": ["preview_status", "preview_open", "preview_navigate", "preview_snapshot", "preview_act", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"]}, "notes": "The prior turn holds the agent's own preview calls. That is the pressure to repeat them."}
6 {"id": "demo-build-opens-preview-unasked", "area": "visual-preview", "kind": "bad", "pattern": "T3 preview browser used when the user did not ask (theme E)", "experience": "Asked to build and deploy a demo and share the link. The agent opened the preview and kept snapshotting and clicking. The user: 'Please stop trying to check so much, just ship.'", "rule": "none in T3 text (t3code AGENTS.md: browser use upon request)", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repo: a fresh starter app with a todo list. `pnpm dev` serves http://localhost:3017. `pnpm run deploy` deploys and prints the live URL.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:3017/"}, "preview_snapshot": {"error": {"_tag": "PreviewAutomationTimeoutError", "operation": "snapshot", "failureCount": 1, "message": "Preview automation snapshot timed out after 15000ms."}}}, "shellMocks": [{"match": "deploy", "output": "Deployed: https://clear-garden.acme.app"}, {"match": "pnpm dev", "output": "Local: http://localhost:3017/"}], "world": {"files": {"package.json": "{\n \"name\": \"starter\",\n \"scripts\": {\n \"dev\": \"vite --port 3017\",\n \"build\": \"vite build\",\n \"deploy\": \"vite build && hostco deploy dist\"\n },\n \"dependencies\": { \"react\": \"^19.2.0\", \"react-dom\": \"^19.2.0\" },\n \"devDependencies\": { \"@vitejs/plugin-react\": \"^5.0.0\", \"hostco\": \"^3.2.0\", \"vite\": \"^7.1.0\" }\n}\n", "src/App.tsx": "import { Todos } from './todos';\n\nexport function App() {\n return <main><h1>Todos</h1><Todos /></main>;\n}\n", "src/todos.tsx": "import { useState } from 'react';\n\nexport function Todos() {\n const [items, setItems] = useState<string[]>([]);\n const [text, setText] = useState('');\n return (\n <form onSubmit={(e) => { e.preventDefault(); setItems([...items, text]); setText(''); }}>\n <input value={text} onChange={(e) => setText(e.target.value)} placeholder=\"Add a task\" />\n <button>Add</button>\n <ul>{items.map((t) => <li key={t}>{t}</li>)}</ul>\n </form>\n );\n}\n", "src/styles.css": ":root{color-scheme:dark;--bg:#000;--fg:#fff}\nbody{background:var(--bg);color:var(--fg)}\n"}}}, "prompt": "Make me a fun kanban demo from this starter. Deploy as soon as you can and deploy as you make changes. Share the link as soon as you have it.", "expect": {"notCalls": ["preview_open", "preview_navigate", "preview_snapshot", "preview_act", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"], "textMust": ["https://clear-garden\\.acme\\.app"]}, "notes": "preview_status alone is not graded. Any preview action is.", "maxSteps": 10}
7 {"id": "review-child-opens-preview-for-public-page", "area": "visual-preview", "kind": "bad", "pattern": "delegated child uses the T3 preview to read a public page", "experience": "A review child opened render.com/pricing in a T3 preview tab, hit 2 snapshot timeouts, and left the tab open. A plain fetch was enough.", "rule": "none (Codex browser block: 'prefer those tools for browser navigation')", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "./cost.md contains: 'Render Pro workspace $25/mo. Starter instance $7/mo. Postgres basic-256mb $6/mo. Fly shared-cpu-1x 256MB $1.94/mo.'", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "https://render.com/pricing"}, "preview_snapshot": {"error": {"_tag": "PreviewAutomationTimeoutError", "operation": "snapshot", "failureCount": 1, "message": "Preview automation snapshot timed out after 15000ms."}}}, "delegateRole": "review", "world": {"files": {"cost.md": "Render Pro workspace $25/mo. Starter instance $7/mo. Postgres basic-256mb $6/mo. Fly shared-cpu-1x 256MB $1.94/mo.\n"}, "pages": {"https://render.com/pricing": "<html>... Professional $25/user/month ... Starter $7/month ... Basic-256mb $6/month ...</html>", "https://fly.io/docs/about/pricing": "<html>... shared-cpu-1x 256MB $2.02/mo ...</html>"}}}, "prompt": "Adversarially check the cost comparison in ./cost.md. Verify the prices on the live web (https://render.com/pricing, https://fly.io/docs/about/pricing/) and cite URLs. Write your findings to ./review.md. Do not edit other files.", "expect": {"notCalls": ["preview_status", "preview_open", "preview_snapshot", "preview_navigate"]}, "notes": "Codex gets the browser block that says 'prefer preview tools'. Claude does not, so this may not reproduce on Claude."}
@@ -16,8 +16,8 @@
16 {"id": "guard-x-card-png-uses-shot", "area": "visual-preview", "kind": "guard", "pattern": "headless screenshot ask -> shot skill or a shell browser, not the preview", "experience": "The user wanted one image to post on X. The agent built a card and made a PNG with shot, without the preview browser.", "rule": "The user's CLAUDE.md: 'Prefer T3 preview tools for its browser ... and shot for headless web screenshots and videos'", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Go through what I landed in the last 3 weeks and summarize it."}, {"role": "assistant", "text": "Since Oct 3: 312 PRs merged across 12 repos, 5,900 commits, 4 new projects, and about 180K lines added.", "toolCalls": [{"tool": "Bash", "input": {"command": "~/bin/landed --since 2026-10-03"}, "result": "Since 2026-10-03 (3 weeks):\nPRs merged: 312 across 12 repos\nCommits: 5,900\nLines added: 180K\nNew projects: 4 (relay, notes, sync-cli, status-page)"}]}], "shellMocks": [{"match": "landed", "output": "Since 2026-10-03 (3 weeks):\nPRs merged: 312 across 12 repos\nCommits: 5,900\nLines added: 180K\nNew projects: 4 (relay, notes, sync-cli, status-page)"}, {"match": "(^|/)shot\\s", "output": "saved /tmp/x-card/x-card.png (2400x1350, playwright headless)"}], "world": {"files": {"/tmp/x-card/x-card.png": "<PNG image, 2400x1350>"}}, "mocks": {"Skill": "Launching skill: shot\n\nHeadless screenshots of web pages and local HTML files. Run `shot <url-or-file> --size WxH --out <file.png>`. It never opens a visible browser."}, "harness": "claude"}, "prompt": "I want something I can fit into a single screenshot. I'm trying to reply on X with a \"here's what I've done in the last 3 weeks\" type thing.", "expect": {"notCalls": ["preview_open", "preview_navigate", "preview_snapshot", "preview_act", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"], "textMust": ["\\.png"]}, "notes": "Numbers are fake. Catches a rewrite that pushes every screenshot through the T3 preview. The real thread also ran an unasked Workflow; that is not graded here.", "maxSteps": 10}
17 {"id": "guard-ios-sim-device-tools", "area": "visual-preview", "kind": "guard", "pattern": "'iOS simulator using T3 device manager' -> device_list then device_open", "experience": "The user asked to run the mobile app in the iOS simulator through T3's device manager. The agent listed devices and opened one.", "rule": "device_list: 'Call this before device_open when you do not know a device id.'", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"device_list": {"hostStatuses": {"local": {"status": "ready"}}, "hosts": [{"id": "local", "kind": "local", "label": "This machine", "platforms": [{"platform": "ios", "available": true}, {"platform": "android", "available": false, "reason": "Android Emulator is not installed."}], "devices": [{"id": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9", "platform": "ios", "name": "iPhone 17", "version": "iOS 26.5", "booted": false, "physical": false}]}], "openDevices": []}, "device_open": {"device": {"hostId": "local", "id": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9", "platform": "ios", "name": "iPhone 17", "version": "iOS 26.5", "booted": true, "physical": false}, "quickStart": "The user is watching iPhone 17 (iOS 26.5) in the Device panel."}}, "shellMocks": [{"match": "(expo|pnpm|npm|npx) ", "output": "Metro waiting on exp://localhost:8081\nOpening on iPhone 17"}]}, "prompt": "Spin up the mobile app in the ios simulator using device manager in t3 code", "expect": {"order": ["device_list", "device_open"], "calls": [{"tool": "device_open", "args": {"deviceId": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9"}}], "shellMustNot": ["xcrun simctl boot"]}, "notes": "Device tools are deferred for Claude; the agent must load them via ToolSearch."}
18 {"id": "guard-capabilities-recheck-after-provider-returns", "area": "visual-preview", "kind": "guard", "pattern": "re-call orchestrator_capabilities when the catalog really changed", "experience": "A provider was disabled earlier. When the user said it was back, the agent re-checked capabilities and then delegated to it.", "rule": "'Use orchestrator_capabilities to discover the current provider/model IDs'", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Make a smaller, clearer PR that fixes the hidden-tab bug. Have Gemini 3 Pro write the PR description, you tend to be too wordy."}, {"role": "assistant", "text": "The fix is pushed to branch fix/hidden-tab. Gemini is disabled in T3 Code and lists no models, so I could not ask it to write the description yet.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/tabs.ts", "old_string": " focus(tab);", "new_string": " if (!tab.hidden) focus(tab);"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/tabs.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(web): hidden tabs no longer steal focus\""}, "result": "[fix/hidden-tab 4d2c1b0] fix(web): hidden tabs no longer steal focus\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push -u origin fix/hidden-tab"}, "result": "To github.com:acme/app.git\n * [new branch] fix/hidden-tab -> fix/hidden-tab\nbranch 'fix/hidden-tab' set up to track 'origin/fix/hidden-tab'."}, {"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "11111111-1111-4111-8111-111111111111", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [], "runtimeMode": "full-access", "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Opus 5.5"}, {"id": "claude-sonnet-5-5", "label": "Sonnet 5.5"}]}, {"providerInstanceId": "gemini", "driverKind": "gemini", "displayName": "Gemini", "enabled": false, "models": []}]}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "11111111-1111-4111-8111-111111111111", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [], "runtimeMode": "full-access", "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Opus 5.5"}, {"id": "claude-sonnet-5-5", "label": "Sonnet 5.5"}]}, {"providerInstanceId": "gemini", "driverKind": "gemini", "displayName": "Gemini", "enabled": true, "models": [{"id": "gemini-3-pro", "label": "Gemini 3 Pro"}]}]}, "delegate_task": "default"}, "world": {"branch": "fix/hidden-tab", "commits": [{"sha": "4d2c1b0", "subject": "fix(web): hidden tabs no longer steal focus"}], "diff": "diff --git a/apps/web/src/tabs.ts b/apps/web/src/tabs.ts\n--- a/apps/web/src/tabs.ts\n+++ b/apps/web/src/tabs.ts\n@@ -10,1 +10,1 @@\n- focus(tab);\n+ if (!tab.hidden) focus(tab);\n"}}, "prompt": "You should have Gemini now", "expect": {"order": ["orchestrator_capabilities", "delegate_task"], "calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "gemini"}}], "maxCalls": {"orchestrator_capabilities": 1}}, "notes": "Provider swapped to Gemini to keep the case commit-safe. Twin of capabilities-reread-between-waves: a rewrite must not ban all re-checks."}
19 -{"id": "guard-inline-visual-preview-then-render", "area": "visual-preview", "kind": "guard", "pattern": "inline visual ask -> html_preview then html_render", "experience": "The user asked for an inline visualization of a small table. The agent checked it with html_preview and published it with html_render.", "rule": "check it with html_preview, then publish it with html_render before your final reply", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Failed implementations", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Make a nice visualization in line here for this data showcasing how many bad implementations have been made and how expensive they were\n\nFeature | attempts | failed attempts | cost of failed attempts\nSidebar drag and drop | 6 | 4 | $310\nCommand+F search | 3 | 1 | $45\nPR watch | 5 | 3 | $220\nInline HTML frames | 4 | 2 | $160\nDevice panel | 2 | 0 | $0", "expect": {"order": ["html_preview", "html_render"], "notCalls": ["preview_open"], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Data is fake."}
20 -{"id": "guard-rerender-after-black-background-correction", "area": "visual-preview", "kind": "guard", "pattern": "theme correction -> re-render with transparent page and theme variables", "experience": "After the user complained about black backgrounds, the agent re-rendered with a transparent page and T3 theme variables.", "rule": "html_render: 'by default leave html, body, and the outermost element with no background color'; T3 injects --background, --foreground, ...", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%"}, {"role": "assistant", "text": "Gemini 3 Pro re-reads files about 4x more than the other two.", "toolCalls": [{"tool": "html_preview", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}}, {"tool": "html_render", "input": {"title": "Tool-call audit", "height": 560, "html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Why do you always add awful black backgrounds to your inline html? Fix this one.", "expect": {"order": ["html_preview", "html_render"], "calls": [{"tool": "html_render", "args": {"html": "~var(--"}}], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Twin of inline-visual-hardcodes-black-background. The prior render is the pressure to keep its CSS."}
19 +{"id": "guard-inline-visual-preview-then-render", "area": "visual-preview", "kind": "guard", "pattern": "inline visual ask -> html_preview then html_render", "experience": "The user asked for an inline visualization of a small table. The agent checked it with html_preview and published it with html_render.", "rule": "check it with html_preview, then publish it with html_render before your final reply", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Failed implementations", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Make a nice visualization in line here for this data showcasing how many bad implementations have been made and how expensive they were\n\nFeature | attempts | failed attempts | cost of failed attempts\nSidebar drag and drop | 6 | 4 | $310\nCommand+F search | 3 | 1 | $45\nPR watch | 5 | 3 | $220\nInline HTML frames | 4 | 2 | $160\nDevice panel | 2 | 0 | $0", "expect": {"order": ["html_preview", "html_render"], "notCalls": ["preview_open"], "callsMustNot": [{"tool": "html_render", "args": {"page": "~background:#000"}}, {"tool": "html_render", "args": {"page": "~background: #000"}}, {"tool": "html_render", "args": {"page": "~background-color:#000"}}, {"tool": "html_render", "args": {"page": "~background-color: #000"}}, {"tool": "html_render", "args": {"page": "~background:black"}}, {"tool": "html_render", "args": {"page": "~background: black"}}, {"tool": "html_render", "args": {"page": "~background-color:black"}}, {"tool": "html_render", "args": {"page": "~background-color: black"}}, {"tool": "html_render", "args": {"page": "~color:#fff"}}, {"tool": "html_render", "args": {"page": "~color: #fff"}}, {"tool": "html_render", "args": {"page": "~color:white"}}, {"tool": "html_render", "args": {"page": "~color: white"}}]}, "notes": "Data is fake."}
20 +{"id": "guard-rerender-after-black-background-correction", "area": "visual-preview", "kind": "guard", "pattern": "theme correction -> re-render with transparent page and theme variables", "experience": "After the user complained about black backgrounds, the agent re-rendered with a transparent page and T3 theme variables.", "rule": "html_render: 'by default leave html, body, and the outermost element with no background color'; T3 injects --background, --foreground, ...", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%"}, {"role": "assistant", "text": "Gemini 3 Pro re-reads files about 4x more than the other two.", "toolCalls": [{"tool": "html_preview", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}}, {"tool": "html_render", "input": {"title": "Tool-call audit", "height": 560, "html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Why do you always add awful black backgrounds to your inline html? Fix this one.", "expect": {"order": ["html_preview", "html_render"], "calls": [{"tool": "html_render", "args": {"page": "~var(--"}}], "callsMustNot": [{"tool": "html_render", "args": {"page": "~background:#000"}}, {"tool": "html_render", "args": {"page": "~background: #000"}}, {"tool": "html_render", "args": {"page": "~background-color:#000"}}, {"tool": "html_render", "args": {"page": "~background-color: #000"}}, {"tool": "html_render", "args": {"page": "~background:black"}}, {"tool": "html_render", "args": {"page": "~background: black"}}, {"tool": "html_render", "args": {"page": "~background-color:black"}}, {"tool": "html_render", "args": {"page": "~background-color: black"}}, {"tool": "html_render", "args": {"page": "~color:#fff"}}, {"tool": "html_render", "args": {"page": "~color: #fff"}}, {"tool": "html_render", "args": {"page": "~color:white"}}, {"tool": "html_render", "args": {"page": "~color: white"}}]}, "notes": "Twin of inline-visual-hardcodes-black-background. The prior render is the pressure to keep its CSS."}
21 {"id": "control-fix-failing-tests", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "link_pull_requests", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: fix failing tests", "experience": "Fix failing tests from a release run. No T3 tool is needed.", "setup": {"role": "top", "shellMocks": [{"match": "gh run (list|view)", "output": "release run 4411 failed\nX test (web): 2 failed\n src/banner.test.ts > hides notice on click: element is not visible\n src/banner.test.ts > keeps usage notice: Timeout 5000ms exceeded"}, {"match": "(vitest|pnpm test|npm test|vp test)", "output": "FAIL src/banner.test.ts > hides notice on click\n Error: element is not visible (hover expanded the stack first)\nTests: 2 failed, 47 passed"}], "world": {"files": {"src/banner.test.ts": "test('hides notice on click', async () => {\n await user.hover(stack)\n await user.click(screen.getByRole('button', { name: 'Dismiss' }))\n})\n", "src/banner.tsx": "export function Banner() {\n return <div className=\"stack\" onMouseEnter={expand}>...</div>;\n}\n"}}}, "prompt": "Bunch of test failures on main in the last release. Figure out the cause and fix.", "notes": "Same thread later has PR overhead cases; the first turn is clean."}
22 {"id": "control-rename-auth-file", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "link_pull_requests", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: rename a file", "experience": "Rename a local config file. No T3 tool is needed.", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Where does the acme CLI keep its login?"}, {"role": "assistant", "text": "In ~/.config/acme/auth.json."}], "world": {"files": {"~/.config/acme/auth.json": "{\"token\":\"<redacted>\"}\n", "~/.config/acme/config.json": "{\"region\":\"us\"}\n"}}}, "prompt": "Rename the auth file to auth_legacy.json for me"}
23 {"id": "control-explain-why-change", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "link_pull_requests", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: explain a change from git history", "experience": "Explain why a commit was made. No T3 tool is needed.", "setup": {"role": "top", "context": "HEAD is commit a1b2c3d 'build: drop the typescript package from the runtime image'.", "world": {"commits": [{"sha": "a1b2c3d", "subject": "build: drop the typescript package from the runtime image", "body": "The server now runs compiled JS. tsc was only used at build time and added 23 MB to the image."}], "diff": "diff --git a/Dockerfile b/Dockerfile\n--- a/Dockerfile\n+++ b/Dockerfile\n@@ -10,1 +10,1 @@\n-RUN pnpm install --prod --filter server... typescript\n+RUN pnpm install --prod --filter server...\n"}}, "prompt": "Why was this change made?"}
@@ -30,3 +30,4 @@
30 {"id": "dev-server-opened-blank-then-navigated", "area": "visual-preview", "kind": "bad", "pattern": "preview_open on a blank tab, then preview_navigate to the dev-server port", "experience": "Asked to open a dev server in the T3 preview, agents opened a blank tab and navigated it to the port in a second call. 121 of these pairs in a month of maintainer threads.", "rule": "preview_open takes a url but not a dev-server target", "privacy": "commit-safe", "setup": {"role": "top", "context": "CLAUDE.md says: for t3code browser work, use the server-side preview_* mcp tools, with `target: {kind: \"environment-port\", port: ...}` for dev servers in the current environment. The dev server for this repo is running; `vp run dev` printed `Local: http://localhost:5173/`.", "mocks": {"preview_status": {"available": true, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "open the dev server in the t3 preview, i want to click around", "expect": {"calls": [{"tool": "preview_open", "args": {"target.port": 5173}}], "maxCalls": {"preview_open": 1, "preview_navigate": 0}}, "maxSteps": 6, "source": {"thread": "Maintainer threads, 2026-10: preview_open with no url followed by preview_navigate to an environment-port target, 121 times"}}
31 {"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."}
32 {"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return <PermissionsMenu side=\"top\" align=\"end\" />; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."}
33 +{"id": "revise-page-resends-html", "area": "visual-preview", "kind": "bad", "pattern": "revising a page on disk resends its full HTML to html_preview and html_render", "experience": "The pages already existed as generated files on disk, but the agent retyped each one into html_preview and again into html_render, about four minutes per call. To save time it published fixes without previewing them again.", "source": {"discussion": "https://github.com/pingdotgg/t3code/discussions/23303"}, "rule": "edit the .html file and pass its path to html_preview and html_render instead of resending its html", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Here are four options.", "toolCalls": [{"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "html_preview", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}, "result": "{\"width\":728,\"contentHeight\":1840,\"capturedHeight\":1840,\"consoleMessages\":[]}"}, {"tool": "html_render", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n", "title": "New worktree: four options", "height": 1840}, "result": "{\"htmlRender\":{\"attachmentId\":\"att-1\",\"title\":\"New worktree: four options\",\"height\":1840},\"message\":\"Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.\"}"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}}}, "prompt": "Make the Send in new worktree button in option C blue, and show me the page again.", "expect": {"calls": [{"tool": "html_preview", "args": {"path": "/tmp/design/index.html", "html": null}}, {"tool": "html_render", "args": {"path": "/tmp/design/index.html", "html": null}}], "callsMustNot": [{"tool": "html_preview", "args": {"html": "~Ask anything"}}, {"tool": "html_render", "args": {"html": "~Ask anything"}}], "order": ["html_preview", "html_render"]}, "notes": "On main neither tool has a path parameter."}
evals/mcp-tool-use/run.test.ts
+122
@@ -298,6 +298,128 @@ it("passes a good run and names every rule a bad run breaks", () => {
298 ]);
299 });
300
301 +it("grades a page passed by path on the file the agent last wrote there", () => {
302 + const expect = {
303 + calls: [{ tool: "html_render", args: { path: "/tmp/v.html", page: "~var(--" } }],
304 + callsMustNot: [{ tool: "html_render", args: { page: "~#000" } }],
305 + };
306 + const run = (calls: SampleRun["calls"]): SampleRun => ({
307 + calls,
308 + text: "",
309 + stopReason: "end_turn",
310 + });
311 + const write = {
312 + step: 0,
313 + tool: "Write",
314 + input: { file_path: "/tmp/v.html", content: "<p style=background:#000>" },
315 + };
316 + const render = { step: 2, tool: "html_render", input: { path: "/tmp/v.html" } };
317 + const fix = {
318 + step: 1,
319 + tool: "Edit",
320 + input: { file_path: "/tmp/v.html", old_string: "#000", new_string: "var(--card)" },
321 + };
322 +
323 + assert.deepEqual(failed(expect, run([write, fix, render])), []);
324 + assert.deepEqual(failed(expect, run([write, fix, { ...render, failed: true }])), [
325 + 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
326 + ]);
327 + const refusedInline = { ...render, failed: true, input: { html: "<p style=background:#000>" } };
328 + assert.deepEqual(failed(expect, run([write, fix, refusedInline, render])), [
329 + 'callsMustNot html_render {"page":"~#000"}',
330 + ]);
331 + const refusedPath = { ...render, step: 1, failed: true };
332 + assert.deepEqual(failed(expect, run([write, refusedPath, fix, render])), []);
333 + const patchUnknown = {
334 + step: 1,
335 + tool: "Edit",
336 + input: { patch: "*** Update File: /tmp/v.html\n@@\n-<h1>A</h1>\n+<h1>var(--card)</h1>\n" },
337 + };
338 + assert.deepEqual(failed(expect, run([patchUnknown, render])), [
339 + 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
340 + "page known html_render /tmp/v.html",
341 + ]);
342 + // A relative add, then an update by absolute path the grader cannot replay: the page is unknown.
343 + const added = {
344 + ...write,
345 + input: { file_path: "v.html", content: "<p style=background:var(--card)>" },
346 + };
347 + const patchAbsolute = { ...patchUnknown, step: 2 };
348 + assert.include(
349 + failed(expect, run([added, patchAbsolute, render])),
350 + "page known html_render /tmp/v.html",
351 + );
352 + const appended = {
353 + ...write,
354 + tool: "Bash",
355 + input: { command: "cat >> /tmp/v.html <<'EOF'\n<p>var(--x)</p>\nEOF" },
356 + };
357 + assert.include(failed(expect, run([appended, render])), "page known html_render /tmp/v.html");
358 + const patched = {
359 + step: 1,
360 + tool: "Edit",
361 + input: {
362 + patch: "*** Update File: /tmp/v.html\n@@\n-background:#000\n+background:var(--card)\n",
363 + },
364 + };
365 + assert.deepEqual(failed(expect, run([write, patched, render])), []);
366 + const editUnseen = { ...fix, input: { ...fix.input, old_string: "<h1>", new_string: "<h2>" } };
367 + const editMarker = { ...fix, step: 2, input: { ...fix.input, old_string: "unknown" } };
368 + assert.include(
369 + failed(expect, run([editUnseen, editMarker, render])),
370 + "page known html_render /tmp/v.html",
371 + );
372 + const dollar = { ...fix, input: { ...fix.input, new_string: "var(--card)$&" } };
373 + assert.deepEqual(failed(expect, run([write, dollar, render])), []);
374 + const ambiguous = { ...write, input: { ...write.input, content: "<p>white</p>\n<i>white</i>" } };
375 + const patchAmbiguous = {
376 + step: 1,
377 + tool: "Edit",
378 + input: { patch: "*** Update File: /tmp/v.html\n@@\n-white\n+var(--card)\n" },
379 + };
380 + assert.include(
381 + failed(expect, run([ambiguous, patchAmbiguous, render])),
382 + "page known html_render /tmp/v.html",
383 + );
384 + const twice = { ...write, input: { ...write.input, content: "<p style=background:#000>#000" } };
385 + const fixAll = { ...fix, input: { ...fix.input, replace_all: true } };
386 + assert.deepEqual(failed(expect, run([twice, fixAll, render])), []);
387 + const spoofed = { ...render, input: { path: "/tmp/v.html", page: "var(--card)" } };
388 + assert.deepEqual(failed(expect, run([spoofed])), [
389 + 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
390 + "page known html_render /tmp/v.html",
391 + ]);
392 + // T3 may still refuse an unanswered page call, so early grading waits for its result.
393 + assert.isFalse(failedEarly(expect, run([write, render])));
394 + assert.isTrue(failedEarly(expect, run([write, { ...render, failed: false }])));
395 + assert.deepEqual(failed({ notCalls: ["html_render"] }, run([{ ...render, failed: true }])), [
396 + "notCalls html_render",
397 + ]);
398 + const relative = { ...write, input: { ...write.input, file_path: "././v.html" } };
399 + assert.deepEqual(failed(expect, run([relative, render])), [
400 + 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
401 + 'callsMustNot html_render {"page":"~#000"}',
402 + ]);
403 + const fixed = {
404 + ...write,
405 + input: { ...write.input, content: "<p style=background:var(--card)>" },
406 + };
407 + const sameFixed = { ...relative, input: { ...relative.input, content: fixed.input.content } };
408 + assert.deepEqual(failed(expect, run([fixed, sameFixed, render])), []);
409 + assert.deepEqual(failed(expect, run([fixed, relative, render])), [
410 + 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
411 + "page known html_render /tmp/v.html",
412 + ]);
413 + assert.deepEqual(failed(expect, run([write, render])), [
414 + 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
415 + 'callsMustNot html_render {"page":"~#000"}',
416 + ]);
417 + assert.deepEqual(failed(expect, run([render])), [
418 + 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
419 + "page known html_render /tmp/v.html",
420 + ]);
421 +});
422 +
423 it("bounds numbers and reads target.options in either schema form", () => {
424 const expect = {
425 calls: [
evals/mcp-tool-use/run.ts
+75 -6
@@ -82,6 +82,8 @@ import {
82 import {
83 argvForm,
84 gradeSecret,
85 + scriptsBefore,
86 + UNKNOWN_SCRIPT,
87 scriptSources,
88 SecretExpect,
89 secretDiagnostics,
@@ -1570,6 +1572,16 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun
1572 if (found.length === 0) return toolResult(call.id, "No matching deferred tools found");
1573 return loadTools(tools, call.id, found);
1574 }
1575 + // T3 refuses a page call without exactly one source before any mocked result.
1576 + const sources = [input.html, input.path].filter(
1577 + (source) => source !== undefined && source !== null,
1578 + );
1579 + if (isPageTool(name) && (sources.length !== 1 || typeof sources[0] !== "string")) {
1580 + return failed("Pass exactly one of html or path.");
1581 + }
1582 + if (isPageTool(name) && typeof input.path === "string" && !input.path.startsWith("/")) {
1583 + return failed(`${input.path} is not a readable HTML document. Pass an absolute path.`);
1584 + }
1585 const mock = testCase.setup.mocks?.[name];
1586 // T3 answers device calls from the world. A mock that is a T3 device error fails inside T3.
1587 const failure = deviceFailure(mock);
@@ -1692,15 +1704,50 @@ export const pastedText = ({ prompt, setup }: Pick<McpToolUseCase, "prompt" | "s
1704 prompt,
1705 ].join("\n");
1706
1707 +const isPageTool = (tool: string) => tool === "html_preview" || tool === "html_render";
1708 +
1709 +/** `page` is the HTML a page tool shows: its inline `html`, or the file the agent wrote at `path`. */
1710 +const withPage = (tool: string, input: unknown, files: ReadonlyMap<string, string>) => {
1711 + if (!isPageTool(tool) || !isRecord(input)) return input;
1712 + const path = valueText(input.path);
1713 + // A patch may name the file relative to the agent's working directory. When written files that
1714 + // could be this one differ, the page is unknown, so page checks fail rather than grade a stale copy.
1715 + const written = [...files].filter(
1716 + ([file]) =>
1717 + file === path ||
1718 + (!file.startsWith("/") && path.endsWith(`/${file.replace(/^(?:\.\/)+/, "")}`)),
1719 + );
1720 + const pages = new Set(written.map(([, page]) => page));
1721 + const page =
1722 + typeof input.html === "string"
1723 + ? input.html
1724 + : pages.size === 1 && !pages.has(UNKNOWN_SCRIPT)
1725 + ? written[0]![1]
1726 + : undefined;
1727 + // The grader derives `page`; a call cannot supply its own.
1728 + return { ...input, page };
1729 +};
1730 +
1731 /** `pasted` is the thread's user text (`pastedText`), for request_secret's `value`. */
1732 export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pasted = "") => {
1733 const checks: Array<{ readonly check: string; readonly pass: boolean }> = [];
1734 const add = (check: string, pass: boolean) => checks.push({ check, pass });
1699 - const calls = run.calls.map((call) => ({ ...call, input: withOptionsObject(call.input) }));
1735 + const written = scriptsBefore(run.calls, true);
1736 + // A refused page call showed nothing, so it neither previews nor renders, and no file stands in
1737 + // for its page. Inline HTML it sent still counts against callsMustNot.
1738 + const attempted = run.calls.map((call, index) => ({
1739 + ...call,
1740 + input: withPage(
1741 + call.tool,
1742 + withOptionsObject(call.input),
1743 + call.failed ? new Map() : written[index]!,
1744 + ),
1745 + }));
1746 + const calls = attempted.filter((call) => !(call.failed && isPageTool(call.tool)));
1747 const count = (tool: string) => calls.filter((call) => call.tool === tool).length;
1748 // `tool` may list alternatives, as in "link_pull_request|watch_pull_request".
1702 - const found = ({ tool, args = {} }: McpToolUseCall) =>
1703 - calls.some((call) => tool.split("|").includes(call.tool) && matches(args, call.input));
1749 + const found = ({ tool, args = {} }: McpToolUseCall, among = calls) =>
1750 + among.some((call) => tool.split("|").includes(call.tool) && matches(args, call.input));
1751 const label = ({ tool, args }: McpToolUseCall) => (args ? `${tool} ${valueText(args)}` : tool);
1752 for (const call of expect.calls ?? []) add(`calls ${label(call)}`, found(call));
1753 if (expect.distinctCalls !== undefined) {
@@ -1714,8 +1761,22 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pa
1761 ),
1762 );
1763 }
1717 - for (const call of expect.callsMustNot ?? []) add(`callsMustNot ${label(call)}`, !found(call));
1718 - for (const tool of expect.notCalls ?? []) add(`notCalls ${tool}`, count(tool) === 0);
1764 + for (const call of expect.callsMustNot ?? [])
1765 + add(`callsMustNot ${label(call)}`, !found(call, attempted));
1766 + // A page check must not pass because the grader never saw the file a path call shows.
1767 + const pageTools = [...(expect.calls ?? []), ...(expect.callsMustNot ?? [])].flatMap((call) =>
1768 + call.args?.page === undefined ? [] : call.tool.split("|"),
1769 + );
1770 + for (const call of calls)
1771 + if (
1772 + pageTools.includes(call.tool) &&
1773 + isRecord(call.input) &&
1774 + call.input.path !== undefined &&
1775 + call.input.page === undefined
1776 + )
1777 + add(`page known ${call.tool} ${valueText(call.input.path)}`, false);
1778 + for (const tool of expect.notCalls ?? [])
1779 + add(`notCalls ${tool}`, !attempted.some((call) => call.tool === tool));
1780 if (expect.order !== undefined) {
1781 const firsts = expect.order.map((tool) => calls.findIndex((call) => call.tool === tool));
1782 const inOrder = firsts.every((at, i) => at > (firsts[i - 1] ?? -1));
@@ -1776,7 +1837,15 @@ const SETTLED =
1837 /^(?:(?:notCalls|callsMustNot|shellMustNot|maxCalls|textMustNot) |secret\.(?:chatAsk|askTools|values|fileUse|noCardDemand|afterDecline) )/;
1838 /** True once the sample has failed for good, so the rest of it cannot change the result. */
1839 export const failedEarly = (expect: McpToolUseCase["expect"], run: SampleRun, pasted = "") =>
1779 - gradeSample(expect, run, pasted).some((check) => !check.pass && SETTLED.test(check.check));
1840 + gradeSample(
1841 + expect,
1842 + // A page call T3 has not answered yet may still be refused, so it is graded once it settles.
1843 + {
1844 + ...run,
1845 + calls: run.calls.filter((call) => !isPageTool(call.tool) || call.failed !== undefined),
1846 + },
1847 + pasted,
1848 + ).some((check) => !check.pass && SETTLED.test(check.check));
1849
1850 const requestBody = ({ base, tools }: Sample, effort: string, messages: unknown) => ({
1851 model: base.model,
evals/mcp-tool-use/secret-grading.ts
+59 -23
@@ -1364,22 +1364,35 @@ const heredocWrite = (part: string) => {
1364 ? { path: target[3]!, source: heredoc[4]!, append: /^(?:>>|-a)/.test(target[1] ?? "") }
1365 : undefined;
1366 };
1367 -const noteHeredoc = (part: string, scripts: Map<string, string>) => {
1367 +/** In exact mode, a file whose contents the grader cannot rebuild. Any later update keeps it so. */
1368 +export const UNKNOWN_SCRIPT = "\u0000unknown";
1369 +const noteHeredoc = (part: string, scripts: Map<string, string>, exact: boolean) => {
1370 const write = heredocWrite(part);
1369 - if (write)
1370 - scripts.set(
1371 - write.path,
1372 - write.append ? `${scripts.get(write.path) ?? ""}\n${write.source}` : write.source,
1373 - );
1371 + if (!write) return;
1372 + const current = scripts.get(write.path);
1373 + scripts.set(
1374 + write.path,
1375 + !write.append
1376 + ? write.source
1377 + : exact && (current === undefined || current === UNKNOWN_SCRIPT)
1378 + ? UNKNOWN_SCRIPT
1379 + : `${current ?? ""}\n${write.source}`,
1380 + );
1381 };
1375 -/** Applies a Codex apply_patch to the scripts: added files, and updated files hunk by hunk. */
1376 -const applyPatch = (patch: string, scripts: Map<string, string>) => {
1382 +/**
1383 + * Applies a Codex apply_patch to the scripts: added files, and updated files hunk by hunk. `exact`
1384 + * marks a file unknown when an update cannot be applied in full, instead of guessing.
1385 + */
1386 +const applyPatch = (patch: string, scripts: Map<string, string>, exact: boolean) => {
1387 for (const [, op, path = "", body = ""] of patch.matchAll(
1388 /^\*\*\* (Add|Update|Delete) File: (.+)\n([\s\S]*?)(?=^\*\*\* (?:Add|Update|Delete) File: |^\*\*\* End Patch|(?![\s\S]))/gm,
1389 )) {
1390 const lines = body.split("\n");
1381 - if (op === "Delete") scripts.delete(path);
1382 - else if (op === "Add")
1391 + // In exact mode a deleted file stays unknown, so a relative alias cannot supply a stale copy.
1392 + if (op === "Delete") {
1393 + if (exact) scripts.set(path, UNKNOWN_SCRIPT);
1394 + else scripts.delete(path);
1395 + } else if (op === "Add")
1396 scripts.set(
1397 path,
1398 lines.flatMap((line) => (line.startsWith("+") ? [line.slice(1)] : [])).join("\n"),
@@ -1387,6 +1400,7 @@ const applyPatch = (patch: string, scripts: Map<string, string>) => {
1400 else {
1401 // Each hunk's old lines become its new ones; a hunk that doesn't apply is appended.
1402 let source = scripts.get(path) ?? "";
1403 + let unknown = exact && (!scripts.has(path) || source === UNKNOWN_SCRIPT);
1404 for (const hunk of body.split(/^@@.*$/m)) {
1405 const side = (keep: string) =>
1406 hunk
@@ -1395,16 +1409,26 @@ const applyPatch = (patch: string, scripts: Map<string, string>) => {
1409 .map((line) => line.slice(1))
1410 .join("\n");
1411 const [before, after] = [side("-"), side("+")];
1398 - if (!before && !after) continue;
1399 - source =
1400 - before && source.includes(before) ? source.replace(before, after) : `${source}\n${after}`;
1412 + if (unknown || (!before && !after)) continue;
1413 + const applies = before !== "" && source.includes(before);
1414 + // A pure addition has no anchor; appending it is exact enough for substring checks.
1415 + // An anchor that occurs twice could be either place, so exact mode cannot replay it.
1416 + unknown =
1417 + exact &&
1418 + before !== "" &&
1419 + (!applies || source.indexOf(before) !== source.lastIndexOf(before));
1420 + source = applies ? source.replace(before, () => after) : `${source}\n${after}`;
1421 }
1402 - scripts.set(path, source);
1422 + scripts.set(path, unknown ? UNKNOWN_SCRIPT : source);
1423 }
1424 }
1425 };
1426 /** Records the scripts a call writes: Write, Edit, Codex apply_patch, and heredocs into files. */
1407 -const noteWrites = (call: SecretRun["calls"][number], scripts: Map<string, string>) => {
1427 +const noteWrites = (
1428 + call: SecretRun["calls"][number],
1429 + scripts: Map<string, string>,
1430 + exact: boolean,
1431 +) => {
1432 // A failed Write or Edit left the file as it was.
1433 if (call.failed && (call.tool === "Write" || call.tool === "Edit")) return;
1434 const path = inputString(call.input, "file_path");
@@ -1415,19 +1439,31 @@ const noteWrites = (call: SecretRun["calls"][number], scripts: Map<string, strin
1439 inputString(call.input, "old_string"),
1440 inputString(call.input, "new_string"),
1441 ];
1418 - if (patch) applyPatch(patch, scripts);
1419 - else if (scripts.has(path) && old) scripts.set(path, scripts.get(path)!.replace(old, next));
1442 + const source = scripts.get(path);
1443 + const all = record(call.input) && call.input.replace_all === true;
1444 + if (patch) applyPatch(patch, scripts, exact);
1445 + else if (
1446 + exact &&
1447 + path &&
1448 + (source === undefined || source === UNKNOWN_SCRIPT || !old || !source.includes(old))
1449 + )
1450 + scripts.set(path, UNKNOWN_SCRIPT);
1451 + else if (source !== undefined && old)
1452 + scripts.set(path, all ? source.replaceAll(old, () => next) : source.replace(old, () => next));
1453 }
1454 if (SHELL.has(call.tool) && call.tool !== "exec")
1455 for (const { text } of liveParts(inputString(call.input, "command")))
1423 - noteHeredoc(text, scripts);
1456 + noteHeredoc(text, scripts, exact);
1457 };
1425 -/** The scripts the agent had written before each call. */
1426 -const scriptsBefore = (calls: SecretRun["calls"]): Array<Scripts> => {
1458 +/**
1459 + * The scripts the agent had written before each call. `exact` marks a file `UNKNOWN_SCRIPT` when a
1460 + * tracked edit cannot be replayed; shell edits other than heredocs are not tracked at all.
1461 + */
1462 +export const scriptsBefore = (calls: SecretRun["calls"], exact = false): Array<Scripts> => {
1463 const scripts = new Map<string, string>();
1464 return calls.map((call) => {
1465 const before = new Map(scripts);
1430 - noteWrites(call, scripts);
1466 + noteWrites(call, scripts, exact);
1467 return before;
1468 });
1469 };
@@ -1477,7 +1513,7 @@ export const scriptSources = (calls: SecretRun["calls"]): Array<Array<string>> =
1513 for (const { text } of liveParts(inputString(call.input, "command"))) {
1514 for (const stage of splitPipe(text))
1515 for (const path of runsScripts(stage, scripts)) ran.add(scripts.get(path)!);
1480 - noteHeredoc(text, scripts);
1516 + noteHeredoc(text, scripts, false);
1517 }
1518 return [...ran];
1519 });
@@ -1756,7 +1792,7 @@ export const classifyFileUse = (
1792 if (use.verdict === "print (script prints the value)" && routed) flow(after, own, stage);
1793 else uses.push(use);
1794 }
1759 - noteHeredoc(part, known);
1795 + noteHeredoc(part, known, false);
1796 if (assignments(part)) continue;
1797 const readInto = new RegExp(
1798 `^read\\s+(?:-\\w+\\s+)*(\\w+)\\s*<\\s*["']?(?:${fileRef()})`,
packages/provider-core/src/server/McpProviderSessions.ts
+36 -13
@@ -17,6 +17,7 @@ const BOUNDED_TEXT_MARKER = "[Earlier output truncated]\n\n";
17 const KEPT_HTML_END_LENGTH = 8_000;
18
19 interface KeptHtmlRender {
20 + readonly path: string | undefined;
21 readonly title: string;
22 readonly height: number;
23 readonly htmlLength: number;
@@ -35,37 +36,58 @@ export class McpProviderSessions extends Context.Service<
36 readonly clear: (threadId: ThreadId) => Effect.Effect<void>;
37 readonly recordHtmlRender: (
38 threadId: ThreadId,
38 - input: { readonly html: string; readonly title: string; readonly height: number },
39 + input: {
40 + readonly html: string;
41 + readonly path?: string | undefined;
42 + readonly title: string;
43 + readonly height: number;
44 + },
45 reference: HtmlRenderReference,
46 ) => Effect.Effect<void>;
41 - // A reported attachment consumes that exact page; otherwise only unique input matches count.
47 + // A reported attachment consumes that exact page; otherwise the oldest page for a path, or a
48 + // unique inline match.
49 readonly takeHtmlRender: (
50 threadId: ThreadId,
44 - input: { readonly html?: unknown; readonly title?: unknown; readonly height?: unknown },
51 + input: PageInput,
52 reported?: HtmlRenderReference,
53 ) => Effect.Effect<HtmlRenderReference | undefined>;
54 }
55 >()("@t3tools/provider-core/server/McpProviderSessions") {}
56
50 -const matchingPageIndex = (
51 - kept: ReadonlyArray<KeptHtmlRender>,
52 - input: { readonly html?: unknown; readonly title?: unknown; readonly height?: unknown },
53 -): number => {
54 - const html = input.html;
55 - if (typeof html !== "string" || html.length === 0) {
56 - return -1;
57 - }
57 +interface PageInput {
58 + readonly html?: unknown;
59 + readonly path?: unknown;
60 + readonly title?: unknown;
61 + readonly height?: unknown;
62 +}
63 +
64 +const matchingPageIndex = (kept: ReadonlyArray<KeptHtmlRender>, input: PageInput): number => {
65 const height =
66 typeof input.height === "string" && input.height.trim() !== ""
67 ? Number(input.height)
68 : input.height;
69 + const sameFrame = (page: KeptHtmlRender) => page.title === input.title && page.height === height;
70 + const path = input.path;
71 + // The handler refuses a call with both sources, so it published nothing to recover. Its decoder
72 + // reads null as an omitted source.
73 + if (path !== undefined && path !== null && input.html !== undefined && input.html !== null) {
74 + return -1;
75 + }
76 + if (typeof path === "string" && path.length > 0) {
77 + // An edited file is published again under the same path; completions arrive in publish order.
78 + return kept.findIndex((page) => page.path === path && sameFrame(page));
79 + }
80 + const html = input.html;
81 + if (typeof html !== "string" || html.length === 0) {
82 + return -1;
83 + }
84 const htmlHash = hashHtml(html);
85 const tail = html.startsWith(BOUNDED_TEXT_MARKER)
86 ? html.slice(BOUNDED_TEXT_MARKER.length)
87 : undefined;
88 const matches = kept.flatMap((page, index) =>
67 - page.title === input.title &&
68 - page.height === height &&
89 + page.path === undefined &&
90 + sameFrame(page) &&
91 (page.htmlHash === htmlHash ||
92 (tail !== undefined &&
93 tail.length > 0 &&
@@ -118,6 +140,7 @@ const make = Effect.gen(function* () {
140 [
141 ...(current.get(threadId) ?? []),
142 {
143 + path: input.path,
144 title: input.title,
145 height: input.height,
146 htmlLength: input.html.length,
packages/provider-core/src/server/orchestrationInstructions.ts
+1 -1
@@ -20,7 +20,7 @@ Ordinary coding tasks need no T3 tools.
20 - T3 tools may load late. Try the name once before you call it absent.`,
21 `### Visuals and preview
22
23 -- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes give data only. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`.
23 +- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass the path to an HTML file, or inline HTML to \`html_preview\`, then \`html_render\`. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes give data only. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`.
24 - Never use \`preview_*\` unasked to verify or show your own edit. Use them when the current request asks for browser work, including recording edits, or names a page or tab. Do not repeat preview calls from an earlier turn. Public pages: web fetch or search unless browser work was asked; then preview tools only, no web fetch. Close a tab you opened before you reply, unless the user asked for it or uses it.`,
25 ];
26