The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 02:00 UTC
feat(server): html tools take a file path, so agents stop retyping pages (#32833)
Agents can pass `path` to an absolute .html file instead of retyping the page as inline `html`. The server reads the file only for full-access threads, with the existing disk-read guards, and snapshots it at publish. ACP render recovery matches path calls by path. The mcp-tool-use eval grades pages passed by path. Takes over #32817. Co-authored-by: Stoli Geyer <stoli@tractorbeam.ai> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Theo Browne committed
Nov 7, 2026 at 21:26 UTC
d6c2b09c52e96a71e22124a734190cc0d14ca25f
12 files changed
+608
-85
apps/server/src/htmlRender/HtmlRender.test.ts
+114
@@ -4,6 +4,7 @@ import { assert, describe, expect, it } from "@effect/vitest";
4
import { EnvironmentId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";
5
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
6
import * as HostProcess from "@t3tools/shared/HostProcess";
7
+import { symlinksSupported } from "@t3tools/shared/testing/symlinks";
8
import * as Deferred from "effect/Deferred";
9
import * as Effect from "effect/Effect";
10
import * as Exit from "effect/Exit";
@@ -259,6 +260,119 @@ describe("HtmlRender", () => {
260
}).pipe(Effect.provide(layerTest)),
261
);
262
263
+ it.effect("publishes a page from an .html file as a snapshot that path calls recover", () =>
264
+ Effect.gen(function* () {
265
+ const fileSystem = yield* FileSystem.FileSystem;
266
+ const path = yield* Path.Path;
267
+ const config = yield* ServerConfig.ServerConfig;
268
+ const htmlRender = yield* HtmlRender.HtmlRender;
269
+ const mcpSessions = yield* McpProviderSessions.McpProviderSessions;
270
+ const threadId = ThreadId.make("thread-html-path");
271
+ const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-html-page-" });
272
+ const file = path.join(directory, "report.html");
273
+ yield* fileSystem.writeFileString(file, "<!doctype html><p>First draft</p>");
274
+ const page = { path: file, title: "Report", height: 300 };
275
+
276
+ const first = yield* htmlRender.publish({ threadId, ...page });
277
+ yield* fileSystem.writeFileString(file, "<!doctype html><p>Second draft</p>");
278
+ const second = yield* htmlRender.publish({ threadId, ...page });
279
+ const inline = { html: "<!doctype html><p>First draft</p>", title: "Report", height: 300 };
280
+ const inlineReference = yield* htmlRender.publish({ threadId, ...inline });
281
+
282
+ const stored = yield* fileSystem.readFileString(
283
+ resolveAttachmentPathById({
284
+ attachmentsDir: config.attachmentsDir,
285
+ attachmentId: first.attachmentId,
286
+ }) ?? "",
287
+ );
288
+ expect(stored).toContain("<p>First draft</p>");
289
+ expect(yield* mcpSessions.takeHtmlRender(threadId, { ...page, path: `${file}x` })).toBe(
290
+ undefined,
291
+ );
292
+ for (const html of [inline.html, false]) {
293
+ expect(yield* mcpSessions.takeHtmlRender(threadId, { ...page, html })).toBe(undefined);
294
+ }
295
+ expect(yield* mcpSessions.takeHtmlRender(threadId, { ...inline, path: null })).toEqual(
296
+ inlineReference,
297
+ );
298
+ expect(yield* mcpSessions.takeHtmlRender(threadId, { ...page, html: null })).toEqual(first);
299
+ expect(yield* mcpSessions.takeHtmlRender(threadId, page)).toEqual(second);
300
+ expect(yield* mcpSessions.takeHtmlRender(threadId, page)).toBe(undefined);
301
+ }).pipe(Effect.provide(layerTest)),
302
+ );
303
+
304
+ it.effect("refuses anything but a regular, singly linked .html file of at most 25 MiB", () =>
305
+ Effect.gen(function* () {
306
+ const fileSystem = yield* FileSystem.FileSystem;
307
+ const path = yield* Path.Path;
308
+ const config = yield* ServerConfig.ServerConfig;
309
+ const htmlRender = yield* HtmlRender.HtmlRender;
310
+ const threadId = ThreadId.make("thread-html-path-guard");
311
+ const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-html-page-" });
312
+ const secret = path.join(directory, "id_rsa");
313
+ yield* fileSystem.writeFileString(secret, "PRIVATE KEY");
314
+ const page = path.join(directory, "page.html");
315
+ yield* fileSystem.writeFileString(page, "<!doctype html><p>linked</p>");
316
+ const hardLinked = path.join(directory, "linked.html");
317
+ yield* fileSystem.link(page, hardLinked);
318
+ // `mv .env renamed.html` keeps one link and an .html name; only its bytes give it away.
319
+ const renamed = path.join(directory, "renamed.html");
320
+ yield* fileSystem.writeFileString(renamed, "API_KEY=abc123\n");
321
+ const folder = path.join(directory, "folder.html");
322
+ yield* fileSystem.makeDirectory(folder);
323
+ const internal = path.join(config.secretsDir, "page.html");
324
+ yield* fileSystem.makeDirectory(config.secretsDir, { recursive: true });
325
+ yield* fileSystem.writeFileString(internal, "<!doctype html><p>internal</p>");
326
+
327
+ for (const file of [
328
+ "page.html",
329
+ "//attacker/share/page.html",
330
+ path.join(directory, "missing.html"),
331
+ secret,
332
+ hardLinked,
333
+ renamed,
334
+ folder,
335
+ internal,
336
+ ]) {
337
+ const error = yield* htmlRender
338
+ .publish({ threadId, path: file, title: "X", height: 200 })
339
+ .pipe(Effect.flip);
340
+ expect(error, file).toBeInstanceOf(HtmlRender.HtmlRenderFileError);
341
+ }
342
+ const huge = path.join(directory, "huge.html");
343
+ yield* fileSystem.writeFileString(huge, "");
344
+ yield* fileSystem.truncate(huge, 25 * 1024 * 1024 + 1);
345
+ const tooLarge = yield* htmlRender
346
+ .publish({ threadId, path: huge, title: "X", height: 200 })
347
+ .pipe(Effect.flip);
348
+ expect(tooLarge).toBeInstanceOf(HtmlRender.HtmlRenderFileTooLargeError);
349
+ }).pipe(Effect.provide(layerTest)),
350
+ );
351
+
352
+ it.effect.skipIf(!symlinksSupported)("refuses an .html symlink to another file", () =>
353
+ Effect.gen(function* () {
354
+ const fileSystem = yield* FileSystem.FileSystem;
355
+ const path = yield* Path.Path;
356
+ const htmlRender = yield* HtmlRender.HtmlRender;
357
+ const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-html-page-" });
358
+ const secret = path.join(directory, "id_rsa");
359
+ yield* fileSystem.writeFileString(secret, "PRIVATE KEY");
360
+ const symlinked = path.join(directory, "symlinked.html");
361
+ yield* fileSystem.symlink(secret, symlinked);
362
+
363
+ const error = yield* htmlRender
364
+ .publish({
365
+ threadId: ThreadId.make("thread-html-path-link"),
366
+ path: symlinked,
367
+ title: "X",
368
+ height: 200,
369
+ })
370
+ .pipe(Effect.flip);
371
+
372
+ expect(error).toBeInstanceOf(HtmlRender.HtmlRenderFileError);
373
+ }).pipe(Effect.provide(layerTest)),
374
+ );
375
+
376
it.effect("removes the page when publishing is interrupted", () =>
377
Effect.gen(function* () {
378
const threadId = ThreadId.make("thread-html-cancel");
apps/server/src/htmlRender/HtmlRender.ts
+125
-27
@@ -1,6 +1,7 @@
1
import type { ThreadId } from "@t3tools/contracts";
2
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
3
import * as HostProcess from "@t3tools/shared/HostProcess";
4
+import { isNetworkPath } from "@t3tools/shared/path";
5
import {
6
clampHtmlRenderHeight,
7
HTML_RENDER_COLUMN_WIDTH,
@@ -22,14 +23,16 @@ import * as Effect from "effect/Effect";
23
import * as FileSystem from "effect/FileSystem";
24
import * as Layer from "effect/Layer";
25
import * as Option from "effect/Option";
25
-import type * as Path from "effect/Path";
26
+import * as Path from "effect/Path";
27
import * as Schema from "effect/Schema";
28
import * as Semaphore from "effect/Semaphore";
29
import * as Stream from "effect/Stream";
30
import type * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
31
32
+import { openMediaFile } from "../assets/MediaFile.ts";
33
import { resolveAttachmentRelativePath } from "../attachmentPaths.ts";
34
import { createAttachmentId } from "../attachmentStore.ts";
35
+import { makeIsServerInternalPath } from "../auth/serverInternalPaths.ts";
36
import { resolveRootCliCommand } from "../cli/invocation.ts";
37
import * as ServerConfig from "../config.ts";
38
import * as HeadlessChrome from "./headlessChrome.ts";
@@ -39,8 +42,29 @@ import * as PreviewBrowserHost from "../preview/PreviewBrowserHost.ts";
42
const MIB = 1024 * 1024;
43
const MAX_IMAGE_BYTES = 10 * MIB;
44
const MAX_PAGE_BYTES = 25 * MIB;
45
+const HTML_FILE_EXTENSION = /\.html?$/i;
46
+// Like the image check, judge the bytes: a renamed .env or key does not start like a page.
47
+const HTML_DOCUMENT_START = /^\s*<(?:!doctype\s+html|html)[\s>]/i;
48
const formatMib = (bytes: number) => `${(bytes / MIB).toFixed(1)} MiB`;
49
50
+export class HtmlRenderFileError extends Schema.TaggedError<HtmlRenderFileError>()(
51
+ "HtmlRenderFileError",
52
+ { path: Schema.String, cause: Schema.optional(Schema.Defect()) },
53
+) {
54
+ override get message(): string {
55
+ return `${this.path} is not a readable HTML document. Pass an absolute path to a regular .html file that starts with <!doctype html> or <html>, or pass html instead.`;
56
+ }
57
+}
58
+
59
+export class HtmlRenderFileTooLargeError extends Schema.TaggedError<HtmlRenderFileTooLargeError>()(
60
+ "HtmlRenderFileTooLargeError",
61
+ { path: Schema.String, sizeBytes: Schema.Number },
62
+) {
63
+ override get message(): string {
64
+ return `${this.path} is ${formatMib(this.sizeBytes)}; a page must be at most ${formatMib(MAX_PAGE_BYTES)}.`;
65
+ }
66
+}
67
+
68
export class HtmlRenderImagesNotFoundError extends Schema.TaggedError<HtmlRenderImagesNotFoundError>()(
69
"HtmlRenderImagesNotFoundError",
70
{ paths: Schema.Array(Schema.String) },
@@ -108,6 +132,8 @@ export type HtmlRenderPrepareError =
132
| HtmlRenderPageTooLargeError
133
| HtmlRenderViewportHeightError;
134
135
+export type HtmlRenderSource = { readonly html: string } | { readonly path: string };
136
+
137
export interface HtmlPreview {
138
readonly png: string;
139
readonly width: number;
@@ -121,18 +147,28 @@ export class HtmlRender extends Context.Service<
147
HtmlRender,
148
{
149
readonly prepare: (html: string) => Effect.Effect<string, HtmlRenderPrepareError>;
124
- readonly publish: (input: {
125
- readonly threadId: ThreadId;
126
- readonly html: string;
127
- readonly title: string;
128
- readonly height: number;
129
- }) => Effect.Effect<HtmlRenderReference, HtmlRenderPrepareError | HtmlRenderStoreError>;
130
- readonly preview: (input: {
131
- readonly html: string;
132
- readonly width?: number | undefined;
133
- readonly appearance?: ThemeAppearance | undefined;
134
- }) => Effect.Effect<
150
+ readonly publish: (
151
+ input: HtmlRenderSource & {
152
+ readonly threadId: ThreadId;
153
+ readonly title: string;
154
+ readonly height: number;
155
+ },
156
+ ) => Effect.Effect<
157
+ HtmlRenderReference,
158
+ | HtmlRenderFileError
159
+ | HtmlRenderFileTooLargeError
160
+ | HtmlRenderPrepareError
161
+ | HtmlRenderStoreError
162
+ >;
163
+ readonly preview: (
164
+ input: HtmlRenderSource & {
165
+ readonly width?: number | undefined;
166
+ readonly appearance?: ThemeAppearance | undefined;
167
+ },
168
+ ) => Effect.Effect<
169
HtmlPreview,
170
+ | HtmlRenderFileError
171
+ | HtmlRenderFileTooLargeError
172
| Exclude<HtmlRenderPrepareError, HtmlRenderImagesNotFoundError>
173
| PreviewBrowser.PreviewBrowserInstallError
174
| PreviewBrowser.PreviewBrowserInstallingError
@@ -331,7 +367,9 @@ const MEASURE_FRAGMENT = htmlRenderThemeFragment(
367
368
const make = Effect.gen(function* () {
369
const fileSystem = yield* FileSystem.FileSystem;
370
+ const path = yield* Path.Path;
371
const config = yield* ServerConfig.ServerConfig;
372
+ const isServerInternalPath = yield* makeIsServerInternalPath;
373
const previewBrowser = yield* PreviewBrowser.PreviewBrowser;
374
const mcpSessions = yield* McpProviderSessions.McpProviderSessions;
375
const services = yield* Effect.context<
@@ -400,6 +438,53 @@ const make = Effect.gen(function* () {
438
),
439
);
440
441
+ const readPageFile = Effect.fn("HtmlRender.readPageFile")(function* (filePath: string) {
442
+ const unreadable = new HtmlRenderFileError({ path: filePath });
443
+ if (!path.isAbsolute(filePath) || isNetworkPath(filePath)) return yield* unreadable;
444
+ // Check the resolved file, so a symlink named .html cannot reach another file.
445
+ const realPath = yield* fileSystem
446
+ .realPath(filePath)
447
+ .pipe(Effect.mapError((cause) => new HtmlRenderFileError({ path: filePath, cause })));
448
+ if (!HTML_FILE_EXTENSION.test(realPath) || (yield* isServerInternalPath(realPath))) {
449
+ return yield* unreadable;
450
+ }
451
+ // Open without following links and recheck the path, so a link or FIFO swapped in is refused.
452
+ const html = yield* Effect.scoped(
453
+ Effect.gen(function* () {
454
+ const file = yield* openMediaFile(realPath).pipe(
455
+ Effect.mapError((cause) => new HtmlRenderFileError({ path: filePath, cause })),
456
+ );
457
+ // Unlike client reads, the caller may be sandboxed: a second hard link could give any
458
+ // file an .html name.
459
+ if (file === null || file.info.nlink !== 1n) return yield* unreadable;
460
+ if (file.info.size > BigInt(MAX_PAGE_BYTES)) {
461
+ return yield* new HtmlRenderFileTooLargeError({
462
+ path: filePath,
463
+ sizeBytes: Number(file.info.size),
464
+ });
465
+ }
466
+ // One read may return fewer bytes, so fill up to the checked size; growth past it is ignored.
467
+ return yield* Effect.tryPromise({
468
+ try: async () => {
469
+ const buffer = Buffer.alloc(Number(file.info.size));
470
+ let filled = 0;
471
+ while (filled < buffer.length) {
472
+ const { bytesRead } = await file.handle.read(buffer, filled, buffer.length - filled);
473
+ if (bytesRead === 0) break;
474
+ filled += bytesRead;
475
+ }
476
+ return buffer.toString("utf8", 0, filled);
477
+ },
478
+ catch: (cause) => new HtmlRenderFileError({ path: filePath, cause }),
479
+ });
480
+ }),
481
+ );
482
+ return HTML_DOCUMENT_START.test(html) ? html : yield* unreadable;
483
+ });
484
+
485
+ const readSource = (input: HtmlRenderSource) =>
486
+ "path" in input ? readPageFile(input.path) : Effect.succeed(input.html);
487
+
488
// The bootstrap goes in first so its head scan never runs over inlined image data.
489
const inline = (html: string) =>
490
inlineLocalImages(injectHtmlRenderBootstrap(html)).pipe(Effect.provideContext(services));
@@ -413,13 +498,15 @@ const make = Effect.gen(function* () {
498
return inlined.html;
499
});
500
416
- const publish = Effect.fn("HtmlRender.publish")(function* (input: {
417
- readonly threadId: ThreadId;
418
- readonly html: string;
419
- readonly title: string;
420
- readonly height: number;
421
- }) {
422
- const html = yield* prepare(input.html);
501
+ const publish = Effect.fn("HtmlRender.publish")(function* (
502
+ input: HtmlRenderSource & {
503
+ readonly threadId: ThreadId;
504
+ readonly title: string;
505
+ readonly height: number;
506
+ },
507
+ ) {
508
+ const source = yield* readSource(input);
509
+ const html = yield* prepare(source);
510
// The `-html` id suffix makes the asset route serve it as a sandboxed text/html document.
511
const attachmentId = createAttachmentId(input.threadId, "html");
512
const filePath =
@@ -447,7 +534,16 @@ const make = Effect.gen(function* () {
534
...(heights === undefined ? {} : { heights }),
535
};
536
return mcpSessions
450
- .recordHtmlRender(input.threadId, input, reference)
537
+ .recordHtmlRender(
538
+ input.threadId,
539
+ {
540
+ html: source,
541
+ path: "path" in input ? input.path : undefined,
542
+ title: input.title,
543
+ height: input.height,
544
+ },
545
+ reference,
546
+ )
547
.pipe(Effect.as(reference));
548
}),
549
// An interrupted publish must leave neither an attachment nor a recoverable reference.
@@ -463,18 +559,20 @@ const make = Effect.gen(function* () {
559
);
560
});
561
466
- const preview = Effect.fn("HtmlRender.preview")(function* (input: {
467
- readonly html: string;
468
- readonly width?: number | undefined;
469
- readonly appearance?: ThemeAppearance | undefined;
470
- }) {
562
+ const preview = Effect.fn("HtmlRender.preview")(function* (
563
+ input: HtmlRenderSource & {
564
+ readonly width?: number | undefined;
565
+ readonly appearance?: ThemeAppearance | undefined;
566
+ },
567
+ ) {
568
const width = Math.min(
569
MAX_PREVIEW_WIDTH,
570
Math.max(MIN_PREVIEW_WIDTH, Math.round(input.width ?? HTML_RENDER_COLUMN_WIDTH)),
571
);
572
const appearance = input.appearance ?? "dark";
476
- yield* rejectViewportHeight(input.html);
477
- const inlined = yield* inline(input.html);
573
+ const source = yield* readSource(input);
574
+ yield* rejectViewportHeight(source);
575
+ const inlined = yield* inline(source);
576
const executable = yield* previewBrowser.executable;
577
const theme = htmlRenderTheme(
578
appearance === "light" ? T3_CODE_LIGHT_THEME_COLORS : T3_CODE_DARK_THEME_COLORS,
apps/server/src/mcp/toolkits/core.test.ts
+24
-2
@@ -252,6 +252,8 @@ it("bounds public command rejections and redacts internal dispatch causes", () =
252
});
253
});
254
255
+const fullAccessThreadId = ThreadId.make("mcp-core-full-access-thread");
256
+
257
it.effect("returns an HTML render reference that Codex and Claude tool rows both carry", () =>
258
Effect.gen(function* () {
259
const server = yield* McpServer.McpServer;
@@ -272,6 +274,24 @@ it.effect("returns an HTML render reference that Codex and Claude tool rows both
274
expect(
275
htmlRenderFromToolItem({ toolName: "mcp__t3-code__html_render", output: result.content }),
276
).toEqual(reference);
277
+ // Only a full-access thread may have the server read a file for it.
278
+ const byPath = (name: string, caller: McpInvocationContext.McpInvocationScope) =>
279
+ server
280
+ .callTool({
281
+ name,
282
+ arguments: { path: "/t3-missing-dir/revenue.html", title: "Revenue", height: 240 },
283
+ })
284
+ .pipe(
285
+ Effect.provideService(McpInvocationContext.McpInvocationContext, caller),
286
+ Effect.provideService(McpSchema.McpServerClient, client),
287
+ // html_preview reports failures as plain text, html_render as a declared failure.
288
+ Effect.map((result) => (result.isError ? JSON.stringify(result.content) : undefined)),
289
+ );
290
+ const fullAccess = { ...scope, thread: { ...scope.thread!, threadId: fullAccessThreadId } };
291
+ for (const name of ["html_preview", "html_render"]) {
292
+ expect(yield* byPath(name, scope), name).toContain("only full-access threads");
293
+ expect(yield* byPath(name, fullAccess), name).toContain("not a readable HTML document");
294
+ }
295
}).pipe(
296
Effect.provide(
297
McpHttpServer.layerHtmlToolkit.pipe(
@@ -287,13 +307,15 @@ it.effect("returns an HTML render reference that Codex and Claude tool rows both
307
Layer.provide(FetchHttpClient.layer),
308
Layer.provide(
309
Layer.mock(ThreadManagement.ThreadManagementService)({
290
- getThreadShell: () =>
310
+ getThreadShell: (id) =>
311
Effect.succeed({
292
- id: threadId,
312
+ id,
313
deletedAt: null,
314
archivedAt: null,
315
activeRunId: RunId.make("mcp-core-run"),
316
providerInstanceId: ProviderInstanceId.make("codex"),
317
+ runtimeMode: id === fullAccessThreadId ? "full-access" : "approval-required",
318
+ interactionMode: "default",
319
} as OrchestrationV2ThreadShell),
320
}),
321
),
apps/server/src/mcp/toolkits/html/handlers.ts
+34
-2
@@ -4,9 +4,12 @@ import * as Effect from "effect/Effect";
4
import * as HtmlRender from "../../../htmlRender/HtmlRender.ts";
5
import * as McpInvocationContext from "../../McpInvocationContext.ts";
6
import * as McpToolAccess from "../../McpToolAccess.ts";
7
+import { loadCaller } from "../../threadAccess.ts";
8
import { HtmlPreviewToolkit, HtmlRenderToolkit, type HtmlToolkit } from "./tools.ts";
9
10
const INVALID_PAGE_ERRORS = new Set([
11
+ "HtmlRenderFileError",
12
+ "HtmlRenderFileTooLargeError",
13
"HtmlRenderImagesNotFoundError",
14
"HtmlRenderImageTooLargeError",
15
"HtmlRenderPageTooLargeError",
@@ -19,12 +22,40 @@ const toFailure = (error: { readonly _tag: string; readonly message: string }) =
22
message: error.message,
23
});
24
25
+// The server reads the file for the caller, so only a thread that reads files without asking may
26
+// pass a path; a supervised thread passes the HTML its provider already let it read.
27
+const pageSource = Effect.fn("HtmlToolkit.pageSource")(function* (input: {
28
+ readonly html?: string | undefined;
29
+ readonly path?: string | undefined;
30
+}) {
31
+ if (input.html !== undefined && input.path === undefined) {
32
+ return { html: input.html } satisfies HtmlRender.HtmlRenderSource;
33
+ }
34
+ if (input.path === undefined || input.html !== undefined) {
35
+ return yield* new OrchestratorMcpFailure({
36
+ code: "invalid_request",
37
+ message: "Pass exactly one of html or path.",
38
+ });
39
+ }
40
+ const { limits } = yield* loadCaller();
41
+ if (limits.runtimeMode !== "full-access") {
42
+ return yield* new OrchestratorMcpFailure({
43
+ code: "capability_denied",
44
+ message: "Pass html instead of path: only full-access threads may pass a path.",
45
+ });
46
+ }
47
+ return { path: input.path } satisfies HtmlRender.HtmlRenderSource;
48
+});
49
+
50
const handlers = {
51
// Local-file access limits this host browser to T3-launched agents.
52
html_preview: McpToolAccess.readsAsCaller((input) =>
53
Effect.gen(function* () {
54
const htmlRender = yield* HtmlRender.HtmlRender;
27
- const { png, ...preview } = yield* htmlRender.preview(input).pipe(Effect.mapError(toFailure));
55
+ const source = yield* pageSource(input);
56
+ const { png, ...preview } = yield* htmlRender
57
+ .preview({ ...source, width: input.width, appearance: input.appearance })
58
+ .pipe(Effect.mapError(toFailure));
59
return {
60
...preview,
61
screenshot: {
@@ -41,8 +72,9 @@ const handlers = {
72
const scope = yield* McpInvocationContext.McpInvocationContext;
73
const { thread } = yield* McpInvocationContext.requireThreadScope(scope, "html_render");
74
const htmlRender = yield* HtmlRender.HtmlRender;
75
+ const source = yield* pageSource(input);
76
const reference = yield* htmlRender
45
- .publish({ threadId: thread.threadId, ...input })
77
+ .publish({ ...source, threadId: thread.threadId, title: input.title, height: input.height })
78
.pipe(Effect.mapError(toFailure));
79
return {
80
htmlRender: reference,
apps/server/src/mcp/toolkits/html/tools.ts
+11
-5
@@ -17,16 +17,21 @@ import * as ThreadManagementService from "../../../orchestration-v2/ThreadManage
17
import * as McpInvocationContext from "../../McpInvocationContext.ts";
18
19
const Html = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(512_000)).annotate({
20
- description: "A complete, self-contained HTML document.",
20
+ description: "Inline HTML.",
21
+});
22
+
23
+const HtmlPath = Schema.String.annotate({
24
+ description: "Absolute .html file path.",
25
});
26
27
const PAGE_RULES =
24
- "Inline <style> and <script>. Local images as absolute paths are inlined. Remote URLs load as-is.";
28
+ "Self-contained: inline <style> and <script>. Local images as absolute paths are inlined. Remote URLs load as-is.";
29
30
export const HtmlPreviewTool = Tool.make("html_preview", {
27
- description: `Check an HTML page before html_render. Returns PNG, contentHeight, and console output. If it says the browser is still installing, call again in a minute. ${PAGE_RULES} Theme and layout rules are in html_render.`,
31
+ description: `Check an HTML page before html_render. Returns PNG, contentHeight, and console output. ${PAGE_RULES} Theme and layout rules are in html_render.`,
32
parameters: Schema.Struct({
29
- html: Html,
33
+ path: Schema.optional(HtmlPath),
34
+ html: Schema.optional(Html),
35
width: Schema.optional(
36
Schema.Int.annotate({
37
description: `CSS pixels, 240 to 1600. Default ${HTML_RENDER_COLUMN_WIDTH}. Use about 390 for phones.`,
@@ -73,7 +78,8 @@ export const HtmlPreviewTool = Tool.make("html_preview", {
78
const HtmlRenderTool = Tool.make(HTML_RENDER_TOOL_NAME, {
79
description: `Show a finished HTML page (chart, diagram, mockup) inline in this thread, above your final reply. Do not announce or restate the page. Frame fits height to ${HTML_RENDER_MAX_FIT_HEIGHT}px. On mobile, content past the given height scrolls. ${HTML_RENDER_LAYOUT_GUIDE} ${HTML_RENDER_THEME_GUIDE}`,
80
parameters: Schema.Struct({
76
- html: Html,
81
+ path: Schema.optional(HtmlPath),
82
+ html: Schema.optional(Html),
83
title: Schema.String.check(
84
Schema.isMinLength(1),
85
Schema.isMaxLength(HTML_RENDER_MAX_TITLE_LENGTH),
apps/server/src/provider/agentContextBudget.json
+3
-3
@@ -1,5 +1,5 @@
1
{
2
- "claudeSystemAppendBytes": 5366,
3
- "codexAdditionalContextBytes": 7096,
4
- "mcpToolListJsonBytes": 48119
2
+ "claudeSystemAppendBytes": 5350,
3
+ "codexAdditionalContextBytes": 7080,
4
+ "mcpToolListJsonBytes": 48116
5
}
evals/mcp-tool-use/cases/visual-preview.ndjson
+4
-3
@@ -1,7 +1,7 @@
1
{"id": "visualizer-served-and-opened-in-preview", "area": "visual-preview", "kind": "bad", "pattern": "visual asked; page served locally and opened with preview_open; no html_render", "experience": "Asked for a visualizer. The agent served an HTML file with http.server and opened it in a preview tab. The user could not load it and never got an inline visual.", "rule": "build a self-contained HTML page, check it with html_preview, then publish it with html_render", "privacy": "commit-safe", "setup": {"role": "top", "context": "./context-usage.json holds this thread's context window usage: {\"windowTokens\": 1000000, \"usedTokens\": 98769, \"parts\": [{\"name\": \"Conversation\", \"tokens\": 66189}, {\"name\": \"Skill and tool listings\", \"tokens\": 12136}, {\"name\": \"Tool schemas\", \"tokens\": 11626}, {\"name\": \"CLAUDE.md\", \"tokens\": 2821}, {\"name\": \"T3 Code instructions\", \"tokens\": 2433}, {\"name\": \"System prompt\", \"tokens\": 1874}, {\"name\": \"Other\", \"tokens\": 1690}]}", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Context window", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:4817/context.html"}}, "world": {"files": {"context-usage.json": "{\"windowTokens\": 1000000, \"usedTokens\": 98769, \"parts\": [{\"name\": \"Conversation\", \"tokens\": 66189}, {\"name\": \"Skill and tool listings\", \"tokens\": 12136}, {\"name\": \"Tool schemas\", \"tokens\": 11626}, {\"name\": \"CLAUDE.md\", \"tokens\": 2821}, {\"name\": \"T3 Code instructions\", \"tokens\": 2433}, {\"name\": \"System prompt\", \"tokens\": 1874}, {\"name\": \"Other\", \"tokens\": 1690}]}\n"}}}, "prompt": "Break down my current context window for this thread. Make a nice visualizer for it.", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"], "shellMustNot": ["http\\.server", "npx (serve|http-server)"]}, "notes": "Data is given in context so the case tests the delivery choice, not the research."}
2
{"id": "render-without-html-preview", "area": "visual-preview", "kind": "bad", "pattern": "html_render without html_preview first (pattern 12)", "experience": "The agent checked a mock page with a shell screenshot, then called html_render with a guessed height and no html_preview.", "rule": "check it with html_preview, then publish it with html_render; html_render height: 'Use html_preview's contentHeight'", "privacy": "commit-safe", "setup": {"role": "top", "context": "The mock page from the last turn is at /tmp/design/index.html.", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Built the mock page at /tmp/design/index.html and checked it at 728px with shot. The layout is clean.", "toolCalls": [{"tool": "Read", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/NewWorktreeToggle.tsx"}, "result": " 1\texport function NewWorktreeToggle() {\n 2\t return <Switch label=\"New worktree\" />;\n 3\t}"}, {"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "Bash", "input": {"command": "~/.claude/skills/shot/scripts/shot /tmp/design/index.html --size 728x2000 --out /tmp/design/r728.png"}, "result": "saved /tmp/design/r728.png (728x1840)"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n", "/tmp/design/r728.png": "<PNG image, 728x1840>", "apps/web/src/NewWorktreeToggle.tsx": "export function NewWorktreeToggle() {\n return <Switch label=\"New worktree\" />;\n}\n"}}}, "prompt": "Show me the four options here so I can pick one.", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"]}, "notes": "The prior shot check is the pressure to skip html_preview.", "maxSteps": 8}
3
{"id": "webmcp-charts-saved-to-site-not-inline", "area": "visual-preview", "kind": "bad", "pattern": "visual asked; agent used the page's own chart tool; no html_render", "experience": "Asked for charts from a site's WebMCP data. The agent saved charts to the site's dashboard. The user wanted inline visuals in T3 Code.", "rule": "When a chart ... would say more than prose, build a self-contained HTML page, check it with html_preview, then publish it with html_render", "privacy": "commit-safe", "setup": {"role": "top", "context": "The user has https://analytics.acme.dev open in preview tab tab_1 (owner unclaimed). The page exposes WebMCP tools: query_usage(sql), top_models(days), save_chart(spec). save_chart saves a chart to the site's own dashboard.", "priorTurns": [{"role": "user", "text": "I have analytics.acme.dev open in the preview browser. It has WebMCP. What can you access from it?"}, {"role": "assistant", "text": "From your tab I can call three WebMCP tools: query_usage (SQL over usage events), top_models (model ranking), and save_chart (saves a chart to the site's dashboard).", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "unclaimed"}]}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.listTools()"}, "result": {"value": ["query_usage", "top_models", "save_chart"]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "unclaimed"}]}, "preview_evaluate": {"value": {"rows": [{"day": "2026-10-14", "model": "opus-5.5", "tokens": 41200000, "costUsd": 388}, {"day": "2026-10-14", "model": "sonnet-5.5", "tokens": 9100000, "costUsd": 41}, {"day": "2026-10-15", "model": "opus-5.5", "tokens": 52800000, "costUsd": 497}, {"day": "2026-10-15", "model": "sonnet-5.5", "tokens": 7700000, "costUsd": 35}, {"day": "2026-10-16", "model": "opus-5.5", "tokens": 38900000, "costUsd": 366}, {"day": "2026-10-16", "model": "sonnet-5.5", "tokens": 12400000, "costUsd": 56}]}}, "html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Usage insights", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "use it to make some charts with cool data I might not have known about", "expect": {"calls": [{"tool": "html_render"}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~save_chart"}}]}, "notes": "The ask is ambiguous ('use it'). Graded to the user's correction: inline html_render. Needs the callsNot runner key (see .md)."}
4
-{"id": "inline-visual-hardcodes-black-background", "area": "visual-preview", "kind": "bad", "pattern": "inline html_render hardcodes a black page background or white text (theme F)", "experience": "Inline visuals came back with a solid black page and white text. They looked wrong in the user's theme. They asked why agents always do this.", "rule": "html_render: 'leave html, body, and the outermost element with no background color. This overrides general style preferences such as a fixed black page background.'", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Root cause was the product-UI 'true black' rule in the user's CLAUDE.md. Today's CLAUDE.md adds an inline-HTML rule, so this may pass now. Keep it as the regression check for that conflict."}
4
+{"id": "inline-visual-hardcodes-black-background", "area": "visual-preview", "kind": "bad", "pattern": "inline html_render hardcodes a black page background or white text (theme F)", "experience": "Inline visuals came back with a solid black page and white text. They looked wrong in the user's theme. They asked why agents always do this.", "rule": "html_render: 'leave html, body, and the outermost element with no background color. This overrides general style preferences such as a fixed black page background.'", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "callsMustNot": [{"tool": "html_render", "args": {"page": "~background:#000"}}, {"tool": "html_render", "args": {"page": "~background: #000"}}, {"tool": "html_render", "args": {"page": "~background-color:#000"}}, {"tool": "html_render", "args": {"page": "~background-color: #000"}}, {"tool": "html_render", "args": {"page": "~background:black"}}, {"tool": "html_render", "args": {"page": "~background: black"}}, {"tool": "html_render", "args": {"page": "~background-color:black"}}, {"tool": "html_render", "args": {"page": "~background-color: black"}}, {"tool": "html_render", "args": {"page": "~color:#fff"}}, {"tool": "html_render", "args": {"page": "~color: #fff"}}, {"tool": "html_render", "args": {"page": "~color:white"}}, {"tool": "html_render", "args": {"page": "~color: white"}}]}, "notes": "Root cause was the product-UI 'true black' rule in the user's CLAUDE.md. Today's CLAUDE.md adds an inline-HTML rule, so this may pass now. Keep it as the regression check for that conflict."}
5
{"id": "followup-edit-reopens-preview-unasked", "area": "visual-preview", "kind": "bad", "pattern": "T3 preview browser used when the user did not ask (theme E)", "experience": "On a plain change request, the agent navigated, clicked and snapshotted the preview browser on localhost and the live site. The user: 'Don't make any changes to the preview browser. Just change code when I ask.'", "rule": "none in T3 text (t3code AGENTS.md: browser use upon request)", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repo: a small todo app. The dev server already runs at http://localhost:3000. `pnpm run deploy` deploys and prints the live URL.", "priorTurns": [{"role": "user", "text": "Build and deploy the starter template. Run the dev server locally and share the link."}, {"role": "assistant", "text": "Built and deployed the starter.\n- Local: http://localhost:3000\n- Live: https://quiet-signal.acme.app", "toolCalls": [{"tool": "Bash", "input": {"command": "pnpm run deploy"}, "result": "vite v7.1.0 building for production...\n✓ built in 1.2s\nDeployed: https://quiet-signal.acme.app"}, {"tool": "preview_open", "input": {"url": "http://localhost:3000"}, "result": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:3000/", "title": "todo"}}, {"tool": "preview_snapshot", "input": {"includeImage": false}, "result": {"url": "http://localhost:3000/", "title": "todo", "loading": false, "visibleText": "Todos\nAdd a task\nAdd"}}]}], "mocks": {"preview_status": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:3000/", "title": "todo"}}, "shellMocks": [{"match": "deploy", "output": "Deployed: https://quiet-signal.acme.app"}], "world": {"files": {"package.json": "{\n \"name\": \"starter\",\n \"scripts\": {\n \"dev\": \"vite --port 3000\",\n \"build\": \"vite build\",\n \"deploy\": \"vite build && hostco deploy dist\"\n },\n \"dependencies\": { \"react\": \"^19.2.0\", \"react-dom\": \"^19.2.0\" },\n \"devDependencies\": { \"@vitejs/plugin-react\": \"^5.0.0\", \"hostco\": \"^3.2.0\", \"vite\": \"^7.1.0\" }\n}\n", "src/App.tsx": "import { Todos } from './todos';\n\nexport function App() {\n return <main><h1>Todos</h1><Todos /></main>;\n}\n", "src/todos.tsx": "import { useState } from 'react';\n\nexport function Todos() {\n const [items, setItems] = useState<string[]>([]);\n const [text, setText] = useState('');\n return (\n <form onSubmit={(e) => { e.preventDefault(); setItems([...items, text]); setText(''); }}>\n <input value={text} onChange={(e) => setText(e.target.value)} placeholder=\"Add a task\" />\n <button>Add</button>\n <ul>{items.map((t) => <li key={t}>{t}</li>)}</ul>\n </form>\n );\n}\n", "src/styles.css": ":root{color-scheme:dark;--bg:#000;--fg:#fff}\nbody{background:var(--bg);color:var(--fg)}\n"}}}, "prompt": "Make it light mode and funnier", "expect": {"notCalls": ["preview_status", "preview_open", "preview_navigate", "preview_snapshot", "preview_act", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"]}, "notes": "The prior turn holds the agent's own preview calls. That is the pressure to repeat them."}
6
{"id": "demo-build-opens-preview-unasked", "area": "visual-preview", "kind": "bad", "pattern": "T3 preview browser used when the user did not ask (theme E)", "experience": "Asked to build and deploy a demo and share the link. The agent opened the preview and kept snapshotting and clicking. The user: 'Please stop trying to check so much, just ship.'", "rule": "none in T3 text (t3code AGENTS.md: browser use upon request)", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repo: a fresh starter app with a todo list. `pnpm dev` serves http://localhost:3017. `pnpm run deploy` deploys and prints the live URL.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:3017/"}, "preview_snapshot": {"error": {"_tag": "PreviewAutomationTimeoutError", "operation": "snapshot", "failureCount": 1, "message": "Preview automation snapshot timed out after 15000ms."}}}, "shellMocks": [{"match": "deploy", "output": "Deployed: https://clear-garden.acme.app"}, {"match": "pnpm dev", "output": "Local: http://localhost:3017/"}], "world": {"files": {"package.json": "{\n \"name\": \"starter\",\n \"scripts\": {\n \"dev\": \"vite --port 3017\",\n \"build\": \"vite build\",\n \"deploy\": \"vite build && hostco deploy dist\"\n },\n \"dependencies\": { \"react\": \"^19.2.0\", \"react-dom\": \"^19.2.0\" },\n \"devDependencies\": { \"@vitejs/plugin-react\": \"^5.0.0\", \"hostco\": \"^3.2.0\", \"vite\": \"^7.1.0\" }\n}\n", "src/App.tsx": "import { Todos } from './todos';\n\nexport function App() {\n return <main><h1>Todos</h1><Todos /></main>;\n}\n", "src/todos.tsx": "import { useState } from 'react';\n\nexport function Todos() {\n const [items, setItems] = useState<string[]>([]);\n const [text, setText] = useState('');\n return (\n <form onSubmit={(e) => { e.preventDefault(); setItems([...items, text]); setText(''); }}>\n <input value={text} onChange={(e) => setText(e.target.value)} placeholder=\"Add a task\" />\n <button>Add</button>\n <ul>{items.map((t) => <li key={t}>{t}</li>)}</ul>\n </form>\n );\n}\n", "src/styles.css": ":root{color-scheme:dark;--bg:#000;--fg:#fff}\nbody{background:var(--bg);color:var(--fg)}\n"}}}, "prompt": "Make me a fun kanban demo from this starter. Deploy as soon as you can and deploy as you make changes. Share the link as soon as you have it.", "expect": {"notCalls": ["preview_open", "preview_navigate", "preview_snapshot", "preview_act", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"], "textMust": ["https://clear-garden\\.acme\\.app"]}, "notes": "preview_status alone is not graded. Any preview action is.", "maxSteps": 10}
7
{"id": "review-child-opens-preview-for-public-page", "area": "visual-preview", "kind": "bad", "pattern": "delegated child uses the T3 preview to read a public page", "experience": "A review child opened render.com/pricing in a T3 preview tab, hit 2 snapshot timeouts, and left the tab open. A plain fetch was enough.", "rule": "none (Codex browser block: 'prefer those tools for browser navigation')", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "./cost.md contains: 'Render Pro workspace $25/mo. Starter instance $7/mo. Postgres basic-256mb $6/mo. Fly shared-cpu-1x 256MB $1.94/mo.'", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "https://render.com/pricing"}, "preview_snapshot": {"error": {"_tag": "PreviewAutomationTimeoutError", "operation": "snapshot", "failureCount": 1, "message": "Preview automation snapshot timed out after 15000ms."}}}, "delegateRole": "review", "world": {"files": {"cost.md": "Render Pro workspace $25/mo. Starter instance $7/mo. Postgres basic-256mb $6/mo. Fly shared-cpu-1x 256MB $1.94/mo.\n"}, "pages": {"https://render.com/pricing": "<html>... Professional $25/user/month ... Starter $7/month ... Basic-256mb $6/month ...</html>", "https://fly.io/docs/about/pricing": "<html>... shared-cpu-1x 256MB $2.02/mo ...</html>"}}}, "prompt": "Adversarially check the cost comparison in ./cost.md. Verify the prices on the live web (https://render.com/pricing, https://fly.io/docs/about/pricing/) and cite URLs. Write your findings to ./review.md. Do not edit other files.", "expect": {"notCalls": ["preview_status", "preview_open", "preview_snapshot", "preview_navigate"]}, "notes": "Codex gets the browser block that says 'prefer preview tools'. Claude does not, so this may not reproduce on Claude."}
@@ -16,8 +16,8 @@
16
{"id": "guard-x-card-png-uses-shot", "area": "visual-preview", "kind": "guard", "pattern": "headless screenshot ask -> shot skill or a shell browser, not the preview", "experience": "The user wanted one image to post on X. The agent built a card and made a PNG with shot, without the preview browser.", "rule": "The user's CLAUDE.md: 'Prefer T3 preview tools for its browser ... and shot for headless web screenshots and videos'", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Go through what I landed in the last 3 weeks and summarize it."}, {"role": "assistant", "text": "Since Oct 3: 312 PRs merged across 12 repos, 5,900 commits, 4 new projects, and about 180K lines added.", "toolCalls": [{"tool": "Bash", "input": {"command": "~/bin/landed --since 2026-10-03"}, "result": "Since 2026-10-03 (3 weeks):\nPRs merged: 312 across 12 repos\nCommits: 5,900\nLines added: 180K\nNew projects: 4 (relay, notes, sync-cli, status-page)"}]}], "shellMocks": [{"match": "landed", "output": "Since 2026-10-03 (3 weeks):\nPRs merged: 312 across 12 repos\nCommits: 5,900\nLines added: 180K\nNew projects: 4 (relay, notes, sync-cli, status-page)"}, {"match": "(^|/)shot\\s", "output": "saved /tmp/x-card/x-card.png (2400x1350, playwright headless)"}], "world": {"files": {"/tmp/x-card/x-card.png": "<PNG image, 2400x1350>"}}, "mocks": {"Skill": "Launching skill: shot\n\nHeadless screenshots of web pages and local HTML files. Run `shot <url-or-file> --size WxH --out <file.png>`. It never opens a visible browser."}, "harness": "claude"}, "prompt": "I want something I can fit into a single screenshot. I'm trying to reply on X with a \"here's what I've done in the last 3 weeks\" type thing.", "expect": {"notCalls": ["preview_open", "preview_navigate", "preview_snapshot", "preview_act", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"], "textMust": ["\\.png"]}, "notes": "Numbers are fake. Catches a rewrite that pushes every screenshot through the T3 preview. The real thread also ran an unasked Workflow; that is not graded here.", "maxSteps": 10}
17
{"id": "guard-ios-sim-device-tools", "area": "visual-preview", "kind": "guard", "pattern": "'iOS simulator using T3 device manager' -> device_list then device_open", "experience": "The user asked to run the mobile app in the iOS simulator through T3's device manager. The agent listed devices and opened one.", "rule": "device_list: 'Call this before device_open when you do not know a device id.'", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"device_list": {"hostStatuses": {"local": {"status": "ready"}}, "hosts": [{"id": "local", "kind": "local", "label": "This machine", "platforms": [{"platform": "ios", "available": true}, {"platform": "android", "available": false, "reason": "Android Emulator is not installed."}], "devices": [{"id": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9", "platform": "ios", "name": "iPhone 17", "version": "iOS 26.5", "booted": false, "physical": false}]}], "openDevices": []}, "device_open": {"device": {"hostId": "local", "id": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9", "platform": "ios", "name": "iPhone 17", "version": "iOS 26.5", "booted": true, "physical": false}, "quickStart": "The user is watching iPhone 17 (iOS 26.5) in the Device panel."}}, "shellMocks": [{"match": "(expo|pnpm|npm|npx) ", "output": "Metro waiting on exp://localhost:8081\nOpening on iPhone 17"}]}, "prompt": "Spin up the mobile app in the ios simulator using device manager in t3 code", "expect": {"order": ["device_list", "device_open"], "calls": [{"tool": "device_open", "args": {"deviceId": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9"}}], "shellMustNot": ["xcrun simctl boot"]}, "notes": "Device tools are deferred for Claude; the agent must load them via ToolSearch."}
18
{"id": "guard-capabilities-recheck-after-provider-returns", "area": "visual-preview", "kind": "guard", "pattern": "re-call orchestrator_capabilities when the catalog really changed", "experience": "A provider was disabled earlier. When the user said it was back, the agent re-checked capabilities and then delegated to it.", "rule": "'Use orchestrator_capabilities to discover the current provider/model IDs'", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Make a smaller, clearer PR that fixes the hidden-tab bug. Have Gemini 3 Pro write the PR description, you tend to be too wordy."}, {"role": "assistant", "text": "The fix is pushed to branch fix/hidden-tab. Gemini is disabled in T3 Code and lists no models, so I could not ask it to write the description yet.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/tabs.ts", "old_string": " focus(tab);", "new_string": " if (!tab.hidden) focus(tab);"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/tabs.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(web): hidden tabs no longer steal focus\""}, "result": "[fix/hidden-tab 4d2c1b0] fix(web): hidden tabs no longer steal focus\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push -u origin fix/hidden-tab"}, "result": "To github.com:acme/app.git\n * [new branch] fix/hidden-tab -> fix/hidden-tab\nbranch 'fix/hidden-tab' set up to track 'origin/fix/hidden-tab'."}, {"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "11111111-1111-4111-8111-111111111111", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [], "runtimeMode": "full-access", "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Opus 5.5"}, {"id": "claude-sonnet-5-5", "label": "Sonnet 5.5"}]}, {"providerInstanceId": "gemini", "driverKind": "gemini", "displayName": "Gemini", "enabled": false, "models": []}]}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "11111111-1111-4111-8111-111111111111", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [], "runtimeMode": "full-access", "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Opus 5.5"}, {"id": "claude-sonnet-5-5", "label": "Sonnet 5.5"}]}, {"providerInstanceId": "gemini", "driverKind": "gemini", "displayName": "Gemini", "enabled": true, "models": [{"id": "gemini-3-pro", "label": "Gemini 3 Pro"}]}]}, "delegate_task": "default"}, "world": {"branch": "fix/hidden-tab", "commits": [{"sha": "4d2c1b0", "subject": "fix(web): hidden tabs no longer steal focus"}], "diff": "diff --git a/apps/web/src/tabs.ts b/apps/web/src/tabs.ts\n--- a/apps/web/src/tabs.ts\n+++ b/apps/web/src/tabs.ts\n@@ -10,1 +10,1 @@\n- focus(tab);\n+ if (!tab.hidden) focus(tab);\n"}}, "prompt": "You should have Gemini now", "expect": {"order": ["orchestrator_capabilities", "delegate_task"], "calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "gemini"}}], "maxCalls": {"orchestrator_capabilities": 1}}, "notes": "Provider swapped to Gemini to keep the case commit-safe. Twin of capabilities-reread-between-waves: a rewrite must not ban all re-checks."}
19
-{"id": "guard-inline-visual-preview-then-render", "area": "visual-preview", "kind": "guard", "pattern": "inline visual ask -> html_preview then html_render", "experience": "The user asked for an inline visualization of a small table. The agent checked it with html_preview and published it with html_render.", "rule": "check it with html_preview, then publish it with html_render before your final reply", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Failed implementations", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Make a nice visualization in line here for this data showcasing how many bad implementations have been made and how expensive they were\n\nFeature | attempts | failed attempts | cost of failed attempts\nSidebar drag and drop | 6 | 4 | $310\nCommand+F search | 3 | 1 | $45\nPR watch | 5 | 3 | $220\nInline HTML frames | 4 | 2 | $160\nDevice panel | 2 | 0 | $0", "expect": {"order": ["html_preview", "html_render"], "notCalls": ["preview_open"], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Data is fake."}
20
-{"id": "guard-rerender-after-black-background-correction", "area": "visual-preview", "kind": "guard", "pattern": "theme correction -> re-render with transparent page and theme variables", "experience": "After the user complained about black backgrounds, the agent re-rendered with a transparent page and T3 theme variables.", "rule": "html_render: 'by default leave html, body, and the outermost element with no background color'; T3 injects --background, --foreground, ...", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%"}, {"role": "assistant", "text": "Gemini 3 Pro re-reads files about 4x more than the other two.", "toolCalls": [{"tool": "html_preview", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}}, {"tool": "html_render", "input": {"title": "Tool-call audit", "height": 560, "html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Why do you always add awful black backgrounds to your inline html? Fix this one.", "expect": {"order": ["html_preview", "html_render"], "calls": [{"tool": "html_render", "args": {"html": "~var(--"}}], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Twin of inline-visual-hardcodes-black-background. The prior render is the pressure to keep its CSS."}
19
+{"id": "guard-inline-visual-preview-then-render", "area": "visual-preview", "kind": "guard", "pattern": "inline visual ask -> html_preview then html_render", "experience": "The user asked for an inline visualization of a small table. The agent checked it with html_preview and published it with html_render.", "rule": "check it with html_preview, then publish it with html_render before your final reply", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Failed implementations", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Make a nice visualization in line here for this data showcasing how many bad implementations have been made and how expensive they were\n\nFeature | attempts | failed attempts | cost of failed attempts\nSidebar drag and drop | 6 | 4 | $310\nCommand+F search | 3 | 1 | $45\nPR watch | 5 | 3 | $220\nInline HTML frames | 4 | 2 | $160\nDevice panel | 2 | 0 | $0", "expect": {"order": ["html_preview", "html_render"], "notCalls": ["preview_open"], "callsMustNot": [{"tool": "html_render", "args": {"page": "~background:#000"}}, {"tool": "html_render", "args": {"page": "~background: #000"}}, {"tool": "html_render", "args": {"page": "~background-color:#000"}}, {"tool": "html_render", "args": {"page": "~background-color: #000"}}, {"tool": "html_render", "args": {"page": "~background:black"}}, {"tool": "html_render", "args": {"page": "~background: black"}}, {"tool": "html_render", "args": {"page": "~background-color:black"}}, {"tool": "html_render", "args": {"page": "~background-color: black"}}, {"tool": "html_render", "args": {"page": "~color:#fff"}}, {"tool": "html_render", "args": {"page": "~color: #fff"}}, {"tool": "html_render", "args": {"page": "~color:white"}}, {"tool": "html_render", "args": {"page": "~color: white"}}]}, "notes": "Data is fake."}
20
+{"id": "guard-rerender-after-black-background-correction", "area": "visual-preview", "kind": "guard", "pattern": "theme correction -> re-render with transparent page and theme variables", "experience": "After the user complained about black backgrounds, the agent re-rendered with a transparent page and T3 theme variables.", "rule": "html_render: 'by default leave html, body, and the outermost element with no background color'; T3 injects --background, --foreground, ...", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%"}, {"role": "assistant", "text": "Gemini 3 Pro re-reads files about 4x more than the other two.", "toolCalls": [{"tool": "html_preview", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}}, {"tool": "html_render", "input": {"title": "Tool-call audit", "height": 560, "html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Why do you always add awful black backgrounds to your inline html? Fix this one.", "expect": {"order": ["html_preview", "html_render"], "calls": [{"tool": "html_render", "args": {"page": "~var(--"}}], "callsMustNot": [{"tool": "html_render", "args": {"page": "~background:#000"}}, {"tool": "html_render", "args": {"page": "~background: #000"}}, {"tool": "html_render", "args": {"page": "~background-color:#000"}}, {"tool": "html_render", "args": {"page": "~background-color: #000"}}, {"tool": "html_render", "args": {"page": "~background:black"}}, {"tool": "html_render", "args": {"page": "~background: black"}}, {"tool": "html_render", "args": {"page": "~background-color:black"}}, {"tool": "html_render", "args": {"page": "~background-color: black"}}, {"tool": "html_render", "args": {"page": "~color:#fff"}}, {"tool": "html_render", "args": {"page": "~color: #fff"}}, {"tool": "html_render", "args": {"page": "~color:white"}}, {"tool": "html_render", "args": {"page": "~color: white"}}]}, "notes": "Twin of inline-visual-hardcodes-black-background. The prior render is the pressure to keep its CSS."}
21
{"id": "control-fix-failing-tests", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "link_pull_requests", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: fix failing tests", "experience": "Fix failing tests from a release run. No T3 tool is needed.", "setup": {"role": "top", "shellMocks": [{"match": "gh run (list|view)", "output": "release run 4411 failed\nX test (web): 2 failed\n src/banner.test.ts > hides notice on click: element is not visible\n src/banner.test.ts > keeps usage notice: Timeout 5000ms exceeded"}, {"match": "(vitest|pnpm test|npm test|vp test)", "output": "FAIL src/banner.test.ts > hides notice on click\n Error: element is not visible (hover expanded the stack first)\nTests: 2 failed, 47 passed"}], "world": {"files": {"src/banner.test.ts": "test('hides notice on click', async () => {\n await user.hover(stack)\n await user.click(screen.getByRole('button', { name: 'Dismiss' }))\n})\n", "src/banner.tsx": "export function Banner() {\n return <div className=\"stack\" onMouseEnter={expand}>...</div>;\n}\n"}}}, "prompt": "Bunch of test failures on main in the last release. Figure out the cause and fix.", "notes": "Same thread later has PR overhead cases; the first turn is clean."}
22
{"id": "control-rename-auth-file", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "link_pull_requests", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: rename a file", "experience": "Rename a local config file. No T3 tool is needed.", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Where does the acme CLI keep its login?"}, {"role": "assistant", "text": "In ~/.config/acme/auth.json."}], "world": {"files": {"~/.config/acme/auth.json": "{\"token\":\"<redacted>\"}\n", "~/.config/acme/config.json": "{\"region\":\"us\"}\n"}}}, "prompt": "Rename the auth file to auth_legacy.json for me"}
23
{"id": "control-explain-why-change", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "link_pull_requests", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: explain a change from git history", "experience": "Explain why a commit was made. No T3 tool is needed.", "setup": {"role": "top", "context": "HEAD is commit a1b2c3d 'build: drop the typescript package from the runtime image'.", "world": {"commits": [{"sha": "a1b2c3d", "subject": "build: drop the typescript package from the runtime image", "body": "The server now runs compiled JS. tsc was only used at build time and added 23 MB to the image."}], "diff": "diff --git a/Dockerfile b/Dockerfile\n--- a/Dockerfile\n+++ b/Dockerfile\n@@ -10,1 +10,1 @@\n-RUN pnpm install --prod --filter server... typescript\n+RUN pnpm install --prod --filter server...\n"}}, "prompt": "Why was this change made?"}
@@ -30,3 +30,4 @@
30
{"id": "dev-server-opened-blank-then-navigated", "area": "visual-preview", "kind": "bad", "pattern": "preview_open on a blank tab, then preview_navigate to the dev-server port", "experience": "Asked to open a dev server in the T3 preview, agents opened a blank tab and navigated it to the port in a second call. 121 of these pairs in a month of maintainer threads.", "rule": "preview_open takes a url but not a dev-server target", "privacy": "commit-safe", "setup": {"role": "top", "context": "CLAUDE.md says: for t3code browser work, use the server-side preview_* mcp tools, with `target: {kind: \"environment-port\", port: ...}` for dev servers in the current environment. The dev server for this repo is running; `vp run dev` printed `Local: http://localhost:5173/`.", "mocks": {"preview_status": {"available": true, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "open the dev server in the t3 preview, i want to click around", "expect": {"calls": [{"tool": "preview_open", "args": {"target.port": 5173}}], "maxCalls": {"preview_open": 1, "preview_navigate": 0}}, "maxSteps": 6, "source": {"thread": "Maintainer threads, 2026-10: preview_open with no url followed by preview_navigate to an environment-port target, 121 times"}}
31
{"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."}
32
{"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return <PermissionsMenu side=\"top\" align=\"end\" />; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."}
33
+{"id": "revise-page-resends-html", "area": "visual-preview", "kind": "bad", "pattern": "revising a page on disk resends its full HTML to html_preview and html_render", "experience": "The pages already existed as generated files on disk, but the agent retyped each one into html_preview and again into html_render, about four minutes per call. To save time it published fixes without previewing them again.", "source": {"discussion": "https://github.com/pingdotgg/t3code/discussions/23303"}, "rule": "edit the .html file and pass its path to html_preview and html_render instead of resending its html", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Here are four options.", "toolCalls": [{"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "html_preview", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}, "result": "{\"width\":728,\"contentHeight\":1840,\"capturedHeight\":1840,\"consoleMessages\":[]}"}, {"tool": "html_render", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n", "title": "New worktree: four options", "height": 1840}, "result": "{\"htmlRender\":{\"attachmentId\":\"att-1\",\"title\":\"New worktree: four options\",\"height\":1840},\"message\":\"Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.\"}"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}}}, "prompt": "Make the Send in new worktree button in option C blue, and show me the page again.", "expect": {"calls": [{"tool": "html_preview", "args": {"path": "/tmp/design/index.html", "html": null}}, {"tool": "html_render", "args": {"path": "/tmp/design/index.html", "html": null}}], "callsMustNot": [{"tool": "html_preview", "args": {"html": "~Ask anything"}}, {"tool": "html_render", "args": {"html": "~Ask anything"}}], "order": ["html_preview", "html_render"]}, "notes": "On main neither tool has a path parameter."}
evals/mcp-tool-use/run.test.ts
+122
@@ -298,6 +298,128 @@ it("passes a good run and names every rule a bad run breaks", () => {
298
]);
299
});
300
301
+it("grades a page passed by path on the file the agent last wrote there", () => {
302
+ const expect = {
303
+ calls: [{ tool: "html_render", args: { path: "/tmp/v.html", page: "~var(--" } }],
304
+ callsMustNot: [{ tool: "html_render", args: { page: "~#000" } }],
305
+ };
306
+ const run = (calls: SampleRun["calls"]): SampleRun => ({
307
+ calls,
308
+ text: "",
309
+ stopReason: "end_turn",
310
+ });
311
+ const write = {
312
+ step: 0,
313
+ tool: "Write",
314
+ input: { file_path: "/tmp/v.html", content: "<p style=background:#000>" },
315
+ };
316
+ const render = { step: 2, tool: "html_render", input: { path: "/tmp/v.html" } };
317
+ const fix = {
318
+ step: 1,
319
+ tool: "Edit",
320
+ input: { file_path: "/tmp/v.html", old_string: "#000", new_string: "var(--card)" },
321
+ };
322
+
323
+ assert.deepEqual(failed(expect, run([write, fix, render])), []);
324
+ assert.deepEqual(failed(expect, run([write, fix, { ...render, failed: true }])), [
325
+ 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
326
+ ]);
327
+ const refusedInline = { ...render, failed: true, input: { html: "<p style=background:#000>" } };
328
+ assert.deepEqual(failed(expect, run([write, fix, refusedInline, render])), [
329
+ 'callsMustNot html_render {"page":"~#000"}',
330
+ ]);
331
+ const refusedPath = { ...render, step: 1, failed: true };
332
+ assert.deepEqual(failed(expect, run([write, refusedPath, fix, render])), []);
333
+ const patchUnknown = {
334
+ step: 1,
335
+ tool: "Edit",
336
+ input: { patch: "*** Update File: /tmp/v.html\n@@\n-<h1>A</h1>\n+<h1>var(--card)</h1>\n" },
337
+ };
338
+ assert.deepEqual(failed(expect, run([patchUnknown, render])), [
339
+ 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
340
+ "page known html_render /tmp/v.html",
341
+ ]);
342
+ // A relative add, then an update by absolute path the grader cannot replay: the page is unknown.
343
+ const added = {
344
+ ...write,
345
+ input: { file_path: "v.html", content: "<p style=background:var(--card)>" },
346
+ };
347
+ const patchAbsolute = { ...patchUnknown, step: 2 };
348
+ assert.include(
349
+ failed(expect, run([added, patchAbsolute, render])),
350
+ "page known html_render /tmp/v.html",
351
+ );
352
+ const appended = {
353
+ ...write,
354
+ tool: "Bash",
355
+ input: { command: "cat >> /tmp/v.html <<'EOF'\n<p>var(--x)</p>\nEOF" },
356
+ };
357
+ assert.include(failed(expect, run([appended, render])), "page known html_render /tmp/v.html");
358
+ const patched = {
359
+ step: 1,
360
+ tool: "Edit",
361
+ input: {
362
+ patch: "*** Update File: /tmp/v.html\n@@\n-background:#000\n+background:var(--card)\n",
363
+ },
364
+ };
365
+ assert.deepEqual(failed(expect, run([write, patched, render])), []);
366
+ const editUnseen = { ...fix, input: { ...fix.input, old_string: "<h1>", new_string: "<h2>" } };
367
+ const editMarker = { ...fix, step: 2, input: { ...fix.input, old_string: "unknown" } };
368
+ assert.include(
369
+ failed(expect, run([editUnseen, editMarker, render])),
370
+ "page known html_render /tmp/v.html",
371
+ );
372
+ const dollar = { ...fix, input: { ...fix.input, new_string: "var(--card)$&" } };
373
+ assert.deepEqual(failed(expect, run([write, dollar, render])), []);
374
+ const ambiguous = { ...write, input: { ...write.input, content: "<p>white</p>\n<i>white</i>" } };
375
+ const patchAmbiguous = {
376
+ step: 1,
377
+ tool: "Edit",
378
+ input: { patch: "*** Update File: /tmp/v.html\n@@\n-white\n+var(--card)\n" },
379
+ };
380
+ assert.include(
381
+ failed(expect, run([ambiguous, patchAmbiguous, render])),
382
+ "page known html_render /tmp/v.html",
383
+ );
384
+ const twice = { ...write, input: { ...write.input, content: "<p style=background:#000>#000" } };
385
+ const fixAll = { ...fix, input: { ...fix.input, replace_all: true } };
386
+ assert.deepEqual(failed(expect, run([twice, fixAll, render])), []);
387
+ const spoofed = { ...render, input: { path: "/tmp/v.html", page: "var(--card)" } };
388
+ assert.deepEqual(failed(expect, run([spoofed])), [
389
+ 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
390
+ "page known html_render /tmp/v.html",
391
+ ]);
392
+ // T3 may still refuse an unanswered page call, so early grading waits for its result.
393
+ assert.isFalse(failedEarly(expect, run([write, render])));
394
+ assert.isTrue(failedEarly(expect, run([write, { ...render, failed: false }])));
395
+ assert.deepEqual(failed({ notCalls: ["html_render"] }, run([{ ...render, failed: true }])), [
396
+ "notCalls html_render",
397
+ ]);
398
+ const relative = { ...write, input: { ...write.input, file_path: "././v.html" } };
399
+ assert.deepEqual(failed(expect, run([relative, render])), [
400
+ 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
401
+ 'callsMustNot html_render {"page":"~#000"}',
402
+ ]);
403
+ const fixed = {
404
+ ...write,
405
+ input: { ...write.input, content: "<p style=background:var(--card)>" },
406
+ };
407
+ const sameFixed = { ...relative, input: { ...relative.input, content: fixed.input.content } };
408
+ assert.deepEqual(failed(expect, run([fixed, sameFixed, render])), []);
409
+ assert.deepEqual(failed(expect, run([fixed, relative, render])), [
410
+ 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
411
+ "page known html_render /tmp/v.html",
412
+ ]);
413
+ assert.deepEqual(failed(expect, run([write, render])), [
414
+ 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
415
+ 'callsMustNot html_render {"page":"~#000"}',
416
+ ]);
417
+ assert.deepEqual(failed(expect, run([render])), [
418
+ 'calls html_render {"path":"/tmp/v.html","page":"~var(--"}',
419
+ "page known html_render /tmp/v.html",
420
+ ]);
421
+});
422
+
423
it("bounds numbers and reads target.options in either schema form", () => {
424
const expect = {
425
calls: [
evals/mcp-tool-use/run.ts
+75
-6
@@ -82,6 +82,8 @@ import {
82
import {
83
argvForm,
84
gradeSecret,
85
+ scriptsBefore,
86
+ UNKNOWN_SCRIPT,
87
scriptSources,
88
SecretExpect,
89
secretDiagnostics,
@@ -1570,6 +1572,16 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun
1572
if (found.length === 0) return toolResult(call.id, "No matching deferred tools found");
1573
return loadTools(tools, call.id, found);
1574
}
1575
+ // T3 refuses a page call without exactly one source before any mocked result.
1576
+ const sources = [input.html, input.path].filter(
1577
+ (source) => source !== undefined && source !== null,
1578
+ );
1579
+ if (isPageTool(name) && (sources.length !== 1 || typeof sources[0] !== "string")) {
1580
+ return failed("Pass exactly one of html or path.");
1581
+ }
1582
+ if (isPageTool(name) && typeof input.path === "string" && !input.path.startsWith("/")) {
1583
+ return failed(`${input.path} is not a readable HTML document. Pass an absolute path.`);
1584
+ }
1585
const mock = testCase.setup.mocks?.[name];
1586
// T3 answers device calls from the world. A mock that is a T3 device error fails inside T3.
1587
const failure = deviceFailure(mock);
@@ -1692,15 +1704,50 @@ export const pastedText = ({ prompt, setup }: Pick<McpToolUseCase, "prompt" | "s
1704
prompt,
1705
].join("\n");
1706
1707
+const isPageTool = (tool: string) => tool === "html_preview" || tool === "html_render";
1708
+
1709
+/** `page` is the HTML a page tool shows: its inline `html`, or the file the agent wrote at `path`. */
1710
+const withPage = (tool: string, input: unknown, files: ReadonlyMap<string, string>) => {
1711
+ if (!isPageTool(tool) || !isRecord(input)) return input;
1712
+ const path = valueText(input.path);
1713
+ // A patch may name the file relative to the agent's working directory. When written files that
1714
+ // could be this one differ, the page is unknown, so page checks fail rather than grade a stale copy.
1715
+ const written = [...files].filter(
1716
+ ([file]) =>
1717
+ file === path ||
1718
+ (!file.startsWith("/") && path.endsWith(`/${file.replace(/^(?:\.\/)+/, "")}`)),
1719
+ );
1720
+ const pages = new Set(written.map(([, page]) => page));
1721
+ const page =
1722
+ typeof input.html === "string"
1723
+ ? input.html
1724
+ : pages.size === 1 && !pages.has(UNKNOWN_SCRIPT)
1725
+ ? written[0]![1]
1726
+ : undefined;
1727
+ // The grader derives `page`; a call cannot supply its own.
1728
+ return { ...input, page };
1729
+};
1730
+
1731
/** `pasted` is the thread's user text (`pastedText`), for request_secret's `value`. */
1732
export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pasted = "") => {
1733
const checks: Array<{ readonly check: string; readonly pass: boolean }> = [];
1734
const add = (check: string, pass: boolean) => checks.push({ check, pass });
1699
- const calls = run.calls.map((call) => ({ ...call, input: withOptionsObject(call.input) }));
1735
+ const written = scriptsBefore(run.calls, true);
1736
+ // A refused page call showed nothing, so it neither previews nor renders, and no file stands in
1737
+ // for its page. Inline HTML it sent still counts against callsMustNot.
1738
+ const attempted = run.calls.map((call, index) => ({
1739
+ ...call,
1740
+ input: withPage(
1741
+ call.tool,
1742
+ withOptionsObject(call.input),
1743
+ call.failed ? new Map() : written[index]!,
1744
+ ),
1745
+ }));
1746
+ const calls = attempted.filter((call) => !(call.failed && isPageTool(call.tool)));
1747
const count = (tool: string) => calls.filter((call) => call.tool === tool).length;
1748
// `tool` may list alternatives, as in "link_pull_request|watch_pull_request".
1702
- const found = ({ tool, args = {} }: McpToolUseCall) =>
1703
- calls.some((call) => tool.split("|").includes(call.tool) && matches(args, call.input));
1749
+ const found = ({ tool, args = {} }: McpToolUseCall, among = calls) =>
1750
+ among.some((call) => tool.split("|").includes(call.tool) && matches(args, call.input));
1751
const label = ({ tool, args }: McpToolUseCall) => (args ? `${tool} ${valueText(args)}` : tool);
1752
for (const call of expect.calls ?? []) add(`calls ${label(call)}`, found(call));
1753
if (expect.distinctCalls !== undefined) {
@@ -1714,8 +1761,22 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun, pa
1761
),
1762
);
1763
}
1717
- for (const call of expect.callsMustNot ?? []) add(`callsMustNot ${label(call)}`, !found(call));
1718
- for (const tool of expect.notCalls ?? []) add(`notCalls ${tool}`, count(tool) === 0);
1764
+ for (const call of expect.callsMustNot ?? [])
1765
+ add(`callsMustNot ${label(call)}`, !found(call, attempted));
1766
+ // A page check must not pass because the grader never saw the file a path call shows.
1767
+ const pageTools = [...(expect.calls ?? []), ...(expect.callsMustNot ?? [])].flatMap((call) =>
1768
+ call.args?.page === undefined ? [] : call.tool.split("|"),
1769
+ );
1770
+ for (const call of calls)
1771
+ if (
1772
+ pageTools.includes(call.tool) &&
1773
+ isRecord(call.input) &&
1774
+ call.input.path !== undefined &&
1775
+ call.input.page === undefined
1776
+ )
1777
+ add(`page known ${call.tool} ${valueText(call.input.path)}`, false);
1778
+ for (const tool of expect.notCalls ?? [])
1779
+ add(`notCalls ${tool}`, !attempted.some((call) => call.tool === tool));
1780
if (expect.order !== undefined) {
1781
const firsts = expect.order.map((tool) => calls.findIndex((call) => call.tool === tool));
1782
const inOrder = firsts.every((at, i) => at > (firsts[i - 1] ?? -1));
@@ -1776,7 +1837,15 @@ const SETTLED =
1837
/^(?:(?:notCalls|callsMustNot|shellMustNot|maxCalls|textMustNot) |secret\.(?:chatAsk|askTools|values|fileUse|noCardDemand|afterDecline) )/;
1838
/** True once the sample has failed for good, so the rest of it cannot change the result. */
1839
export const failedEarly = (expect: McpToolUseCase["expect"], run: SampleRun, pasted = "") =>
1779
- gradeSample(expect, run, pasted).some((check) => !check.pass && SETTLED.test(check.check));
1840
+ gradeSample(
1841
+ expect,
1842
+ // A page call T3 has not answered yet may still be refused, so it is graded once it settles.
1843
+ {
1844
+ ...run,
1845
+ calls: run.calls.filter((call) => !isPageTool(call.tool) || call.failed !== undefined),
1846
+ },
1847
+ pasted,
1848
+ ).some((check) => !check.pass && SETTLED.test(check.check));
1849
1850
const requestBody = ({ base, tools }: Sample, effort: string, messages: unknown) => ({
1851
model: base.model,
evals/mcp-tool-use/secret-grading.ts
+59
-23
@@ -1364,22 +1364,35 @@ const heredocWrite = (part: string) => {
1364
? { path: target[3]!, source: heredoc[4]!, append: /^(?:>>|-a)/.test(target[1] ?? "") }
1365
: undefined;
1366
};
1367
-const noteHeredoc = (part: string, scripts: Map<string, string>) => {
1367
+/** In exact mode, a file whose contents the grader cannot rebuild. Any later update keeps it so. */
1368
+export const UNKNOWN_SCRIPT = "\u0000unknown";
1369
+const noteHeredoc = (part: string, scripts: Map<string, string>, exact: boolean) => {
1370
const write = heredocWrite(part);
1369
- if (write)
1370
- scripts.set(
1371
- write.path,
1372
- write.append ? `${scripts.get(write.path) ?? ""}\n${write.source}` : write.source,
1373
- );
1371
+ if (!write) return;
1372
+ const current = scripts.get(write.path);
1373
+ scripts.set(
1374
+ write.path,
1375
+ !write.append
1376
+ ? write.source
1377
+ : exact && (current === undefined || current === UNKNOWN_SCRIPT)
1378
+ ? UNKNOWN_SCRIPT
1379
+ : `${current ?? ""}\n${write.source}`,
1380
+ );
1381
};
1375
-/** Applies a Codex apply_patch to the scripts: added files, and updated files hunk by hunk. */
1376
-const applyPatch = (patch: string, scripts: Map<string, string>) => {
1382
+/**
1383
+ * Applies a Codex apply_patch to the scripts: added files, and updated files hunk by hunk. `exact`
1384
+ * marks a file unknown when an update cannot be applied in full, instead of guessing.
1385
+ */
1386
+const applyPatch = (patch: string, scripts: Map<string, string>, exact: boolean) => {
1387
for (const [, op, path = "", body = ""] of patch.matchAll(
1388
/^\*\*\* (Add|Update|Delete) File: (.+)\n([\s\S]*?)(?=^\*\*\* (?:Add|Update|Delete) File: |^\*\*\* End Patch|(?![\s\S]))/gm,
1389
)) {
1390
const lines = body.split("\n");
1381
- if (op === "Delete") scripts.delete(path);
1382
- else if (op === "Add")
1391
+ // In exact mode a deleted file stays unknown, so a relative alias cannot supply a stale copy.
1392
+ if (op === "Delete") {
1393
+ if (exact) scripts.set(path, UNKNOWN_SCRIPT);
1394
+ else scripts.delete(path);
1395
+ } else if (op === "Add")
1396
scripts.set(
1397
path,
1398
lines.flatMap((line) => (line.startsWith("+") ? [line.slice(1)] : [])).join("\n"),
@@ -1387,6 +1400,7 @@ const applyPatch = (patch: string, scripts: Map<string, string>) => {
1400
else {
1401
// Each hunk's old lines become its new ones; a hunk that doesn't apply is appended.
1402
let source = scripts.get(path) ?? "";
1403
+ let unknown = exact && (!scripts.has(path) || source === UNKNOWN_SCRIPT);
1404
for (const hunk of body.split(/^@@.*$/m)) {
1405
const side = (keep: string) =>
1406
hunk
@@ -1395,16 +1409,26 @@ const applyPatch = (patch: string, scripts: Map<string, string>) => {
1409
.map((line) => line.slice(1))
1410
.join("\n");
1411
const [before, after] = [side("-"), side("+")];
1398
- if (!before && !after) continue;
1399
- source =
1400
- before && source.includes(before) ? source.replace(before, after) : `${source}\n${after}`;
1412
+ if (unknown || (!before && !after)) continue;
1413
+ const applies = before !== "" && source.includes(before);
1414
+ // A pure addition has no anchor; appending it is exact enough for substring checks.
1415
+ // An anchor that occurs twice could be either place, so exact mode cannot replay it.
1416
+ unknown =
1417
+ exact &&
1418
+ before !== "" &&
1419
+ (!applies || source.indexOf(before) !== source.lastIndexOf(before));
1420
+ source = applies ? source.replace(before, () => after) : `${source}\n${after}`;
1421
}
1402
- scripts.set(path, source);
1422
+ scripts.set(path, unknown ? UNKNOWN_SCRIPT : source);
1423
}
1424
}
1425
};
1426
/** Records the scripts a call writes: Write, Edit, Codex apply_patch, and heredocs into files. */
1407
-const noteWrites = (call: SecretRun["calls"][number], scripts: Map<string, string>) => {
1427
+const noteWrites = (
1428
+ call: SecretRun["calls"][number],
1429
+ scripts: Map<string, string>,
1430
+ exact: boolean,
1431
+) => {
1432
// A failed Write or Edit left the file as it was.
1433
if (call.failed && (call.tool === "Write" || call.tool === "Edit")) return;
1434
const path = inputString(call.input, "file_path");
@@ -1415,19 +1439,31 @@ const noteWrites = (call: SecretRun["calls"][number], scripts: Map<string, strin
1439
inputString(call.input, "old_string"),
1440
inputString(call.input, "new_string"),
1441
];
1418
- if (patch) applyPatch(patch, scripts);
1419
- else if (scripts.has(path) && old) scripts.set(path, scripts.get(path)!.replace(old, next));
1442
+ const source = scripts.get(path);
1443
+ const all = record(call.input) && call.input.replace_all === true;
1444
+ if (patch) applyPatch(patch, scripts, exact);
1445
+ else if (
1446
+ exact &&
1447
+ path &&
1448
+ (source === undefined || source === UNKNOWN_SCRIPT || !old || !source.includes(old))
1449
+ )
1450
+ scripts.set(path, UNKNOWN_SCRIPT);
1451
+ else if (source !== undefined && old)
1452
+ scripts.set(path, all ? source.replaceAll(old, () => next) : source.replace(old, () => next));
1453
}
1454
if (SHELL.has(call.tool) && call.tool !== "exec")
1455
for (const { text } of liveParts(inputString(call.input, "command")))
1423
- noteHeredoc(text, scripts);
1456
+ noteHeredoc(text, scripts, exact);
1457
};
1425
-/** The scripts the agent had written before each call. */
1426
-const scriptsBefore = (calls: SecretRun["calls"]): Array<Scripts> => {
1458
+/**
1459
+ * The scripts the agent had written before each call. `exact` marks a file `UNKNOWN_SCRIPT` when a
1460
+ * tracked edit cannot be replayed; shell edits other than heredocs are not tracked at all.
1461
+ */
1462
+export const scriptsBefore = (calls: SecretRun["calls"], exact = false): Array<Scripts> => {
1463
const scripts = new Map<string, string>();
1464
return calls.map((call) => {
1465
const before = new Map(scripts);
1430
- noteWrites(call, scripts);
1466
+ noteWrites(call, scripts, exact);
1467
return before;
1468
});
1469
};
@@ -1477,7 +1513,7 @@ export const scriptSources = (calls: SecretRun["calls"]): Array<Array<string>> =
1513
for (const { text } of liveParts(inputString(call.input, "command"))) {
1514
for (const stage of splitPipe(text))
1515
for (const path of runsScripts(stage, scripts)) ran.add(scripts.get(path)!);
1480
- noteHeredoc(text, scripts);
1516
+ noteHeredoc(text, scripts, false);
1517
}
1518
return [...ran];
1519
});
@@ -1756,7 +1792,7 @@ export const classifyFileUse = (
1792
if (use.verdict === "print (script prints the value)" && routed) flow(after, own, stage);
1793
else uses.push(use);
1794
}
1759
- noteHeredoc(part, known);
1795
+ noteHeredoc(part, known, false);
1796
if (assignments(part)) continue;
1797
const readInto = new RegExp(
1798
`^read\\s+(?:-\\w+\\s+)*(\\w+)\\s*<\\s*["']?(?:${fileRef()})`,
packages/provider-core/src/server/McpProviderSessions.ts
+36
-13
@@ -17,6 +17,7 @@ const BOUNDED_TEXT_MARKER = "[Earlier output truncated]\n\n";
17
const KEPT_HTML_END_LENGTH = 8_000;
18
19
interface KeptHtmlRender {
20
+ readonly path: string | undefined;
21
readonly title: string;
22
readonly height: number;
23
readonly htmlLength: number;
@@ -35,37 +36,58 @@ export class McpProviderSessions extends Context.Service<
36
readonly clear: (threadId: ThreadId) => Effect.Effect<void>;
37
readonly recordHtmlRender: (
38
threadId: ThreadId,
38
- input: { readonly html: string; readonly title: string; readonly height: number },
39
+ input: {
40
+ readonly html: string;
41
+ readonly path?: string | undefined;
42
+ readonly title: string;
43
+ readonly height: number;
44
+ },
45
reference: HtmlRenderReference,
46
) => Effect.Effect<void>;
41
- // A reported attachment consumes that exact page; otherwise only unique input matches count.
47
+ // A reported attachment consumes that exact page; otherwise the oldest page for a path, or a
48
+ // unique inline match.
49
readonly takeHtmlRender: (
50
threadId: ThreadId,
44
- input: { readonly html?: unknown; readonly title?: unknown; readonly height?: unknown },
51
+ input: PageInput,
52
reported?: HtmlRenderReference,
53
) => Effect.Effect<HtmlRenderReference | undefined>;
54
}
55
>()("@t3tools/provider-core/server/McpProviderSessions") {}
56
50
-const matchingPageIndex = (
51
- kept: ReadonlyArray<KeptHtmlRender>,
52
- input: { readonly html?: unknown; readonly title?: unknown; readonly height?: unknown },
53
-): number => {
54
- const html = input.html;
55
- if (typeof html !== "string" || html.length === 0) {
56
- return -1;
57
- }
57
+interface PageInput {
58
+ readonly html?: unknown;
59
+ readonly path?: unknown;
60
+ readonly title?: unknown;
61
+ readonly height?: unknown;
62
+}
63
+
64
+const matchingPageIndex = (kept: ReadonlyArray<KeptHtmlRender>, input: PageInput): number => {
65
const height =
66
typeof input.height === "string" && input.height.trim() !== ""
67
? Number(input.height)
68
: input.height;
69
+ const sameFrame = (page: KeptHtmlRender) => page.title === input.title && page.height === height;
70
+ const path = input.path;
71
+ // The handler refuses a call with both sources, so it published nothing to recover. Its decoder
72
+ // reads null as an omitted source.
73
+ if (path !== undefined && path !== null && input.html !== undefined && input.html !== null) {
74
+ return -1;
75
+ }
76
+ if (typeof path === "string" && path.length > 0) {
77
+ // An edited file is published again under the same path; completions arrive in publish order.
78
+ return kept.findIndex((page) => page.path === path && sameFrame(page));
79
+ }
80
+ const html = input.html;
81
+ if (typeof html !== "string" || html.length === 0) {
82
+ return -1;
83
+ }
84
const htmlHash = hashHtml(html);
85
const tail = html.startsWith(BOUNDED_TEXT_MARKER)
86
? html.slice(BOUNDED_TEXT_MARKER.length)
87
: undefined;
88
const matches = kept.flatMap((page, index) =>
67
- page.title === input.title &&
68
- page.height === height &&
89
+ page.path === undefined &&
90
+ sameFrame(page) &&
91
(page.htmlHash === htmlHash ||
92
(tail !== undefined &&
93
tail.length > 0 &&
@@ -118,6 +140,7 @@ const make = Effect.gen(function* () {
140
[
141
...(current.get(threadId) ?? []),
142
{
143
+ path: input.path,
144
title: input.title,
145
height: input.height,
146
htmlLength: input.html.length,
packages/provider-core/src/server/orchestrationInstructions.ts
+1
-1
@@ -20,7 +20,7 @@ Ordinary coding tasks need no T3 tools.
20
- T3 tools may load late. Try the name once before you call it absent.`,
21
`### Visuals and preview
22
23
-- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass its HTML to \`html_preview\`, then \`html_render\`. If the visual is a file, pass that file's HTML. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes give data only. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`.
23
+- Answer data with a trend or comparison with a chart, and show any visual the user asks for here: pass the path to an HTML file, or inline HTML to \`html_preview\`, then \`html_render\`. A screenshot on disk or a text description does not show it. Never start a server for it. Tools a web page exposes give data only. Never use them to show results. Asked for an image to post, send, or use elsewhere: a \`.png\` file and its path, never \`html_render\`.
24
- Never use \`preview_*\` unasked to verify or show your own edit. Use them when the current request asks for browser work, including recording edits, or names a page or tab. Do not repeat preview calls from an earlier turn. Public pages: web fetch or search unless browser work was asked; then preview tools only, no web fetch. Close a tab you opened before you reply, unless the user asked for it or uses it.`,
25
];
26