The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 05:45 UTC
fix(evals): resume browser verification after unavailable status (#30298)
Yash Singh committed
Nov 3, 2026 at 13:28 UTC
1a3f76e80afd33785d06c2acaf0e80eb1785c1e6
5 files changed
+80
-19
.agents/skills/test-t3-app/SKILL.md
+11
-6
@@ -5,10 +5,10 @@ description: Test T3 Code's web and desktop UI through its built-in Browser pane
5
6
# Test T3 web and desktop
7
8
-Use T3's built-in Browser panel for verification. If its tools are absent or
9
-the panel reports unavailable, explain the blocker and stop verification.
10
-Do not install or switch to another automation system. For native mobile
11
-testing, use [test-t3-mobile](../test-t3-mobile/SKILL.md).
8
+Use T3's built-in Browser panel for verification. Follow the recovery steps
9
+below before declaring it unavailable. Do not install or switch to another
10
+automation system. For native mobile testing, use
11
+[test-t3-mobile](../test-t3-mobile/SKILL.md).
12
13
## Start the app
14
@@ -36,8 +36,13 @@ model, such as Haiku for Claude or Luna for Codex.
36
37
## Use the Browser panel
38
39
-Call `preview_status`, then `preview_open` if the Browser panel is
40
-closed. Navigate to the complete startup pairing URL once with
39
+Call `preview_status`. If no tab can be automated, call `preview_open`, including
40
+when status reports `available: false` or `tabId: null`. Status can report
41
+unavailable simply because no tab exists yet; opening a tab needs no user action.
42
+Stop verification and explain the blocker only if the tools are absent or
43
+`preview_open` reports unavailable or fails.
44
+
45
+Navigate to the complete startup pairing URL once with
46
`preview_navigate`, then use `preview_snapshot` and T3's interaction tools.
47
If the token was consumed or expired, run `node apps/server/src/bin.ts pair`
48
for a fresh one. Keep using the same tab.
evals/mcp-tool-use/README.md
+1
@@ -26,6 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts`
26
- `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work.
27
- `experience`: the real mistake, in a sentence. `source`: where it happened.
28
- `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results.
29
+- `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt.
30
- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers.
31
32
Check that the case builds with `--dry-run --case <id>`. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass.
evals/mcp-tool-use/cases/visual-preview.ndjson
+2
@@ -28,3 +28,5 @@
28
{"id": "proactive-visual-ci-time-breakdown", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "data that reads better as a chart answered in prose only", "rule": "When a chart, diagram, or mockup says more than prose: html_preview, then html_render", "source": {"report": "agents used inline HTML visuals less often after #24850 made the rule ask-only"}, "experience": "The user shared CI job times over several runs and asked what eats the time. The agent answered in prose; a stacked chart would have shown it.", "setup": {"role": "top"}, "prompt": "CI got slow. Here are job times in minutes for the last 8 runs on main. What's eating the time?\n\nRun | install | typecheck | lint | test-server | test-web | build | e2e\n1 | 2.1 | 3.0 | 1.2 | 6.8 | 4.1 | 3.3 | 7.9\n2 | 2.0 | 3.1 | 1.1 | 6.9 | 4.0 | 3.4 | 8.2\n3 | 2.2 | 3.0 | 1.2 | 7.4 | 4.2 | 3.3 | 8.0\n4 | 2.1 | 3.2 | 1.3 | 9.8 | 4.1 | 3.5 | 8.1\n5 | 2.3 | 3.1 | 1.2 | 12.6 | 4.3 | 3.4 | 8.4\n6 | 2.2 | 3.3 | 1.2 | 13.1 | 4.2 | 3.6 | 12.9\n7 | 2.1 | 3.2 | 1.3 | 13.4 | 4.4 | 3.5 | 13.2\n8 | 2.2 | 3.2 | 1.2 | 13.0 | 4.3 | 3.4 | 13.5", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"]}}
29
{"id": "requested-after-video-for-edit", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "agent refuses requested browser recording of its own edit", "experience": "A user asked for an after video of a UI fix. The agent declined because the injected instructions prohibited preview for its own edits.", "rule": "Use preview when the user asks for browser work, including recording an edit.", "setup": {"role": "top", "context": "The UI fix is complete and a development server is already running at http://localhost:3000. Browser verification and recording are authorized.", "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:3000/"}, "preview_recording_start": {"recording": true, "tabId": "tab_1", "startedAt": "2026-01-01T00:00:00.000Z"}, "preview_recording_stop": {"id": "recording_1", "tabId": "tab_1", "path": "/tmp/after.webm", "mimeType": "video/webm", "sizeBytes": 1024, "createdAt": "2026-01-01T00:00:01.000Z"}}}, "prompt": "Use the T3 browser to verify the UI fix you just made and record an after video.", "expect": {"calls": [{"tool": "preview_recording_start"}, {"tool": "preview_recording_stop"}], "order": ["preview_recording_start", "preview_recording_stop"]}, "source": {"report": "A user requested an after video of a UI fix; the agent declined because preview was prohibited for its own edits."}}
30
{"id": "dev-server-opened-blank-then-navigated", "area": "visual-preview", "kind": "bad", "pattern": "preview_open on a blank tab, then preview_navigate to the dev-server port", "experience": "Asked to open a dev server in the T3 preview, agents opened a blank tab and navigated it to the port in a second call. 121 of these pairs in a month of maintainer threads.", "rule": "preview_open takes a url but not a dev-server target", "privacy": "commit-safe", "setup": {"role": "top", "context": "CLAUDE.md says: for t3code browser work, use the server-side preview_* mcp tools, with `target: {kind: \"environment-port\", port: ...}` for dev servers in the current environment. The dev server for this repo is running; `vp run dev` printed `Local: http://localhost:5173/`.", "mocks": {"preview_status": {"available": true, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}, "preview_navigate": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "title": "Acme", "loading": false, "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "open the dev server in the t3 preview, i want to click around", "expect": {"calls": [{"tool": "preview_open", "args": {"target.port": 5173}}], "maxCalls": {"preview_open": 1, "preview_navigate": 0}}, "maxSteps": 6, "source": {"thread": "Maintainer threads, 2026-10: preview_open with no url followed by preview_navigate to an environment-port target, 121 times"}}
31
+{"id": "browser-status-reuses-existing-tab", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "pattern": "usable Browser tab needs no new tab", "experience": "An already paired, usable tab should be reused during verification.", "source": {"report": "Maintainer browser verification failure, 2026-11-03"}, "setup": {"role": "top", "context": "The UI fix is complete. The test server is healthy at http://localhost:5173 and its browser profile is already paired. The current tab can be automated.", "mocks": {"preview_status": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "priorTurns": [{"role": "user", "text": "Use the Browser panel to verify the UI fix and capture an after screenshot."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "visible": true, "tabId": "tab_1", "url": "http://localhost:5173/", "tabs": [{"tabId": "tab_1", "url": "http://localhost:5173/", "owner": "agent", "ownedByCaller": true}], "control": {"owner": "agent", "ownedByCaller": true, "generation": 0}}}]}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_snapshot"}], "callsMustNot": [{"tool": "preview_open", "args": {"reuseExistingTab": false}}]}, "maxSteps": 10, "notes": "Checks reuse of an existing usable tab. Screenshot saving is outside this control. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."}
32
+{"id": "browser-resume-after-unavailable-status", "area": "visual-preview", "kind": "bad", "privacy": "commit-safe", "pattern": "closed Browser panel treated as unavailable automation", "experience": "The agent read test-t3-app, saw available false with no tab, and asked the user to open the panel instead of calling preview_open. Opening a tab succeeded on the next turn.", "source": {"report": "Maintainer browser verification failure, 2026-11-03; fresh skill read followed by unavailable status. Opus replay reproduces the stop on the old skill."}, "setup": {"role": "top", "context": "Repository instructions: Before web or desktop UI verification, read .agents/skills/test-t3-app/SKILL.md and follow it. If a skill causes you to stop or leave requested work unfinished, link the SKILL.md, quote the relevant instruction, and explain how it applies.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5173/"}, "preview_snapshot": {"available": true, "tabId": "tab_1", "url": "http://localhost:5173/", "screenshotPath": "/tmp/after.png"}}, "world": {"repo": "acme/app", "branch": "fix/permissions-popup", "status": "", "files": {"apps/web/src/components/ComposerContextPermissions.tsx": "export function ComposerContextPermissions() { return <PermissionsMenu side=\"top\" align=\"end\" />; }\n"}, "description": "The requested fix is already committed as a1b2c3d on fix/permissions-popup, ahead of main. It changes ComposerContextPermissions.tsx to open the permissions popup above its trigger. Dependencies are installed. No development server is running and no .t3 directory exists.", "commits": [{"sha": "a1b2c3d", "subject": "fix(web): open composer permissions above its button"}]}, "priorTurns": [{"role": "user", "text": "When I click Full access in the composer, open its permissions menu directly above that button."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "git commit -am \"fix(web): open composer permissions above its button\""}, "result": "[fix/permissions-popup a1b2c3d] fix(web): open composer permissions above its button\n 1 file changed, 3 insertions(+), 1 deletion(-)"}]}, {"role": "assistant", "text": "Implemented. Clicking Full access opens the permissions menu above the button. Focused checks pass. Full-client visual verification remains undone."}, {"role": "user", "text": "before/after screenshot"}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "cat .agents/skills/test-t3-app/SKILL.md"}, "result": "", "resultFile": ".agents/skills/test-t3-app/SKILL.md"}]}, {"role": "assistant", "text": "I'll use the test-t3-app skill to capture the before and after states in the app."}, {"role": "assistant", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}}]}], "shellMocks": [{"match": "git status|ls .*\\.t3", "output": "ls: .t3: No such file or directory"}], "harness": "claude"}, "prompt": "", "expect": {"calls": [{"tool": "preview_open"}]}, "maxSteps": 8, "notes": "Resume at the failed decision after reading the skill and receiving available=false. Opus reproduces the premature stop; Sonnet did not reliably reproduce it. This checks recovery through preview_open, not completion of screenshots. Prior tool-result replay is validated on the Claude harness; Codex output wrapping needs separate validation."}
evals/mcp-tool-use/run.test.ts
+22
@@ -394,6 +394,28 @@ it("replays earlier capability results with the model under test as the caller",
394
.flatMap((message) => message.content)
395
.find((block) => block.type === "tool_result" && String(block.content).includes("inherited"));
396
assert.include(String(replayed?.content), '"inheritedModel":"claude-opus-5-5"');
397
+ const resumed = buildMessages(makeSample({ ...testCase, prompt: "" }, base));
398
+ assert.equal(resumed.at(-1)?.role, "user");
399
+ assert.equal(resumed.at(-1)?.content.at(-1)?.type, "tool_result");
400
+ for (const setup of [
401
+ {
402
+ ...testCase.setup,
403
+ sideChat: { parentThreadId: "thread-parent", digest: ["The permissions fix is committed."] },
404
+ },
405
+ { ...testCase.setup, role: "native-subagent" as const },
406
+ { ...testCase.setup, role: "delegated-child" as const },
407
+ ]) {
408
+ assert.throws(
409
+ () =>
410
+ buildMessages(
411
+ makeSample(
412
+ { ...testCase, prompt: "", setup },
413
+ { ...base, subagentContext: "Hook context" },
414
+ ),
415
+ ),
416
+ /Empty-prompt replay/,
417
+ );
418
+ }
419
});
420
421
it("gives each launched thread its own id, and reading it back shows that thread's brief", () => {
evals/mcp-tool-use/run.ts
+44
-13
@@ -140,6 +140,8 @@ const PriorToolCall = Schema.Struct({
140
input: Schema.Unknown,
141
/** For a device tool, `"default"` runs the call on the world's devices for T3's result. */
142
result: Schema.Unknown,
143
+ /** Repository-relative file whose current contents supply this tool result. */
144
+ resultFile: Schema.optional(Schema.String),
145
});
146
147
/**
@@ -275,6 +277,7 @@ export const McpToolUseCase = Schema.Struct({
277
Schema.Struct({ parentThreadId: Schema.String, digest: Schema.Array(Schema.String) }),
278
),
279
}),
280
+ /** Empty to resume a top-level thread without side-chat context after its last tool result. */
281
prompt: Schema.String,
282
/** Records for the context chips `prompt` links, projected as T3 sends them. */
283
promptContext: Schema.optional(Schema.Struct({ records: Schema.Array(ComposerContextRecord) })),
@@ -880,6 +883,19 @@ export const buildMessages = (sample: Sample) => {
883
else messages[messages.length - 1] = { role, content: [...last.content, ...blocks] };
884
};
885
const { role, context, delegateRole, priorTurns = [] } = testCase.setup;
886
+ const sideChat = testCase.setup.sideChat;
887
+ if (testCase.prompt === "" && (role !== "top" || sideChat !== undefined)) {
888
+ throw new Error(
889
+ `${testCase.id}: Empty-prompt replay requires a top-level thread without side-chat context.`,
890
+ );
891
+ }
892
+ const sideChatContext =
893
+ sideChat === undefined
894
+ ? undefined
895
+ : makeSideChatContext({
896
+ parentThreadId: ThreadId.make(sideChat.parentThreadId),
897
+ digest: sideChat.digest,
898
+ });
899
const native = role === "native-subagent";
900
const notes = [
901
native
@@ -926,19 +942,11 @@ export const buildMessages = (sample: Sample) => {
942
testCase.promptContext === undefined
943
? testCase.prompt
944
: projectComposerContextForProvider({ text: testCase.prompt, ...testCase.promptContext });
929
- const sideChat = testCase.setup.sideChat;
945
const prompt =
931
- sideChat === undefined
932
- ? userText
933
- : withInlineContext(
934
- makeSideChatContext({
935
- parentThreadId: ThreadId.make(sideChat.parentThreadId),
936
- digest: sideChat.digest,
937
- }),
938
- userText,
939
- );
946
+ sideChatContext === undefined ? userText : withInlineContext(sideChatContext, userText);
947
const childPrompt = () => taskPrompt({ task: prompt, role: delegateRole });
941
- push("user", text(role === "delegated-child" ? childPrompt() : prompt));
948
+ if (testCase.prompt !== "")
949
+ push("user", text(role === "delegated-child" ? childPrompt() : prompt));
950
// Claude Code gives a subagent its SubagentStart hooks' context as a reminder after the task.
951
if (native && base.subagentContext) {
952
push("user", reminder(`SubagentStart hook additional context: ${base.subagentContext}`));
@@ -2125,12 +2133,35 @@ const main = Effect.gen(function* () {
2133
const wanted = flags.case?.flatMap((ids) => ids.split(","));
2134
const codex = isCodex(flags.model);
2135
const harness = codex ? "codex" : "claude";
2128
- const selected = cases.filter(
2136
+ const selectedCases = cases.filter(
2137
({ id, setup }) =>
2138
(wanted === undefined || wanted.includes(id)) &&
2139
(setup.harness === undefined || setup.harness === harness),
2140
);
2133
- if (selected.length === 0) return yield* Effect.die(new Error("No case matches --case."));
2141
+ if (selectedCases.length === 0) return yield* Effect.die(new Error("No case matches --case."));
2142
+ const selected = yield* Effect.forEach(selectedCases, (testCase) =>
2143
+ Effect.gen(function* () {
2144
+ const priorTurns = yield* Effect.forEach(testCase.setup.priorTurns ?? [], (turn) =>
2145
+ Effect.gen(function* () {
2146
+ const toolCalls = yield* Effect.forEach(turn.toolCalls ?? [], (call) =>
2147
+ call.resultFile === undefined
2148
+ ? Effect.succeed(call)
2149
+ : fs
2150
+ .readFileString(path.resolve(import.meta.dirname, "../..", call.resultFile))
2151
+ .pipe(Effect.map((result) => ({ ...call, result }))),
2152
+ );
2153
+ return { ...turn, toolCalls };
2154
+ }),
2155
+ );
2156
+ return {
2157
+ ...testCase,
2158
+ setup: {
2159
+ ...testCase.setup,
2160
+ priorTurns,
2161
+ },
2162
+ };
2163
+ }),
2164
+ );
2165
// Trimmed captures without user data ship beside the cases. `--template` also takes a raw
2166
// CLIProxyAPI capture.
2167
const templatePath =