@devries-cameron20 / t3code-1 / commits / d51bc58f5e
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(tools): show images from MCP and OpenCode outputs (#30389)

Preserve MCP and OpenCode image outputs so tool details show screenshots before their inputs. Highlight HTML preview input and valid JSON arguments and results across clients, while keeping native file text readable. Sanitize CI test logs to avoid a Unicode runner crash without changing test exit codes. Validation: all CI checks passed, 729 affected tests and the full 156-test Pi suite passed locally, web and iOS details verified. Model: GPT 6.1 Sol. Harness: Codex.

Yash Singh committed Nov 3, 2026 at 20:05 UTC d51bc58f5e05123a416a3737ac4bce42d1d76b7c
20 files changed +401 -105
.github/workflows/ci.yml
+7 -1
@@ -272,12 +272,18 @@ jobs:
272 env:
273 TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }}
274 run: |
275 + set -o pipefail
276 filters=(--filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web')
277 if [[ -n "$TEST_PACKAGES" ]]; then
278 filters=()
279 for package in $TEST_PACKAGES; do filters+=(--filter "$package"); done
280 fi
280 - vp run --parallel --concurrency-limit 4 "${filters[@]}" test
281 + # The runner's trace writer can crash on surrogate pairs in test output.
282 + vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c '
283 + import sys
284 + for line in sys.stdin:
285 + sys.stdout.write(line.encode("ascii", "backslashreplace").decode("ascii"))
286 + '
287
288 # The web suite is the largest outside the server. In the shared Test job it
289 # started last and ran alone for about 40s after everything else finished.
apps/mobile/src/features/threads/thread-work-log.tsx
+43 -12
@@ -69,6 +69,7 @@ import {
69 workEntryRowLabel,
70 } from "../../lib/threadActivity";
71 import {
72 + isJsonToolText,
73 toolCallLines,
74 turnItemOutputImages,
75 turnItemOutputText,
@@ -874,14 +875,16 @@ function WorkspacePreparationRetryButton(props: {
875 );
876 }
877
877 -function ReplCodeText({
878 +function ToolCodeText({
879 code,
880 + language,
881 theme,
882 }: {
883 readonly code: string;
884 + readonly language: "typescript" | "html" | "json";
885 readonly theme: "light" | "dark";
886 }) {
884 - const lines = useMarkdownCodeHighlight({ code, language: "typescript", theme, enabled: true });
887 + const lines = useMarkdownCodeHighlight({ code, language, theme, enabled: true });
888 let offset = 0;
889 const highlighted = lines?.flatMap((line, lineIndex) => {
890 const separator = lineIndex > 0 ? "\n" : "";
@@ -1255,20 +1258,32 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
1258 ) : call ? (
1259 [
1260 call.command,
1258 - ...(call.args ?? []).map(([key, value]) =>
1259 - key === "code" &&
1260 - shownItem.type === "dynamic_tool" &&
1261 - isJavaScriptReplTool(shownItem.toolName) ? (
1261 + ...(call.args ?? []).map(([key, value]) => {
1262 + const language =
1263 + key === "html"
1264 + ? "html"
1265 + : key === "code" &&
1266 + shownItem.type === "dynamic_tool" &&
1267 + isJavaScriptReplTool(shownItem.toolName)
1268 + ? "typescript"
1269 + : isJsonToolText(value)
1270 + ? "json"
1271 + : null;
1272 + return language ? (
1273 <View key={key}>
1274 <Text className="font-mono text-2xs leading-normal text-foreground-muted">
1275 {key}
1276 </Text>
1266 - <ReplCodeText code={value} theme={props.themeAppearance} />
1277 + <ToolCodeText
1278 + code={value}
1279 + language={language}
1280 + theme={props.themeAppearance}
1281 + />
1282 </View>
1283 ) : (
1284 `${key} ${value}`
1270 - ),
1271 - ),
1285 + );
1286 + }),
1287 call.argsText,
1288 ]
1289 .filter((line) => Boolean(line))
@@ -1281,7 +1296,11 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
1296 selectable
1297 className="font-mono text-2xs leading-normal text-foreground"
1298 >
1284 - {line}
1299 + {isJsonToolText(line) ? (
1300 + <ToolCodeText code={line} language="json" theme={props.themeAppearance} />
1301 + ) : (
1302 + line
1303 + )}
1304 </Text>
1305 ) : (
1306 line
@@ -1289,7 +1308,11 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
1308 )
1309 ) : fullDetail ? (
1310 <Text selectable className="font-mono text-2xs leading-normal text-foreground-muted">
1292 - {fullDetail}
1311 + {isJsonToolText(fullDetail) ? (
1312 + <ToolCodeText code={fullDetail} language="json" theme={props.themeAppearance} />
1313 + ) : (
1314 + fullDetail
1315 + )}
1316 </Text>
1317 ) : null}
1318 {fetchedOutput ? (
@@ -1300,7 +1323,15 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
1323 (!call || call.command || call.args || call.argsText) && "mt-1.5",
1324 )}
1325 >
1303 - {fetchedOutput}
1326 + {isJsonToolText(fetchedOutput) ? (
1327 + <ToolCodeText
1328 + code={fetchedOutput}
1329 + language="json"
1330 + theme={props.themeAppearance}
1331 + />
1332 + ) : (
1333 + fetchedOutput
1334 + )}
1335 </Text>
1336 ) : null}
1337 {streamsOutput ? null : exitLine}
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+13 -1
@@ -26,6 +26,7 @@ import type {
26 } from "@anthropic-ai/claude-agent-sdk/sdk-tools";
27 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
28 import * as NodeServices from "@effect/platform-node/NodeServices";
29 +import { toolOutputImages } from "@t3tools/shared/toolOutput";
30 import {
31 ChatAttachmentId,
32 type ChatAttachment,
@@ -4844,11 +4845,19 @@ describe("ClaudeAdapterV2 background wake turns", () => {
4845 {
4846 type: "tool_result",
4847 tool_use_id: id,
4847 - content: "Weather result",
4848 + content: [
4849 + { type: "text", text: "Weather result" },
4850 + {
4851 + type: "image",
4852 + source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
4853 + },
4854 + ],
4855 is_error: isError,
4856 },
4857 ],
4858 },
4859 + tool_use_result:
4860 + title === undefined ? { screenshot: { mimeType: "image/png" } } : undefined,
4861 }),
4862 );
4863 yield* Queue.offer(
@@ -4881,6 +4890,9 @@ describe("ClaudeAdapterV2 background wake turns", () => {
4890 assert.deepEqual(item.input, { city: "Berlin" });
4891 }
4892 assert.equal(new Set(items.map((item) => item.id)).size, 1);
4893 + assert.deepEqual(toolOutputImages(items.at(-1)?.output), [
4894 + { mimeType: "image/png", data: "iVBORw0KGgo=" },
4895 + ]);
4896 }).pipe(
4897 Effect.scoped,
4898 Effect.provide(
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+8
@@ -1,4 +1,5 @@
1 import * as DocumentText from "../../provider/DocumentText.ts";
2 +import { mcpToolResultOutput, toolOutputImages } from "@t3tools/shared/toolOutput";
3 import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts";
4 import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts";
5 import {
@@ -3084,7 +3085,14 @@ function claudeNativeToolOutputValue(output: ClaudeNativeToolOutput): unknown |
3085 case "none":
3086 return undefined;
3087 case "content_block":
3088 + return output.value;
3089 case "structured_tool_use_result":
3090 + if (toolOutputImages(output.value).length === 0) {
3091 + return mcpToolResultOutput({
3092 + content: output.fallbackValue,
3093 + structuredContent: output.value,
3094 + });
3095 + }
3096 return output.value;
3097 default:
3098 return assertNever(output);
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+2 -3
@@ -50,6 +50,7 @@ import {
50 import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model";
51 import { appendGeneratedImageEmbeds } from "@t3tools/shared/generatedImageMarkdown";
52 import { resolveSpawnCommand } from "@t3tools/shared/shell";
53 +import { mcpToolResultOutput } from "@t3tools/shared/toolOutput";
54 import type {
55 ChatAttachment,
56 OrchestrationV2AppThread,
@@ -490,9 +491,7 @@ function codexMcpToolOutput(
491 const resultOutput =
492 item.result === null || item.result === undefined
493 ? undefined
493 - : item.result.structuredContent !== null && item.result.structuredContent !== undefined
494 - ? item.result.structuredContent
495 - : item.result.content;
494 + : mcpToolResultOutput(item.result);
495
496 if (item.error === null || item.error === undefined) {
497 return resultOutput;
apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts
+62 -42
@@ -1,4 +1,5 @@
1 import * as NodeURL from "node:url";
2 +import { toolOutputImages } from "@t3tools/shared/toolOutput";
3 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
4 import * as NodeServices from "@effect/platform-node/NodeServices";
5 import { assert, it } from "@effect/vitest";
@@ -2715,48 +2716,67 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
2716 }).pipe(Effect.scoped),
2717 );
2718
2718 - it.effect("keeps thread-scoped T3 waits as dynamic tools through their lifecycle", () =>
2719 - Effect.gen(function* () {
2720 - const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };
2721 - const name = "t3-code-web_delegate_task";
2722 - const input = { task: "Review", mode: "wait" };
2723 - const { runtime, thread } = yield* resumed([
2724 - out("session.prompt", { sessionID: SESSION, text: "<any>" }),
2725 - promptAccepted,
2726 - event("session.tool.input.started", { ...tool, name }),
2727 - event("session.tool.called", { ...tool, name, input, executed: true }),
2728 - event("session.tool.success", {
2729 - ...tool,
2730 - content: [{ type: "text", text: "Reviewed" }],
2731 - metadata: {},
2732 - executed: true,
2733 - }),
2734 - event("session.execution.succeeded", { sessionID: SESSION }),
2735 - ]);
2736 - const events = yield* runtime.events.pipe(
2737 - Stream.takeUntil((event) => event.type === "turn.terminal"),
2738 - Stream.runCollect,
2739 - Effect.forkScoped,
2740 - );
2741 - yield* runtime.startTurn(turnInput(thread));
2742 - const items = (yield* Fiber.join(events)).flatMap((event) =>
2743 - event.type === "turn_item.updated" &&
2744 - event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)
2745 - ? [event.turnItem]
2746 - : [],
2747 - );
2748 - assert.deepEqual(
2749 - items.map((item) => item.type),
2750 - ["dynamic_tool", "dynamic_tool", "dynamic_tool"],
2751 - );
2752 - assert.deepEqual(
2753 - items.map((item) => item.status),
2754 - ["running", "running", "completed"],
2755 - );
2756 - const last = items.at(-1);
2757 - assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);
2758 - assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);
2759 - }).pipe(Effect.scoped),
2719 + it.effect.each(["delegate_task", "html_preview", "mcp__acme__edit_document"])(
2720 + "keeps MCP %s as a dynamic tool through its lifecycle",
2721 + (toolName) =>
2722 + Effect.gen(function* () {
2723 + const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };
2724 + const name = toolName.startsWith("mcp__") ? toolName : `t3-code-web_${toolName}`;
2725 + const input =
2726 + toolName !== "delegate_task"
2727 + ? { html: "<p>Preview</p>" }
2728 + : { task: "Review", mode: "wait" };
2729 + const { runtime, thread } = yield* resumed([
2730 + out("session.prompt", { sessionID: SESSION, text: "<any>" }),
2731 + promptAccepted,
2732 + event("session.tool.input.started", { ...tool, name }),
2733 + event("session.tool.called", { ...tool, name, input, executed: true }),
2734 + event(
2735 + "session.tool.success",
2736 + {
2737 + ...tool,
2738 + content: [
2739 + { type: "text", text: "Reviewed" },
2740 + ...(toolName !== "delegate_task"
2741 + ? [{ type: "file", uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" }]
2742 + : []),
2743 + ],
2744 + metadata: {},
2745 + executed: true,
2746 + },
2747 + undefined,
2748 + 2,
2749 + ),
2750 + event("session.execution.succeeded", { sessionID: SESSION }),
2751 + ]);
2752 + const events = yield* runtime.events.pipe(
2753 + Stream.takeUntil((event) => event.type === "turn.terminal"),
2754 + Stream.runCollect,
2755 + Effect.forkScoped,
2756 + );
2757 + yield* runtime.startTurn(turnInput(thread));
2758 + const items = (yield* Fiber.join(events)).flatMap((event) =>
2759 + event.type === "turn_item.updated" &&
2760 + event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)
2761 + ? [event.turnItem]
2762 + : [],
2763 + );
2764 + assert.deepEqual(
2765 + items.map((item) => item.type),
2766 + ["dynamic_tool", "dynamic_tool", "dynamic_tool"],
2767 + );
2768 + assert.deepEqual(
2769 + items.map((item) => item.status),
2770 + ["running", "running", "completed"],
2771 + );
2772 + const last = items.at(-1);
2773 + assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);
2774 + assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);
2775 + assert.deepEqual(
2776 + toolOutputImages(last?.type === "dynamic_tool" ? last.output : null),
2777 + toolName !== "delegate_task" ? [{ mimeType: "image/png", data: "iVBORw0KGgo=" }] : [],
2778 + );
2779 + }).pipe(Effect.scoped),
2780 );
2781
2782 it.effect("stores a shell command that exits non-zero as failed", () =>
apps/web/src/components/chat/V2ItemInspector.tsx
+19 -25
@@ -1,6 +1,7 @@
1 import type { ToolCallDiffTarget } from "../../diffPanelStore";
2 import type { EnvironmentId, OrchestrationV2ProjectedTurnItem } from "@t3tools/contracts";
3 import {
4 + isJsonToolText,
5 toolCallLines,
6 turnItemDetailRevision,
7 turnItemNeedsDetailFetch,
@@ -43,7 +44,7 @@ function SyntaxTokens({
44 language,
45 }: {
46 readonly text: string;
46 - readonly language: "json" | "typescript";
47 + readonly language: "json" | "typescript" | "html";
48 }) {
49 const resolvedTheme = useRenderedAppearance();
50 const highlighter = use(getSyntaxHighlighterPromise(language));
@@ -64,16 +65,6 @@ function SyntaxTokens({
65 const monoClassName =
66 "font-mono text-(length:--font-size-code,var(--text-2xs)) leading-relaxed whitespace-pre-wrap break-words select-text";
67
67 -function isJsonText(text: string): boolean {
68 - if (!text) return false;
69 - try {
70 - JSON.parse(text);
71 - return true;
72 - } catch {
73 - return false;
74 - }
75 -}
76 -
68 function StructuredValue({
69 value,
70 highlightJson = false,
@@ -81,10 +72,10 @@ function StructuredValue({
72 }: {
73 readonly value: unknown;
74 readonly highlightJson?: boolean;
84 - readonly language?: "typescript";
75 + readonly language?: "typescript" | "html" | "json";
76 }) {
77 const text = typeof value === "string" ? value : JSON.stringify(value, null, 2);
87 - const isJson = highlightJson && isJsonText(text);
78 + const isJson = highlightJson && isJsonToolText(text);
79 if (!text) return null;
80 return (
81 <pre className={cn("max-h-80 overflow-auto text-muted-foreground", monoClassName)}>
@@ -150,8 +141,8 @@ interface ToolOutputState {
141 readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
142 }
143
153 -function ToolOutput(props: ToolOutputState) {
154 - const images = props.images.map((resource) => (
144 +function ToolOutputImages(props: ToolOutputState) {
145 + return props.images.map((resource) => (
146 <ChatMarkdownAssetImage
147 key={resource.index}
148 environmentId={props.environmentId}
@@ -161,21 +152,18 @@ function ToolOutput(props: ToolOutputState) {
152 onImageExpand={props.onImageExpand}
153 />
154 ));
164 - const text = props.output ? (
165 - <div className="max-h-80 overflow-auto text-muted-foreground">{props.output}</div>
155 +}
156 +
157 +function ToolOutput(props: ToolOutputState) {
158 + return props.output ? (
159 + <StructuredValue value={props.output} highlightJson />
160 ) : props.pending ? (
161 <div className="text-muted-foreground italic">Loading output…</div>
162 ) : props.error ? (
163 <div className="text-destructive">Couldn&apos;t load output: {props.error}</div>
170 - ) : props.empty && images.length === 0 ? (
164 + ) : props.empty && props.images.length === 0 ? (
165 <div className="text-muted-foreground italic">No output.</div>
166 ) : null;
173 - return (
174 - <>
175 - {images}
176 - {text}
177 - </>
178 - );
167 }
168
169 export function FetchedToolOutput(props: {
@@ -187,6 +175,7 @@ export function FetchedToolOutput(props: {
175 const { output } = useFetchedTurnItem(props.projectedItem, props.environmentId, props.source);
176 return (
177 <div className={cn("mt-1.5 space-y-1.5", monoClassName)}>
178 + <ToolOutputImages {...output} onImageExpand={props.onImageExpand} />
179 <ToolOutput {...output} onImageExpand={props.onImageExpand} />
180 </div>
181 );
@@ -205,14 +194,19 @@ function ToolCallBody(
194 const call = toolCallLines({ command: props.command, args: props.args });
195 return (
196 <div className={cn("space-y-1.5", monoClassName)}>
197 + <ToolOutputImages {...props} />
198 {call.command ? <ShellCommandBlock command={call.command} /> : null}
199 {call.args ? (
200 <div className="text-foreground/85">
201 {call.args.map(([key, value]) => (
202 <div key={key}>
203 <span className="text-muted-foreground">{key} </span>
214 - {key === "code" && isJavaScriptReplTool(props.toolName) ? (
204 + {key === "html" ? (
205 + <StructuredValue value={value} language="html" />
206 + ) : key === "code" && isJavaScriptReplTool(props.toolName) ? (
207 <StructuredValue value={value} language="typescript" />
208 + ) : isJsonToolText(value) ? (
209 + <StructuredValue value={value} language="json" />
210 ) : (
211 value
212 )}
packages/client-runtime/src/work-log/itemDetail.test.ts
+66 -5
@@ -2,7 +2,26 @@ import { describe, expect, it } from "vite-plus/test";
2 import { ThreadId, TurnItemId, type OrchestrationV2TurnItem } from "@t3tools/contracts";
3 import * as DateTime from "effect/DateTime";
4
5 -import { turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
5 +import { isJsonToolText, turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
6 +
7 +describe("isJsonToolText", () => {
8 + it.each([
9 + ['{"screenshot":{"mimeType":"image/png"}}', true],
10 + ['[{"threadId":"thread-1"}]', true],
11 + ['{\n\n "status": "completed"\n}', true],
12 + ['{\n "count": 1\n}\n\n{\n "count": 2\n}', true],
13 + ["false", true],
14 + ["null", true],
15 + ["42", true],
16 + ['"completed"', true],
17 + ['{"partial":', false],
18 + ['{"count":1}\n\nFinished.', false],
19 + ["No output.", false],
20 + ["", false],
21 + ] as const)("classifies %j as JSON: %s", (text, expected) => {
22 + expect(isJsonToolText(text)).toBe(expected);
23 + });
24 +});
25
26 function toolItem(output: unknown): OrchestrationV2TurnItem {
27 return {
@@ -28,15 +47,54 @@ function toolItem(output: unknown): OrchestrationV2TurnItem {
47 }
48
49 describe("tool image output parsing", () => {
31 - it.each(["array", "content"] as const)(
50 + it.each(["image-only", "summary", "json"] as const)(
51 + "keeps structured fields beside %s content",
52 + (shape) => {
53 + const content = [
54 + { type: "image", mimeType: "image/png" },
55 + ...(shape === "image-only"
56 + ? []
57 + : [{ type: "text", text: shape === "summary" ? "Sunny" : '{"temp":21}' }]),
58 + ];
59 + for (const structuredContent of [
60 + { temp: 21 },
61 + { type: "text", text: "Sunny", temp: 21 },
62 + { content: [{ type: "text", text: "Sunny" }], temp: 21 },
63 + ]) {
64 + const item = toolItem({ structuredContent, content });
65 + expect(turnItemOutputText(item)).toBe(JSON.stringify(structuredContent, null, 2));
66 + expect(turnItemOutputImages(item)).toHaveLength(1);
67 + }
68 + for (const structuredContent of [null, {}, [], ""]) {
69 + const item = toolItem({ structuredContent, content: [{ type: "text", text: "Sunny" }] });
70 + expect(turnItemOutputText(item)).toBe("Sunny");
71 + }
72 + expect(
73 + turnItemOutputText(
74 + toolItem({
75 + content: [{ type: "text", text: "File contents" }],
76 + structuredContent: { path: "read.ts", ready: true },
77 + }),
78 + ),
79 + ).toBe("File contents");
80 + },
81 + );
82 + it.each(["array", "content", "structured"] as const)(
83 "keeps markers for images beyond the served positions in %s output",
84 (shape) => {
85 // Detail responses retain image metadata after the server strips image bytes.
86 const image = { type: "image", mimeType: "image/png" };
87 const blocks = Array.from({ length: 8 }, () => ({ ...image }));
37 - const wrap = (content: unknown[]) => (shape === "array" ? content : { content });
88 + const wrap = (content: unknown[]) =>
89 + shape === "array"
90 + ? content
91 + : shape === "structured"
92 + ? { content, structuredContent: { temp: 21 } }
93 + : { content };
94 const withinLimit = toolItem(wrap(blocks));
39 - expect(turnItemOutputText(withinLimit)).toBeNull();
95 + expect(turnItemOutputText(withinLimit)).toBe(
96 + shape === "structured" ? '{\n "temp": 21\n}' : null,
97 + );
98 expect(turnItemOutputImages(withinLimit)).toHaveLength(8);
99
100 const overflow = toolItem(
@@ -48,7 +106,10 @@ describe("tool image output parsing", () => {
106 ]),
107 );
108 expect(turnItemOutputImages(overflow)).toHaveLength(8);
51 - expect(turnItemOutputText(overflow)).toBe("Two additional screenshots:\n[image]\n[image]");
109 + expect(turnItemOutputText(overflow)).toBe(
110 + (shape === "structured" ? '{\n "temp": 21\n}' : "Two additional screenshots:") +
111 + "\n[image]\n[image]",
112 + );
113 },
114 );
115 });
packages/client-runtime/src/work-log/itemDetail.ts
+24 -4
@@ -39,6 +39,14 @@ function textFromBlocks(
39 return textFromBlocks(value.content, depth + 1, imageBlocks);
40 if (keys.length === 2 && keys.includes("content") && keys.includes("structuredContent")) {
41 const text = textFromBlocks(value.content, depth + 1, imageBlocks);
42 + const images = Array.isArray(value.content)
43 + ? value.content.filter((block) => isRecord(block) && block.type === "image")
44 + : [];
45 + const structured = formatJsonValue(value.structuredContent);
46 + if (structured !== null && (images.length > 0 || !text?.trim())) {
47 + const markers = textFromBlocks(images, depth + 1, imageBlocks);
48 + return [structured, markers].filter(Boolean).join("\n");
49 + }
50 return text?.trim() ? text : null;
51 }
52 return null;
@@ -52,6 +60,12 @@ function parseJson(text: string): unknown {
60 }
61 }
62
63 +export function isJsonToolText(text: string): boolean {
64 + if (parseJson(text) !== undefined) return true;
65 + const documents = text.trim().split(/\n\s*\n/);
66 + return documents.length > 1 && documents.every((document) => parseJson(document) !== undefined);
67 +}
68 +
69 function prettyJsonText(text: string): string {
70 const trimmed = text.trim();
71 if (!/^[[{]/.test(trimmed)) return text;
@@ -63,11 +77,9 @@ function prettyJsonText(text: string): string {
77 return documents.map((document) => JSON.stringify(document, null, 2)).join("\n\n");
78 }
79
66 -function formatToolValue(value: unknown, hasImagePreview = false): string | null {
80 +function formatJsonValue(value: unknown): string | null {
81 if (value === undefined || value === null) return null;
68 - if (hasImagePreview && isRecord(value) && value.type === "image") return null;
69 - const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));
70 - if (text !== null) return text.trim() ? prettyJsonText(text) : null;
82 + if (typeof value === "string" && !value.trim()) return null;
83 let json: string | undefined;
84 try {
85 json = JSON.stringify(value, null, 2);
@@ -78,6 +90,14 @@ function formatToolValue(value: unknown, hasImagePreview = false): string | null
90 return json;
91 }
92
93 +function formatToolValue(value: unknown, hasImagePreview = false): string | null {
94 + if (value === undefined || value === null) return null;
95 + if (hasImagePreview && isRecord(value) && value.type === "image") return null;
96 + const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));
97 + if (text !== null) return text.trim() ? prettyJsonText(text) : null;
98 + return formatJsonValue(value);
99 +}
100 +
101 export function toolCallLines(input: {
102 readonly command?: string | undefined;
103 readonly args?: unknown;
packages/provider-acp/src/server/adapter.ts
+6 -3
@@ -39,7 +39,11 @@ import { HTML_RENDER_TOOL_NAME } from "@t3tools/shared/htmlRender";
39 import { modelSelectionsEqual } from "@t3tools/shared/model";
40 import { type SelfInvocation, selfInvocationArgs } from "@t3tools/shared/nodeRuntime";
41 import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation";
42 -import { compactDynamicToolOutput, htmlRenderFromToolItem } from "@t3tools/shared/toolOutput";
42 +import {
43 + compactDynamicToolOutput,
44 + htmlRenderFromToolItem,
45 + mcpToolResultOutput,
46 +} from "@t3tools/shared/toolOutput";
47 import { FILE_HEADERS_ONLY, formatPatch, structuredPatch } from "diff";
48 import * as Cause from "effect/Cause";
49 import * as Clock from "effect/Clock";
@@ -923,8 +927,7 @@ function acpMcpToolCallOutput(rawOutput: unknown): unknown {
927 if (record === undefined) return rawOutput;
928 if (!("result" in record) && !("error" in record)) return rawOutput;
929 const result = unknownRecord(record.result);
926 - const resultOutput =
927 - result === undefined ? undefined : (result.structuredContent ?? result.content ?? undefined);
930 + const resultOutput = result === undefined ? undefined : mcpToolResultOutput(result);
931 const errorMessage = unknownRecord(record.error)?.message;
932 if (typeof errorMessage !== "string") {
933 return resultOutput ?? rawOutput;
packages/provider-cursor/src/server/adapter.test.ts
+16 -1
@@ -1,6 +1,7 @@
1 import * as NodeServices from "@effect/platform-node/NodeServices";
2 import { assert, describe, it } from "@effect/vitest";
3 import type { InteractionUpdate } from "@cursor/sdk";
4 +import { toolOutputImages } from "@t3tools/shared/toolOutput";
5 import {
6 EnvironmentId,
7 MessageId,
@@ -844,7 +845,18 @@ describe("CursorAdapterV2", () => {
845 args: { city: "Berlin" },
846 },
847 ...(type === "tool-call-completed"
847 - ? { result: { status: "success" as const, value: { content: [], isError: false } } }
848 + ? {
849 + result: {
850 + status: "success" as const,
851 + value: {
852 + content: [
853 + { text: { text: "Preview complete" } },
854 + { image: { data: "iVBORw0KGgo=", mimeType: "image/png" } },
855 + ],
856 + isError: false,
857 + },
858 + },
859 + }
860 : {}),
861 },
862 })),
@@ -1155,6 +1167,9 @@ describe("CursorAdapterV2", () => {
1167 });
1168 assert.deepEqual(item.input, { city: "Berlin" });
1169 }
1170 + assert.deepEqual(toolOutputImages(mcpItems.at(-1)?.output), [
1171 + { mimeType: "image/png", data: "iVBORw0KGgo=" },
1172 + ]);
1173 const fileSearchItems = events.flatMap((event) =>
1174 event.type === "turn_item.updated" &&
1175 event.turnItem.type === "file_search" &&
packages/provider-cursor/src/server/adapter.ts
+11
@@ -377,6 +377,17 @@ function cursorToolOutput(toolCall: ToolCall): unknown {
377 if (result === undefined) {
378 return undefined;
379 }
380 + if (toolCall.type === "mcp" && toolCall.result?.status === "success") {
381 + return {
382 + ...toolCall.result.value,
383 + content: toolCall.result.value.content.flatMap((part) => [
384 + ...(part.text === undefined ? [] : [{ type: "text", text: part.text.text }]),
385 + ...(part.image === undefined
386 + ? []
387 + : [{ type: "image", mimeType: part.image.mimeType, data: part.image.data }]),
388 + ]),
389 + };
390 + }
391 return result.status === "success" ? result.value : result.error;
392 }
393
packages/provider-opencode/src/server/adapter.test.ts
+11 -1
@@ -6,6 +6,7 @@ import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts";
6 import { assert, describe, it } from "@effect/vitest";
7 import * as NodeServices from "@effect/platform-node/NodeServices";
8 import type { OpencodeClient, ToolPart } from "@opencode-ai/sdk/v2";
9 +import { toolOutputImages } from "@t3tools/shared/toolOutput";
10 import {
11 CheckpointId,
12 NodeId,
@@ -1273,6 +1274,13 @@ describe("OpenCodeAdapterV2", () => {
1274 title: "Edit remote document",
1275 metadata: {},
1276 output,
1277 + attachments: [
1278 + {
1279 + type: "file",
1280 + mime: "image/png",
1281 + url: "data:image/png;base64,iVBORw0KGgo=",
1282 + },
1283 + ],
1284 error: "failed",
1285 time: { start: 1, end: 2 },
1286 },
@@ -1365,7 +1373,9 @@ describe("OpenCodeAdapterV2", () => {
1373 });
1374 assert.deepEqual(item.input, { document: "remote-doc" });
1375 }
1368 - assert.equal(items[1]?.output, output);
1376 + assert.deepEqual(toolOutputImages(items[1]?.output), [
1377 + { mimeType: "image/png", data: "iVBORw0KGgo=" },
1378 + ]);
1379 assert.equal(items[3]?.toolName, "ambiguous_server_edit_document");
1380 assert.isUndefined(items[3]?.toolSource);
1381 assert.equal(statusReads, 1);
packages/provider-opencode/src/server/adapter.ts
+9 -2
@@ -80,7 +80,7 @@ import {
80 makeSubagentChildThread,
81 subagentThreadTitle,
82 } from "@t3tools/provider-core/server/subagentProjection";
83 -import { openCodeCommandFailed, openCodeToolTurnItem } from "./toolItems.ts";
83 +import { openCodeCommandFailed, openCodeToolOutput, openCodeToolTurnItem } from "./toolItems.ts";
84
85 export const OPENCODE_PROVIDER = ProviderDriverKind.make("opencode");
86 export const OPENCODE_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(OPENCODE_PROVIDER);
@@ -1605,6 +1605,10 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function
1605 toolName: part.tool.slice(serverName.replace(/[^a-zA-Z0-9_-]/g, "_").length + 1),
1606 title: toolTitle(part) === part.tool ? undefined : toolTitle(part),
1607 });
1608 + const files =
1609 + part.state.status === "completed"
1610 + ? part.state.attachments?.map((file) => ({ uri: file.url, mime: file.mime }))
1611 + : undefined;
1612 const turnItem: OrchestrationV2TurnItem = matchingServers?.length
1613 ? {
1614 ...base,
@@ -1612,12 +1616,15 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function
1616 ...presentation,
1617 toolName: part.tool,
1618 input,
1615 - ...(output === undefined ? {} : { output }),
1619 + ...(output === undefined && !files?.length
1620 + ? {}
1621 + : { output: openCodeToolOutput(output, files) }),
1622 }
1623 : openCodeToolTurnItem(base, {
1624 name: part.tool,
1625 input,
1626 output,
1627 + files,
1628 completedMetadata:
1629 part.state.status === "completed" ? part.state.metadata : undefined,
1630 });
packages/provider-opencode/src/server/toolItems.test.ts new
+43
@@ -0,0 +1,43 @@
1 +import { describe, expect, it } from "vite-plus/test";
2 +import {
3 + MAX_TOOL_OUTPUT_IMAGES,
4 + MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH,
5 + toolOutputImages,
6 +} from "@t3tools/shared/toolOutput";
7 +
8 +import { openCodeToolOutput } from "./toolItems.ts";
9 +
10 +describe("openCodeToolOutput", () => {
11 + it("preserves inline raster file content and leaves other files unserved", () => {
12 + const output = openCodeToolOutput("Preview complete", [
13 + { uri: "https://example.com/preview.png", mime: "image/png" },
14 + { uri: "file:///tmp/preview.png", mime: "image/png" },
15 + { uri: "data:image/svg+xml;base64,AAAA", mime: "image/svg+xml" },
16 + { uri: "data:image/png;base64,AAAA", mime: "image/jpeg" },
17 + { uri: "data:image/png;base64,", mime: "image/png" },
18 + { uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" },
19 + ]);
20 + expect(toolOutputImages(output)).toEqual([{ mimeType: "image/png", data: "iVBORw0KGgo=" }]);
21 + expect(output).toEqual({
22 + content: [
23 + { type: "text", text: "Preview complete" },
24 + { type: "image", mimeType: "image/png", data: "iVBORw0KGgo=" },
25 + ],
26 + });
27 + expect(openCodeToolOutput("Text result")).toBe("Text result");
28 + const oversized = "A".repeat(MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4);
29 + const bounded = openCodeToolOutput(
30 + "Preview complete",
31 + Array.from({ length: MAX_TOOL_OUTPUT_IMAGES + 2 }, (_, index) => ({
32 + uri: `data:image/png;base64,${index === 0 ? oversized : "AQ=="}`,
33 + mime: "image/png",
34 + })),
35 + );
36 + expect(typeof bounded === "object" ? bounded.content.length : 0).toBe(
37 + MAX_TOOL_OUTPUT_IMAGES + 1,
38 + );
39 + const images = toolOutputImages(bounded);
40 + expect(images[0]?.data?.length).toBe(oversized.length);
41 + expect(images[1]?.data).toBe("AQ==");
42 + });
43 +});
packages/provider-opencode/src/server/toolItems.ts
+28 -2
@@ -1,6 +1,29 @@
1 import type { OrchestrationV2TurnItem } from "@t3tools/contracts";
2 import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation";
3 import { formatReadToolLabel, formatSearchToolLabel } from "@t3tools/shared/toolActivity";
4 +import {
5 + MAX_TOOL_OUTPUT_IMAGES,
6 + readToolOutputImage,
7 + type ToolOutputImage,
8 +} from "@t3tools/shared/toolOutput";
9 +
10 +export function openCodeToolOutput(
11 + output: string | undefined,
12 + files: ReadonlyArray<{ readonly uri: string; readonly mime: string }> = [],
13 +) {
14 + const images: Array<ToolOutputImage & { type: "image" }> = [];
15 + for (const file of files) {
16 + const match = /^data:(image\/[a-z0-9.+-]+);base64,([A-Za-z0-9+/]+={0,2})$/i.exec(file.uri);
17 + if (match?.[1]?.toLowerCase() !== file.mime.toLowerCase()) continue;
18 + const image = readToolOutputImage({ type: "image", mimeType: file.mime, data: match?.[2] });
19 + if (image === null) continue;
20 + images.push({ type: "image", ...image });
21 + if (images.length === MAX_TOOL_OUTPUT_IMAGES) break;
22 + }
23 + return images.length === 0
24 + ? output
25 + : { content: [...(output ? [{ type: "text", text: output }] : []), ...images] };
26 +}
27
28 // Search results stay on the timeline wire, so keep their text a preview.
29 const SEARCH_PREVIEW_MAX_CHARS = 8_000;
@@ -13,7 +36,7 @@ type ToolItemBase = Omit<
36 function openCodeToolProjectionKind(
37 toolName: string,
38 ): "command_execution" | "file_change" | "file_search" | "web_search" | "dynamic_tool" {
16 - if (resolveT3McpToolId(toolName) !== null) return "dynamic_tool";
39 + if (toolName.startsWith("mcp__") || resolveT3McpToolId(toolName) !== null) return "dynamic_tool";
40 const normalized = toolName.toLowerCase();
41 if (normalized === "todowrite") {
42 return "dynamic_tool";
@@ -98,6 +121,7 @@ export function openCodeToolTurnItem(
121 readonly input: Record<string, unknown>;
122 readonly output: string | undefined;
123 readonly completedMetadata: unknown;
124 + readonly files?: ReadonlyArray<{ readonly uri: string; readonly mime: string }> | undefined;
125 },
126 ): OrchestrationV2TurnItem {
127 const { input, output } = tool;
@@ -178,7 +202,9 @@ export function openCodeToolTurnItem(
202 type: "dynamic_tool",
203 toolName: tool.name,
204 input,
181 - ...(cleanedOutput === undefined ? {} : { output: cleanedOutput }),
205 + ...(cleanedOutput === undefined && !tool.files?.length
206 + ? {}
207 + : { output: openCodeToolOutput(cleanedOutput, tool.files) }),
208 };
209 }
210 }
packages/provider-opencode/src/server/v2/adapter.ts
+10 -1
@@ -996,7 +996,11 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
996 turn: ActiveTurn,
997 id: string,
998 status: "running" | "completed" | "failed" | "interrupted",
999 - result?: { readonly output: string | undefined; readonly metadata: unknown },
999 + result?: {
1000 + readonly output: string | undefined;
1001 + readonly metadata: unknown;
1002 + readonly files?: ReadonlyArray<{ readonly uri: string; readonly mime: string }>;
1003 + },
1004 ) {
1005 const tool = turn.tools.get(id);
1006 if (tool === undefined) return;
@@ -1018,6 +1022,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
1022 name: tool.name,
1023 input: tool.input,
1024 output: result?.output,
1025 + files: result?.files,
1026 completedMetadata: status === "completed" ? result?.metadata : undefined,
1027 },
1028 ),
@@ -2097,6 +2102,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
2102 const output = textOf(event.data.content);
2103 yield* emitTool(state, turn, event.data.id, "completed", {
2104 output,
2105 + files: event.data.content.flatMap((part) => (part.type === "file" ? [part] : [])),
2106 metadata: event.data.metadata,
2107 });
2108 turn.tools.delete(event.data.id);
@@ -2555,6 +2561,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
2561 if (part.state.status === "completed") {
2562 yield* emitTool(state, turn, part.id, "completed", {
2563 output: textOf(part.state.content),
2564 + files: part.state.content.flatMap((content) =>
2565 + content.type === "file" ? [content] : [],
2566 + ),
2567 metadata: part.state.metadata,
2568 });
2569 turn.tools.delete(part.id);
packages/provider-pi/src/server/adapter.test.ts
+5 -2
@@ -2328,7 +2328,10 @@ describe("PiAdapterV2", () => {
2328 if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")
2329 return yield* Effect.die("Expected a read image tool item");
2330 assert.deepEqual(event.turnItem.output, { content: imageContent, structuredContent });
2331 - assert.equal(turnItemOutputText(event.turnItem), note);
2331 + assert.equal(
2332 + turnItemOutputText(event.turnItem),
2333 + JSON.stringify(structuredContent, null, 2),
2334 + );
2335 }
2336 }).pipe(Effect.scoped, Effect.provide(layerTest)),
2337 );
@@ -2385,7 +2388,7 @@ describe("PiAdapterV2", () => {
2388 assert.equal(event.turnItem.toolName, tool.name);
2389 assert.equal(event.turnItem.status, "completed");
2390 assert.deepEqual(event.turnItem.output, { content, structuredContent: metadata });
2388 - assert.equal(turnItemOutputText(event.turnItem), "Screenshot captured");
2391 + assert.equal(turnItemOutputText(event.turnItem), JSON.stringify(metadata, null, 2));
2392 const image = toolOutputImages(event.turnItem.output)[0];
2393 assert.equal(image?.mimeType, "image/png");
2394 assert.deepEqual(Buffer.from(image!.data!, "base64"), pixels);
packages/shared/src/toolOutput.test.ts
+8
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vite-plus/test";
3 import {
4 compactDynamicToolOutput,
5 MAX_TOOL_OUTPUT_IMAGES,
6 + mcpToolResultOutput,
7 toolOutputImages,
8 toolOutputIndicatesFailure,
9 } from "./toolOutput.ts";
@@ -187,6 +188,13 @@ describe("toolOutputImages", () => {
188 ]);
189 expect(toolOutputImages([mcpImage])).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
190 expect(toolOutputImages(mcpImage)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
191 + const structuredContent = { screenshot: { mimeType: "image/png", width: 728, height: 354 } };
192 + const content = [text, mcpImage];
193 + const output = mcpToolResultOutput({ content, structuredContent });
194 + expect(toolOutputImages(output)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
195 + expect(output).toEqual({ content, structuredContent });
196 + expect(mcpToolResultOutput({ content: [text], structuredContent })).toBe(structuredContent);
197 + expect(mcpToolResultOutput({ content })).toBe(content);
198 });
199
200 it("skips types the server must not serve inline", () => {
packages/shared/src/toolOutput.ts
+10
@@ -274,6 +274,16 @@ export function toolOutputImages(value: unknown): ReadonlyArray<ToolOutputImage>
274 return toolOutputImageBlocks(value).flatMap((block) => readToolOutputImage(block) ?? []);
275 }
276
277 +export function mcpToolResultOutput(result: {
278 + readonly content?: unknown;
279 + readonly structuredContent?: unknown;
280 +}): unknown {
281 + if (result.structuredContent != null && toolOutputImages(result.content).length > 0) {
282 + return { content: result.content, structuredContent: result.structuredContent };
283 + }
284 + return result.structuredContent ?? result.content;
285 +}
286 +
287 export function omitToolOutputImageData(value: unknown): unknown {
288 const omit = (block: unknown) => {
289 const image = readToolOutputImage(block);