The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 04:30 UTC
fix(tools): show images from MCP and OpenCode outputs (#30389)
Preserve MCP and OpenCode image outputs so tool details show screenshots before their inputs. Highlight HTML preview input and valid JSON arguments and results across clients, while keeping native file text readable. Sanitize CI test logs to avoid a Unicode runner crash without changing test exit codes. Validation: all CI checks passed, 729 affected tests and the full 156-test Pi suite passed locally, web and iOS details verified. Model: GPT 6.1 Sol. Harness: Codex.
Yash Singh committed
Nov 3, 2026 at 20:05 UTC
d51bc58f5e05123a416a3737ac4bce42d1d76b7c
20 files changed
+401
-105
.github/workflows/ci.yml
+7
-1
@@ -272,12 +272,18 @@ jobs:
272
env:
273
TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }}
274
run: |
275
+ set -o pipefail
276
filters=(--filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web')
277
if [[ -n "$TEST_PACKAGES" ]]; then
278
filters=()
279
for package in $TEST_PACKAGES; do filters+=(--filter "$package"); done
280
fi
280
- vp run --parallel --concurrency-limit 4 "${filters[@]}" test
281
+ # The runner's trace writer can crash on surrogate pairs in test output.
282
+ vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c '
283
+ import sys
284
+ for line in sys.stdin:
285
+ sys.stdout.write(line.encode("ascii", "backslashreplace").decode("ascii"))
286
+ '
287
288
# The web suite is the largest outside the server. In the shared Test job it
289
# started last and ran alone for about 40s after everything else finished.
apps/mobile/src/features/threads/thread-work-log.tsx
+43
-12
@@ -69,6 +69,7 @@ import {
69
workEntryRowLabel,
70
} from "../../lib/threadActivity";
71
import {
72
+ isJsonToolText,
73
toolCallLines,
74
turnItemOutputImages,
75
turnItemOutputText,
@@ -874,14 +875,16 @@ function WorkspacePreparationRetryButton(props: {
875
);
876
}
877
877
-function ReplCodeText({
878
+function ToolCodeText({
879
code,
880
+ language,
881
theme,
882
}: {
883
readonly code: string;
884
+ readonly language: "typescript" | "html" | "json";
885
readonly theme: "light" | "dark";
886
}) {
884
- const lines = useMarkdownCodeHighlight({ code, language: "typescript", theme, enabled: true });
887
+ const lines = useMarkdownCodeHighlight({ code, language, theme, enabled: true });
888
let offset = 0;
889
const highlighted = lines?.flatMap((line, lineIndex) => {
890
const separator = lineIndex > 0 ? "\n" : "";
@@ -1255,20 +1258,32 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
1258
) : call ? (
1259
[
1260
call.command,
1258
- ...(call.args ?? []).map(([key, value]) =>
1259
- key === "code" &&
1260
- shownItem.type === "dynamic_tool" &&
1261
- isJavaScriptReplTool(shownItem.toolName) ? (
1261
+ ...(call.args ?? []).map(([key, value]) => {
1262
+ const language =
1263
+ key === "html"
1264
+ ? "html"
1265
+ : key === "code" &&
1266
+ shownItem.type === "dynamic_tool" &&
1267
+ isJavaScriptReplTool(shownItem.toolName)
1268
+ ? "typescript"
1269
+ : isJsonToolText(value)
1270
+ ? "json"
1271
+ : null;
1272
+ return language ? (
1273
<View key={key}>
1274
<Text className="font-mono text-2xs leading-normal text-foreground-muted">
1275
{key}
1276
</Text>
1266
- <ReplCodeText code={value} theme={props.themeAppearance} />
1277
+ <ToolCodeText
1278
+ code={value}
1279
+ language={language}
1280
+ theme={props.themeAppearance}
1281
+ />
1282
</View>
1283
) : (
1284
`${key} ${value}`
1270
- ),
1271
- ),
1285
+ );
1286
+ }),
1287
call.argsText,
1288
]
1289
.filter((line) => Boolean(line))
@@ -1281,7 +1296,11 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
1296
selectable
1297
className="font-mono text-2xs leading-normal text-foreground"
1298
>
1284
- {line}
1299
+ {isJsonToolText(line) ? (
1300
+ <ToolCodeText code={line} language="json" theme={props.themeAppearance} />
1301
+ ) : (
1302
+ line
1303
+ )}
1304
</Text>
1305
) : (
1306
line
@@ -1289,7 +1308,11 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
1308
)
1309
) : fullDetail ? (
1310
<Text selectable className="font-mono text-2xs leading-normal text-foreground-muted">
1292
- {fullDetail}
1311
+ {isJsonToolText(fullDetail) ? (
1312
+ <ToolCodeText code={fullDetail} language="json" theme={props.themeAppearance} />
1313
+ ) : (
1314
+ fullDetail
1315
+ )}
1316
</Text>
1317
) : null}
1318
{fetchedOutput ? (
@@ -1300,7 +1323,15 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
1323
(!call || call.command || call.args || call.argsText) && "mt-1.5",
1324
)}
1325
>
1303
- {fetchedOutput}
1326
+ {isJsonToolText(fetchedOutput) ? (
1327
+ <ToolCodeText
1328
+ code={fetchedOutput}
1329
+ language="json"
1330
+ theme={props.themeAppearance}
1331
+ />
1332
+ ) : (
1333
+ fetchedOutput
1334
+ )}
1335
</Text>
1336
) : null}
1337
{streamsOutput ? null : exitLine}
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+13
-1
@@ -26,6 +26,7 @@ import type {
26
} from "@anthropic-ai/claude-agent-sdk/sdk-tools";
27
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
28
import * as NodeServices from "@effect/platform-node/NodeServices";
29
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
30
import {
31
ChatAttachmentId,
32
type ChatAttachment,
@@ -4844,11 +4845,19 @@ describe("ClaudeAdapterV2 background wake turns", () => {
4845
{
4846
type: "tool_result",
4847
tool_use_id: id,
4847
- content: "Weather result",
4848
+ content: [
4849
+ { type: "text", text: "Weather result" },
4850
+ {
4851
+ type: "image",
4852
+ source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
4853
+ },
4854
+ ],
4855
is_error: isError,
4856
},
4857
],
4858
},
4859
+ tool_use_result:
4860
+ title === undefined ? { screenshot: { mimeType: "image/png" } } : undefined,
4861
}),
4862
);
4863
yield* Queue.offer(
@@ -4881,6 +4890,9 @@ describe("ClaudeAdapterV2 background wake turns", () => {
4890
assert.deepEqual(item.input, { city: "Berlin" });
4891
}
4892
assert.equal(new Set(items.map((item) => item.id)).size, 1);
4893
+ assert.deepEqual(toolOutputImages(items.at(-1)?.output), [
4894
+ { mimeType: "image/png", data: "iVBORw0KGgo=" },
4895
+ ]);
4896
}).pipe(
4897
Effect.scoped,
4898
Effect.provide(
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+8
@@ -1,4 +1,5 @@
1
import * as DocumentText from "../../provider/DocumentText.ts";
2
+import { mcpToolResultOutput, toolOutputImages } from "@t3tools/shared/toolOutput";
3
import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts";
4
import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts";
5
import {
@@ -3084,7 +3085,14 @@ function claudeNativeToolOutputValue(output: ClaudeNativeToolOutput): unknown |
3085
case "none":
3086
return undefined;
3087
case "content_block":
3088
+ return output.value;
3089
case "structured_tool_use_result":
3090
+ if (toolOutputImages(output.value).length === 0) {
3091
+ return mcpToolResultOutput({
3092
+ content: output.fallbackValue,
3093
+ structuredContent: output.value,
3094
+ });
3095
+ }
3096
return output.value;
3097
default:
3098
return assertNever(output);
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+2
-3
@@ -50,6 +50,7 @@ import {
50
import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model";
51
import { appendGeneratedImageEmbeds } from "@t3tools/shared/generatedImageMarkdown";
52
import { resolveSpawnCommand } from "@t3tools/shared/shell";
53
+import { mcpToolResultOutput } from "@t3tools/shared/toolOutput";
54
import type {
55
ChatAttachment,
56
OrchestrationV2AppThread,
@@ -490,9 +491,7 @@ function codexMcpToolOutput(
491
const resultOutput =
492
item.result === null || item.result === undefined
493
? undefined
493
- : item.result.structuredContent !== null && item.result.structuredContent !== undefined
494
- ? item.result.structuredContent
495
- : item.result.content;
494
+ : mcpToolResultOutput(item.result);
495
496
if (item.error === null || item.error === undefined) {
497
return resultOutput;
apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts
+62
-42
@@ -1,4 +1,5 @@
1
import * as NodeURL from "node:url";
2
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
3
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
4
import * as NodeServices from "@effect/platform-node/NodeServices";
5
import { assert, it } from "@effect/vitest";
@@ -2715,48 +2716,67 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
2716
}).pipe(Effect.scoped),
2717
);
2718
2718
- it.effect("keeps thread-scoped T3 waits as dynamic tools through their lifecycle", () =>
2719
- Effect.gen(function* () {
2720
- const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };
2721
- const name = "t3-code-web_delegate_task";
2722
- const input = { task: "Review", mode: "wait" };
2723
- const { runtime, thread } = yield* resumed([
2724
- out("session.prompt", { sessionID: SESSION, text: "<any>" }),
2725
- promptAccepted,
2726
- event("session.tool.input.started", { ...tool, name }),
2727
- event("session.tool.called", { ...tool, name, input, executed: true }),
2728
- event("session.tool.success", {
2729
- ...tool,
2730
- content: [{ type: "text", text: "Reviewed" }],
2731
- metadata: {},
2732
- executed: true,
2733
- }),
2734
- event("session.execution.succeeded", { sessionID: SESSION }),
2735
- ]);
2736
- const events = yield* runtime.events.pipe(
2737
- Stream.takeUntil((event) => event.type === "turn.terminal"),
2738
- Stream.runCollect,
2739
- Effect.forkScoped,
2740
- );
2741
- yield* runtime.startTurn(turnInput(thread));
2742
- const items = (yield* Fiber.join(events)).flatMap((event) =>
2743
- event.type === "turn_item.updated" &&
2744
- event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)
2745
- ? [event.turnItem]
2746
- : [],
2747
- );
2748
- assert.deepEqual(
2749
- items.map((item) => item.type),
2750
- ["dynamic_tool", "dynamic_tool", "dynamic_tool"],
2751
- );
2752
- assert.deepEqual(
2753
- items.map((item) => item.status),
2754
- ["running", "running", "completed"],
2755
- );
2756
- const last = items.at(-1);
2757
- assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);
2758
- assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);
2759
- }).pipe(Effect.scoped),
2719
+ it.effect.each(["delegate_task", "html_preview", "mcp__acme__edit_document"])(
2720
+ "keeps MCP %s as a dynamic tool through its lifecycle",
2721
+ (toolName) =>
2722
+ Effect.gen(function* () {
2723
+ const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };
2724
+ const name = toolName.startsWith("mcp__") ? toolName : `t3-code-web_${toolName}`;
2725
+ const input =
2726
+ toolName !== "delegate_task"
2727
+ ? { html: "<p>Preview</p>" }
2728
+ : { task: "Review", mode: "wait" };
2729
+ const { runtime, thread } = yield* resumed([
2730
+ out("session.prompt", { sessionID: SESSION, text: "<any>" }),
2731
+ promptAccepted,
2732
+ event("session.tool.input.started", { ...tool, name }),
2733
+ event("session.tool.called", { ...tool, name, input, executed: true }),
2734
+ event(
2735
+ "session.tool.success",
2736
+ {
2737
+ ...tool,
2738
+ content: [
2739
+ { type: "text", text: "Reviewed" },
2740
+ ...(toolName !== "delegate_task"
2741
+ ? [{ type: "file", uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" }]
2742
+ : []),
2743
+ ],
2744
+ metadata: {},
2745
+ executed: true,
2746
+ },
2747
+ undefined,
2748
+ 2,
2749
+ ),
2750
+ event("session.execution.succeeded", { sessionID: SESSION }),
2751
+ ]);
2752
+ const events = yield* runtime.events.pipe(
2753
+ Stream.takeUntil((event) => event.type === "turn.terminal"),
2754
+ Stream.runCollect,
2755
+ Effect.forkScoped,
2756
+ );
2757
+ yield* runtime.startTurn(turnInput(thread));
2758
+ const items = (yield* Fiber.join(events)).flatMap((event) =>
2759
+ event.type === "turn_item.updated" &&
2760
+ event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)
2761
+ ? [event.turnItem]
2762
+ : [],
2763
+ );
2764
+ assert.deepEqual(
2765
+ items.map((item) => item.type),
2766
+ ["dynamic_tool", "dynamic_tool", "dynamic_tool"],
2767
+ );
2768
+ assert.deepEqual(
2769
+ items.map((item) => item.status),
2770
+ ["running", "running", "completed"],
2771
+ );
2772
+ const last = items.at(-1);
2773
+ assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);
2774
+ assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);
2775
+ assert.deepEqual(
2776
+ toolOutputImages(last?.type === "dynamic_tool" ? last.output : null),
2777
+ toolName !== "delegate_task" ? [{ mimeType: "image/png", data: "iVBORw0KGgo=" }] : [],
2778
+ );
2779
+ }).pipe(Effect.scoped),
2780
);
2781
2782
it.effect("stores a shell command that exits non-zero as failed", () =>
apps/web/src/components/chat/V2ItemInspector.tsx
+19
-25
@@ -1,6 +1,7 @@
1
import type { ToolCallDiffTarget } from "../../diffPanelStore";
2
import type { EnvironmentId, OrchestrationV2ProjectedTurnItem } from "@t3tools/contracts";
3
import {
4
+ isJsonToolText,
5
toolCallLines,
6
turnItemDetailRevision,
7
turnItemNeedsDetailFetch,
@@ -43,7 +44,7 @@ function SyntaxTokens({
44
language,
45
}: {
46
readonly text: string;
46
- readonly language: "json" | "typescript";
47
+ readonly language: "json" | "typescript" | "html";
48
}) {
49
const resolvedTheme = useRenderedAppearance();
50
const highlighter = use(getSyntaxHighlighterPromise(language));
@@ -64,16 +65,6 @@ function SyntaxTokens({
65
const monoClassName =
66
"font-mono text-(length:--font-size-code,var(--text-2xs)) leading-relaxed whitespace-pre-wrap break-words select-text";
67
67
-function isJsonText(text: string): boolean {
68
- if (!text) return false;
69
- try {
70
- JSON.parse(text);
71
- return true;
72
- } catch {
73
- return false;
74
- }
75
-}
76
-
68
function StructuredValue({
69
value,
70
highlightJson = false,
@@ -81,10 +72,10 @@ function StructuredValue({
72
}: {
73
readonly value: unknown;
74
readonly highlightJson?: boolean;
84
- readonly language?: "typescript";
75
+ readonly language?: "typescript" | "html" | "json";
76
}) {
77
const text = typeof value === "string" ? value : JSON.stringify(value, null, 2);
87
- const isJson = highlightJson && isJsonText(text);
78
+ const isJson = highlightJson && isJsonToolText(text);
79
if (!text) return null;
80
return (
81
<pre className={cn("max-h-80 overflow-auto text-muted-foreground", monoClassName)}>
@@ -150,8 +141,8 @@ interface ToolOutputState {
141
readonly onImageExpand?: ((preview: ExpandedImagePreview) => void) | undefined;
142
}
143
153
-function ToolOutput(props: ToolOutputState) {
154
- const images = props.images.map((resource) => (
144
+function ToolOutputImages(props: ToolOutputState) {
145
+ return props.images.map((resource) => (
146
<ChatMarkdownAssetImage
147
key={resource.index}
148
environmentId={props.environmentId}
@@ -161,21 +152,18 @@ function ToolOutput(props: ToolOutputState) {
152
onImageExpand={props.onImageExpand}
153
/>
154
));
164
- const text = props.output ? (
165
- <div className="max-h-80 overflow-auto text-muted-foreground">{props.output}</div>
155
+}
156
+
157
+function ToolOutput(props: ToolOutputState) {
158
+ return props.output ? (
159
+ <StructuredValue value={props.output} highlightJson />
160
) : props.pending ? (
161
<div className="text-muted-foreground italic">Loading output…</div>
162
) : props.error ? (
163
<div className="text-destructive">Couldn't load output: {props.error}</div>
170
- ) : props.empty && images.length === 0 ? (
164
+ ) : props.empty && props.images.length === 0 ? (
165
<div className="text-muted-foreground italic">No output.</div>
166
) : null;
173
- return (
174
- <>
175
- {images}
176
- {text}
177
- </>
178
- );
167
}
168
169
export function FetchedToolOutput(props: {
@@ -187,6 +175,7 @@ export function FetchedToolOutput(props: {
175
const { output } = useFetchedTurnItem(props.projectedItem, props.environmentId, props.source);
176
return (
177
<div className={cn("mt-1.5 space-y-1.5", monoClassName)}>
178
+ <ToolOutputImages {...output} onImageExpand={props.onImageExpand} />
179
<ToolOutput {...output} onImageExpand={props.onImageExpand} />
180
</div>
181
);
@@ -205,14 +194,19 @@ function ToolCallBody(
194
const call = toolCallLines({ command: props.command, args: props.args });
195
return (
196
<div className={cn("space-y-1.5", monoClassName)}>
197
+ <ToolOutputImages {...props} />
198
{call.command ? <ShellCommandBlock command={call.command} /> : null}
199
{call.args ? (
200
<div className="text-foreground/85">
201
{call.args.map(([key, value]) => (
202
<div key={key}>
203
<span className="text-muted-foreground">{key} </span>
214
- {key === "code" && isJavaScriptReplTool(props.toolName) ? (
204
+ {key === "html" ? (
205
+ <StructuredValue value={value} language="html" />
206
+ ) : key === "code" && isJavaScriptReplTool(props.toolName) ? (
207
<StructuredValue value={value} language="typescript" />
208
+ ) : isJsonToolText(value) ? (
209
+ <StructuredValue value={value} language="json" />
210
) : (
211
value
212
)}
packages/client-runtime/src/work-log/itemDetail.test.ts
+66
-5
@@ -2,7 +2,26 @@ import { describe, expect, it } from "vite-plus/test";
2
import { ThreadId, TurnItemId, type OrchestrationV2TurnItem } from "@t3tools/contracts";
3
import * as DateTime from "effect/DateTime";
4
5
-import { turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
5
+import { isJsonToolText, turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
6
+
7
+describe("isJsonToolText", () => {
8
+ it.each([
9
+ ['{"screenshot":{"mimeType":"image/png"}}', true],
10
+ ['[{"threadId":"thread-1"}]', true],
11
+ ['{\n\n "status": "completed"\n}', true],
12
+ ['{\n "count": 1\n}\n\n{\n "count": 2\n}', true],
13
+ ["false", true],
14
+ ["null", true],
15
+ ["42", true],
16
+ ['"completed"', true],
17
+ ['{"partial":', false],
18
+ ['{"count":1}\n\nFinished.', false],
19
+ ["No output.", false],
20
+ ["", false],
21
+ ] as const)("classifies %j as JSON: %s", (text, expected) => {
22
+ expect(isJsonToolText(text)).toBe(expected);
23
+ });
24
+});
25
26
function toolItem(output: unknown): OrchestrationV2TurnItem {
27
return {
@@ -28,15 +47,54 @@ function toolItem(output: unknown): OrchestrationV2TurnItem {
47
}
48
49
describe("tool image output parsing", () => {
31
- it.each(["array", "content"] as const)(
50
+ it.each(["image-only", "summary", "json"] as const)(
51
+ "keeps structured fields beside %s content",
52
+ (shape) => {
53
+ const content = [
54
+ { type: "image", mimeType: "image/png" },
55
+ ...(shape === "image-only"
56
+ ? []
57
+ : [{ type: "text", text: shape === "summary" ? "Sunny" : '{"temp":21}' }]),
58
+ ];
59
+ for (const structuredContent of [
60
+ { temp: 21 },
61
+ { type: "text", text: "Sunny", temp: 21 },
62
+ { content: [{ type: "text", text: "Sunny" }], temp: 21 },
63
+ ]) {
64
+ const item = toolItem({ structuredContent, content });
65
+ expect(turnItemOutputText(item)).toBe(JSON.stringify(structuredContent, null, 2));
66
+ expect(turnItemOutputImages(item)).toHaveLength(1);
67
+ }
68
+ for (const structuredContent of [null, {}, [], ""]) {
69
+ const item = toolItem({ structuredContent, content: [{ type: "text", text: "Sunny" }] });
70
+ expect(turnItemOutputText(item)).toBe("Sunny");
71
+ }
72
+ expect(
73
+ turnItemOutputText(
74
+ toolItem({
75
+ content: [{ type: "text", text: "File contents" }],
76
+ structuredContent: { path: "read.ts", ready: true },
77
+ }),
78
+ ),
79
+ ).toBe("File contents");
80
+ },
81
+ );
82
+ it.each(["array", "content", "structured"] as const)(
83
"keeps markers for images beyond the served positions in %s output",
84
(shape) => {
85
// Detail responses retain image metadata after the server strips image bytes.
86
const image = { type: "image", mimeType: "image/png" };
87
const blocks = Array.from({ length: 8 }, () => ({ ...image }));
37
- const wrap = (content: unknown[]) => (shape === "array" ? content : { content });
88
+ const wrap = (content: unknown[]) =>
89
+ shape === "array"
90
+ ? content
91
+ : shape === "structured"
92
+ ? { content, structuredContent: { temp: 21 } }
93
+ : { content };
94
const withinLimit = toolItem(wrap(blocks));
39
- expect(turnItemOutputText(withinLimit)).toBeNull();
95
+ expect(turnItemOutputText(withinLimit)).toBe(
96
+ shape === "structured" ? '{\n "temp": 21\n}' : null,
97
+ );
98
expect(turnItemOutputImages(withinLimit)).toHaveLength(8);
99
100
const overflow = toolItem(
@@ -48,7 +106,10 @@ describe("tool image output parsing", () => {
106
]),
107
);
108
expect(turnItemOutputImages(overflow)).toHaveLength(8);
51
- expect(turnItemOutputText(overflow)).toBe("Two additional screenshots:\n[image]\n[image]");
109
+ expect(turnItemOutputText(overflow)).toBe(
110
+ (shape === "structured" ? '{\n "temp": 21\n}' : "Two additional screenshots:") +
111
+ "\n[image]\n[image]",
112
+ );
113
},
114
);
115
});
packages/client-runtime/src/work-log/itemDetail.ts
+24
-4
@@ -39,6 +39,14 @@ function textFromBlocks(
39
return textFromBlocks(value.content, depth + 1, imageBlocks);
40
if (keys.length === 2 && keys.includes("content") && keys.includes("structuredContent")) {
41
const text = textFromBlocks(value.content, depth + 1, imageBlocks);
42
+ const images = Array.isArray(value.content)
43
+ ? value.content.filter((block) => isRecord(block) && block.type === "image")
44
+ : [];
45
+ const structured = formatJsonValue(value.structuredContent);
46
+ if (structured !== null && (images.length > 0 || !text?.trim())) {
47
+ const markers = textFromBlocks(images, depth + 1, imageBlocks);
48
+ return [structured, markers].filter(Boolean).join("\n");
49
+ }
50
return text?.trim() ? text : null;
51
}
52
return null;
@@ -52,6 +60,12 @@ function parseJson(text: string): unknown {
60
}
61
}
62
63
+export function isJsonToolText(text: string): boolean {
64
+ if (parseJson(text) !== undefined) return true;
65
+ const documents = text.trim().split(/\n\s*\n/);
66
+ return documents.length > 1 && documents.every((document) => parseJson(document) !== undefined);
67
+}
68
+
69
function prettyJsonText(text: string): string {
70
const trimmed = text.trim();
71
if (!/^[[{]/.test(trimmed)) return text;
@@ -63,11 +77,9 @@ function prettyJsonText(text: string): string {
77
return documents.map((document) => JSON.stringify(document, null, 2)).join("\n\n");
78
}
79
66
-function formatToolValue(value: unknown, hasImagePreview = false): string | null {
80
+function formatJsonValue(value: unknown): string | null {
81
if (value === undefined || value === null) return null;
68
- if (hasImagePreview && isRecord(value) && value.type === "image") return null;
69
- const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));
70
- if (text !== null) return text.trim() ? prettyJsonText(text) : null;
82
+ if (typeof value === "string" && !value.trim()) return null;
83
let json: string | undefined;
84
try {
85
json = JSON.stringify(value, null, 2);
@@ -78,6 +90,14 @@ function formatToolValue(value: unknown, hasImagePreview = false): string | null
90
return json;
91
}
92
93
+function formatToolValue(value: unknown, hasImagePreview = false): string | null {
94
+ if (value === undefined || value === null) return null;
95
+ if (hasImagePreview && isRecord(value) && value.type === "image") return null;
96
+ const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));
97
+ if (text !== null) return text.trim() ? prettyJsonText(text) : null;
98
+ return formatJsonValue(value);
99
+}
100
+
101
export function toolCallLines(input: {
102
readonly command?: string | undefined;
103
readonly args?: unknown;
packages/provider-acp/src/server/adapter.ts
+6
-3
@@ -39,7 +39,11 @@ import { HTML_RENDER_TOOL_NAME } from "@t3tools/shared/htmlRender";
39
import { modelSelectionsEqual } from "@t3tools/shared/model";
40
import { type SelfInvocation, selfInvocationArgs } from "@t3tools/shared/nodeRuntime";
41
import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation";
42
-import { compactDynamicToolOutput, htmlRenderFromToolItem } from "@t3tools/shared/toolOutput";
42
+import {
43
+ compactDynamicToolOutput,
44
+ htmlRenderFromToolItem,
45
+ mcpToolResultOutput,
46
+} from "@t3tools/shared/toolOutput";
47
import { FILE_HEADERS_ONLY, formatPatch, structuredPatch } from "diff";
48
import * as Cause from "effect/Cause";
49
import * as Clock from "effect/Clock";
@@ -923,8 +927,7 @@ function acpMcpToolCallOutput(rawOutput: unknown): unknown {
927
if (record === undefined) return rawOutput;
928
if (!("result" in record) && !("error" in record)) return rawOutput;
929
const result = unknownRecord(record.result);
926
- const resultOutput =
927
- result === undefined ? undefined : (result.structuredContent ?? result.content ?? undefined);
930
+ const resultOutput = result === undefined ? undefined : mcpToolResultOutput(result);
931
const errorMessage = unknownRecord(record.error)?.message;
932
if (typeof errorMessage !== "string") {
933
return resultOutput ?? rawOutput;
packages/provider-cursor/src/server/adapter.test.ts
+16
-1
@@ -1,6 +1,7 @@
1
import * as NodeServices from "@effect/platform-node/NodeServices";
2
import { assert, describe, it } from "@effect/vitest";
3
import type { InteractionUpdate } from "@cursor/sdk";
4
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
5
import {
6
EnvironmentId,
7
MessageId,
@@ -844,7 +845,18 @@ describe("CursorAdapterV2", () => {
845
args: { city: "Berlin" },
846
},
847
...(type === "tool-call-completed"
847
- ? { result: { status: "success" as const, value: { content: [], isError: false } } }
848
+ ? {
849
+ result: {
850
+ status: "success" as const,
851
+ value: {
852
+ content: [
853
+ { text: { text: "Preview complete" } },
854
+ { image: { data: "iVBORw0KGgo=", mimeType: "image/png" } },
855
+ ],
856
+ isError: false,
857
+ },
858
+ },
859
+ }
860
: {}),
861
},
862
})),
@@ -1155,6 +1167,9 @@ describe("CursorAdapterV2", () => {
1167
});
1168
assert.deepEqual(item.input, { city: "Berlin" });
1169
}
1170
+ assert.deepEqual(toolOutputImages(mcpItems.at(-1)?.output), [
1171
+ { mimeType: "image/png", data: "iVBORw0KGgo=" },
1172
+ ]);
1173
const fileSearchItems = events.flatMap((event) =>
1174
event.type === "turn_item.updated" &&
1175
event.turnItem.type === "file_search" &&
packages/provider-cursor/src/server/adapter.ts
+11
@@ -377,6 +377,17 @@ function cursorToolOutput(toolCall: ToolCall): unknown {
377
if (result === undefined) {
378
return undefined;
379
}
380
+ if (toolCall.type === "mcp" && toolCall.result?.status === "success") {
381
+ return {
382
+ ...toolCall.result.value,
383
+ content: toolCall.result.value.content.flatMap((part) => [
384
+ ...(part.text === undefined ? [] : [{ type: "text", text: part.text.text }]),
385
+ ...(part.image === undefined
386
+ ? []
387
+ : [{ type: "image", mimeType: part.image.mimeType, data: part.image.data }]),
388
+ ]),
389
+ };
390
+ }
391
return result.status === "success" ? result.value : result.error;
392
}
393
packages/provider-opencode/src/server/adapter.test.ts
+11
-1
@@ -6,6 +6,7 @@ import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts";
6
import { assert, describe, it } from "@effect/vitest";
7
import * as NodeServices from "@effect/platform-node/NodeServices";
8
import type { OpencodeClient, ToolPart } from "@opencode-ai/sdk/v2";
9
+import { toolOutputImages } from "@t3tools/shared/toolOutput";
10
import {
11
CheckpointId,
12
NodeId,
@@ -1273,6 +1274,13 @@ describe("OpenCodeAdapterV2", () => {
1274
title: "Edit remote document",
1275
metadata: {},
1276
output,
1277
+ attachments: [
1278
+ {
1279
+ type: "file",
1280
+ mime: "image/png",
1281
+ url: "data:image/png;base64,iVBORw0KGgo=",
1282
+ },
1283
+ ],
1284
error: "failed",
1285
time: { start: 1, end: 2 },
1286
},
@@ -1365,7 +1373,9 @@ describe("OpenCodeAdapterV2", () => {
1373
});
1374
assert.deepEqual(item.input, { document: "remote-doc" });
1375
}
1368
- assert.equal(items[1]?.output, output);
1376
+ assert.deepEqual(toolOutputImages(items[1]?.output), [
1377
+ { mimeType: "image/png", data: "iVBORw0KGgo=" },
1378
+ ]);
1379
assert.equal(items[3]?.toolName, "ambiguous_server_edit_document");
1380
assert.isUndefined(items[3]?.toolSource);
1381
assert.equal(statusReads, 1);
packages/provider-opencode/src/server/adapter.ts
+9
-2
@@ -80,7 +80,7 @@ import {
80
makeSubagentChildThread,
81
subagentThreadTitle,
82
} from "@t3tools/provider-core/server/subagentProjection";
83
-import { openCodeCommandFailed, openCodeToolTurnItem } from "./toolItems.ts";
83
+import { openCodeCommandFailed, openCodeToolOutput, openCodeToolTurnItem } from "./toolItems.ts";
84
85
export const OPENCODE_PROVIDER = ProviderDriverKind.make("opencode");
86
export const OPENCODE_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(OPENCODE_PROVIDER);
@@ -1605,6 +1605,10 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function
1605
toolName: part.tool.slice(serverName.replace(/[^a-zA-Z0-9_-]/g, "_").length + 1),
1606
title: toolTitle(part) === part.tool ? undefined : toolTitle(part),
1607
});
1608
+ const files =
1609
+ part.state.status === "completed"
1610
+ ? part.state.attachments?.map((file) => ({ uri: file.url, mime: file.mime }))
1611
+ : undefined;
1612
const turnItem: OrchestrationV2TurnItem = matchingServers?.length
1613
? {
1614
...base,
@@ -1612,12 +1616,15 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function
1616
...presentation,
1617
toolName: part.tool,
1618
input,
1615
- ...(output === undefined ? {} : { output }),
1619
+ ...(output === undefined && !files?.length
1620
+ ? {}
1621
+ : { output: openCodeToolOutput(output, files) }),
1622
}
1623
: openCodeToolTurnItem(base, {
1624
name: part.tool,
1625
input,
1626
output,
1627
+ files,
1628
completedMetadata:
1629
part.state.status === "completed" ? part.state.metadata : undefined,
1630
});
packages/provider-opencode/src/server/toolItems.test.ts
new
+43
@@ -0,0 +1,43 @@
1
+import { describe, expect, it } from "vite-plus/test";
2
+import {
3
+ MAX_TOOL_OUTPUT_IMAGES,
4
+ MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH,
5
+ toolOutputImages,
6
+} from "@t3tools/shared/toolOutput";
7
+
8
+import { openCodeToolOutput } from "./toolItems.ts";
9
+
10
+describe("openCodeToolOutput", () => {
11
+ it("preserves inline raster file content and leaves other files unserved", () => {
12
+ const output = openCodeToolOutput("Preview complete", [
13
+ { uri: "https://example.com/preview.png", mime: "image/png" },
14
+ { uri: "file:///tmp/preview.png", mime: "image/png" },
15
+ { uri: "data:image/svg+xml;base64,AAAA", mime: "image/svg+xml" },
16
+ { uri: "data:image/png;base64,AAAA", mime: "image/jpeg" },
17
+ { uri: "data:image/png;base64,", mime: "image/png" },
18
+ { uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" },
19
+ ]);
20
+ expect(toolOutputImages(output)).toEqual([{ mimeType: "image/png", data: "iVBORw0KGgo=" }]);
21
+ expect(output).toEqual({
22
+ content: [
23
+ { type: "text", text: "Preview complete" },
24
+ { type: "image", mimeType: "image/png", data: "iVBORw0KGgo=" },
25
+ ],
26
+ });
27
+ expect(openCodeToolOutput("Text result")).toBe("Text result");
28
+ const oversized = "A".repeat(MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4);
29
+ const bounded = openCodeToolOutput(
30
+ "Preview complete",
31
+ Array.from({ length: MAX_TOOL_OUTPUT_IMAGES + 2 }, (_, index) => ({
32
+ uri: `data:image/png;base64,${index === 0 ? oversized : "AQ=="}`,
33
+ mime: "image/png",
34
+ })),
35
+ );
36
+ expect(typeof bounded === "object" ? bounded.content.length : 0).toBe(
37
+ MAX_TOOL_OUTPUT_IMAGES + 1,
38
+ );
39
+ const images = toolOutputImages(bounded);
40
+ expect(images[0]?.data?.length).toBe(oversized.length);
41
+ expect(images[1]?.data).toBe("AQ==");
42
+ });
43
+});
packages/provider-opencode/src/server/toolItems.ts
+28
-2
@@ -1,6 +1,29 @@
1
import type { OrchestrationV2TurnItem } from "@t3tools/contracts";
2
import { resolveT3McpToolId } from "@t3tools/shared/t3McpToolPresentation";
3
import { formatReadToolLabel, formatSearchToolLabel } from "@t3tools/shared/toolActivity";
4
+import {
5
+ MAX_TOOL_OUTPUT_IMAGES,
6
+ readToolOutputImage,
7
+ type ToolOutputImage,
8
+} from "@t3tools/shared/toolOutput";
9
+
10
+export function openCodeToolOutput(
11
+ output: string | undefined,
12
+ files: ReadonlyArray<{ readonly uri: string; readonly mime: string }> = [],
13
+) {
14
+ const images: Array<ToolOutputImage & { type: "image" }> = [];
15
+ for (const file of files) {
16
+ const match = /^data:(image\/[a-z0-9.+-]+);base64,([A-Za-z0-9+/]+={0,2})$/i.exec(file.uri);
17
+ if (match?.[1]?.toLowerCase() !== file.mime.toLowerCase()) continue;
18
+ const image = readToolOutputImage({ type: "image", mimeType: file.mime, data: match?.[2] });
19
+ if (image === null) continue;
20
+ images.push({ type: "image", ...image });
21
+ if (images.length === MAX_TOOL_OUTPUT_IMAGES) break;
22
+ }
23
+ return images.length === 0
24
+ ? output
25
+ : { content: [...(output ? [{ type: "text", text: output }] : []), ...images] };
26
+}
27
28
// Search results stay on the timeline wire, so keep their text a preview.
29
const SEARCH_PREVIEW_MAX_CHARS = 8_000;
@@ -13,7 +36,7 @@ type ToolItemBase = Omit<
36
function openCodeToolProjectionKind(
37
toolName: string,
38
): "command_execution" | "file_change" | "file_search" | "web_search" | "dynamic_tool" {
16
- if (resolveT3McpToolId(toolName) !== null) return "dynamic_tool";
39
+ if (toolName.startsWith("mcp__") || resolveT3McpToolId(toolName) !== null) return "dynamic_tool";
40
const normalized = toolName.toLowerCase();
41
if (normalized === "todowrite") {
42
return "dynamic_tool";
@@ -98,6 +121,7 @@ export function openCodeToolTurnItem(
121
readonly input: Record<string, unknown>;
122
readonly output: string | undefined;
123
readonly completedMetadata: unknown;
124
+ readonly files?: ReadonlyArray<{ readonly uri: string; readonly mime: string }> | undefined;
125
},
126
): OrchestrationV2TurnItem {
127
const { input, output } = tool;
@@ -178,7 +202,9 @@ export function openCodeToolTurnItem(
202
type: "dynamic_tool",
203
toolName: tool.name,
204
input,
181
- ...(cleanedOutput === undefined ? {} : { output: cleanedOutput }),
205
+ ...(cleanedOutput === undefined && !tool.files?.length
206
+ ? {}
207
+ : { output: openCodeToolOutput(cleanedOutput, tool.files) }),
208
};
209
}
210
}
packages/provider-opencode/src/server/v2/adapter.ts
+10
-1
@@ -996,7 +996,11 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
996
turn: ActiveTurn,
997
id: string,
998
status: "running" | "completed" | "failed" | "interrupted",
999
- result?: { readonly output: string | undefined; readonly metadata: unknown },
999
+ result?: {
1000
+ readonly output: string | undefined;
1001
+ readonly metadata: unknown;
1002
+ readonly files?: ReadonlyArray<{ readonly uri: string; readonly mime: string }>;
1003
+ },
1004
) {
1005
const tool = turn.tools.get(id);
1006
if (tool === undefined) return;
@@ -1018,6 +1022,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
1022
name: tool.name,
1023
input: tool.input,
1024
output: result?.output,
1025
+ files: result?.files,
1026
completedMetadata: status === "completed" ? result?.metadata : undefined,
1027
},
1028
),
@@ -2097,6 +2102,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
2102
const output = textOf(event.data.content);
2103
yield* emitTool(state, turn, event.data.id, "completed", {
2104
output,
2105
+ files: event.data.content.flatMap((part) => (part.type === "file" ? [part] : [])),
2106
metadata: event.data.metadata,
2107
});
2108
turn.tools.delete(event.data.id);
@@ -2555,6 +2561,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
2561
if (part.state.status === "completed") {
2562
yield* emitTool(state, turn, part.id, "completed", {
2563
output: textOf(part.state.content),
2564
+ files: part.state.content.flatMap((content) =>
2565
+ content.type === "file" ? [content] : [],
2566
+ ),
2567
metadata: part.state.metadata,
2568
});
2569
turn.tools.delete(part.id);
packages/provider-pi/src/server/adapter.test.ts
+5
-2
@@ -2328,7 +2328,10 @@ describe("PiAdapterV2", () => {
2328
if (event.type !== "turn_item.updated" || event.turnItem.type !== "dynamic_tool")
2329
return yield* Effect.die("Expected a read image tool item");
2330
assert.deepEqual(event.turnItem.output, { content: imageContent, structuredContent });
2331
- assert.equal(turnItemOutputText(event.turnItem), note);
2331
+ assert.equal(
2332
+ turnItemOutputText(event.turnItem),
2333
+ JSON.stringify(structuredContent, null, 2),
2334
+ );
2335
}
2336
}).pipe(Effect.scoped, Effect.provide(layerTest)),
2337
);
@@ -2385,7 +2388,7 @@ describe("PiAdapterV2", () => {
2388
assert.equal(event.turnItem.toolName, tool.name);
2389
assert.equal(event.turnItem.status, "completed");
2390
assert.deepEqual(event.turnItem.output, { content, structuredContent: metadata });
2388
- assert.equal(turnItemOutputText(event.turnItem), "Screenshot captured");
2391
+ assert.equal(turnItemOutputText(event.turnItem), JSON.stringify(metadata, null, 2));
2392
const image = toolOutputImages(event.turnItem.output)[0];
2393
assert.equal(image?.mimeType, "image/png");
2394
assert.deepEqual(Buffer.from(image!.data!, "base64"), pixels);
packages/shared/src/toolOutput.test.ts
+8
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vite-plus/test";
3
import {
4
compactDynamicToolOutput,
5
MAX_TOOL_OUTPUT_IMAGES,
6
+ mcpToolResultOutput,
7
toolOutputImages,
8
toolOutputIndicatesFailure,
9
} from "./toolOutput.ts";
@@ -187,6 +188,13 @@ describe("toolOutputImages", () => {
188
]);
189
expect(toolOutputImages([mcpImage])).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
190
expect(toolOutputImages(mcpImage)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
191
+ const structuredContent = { screenshot: { mimeType: "image/png", width: 728, height: 354 } };
192
+ const content = [text, mcpImage];
193
+ const output = mcpToolResultOutput({ content, structuredContent });
194
+ expect(toolOutputImages(output)).toEqual([{ mimeType: "image/png", data: "AAAA" }]);
195
+ expect(output).toEqual({ content, structuredContent });
196
+ expect(mcpToolResultOutput({ content: [text], structuredContent })).toBe(structuredContent);
197
+ expect(mcpToolResultOutput({ content })).toBe(content);
198
});
199
200
it("skips types the server must not serve inline", () => {
packages/shared/src/toolOutput.ts
+10
@@ -274,6 +274,16 @@ export function toolOutputImages(value: unknown): ReadonlyArray<ToolOutputImage>
274
return toolOutputImageBlocks(value).flatMap((block) => readToolOutputImage(block) ?? []);
275
}
276
277
+export function mcpToolResultOutput(result: {
278
+ readonly content?: unknown;
279
+ readonly structuredContent?: unknown;
280
+}): unknown {
281
+ if (result.structuredContent != null && toolOutputImages(result.content).length > 0) {
282
+ return { content: result.content, structuredContent: result.structuredContent };
283
+ }
284
+ return result.structuredContent ?? result.content;
285
+}
286
+
287
export function omitToolOutputImageData(value: unknown): unknown {
288
const omit = (block: unknown) => {
289
const image = readToolOutputImage(block);