@devries-cameron20 / t3code Mirror · read-only / commits / 5b11efe5df
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 23:30 UTC

fix(clients): images the agent opens stay visible in the chat (#30473)

Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Federico Infelici committed Nov 11, 2026 at 06:35 UTC 5b11efe5df2f44741d2a89733ee49b00b753e5d1
9 files changed +332 -15
apps/mobile/src/features/threads/ThreadFeed.tsx
+14
@@ -286,6 +286,7 @@ const WIDE_MARKDOWN_BLOCK_OPTIONS = {
286
287 const TURN_FOLD_HEIGHT = 42;
288 const FEED_PADDING_TOP = 12;
289 +const VIEWED_IMAGE_GAP = 8;
290 const USER_BUBBLE_HORIZONTAL_PADDING = 3.5 * 3.5;
291 const ASSISTANT_ROW_HORIZONTAL_PADDING = 3.5;
292 // Virtualization remounts old rows; only newly created content should replay an entrance.
@@ -1722,6 +1723,19 @@ function renderFeedEntry(
1723 );
1724 }
1725
1726 + if (entry.type === "viewed-images") {
1727 + const thumbnailWidth = Math.floor((props.contentWidth - VIEWED_IMAGE_GAP) / 2);
1728 + return (
1729 + <View className="flex-row flex-wrap pb-2" style={{ gap: VIEWED_IMAGE_GAP }}>
1730 + {entry.paths.map((path) => (
1731 + <View key={path} style={{ width: thumbnailWidth }}>
1732 + {props.renderViewedImage({ href: path, alt: null, title: null })}
1733 + </View>
1734 + ))}
1735 + </View>
1736 + );
1737 + }
1738 +
1739 if (entry.type === "mcp-app") {
1740 return (
1741 <ThreadMcpApp
apps/mobile/src/lib/threadActivity.ts
+39 -5
@@ -27,6 +27,7 @@ import {
27 resolveWorkEntryToolPresentation,
28 summarizeToolGroup,
29 toolGroupSummaryKind,
30 + workEntriesViewedImagePaths,
31 type ToolGroupSummaryKind,
32 type WorkLogPresentationEntry,
33 type WorkLogToolLifecycleStatus,
@@ -224,6 +225,14 @@ type ThreadFeedEntryContent =
225 readonly shimmer: boolean;
226 readonly thought?: string;
227 }
228 + | {
229 + /** Images the agent opened in the tool group above, kept visible while the group is collapsed. */
230 + readonly type: "viewed-images";
231 + readonly id: string;
232 + readonly createdAt: string;
233 + readonly runId: RunId | null;
234 + readonly paths: ReadonlyArray<string>;
235 + }
236 | {
237 readonly type: "run-fold";
238 readonly deferredWorkItemIds?: ReadonlyArray<string>;
@@ -1252,7 +1261,10 @@ const supersededReasoningGroups = new WeakMap<ThreadFeedActivityGroup, ThreadFee
1261 const trailingReasoningGroups = new WeakMap<ThreadFeedActivityGroup, ThreadFeedActivityGroup>();
1262
1263 function settleSupersededReasoning(
1255 - entry: Exclude<ThreadFeedEntry, { readonly type: "run-fold" | "work-toggle" | "thinking" }>,
1264 + entry: Exclude<
1265 + ThreadFeedEntry,
1266 + { readonly type: "run-fold" | "work-toggle" | "thinking" | "viewed-images" }
1267 + >,
1268 tail: boolean,
1269 ) {
1270 if (entry.type !== "activity-group") return entry;
@@ -1527,7 +1539,10 @@ export function deriveThreadFeedPresentation(
1539 const lastEntryByRun = new Map<RunId, number>();
1540
1541 const retainedFeed: Array<
1530 - Exclude<ThreadFeedEntry, { readonly type: "run-fold" | "work-toggle" | "thinking" }>
1542 + Exclude<
1543 + ThreadFeedEntry,
1544 + { readonly type: "run-fold" | "work-toggle" | "thinking" | "viewed-images" }
1545 + >
1546 > = [];
1547 const retainedFeedIndices: number[] = [];
1548 const { toolCallVisibility, revealedMessageId } = display;
@@ -1540,7 +1555,12 @@ export function deriveThreadFeedPresentation(
1555 : null;
1556 for (let index = start; index < feed.length; index += 1) {
1557 let entry = feed[index]!;
1543 - if (entry.type === "run-fold" || entry.type === "work-toggle" || entry.type === "thinking")
1558 + if (
1559 + entry.type === "run-fold" ||
1560 + entry.type === "work-toggle" ||
1561 + entry.type === "thinking" ||
1562 + entry.type === "viewed-images"
1563 + )
1564 continue;
1565 const runId = feedEntryRunId(entry);
1566 if (runId !== null) {
@@ -1819,7 +1839,10 @@ function thinkingRow(createdAt: string, runId: RunId | null, label: string | nul
1839
1840 function appendPresentedFeedEntry(
1841 result: ThreadFeedEntry[],
1822 - entry: Exclude<ThreadFeedEntry, { readonly type: "run-fold" | "work-toggle" | "thinking" }>,
1842 + entry: Exclude<
1843 + ThreadFeedEntry,
1844 + { readonly type: "run-fold" | "work-toggle" | "thinking" | "viewed-images" }
1845 + >,
1846 expandedWorkGroupIds: ReadonlySet<string>,
1847 activeRunId: RunId | null,
1848 isWorking: boolean,
@@ -1890,16 +1913,27 @@ function appendActivityGroupRows(
1913 let groupableRun: ThreadFeedActivity[] = [];
1914 const flushGroupableRun = (isTrailingRun: boolean) => {
1915 if (groupableRun.length === 0) return;
1916 + const groupId = `work-group:${groupAnchorIdByActivityId.get(groupableRun[0]!.id) ?? groupableRun[0]!.id}`;
1917 appendToolGroupRows(
1918 result,
1919 entry,
1920 groupableRun,
1897 - `work-group:${groupAnchorIdByActivityId.get(groupableRun[0]!.id) ?? groupableRun[0]!.id}`,
1921 + groupId,
1922 expandedWorkGroupIds,
1923 activeRunId,
1924 isWorking,
1925 activeTail && isTrailingRun,
1926 );
1927 + const paths = workEntriesViewedImagePaths(groupableRun.map((activity) => activity.workEntry));
1928 + if (paths.length > 0) {
1929 + result.push({
1930 + type: "viewed-images",
1931 + id: `${groupId}:viewed-images`,
1932 + createdAt: groupableRun[0]!.createdAt,
1933 + runId: groupableRun[0]!.runId,
1934 + paths,
1935 + });
1936 + }
1937 groupableRun = [];
1938 };
1939 for (const activity of activities) {
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+68
@@ -1375,6 +1375,74 @@ describe("CodexAdapterV2 post-settle continuation", () => {
1375 };
1376 });
1377
1378 + it.effect("keeps native image views available to the clients", () =>
1379 + Effect.gen(function* () {
1380 + const nativeThreadId = "image-view-thread";
1381 + const nativeTurnId = "image-view-turn";
1382 + const imagePath = "file:///workspace/reference.png";
1383 + const harness = yield* makeCodexReplayHarness(
1384 + makeCodexReplayTranscript({
1385 + scenario: "image-view",
1386 + entries: [
1387 + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Show the reference." }),
1388 + ...(["started", "completed"] as const).map((phase) => ({
1389 + type: "emit_inbound" as const,
1390 + label: `image ${phase}`,
1391 + frame: {
1392 + method: `item/${phase}`,
1393 + params: {
1394 + threadId: nativeThreadId,
1395 + turnId: nativeTurnId,
1396 + item: { type: "imageView", id: "image-view", path: imagePath },
1397 + ...(phase === "started" ? { startedAtMs: 1 } : { completedAtMs: 2 }),
1398 + },
1399 + },
1400 + })),
1401 + {
1402 + type: "emit_inbound",
1403 + label: "turn completed",
1404 + frame: {
1405 + method: "turn/completed",
1406 + params: {
1407 + threadId: nativeThreadId,
1408 + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }),
1409 + },
1410 + },
1411 + },
1412 + ],
1413 + }),
1414 + );
1415 + yield* harness.runtime.startTurn(
1416 + makeCodexTestTurnInput({
1417 + threadId: harness.threadId,
1418 + providerThread: harness.providerThread,
1419 + now: yield* DateTime.now,
1420 + attemptId: RunAttemptId.make("image-view-attempt"),
1421 + text: "Show the reference.",
1422 + }),
1423 + );
1424 + yield* harness.firstTerminal;
1425 + const images = harness.events.flatMap((event) =>
1426 + event.type === "turn_item.updated" && event.turnItem.type === "dynamic_tool"
1427 + ? [event.turnItem]
1428 + : [],
1429 + );
1430 + assert.lengthOf(images, 1);
1431 + assert.equal(images[0]?.viewedImagePath, imagePath);
1432 + assert.equal(images[0]?.status, "completed");
1433 + }).pipe(
1434 + Effect.scoped,
1435 + Effect.provide(
1436 + Layer.mergeAll(
1437 + IdAllocator.layer,
1438 + McpProviderSessions.layer,
1439 + NodeServices.layer,
1440 + NodeCrypto.layer,
1441 + ),
1442 + ),
1443 + ),
1444 + );
1445 +
1446 it.effect.each(["before", "after"] as const)(
1447 "rejects a previous run's native turn when it completes %s the new start response",
1448 (completionOrder) =>
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+21
@@ -5357,6 +5357,27 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5357 return;
5358 }
5359
5360 + if (payload.item.type === "imageView") {
5361 + const artifacts = yield* buildDynamicToolArtifacts(context, {
5362 + type: "dynamicToolCall",
5363 + id: payload.item.id,
5364 + tool: "view_image",
5365 + arguments: { path: payload.item.path },
5366 + status: "completed",
5367 + });
5368 + yield* emitProviderEvent({
5369 + type: "node.updated",
5370 + driver: CODEX_PROVIDER,
5371 + node: artifacts.node,
5372 + });
5373 + yield* emitProviderEvent({
5374 + type: "turn_item.updated",
5375 + driver: CODEX_PROVIDER,
5376 + turnItem: { ...artifacts.turnItem, viewedImagePath: payload.item.path },
5377 + });
5378 + return;
5379 + }
5380 +
5381 if (payload.item.type !== "agentMessage") {
5382 return;
5383 }
apps/web/src/components/chat/MessagesTimeline.logic.test.ts
+99
@@ -2210,6 +2210,105 @@ describe("deriveMessagesTimelineRows", () => {
2210 ]);
2211 });
2212
2213 + it("shows images the agent viewed outside their collapsed tool group", () => {
2214 + const threadId = ThreadId.make("viewed-images-thread");
2215 + const runId = RunId.make("viewed-images-run");
2216 + const at = (second: number) =>
2217 + DateTime.makeUnsafe(new Date(Date.UTC(2026, 10, 4, 12, 0, second)).toISOString());
2218 + const base = (id: string, ordinal: number) => ({
2219 + id: TurnItemId.make(id),
2220 + threadId,
2221 + runId,
2222 + nodeId: null,
2223 + providerThreadId: null,
2224 + providerTurnId: null,
2225 + nativeItemRef: null,
2226 + parentItemId: null,
2227 + ordinal,
2228 + status: "completed" as const,
2229 + title: null,
2230 + startedAt: at(ordinal),
2231 + completedAt: at(ordinal),
2232 + updatedAt: at(ordinal),
2233 + });
2234 + const read = (id: string, ordinal: number, filePath: string) => ({
2235 + ...base(id, ordinal),
2236 + type: "dynamic_tool" as const,
2237 + toolName: "Read",
2238 + // The Claude adapter tags reads of image files.
2239 + ...(filePath.endsWith(".png") ? { viewedImagePath: filePath } : {}),
2240 + input: { file_path: filePath },
2241 + });
2242 + const items = (settled: boolean) =>
2243 + [
2244 + {
2245 + ...base("prompt", 0),
2246 + type: "user_message" as const,
2247 + messageId: MessageId.make("prompt"),
2248 + text: "Show me the mockups",
2249 + attachments: [],
2250 + inputIntent: "turn_start" as const,
2251 + createdBy: "user" as const,
2252 + creationSource: "web" as const,
2253 + },
2254 + read("read-sidebar", 1, "/repo/docs/sidebar.png"),
2255 + read("read-notes", 2, "/repo/docs/notes.md"),
2256 + read("read-sidebar-again", 3, "/repo/docs/sidebar.png"),
2257 + read("read-topbar", 4, "/repo/docs/topbar.png"),
2258 + settled
2259 + ? {
2260 + ...base("answer", 5),
2261 + type: "assistant_message" as const,
2262 + messageId: MessageId.make("answer"),
2263 + text: "Here are both layouts",
2264 + streaming: false,
2265 + }
2266 + : {
2267 + ...base("ask-again", 5),
2268 + status: "waiting" as const,
2269 + completedAt: null,
2270 + type: "user_input_request" as const,
2271 + requestId: RuntimeRequestId.make("layout-question"),
2272 + questions: [],
2273 + },
2274 + ].map((item, position) => ({
2275 + position,
2276 + visibility: "local" as const,
2277 + sourceThreadId: threadId,
2278 + sourceItemId: item.id,
2279 + item,
2280 + }));
2281 + const viewedImages = (settled: boolean, expandedRunIds?: ReadonlySet<RunId>) => {
2282 + const rows = deriveMessagesTimelineRows({
2283 + timelineEntries: deriveTimelineEntriesFromVisibleTurnItems({
2284 + visibleTurnItems: items(settled),
2285 + optimisticMessages: [],
2286 + }),
2287 + latestRun: {
2288 + runId,
2289 + status: settled ? "completed" : "running",
2290 + startedAt: DateTime.formatIso(at(0)),
2291 + completedAt: settled ? DateTime.formatIso(at(5)) : null,
2292 + },
2293 + runningRunId: settled ? null : runId,
2294 + isWorking: !settled,
2295 + activeTurnStartedAt: settled ? null : DateTime.formatIso(at(0)),
2296 + ...(expandedRunIds === undefined ? {} : { expandedRunIds }),
2297 + turnDiffSummaries: [],
2298 + supportsConversationRollback: false,
2299 + });
2300 + return rows.flatMap((row, index) =>
2301 + row.kind === "viewed-images" ? [{ after: rows[index - 1]?.kind, paths: row.paths }] : [],
2302 + );
2303 + };
2304 + const paths = ["/repo/docs/sidebar.png", "/repo/docs/topbar.png"];
2305 +
2306 + expect(viewedImages(false)).toEqual([{ after: "work-live", paths }]);
2307 + // A settled turn's fold keeps hiding its work, images included, until it is opened.
2308 + expect(viewedImages(true)).toEqual([]);
2309 + expect(viewedImages(true, new Set([runId]))).toEqual([{ after: "work-toggle", paths }]);
2310 + });
2311 +
2312 it("keeps imported V1 turns folded once the thread's first V2 run starts", () => {
2313 const at = (second: number) => `2026-01-01T00:00:${String(second).padStart(2, "0")}Z`;
2314 const message = (
apps/web/src/components/chat/MessagesTimeline.logic.ts
+48 -9
@@ -27,6 +27,7 @@ import {
27 toolGroupAction,
28 toolGroupSummaryKind,
29 contextCompactionTokenCounts,
30 + workEntriesViewedImagePaths,
31 type ToolGroupSummaryKind,
32 liveThoughtLine,
33 workEntryHiddenByToolCallVisibility,
@@ -549,6 +550,13 @@ type MessagesTimelineRowContent =
550 toolIcon?: WorkLogEntry["toolIcon"];
551 hasFailure: boolean;
552 }
553 + | {
554 + /** Images the agent opened in the tool group above, kept visible while the group is collapsed. */
555 + kind: "viewed-images";
556 + id: string;
557 + createdAt: string;
558 + paths: ReadonlyArray<string>;
559 + }
560 | {
561 kind: "turn-fold";
562 deferredWorkItemIds?: ReadonlyArray<string>;
@@ -662,6 +670,17 @@ function expandedWorkGroupRow(
670 };
671 }
672
673 +function viewedImagesRows(
674 + groupId: string,
675 + createdAt: string,
676 + groupedEntries: ReadonlyArray<WorkLogEntry>,
677 +): Array<Extract<MessagesTimelineRow, { kind: "viewed-images" }>> {
678 + const paths = workEntriesViewedImagePaths(groupedEntries);
679 + return paths.length === 0
680 + ? []
681 + : [{ kind: "viewed-images", id: `${groupId}:viewed-images`, createdAt, paths }];
682 +}
683 +
684 export function resolveAssistantMessageCopyState({
685 text,
686 showCopyButton,
@@ -1792,9 +1811,17 @@ export function deriveMessagesTimelineRows(input: {
1811 if (activeWorkRow === null) return;
1812 nextRows.push(activeWorkRow);
1813 hasActivityRow ||= activeWorkRow.active;
1795 - if (!activeWorkRow.expanded) return;
1814 + if (activeWorkRow.expanded) {
1815 + nextRows.push(
1816 + expandedWorkGroupRow(
1817 + activeWorkRow.groupId,
1818 + activeWorkRow.createdAt,
1819 + activeWorkRow.groupedEntries,
1820 + ),
1821 + );
1822 + }
1823 nextRows.push(
1797 - expandedWorkGroupRow(
1824 + ...viewedImagesRows(
1825 activeWorkRow.groupId,
1826 activeWorkRow.createdAt,
1827 activeWorkRow.groupedEntries,
@@ -2058,6 +2085,13 @@ export function deriveMessagesTimelineRows(input: {
2085 );
2086 }
2087 }
2088 + nextRows.push(
2089 + ...viewedImagesRows(
2090 + workGroupId(timelineEntry.id),
2091 + timelineEntry.createdAt,
2092 + visibleGroupedEntries,
2093 + ),
2094 + );
2095 }
2096 index = cursor - 1;
2097 continue;
@@ -2241,6 +2275,7 @@ export function deriveMessagesTimelineRows(input: {
2275 if (failedGroupAnchor) {
2276 const groupId = workGroupId(failedGroupAnchor.id);
2277 const expanded = input.expandedWorkGroupIds?.has(groupId) ?? false;
2278 + const groupedEntries = visibleActiveToolEntries.map((entry) => entry.entry);
2279 nextRows.push({
2280 kind: "thinking",
2281 id: LIVE_ACTIVITY_ROW_ID,
@@ -2249,14 +2284,9 @@ export function deriveMessagesTimelineRows(input: {
2284 expanded,
2285 });
2286 if (expanded) {
2252 - nextRows.push(
2253 - expandedWorkGroupRow(
2254 - groupId,
2255 - failedGroupAnchor.createdAt,
2256 - visibleActiveToolEntries.map((entry) => entry.entry),
2257 - ),
2258 - );
2287 + nextRows.push(expandedWorkGroupRow(groupId, failedGroupAnchor.createdAt, groupedEntries));
2288 }
2289 + nextRows.push(...viewedImagesRows(groupId, failedGroupAnchor.createdAt, groupedEntries));
2290 } else {
2291 nextRows.push({
2292 kind: "thinking",
@@ -2573,6 +2603,15 @@ function isRowUnchanged(a: MessagesTimelineRow, b: MessagesTimelineRow): boolean
2603 case "proposed-plan":
2604 return a.proposedPlan === (b as typeof a).proposedPlan;
2605
2606 + case "viewed-images": {
2607 + const bv = b as typeof a;
2608 + return (
2609 + a.createdAt === bv.createdAt &&
2610 + a.paths.length === bv.paths.length &&
2611 + a.paths.every((path, index) => path === bv.paths[index])
2612 + );
2613 + }
2614 +
2615 case "html-render": {
2616 // Tool updates rebuild entries; preserve frames for unchanged pages.
2617 const bh = b as typeof a;
apps/web/src/components/chat/MessagesTimeline.tsx
+4 -1
@@ -117,6 +117,7 @@ import {
117 LiveWorkEntryTimelineRow,
118 WorkGroupToggleTimelineRow,
119 } from "./TimelineWorkGroupRows";
120 +import { ViewedImagesTimelineRow } from "./TimelineWorkEntryRow";
121 import { UserMessageRow } from "./TimelineUserMessageRow";
122 import {
123 AssistantTimelineRow,
@@ -1453,7 +1454,8 @@ const TimelineRowContent = memo(function TimelineRowContent({ row }: { row: Time
1454 row.kind === "event" ||
1455 row.kind === "attempt-fold" ||
1456 row.kind === "html-render" ||
1456 - row.kind === "mcp-app"
1457 + row.kind === "mcp-app" ||
1458 + row.kind === "viewed-images"
1459 ? "pb-2"
1460 : "pb-4",
1461 (row.kind === "message" && row.message.role === "assistant") ||
@@ -1502,6 +1504,7 @@ const TimelineRowContent = memo(function TimelineRowContent({ row }: { row: Time
1504 {row.kind === "assistant-meta" ? <AssistantMetaTimelineRow row={row} /> : null}
1505 {row.kind === "proposed-plan" ? <ProposedPlanTimelineRow row={row} /> : null}
1506 {row.kind === "html-render" ? <HtmlRenderTimelineRow row={row} /> : null}
1507 + {row.kind === "viewed-images" ? <ViewedImagesTimelineRow row={row} /> : null}
1508 {row.kind === "mcp-app" ? <McpAppTimelineRow row={row} /> : null}
1509 {row.kind === "working" ? <WorkingTimelineRow row={row} /> : null}
1510 {row.kind === "worktree-setup" ? <WorktreeSetupTimelineRow row={row} /> : null}
apps/web/src/components/chat/TimelineWorkEntryRow.tsx
+32
@@ -51,6 +51,7 @@ import {
51 TimelineRowCtx,
52 TimelineRowActivityCtx,
53 WorkGroupViewCtx,
54 + type TimelineRow,
55 type TimelineWorkEntry,
56 } from "./timelineRowContext";
57 import { MESSAGE_HEADING_LEVEL, TimelineRowTimestamp } from "./TimelineRowParts";
@@ -171,6 +172,37 @@ export const SimpleWorkEntryRow = memo(function SimpleWorkEntryRow(props: WorkEn
172 );
173 });
174
175 +export function ViewedImagesTimelineRow({
176 + row,
177 +}: {
178 + row: Extract<TimelineRow, { kind: "viewed-images" }>;
179 +}) {
180 + const { threadRef, workspaceRoot, onImageExpand } = use(TimelineRowCtx);
181 + if (!threadRef) return null;
182 + return (
183 + <div className="flex flex-wrap items-start gap-2 px-0.5">
184 + {row.paths.map((path) => {
185 + const image = resolveViewedImageAsset(path, {
186 + threadId: threadRef.threadId,
187 + workspaceRoot,
188 + });
189 + return image ? (
190 + <ChatMarkdownAssetImage
191 + key={path}
192 + environmentId={threadRef.environmentId}
193 + resource={image.resource}
194 + alt={image.alt}
195 + srcFragment={image.srcFragment}
196 + workspaceRoot={workspaceRoot}
197 + maxHeightRem={10}
198 + onImageExpand={onImageExpand}
199 + />
200 + ) : null;
201 + })}
202 + </div>
203 + );
204 +}
205 +
206 function WorkEntryLogRow(props: WorkEntryRowProps) {
207 const { workEntry, workspaceRoot, displayLabel } = props;
208 const ctx = use(TimelineRowCtx);
packages/client-runtime/src/work-log/presentation.ts
+7
@@ -414,6 +414,13 @@ export function workEntryViewedImagePath(entry: WorkLogPresentationEntry): strin
414 : null;
415 }
416
417 +/** Distinct images the agent opened across a tool group, in viewing order. */
418 +export function workEntriesViewedImagePaths(
419 + entries: ReadonlyArray<WorkLogPresentationEntry>,
420 +): string[] {
421 + return [...new Set(entries.flatMap((entry) => workEntryViewedImagePath(entry) ?? []))];
422 +}
423 +
424 export interface ViewedImageAsset {
425 readonly resource: Extract<AssetResource, { readonly _tag: "media-file" }>;
426 readonly alt: string;