@devries-cameron20 / t3code Mirror · read-only / commits / 50d3a31ffd
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 13:15 UTC

fix: guests see the screenshots the agent shows in its replies (#26323)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 27, 2026 at 23:52 UTC 50d3a31ffd8a24965ebf72bcc77fd28408be2739
15 files changed +644 -79
apps/server/src/guest/GuestShareService.test.ts
+271
@@ -5,22 +5,26 @@ import {
5 CommandId,
6 EventId,
7 GuestShareId,
8 + type GuestStreamEvent,
9 MessageId,
10 type OrchestrationV2TurnItem,
11 ProjectId,
12 ProviderDriverKind,
13 ProviderInstanceId,
14 + RunId,
15 ThreadId,
16 TurnItemId,
17 } from "@t3tools/contracts";
18 import * as Context from "effect/Context";
19 import * as DateTime from "effect/DateTime";
20 +import * as Deferred from "effect/Deferred";
21 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
22 import * as Effect from "effect/Effect";
23 import * as Exit from "effect/Exit";
24 import * as FileSystem from "effect/FileSystem";
25 import * as Fiber from "effect/Fiber";
26 import * as Layer from "effect/Layer";
27 +import * as Queue from "effect/Queue";
28 import * as Ref from "effect/Ref";
29 import * as Stream from "effect/Stream";
30 import * as TestClock from "effect/testing/TestClock";
@@ -842,3 +846,270 @@ it.effect("serves the shared thread's files to its guest, never another thread's
846 assert.isDefined(shareId);
847 }).pipe(Effect.scoped, Effect.provide(layerTest)),
848 );
849 +
850 +it.effect("serves an image on the host only when an agent reply the guest sees shows it", () =>
851 + Effect.gen(function* () {
852 + const { orchestrator, guestShares } = yield* shareThread;
853 + const runId = yield* ownerRun;
854 + const fileSystem = yield* FileSystem.FileSystem;
855 + const directory = yield* fileSystem.makeTempDirectoryScoped();
856 + const shot = `${directory}/my shot.png`;
857 + const fileUrl = `${directory}/file-url.png`;
858 + const page = `${directory}/page.png`;
859 + const fromUser = `${directory}/from-user.png`;
860 + const unnamed = `${directory}/unnamed.png`;
861 + const later = `${directory}/later.png`;
862 + for (const path of [shot, fileUrl, fromUser, unnamed, later]) {
863 + yield* fileSystem.writeFile(path, PNG);
864 + }
865 + yield* fileSystem.writeFileString(page, "<script>alert(1)</script>");
866 + const eventSink = yield* EventSink.EventSinkV2;
867 + const now = yield* DateTime.now;
868 + const run = (yield* orchestrator.getThreadRecords(threadId, ["runs"])).runs[0];
869 + const item = (
870 + id: string,
871 + ordinal: number,
872 + fields:
873 + | { readonly type: "assistant_message"; readonly text: string }
874 + | { readonly type: "user_message"; readonly text: string },
875 + target: { readonly threadId: ThreadId; readonly runId: RunId } = { threadId, runId },
876 + ) => ({
877 + id: EventId.make(`event:${id}`),
878 + type: "turn-item.updated" as const,
879 + threadId: target.threadId,
880 + driver: ProviderDriverKind.make("codex"),
881 + providerInstanceId: instanceId,
882 + occurredAt: now,
883 + payload: {
884 + id: TurnItemId.make(id),
885 + threadId: target.threadId,
886 + runId: target.runId,
887 + nodeId: null,
888 + providerThreadId: null,
889 + providerTurnId: null,
890 + nativeItemRef: null,
891 + parentItemId: null,
892 + ordinal,
893 + title: null,
894 + status: "completed" as const,
895 + startedAt: now,
896 + completedAt: now,
897 + updatedAt: now,
898 + messageId: MessageId.make(id),
899 + ...(fields.type === "assistant_message"
900 + ? { ...fields, streaming: false }
901 + : {
902 + ...fields,
903 + createdBy: "user" as const,
904 + creationSource: "web" as const,
905 + inputIntent: "turn_start" as const,
906 + attachments: [],
907 + }),
908 + },
909 + });
910 + yield* eventSink.write({
911 + events: [
912 + item("item:agent-images", 1, {
913 + type: "assistant_message",
914 + text: `Done: ![Shot](<${shot}>) ![Other](file://${encodeURI(fileUrl)}) ![Page](${page})`,
915 + }),
916 + // A guest's sent message is a user message; it must not open files on the host.
917 + item("item:guest-image", 2, { type: "user_message", text: `![Mine](${fromUser})` }),
918 + {
919 + id: EventId.make("event:run-done"),
920 + type: "run.updated" as const,
921 + threadId,
922 + runId,
923 + providerInstanceId: instanceId,
924 + occurredAt: now,
925 + payload: { ...run!, status: "completed" as const, completedAt: now },
926 + },
927 + ],
928 + });
929 + // The guest watches a fork, which shows these replies as inherited history.
930 + const forkId = ThreadId.make("thread:guest-share-fork");
931 + yield* orchestrator.dispatch({
932 + type: "thread.fork",
933 + commandId: CommandId.make("fork"),
934 + sourceThreadId: threadId,
935 + targetThreadId: forkId,
936 + sourcePoint: { type: "run", runId },
937 + createdBy: "user",
938 + creationSource: "web",
939 + });
940 + const { url } = yield* guestShares.create({ threadId: forkId });
941 + const { token: session } = yield* guestShares.redeem({
942 + credential: url.slice(url.indexOf("#") + 1),
943 + name: "Sam",
944 + });
945 +
946 + // The guest's page opens, and its stream sends the replies the fork inherits.
947 + const sent = yield* Queue.unbounded<GuestStreamEvent>();
948 + yield* Stream.runForEach(yield* guestShares.stream(session), (event) =>
949 + Queue.offer(sent, event),
950 + ).pipe(Effect.forkScoped);
951 + const sentItem = Effect.fn(function* (id: string) {
952 + for (;;) {
953 + const event = yield* Queue.take(sent);
954 + if (
955 + event.type === "thread" &&
956 + event.item.kind === "event" &&
957 + event.item.event.type === "turn-item.updated" &&
958 + event.item.event.payload.id === id
959 + ) {
960 + return;
961 + }
962 + }
963 + });
964 +
965 + // The agent shows another screenshot live, then enough rows to push the first reply out of
966 + // the window, before the page asks for any image.
967 + yield* orchestrator.dispatch({
968 + type: "message.dispatch",
969 + commandId: CommandId.make("fork-message"),
970 + threadId: forkId,
971 + messageId: MessageId.make("message:fork"),
972 + text: "One more",
973 + attachments: [],
974 + dispatchMode: { type: "defer_start" },
975 + createdBy: "user",
976 + creationSource: "web",
977 + });
978 + const forkRun = (yield* orchestrator.getThreadRecords(forkId, ["runs"])).runs.at(-1)!;
979 + const inFork = { threadId: forkId, runId: forkRun.id };
980 + yield* eventSink.write({
981 + events: [
982 + item(
983 + "item:later-image",
984 + 1,
985 + { type: "assistant_message", text: `![Later](${later})` },
986 + inFork,
987 + ),
988 + ...Array.from({ length: 600 }, (_, index) =>
989 + item(
990 + `item:filler-${index}`,
991 + index + 2,
992 + { type: "assistant_message", text: "Working." },
993 + inFork,
994 + ),
995 + ),
996 + ],
997 + });
998 + yield* sentItem("item:later-image");
999 +
1000 + // The page still shows every reply it was sent, so each image loads.
1001 + for (const path of [shot, fileUrl, later]) {
1002 + assert.deepStrictEqual(yield* guestShares.media(session, path), {
1003 + path,
1004 + mimeType: "image/png",
1005 + });
1006 + }
1007 + for (const path of [page, fromUser, unnamed, `${directory}/missing.png`]) {
1008 + assert.equal((yield* Effect.flip(guestShares.media(session, path))).reason, "unavailable");
1009 + }
1010 +
1011 + // A reply edited to drop its image stops allowing it.
1012 + yield* eventSink.write({
1013 + events: [
1014 + {
1015 + ...item("item:later-image", 1, { type: "assistant_message", text: "Gone." }, inFork),
1016 + id: EventId.make("event:later-image-edited"),
1017 + },
1018 + ],
1019 + });
1020 + yield* sentItem("item:later-image");
1021 + assert.equal((yield* Effect.flip(guestShares.media(session, later))).reason, "unavailable");
1022 + }).pipe(Effect.scoped, Effect.provide(layerTest)),
1023 +);
1024 +
1025 +it.effect("keeps a connected guest's images while other shares come and go", () =>
1026 + Effect.gen(function* () {
1027 + const fileSystem = yield* FileSystem.FileSystem;
1028 + const directory = yield* fileSystem.makeTempDirectoryScoped();
1029 + const shot = `${directory}/shot.png`;
1030 + yield* fileSystem.writeFile(shot, PNG);
1031 + const eventSink = yield* EventSink.EventSinkV2;
1032 + const now = yield* DateTime.now;
1033 + /** A shared thread whose agent reply shows the screenshot, and its guest's session. */
1034 + const sharedWithImage = Effect.fn(function* (id: ThreadId) {
1035 + const { orchestrator, guestShares, credential } = yield* shareThreadFor(id);
1036 + yield* orchestrator.dispatch({
1037 + type: "message.dispatch",
1038 + commandId: CommandId.make(`message:${id}`),
1039 + threadId: id,
1040 + messageId: MessageId.make(`message:${id}`),
1041 + text: "Show me",
1042 + attachments: [],
1043 + dispatchMode: { type: "defer_start" },
1044 + createdBy: "user",
1045 + creationSource: "web",
1046 + });
1047 + const runId = (yield* orchestrator.getThreadRecords(id, ["runs"])).runs[0]!.id;
1048 + yield* eventSink.write({
1049 + events: [
1050 + {
1051 + id: EventId.make(`event:reply:${id}`),
1052 + type: "turn-item.updated",
1053 + threadId: id,
1054 + driver: ProviderDriverKind.make("codex"),
1055 + providerInstanceId: instanceId,
1056 + occurredAt: now,
1057 + payload: {
1058 + id: TurnItemId.make(`item:reply:${id}`),
1059 + threadId: id,
1060 + runId,
1061 + nodeId: null,
1062 + providerThreadId: null,
1063 + providerTurnId: null,
1064 + nativeItemRef: null,
1065 + parentItemId: null,
1066 + ordinal: 1,
1067 + title: null,
1068 + status: "completed",
1069 + startedAt: now,
1070 + completedAt: now,
1071 + updatedAt: now,
1072 + type: "assistant_message",
1073 + messageId: MessageId.make(`item:reply:${id}`),
1074 + text: `![Shot](${shot})`,
1075 + streaming: false,
1076 + },
1077 + },
1078 + ],
1079 + });
1080 + const { token } = yield* guestShares.redeem({ credential, name: "Sam" });
1081 + return { guestShares, token };
1082 + });
1083 +
1084 + // This guest's page stays open while more guests than the record keeps open and close theirs.
1085 + const { guestShares, token } = yield* sharedWithImage(threadId);
1086 + const opened = yield* Deferred.make<void>();
1087 + yield* Stream.runForEach(yield* guestShares.stream(token), () =>
1088 + Deferred.succeed(opened, undefined),
1089 + ).pipe(Effect.forkScoped);
1090 + yield* Deferred.await(opened);
1091 + const closedTokens: Array<string> = [];
1092 + for (let index = 0; index < 65; index++) {
1093 + const other = yield* sharedWithImage(ThreadId.make(`thread:other-${index}`));
1094 + const events = yield* guestShares.stream(other.token);
1095 + // The first is built but never read, as for a HEAD request; the rest open and close.
1096 + if (index > 0) yield* Stream.runDrain(Stream.take(events, 2));
1097 + closedTokens.push(other.token);
1098 + }
1099 + assert.equal((yield* guestShares.media(token, shot)).mimeType, "image/png");
1100 + // Records of guests with no open page were dropped to make room.
1101 + for (const closed of closedTokens.slice(0, 2)) {
1102 + assert.equal((yield* Effect.flip(guestShares.media(closed, shot))).reason, "unavailable");
1103 + }
1104 +
1105 + // A new link on the thread starts empty: its guest gets nothing the last guest was sent.
1106 + const { url } = yield* guestShares.create({ threadId });
1107 + const next = yield* guestShares.redeem({
1108 + credential: url.slice(url.indexOf("#") + 1),
1109 + name: "Kim",
1110 + });
1111 + assert.equal((yield* Effect.flip(guestShares.media(next.token, shot))).reason, "unavailable");
1112 + yield* Stream.runDrain(Stream.take(yield* guestShares.stream(next.token), 2));
1113 + assert.equal((yield* guestShares.media(next.token, shot)).mimeType, "image/png");
1114 + }).pipe(Effect.scoped, Effect.provide(layerTest)),
1115 +);
apps/server/src/guest/GuestShareService.ts
+106 -8
@@ -15,9 +15,11 @@ import {
15 type OrchestrationV2CommandOutputFrame,
16 type OrchestrationV2DomainEvent,
17 type OrchestrationV2GetTurnItemResult,
18 + type OrchestrationV2TurnItem,
19 ThreadId,
20 TurnItemId,
21 } from "@t3tools/contracts";
22 +import { chatMarkdownImageSources } from "@t3tools/shared/markdownPipeline";
23 import { htmlRenderFromToolItem } from "@t3tools/shared/toolOutput";
24 import * as Context from "effect/Context";
25 import * as Crypto from "effect/Crypto";
@@ -63,7 +65,10 @@ import { guestShareView, guestThreadModel } from "./guestView.ts";
65 import {
66 classifyGuestUploads,
67 GUEST_UPLOAD_HEADER_BYTES,
68 + guestHostImageType,
69 type GuestUpload,
70 + matchesImageSignature,
71 + repliesShowImage,
72 } from "./guestAttachments.ts";
73
74 const SIGNING_SECRET_NAME = "guest-share-signing-key";
@@ -71,6 +76,8 @@ const SHARE_ID_BYTES = 16;
76 const INVITE_SIGNATURE_BYTES = 16;
77 const SNAPSHOT_ROW_LIMIT = 600;
78 const SNAPSHOT_USER_TURN_LIMIT = 20;
79 +// Shares with no open guest stream whose sent image replies are still kept.
80 +const SHARED_THREAD_REPLIES_LIMIT = 64;
81 // How soon an open read notices that its link stopped working, such as Connect no longer serving
82 // the host. Closing or replacing the link ends the main stream at once.
83 const ACCESS_RECHECK_INTERVAL = Duration.seconds(30);
@@ -190,9 +197,20 @@ export class GuestShareService extends Context.Service<
197 { readonly path: string; readonly attachment: ChatAttachment },
198 GuestShareAccessError
199 >;
200 + /** An image on the host that an agent reply the guest sees shows, and its type. */
201 + readonly media: (
202 + session: string,
203 + path: string,
204 + ) => Effect.Effect<{ readonly path: string; readonly mimeType: string }, GuestShareAccessError>;
205 }
206 >()("t3/guest/GuestShareService") {}
207
208 +/** A reply a guest's stream sent, with its image sources once a media request parsed them. */
209 +interface SentReply {
210 + readonly text: string;
211 + sources?: ReadonlyArray<string>;
212 +}
213 +
214 /** The thread record an event carries, for the share and the view. */
215 const threadOf = (event: OrchestrationV2DomainEvent): OrchestrationV2AppThread | null => {
216 switch (event.type) {
@@ -464,6 +482,40 @@ const make = Effect.gen(function* () {
482 Effect.ignore,
483 );
484
485 + // The image replies each share's guest stream has sent, in its opening snapshot or live, so an
486 + // image loads while the guest's page can show its reply, inherited ones included. Keyed by
487 + // share, so a new link on the thread starts empty. A share with an open stream keeps its
488 + // record; past the limit, closed ones are dropped.
489 + const sentImageReplies = new Map<
490 + GuestShareId,
491 + { openStreams: number; readonly replies: Map<string, SentReply> }
492 + >();
493 + const sentRecord = (shareId: GuestShareId) => {
494 + let record = sentImageReplies.get(shareId);
495 + if (record === undefined) {
496 + record = { openStreams: 0, replies: new Map() };
497 + sentImageReplies.set(shareId, record);
498 + for (const [id, other] of sentImageReplies) {
499 + if (sentImageReplies.size <= SHARED_THREAD_REPLIES_LIMIT) break;
500 + if (other.openStreams === 0 && id !== shareId) sentImageReplies.delete(id);
501 + }
502 + }
503 + return record;
504 + };
505 + const rememberSent = (shareId: GuestShareId, items: Iterable<OrchestrationV2TurnItem>) => {
506 + for (const item of items) {
507 + if (item.type !== "assistant_message") continue;
508 + const key = `${item.threadId}\u0000${item.id}`;
509 + // Only replies with image syntax can show an image. One edited to drop its images stops
510 + // allowing them.
511 + if (item.text.includes("![") || /<img/i.test(item.text)) {
512 + sentRecord(shareId).replies.set(key, { text: item.text });
513 + } else {
514 + sentImageReplies.get(shareId)?.replies.delete(key);
515 + }
516 + }
517 + };
518 +
519 /**
520 * The thread as the owner's thread subscription sends it: a bounded snapshot, then the same
521 * wire events, batched the same way. Only the share's own state is added.
@@ -511,6 +563,7 @@ const make = Effect.gen(function* () {
563 Stream.map((item): ReadonlyArray<GuestStreamEvent> => {
564 if (item.kind !== "event") return [];
565 if (endsShare(item.event)) return [ended];
566 + if (item.event.type === "turn-item.updated") rememberSent(share.id, [item.event.payload]);
567 const updated = threadOf(item.event);
568 const view = updated === null ? null : viewOf(updated);
569 return view === null
@@ -528,17 +581,15 @@ const make = Effect.gen(function* () {
581 const untilExpiry = Effect.sleep(
582 Math.max(0, Date.parse(share.expiresAt) - DateTime.toEpochMillis(yield* DateTime.now)),
583 );
584 + const openingSnapshot = buildBoundedThreadStreamSnapshot({
585 + snapshotSequence: snapshot.snapshotSequence,
586 + projection: snapshot.projection,
587 + });
588 const opening: ReadonlyArray<GuestStreamEvent> = [
589 { type: "view", view: initialView },
533 - {
534 - type: "thread",
535 - item: buildBoundedThreadStreamSnapshot({
536 - snapshotSequence: snapshot.snapshotSequence,
537 - projection: snapshot.projection,
538 - }),
539 - },
590 + { type: "thread", item: openingSnapshot },
591 ];
541 - return Stream.fromIterable(opening).pipe(
592 + const events = Stream.fromIterable(opening).pipe(
593 Stream.concat(
594 Stream.merge(live, Stream.fromEffect(Effect.as(untilExpiry, ended)), {
595 haltStrategy: "either",
@@ -557,6 +608,25 @@ const make = Effect.gen(function* () {
608 ),
609 ),
610 );
611 + // The record is opened and counted when the stream runs, paired with its release, so a
612 + // response whose body never runs (HEAD) touches nothing.
613 + return Stream.unwrap(
614 + Effect.acquireRelease(
615 + Effect.sync(() => {
616 + const record = sentRecord(share.id);
617 + record.openStreams += 1;
618 + rememberSent(
619 + share.id,
620 + openingSnapshot.projection.visibleTurnItems.map((row) => row.item),
621 + );
622 + return record;
623 + }),
624 + (record) =>
625 + Effect.sync(() => {
626 + record.openStreams -= 1;
627 + }),
628 + ).pipe(Effect.as(events)),
629 + );
630 });
631
632 const unavailable = new GuestShareAccessError({ reason: "unavailable" });
@@ -630,6 +700,33 @@ const make = Effect.gen(function* () {
700 return exists ? { path, attachment: found } : yield* unavailable;
701 });
702
703 + const media = Effect.fn("GuestShareService.media")(function* (session: string, path: string) {
704 + const { share } = yield* authorize(session, "session");
705 + const mimeType = guestHostImageType(path);
706 + if (mimeType === null) return yield* unavailable;
707 + const sent = sentImageReplies.get(share.id)?.replies.values() ?? [];
708 + // Each sent reply is parsed once, on the first request that needs it.
709 + if (
710 + !repliesShowImage(
711 + path,
712 + [...sent],
713 + (reply) => (reply.sources ??= chatMarkdownImageSources(reply.text)),
714 + )
715 + ) {
716 + return yield* unavailable;
717 + }
718 + // The bytes must be the image the name says, as for a guest's own upload.
719 + const header = yield* fileSystem.open(path, { flag: "r" }).pipe(
720 + Effect.flatMap((file) => file.readAlloc(GUEST_UPLOAD_HEADER_BYTES)),
721 + Effect.scoped,
722 + Effect.mapError(() => unavailable),
723 + );
724 + if (Option.isNone(header) || !matchesImageSignature(mimeType, header.value)) {
725 + return yield* unavailable;
726 + }
727 + return { path, mimeType };
728 + });
729 +
730 return GuestShareService.of({
731 create,
732 redeem,
@@ -640,6 +737,7 @@ const make = Effect.gen(function* () {
737 item,
738 commandOutput,
739 attachment,
740 + media,
741 });
742 });
743
apps/server/src/guest/guestAttachments.test.ts new
+33
@@ -0,0 +1,33 @@
1 +import { describe, expect, it } from "vite-plus/test";
2 +
3 +import { guestHostImageType, repliesShowImage } from "./guestAttachments.ts";
4 +
5 +describe("guest host images", () => {
6 + it.each([
7 + ["![a](/w/a.png)", "/w/a.png", "image/png"],
8 + ["![a](</w/my shot.jpg>)", "/w/my shot.jpg", "image/jpeg"],
9 + ["![a](/w/shot(1).png)", "/w/shot(1).png", "image/png"],
10 + ["![a](/w/shot\\_1.png) ![b](/w/a&amp;b.png)", "/w/shot_1.png", "image/png"],
11 + ["![a](/w/shot\\_1.png) ![b](/w/a&amp;b.png)", "/w/a&b.png", "image/png"],
12 + ["![a](file:///w/caf%c3%a9.png)", "/w/café.png", "image/png"],
13 + ["![a](file:///w/%73hot.png) ![b](/w/sh&#111;t2.png)", "/w/shot.png", "image/png"],
14 + ["![a](file:///w/%73hot.png) ![b](/w/sh&#111;t2.png)", "/w/shot2.png", "image/png"],
15 + ["![a][r]\n\n[r]: /w/ref.png", "/w/ref.png", "image/png"],
16 + ["![a](file:///w/url%20form.png)", "/w/url form.png", "image/png"],
17 + ["![a](C:\\w\\win.png)", "C:\\w\\win.png", "image/png"],
18 + ["![a][r]\n\n[r]: C:\\w\\ref.png", "C:/w/ref.png", "image/png"],
19 + // Shown as a link, in code, or not at all: the guest sees no image, so none loads.
20 + ["[a](/w/link.png)", "/w/link.png", null],
21 + ["```\n![a](/w/code.png)\n```", "/w/code.png", null],
22 + ["`![a](/w/span.png)`", "/w/span.png", null],
23 + ["<!-- ![a](/w/comment.png) -->", "/w/comment.png", null],
24 + ["![a](/w/a.png)", "/a.png", null],
25 + ["![a](//host/s/x.png)", "//host/s/x.png", null],
26 + ["![a](\\\\host\\s\\x.png)", "\\\\host\\s\\x.png", null],
27 + ["![a](/w/a.svg)", "/w/a.svg", null],
28 + ["![a](/w/a.constructor)", "/w/a.constructor", null],
29 + ])("%j serves %j as %j", (text, path, type) => {
30 + const served = guestHostImageType(path) !== null && repliesShowImage(path, [{ text }]);
31 + expect(served ? guestHostImageType(path) : null).toBe(type);
32 + });
33 +});
apps/server/src/guest/guestAttachments.ts
+42 -3
@@ -3,6 +3,9 @@ import {
3 GUEST_MESSAGE_MAX_ATTACHMENTS,
4 isProviderSendTurnSupportedImageMimeType,
5 } from "@t3tools/contracts";
6 +import { markdownDestinationHostPath } from "@t3tools/shared/markdownLinks";
7 +import { chatMarkdownImageSources } from "@t3tools/shared/markdownPipeline";
8 +import { isAbsolutePath, isNetworkPath } from "@t3tools/shared/path";
9
10 /** A file from a guest's message request, saved by the multipart parser to a temporary path. */
11 export interface GuestUpload {
@@ -46,6 +49,44 @@ const IMAGE_SIGNATURES: Readonly<Record<string, (header: Uint8Array) => boolean>
49 startsWith(header, ascii("RIFF")) && startsWith(header, ascii("WEBP"), 8),
50 };
51
52 +export const matchesImageSignature = (mimeType: string, header: Uint8Array) =>
53 + IMAGE_SIGNATURES[mimeType]?.(header) === true;
54 +
55 +const IMAGE_TYPE_BY_EXTENSION: ReadonlyMap<string, string> = new Map([
56 + ["png", "image/png"],
57 + ["jpg", "image/jpeg"],
58 + ["jpeg", "image/jpeg"],
59 + ["gif", "image/gif"],
60 + ["webp", "image/webp"],
61 +]);
62 +
63 +/**
64 + * The type of a host image a guest may ask for by path, or null. A network path never qualifies:
65 + * it would make the host reach out to another machine.
66 + */
67 +export function guestHostImageType(path: string): string | null {
68 + if (!isAbsolutePath(path) || isNetworkPath(path)) return null;
69 + return IMAGE_TYPE_BY_EXTENSION.get(path.split(".").at(-1)!.toLowerCase()) ?? null;
70 +}
71 +
72 +/**
73 + * Whether an agent reply shows the image at `path`, so the guest already sees it. `imageSources`
74 + * lets the caller reuse a reply's parse across requests.
75 + */
76 +export function repliesShowImage<Reply extends { readonly text: string }>(
77 + path: string,
78 + replies: ReadonlyArray<Reply>,
79 + imageSources: (reply: Reply) => ReadonlyArray<string> = (reply) =>
80 + chatMarkdownImageSources(reply.text),
81 +) {
82 + // Parsing is the slow part; a reply with no image syntax shows no image.
83 + return replies.some(
84 + (reply) =>
85 + (reply.text.includes("![") || /<img/i.test(reply.text)) &&
86 + imageSources(reply).some((source) => markdownDestinationHostPath(source) === path),
87 + );
88 +}
89 +
90 /**
91 * Control characters and separators would break the name in headers and on disk, and direction
92 * overrides would let `invoice\u202Efdp.exe` show as a PDF.
@@ -76,11 +117,9 @@ export function classifyGuestUploads(
117 upload.contentType.split(";", 1)[0]!.trim().toLowerCase() || "application/octet-stream";
118 if (upload.sizeBytes === 0) return { ok: false, reason: "unsupported_file" };
119 if (mimeType.startsWith("image/")) {
79 - const signatureMatches = IMAGE_SIGNATURES[mimeType];
120 if (
121 !isProviderSendTurnSupportedImageMimeType(mimeType) ||
82 - signatureMatches === undefined ||
83 - !signatureMatches(upload.header)
122 + !matchesImageSignature(mimeType, upload.header)
123 ) {
124 return { ok: false, reason: "unsupported_file" };
125 }
apps/server/src/guest/http.ts
+28 -3
@@ -2,6 +2,7 @@ import {
2 GUEST_ATTACHMENT_ROUTE,
3 GUEST_COMMAND_OUTPUT_ROUTE,
4 GUEST_ITEM_ROUTE,
5 + GUEST_MEDIA_ROUTE,
6 GUEST_RENDER_ROUTE,
7 GUEST_ROUTE_PREFIX,
8 GUEST_SHARE_PARAM,
@@ -34,6 +35,12 @@ import { GuestShareAccessError, GuestShareService } from "./GuestShareService.ts
35 // Cloudflare closes idle streams after about 100 seconds.
36 const HEARTBEAT_INTERVAL = Duration.seconds(20);
37 const NO_STORE = { "cache-control": "no-store" };
38 +// A file never runs as a page on this origin, whatever its bytes.
39 +const FILE_HEADERS = {
40 + ...NO_STORE,
41 + "x-content-type-options": "nosniff",
42 + "content-security-policy": "default-src 'none'; sandbox",
43 +};
44
45 const decodeRedeem = Schema.decodeUnknownEffect(GuestRedeemRequest);
46 const decodeSubmit = Schema.decodeUnknownEffect(GuestSubmitRequest);
@@ -315,9 +322,7 @@ const attachmentRoute = HttpRouter.add(
322 Effect.flatMap(({ path, attachment }) =>
323 HttpServerResponse.file(path, {
324 headers: {
318 - ...NO_STORE,
319 - "x-content-type-options": "nosniff",
320 - "content-security-policy": "default-src 'none'; sandbox",
325 + ...FILE_HEADERS,
326 ...(attachment.type === "image" &&
327 isProviderSendTurnSupportedImageMimeType(attachment.mimeType)
328 ? { "content-type": attachment.mimeType }
@@ -334,6 +339,25 @@ const attachmentRoute = HttpRouter.add(
339 }),
340 );
341
342 +// Only an image an agent reply shows loads; anything else on the host is refused.
343 +const mediaRoute = HttpRouter.add(
344 + "GET",
345 + GUEST_MEDIA_ROUTE,
346 + Effect.gen(function* () {
347 + const request = yield* HttpServerRequest.HttpServerRequest;
348 + const path = new URLSearchParams(request.url.split("?", 2)[1] ?? "").get("path");
349 + if (!path) return yield* badRequest;
350 + const guestShares = yield* GuestShareService;
351 + return yield* guestShares.media(yield* sessionToken, path).pipe(
352 + Effect.flatMap(({ path, mimeType }) =>
353 + HttpServerResponse.file(path, { headers: { ...FILE_HEADERS, "content-type": mimeType } }),
354 + ),
355 + Effect.catchTags({ GuestShareAccessError: errorResponse }),
356 + Effect.orElseSucceed(() => HttpServerResponse.empty({ status: 500, headers: NO_STORE })),
357 + );
358 + }),
359 +);
360 +
361 export const layer = Layer.mergeAll(
362 redeemRoute,
363 sessionRoute,
@@ -343,4 +367,5 @@ export const layer = Layer.mergeAll(
367 itemRoute,
368 commandOutputRoute,
369 attachmentRoute,
370 + mediaRoute,
371 );
apps/web/src/components/ChatMarkdown.browser.test.tsx
+17
@@ -10,6 +10,7 @@ import "../index.css";
10 import { filesystemEnvironment } from "../state/filesystem";
11 import { useRightPanelStore } from "../rightPanelStore";
12 import { setMarkdownTaskChecked } from "./files/filePreviewMode";
13 +import { HostImageUrlContext } from "./chat/hostImageUrl";
14
15 const { workspaceSearch, filesystemMetadata } = vi.hoisted(() => ({
16 workspaceSearch: vi.fn(),
@@ -122,6 +123,22 @@ it("shows unavailable audio when a workspace embed has no thread", async () => {
123 expect(screen.container.querySelector("audio")).toBeNull();
124 });
125
126 +it("loads a host image through the page's own route when there is no environment", async () => {
127 + setup();
128 + const pixel =
129 + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
130 + const hostImageUrl = vi.fn((_path: string) => pixel);
131 + await using screen = await render(
132 + <HostImageUrlContext value={hostImageUrl}>
133 + <ChatMarkdown cwd={undefined} text="![Shot](</work/my shot.png>)" />
134 + </HostImageUrlContext>,
135 + );
136 + await expect.element(screen.getByAltText("Shot")).toHaveAttribute("src", pixel);
137 + expect(new Set(hostImageUrl.mock.calls.map(([path]) => path))).toEqual(
138 + new Set(["/work/my shot.png"]),
139 + );
140 +});
141 +
142 describe("ChatMarkdown favicon privacy", () => {
143 it("suppresses private link images while preserving public links across updates", async () => {
144 setup();
apps/web/src/components/ChatMarkdown.tsx
+19
@@ -65,6 +65,7 @@ import {
65 import { mediaUrlReference } from "@t3tools/client-runtime/media-reference";
66 import { mediaKindFromPath, mediaMimeTypeFromExtension } from "@t3tools/shared/mediaPreview";
67 import { InlineAudioPreview } from "./files/InlineAudioPreview";
68 +import { HostImageUrlContext } from "./chat/hostImageUrl";
69 import * as Cause from "effect/Cause";
70 import * as Option from "effect/Option";
71 import * as Schema from "effect/Schema";
@@ -1483,6 +1484,24 @@ const CHAT_MARKDOWN_COMPONENTS = {
1484 />
1485 );
1486 }
1487 + const hostImageUrl = use(HostImageUrlContext);
1488 + if (imageSource._tag === "WorkspaceFile" && kind === "image" && hostImageUrl !== null) {
1489 + const hostSrc = hostImageUrl(imageSource.path);
1490 + return (
1491 + <ChatMarkdownImage
1492 + key={hostSrc}
1493 + src={hostSrc}
1494 + alt={altText}
1495 + copyMarkdown={copyMarkdown}
1496 + standalone={standalone}
1497 + className={className}
1498 + style={authoredSizeStyle}
1499 + imageProps={imageProps}
1500 + actionsSource={{ kind, name: altText || kind, src: hostSrc }}
1501 + onImageExpand={imageExpand}
1502 + />
1503 + );
1504 + }
1505 if (imageSource._tag === "WorkspaceFile" && threadRef) {
1506 return (
1507 <ChatMarkdownAssetImage
apps/web/src/components/chat/hostImageUrl.ts new
+7
@@ -0,0 +1,7 @@
1 +import { createContext } from "react";
2 +
3 +/**
4 + * Where markdown loads an image file on the host by its absolute path, when the page has no
5 + * environment to ask. The guest page sets it to its own route.
6 + */
7 +export const HostImageUrlContext = createContext<((path: string) => string) | null>(null);
apps/web/src/components/guest/GuestPage.tsx
+49 -44
@@ -45,6 +45,7 @@ import type { ChatComposerHandle } from "../chat/ChatComposer";
45 import { ComposerControl, ComposerControlIcon } from "../chat/ComposerControl";
46 import type { ExpandedImagePreview } from "../chat/ExpandedImagePreview";
47 import { GuestMessageAttachments } from "../chat/GuestMessageAttachments";
48 +import { HostImageUrlContext } from "../chat/hostImageUrl";
49 import { PierreEntryIcon } from "../chat/PierreEntryIcon";
50 import { GuestMessageName, guestBubbleClassName } from "../chat/guestMessageStyle";
51 import { MessagesTimeline } from "../chat/MessagesTimeline";
@@ -60,6 +61,7 @@ import {
61 applyGuestStreamEvent,
62 formatGuestExpiry,
63 guestAttachmentUrl,
64 + guestMediaUrl,
65 createGuestAttachmentUrls,
66 guestFilesProblem,
67 initialGuestStreamState,
@@ -387,6 +389,7 @@ function GuestThread({
389 const openFile = useStableCallback((file: { readonly id: string }) =>
390 openInNewTab(guestAttachmentUrl(shareId, file.id)),
391 );
392 + const hostImageUrl = useStableCallback((path: string) => guestMediaUrl(shareId, path));
393 const activityRun = deriveThreadActivityRun(projection);
394 const runtime = deriveThreadStructureRuntime(projection);
395 const runlessWorkStartedAt = deriveRunlessWorkStartedAt(projection);
@@ -448,50 +451,52 @@ function GuestThread({
451 <ExpiryLabel expiresAt={expiresAt} />
452 </header>
453 <div className="relative min-h-0 flex-1">
451 - <MessagesTimeline
452 - listRef={listRef}
453 - timelineEntries={entries}
454 - isWorking={isWorking}
455 - activeTurnStartedAt={workStartedAt}
456 - latestRun={activityRun}
457 - runningRunId={runningRunId}
458 - turnDiffSummaries={NO_TURN_DIFFS}
459 - runs={projection.runs}
460 - routeThreadKey={threadKey}
461 - onOpenThread={ignore}
462 - supportsConversationRollback={false}
463 - onRevertToTurnCount={ignore}
464 - isRevertingCheckpoint={false}
465 - onImageExpand={openExpandedImage}
466 - onFileOpen={openFile}
467 - onFileDownload={openFile}
468 - renderHtmlRender={renderHtmlRender}
469 - turnItemSource={turnItemSource}
470 - activeThreadEnvironmentId={GUEST_ENVIRONMENT_ID}
471 - markdownCwd={undefined}
472 - resolvedTheme={resolvedTheme}
473 - timestampFormat="locale"
474 - workspaceRoot={undefined}
475 - anchorMessageId={null}
476 - onAnchorReady={ignore}
477 - onAnchorSizeChanged={ignore}
478 - contentInsetEndAdjustment={0}
479 - emptyPlaceholderInset={0}
480 - liveFollowEnabled={follow.timelineLiveFollowEnabled}
481 - onIsAtEndChange={follow.onIsAtEndChangeStable}
482 - onViewportAttach={follow.attachTimelineViewport}
483 - footer={
484 - unplaced.length === 0 ? null : (
485 - <ol ref={footerRef} className="chat-content-lane flex flex-col gap-4 pb-4">
486 - {unplaced.map((message) => (
487 - <li key={message.id}>
488 - <UnplacedGuestMessage shareId={shareId} message={message} />
489 - </li>
490 - ))}
491 - </ol>
492 - )
493 - }
494 - />
454 + <HostImageUrlContext value={hostImageUrl}>
455 + <MessagesTimeline
456 + listRef={listRef}
457 + timelineEntries={entries}
458 + isWorking={isWorking}
459 + activeTurnStartedAt={workStartedAt}
460 + latestRun={activityRun}
461 + runningRunId={runningRunId}
462 + turnDiffSummaries={NO_TURN_DIFFS}
463 + runs={projection.runs}
464 + routeThreadKey={threadKey}
465 + onOpenThread={ignore}
466 + supportsConversationRollback={false}
467 + onRevertToTurnCount={ignore}
468 + isRevertingCheckpoint={false}
469 + onImageExpand={openExpandedImage}
470 + onFileOpen={openFile}
471 + onFileDownload={openFile}
472 + renderHtmlRender={renderHtmlRender}
473 + turnItemSource={turnItemSource}
474 + activeThreadEnvironmentId={GUEST_ENVIRONMENT_ID}
475 + markdownCwd={undefined}
476 + resolvedTheme={resolvedTheme}
477 + timestampFormat="locale"
478 + workspaceRoot={undefined}
479 + anchorMessageId={null}
480 + onAnchorReady={ignore}
481 + onAnchorSizeChanged={ignore}
482 + contentInsetEndAdjustment={0}
483 + emptyPlaceholderInset={0}
484 + liveFollowEnabled={follow.timelineLiveFollowEnabled}
485 + onIsAtEndChange={follow.onIsAtEndChangeStable}
486 + onViewportAttach={follow.attachTimelineViewport}
487 + footer={
488 + unplaced.length === 0 ? null : (
489 + <ol ref={footerRef} className="chat-content-lane flex flex-col gap-4 pb-4">
490 + {unplaced.map((message) => (
491 + <li key={message.id}>
492 + <UnplacedGuestMessage shareId={shareId} message={message} />
493 + </li>
494 + ))}
495 + </ol>
496 + )
497 + }
498 + />
499 + </HostImageUrlContext>
500 {follow.showScrollToBottom ? (
501 <ScrollToEndButton
502 summary={null}
apps/web/src/components/guest/guestStream.logic.ts
+5
@@ -1,5 +1,6 @@
1 import {
2 GUEST_ATTACHMENT_ROUTE,
3 + GUEST_MEDIA_ROUTE,
4 GUEST_MESSAGE_MAX_ATTACHMENT_BYTES,
5 GUEST_MESSAGE_MAX_ATTACHMENTS,
6 GUEST_SHARE_PARAM,
@@ -184,6 +185,10 @@ export function inviteShareId(credential: string): string | null {
185 export const guestAttachmentUrl = (shareId: string, attachmentId: string) =>
186 `${GUEST_ATTACHMENT_ROUTE}?${GUEST_SHARE_PARAM}=${encodeURIComponent(shareId)}&id=${encodeURIComponent(attachmentId)}`;
187
188 +/** Where the guest page loads an image file on the host that an agent message embeds. */
189 +export const guestMediaUrl = (shareId: string, path: string) =>
190 + `${GUEST_MEDIA_ROUTE}?${GUEST_SHARE_PARAM}=${encodeURIComponent(shareId)}&path=${encodeURIComponent(path)}`;
191 +
192 /**
193 * Why these files cannot go with one message, or null. The server checks the same limits; this
194 * only says so before the upload.
docs/user/guest-links.md
+3 -3
@@ -26,9 +26,9 @@ unlinking the machine from T3 Connect also ends it. You can send or dismiss gues
26 ## What the guest sees
27
28 The guest sees the thread's recent history and everything after, in the same timeline you see, as
29 -it streams: messages and their files, the agent's replies and reasoning, plans, questions, and its
30 -work, with command output when they expand a command. Their composer shows the thread's model, but
31 -they cannot change it. They do not see diffs, and they cannot act on the thread: no forking,
29 +it streams: messages and their files, the agent's replies with the images they show from your
30 +machine, its reasoning, plans, questions, and its work, with command output when they expand a
31 +command. Their composer shows the thread's model, but they cannot change it. They do not see diffs, and they cannot act on the thread: no forking,
32 reverting, opening files on your machine, or answering the agent for you.
33
34 Anything the agent read or printed can show, such as a value from a file. Share only threads you
packages/client-runtime/src/markdownImages.ts
+4 -16
@@ -1,16 +1,14 @@
1 import {
2 isNetworkPath,
3 isWindowsAbsolutePath,
4 - stripSlashPrefixedWindowsDrive,
4 workspaceRelativeFilePath,
5 } from "@t3tools/shared/path";
6
7 import {
8 + markdownDestinationHostPath,
9 normalizeMarkdownLinkDestination,
10 - safeDecodeURIComponent,
10 splitMarkdownLinkSearchAndHash,
11 } from "@t3tools/shared/markdownLinks";
13 -import { parseFileUrlHref } from "@t3tools/shared/fileLinks";
12
13 const DIRECT_IMAGE_SOURCE_PATTERN = /^(?:https?:|data:|blob:|\/\/)/i;
14 const URI_SCHEME_PATTERN = /^[A-Za-z][A-Za-z0-9+.-]*:/;
@@ -59,19 +57,9 @@ export function classifyMarkdownImageSource(
57 return { _tag: "Direct", uri: source };
58 }
59
62 - if (/^file:/i.test(source)) {
63 - const target = parseFileUrlHref(source);
64 - return target === null
65 - ? { _tag: "Blocked" }
66 - : hostFileSource(
67 - stripSlashPrefixedWindowsDrive(safeDecodeURIComponent(target.path)),
68 - workspaceRoot,
69 - );
70 - }
71 -
72 - const path = stripSlashPrefixedWindowsDrive(
73 - safeDecodeURIComponent(splitMarkdownLinkSearchAndHash(source).path),
74 - );
60 + const path = markdownDestinationHostPath(source);
61 + if (path === null) return { _tag: "Blocked" };
62 + if (/^file:/i.test(source)) return hostFileSource(path, workspaceRoot);
63 if (path.length === 0) return { _tag: "Blocked" };
64 if (path.startsWith("/") || isWindowsAbsolutePath(path)) {
65 return hostFileSource(path, workspaceRoot);
packages/contracts/src/guestShare.ts
+5
@@ -136,3 +136,8 @@ export const GUEST_COMMAND_OUTPUT_ROUTE = `${GUEST_ROUTE_PREFIX}/command-output`
136 * Images load inline; other files download.
137 */
138 export const GUEST_ATTACHMENT_ROUTE = `${GUEST_ROUTE_PREFIX}/attachment`;
139 +/**
140 + * An image file on the host that an agent message in the shared thread embeds:
141 + * `?share=<share id>&path=<absolute path>`.
142 + */
143 +export const GUEST_MEDIA_ROUTE = `${GUEST_ROUTE_PREFIX}/media`;
packages/shared/src/markdownLinks.ts
+16
@@ -227,6 +227,22 @@ export function splitMarkdownLinkSearchAndHash(value: string): {
227 };
228 }
229
230 +/**
231 + * The decoded path a markdown destination names on the host, before any workspace resolution, or
232 + * null for a `file:` URL that does not parse. Clients and the guest media route both read it here.
233 + */
234 +export function markdownDestinationHostPath(source: string): string | null {
235 + if (/^file:/i.test(source)) {
236 + const target = parseFileUrlHref(source);
237 + return target === null
238 + ? null
239 + : stripSlashPrefixedWindowsDrive(safeDecodeURIComponent(target.path));
240 + }
241 + return stripSlashPrefixedWindowsDrive(
242 + safeDecodeURIComponent(splitMarkdownLinkSearchAndHash(source).path),
243 + );
244 +}
245 +
246 export function isMarkdownFileLinkLabel(label: string, href: string): boolean {
247 const destination = parseMarkdownFileLink(href);
248 if (!destination) return false;
packages/shared/src/markdownPipeline.ts
+39 -2
@@ -1,6 +1,8 @@
1 import type { Paragraph, PhrasingContent, Root, RootContent } from "mdast";
2 import { remarkKeepWindowsPaths } from "./markdownWindowsPaths.ts";
3 -import type { PluggableList } from "unified";
3 +import { type PluggableList, unified } from "unified";
4 +import remarkParse from "remark-parse";
5 +import remarkRehype from "remark-rehype";
6 import rehypeRaw from "rehype-raw";
7 import rehypeSanitize, { defaultSchema } from "rehype-sanitize";
8 import remarkBreaks from "remark-breaks";
@@ -11,7 +13,7 @@ import {
13 CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES,
14 remarkCodexDirectives,
15 } from "./codexMarkdownDirectives.ts";
14 -import { isWindowsDrivePathHref } from "./markdownLinks.ts";
16 +import { isWindowsDrivePathHref, normalizeMarkdownLinkDestination } from "./markdownLinks.ts";
17 import { THREAD_LINK_PROTOCOL } from "./threadLinks.ts";
18
19 type MarkdownImageHastNode = {
@@ -304,3 +306,38 @@ function remarkNormalizeLinksAndTagInlineCode() {
306 export function shouldPreserveAssistantLineBreaks(text: string): boolean {
307 return /^★ Insight(?:\s|─)/mu.test(text);
308 }
309 +
310 +const imageSourceProcessor = unified()
311 + .use(remarkParse)
312 + .use(CHAT_MARKDOWN_REMARK_PLUGINS)
313 + .use(remarkRehype, { allowDangerousHtml: true })
314 + .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
315 +
316 +interface ImageSourceNode {
317 + readonly type: string;
318 + readonly tagName?: string;
319 + readonly properties?: { readonly src?: unknown; readonly dataLocalSrc?: unknown };
320 + readonly children?: ReadonlyArray<ImageSourceNode>;
321 +}
322 +
323 +/**
324 + * The sources of the images chat markdown shows, read as ChatMarkdown's `img` renderer reads
325 + * them. Code, comments, and plain links show no image, so they add none.
326 + */
327 +export function chatMarkdownImageSources(markdown: string): Array<string> {
328 + const sources: Array<string> = [];
329 + const visit = (node: ImageSourceNode) => {
330 + if (node.type === "element" && node.tagName === "img") {
331 + const { src, dataLocalSrc } = node.properties ?? {};
332 + // The sanitizer drops a Windows drive path from `src`; the renderer reads it from here.
333 + if (typeof dataLocalSrc === "string") {
334 + sources.push(normalizeMarkdownLinkDestination(dataLocalSrc).replaceAll("\\", "/"));
335 + } else if (typeof src === "string") {
336 + sources.push(normalizeMarkdownLinkDestination(src));
337 + }
338 + }
339 + node.children?.forEach(visit);
340 + };
341 + visit(imageSourceProcessor.runSync(imageSourceProcessor.parse(markdown)) as ImageSourceNode);
342 + return sources;
343 +}