The best way to control Claude Code, Codex and more from any computer or phone

main
ts 203 lines 6.82 KB
Raw
1 import {
2 createPreviewFramePainter,
3 createPreviewStreamClient,
4 type PreviewStreamClient,
5 } from "@t3tools/client-runtime/preview/server-browser-stream";
6 import type { EnvironmentId, ThreadId } from "@t3tools/contracts";
7 import { scopedThreadKey } from "@t3tools/client-runtime/environment";
8 import { previewPdfAssetUrl } from "@t3tools/shared/preview";
9 import { useSyncExternalStore } from "react";
10
11 import { readPreviewStreamAccess } from "~/state/previewStream";
12 import { readPreparedConnection } from "~/state/session";
13 import { previewStateAtom, type ThreadPreviewState } from "~/previewStateStore";
14 import { appAtomRegistry } from "~/rpc/atomRegistry";
15
16 interface WebKitVideo {
17 webkitSetPresentationMode?: (mode: string) => void;
18 webkitPresentationMode?: string;
19 }
20
21 interface ActivePictureInPicture {
22 readonly key: string;
23 readonly stop: () => void;
24 }
25
26 // Owns its stream outside React so the floating window survives panel unmounts.
27 let active: ActivePictureInPicture | null = null;
28 const listeners = new Set<() => void>();
29
30 const emit = () => {
31 for (const listener of listeners) listener();
32 };
33
34 const subscribe = (listener: () => void) => {
35 listeners.add(listener);
36 return () => listeners.delete(listener);
37 };
38
39 export const serverPictureInPictureKey = (threadId: string, tabId: string) =>
40 JSON.stringify([threadId, tabId]);
41
42 export function supportsServerPictureInPicture(): boolean {
43 if (typeof document === "undefined") return false;
44 if (typeof HTMLCanvasElement.prototype.captureStream !== "function") return false;
45 const video = HTMLVideoElement.prototype as HTMLVideoElement & WebKitVideo;
46 return (
47 (document.pictureInPictureEnabled && typeof video.requestPictureInPicture === "function") ||
48 typeof video.webkitSetPresentationMode === "function"
49 );
50 }
51
52 export function useServerPictureInPictureKey(): string | null {
53 return useSyncExternalStore(
54 subscribe,
55 () => active?.key ?? null,
56 () => null,
57 );
58 }
59
60 export function closeServerPictureInPicture(): void {
61 const current = active;
62 active = null;
63 current?.stop();
64 emit();
65 }
66
67 /** Open during user activation, seeding the video with the currently visible frame. */
68 export async function openServerPictureInPicture(input: {
69 readonly environmentId: EnvironmentId;
70 readonly threadId: ThreadId;
71 readonly tabId: string;
72 readonly seed: HTMLCanvasElement | null;
73 }): Promise<void> {
74 closeServerPictureInPicture();
75 const canvas = document.createElement("canvas");
76 const context = canvas.getContext("2d");
77 const seed = input.seed && input.seed.width > 0 && input.seed.height > 0 ? input.seed : null;
78 canvas.width = seed?.width ?? 640;
79 canvas.height = seed?.height ?? 400;
80 // The capture emits a frame per draw, so the seed is drawn after it starts.
81 const stream = canvas.captureStream();
82 if (seed) context?.drawImage(seed, 0, 0);
83 else context?.fillRect(0, 0, canvas.width, canvas.height);
84 const video = document.createElement("video") as HTMLVideoElement & WebKitVideo;
85 video.muted = true;
86 video.playsInline = true;
87 video.srcObject = stream;
88 // Safari only floats a video that is in the document.
89 video.style.cssText =
90 "position:fixed;right:0;bottom:0;width:1px;height:1px;opacity:0;pointer-events:none";
91 document.body.append(video);
92
93 const painter = createPreviewFramePainter(canvas);
94 let client: PreviewStreamClient | null = null;
95 let stopped = false;
96 let refusals = 0;
97 let retryTimer: ReturnType<typeof setTimeout> | null = null;
98 let stopWatchingPreview = () => {};
99 const onLeave = () => {
100 if (active === current) closeServerPictureInPicture();
101 };
102 const connect = async (refresh: boolean) => {
103 const access = await readPreviewStreamAccess(input.environmentId, refresh);
104 if (stopped) return;
105 if (!access) {
106 closeServerPictureInPicture();
107 return;
108 }
109 client = createPreviewStreamClient(
110 {
111 access,
112 threadId: input.threadId,
113 tabId: input.tabId,
114 maxWidth: 1280,
115 maxHeight: 1280,
116 interactive: false,
117 },
118 {
119 onFrame: (jpeg) => {
120 refusals = 0;
121 painter.paint(jpeg);
122 },
123 onViewport: () => undefined,
124 onConnectedChange: () => undefined,
125 onUnauthorized: () => {
126 if (stopped) return;
127 if (refusals >= 5) {
128 onLeave();
129 return;
130 }
131 retryTimer = setTimeout(() => void connect(true), 1_000 * 2 ** refusals++);
132 },
133 onGone: onLeave,
134 },
135 );
136 };
137
138 const onPresentationMode = () => {
139 if (video.webkitPresentationMode !== "picture-in-picture") onLeave();
140 };
141 const current: ActivePictureInPicture = {
142 key: serverPictureInPictureKey(input.threadId, input.tabId),
143 stop: () => {
144 stopped = true;
145 stopWatchingPreview();
146 if (retryTimer !== null) clearTimeout(retryTimer);
147 client?.stop();
148 painter.stop();
149 video.removeEventListener("leavepictureinpicture", onLeave);
150 video.removeEventListener("webkitpresentationmodechanged", onPresentationMode);
151 if (document.pictureInPictureElement === video) {
152 void document.exitPictureInPicture().catch(() => undefined);
153 } else if (video.webkitPresentationMode === "picture-in-picture") {
154 video.webkitSetPresentationMode?.("inline");
155 }
156 for (const track of stream.getTracks()) track.stop();
157 video.srcObject = null;
158 video.remove();
159 },
160 };
161 active = current;
162 emit();
163
164 try {
165 await video.play();
166 if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
167 await new Promise((resolve) =>
168 video.addEventListener("loadedmetadata", resolve, { once: true }),
169 );
170 }
171 if (document.pictureInPictureEnabled && typeof video.requestPictureInPicture === "function") {
172 video.addEventListener("leavepictureinpicture", onLeave);
173 await video.requestPictureInPicture();
174 } else {
175 video.addEventListener("webkitpresentationmodechanged", onPresentationMode);
176 video.webkitSetPresentationMode?.("picture-in-picture");
177 }
178 } catch (error) {
179 if (stopped) return;
180 onLeave();
181 throw error;
182 }
183 if (active === current) {
184 const atom = previewStateAtom(scopedThreadKey(input));
185 const closeForPdf = (state: ThreadPreviewState) => {
186 const navigation = state.sessions[input.tabId]?.navStatus;
187 if (
188 navigation?._tag === "Success" &&
189 // The base the client loads asset PDFs through now; a connection can change route.
190 previewPdfAssetUrl(
191 navigation.url,
192 readPreparedConnection(input.environmentId)?.httpBaseUrl ?? null,
193 ) !== null
194 ) {
195 onLeave();
196 }
197 };
198 // PDFs render in the client and cannot appear in the server's video stream.
199 stopWatchingPreview = appAtomRegistry.subscribe(atom, closeForPdf);
200 closeForPdf(appAtomRegistry.get(atom));
201 if (active === current) void connect(false);
202 }
203 }