@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 488 lines 15.1 KB
Raw
1 // @effect-diagnostics globalTimers:off globalFetch:off - This browser and WebView transport runs without an Effect runtime.
2 import { type DeviceHubAccess, withDeviceHubQuery } from "../device/hubAccess.ts";
3 import {
4 PREVIEW_STREAM_HOST_SETUP_CLOSE_CODE,
5 type PreviewStreamHostSetup,
6 type PreviewViewportSetting,
7 } from "@t3tools/contracts";
8 import { mFallback, msg } from "gt-i18n";
9 import type { MFunctionType } from "gt-i18n/types";
10
11 export const PREVIEW_STREAM_BASE_PATH = "/api/preview-stream";
12
13 export interface PreviewStreamViewport {
14 readonly width: number;
15 readonly height: number;
16 }
17
18 export type PreviewStreamMouseButton = "none" | "left" | "middle" | "right";
19
20 export interface PreviewStreamControl {
21 readonly canOperate: boolean;
22 readonly controller: "agent" | "you" | "another-viewer" | "unclaimed";
23 readonly generation: number;
24 readonly dialog: null | {
25 readonly type: string;
26 readonly message: string;
27 readonly defaultValue: string;
28 };
29 }
30
31 export const previewStreamControlLabel = (
32 control: PreviewStreamControl | null,
33 resolve: MFunctionType = mFallback,
34 ): string =>
35 !control
36 ? resolve(msg("Connecting..."))
37 : !control.canOperate
38 ? resolve(msg("Read-only"))
39 : control.controller === "you"
40 ? resolve(msg("You have control"))
41 : control.controller === "agent"
42 ? resolve(msg("Agent has control"))
43 : control.controller === "another-viewer"
44 ? resolve(msg("Another viewer has control"))
45 : resolve(
46 msg("Watching", { $context: "preview stream status, viewing without control" }),
47 );
48
49 const isPreviewStreamDialog = (value: unknown): value is PreviewStreamControl["dialog"] =>
50 value === null ||
51 (typeof value === "object" &&
52 "type" in value &&
53 typeof value.type === "string" &&
54 "message" in value &&
55 typeof value.message === "string" &&
56 "defaultValue" in value &&
57 typeof value.defaultValue === "string");
58
59 /** Coordinates are page CSS px. */
60 export type PreviewStreamInput =
61 | { readonly type: "takeControl" }
62 | { readonly type: "releaseControl" }
63 | { readonly type: "dialog"; readonly accept: boolean; readonly promptText?: string }
64 | { readonly type: "viewport"; readonly setting: PreviewViewportSetting }
65 | {
66 readonly type: "mouse";
67 readonly action: "move" | "down" | "up";
68 readonly x: number;
69 readonly y: number;
70 readonly button: PreviewStreamMouseButton;
71 /** Pressed buttons: left 1, right 2, middle 4. */
72 readonly buttons: number;
73 readonly clickCount: number;
74 readonly modifiers: number;
75 }
76 | {
77 readonly type: "wheel";
78 readonly x: number;
79 readonly y: number;
80 readonly deltaX: number;
81 readonly deltaY: number;
82 readonly modifiers: number;
83 }
84 | {
85 readonly type: "key";
86 readonly action: "down" | "up";
87 readonly key: string;
88 readonly code: string;
89 /** Windows virtual key code (DOM `keyCode`); Enter, Backspace, and arrows need it. */
90 readonly keyCode?: number;
91 /** Only for printable keys without Ctrl or Meta. */
92 readonly text?: string;
93 readonly modifiers: number;
94 }
95 | { readonly type: "text"; readonly text: string }
96 | { readonly type: "resize"; readonly width: number; readonly height: number }
97 | { readonly type: "navigate"; readonly url: string }
98 | { readonly type: "history"; readonly delta: -1 | 1 }
99 | { readonly type: "reload"; readonly ignoreCache?: boolean }
100 | { readonly type: "stopLoading" }
101 | { readonly type: "probe"; readonly x: number; readonly y: number };
102
103 export interface PreviewStreamDownload {
104 readonly fileName: string;
105 readonly sizeBytes: number;
106 /** Authenticated with the stream's own access; cookie sessions must send credentials. */
107 readonly url: string;
108 }
109
110 export interface PreviewStreamFileChooser {
111 readonly multiple: boolean;
112 readonly accept: string;
113 /** POST multipart `file` parts here; an empty form cancels the page's picker. */
114 readonly uploadUrl: string;
115 readonly credentials: boolean;
116 }
117
118 const previewStreamUploadUrl = (
119 target: Pick<PreviewStreamTarget, "access" | "threadId" | "tabId">,
120 chooser: string,
121 ): string =>
122 withDeviceHubQuery(
123 `${target.access.httpBase}/upload?${new URLSearchParams({
124 threadId: target.threadId,
125 tabId: target.tabId,
126 chooser,
127 }).toString()}`,
128 target.access,
129 );
130
131 export async function uploadPreviewStreamFiles(
132 chooser: PreviewStreamFileChooser,
133 files: ReadonlyArray<Blob & { readonly name?: string }>,
134 ): Promise<void> {
135 const body = new FormData();
136 for (const file of files) body.append("file", file, file.name ?? "file");
137 const response = await fetch(chooser.uploadUrl, {
138 method: "POST",
139 body,
140 credentials: chooser.credentials ? "include" : "omit",
141 });
142 if (!response.ok) throw new Error((await response.text()) || "The upload was refused.");
143 }
144
145 export const previewStreamDownloadUrl = (
146 target: Pick<PreviewStreamTarget, "access" | "threadId" | "tabId">,
147 id: string,
148 ): string =>
149 withDeviceHubQuery(
150 `${target.access.httpBase}/download?${new URLSearchParams({
151 threadId: target.threadId,
152 tabId: target.tabId,
153 id,
154 }).toString()}`,
155 target.access,
156 );
157
158 export interface PreviewStreamProbe {
159 readonly x: number;
160 readonly y: number;
161 readonly editable: boolean;
162 }
163
164 export interface PreviewStreamPointer {
165 readonly phase: "move" | "click";
166 readonly x: number;
167 readonly y: number;
168 readonly sequence: number;
169 }
170
171 export const previewStreamModifiers = (event: {
172 readonly altKey: boolean;
173 readonly ctrlKey: boolean;
174 readonly metaKey: boolean;
175 readonly shiftKey: boolean;
176 }): number =>
177 (event.altKey ? 1 : 0) |
178 (event.ctrlKey ? 2 : 0) |
179 (event.metaKey ? 4 : 0) |
180 (event.shiftKey ? 8 : 0);
181
182 export interface PreviewStreamTarget {
183 readonly access: DeviceHubAccess;
184 readonly threadId: string;
185 readonly tabId: string;
186 /** Viewer backing-store size in device px. The server never sends larger frames. */
187 readonly maxWidth: number;
188 readonly maxHeight: number;
189 /** Passive viewers reduce their own access, including automatic control grants. */
190 readonly interactive?: boolean;
191 }
192
193 export interface PreviewStreamEvents {
194 readonly onFrame: (jpeg: ArrayBuffer) => void;
195 readonly onViewport: (viewport: PreviewStreamViewport) => void;
196 readonly onProbe?: (probe: PreviewStreamProbe) => void;
197 readonly onPointer?: (pointer: PreviewStreamPointer) => void;
198 readonly onControl?: (control: PreviewStreamControl) => void;
199 readonly onClipboard?: (text: string) => void;
200 readonly onDownload?: (download: PreviewStreamDownload) => void;
201 readonly onPopup?: (tabId: string) => void;
202 readonly onFileChooser?: (chooser: PreviewStreamFileChooser | null) => void;
203 readonly onConnectedChange: (connected: boolean) => void;
204 /** The upgrade was refused; refresh access and start a new client. */
205 readonly onUnauthorized: () => void;
206 readonly onGone?: () => void;
207 readonly onHostSetup?: (setup: PreviewStreamHostSetup) => void;
208 }
209
210 export interface PreviewStreamClient {
211 readonly send: (input: PreviewStreamInput) => boolean;
212 readonly stop: () => void;
213 }
214
215 const ACK_MESSAGE = JSON.stringify({ type: "ack" });
216 export function createPreviewStreamClient(
217 target: PreviewStreamTarget,
218 events: PreviewStreamEvents,
219 ): PreviewStreamClient {
220 const query = new URLSearchParams({
221 threadId: target.threadId,
222 tabId: target.tabId,
223 maxWidth: String(Math.max(1, Math.round(target.maxWidth))),
224 maxHeight: String(Math.max(1, Math.round(target.maxHeight))),
225 });
226 if (target.interactive === false) query.set("interactive", "false");
227 const url = withDeviceHubQuery(`${target.access.wsBase}/ws?${query.toString()}`, target.access);
228 let stopped = false;
229 let socket: WebSocket | null = null;
230 let retryTimer: ReturnType<typeof setTimeout> | null = null;
231 let failures = 0;
232 let control: PreviewStreamControl | null = null;
233 let fileChooser: string | null = null;
234
235 // The server re-offers a picker still open, under a new id, when this viewer reattaches.
236 const withdrawFileChooser = () => {
237 if (fileChooser === null) return;
238 fileChooser = null;
239 events.onFileChooser?.(null);
240 };
241
242 const connect = () => {
243 if (stopped) return;
244 const ws = new WebSocket(url);
245 ws.binaryType = "arraybuffer";
246 socket = ws;
247 let opened = false;
248 ws.addEventListener("open", () => {
249 if (socket !== ws) return;
250 opened = true;
251 events.onConnectedChange(true);
252 });
253 ws.addEventListener("message", (event) => {
254 if (socket !== ws) return;
255 if (event.data instanceof ArrayBuffer) {
256 failures = 0;
257 ws.send(ACK_MESSAGE);
258 events.onFrame(event.data);
259 return;
260 }
261 if (typeof event.data !== "string") return;
262 let message: unknown;
263 try {
264 message = JSON.parse(event.data);
265 } catch {
266 return;
267 }
268 if (typeof message !== "object" || message === null) return;
269 const {
270 type,
271 x,
272 y,
273 width,
274 height,
275 editable,
276 canOperate,
277 controller,
278 generation,
279 dialog,
280 text,
281 id,
282 fileName,
283 sizeBytes,
284 multiple,
285 accept,
286 phase,
287 sequence,
288 tabId,
289 } = message as Record<string, unknown>;
290 if (
291 type === "fileChooser" &&
292 typeof id === "string" &&
293 typeof multiple === "boolean" &&
294 typeof accept === "string"
295 ) {
296 fileChooser = id;
297 events.onFileChooser?.({
298 multiple,
299 accept,
300 uploadUrl: previewStreamUploadUrl(target, id),
301 credentials: target.access.credentials,
302 });
303 } else if (type === "fileChooserClosed" && typeof id === "string") {
304 if (fileChooser !== id) return;
305 fileChooser = null;
306 events.onFileChooser?.(null);
307 } else if (
308 type === "pointer" &&
309 (phase === "move" || phase === "click") &&
310 typeof x === "number" &&
311 typeof y === "number" &&
312 typeof sequence === "number"
313 ) {
314 events.onPointer?.({ phase, x, y, sequence });
315 } else if (type === "popup" && typeof tabId === "string") {
316 events.onPopup?.(tabId);
317 } else if (type === "clipboard" && typeof text === "string") {
318 events.onClipboard?.(text);
319 } else if (
320 type === "download" &&
321 typeof id === "string" &&
322 typeof fileName === "string" &&
323 typeof sizeBytes === "number"
324 ) {
325 events.onDownload?.({
326 fileName,
327 sizeBytes,
328 url: previewStreamDownloadUrl(target, id),
329 });
330 } else if (type === "viewport" && typeof width === "number" && typeof height === "number") {
331 failures = 0;
332 events.onViewport({ width, height });
333 } else if (
334 type === "probe" &&
335 typeof x === "number" &&
336 typeof y === "number" &&
337 typeof editable === "boolean"
338 ) {
339 events.onProbe?.({ x, y, editable });
340 } else if (
341 type === "control" &&
342 typeof canOperate === "boolean" &&
343 (controller === "agent" ||
344 controller === "you" ||
345 controller === "another-viewer" ||
346 controller === "unclaimed") &&
347 typeof generation === "number" &&
348 isPreviewStreamDialog(dialog)
349 ) {
350 const nextControl: PreviewStreamControl = {
351 canOperate,
352 controller,
353 generation,
354 dialog,
355 };
356 control = nextControl;
357 events.onControl?.(nextControl);
358 }
359 });
360 ws.addEventListener("close", (event) => {
361 if (socket !== ws) return;
362 socket = null;
363 control = null;
364 withdrawFileChooser();
365 if (opened) events.onConnectedChange(false);
366 if (stopped) return;
367 if (event.code === 4404) {
368 stopped = true;
369 events.onGone?.();
370 return;
371 }
372 if (event.code === PREVIEW_STREAM_HOST_SETUP_CLOSE_CODE) {
373 stopped = true;
374 events.onHostSetup?.(decodeHostSetup(event.reason));
375 return;
376 }
377 // Rejected upgrades surface as 1006 before open for both cookies and tickets.
378 if (event.code === 1008 || event.code === 4401 || (!opened && event.code === 1006)) {
379 stopped = true;
380 events.onUnauthorized();
381 return;
382 }
383 retryTimer = setTimeout(connect, Math.min(500 * 2 ** failures++, 10_000));
384 });
385 ws.addEventListener("error", () => ws.close());
386 };
387
388 connect();
389
390 return {
391 send: (input) => {
392 if (socket?.readyState !== WebSocket.OPEN) return false;
393 if (!control?.canOperate) return false;
394 if (input.type !== "takeControl" && control.controller !== "you") return false;
395 socket.send(JSON.stringify(input));
396 return true;
397 },
398 stop: () => {
399 stopped = true;
400 if (retryTimer !== null) clearTimeout(retryTimer);
401 const ws = socket;
402 socket = null;
403 control = null;
404 withdrawFileChooser();
405 ws?.close();
406 },
407 };
408 }
409
410 export interface PreviewFramePainter {
411 readonly paint: (jpeg: ArrayBuffer) => void;
412 readonly stop: () => void;
413 }
414
415 /** One decode at a time, keeping only the latest waiting frame for slow viewers. */
416 export function createPreviewFramePainter(
417 canvas: HTMLCanvasElement,
418 onPainted?: () => void,
419 ): PreviewFramePainter {
420 const context = canvas.getContext("2d");
421 let stopped = false;
422 let decoding = false;
423 let waiting: ArrayBuffer | null = null;
424 const draw = (jpeg: ArrayBuffer) => {
425 decoding = true;
426 void createImageBitmap(new Blob([jpeg], { type: "image/jpeg" }))
427 .then((bitmap) => {
428 if (stopped || !context) {
429 bitmap.close();
430 return;
431 }
432 if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) {
433 canvas.width = bitmap.width;
434 canvas.height = bitmap.height;
435 }
436 context.drawImage(bitmap, 0, 0);
437 bitmap.close();
438 onPainted?.();
439 })
440 .catch(() => undefined)
441 .finally(() => {
442 decoding = false;
443 const next = waiting;
444 waiting = null;
445 if (next && !stopped) draw(next);
446 });
447 };
448 return {
449 paint: (jpeg) => {
450 if (stopped) return;
451 if (decoding) waiting = jpeg;
452 else draw(jpeg);
453 },
454 stop: () => {
455 stopped = true;
456 waiting = null;
457 },
458 };
459 }
460
461 const decodeHostSetup = (reason: string): PreviewStreamHostSetup => {
462 try {
463 const value = JSON.parse(reason) as Partial<PreviewStreamHostSetup>;
464 if (
465 (value.need === "sandbox" || value.need === "libraries") &&
466 typeof value.command === "string"
467 ) {
468 return { need: value.need, command: value.command };
469 }
470 } catch {}
471 return { need: "sandbox", command: "sudo npx t3 browser setup" };
472 };
473
474 export const previewStreamHostSetupMessage = (
475 setup: PreviewStreamHostSetup,
476 resolve: MFunctionType = mFallback,
477 ) =>
478 setup.need === "sandbox"
479 ? resolve(
480 msg(
481 "This server's host blocks the sandbox its browser runs in. Run this once on the host, then try again:",
482 ),
483 )
484 : resolve(
485 msg(
486 "This server's host is missing libraries its browser needs. Run this once on the host, then try again:",
487 ),
488 );