The best way to control Claude Code, Codex and more from any computer or phone

main
tsx 520 lines 20.3 KB
Raw
1 import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
2 import { useAtomValue } from "@effect/atom-react";
3 import { AsyncResult, Atom } from "effect/reactivity";
4 import { T, useGT } from "gt-react";
5 import { useEffect, useId, useRef, useState, type ReactNode } from "react";
6
7 import { cn } from "~/lib/utils";
8 import { Button } from "~/components/ui/button";
9 import { renewDeviceHubAccess, useDeviceHubAccess } from "~/state/device";
10 import { resolveDeviceShape } from "@t3tools/client-runtime/device/shape-profile";
11 import { deviceKeyboard, deviceModel } from "./deviceModels";
12 import { fitDeviceFrame } from "./deviceFrameLayout";
13 import { DeviceDuoViewport } from "./DeviceDuoViewport";
14 import { DeviceDuoControls } from "./DeviceDuoControls";
15 import { DeviceAndroidFoldControls } from "./DeviceAndroidFoldControls";
16 import { DevicePhoneViewport } from "./DevicePhoneViewport";
17 import { DeviceLoadingView } from "./DeviceLoadingView";
18 import type { DeviceAxElement } from "./deviceHubApi";
19 import { androidFoldViewAtom, deviceAxElementsAtom } from "./deviceHubAtoms";
20 import {
21 deviceScreenKey,
22 deviceStreamSessionAtom,
23 type DeviceStreamSession,
24 IDLE_DEVICE_STREAM_STATE,
25 } from "./deviceStreamSession";
26 import type { DeviceHardwareButton, DeviceScreenSize } from "@t3tools/client-runtime/device/stream";
27 import { observeResize } from "~/lib/observeResize";
28
29 const NO_SESSION_ATOM = Atom.make<DeviceStreamSession | null>(null).pipe(
30 Atom.withLabel("device-stream:none"),
31 );
32 const NO_AX_ELEMENTS: ReadonlyArray<DeviceAxElement> = [];
33 const NO_AX_ATOM = Atom.make(AsyncResult.initial<ReadonlyArray<DeviceAxElement>, never>()).pipe(
34 Atom.withLabel("device-ax-elements:none"),
35 );
36 const CONTROLS_RAIL_WIDTH = 56;
37
38 function screenAspect(platform: DevicePlatform, screen: DeviceScreenSize | null): number {
39 if (!screen) return platform === "ios" ? 9 / 19.5 : 9 / 20;
40 const landscape =
41 screen.orientation === "landscape_left" || screen.orientation === "landscape_right";
42 const w = landscape
43 ? Math.max(screen.width, screen.height)
44 : Math.min(screen.width, screen.height);
45 const h = landscape
46 ? Math.min(screen.width, screen.height)
47 : Math.max(screen.width, screen.height);
48 return w / h;
49 }
50
51 // serve-sim sends raw portrait framebuffers even when iOS reports landscape; rotate the display.
52 function screenRotation(platform: DevicePlatform, screen: DeviceScreenSize | null): number {
53 if (platform !== "ios" || !screen || screen.width > screen.height) return 0;
54 switch (screen.orientation) {
55 case "landscape_left":
56 return 90;
57 case "landscape_right":
58 return -90;
59 case "portrait_upside_down":
60 return 180;
61 default:
62 return 0;
63 }
64 }
65
66 export interface DeviceViewControls {
67 readonly phone: boolean;
68 readonly streaming: boolean;
69 readonly phoneUnavailableReason: string | null;
70 readonly showPhone: () => void;
71 readonly showFlat: () => void;
72 readonly resetView: () => void;
73 readonly foldingControls?: ReactNode;
74 readonly keyboard: { readonly attached: boolean; readonly toggle: () => void } | null;
75 readonly handle: DeviceStreamHandle | null;
76 }
77
78 export interface DeviceStreamHandle {
79 readonly pressButton: (button: DeviceHardwareButton) => void;
80 readonly rotate: () => void;
81 readonly inputConnected: boolean;
82 }
83
84 export function DeviceStreamView(props: {
85 readonly environmentId: EnvironmentId;
86 readonly platform: DevicePlatform;
87 readonly deviceId: string;
88 readonly deviceName?: string;
89 readonly deviceDescription?: string;
90 readonly visible: boolean;
91 readonly hostId: string;
92 readonly allowPhoneView?: boolean;
93 readonly renderControls?: (view: DeviceViewControls) => ReactNode;
94 readonly axOverlay?: boolean;
95 /** Lets a parent read this view's own stream state, such as its screen size. */
96 readonly viewId?: string;
97 /** The host serves only serve-sim's stream and input socket (Simlock). */
98 readonly serveSimOnly?: boolean;
99 }) {
100 const model = deviceModel(props.platform, props.deviceName ?? "");
101 const isDuo = model?.id === "iphone-duo";
102 const gt = useGT();
103 const [presentation, setPresentation] = useState<"phone" | "flat">("phone");
104 const [keyboardAttached, setKeyboardAttached] = useState(false);
105 const cancelPhoneInputRef = useRef<(() => void) | null>(null);
106 const cancelPhoneInput = () => cancelPhoneInputRef.current?.();
107 const resetViewRef = useRef<(() => void) | null>(null);
108 const onResetReady = (reset: (() => void) | null) => {
109 resetViewRef.current = reset;
110 };
111 const onInputCancel = (cancel: (() => void) | null) => {
112 cancelPhoneInputRef.current = cancel;
113 };
114 const access = useDeviceHubAccess(props.environmentId, props.hostId);
115 const ownViewId = useId();
116 const viewId = props.viewId ?? ownViewId;
117 const session = useAtomValue(
118 access && props.visible
119 ? deviceStreamSessionAtom({
120 environmentId: props.environmentId,
121 hostId: props.hostId,
122 platform: props.platform,
123 deviceId: props.deviceId,
124 access,
125 serveSimOnly: props.serveSimOnly === true,
126 viewId,
127 })
128 : NO_SESSION_ATOM,
129 );
130 const client = session?.stream ?? null;
131 const canvas = session?.canvas ?? null;
132 const { status, detail, hasFrame, screen, mjpegUrl, mjpegGeneration, duoControl } =
133 session?.state ?? IDLE_DEVICE_STREAM_STATE;
134 const inputState = session?.state.input ?? IDLE_DEVICE_STREAM_STATE.input;
135 const phoneUnavailable = session?.state.phoneUnavailable ?? false;
136 const foldAngle = useAtomValue(
137 androidFoldViewAtom(deviceScreenKey(props.environmentId, props.hostId, props.deviceId)),
138 ).angle;
139 const attachMjpegImage = (image: HTMLImageElement | null) => {
140 if (!client || !image) return;
141 client.setMjpegImage(image);
142 return () => client.setMjpegImage(null);
143 };
144 const mountCanvas = (node: HTMLDivElement | null) => {
145 if (!node || !canvas) return;
146 node.append(canvas);
147 return () => canvas.remove();
148 };
149
150 const aspect = screenAspect(props.platform, screen);
151
152 const retainingFrame =
153 status === "connecting" && hasFrame && inputState.connected && screen !== null;
154 const showPhone =
155 props.allowPhoneView &&
156 (status === "streaming" || retainingFrame) &&
157 props.visible &&
158 presentation === "phone" &&
159 !phoneUnavailable &&
160 !mjpegUrl &&
161 !props.axOverlay &&
162 (!isDuo || screen?.supportsHingeAngle === true);
163 const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
164
165 // Fit the frame in both dimensions; pinned CSS height plus a width clamp distorts aspect-ratio.
166 const [host, setHost] = useState({ width: 0, height: 0 });
167 const hostRef = (element: HTMLDivElement | null) => {
168 if (!element) return;
169 const update = () => {
170 const rect = element.getBoundingClientRect();
171 setHost((current) =>
172 current.width === rect.width && current.height === rect.height
173 ? current
174 : { width: rect.width, height: rect.height },
175 );
176 };
177 update();
178 return observeResize(element, update);
179 };
180 const frame = fitDeviceFrame(aspect, host.width, host.height, controlsInset);
181
182 const rotation = screenRotation(props.platform, screen);
183
184 const sideways = rotation === 90 || rotation === -90;
185 const mediaStyle: React.CSSProperties = sideways
186 ? {
187 width: frame.height,
188 height: frame.width,
189 left: (frame.width - frame.height) / 2,
190 top: (frame.height - frame.width) / 2,
191 transform: `rotate(${rotation}deg)`,
192 }
193 : {
194 width: frame.width,
195 height: frame.height,
196 ...(rotation ? { transform: `rotate(${rotation}deg)` } : {}),
197 };
198
199 const axResult = useAtomValue(
200 props.axOverlay && access && props.visible
201 ? deviceAxElementsAtom({ access, platform: props.platform, deviceId: props.deviceId })
202 : NO_AX_ATOM,
203 );
204 const axElements = AsyncResult.getOrElse(axResult, () => NO_AX_ELEMENTS);
205
206 const activePointer = useRef<{ id: number; x: number; y: number } | null>(null);
207 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- clears pointer ownership when its device input connection disconnects or is replaced
208 useEffect(() => {
209 if (!client || !inputState.connected) return;
210 return () => {
211 activePointer.current = null;
212 };
213 }, [client, inputState.connected]);
214 const normalizedPoint = (event: React.PointerEvent<HTMLElement>) => {
215 const rect = event.currentTarget.getBoundingClientRect();
216 const x = (event.clientX - rect.left) / rect.width;
217 const y = (event.clientY - rect.top) / rect.height;
218 return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
219 };
220
221 const endPointer = (event: React.PointerEvent<HTMLElement>) => {
222 const pointer = activePointer.current;
223 if (pointer?.id !== event.pointerId) return;
224 activePointer.current = null;
225 const { x, y } = event.type === "lostpointercapture" ? pointer : normalizedPoint(event);
226 client?.sendTouch("end", x, y);
227 };
228
229 const phoneUnavailableReason =
230 isDuo && !screen?.supportsHingeAngle
231 ? gt("iPhone Duo 3D requires Device Hub {version} or newer", { version: "0.11.0" })
232 : phoneUnavailable
233 ? gt("3D is unavailable on this browser")
234 : mjpegUrl
235 ? gt("3D requires the H.264 stream")
236 : props.axOverlay
237 ? gt("Turn off accessibility frames to use 3D")
238 : null;
239
240 const keyboardSource = deviceKeyboard(props.platform, props.deviceName ?? "");
241 const resetView = () => {
242 if (!isDuo) {
243 const orientation = keyboardAttached ? "landscape_right" : "portrait";
244 if (screen?.orientation !== orientation) client?.setOrientation(orientation);
245 }
246 resetViewRef.current?.();
247 };
248 const profile = resolveDeviceShape({
249 platform: props.platform,
250 name: props.deviceName ?? "",
251 portraitAspect: Math.min(aspect, 1 / aspect),
252 });
253
254 return (
255 <div
256 className={cn(
257 "relative flex size-full min-h-0 min-w-0",
258 props.allowPhoneView ? "bg-background" : "bg-black/90",
259 )}
260 >
261 {props.renderControls ? (
262 <DeviceControlsSlot
263 renderControls={props.renderControls}
264 view={{
265 phone: !!showPhone,
266 streaming: status === "streaming",
267 phoneUnavailableReason,
268 foldingControls:
269 props.platform === "android" && access ? (
270 <DeviceAndroidFoldControls
271 key={`${props.hostId}:${props.deviceId}`}
272 access={access}
273 deviceId={props.deviceId}
274 visible={props.visible}
275 enabled={status === "streaming"}
276 screenWidth={screen?.width}
277 screenHeight={screen?.height}
278 deviceKey={deviceScreenKey(props.environmentId, props.hostId, props.deviceId)}
279 renewAccess={() => renewDeviceHubAccess(props.environmentId, props.hostId)}
280 />
281 ) : showPhone && isDuo && screen?.supportsHingeAngle ? (
282 <DeviceDuoControls
283 screen={screen}
284 state={duoControl}
285 enabled={inputState.connected}
286 onCommand={(command) => {
287 cancelPhoneInput();
288 client?.controlDuo(command);
289 }}
290 />
291 ) : null,
292 keyboard:
293 showPhone && keyboardSource
294 ? {
295 attached: keyboardAttached,
296 toggle: () => {
297 cancelPhoneInput();
298 if (!keyboardAttached && screen?.orientation !== "landscape_right")
299 client?.setOrientation("landscape_right");
300 setKeyboardAttached(!keyboardAttached);
301 },
302 }
303 : null,
304 showPhone: () => setPresentation("phone"),
305 showFlat: () => setPresentation("flat"),
306 handle: client
307 ? {
308 pressButton: client.pressButton,
309 rotate: client.rotate,
310 inputConnected: inputState.connected,
311 }
312 : null,
313 }}
314 onResetView={resetView}
315 />
316 ) : null}
317 <div
318 ref={hostRef}
319 className={cn(
320 "relative flex min-h-0 min-w-0 flex-1 items-center justify-center overflow-hidden outline-none",
321 // Overlay so the ring paints above the stream canvas.
322 "after:pointer-events-none after:absolute after:inset-0 after:z-10 focus-visible:after:ring-2 focus-visible:after:ring-inset focus-visible:after:ring-ring",
323 controlsInset && "pr-14",
324 )}
325 tabIndex={0}
326 role="application"
327 aria-label={
328 props.platform === "ios" ? gt("iOS Simulator screen") : gt("Android Emulator screen")
329 }
330 onKeyDown={(event) => {
331 if (event.target !== event.currentTarget) return;
332 if (event.metaKey && !["r", "R"].includes(event.key)) return;
333 // Let browser paste chords fire the paste event.
334 if (
335 (event.ctrlKey && (event.key === "v" || event.key === "V")) ||
336 (event.shiftKey && event.key === "Insert")
337 ) {
338 return;
339 }
340 event.preventDefault();
341 client?.sendKey(event.nativeEvent, "down");
342 }}
343 onPaste={(event) => {
344 if (event.target !== event.currentTarget) return;
345 event.preventDefault();
346 client?.sendText(event.clipboardData.getData("text/plain"));
347 }}
348 onKeyUp={(event) => {
349 if (event.target !== event.currentTarget) return;
350 client?.sendKey(event.nativeEvent, "up");
351 }}
352 >
353 <div
354 className={cn("relative select-none", showPhone && "invisible pointer-events-none")}
355 style={{ width: frame.width, height: frame.height }}
356 onPointerDown={(event) => {
357 if (event.button !== 0 || !inputState.connected || activePointer.current !== null)
358 return;
359 event.currentTarget.setPointerCapture(event.pointerId);
360 (event.currentTarget.parentElement as HTMLElement | null)?.focus();
361 const { x, y } = normalizedPoint(event);
362 activePointer.current = { id: event.pointerId, x, y };
363 client?.sendTouch("begin", x, y);
364 }}
365 onPointerMove={(event) => {
366 if (activePointer.current?.id !== event.pointerId) return;
367 const { x, y } = normalizedPoint(event);
368 activePointer.current = { id: event.pointerId, x, y };
369 client?.sendTouch("move", x, y);
370 }}
371 onPointerUp={endPointer}
372 onPointerCancel={endPointer}
373 onLostPointerCapture={endPointer}
374 >
375 <div
376 ref={mountCanvas}
377 className={cn("absolute top-0 left-0", mjpegUrl && "hidden")}
378 style={mediaStyle}
379 />
380 {props.visible && access && mjpegUrl ? (
381 <img
382 key={mjpegGeneration}
383 ref={attachMjpegImage}
384 alt=""
385 draggable={false}
386 className="absolute top-0 left-0 object-contain"
387 style={mediaStyle}
388 />
389 ) : null}
390 {axElements.length > 0 ? (
391 <div className="pointer-events-none absolute inset-0" aria-hidden>
392 {axElements.map((element) => (
393 <div
394 key={element.id}
395 className="absolute border border-info/80 bg-info/10"
396 style={{
397 left: `${element.x * 100}%`,
398 top: `${element.y * 100}%`,
399 width: `${element.width * 100}%`,
400 height: `${element.height * 100}%`,
401 }}
402 >
403 {element.label ? (
404 <span className="absolute -top-3.5 left-0 max-w-full truncate rounded-sm bg-info px-1 text-3xs leading-3.5 text-white">
405 {element.label}
406 </span>
407 ) : null}
408 </div>
409 ))}
410 </div>
411 ) : null}
412 </div>
413 {session && showPhone && isDuo && model ? (
414 <DeviceDuoViewport
415 onFrameListener={session.setFrameListener}
416 onControlRejectedListener={session.setControlRejectedListener}
417 model={model}
418 hingePreview={
419 duoControl.requested?.control === "angle" ? duoControl.requested.value : null
420 }
421 source={session.canvas}
422 client={session.stream}
423 onInputCancel={onInputCancel}
424 onResetReady={onResetReady}
425 screen={screen}
426 onUnavailable={session.markPhoneUnavailable}
427 />
428 ) : session && showPhone ? (
429 <DevicePhoneViewport
430 profile={profile}
431 model={deviceModel(props.platform, props.deviceName ?? "")}
432 accessory={keyboardAttached ? keyboardSource : null}
433 source={session.canvas}
434 onFrameListener={session.setFrameListener}
435 client={session.stream}
436 onInputCancel={onInputCancel}
437 onResetReady={onResetReady}
438 screen={screen}
439 foldAngle={foldAngle}
440 onUnavailable={session.markPhoneUnavailable}
441 />
442 ) : null}
443 {props.allowPhoneView && !props.renderControls && status === "streaming" ? (
444 <div className="absolute top-3 left-3 flex gap-1 rounded-lg border border-border/50 bg-background/90 p-1 shadow-sm">
445 <Button
446 variant={showPhone ? "secondary" : "ghost"}
447 size="xs"
448 aria-pressed={!!showPhone}
449 disabled={!!phoneUnavailableReason}
450 title={phoneUnavailableReason ?? gt("Show 3D phone")}
451 onClick={() => setPresentation("phone")}
452 >
453 3D
454 </Button>
455 <Button
456 variant={!showPhone ? "secondary" : "ghost"}
457 size="xs"
458 aria-pressed={!showPhone}
459 onClick={() => setPresentation("flat")}
460 >
461 <T context="flat device view">Flat</T>
462 </Button>
463 </div>
464 ) : null}
465 {status === "streaming" && !inputState.connected ? (
466 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
467 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
468 {inputState.detail
469 ? gt("Input disconnected ({detail}), reconnecting…", { detail: inputState.detail })
470 : gt("Input disconnected, reconnecting…")}
471 </span>
472 </div>
473 ) : null}
474 {retainingFrame && showPhone ? (
475 // Shown only once the device has been silent for two seconds.
476 <div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2 transition-opacity delay-2000 duration-0 starting:opacity-0">
477 <span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
478 <T>Waiting for device video…</T>
479 </span>
480 </div>
481 ) : null}
482 {status !== "streaming" && !(retainingFrame && showPhone) ? (
483 <div className="absolute inset-0">
484 <DeviceLoadingView
485 name={props.deviceName ?? gt("Device")}
486 description={props.deviceDescription ?? ""}
487 stage="stream"
488 message={
489 status === "error" ? (detail ?? gt("Stream failed.")) : gt("Connecting video…")
490 }
491 error={status === "error"}
492 >
493 {status === "error" ? (
494 <Button
495 size="sm"
496 variant="outline"
497 onClick={() => {
498 // An expired ticket surfaces as unauthorized on restart and
499 // refreshes access, which starts a new session.
500 session?.reconnect();
501 }}
502 >
503 <T>Reconnect</T>
504 </Button>
505 ) : null}
506 </DeviceLoadingView>
507 </div>
508 ) : null}
509 </div>
510 </div>
511 );
512 }
513
514 function DeviceControlsSlot(props: {
515 renderControls: (view: DeviceViewControls) => ReactNode;
516 view: Omit<DeviceViewControls, "resetView">;
517 onResetView: () => void;
518 }) {
519 return props.renderControls({ ...props.view, resetView: props.onResetView });
520 }