@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 1,499 lines 52.5 KB
Raw
1 // @effect-diagnostics globalFetch:off globalTimers:off - This browser and WebView transport runs without an Effect runtime.
2
3 import {
4 createDuoControl,
5 type DuoCommand,
6 type DuoControlState,
7 type DuoPose,
8 } from "./duoControl.ts";
9 import * as Schema from "effect/Schema";
10 import * as Option from "effect/Option";
11 import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
12 import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
13 import type { DevicePlatform } from "@t3tools/contracts";
14 import { mFallback, msg } from "gt-i18n";
15 import type { MFunctionType } from "gt-i18n/types";
16
17 export type DeviceStreamStatus = "connecting" | "streaming" | "error";
18
19 export interface DeviceScreenSize {
20 readonly width: number;
21 readonly height: number;
22 readonly orientation: "portrait" | "portrait_upside_down" | "landscape_left" | "landscape_right";
23 readonly screenId?: number;
24 readonly supportsHingeAngle?: boolean;
25 readonly supportsPhysicalOrientation?: boolean;
26 readonly hingeAngle?: number;
27 readonly hingePose?: DuoPose | null;
28 readonly tableMode?: boolean;
29 readonly tableModeAvailable?: boolean;
30 }
31
32 const screenConfigSchema = Schema.Struct({
33 width: Schema.Finite.check(Schema.isGreaterThan(0)),
34 height: Schema.Finite.check(Schema.isGreaterThan(0)),
35 orientation: Schema.Literals([
36 "portrait",
37 "portrait_upside_down",
38 "landscape_left",
39 "landscape_right",
40 ]),
41 inputUnavailable: Schema.optionalKey(Schema.Boolean),
42 screenId: Schema.optionalKey(Schema.Number),
43 supportsHingeAngle: Schema.optionalKey(Schema.Boolean),
44 supportsPhysicalOrientation: Schema.optionalKey(Schema.Boolean),
45 hingeAngle: Schema.optionalKey(
46 Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 180 })),
47 ),
48 hingePose: Schema.optionalKey(
49 Schema.NullOr(Schema.Literals(["closed", "book", "open", "laptop", "tent"])),
50 ),
51 tableMode: Schema.optionalKey(Schema.Boolean),
52 tableModeAvailable: Schema.optionalKey(Schema.Boolean),
53 });
54 const controlReplySchema = Schema.Struct({
55 requestId: Schema.Int,
56 ok: Schema.Boolean,
57 error: Schema.optionalKey(Schema.String),
58 });
59 const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema);
60 const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema);
61 const iceServerSchema = Schema.Struct({
62 urls: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
63 username: Schema.optionalKey(Schema.String),
64 credential: Schema.optionalKey(Schema.String),
65 });
66 const webRtcSettingsSchema = Schema.Struct({
67 transport: Schema.Literal("webrtc"),
68 iceServers: Schema.optionalKey(Schema.Array(iceServerSchema)),
69 iceTransportPolicy: Schema.optionalKey(Schema.Literals(["all", "relay"])),
70 });
71 // serve-sim reports its stream settings as `streamSettings` (Expo) or `stream`.
72 const serveSimApiSchema = Schema.Struct({
73 streamSettings: Schema.optionalKey(Schema.Unknown),
74 stream: Schema.optionalKey(Schema.Unknown),
75 });
76 const decodeServeSimApi = Schema.decodeUnknownOption(serveSimApiSchema);
77 const decodeWebRtcSettings = Schema.decodeUnknownOption(webRtcSettingsSchema);
78 type WebRtcSettings = typeof webRtcSettingsSchema.Type;
79 const decodeLeaseEnd = Schema.decodeUnknownOption(
80 Schema.fromJsonString(Schema.Struct({ reason: Schema.String })),
81 );
82 const leaseEnded = (reason: string, resolve: MFunctionType) =>
83 resolve(msg("Device lease ended: {reason}", { reason }));
84 const decodeTransportLocked = Schema.decodeUnknownOption(
85 Schema.Struct({
86 error: Schema.Literal("stream_transport_locked"),
87 transport: Schema.Literal("webrtc"),
88 }),
89 );
90 const decodeWebRtcAnswer = Schema.decodeUnknownOption(
91 Schema.Struct({ type: Schema.Literal("answer"), sdp: Schema.String }),
92 );
93
94 export interface DuoPanelSinks {
95 readonly cover: DeviceFrameSink;
96 readonly inner: DeviceFrameSink;
97 /** Invalidate captured input synchronously, before React can commit the new layout. */
98 readonly onScreen?: (screen: DeviceScreenSize) => void;
99 }
100
101 export interface DeviceStreamEvents {
102 readonly onDuoControl?: (state: DuoControlState) => void;
103 readonly onDuoUnavailable?: (detail?: string) => void;
104 readonly onStatus: (status: DeviceStreamStatus, detail?: string) => void;
105 readonly onScreen: (screen: DeviceScreenSize) => void;
106 readonly onUnauthorized: () => void;
107 /**
108 * H.264 cannot be decoded here (no WebCodecs, or the simulator's profile is
109 * unsupported); the owner should show an `<img>` instead of the canvas
110 * and attach it with `setMjpegImage` so the client can observe real frames.
111 */
112 readonly onMjpegFallback: (url: string) => void;
113 /**
114 * The stream moved from MJPEG back to painting the canvas, because the
115 * device serves WebRTC only; the owner should show the canvas again.
116 */
117 readonly onCanvasVideo?: () => void;
118 readonly onInputConnected: (connected: boolean, detail?: string) => void;
119 }
120
121 export interface DeviceStreamTarget {
122 readonly platform: DevicePlatform;
123 readonly deviceId: string;
124 readonly access: DeviceHubAccess;
125 /** Native iOS WebViews can use MJPEG without cross-origin fetch or secure-context support. */
126 readonly preferMjpeg?: boolean;
127 readonly panelId?: 1 | 3;
128 readonly videoOnly?: boolean;
129 /** The host speaks only serve-sim's AVCC stream and input socket, on both platforms (Simlock). */
130 readonly serveSimOnly?: boolean;
131 }
132
133 export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appSwitcher";
134
135 const RETRY_DELAY_MS = 1_000;
136 const FIRST_FRAME_TIMEOUT_MS = 15_000;
137 // A WebRTC feed that shows no frame for this long is reconnected; checked
138 // every WEBRTC_STALL_CHECK_MS, so a transient `disconnected` peer that
139 // recovers is left alone.
140 const WEBRTC_STALL_TIMEOUT_MS = FIRST_FRAME_TIMEOUT_MS;
141 const WEBRTC_STALL_CHECK_MS = 5_000;
142 const TRANSPORT_PROBE_TIMEOUT_MS = 5_000;
143 // Offers carry every candidate: serve-sim does not accept trickled ICE.
144 const ICE_GATHERING_TIMEOUT_MS = 2_000;
145 // A gathering cut short without a reflexive or relay candidate is retried
146 // with twice the deadline, up to this; then the offer goes as gathered. The
147 // whole sequence stays inside the first-frame deadline, which retries do
148 // not extend.
149 const ICE_GATHERING_MAX_TIMEOUT_MS = 4_000;
150 const MJPEG_FRAME_CHECK_MS = 250;
151 const FRAME_DURATION_US = 16_667;
152 const SEMU_MAGIC = 0x53454d55;
153 const SEMU_HEADER_BYTES = 16;
154 const SEMU_FLAG_KEY = 1;
155 const SOFT_DECODE_QUEUE = 8;
156 // expo-device-hub 0.15.3 serve-emu input.js MAX_TEXT_BYTES limits packets to 300 UTF-8 bytes.
157 const ANDROID_MAX_TEXT_BYTES = 300;
158 // The pinned hub queue depth is 128; the client does not read queue rejections.
159 const ANDROID_MAX_TEXT_CHUNKS = 64;
160
161 const IOS_MSG_TOUCH = 0x03;
162 const IOS_MSG_BUTTON = 0x04;
163 const IOS_MSG_KEY = 0x06;
164 const IOS_MSG_ORIENTATION = 0x07;
165 const IOS_TAG_SCREEN_CONFIG = 0x82;
166 const IOS_TAG_INPUT_ADMITTED = 0x83;
167
168 const encoder = new TextEncoder();
169 const decoder = new TextDecoder();
170
171 /** serve-sim requires a v4 UUID; `crypto.randomUUID` is missing over plain-HTTP LAN origins. */
172 function webRtcSessionId(): string {
173 const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16));
174 bytes[6] = (bytes[6]! & 0x0f) | 0x40;
175 bytes[8] = (bytes[8]! & 0x3f) | 0x80;
176 const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
177 return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
178 }
179
180 const isWebCodecsSupported = (): boolean =>
181 typeof globalThis !== "undefined" &&
182 "VideoDecoder" in globalThis &&
183 "EncodedVideoChunk" in globalThis;
184
185 function utf8ByteLength(codePoint: number): number {
186 if (codePoint <= 0x7f) return 1;
187 if (codePoint <= 0x7ff) return 2;
188 if (codePoint <= 0xffff) return 3;
189 return 4;
190 }
191
192 function taggedJson(tag: number, payload: unknown): Uint8Array<ArrayBuffer> {
193 const json = encoder.encode(JSON.stringify(payload));
194 const out = new Uint8Array(1 + json.length);
195 out[0] = tag;
196 out.set(json, 1);
197 return out;
198 }
199
200 export function avcCodecString(bytes: Uint8Array): string {
201 if (bytes.length < 4) return "avc1.42E01E";
202 const hex = (byte: number) => byte.toString(16).padStart(2, "0");
203 return `avc1.${hex(bytes[1]!)}${hex(bytes[2]!)}${hex(bytes[3]!)}`;
204 }
205
206 export function parseSemuPacket(raw: ArrayBuffer): {
207 readonly data: Uint8Array;
208 readonly isKey: boolean | null;
209 readonly timestamp: number | null;
210 } {
211 const bytes = new Uint8Array(raw);
212 if (bytes.byteLength > SEMU_HEADER_BYTES) {
213 const view = new DataView(raw, 0, SEMU_HEADER_BYTES);
214 if (view.getUint32(0, false) === SEMU_MAGIC && view.getUint8(4) === 1) {
215 const pts = view.getBigUint64(8, false);
216 return {
217 data: bytes.subarray(SEMU_HEADER_BYTES),
218 isKey: (view.getUint8(5) & SEMU_FLAG_KEY) !== 0,
219 timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null,
220 };
221 }
222 }
223 return { data: bytes, isKey: null, timestamp: null };
224 }
225
226 const isVideoSessionMessage = (text: string) => {
227 try {
228 const message = JSON.parse(text) as { type?: unknown };
229 return message.type === "video-session";
230 } catch {
231 return false;
232 }
233 };
234
235 export function scanAccessUnit(buf: Uint8Array): { isKey: boolean; sps: Uint8Array | null } {
236 let isKey = false;
237 let sps: Uint8Array | null = null;
238 const len = buf.length;
239 let i = 0;
240 while (i + 2 < len) {
241 if (buf[i] === 0 && buf[i + 1] === 0) {
242 let codeLen = 0;
243 if (buf[i + 2] === 1) codeLen = 3;
244 else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4;
245 if (codeLen) {
246 const nalType = buf[i + codeLen]! & 0x1f;
247 if (nalType === 7 && !sps) sps = buf.subarray(i + codeLen);
248 if (nalType === 5) isKey = true;
249 i += codeLen + 1;
250 continue;
251 }
252 }
253 i++;
254 }
255 return { isKey, sps };
256 }
257
258 export type AvccChunk = {
259 readonly type: "description" | "keyframe" | "delta" | "seed" | "end";
260 readonly payload: Uint8Array;
261 };
262
263 const AVCC_TAGS: Record<number, AvccChunk["type"] | undefined> = {
264 1: "description",
265 2: "keyframe",
266 3: "delta",
267 4: "seed",
268 // Simlock's final frame on a lease that ended, carrying `{ reason }`.
269 7: "end",
270 };
271
272 export class AvccDemuxer {
273 private buffer = new Uint8Array(64 * 1024);
274 private length = 0;
275
276 push(bytes: Uint8Array): AvccChunk[] {
277 if (this.length + bytes.length > this.buffer.length) {
278 let capacity = this.buffer.length;
279 while (capacity < this.length + bytes.length) capacity *= 2;
280 const grown = new Uint8Array(capacity);
281 grown.set(this.buffer.subarray(0, this.length));
282 this.buffer = grown;
283 }
284 this.buffer.set(bytes, this.length);
285 this.length += bytes.length;
286
287 const chunks: AvccChunk[] = [];
288 let offset = 0;
289 while (this.length - offset >= 4) {
290 const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + offset, 4);
291 const frameLength = view.getUint32(0, false);
292 if (this.length - offset - 4 < frameLength) break;
293 if (frameLength >= 1) {
294 const type = AVCC_TAGS[this.buffer[offset + 4]!];
295 if (type) {
296 chunks.push({ type, payload: this.buffer.slice(offset + 5, offset + 4 + frameLength) });
297 }
298 }
299 offset += 4 + frameLength;
300 }
301 if (offset > 0) {
302 this.buffer.copyWithin(0, offset, this.length);
303 this.length -= offset;
304 }
305 return chunks;
306 }
307
308 reset(): void {
309 this.length = 0;
310 }
311 }
312
313 export interface DeviceStreamClient {
314 readonly start: () => void;
315 readonly stop: () => void;
316 /**
317 * `stop()` detaches it; attach a fresh image for each restart.
318 */
319 readonly setMjpegImage: (image: HTMLImageElement | null) => void;
320 /** Normalized 0..1 coordinates in the displayed frame. */
321 readonly sendTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
322 readonly sendKey: (event: KeyboardEvent, phase: "down" | "up") => void;
323 /** Android only: iOS input is HID key codes, which cannot carry arbitrary text. */
324 readonly sendText: (text: string) => void;
325 readonly pressButton: (button: DeviceHardwareButton) => void;
326 readonly rotate: () => void;
327 readonly setOrientation: (orientation: DeviceScreenSize["orientation"]) => void;
328 readonly controlDuo: (command: DuoCommand) => void;
329 readonly setDuoPanels: (panels: DuoPanelSinks | null) => void;
330 readonly sendRawTouch: (phase: "begin" | "move" | "end", x: number, y: number) => void;
331 }
332
333 const HID_USAGE_BY_CODE: Readonly<Record<string, number>> = {
334 Enter: 0x28,
335 Escape: 0x29,
336 Backspace: 0x2a,
337 Tab: 0x2b,
338 Space: 0x2c,
339 Minus: 0x2d,
340 Equal: 0x2e,
341 BracketLeft: 0x2f,
342 BracketRight: 0x30,
343 Backslash: 0x31,
344 Semicolon: 0x33,
345 Quote: 0x34,
346 Backquote: 0x35,
347 Comma: 0x36,
348 Period: 0x37,
349 Slash: 0x38,
350 Delete: 0x4c,
351 ArrowRight: 0x4f,
352 ArrowLeft: 0x50,
353 ArrowDown: 0x51,
354 ArrowUp: 0x52,
355 NumpadDivide: 0x54,
356 NumpadMultiply: 0x55,
357 NumpadSubtract: 0x56,
358 NumpadAdd: 0x57,
359 NumpadEnter: 0x58,
360 Numpad1: 0x59,
361 Numpad2: 0x5a,
362 Numpad3: 0x5b,
363 Numpad4: 0x5c,
364 Numpad5: 0x5d,
365 Numpad6: 0x5e,
366 Numpad7: 0x5f,
367 Numpad8: 0x60,
368 Numpad9: 0x61,
369 Numpad0: 0x62,
370 NumpadDecimal: 0x63,
371 NumpadEqual: 0x67,
372 ControlLeft: 0xe0,
373 ShiftLeft: 0xe1,
374 AltLeft: 0xe2,
375 MetaLeft: 0xe3,
376 ControlRight: 0xe4,
377 ShiftRight: 0xe5,
378 AltRight: 0xe6,
379 MetaRight: 0xe7,
380 };
381
382 function hidUsageForCode(code: string): number | null {
383 if (/^Key[A-Z]$/.test(code)) return 0x04 + (code.charCodeAt(3) - 65);
384 if (/^Digit[1-9]$/.test(code)) return 0x1e + (code.charCodeAt(5) - 49);
385 if (code === "Digit0") return 0x27;
386 return HID_USAGE_BY_CODE[code] ?? null;
387 }
388
389 const ANDROID_KEYCODE_BY_KEY: Readonly<Record<string, number>> = {
390 ArrowUp: 19,
391 ArrowDown: 20,
392 ArrowLeft: 21,
393 ArrowRight: 22,
394 Tab: 61,
395 Enter: 66,
396 Backspace: 67,
397 Delete: 112,
398 Home: 122,
399 End: 123,
400 PageUp: 92,
401 PageDown: 93,
402 };
403
404 const IOS_ORIENTATIONS: ReadonlyArray<DeviceScreenSize["orientation"]> = [
405 "portrait",
406 "landscape_left",
407 "portrait_upside_down",
408 "landscape_right",
409 ];
410
411 export function createDeviceStreamClient(
412 target: DeviceStreamTarget,
413 output: HTMLCanvasElement | DeviceFrameSink,
414 events: DeviceStreamEvents,
415 resolve: MFunctionType = mFallback,
416 ): DeviceStreamClient {
417 const { access, platform, deviceId } = target;
418 const sink = "present" in output ? output : createCanvasFrameSink(output);
419 // Android normally streams over serve-emu's socket; a serve-sim-only host serves it like iOS.
420 const serveSimOnly = target.serveSimOnly === true;
421 const avcc = platform === "ios" || serveSimOnly;
422 const vendor = avcc ? "/vendor/serve-sim" : "/vendor/serve-emu";
423 const device = encodeURIComponent(deviceId);
424 const httpUrl = (path: string) =>
425 withDeviceHubQuery(`${access.httpBase}${vendor}${path}`, access);
426 const wsUrl = (path: string) => withDeviceHubQuery(`${access.wsBase}${vendor}${path}`, access);
427 const useWebCodecs =
428 isWebCodecsSupported() && !(platform === "ios" && target.preferMjpeg && !serveSimOnly);
429
430 let stopped = true;
431 let socket: WebSocket | null = null;
432 let controller: AbortController | null = null;
433 const retryTimers = new Map<"video" | "input", ReturnType<typeof setTimeout>>();
434 let primeController: AbortController | null = null;
435 let videoDecoder: VideoDecoder | null = null;
436 let decodedVideoGeneration: number | null = null;
437 let iosInputAdmitted = false;
438 let iosInputUnavailable = false;
439 let timestamp = 0;
440 let awaitingKeyframe = true;
441 let screen: DeviceScreenSize | null = null;
442 let firstFrame = false;
443 let configuring = false;
444 let mjpeg = false;
445 let generation = 0;
446 let decoderEpoch = 0;
447 let frameTimer: ReturnType<typeof setTimeout> | null = null;
448 let videoReadTimer: ReturnType<typeof setTimeout> | null = null;
449 let mjpegImage: HTMLImageElement | null = null;
450 let releaseImage: (() => void) | null = null;
451 let videoGeneration = 0;
452 let panelClients: DeviceStreamClient[] = [];
453 let panelSinks: DuoPanelSinks | null = null;
454 let rotationCursor: DeviceScreenSize["orientation"] | null = null;
455 let pendingOrientation: { requestId: number } | null = null;
456 let iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
457 let peer: { connection: RTCPeerConnection; sessionId: string; video: HTMLVideoElement } | null =
458 null;
459 const duoControl = createDuoControl({
460 send(request) {
461 if (
462 !iosInputAdmitted ||
463 iosInputUnavailable ||
464 socket?.readyState !== WebSocket.OPEN ||
465 !screen?.supportsHingeAngle
466 )
467 return false;
468 if (request.command.control === "physical" && !screen.supportsPhysicalOrientation)
469 return false;
470 try {
471 pendingOrientation = null;
472 if (request.command.control === "orientation") {
473 const value = request.command.value;
474 pendingOrientation = { requestId: request.requestId };
475 rotationCursor = value;
476 // Upstream serializes orientation with hinge commands, then broadcasts config.
477 // An orientation-locked app can keep its framebuffer orientation after the sensor rotates.
478 socket.send(taggedJson(IOS_MSG_ORIENTATION, { orientation: value }));
479 } else socket.send(taggedJson(0x10, request));
480 return true;
481 } catch {
482 return false;
483 }
484 },
485 onChange(state) {
486 if (!state.pending) pendingOrientation = null;
487 events.onDuoControl?.(state);
488 },
489 });
490 const videoPath = `/helper/${device}${target.panelId ? `/panel/${target.panelId}` : ""}/stream.avcc`;
491
492 const mjpegUrl = () => httpUrl(`/helper/${device}/stream.mjpeg`);
493
494 const setStatus = (status: DeviceStreamStatus, detail?: string) => {
495 if (!stopped) events.onStatus(status, detail);
496 };
497
498 const clearFrameTimer = () => {
499 if (frameTimer !== null) clearTimeout(frameTimer);
500 frameTimer = null;
501 };
502
503 const fail = (detail: string) => {
504 if (stopped) return;
505 stop();
506 events.onInputConnected(false, detail);
507 events.onStatus("error", detail);
508 };
509
510 const connecting = (detail?: string) => {
511 firstFrame = false;
512 if (frameTimer === null) {
513 frameTimer = setTimeout(
514 () => fail(resolve(msg("No video received from the device. Reconnect to try again."))),
515 FIRST_FRAME_TIMEOUT_MS,
516 );
517 }
518 setStatus("connecting", detail);
519 };
520
521 const frameReceived = () => {
522 if (firstFrame) return;
523 firstFrame = true;
524 clearFrameTimer();
525 setStatus("streaming");
526 };
527
528 const observeMjpegImage = () => {
529 releaseImage?.();
530 releaseImage = null;
531 const image = mjpegImage;
532 if (!image || stopped || !mjpeg) return;
533 let timer: ReturnType<typeof setTimeout> | null = null;
534 let released = false;
535 const check = () => {
536 if (released || stopped) return;
537 if (timer !== null) clearTimeout(timer);
538 timer = null;
539 if (image.naturalWidth > 0 && image.naturalHeight > 0) frameReceived();
540 else {
541 // Multipart images may not emit load until the response ends.
542 timer = setTimeout(check, MJPEG_FRAME_CHECK_MS);
543 }
544 };
545 const error = () => {
546 if (released) return;
547 const failure = resolve(msg("Could not receive the device stream. Reconnect to try again."));
548 // An <img> cannot read the 409 that a WebRTC-only serve-sim answers with.
549 if (typeof RTCPeerConnection === "undefined") fail(failure);
550 else void switchToWebRtc(failure, false);
551 };
552 image.addEventListener("load", check);
553 image.addEventListener("error", error);
554 releaseImage = () => {
555 released = true;
556 if (timer !== null) clearTimeout(timer);
557 image.removeEventListener("load", check);
558 image.removeEventListener("error", error);
559 image.removeAttribute("src");
560 };
561 image.src = mjpegUrl();
562 check();
563 };
564
565 const setMjpegImage = (image: HTMLImageElement | null) => {
566 if (mjpegImage === image) return;
567 releaseImage?.();
568 releaseImage = null;
569 mjpegImage = image;
570 observeMjpegImage();
571 };
572
573 const fallBackToMjpeg = () => {
574 if (stopped || mjpeg) return;
575 mjpeg = true;
576 controller?.abort();
577 controller = null;
578 closeDecoder();
579 connecting();
580 events.onMjpegFallback(mjpegUrl());
581 if (!releaseImage) observeMjpegImage();
582 };
583
584 const paint = (source: CanvasImageSource, width: number, height: number) => {
585 if (stopped) return;
586 if (!avcc && (screen?.width !== width || screen.height !== height)) {
587 screen = { width, height, orientation: width > height ? "landscape_left" : "portrait" };
588 events.onScreen(screen);
589 }
590 if (!sink.present(source, width, height)) {
591 fail(resolve(msg("Could not display the device stream. Reconnect to try again.")));
592 return;
593 }
594 frameReceived();
595 };
596
597 const closeDecoder = () => {
598 decoderEpoch++;
599 try {
600 videoDecoder?.close();
601 } catch {}
602 videoDecoder = null;
603 awaitingKeyframe = true;
604 };
605
606 const recoverDecoder = () => {
607 if (serveSimOnly) {
608 // A new AVCC request starts a fresh capture with a keyframe; there is no MJPEG.
609 closeDecoder();
610 controller?.abort();
611 } else if (platform === "ios") fallBackToMjpeg();
612 else {
613 closeDecoder();
614 connecting(resolve(msg("Video decoder restarted.")));
615 requestKeyframe();
616 }
617 };
618
619 const makeDecoder = () => {
620 const feedGeneration = videoGeneration;
621 const decoder = new VideoDecoder({
622 output: (frame) => {
623 try {
624 if (videoDecoder === decoder && (!avcc || feedGeneration === videoGeneration)) {
625 decodedVideoGeneration = feedGeneration;
626 paint(frame, frame.displayWidth, frame.displayHeight);
627 }
628 } finally {
629 frame.close();
630 }
631 },
632 error: () => {
633 if (stopped || videoDecoder !== decoder || (avcc && feedGeneration !== videoGeneration))
634 return;
635 recoverDecoder();
636 },
637 });
638 return decoder;
639 };
640
641 const configureDecoder = async (
642 config: VideoDecoderConfig,
643 isCurrent = () => !stopped,
644 ): Promise<boolean> => {
645 const epoch = decoderEpoch;
646 const full: VideoDecoderConfig = { ...config, optimizeForLatency: true };
647 const support = await VideoDecoder.isConfigSupported(full).catch(() => ({ supported: false }));
648 if (!isCurrent() || epoch !== decoderEpoch) return false;
649 if (!support.supported) {
650 if (platform === "android")
651 fail(resolve(msg("This browser cannot decode {codec}.", { codec: config.codec })));
652 return false;
653 }
654 try {
655 if (!videoDecoder || videoDecoder.state === "closed") videoDecoder = makeDecoder();
656 videoDecoder.configure(full);
657 return true;
658 } catch (cause) {
659 if (platform === "android")
660 fail(resolve(msg("Video decoder: {message}", { message: (cause as Error).message })));
661 return false;
662 }
663 };
664
665 const decode = (isKey: boolean, data: Uint8Array, pts?: number | null) => {
666 if (!videoDecoder || videoDecoder.state !== "configured") return;
667 if (awaitingKeyframe) {
668 if (!isKey) return;
669 awaitingKeyframe = false;
670 }
671 if (videoDecoder.decodeQueueSize > SOFT_DECODE_QUEUE) {
672 if (!avcc) {
673 recoverDecoder();
674 return;
675 }
676 // Native IDRs are spaced by frame count, so an idle simulator may take
677 // minutes to produce one. A new AVCC subscription forces a fresh IDR.
678 controller?.abort();
679 closeDecoder();
680 return;
681 }
682 try {
683 videoDecoder.decode(
684 new EncodedVideoChunk({
685 type: isKey ? "key" : "delta",
686 timestamp: pts ?? timestamp,
687 data,
688 }),
689 );
690 timestamp += FRAME_DURATION_US;
691 } catch {
692 recoverDecoder();
693 }
694 };
695
696 const requestKeyframe = () => {
697 if (platform === "android" && socket?.readyState === WebSocket.OPEN) {
698 socket.send(JSON.stringify({ type: "reset-video", ack: false }));
699 }
700 };
701
702 const scheduleRetry = (channel: "video" | "input", run: () => void) => {
703 if (stopped || retryTimers.has(channel)) return;
704 retryTimers.set(
705 channel,
706 setTimeout(() => {
707 retryTimers.delete(channel);
708 run();
709 }, RETRY_DELAY_MS),
710 );
711 };
712
713 const handleUnauthorized = () => {
714 stop();
715 events.onInputConnected(false);
716 events.onUnauthorized();
717 };
718
719 const readIosVideo = async () => {
720 const lifecycle = generation;
721 const feedGeneration = ++videoGeneration;
722 const demuxer = new AvccDemuxer();
723 const videoController = new AbortController();
724 controller = videoController;
725 const isCurrent = () =>
726 !stopped &&
727 generation === lifecycle &&
728 videoGeneration === feedGeneration &&
729 controller === videoController;
730 let retryDetail: string | undefined;
731 let leaseEnd: string | undefined;
732 try {
733 const response = await fetch(httpUrl(videoPath), {
734 signal: videoController.signal,
735 credentials: access.credentials ? "include" : "same-origin",
736 });
737 if (!isCurrent()) {
738 await response.body?.cancel();
739 return;
740 }
741 if (response.status === 401 || response.status === 403) return handleUnauthorized();
742 if (response.status === 409 && !target.panelId) {
743 const locked = decodeTransportLocked(await response.json().catch(() => null));
744 if (Option.isSome(locked)) {
745 if (!isCurrent()) return;
746 controller = null;
747 void switchToWebRtc(
748 resolve(msg("This browser cannot play this device's WebRTC stream.")),
749 true,
750 );
751 return;
752 }
753 }
754 if (target.panelId && [400, 404, 405, 410].includes(response.status)) {
755 await response.body?.cancel();
756 setStatus(
757 "error",
758 resolve(msg("This Device Hub does not provide fixed Duo display feeds.")),
759 );
760 return;
761 }
762 if (!response.ok || !response.body) throw new Error(`stream ${response.status}`);
763 const reader = response.body.getReader();
764 for (;;) {
765 // An AVCC body can stay open after its helper stops producing frames.
766 const timer = setTimeout(() => {
767 if (!isCurrent()) return;
768 retryDetail = resolve(msg("Device stream stopped receiving video. Reconnecting…"));
769 videoController.abort();
770 }, FIRST_FRAME_TIMEOUT_MS);
771 videoReadTimer = timer;
772 let result: ReadableStreamReadResult<Uint8Array>;
773 try {
774 result = await reader.read();
775 } finally {
776 clearTimeout(timer);
777 if (videoReadTimer === timer) videoReadTimer = null;
778 }
779 const { done, value } = result;
780 if (!isCurrent()) return;
781 if (done) break;
782 for (const chunk of demuxer.push(value)) {
783 if (videoController.signal.aborted) break;
784 switch (chunk.type) {
785 case "seed":
786 void createImageBitmap(new Blob([chunk.payload as BlobPart], { type: "image/jpeg" }))
787 .then((bitmap) => {
788 try {
789 if (isCurrent() && decodedVideoGeneration !== feedGeneration)
790 paint(bitmap, bitmap.width, bitmap.height);
791 } finally {
792 bitmap.close();
793 }
794 })
795 .catch(() => {});
796 break;
797 case "description": {
798 awaitingKeyframe = true;
799 const configured = await configureDecoder(
800 {
801 codec: avcCodecString(chunk.payload),
802 description: chunk.payload,
803 },
804 isCurrent,
805 );
806 if (!isCurrent()) return;
807 if (!configured) {
808 await reader.cancel().catch(() => {});
809 if (!isCurrent()) return;
810 if (serveSimOnly)
811 setStatus(
812 "error",
813 resolve(
814 msg("This browser cannot decode this device's {codec} stream.", {
815 codec: avcCodecString(chunk.payload),
816 }),
817 ),
818 );
819 else if (target.videoOnly)
820 setStatus(
821 "error",
822 resolve(
823 msg("This browser cannot decode the Duo panel's {codec} stream.", {
824 codec: avcCodecString(chunk.payload),
825 }),
826 ),
827 );
828 else fallBackToMjpeg();
829 return;
830 }
831 break;
832 }
833 case "keyframe":
834 case "delta":
835 decode(chunk.type === "keyframe", chunk.payload);
836 break;
837 case "end": {
838 const end = decodeLeaseEnd(decoder.decode(chunk.payload));
839 leaseEnd = Option.isSome(end) ? end.value.reason : "ended";
840 break;
841 }
842 }
843 }
844 }
845 } catch (cause) {
846 if (!isCurrent()) return;
847 retryDetail ??= videoController.signal.aborted
848 ? resolve(msg("Video decoder fell behind. Reconnecting…"))
849 : (cause as Error).message;
850 }
851 if (isCurrent()) {
852 if (leaseEnd !== undefined) return fail(leaseEnded(leaseEnd, resolve));
853 controller = null;
854 videoController.abort();
855 closeDecoder();
856 connecting(retryDetail);
857 scheduleRetry("video", () => void readIosVideo());
858 }
859 };
860
861 /**
862 * serve-sim's helper only accepts HID and pushes its screen config once
863 * screen capture is running, and the AVCC stream does not reliably start
864 * it. Touching the MJPEG endpoint does; one aborted request is enough.
865 */
866 const primeIosHelper = async (session: number) => {
867 const controller = new AbortController();
868 primeController = controller;
869 const timeout = setTimeout(() => controller.abort(), 2_000);
870 try {
871 const response = await fetch(httpUrl(`/helper/${device}/stream.mjpeg`), {
872 signal: controller.signal,
873 credentials: access.credentials ? "include" : "same-origin",
874 });
875 if (stopped || generation !== session) return;
876 if (response.status === 401 || response.status === 403) return handleUnauthorized();
877 await response.body?.getReader().read();
878 } catch {
879 } finally {
880 clearTimeout(timeout);
881 controller.abort();
882 if (primeController === controller) primeController = null;
883 }
884 };
885
886 /**
887 * serve-sim can be locked to WebRTC (an Expo Simulator always is), and then
888 * refuses AVCC and MJPEG; serve-emu advertises WebRTC the same way. An
889 * unreadable answer keeps the default path.
890 */
891 const probeWebRtc = async (): Promise<WebRtcSettings | null> => {
892 const probe = new AbortController();
893 const timeout = setTimeout(() => probe.abort(), TRANSPORT_PROBE_TIMEOUT_MS);
894 try {
895 const response = await fetch(httpUrl(`/api?device=${device}`), {
896 signal: probe.signal,
897 credentials: access.credentials ? "include" : "same-origin",
898 });
899 if (!response.ok) return null;
900 const api = decodeServeSimApi(await response.json());
901 if (Option.isNone(api)) return null;
902 const settings = decodeWebRtcSettings(api.value.streamSettings ?? api.value.stream);
903 return Option.isSome(settings) ? settings.value : null;
904 } catch {
905 return null;
906 } finally {
907 clearTimeout(timeout);
908 }
909 };
910
911 const webRtcUrl = (action: "offer" | "close") =>
912 httpUrl(avcc ? `/helper/${device}/webrtc/${action}` : `/webrtc/${action}?device=${device}`);
913
914 const closePeer = () => {
915 const current = peer;
916 if (!current) return;
917 peer = null;
918 current.connection.close();
919 current.video.srcObject = null;
920 current.video.remove();
921 // Releases the capture session; best effort because the page may be unloading.
922 void fetch(webRtcUrl("close"), {
923 method: "POST",
924 keepalive: true,
925 headers: { "content-type": "application/json" },
926 body: JSON.stringify({ sessionId: current.sessionId }),
927 credentials: access.credentials ? "include" : "same-origin",
928 }).catch(() => {});
929 };
930
931 const readWebRtc = async (settings: WebRtcSettings) => {
932 const lifecycle = generation;
933 const feedGeneration = ++videoGeneration;
934 closePeer();
935 const connection = new RTCPeerConnection({
936 ...(settings.iceServers
937 ? {
938 iceServers: settings.iceServers.map((server) => ({
939 ...server,
940 urls: typeof server.urls === "string" ? server.urls : [...server.urls],
941 })),
942 }
943 : {}),
944 iceTransportPolicy: settings.iceTransportPolicy ?? "all",
945 });
946 // Frames only reach requestVideoFrameCallback from a video that is in the
947 // document; it stays invisible because the sink paints them.
948 const video = document.createElement("video");
949 video.muted = true;
950 video.playsInline = true;
951 video.setAttribute("aria-hidden", "true");
952 video.style.cssText =
953 "position:fixed;width:1px;height:1px;opacity:0;pointer-events:none;left:0;top:0";
954 document.body.append(video);
955 const current = { connection, sessionId: webRtcSessionId(), video };
956 peer = current;
957 const isCurrent = () =>
958 !stopped &&
959 generation === lifecycle &&
960 videoGeneration === feedGeneration &&
961 peer === current;
962 // After the first frame nothing else bounds a feed that stops: the peer
963 // can sit `disconnected` with the last frame on the canvas. A long gap
964 // between frames reconnects through the same path as a failed connection.
965 let checksWithoutFrame = 0;
966 let watchdog: ReturnType<typeof setInterval> | null = null;
967 const clearWatchdog = () => {
968 if (watchdog !== null) clearInterval(watchdog);
969 watchdog = null;
970 };
971 const retry = (detail: string) => {
972 clearWatchdog();
973 if (!isCurrent()) return;
974 closePeer();
975 connecting(detail);
976 scheduleRetry("video", () => void readWebRtc(settings));
977 };
978 const presentFrame = () => {
979 if (!isCurrent()) {
980 clearWatchdog();
981 return;
982 }
983 if (video.videoWidth > 0 && video.videoHeight > 0) {
984 paint(video, video.videoWidth, video.videoHeight);
985 checksWithoutFrame = 0;
986 watchdog ??= setInterval(() => {
987 if (!isCurrent()) return clearWatchdog();
988 checksWithoutFrame++;
989 if (checksWithoutFrame * WEBRTC_STALL_CHECK_MS >= WEBRTC_STALL_TIMEOUT_MS) {
990 retry(resolve(msg("Device stream stalled. Reconnecting…")));
991 }
992 }, WEBRTC_STALL_CHECK_MS);
993 }
994 video.requestVideoFrameCallback(presentFrame);
995 };
996 const transceiver = connection.addTransceiver("video", { direction: "recvonly" });
997 const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? [];
998 if (codecs.length > 0) {
999 // serve-sim encodes H.264; other codecs stay as fallbacks it may refuse.
1000 transceiver.setCodecPreferences([
1001 ...codecs.filter((codec) => codec.mimeType.toLowerCase() === "video/h264"),
1002 ...codecs.filter((codec) => codec.mimeType.toLowerCase() !== "video/h264"),
1003 ]);
1004 }
1005 connection.ontrack = (event) => {
1006 if (!isCurrent()) return;
1007 iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
1008 video.srcObject = event.streams[0] ?? new MediaStream([event.track]);
1009 void video.play().catch(() => {});
1010 video.requestVideoFrameCallback(presentFrame);
1011 };
1012 connection.onconnectionstatechange = () => {
1013 if (connection.connectionState === "failed")
1014 retry(resolve(msg("Device stream connection failed. Reconnecting…")));
1015 };
1016 try {
1017 await connection.setLocalDescription(await connection.createOffer());
1018 const gathered = await new Promise<boolean>((resolve) => {
1019 if (connection.iceGatheringState === "complete") return resolve(true);
1020 const timer = setTimeout(() => done(false), iceGatheringDeadlineMs);
1021 function done(complete: boolean) {
1022 clearTimeout(timer);
1023 connection.removeEventListener("icegatheringstatechange", changed);
1024 resolve(complete);
1025 }
1026 function changed() {
1027 if (connection.iceGatheringState === "complete") done(true);
1028 }
1029 connection.addEventListener("icegatheringstatechange", changed);
1030 });
1031 if (!isCurrent()) return;
1032 // serve-sim takes all candidates in the offer; later candidates never trickle.
1033 // Cloud simulators need a reflexive or relay candidate, so retry incomplete offers with
1034 // longer deadlines up to the bound, then send what was gathered.
1035 const sdp = connection.localDescription?.sdp ?? "";
1036 if (
1037 !gathered &&
1038 !/^a=candidate:.* typ (?:srflx|relay)\b/m.test(sdp) &&
1039 iceGatheringDeadlineMs < ICE_GATHERING_MAX_TIMEOUT_MS
1040 ) {
1041 iceGatheringDeadlineMs = Math.min(iceGatheringDeadlineMs * 2, ICE_GATHERING_MAX_TIMEOUT_MS);
1042 return retry(resolve(msg("Device stream found no network candidates yet. Retrying…")));
1043 }
1044 const response = await fetch(webRtcUrl("offer"), {
1045 method: "POST",
1046 headers: { "content-type": "application/json" },
1047 body: JSON.stringify({
1048 type: "offer",
1049 sdp: connection.localDescription?.sdp,
1050 sessionId: current.sessionId,
1051 codec: "h264",
1052 }),
1053 credentials: access.credentials ? "include" : "same-origin",
1054 });
1055 if (!isCurrent()) {
1056 await response.body?.cancel();
1057 return;
1058 }
1059 if (response.status === 401 || response.status === 403) return handleUnauthorized();
1060 // 409: another viewer is still negotiating with this simulator.
1061 if (!response.ok)
1062 return retry(
1063 resolve(
1064 msg("Device stream offer failed ({status}). Retrying…", { status: response.status }),
1065 ),
1066 );
1067 const answer = decodeWebRtcAnswer(await response.json());
1068 if (!isCurrent()) return;
1069 if (Option.isNone(answer))
1070 return retry(resolve(msg("Device stream returned an invalid answer. Retrying…")));
1071 await connection.setRemoteDescription(answer.value);
1072 } catch (cause) {
1073 retry((cause as Error).message);
1074 }
1075 };
1076
1077 /**
1078 * Called once AVCC or MJPEG was refused. `locked` means serve-sim already
1079 * said it streams WebRTC only, so a missing /api answer still switches.
1080 */
1081 const switchToWebRtc = async (failure: string, locked: boolean) => {
1082 const session = generation;
1083 if (typeof RTCPeerConnection === "undefined") return fail(failure);
1084 const settings = (await probeWebRtc()) ?? (locked ? { transport: "webrtc" as const } : null);
1085 if (stopped || generation !== session) return;
1086 if (!settings) return fail(failure);
1087 if (mjpeg) {
1088 mjpeg = false;
1089 releaseImage?.();
1090 releaseImage = null;
1091 events.onCanvasVideo?.();
1092 }
1093 // MJPEG may have shown frames already: say the stream is connecting again
1094 // rather than leave a blank canvas until the first WebRTC frame.
1095 connecting(resolve(msg("Switching to WebRTC.")));
1096 void readWebRtc(settings);
1097 };
1098
1099 const startDuoVideo = (panels: DuoPanelSinks) => {
1100 for (const panel of panelClients) panel.stop();
1101 // Physical handoff elects a native surface. Fixed-panel encoders can keep
1102 // an inactive shutdown frame after election, so this build uses one active
1103 // feed instead of decoding a third stream alongside the two fixed feeds.
1104 const ids = screen?.supportsPhysicalOrientation ? ([null] as const) : ([1, 3] as const);
1105 panelClients = ids.map((id) => {
1106 const output = id === 1 ? panels.cover : panels.inner;
1107 return createDeviceStreamClient(
1108 { ...target, ...(id === null ? {} : { panelId: id }), videoOnly: true },
1109 {
1110 present(source, width, height) {
1111 if (id === null) {
1112 return sink.present(source, width, height);
1113 }
1114 // An inactive native LCD can emit its shutdown black frame. Retain its last useful image.
1115 if (screen?.screenId !== id) return true;
1116 const retained = output.present(source, width, height);
1117 const primary = sink.present(source, width, height);
1118 return retained && primary;
1119 },
1120 },
1121 {
1122 onStatus: (status, detail) => {
1123 if (status === "error") events.onDuoUnavailable?.(detail);
1124 },
1125 onScreen: () => {},
1126 onInputConnected: () => {},
1127 onMjpegFallback: () => {},
1128 onUnauthorized: handleUnauthorized,
1129 },
1130 );
1131 });
1132 for (const panel of panelClients) panel.start();
1133 };
1134
1135 const connectIosInput = async () => {
1136 if (stopped) return;
1137 const session = generation;
1138 if (!serveSimOnly) await primeIosHelper(session);
1139 if (stopped || generation !== session) return;
1140 const ws = new WebSocket(wsUrl(`/helper/ws?device=${device}`));
1141 ws.binaryType = "arraybuffer";
1142 socket = ws;
1143 iosInputAdmitted = false;
1144 iosInputUnavailable = false;
1145 ws.onmessage = (event) => {
1146 if (stopped || socket !== ws) return;
1147 if (!(event.data instanceof ArrayBuffer)) return;
1148 const bytes = new Uint8Array(event.data);
1149 if (socket !== ws || stopped || bytes.length < 1) return;
1150 if (bytes[0] === IOS_TAG_INPUT_ADMITTED) {
1151 iosInputAdmitted = true;
1152 events.onInputConnected(!iosInputUnavailable);
1153 return;
1154 }
1155 try {
1156 const payload: unknown = JSON.parse(decoder.decode(bytes.subarray(1)));
1157 if (bytes[0] === 0x90) {
1158 const reply = decodeControlReply(payload);
1159 if (Option.isSome(reply)) duoControl.receive(reply.value);
1160 } else if (bytes[0] === IOS_TAG_SCREEN_CONFIG) {
1161 const config = decodeScreenConfig(payload);
1162 if (Option.isSome(config)) {
1163 iosInputUnavailable = config.value.inputUnavailable === true;
1164 events.onInputConnected(
1165 iosInputAdmitted && !iosInputUnavailable,
1166 iosInputUnavailable ? resolve(msg("Simulator input is unavailable")) : undefined,
1167 );
1168 const previous = screen;
1169 screen = config.value;
1170 if (screen.hingePose && screen.hingePose !== previous?.hingePose)
1171 rotationCursor = screen.hingePose === "laptop" ? "landscape_left" : "portrait";
1172 else if (screen.orientation !== previous?.orientation)
1173 rotationCursor = screen.orientation;
1174 panelSinks?.onScreen?.(screen);
1175 events.onScreen(screen);
1176 // A surface election can leave an existing decoder on the former
1177 // encoder description. Reopen only video to acquire the elected
1178 // surface's seed and codec configuration; HID and the viewer stay.
1179 if (
1180 panelSinks &&
1181 screen.supportsPhysicalOrientation &&
1182 previous &&
1183 screen.screenId !== previous.screenId
1184 )
1185 startDuoVideo(panelSinks);
1186 if (pendingOrientation) {
1187 const receipt = pendingOrientation;
1188 pendingOrientation = null;
1189 duoControl.receive({
1190 requestId: receipt.requestId,
1191 ok: true,
1192 });
1193 }
1194 }
1195 }
1196 } catch {}
1197 };
1198 ws.onclose = (event) => {
1199 if (socket !== ws) return;
1200 socket = null;
1201 iosInputAdmitted = false;
1202 duoControl.clear();
1203 rotationCursor = null;
1204 if (stopped) return;
1205 if (event.code === 4001)
1206 return fail(
1207 leaseEnded(
1208 event.reason || resolve(msg("ended", { $context: "device lease end reason" })),
1209 resolve,
1210 ),
1211 );
1212 events.onInputConnected(
1213 false,
1214 event.reason || (event.code === 1006 ? "input socket refused" : `closed ${event.code}`),
1215 );
1216 // A rejected HTTP upgrade surfaces as 1006, including an expired stream ticket.
1217 if (
1218 event.code === 1008 ||
1219 event.code === 4401 ||
1220 (event.code === 1006 && access.query.wsTicket)
1221 )
1222 return handleUnauthorized();
1223 scheduleRetry("input", () => void connectIosInput());
1224 };
1225 ws.onerror = () => ws.close();
1226 };
1227
1228 /** `inputOnly` keeps a socket for input beside a WebRTC feed, so its reconnects leave video alone. */
1229 const connectAndroid = (inputOnly = false) => {
1230 if (stopped) return;
1231 const ws = new WebSocket(
1232 wsUrl(`/ws?device=${device}&${inputOnly ? "video=0" : "frame-meta=1"}`),
1233 );
1234 ws.binaryType = "arraybuffer";
1235 socket = ws;
1236 ws.onopen = () => {
1237 if (stopped || socket !== ws) return;
1238 if (!inputOnly) connecting();
1239 events.onInputConnected(true);
1240 };
1241 ws.onmessage = (event) => {
1242 if (stopped || socket !== ws || inputOnly) return;
1243 if (typeof event.data === "string") {
1244 // The encoder restarts at a new size when the device rotates; the
1245 // next keyframe carries a fresh SPS, so the decoder is rebuilt from it.
1246 if (isVideoSessionMessage(event.data)) {
1247 closeDecoder();
1248 configuring = false;
1249 connecting();
1250 requestKeyframe();
1251 }
1252 return;
1253 }
1254 if (!(event.data instanceof ArrayBuffer)) return;
1255 const packet = parseSemuPacket(event.data);
1256 const needsScan =
1257 packet.isKey === null ||
1258 (packet.isKey && (!videoDecoder || videoDecoder.state !== "configured"));
1259 const scanned = needsScan ? scanAccessUnit(packet.data) : null;
1260 const isKey = packet.isKey ?? scanned?.isKey ?? false;
1261 if (scanned?.sps && (!videoDecoder || videoDecoder.state !== "configured")) {
1262 if (configuring) return;
1263 configuring = true;
1264 const epoch = decoderEpoch;
1265 const isCurrent = () => !stopped && socket === ws;
1266 void configureDecoder({ codec: avcCodecString(scanned.sps) }, isCurrent).then(
1267 (configured) => {
1268 if (!isCurrent() || epoch !== decoderEpoch) return;
1269 configuring = false;
1270 awaitingKeyframe = true;
1271 if (configured) requestKeyframe();
1272 },
1273 );
1274 return;
1275 }
1276 if (!videoDecoder || videoDecoder.state !== "configured") {
1277 if (!isKey) requestKeyframe();
1278 return;
1279 }
1280 decode(isKey, packet.data, packet.timestamp);
1281 };
1282 ws.onclose = (event) => {
1283 if (socket !== ws) return;
1284 socket = null;
1285 closeDecoder();
1286 if (stopped) return;
1287 events.onInputConnected(false, event.reason || `closed ${event.code}`);
1288 if (
1289 event.code === 1008 ||
1290 event.code === 4401 ||
1291 (event.code === 1006 && access.query.wsTicket)
1292 )
1293 return handleUnauthorized();
1294 if (!inputOnly) {
1295 configuring = false;
1296 connecting(event.reason || undefined);
1297 }
1298 // A full socket re-probes, so one failed /api read does not pin SEMU on a WebRTC host.
1299 scheduleRetry("input", () => (inputOnly ? connectAndroid(true) : void startAndroid()));
1300 };
1301 ws.onerror = () => ws.close();
1302 };
1303
1304 const startAndroid = async () => {
1305 const session = generation;
1306 const settings = await probeWebRtc();
1307 if (stopped || generation !== session) return;
1308 if (settings) {
1309 if (typeof RTCPeerConnection === "undefined")
1310 return fail(resolve(msg("This browser cannot receive the device's WebRTC stream.")));
1311 if (!target.videoOnly) connectAndroid(true);
1312 void readWebRtc(settings);
1313 } else if (useWebCodecs) connectAndroid();
1314 else
1315 fail(resolve(msg("This browser cannot decode the Android stream (WebCodecs unavailable).")));
1316 };
1317
1318 const start = () => {
1319 if (!stopped) return;
1320 stopped = false;
1321 generation++;
1322 configuring = false;
1323 iosInputAdmitted = false;
1324 iosInputUnavailable = false;
1325 connecting();
1326 if (serveSimOnly) {
1327 if (!target.videoOnly) void connectIosInput();
1328 if (useWebCodecs) void readIosVideo();
1329 else
1330 fail(
1331 resolve(msg("This browser cannot decode this device's stream (WebCodecs unavailable).")),
1332 );
1333 } else if (platform === "ios") {
1334 if (!target.videoOnly) void connectIosInput();
1335 if (useWebCodecs) void readIosVideo();
1336 else fallBackToMjpeg();
1337 } else void startAndroid();
1338 };
1339
1340 const stop = () => {
1341 if (stopped) return;
1342 stopped = true;
1343 generation++;
1344 videoGeneration++;
1345 duoControl.clear();
1346 rotationCursor = null;
1347 for (const panel of panelClients) panel.stop();
1348 panelClients = [];
1349 panelSinks = null;
1350 mjpeg = false;
1351 clearFrameTimer();
1352 if (videoReadTimer !== null) clearTimeout(videoReadTimer);
1353 videoReadTimer = null;
1354 releaseImage?.();
1355 releaseImage = null;
1356 mjpegImage = null;
1357 for (const timer of retryTimers.values()) clearTimeout(timer);
1358 retryTimers.clear();
1359 primeController?.abort();
1360 primeController = null;
1361 controller?.abort();
1362 controller = null;
1363 closePeer();
1364 const discarded = socket;
1365 socket = null;
1366 discarded?.close();
1367 closeDecoder();
1368 };
1369
1370 const send = (payload: Uint8Array<ArrayBuffer> | string) => {
1371 if (
1372 !stopped &&
1373 (!avcc || (iosInputAdmitted && !iosInputUnavailable)) &&
1374 socket?.readyState === WebSocket.OPEN
1375 )
1376 socket.send(payload);
1377 };
1378
1379 const rawPoint = (x: number, y: number) => {
1380 // serve-sim streams the raw framebuffer; rotated devices need input
1381 // remapped into that raw space.
1382 if (platform !== "ios" || !screen || screen.width > screen.height) return { x, y };
1383 switch (screen.orientation) {
1384 case "landscape_left":
1385 return { x: y, y: 1 - x };
1386 case "landscape_right":
1387 return { x: 1 - y, y: x };
1388 case "portrait_upside_down":
1389 return { x: 1 - x, y: 1 - y };
1390 default:
1391 return { x, y };
1392 }
1393 };
1394
1395 return {
1396 start,
1397 stop,
1398 setMjpegImage,
1399 controlDuo: duoControl.enqueue,
1400 sendRawTouch: (phase, x, y) => {
1401 if (platform === "ios") send(taggedJson(IOS_MSG_TOUCH, { type: phase, x, y }));
1402 },
1403 setDuoPanels(panels) {
1404 if (platform !== "ios" || target.videoOnly || stopped || panelSinks === panels) return;
1405 if (panels && !screen?.supportsHingeAngle) return;
1406 panelSinks = panels;
1407 videoGeneration++;
1408 controller?.abort();
1409 controller = null;
1410 closePeer();
1411 closeDecoder();
1412 const retry = retryTimers.get("video");
1413 if (retry) clearTimeout(retry);
1414 retryTimers.delete("video");
1415 for (const panel of panelClients) panel.stop();
1416 panelClients = [];
1417 if (!panels) {
1418 if (useWebCodecs) void readIosVideo();
1419 return;
1420 }
1421 startDuoVideo(panels);
1422 },
1423 sendTouch: (phase, x, y) => {
1424 if (avcc) {
1425 send(taggedJson(IOS_MSG_TOUCH, { type: phase, ...rawPoint(x, y) }));
1426 return;
1427 }
1428 const action = phase === "begin" ? "down" : phase === "move" ? "move" : "up";
1429 send(JSON.stringify({ type: "touch", action, x, y }));
1430 },
1431 sendKey: (event, phase) => {
1432 if (avcc) {
1433 const usage = hidUsageForCode(event.code);
1434 if (usage !== null) send(taggedJson(IOS_MSG_KEY, { type: phase, usage }));
1435 return;
1436 }
1437 if (phase !== "down") return;
1438 if (event.key === "Escape") return send(JSON.stringify({ type: "back" }));
1439 const keycode = ANDROID_KEYCODE_BY_KEY[event.key];
1440 if (keycode !== undefined) return send(JSON.stringify({ type: "key", keycode }));
1441 if (event.key.length === 1 && !event.metaKey && !event.ctrlKey) {
1442 send(JSON.stringify({ type: "text", text: event.key }));
1443 }
1444 },
1445 sendText: (text) => {
1446 if (avcc) return;
1447 let chunk = "";
1448 let chunkBytes = 0;
1449 let sentChunks = 0;
1450 for (const character of text) {
1451 const characterBytes = utf8ByteLength(character.codePointAt(0)!);
1452 if (chunkBytes + characterBytes > ANDROID_MAX_TEXT_BYTES) {
1453 send(JSON.stringify({ type: "text", text: chunk }));
1454 if (++sentChunks >= ANDROID_MAX_TEXT_CHUNKS) return;
1455 chunk = "";
1456 chunkBytes = 0;
1457 }
1458 chunk += character;
1459 chunkBytes += characterBytes;
1460 }
1461 if (chunk.length > 0) send(JSON.stringify({ type: "text", text: chunk }));
1462 },
1463 pressButton: (button) => {
1464 if (avcc) {
1465 const name =
1466 button === "home"
1467 ? "home"
1468 : button === "appSwitcher" || button === "recents"
1469 ? "app_switcher"
1470 : button === "power"
1471 ? "lock"
1472 : button === "back" && platform === "android"
1473 ? "back"
1474 : null;
1475 if (name) send(taggedJson(IOS_MSG_BUTTON, { button: name }));
1476 return;
1477 }
1478 const type = button === "appSwitcher" ? "recents" : button;
1479 if (type === "home" || type === "back" || type === "recents" || type === "power") {
1480 send(JSON.stringify({ type }));
1481 }
1482 },
1483 rotate: () => {
1484 if (platform !== "ios") return;
1485 const current = screen?.supportsHingeAngle
1486 ? (rotationCursor ?? screen.orientation)
1487 : (screen?.orientation ?? "portrait");
1488 const next =
1489 IOS_ORIENTATIONS[(IOS_ORIENTATIONS.indexOf(current) + 1) % IOS_ORIENTATIONS.length]!;
1490 if (screen?.supportsHingeAngle) {
1491 rotationCursor = next;
1492 duoControl.enqueue({ control: "orientation", value: next });
1493 } else send(taggedJson(IOS_MSG_ORIENTATION, { orientation: next }));
1494 },
1495 setOrientation: (orientation) => {
1496 if (platform === "ios") send(taggedJson(IOS_MSG_ORIENTATION, { orientation }));
1497 },
1498 };
1499 }