@devries-cameron20 / t3code Mirror · read-only / commits / 6dc63bd85b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 22:00 UTC

feat(device): stream iOS devices that serve WebRTC only (#21841)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 20, 2026 at 00:57 UTC 6dc63bd85bf6930191e4ee858fdc7d940ab5bd5d
7 files changed +592 -2
apps/mobile/src/features/devices/device-stream.browser.ts
+4
@@ -140,6 +140,10 @@ export function start(configuration: DeviceStreamConfiguration) {
140 canvas.style.display = "none";
141 image.style.display = "block";
142 },
143 + onCanvasVideo: () => {
144 + image.style.display = "none";
145 + canvas.style.display = "";
146 + },
147 onUnauthorized: unauthorized,
148 onInputConnected: (connected) => {
149 inputConnected = connected;
apps/server/src/device/DeviceHubProxy.test.ts
+15
@@ -152,6 +152,21 @@ describe("device hub proxy", () => {
152 expect(headers[0]).not.toHaveProperty("x-forwarded-proto");
153 });
154
155 + it("lets a viewer negotiate a WebRTC stream with read scope", async () => {
156 + const { handler, requests, headers } = fixture([AuthOrchestrationReadScope]);
157 + const offer = await handler(
158 + new Request("http://t3.test/api/device-hub/vendor/serve-sim/helper/udid-1/webrtc/offer", {
159 + method: "POST",
160 + headers: { "content-type": "application/json" },
161 + body: '{"type":"offer"}',
162 + }),
163 + );
164 + expect(offer.status).toBe(200);
165 + await offer.text();
166 + expect(requests).toEqual(["http://hub.test/vendor/serve-sim/helper/udid-1/webrtc/offer"]);
167 + expect(headers[0]?.["content-type"]).toBe("application/json");
168 + });
169 +
170 it("never forwards the vendor shell endpoint", async () => {
171 const { handler, requests } = fixture([AuthOrchestrationOperateScope]);
172 expect(
apps/server/src/device/DeviceHubProxy.ts
+8 -1
@@ -28,6 +28,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
28 /^\/vendor\/serve-sim\/api\/event-log(\/events)?$/,
29 /^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
30 /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
31 + /^\/vendor\/serve-sim\/helper\/[^/]+\/webrtc\/(offer|close)$/,
32 /^\/vendor\/serve-sim\/appstate$/,
33 /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/,
34 /^\/vendor\/serve-emu\/health$/,
@@ -35,6 +36,8 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
36
37 const MUTABLE_PATHS: ReadonlyArray<RegExp> = [
38 /^\/vendor\/serve-sim\/api\/screenshot$/,
39 + // WebRTC signalling: opening and closing a view of the stream, so read scope.
40 + /^\/vendor\/serve-sim\/helper\/[^/]+\/webrtc\/(offer|close)$/,
41 /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/,
42 /^\/vendor\/serve-emu\/api\/fold$/,
43 ];
@@ -120,7 +123,11 @@ const proxyHttp = Effect.fn("DeviceHubProxy.proxyHttp")(function* (
123 HttpClientRequest.setHeaders(forwardHeaders(request, hubOrigin)),
124 method === "GET" || method === "HEAD"
125 ? (self) => self
123 - : HttpClientRequest.bodyStream(request.stream),
126 + : // A bare stream body is sent as octet-stream; serve-sim refuses WebRTC
127 + // offers that are not application/json.
128 + HttpClientRequest.bodyStream(request.stream, {
129 + contentType: request.headers["content-type"],
130 + }),
131 );
132 const response = yield* httpClient.execute(upstreamRequest);
133 const headers: Record<string, string> = {};
apps/web/src/components/device/DeviceStreamView.tsx
+1
@@ -164,6 +164,7 @@ export function DeviceStreamView(props: {
164 setMjpegUrl(url);
165 setMjpegGeneration((generation) => generation + 1);
166 },
167 + onCanvasVideo: () => setMjpegUrl(null),
168 onInputConnected: (connected, detail) => {
169 setInputState({ connected, ...(detail ? { detail } : {}) });
170 if (!connected) setShowRestartNotice(false);
docs/internals/devices.md
+7
@@ -91,3 +91,10 @@ Simulators encode H.264 High 5.1. Hardware decoders on some machines and all
91 headless browsers reject that profile, and WebCodecs is secure-context only, so
92 the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on
93 iOS. Android has no MJPEG; there the panel reports that it cannot decode.
94 +
95 +serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses
96 +AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so
97 +hubs that serve it pay nothing, and switches on that 409, or when its MJPEG
98 +image fails in a browser that has `RTCPeerConnection`. Offers are not trickled:
99 +serve-sim takes one offer with every candidate. The video element only feeds
100 +the same canvas sink, so the panel and the mobile WebView keep one surface.
packages/client-runtime/src/device/stream.test.ts
+277
@@ -923,3 +923,280 @@ it("reopens a stalled AVCC body without disconnecting input, then receives fresh
923 vi.unstubAllGlobals();
924 }
925 });
926 +
927 +describe("WebRTC-only iOS devices", () => {
928 + afterEach(() => {
929 + vi.useRealTimers();
930 + vi.unstubAllGlobals();
931 + });
932 +
933 + function webRtcFixture(options: {
934 + webCodecs: boolean;
935 + offerStatuses?: number[];
936 + /** The fake peer's ICE gathering state; `gathering` never completes. */
937 + iceGatheringState?: "complete" | "gathering";
938 + }) {
939 + vi.useFakeTimers();
940 + if (options.webCodecs) {
941 + vi.stubGlobal("VideoDecoder", vi.fn());
942 + vi.stubGlobal("EncodedVideoChunk", vi.fn());
943 + }
944 + class FakeSocket {
945 + static OPEN = 1;
946 + readyState = 1;
947 + binaryType = "";
948 + onmessage: unknown = null;
949 + onclose: unknown = null;
950 + onerror: unknown = null;
951 + send = vi.fn();
952 + close = vi.fn();
953 + }
954 + vi.stubGlobal("WebSocket", FakeSocket);
955 + const peers: FakePeer[] = [];
956 + class FakePeer {
957 + iceGatheringState = options.iceGatheringState ?? "complete";
958 + connectionState = "new";
959 + localDescription: { type: string; sdp: string } | null = null;
960 + remoteDescription: unknown = null;
961 + ontrack: ((event: { streams: unknown[]; track: unknown }) => void) | null = null;
962 + onconnectionstatechange: (() => void) | null = null;
963 + closed = false;
964 + readonly config: unknown;
965 + readonly codecPreferences = vi.fn();
966 + constructor(config: unknown) {
967 + this.config = config;
968 + peers.push(this);
969 + }
970 + addTransceiver() {
971 + return { setCodecPreferences: this.codecPreferences };
972 + }
973 + async createOffer() {
974 + return { type: "offer", sdp: "v=0 offer" };
975 + }
976 + async setLocalDescription(description: { type: string; sdp: string }) {
977 + this.localDescription = description;
978 + }
979 + async setRemoteDescription(description: unknown) {
980 + this.remoteDescription = description;
981 + }
982 + addEventListener() {}
983 + removeEventListener() {}
984 + close() {
985 + this.closed = true;
986 + }
987 + }
988 + vi.stubGlobal("RTCPeerConnection", FakePeer);
989 + vi.stubGlobal("RTCRtpReceiver", {
990 + getCapabilities: () => ({ codecs: [{ mimeType: "video/VP8" }, { mimeType: "video/H264" }] }),
991 + });
992 + const frameCallbacks: Array<() => void> = [];
993 + const video = {
994 + muted: false,
995 + playsInline: false,
996 + srcObject: null as unknown,
997 + videoWidth: 402,
998 + videoHeight: 874,
999 + style: { cssText: "" },
1000 + setAttribute: vi.fn(),
1001 + remove: vi.fn(),
1002 + play: vi.fn(() => Promise.resolve()),
1003 + requestVideoFrameCallback: vi.fn((callback: () => void) => {
1004 + frameCallbacks.push(callback);
1005 + }),
1006 + };
1007 + vi.stubGlobal("document", { createElement: () => video, body: { append: vi.fn() } });
1008 + const offerStatuses = [...(options.offerStatuses ?? [])];
1009 + const fetch = vi.fn((url: string) => {
1010 + const path = new URL(url).pathname;
1011 + if (path.endsWith("/stream.avcc"))
1012 + return Promise.resolve(
1013 + Response.json({ error: "stream_transport_locked", transport: "webrtc" }, { status: 409 }),
1014 + );
1015 + if (path.endsWith("/vendor/serve-sim/api"))
1016 + return Promise.resolve(
1017 + Response.json({
1018 + streamSettings: { transport: "webrtc", iceServers: [{ urls: ["stun:stun.test"] }] },
1019 + }),
1020 + );
1021 + if (path.endsWith("/webrtc/offer")) {
1022 + const status = offerStatuses.shift() ?? 200;
1023 + return Promise.resolve(
1024 + status === 200
1025 + ? Response.json({ type: "answer", sdp: "v=0 answer" })
1026 + : Response.json({ error: "webrtc_session_busy" }, { status }),
1027 + );
1028 + }
1029 + if (path.endsWith("/webrtc/close"))
1030 + return Promise.resolve(new Response(null, { status: 204 }));
1031 + return Promise.resolve(new Response("refused", { status: 409 }));
1032 + });
1033 + vi.stubGlobal("fetch", fetch);
1034 + const presented: Array<[unknown, number, number]> = [];
1035 + const events = {
1036 + onStatus: vi.fn(),
1037 + onScreen: vi.fn(),
1038 + onUnauthorized: vi.fn(),
1039 + onMjpegFallback: vi.fn(),
1040 + onCanvasVideo: vi.fn(),
1041 + onInputConnected: vi.fn(),
1042 + };
1043 + const client = createDeviceStreamClient(
1044 + {
1045 + platform: "ios",
1046 + deviceId: "udid-1",
1047 + preferMjpeg: !options.webCodecs,
1048 + access: {
1049 + httpBase: "https://environment.test/api/device-hub",
1050 + wsBase: "wss://environment.test/api/device-hub",
1051 + credentials: false,
1052 + query: { hostId: "eas" },
1053 + },
1054 + },
1055 + {
1056 + present: (source, width, height) => {
1057 + presented.push([source, width, height]);
1058 + return true;
1059 + },
1060 + },
1061 + events,
1062 + );
1063 + const requests = (suffix: string) =>
1064 + fetch.mock.calls.filter(([url]) => new URL(url).pathname.endsWith(suffix));
1065 + const bodyOf = (call: unknown[]) =>
1066 + JSON.parse(String((call[1] as RequestInit | undefined)?.body)) as Record<string, unknown>;
1067 + const settle = async () => {
1068 + for (let turn = 0; turn < 20; turn++) await vi.advanceTimersByTimeAsync(0);
1069 + };
1070 + return { client, peers, video, frameCallbacks, presented, events, requests, bodyOf, settle };
1071 + }
1072 +
1073 + it("switches to WebRTC when serve-sim refuses AVCC and releases the session on stop", async () => {
1074 + const fixture = webRtcFixture({ webCodecs: true });
1075 + fixture.client.start();
1076 + await fixture.settle();
1077 + expect(fixture.peers).toHaveLength(1);
1078 + const peer = fixture.peers[0]!;
1079 + expect(peer.config).toEqual({
1080 + iceServers: [{ urls: ["stun:stun.test"] }],
1081 + iceTransportPolicy: "all",
1082 + });
1083 + expect(peer.codecPreferences).toHaveBeenCalledWith([
1084 + { mimeType: "video/H264" },
1085 + { mimeType: "video/VP8" },
1086 + ]);
1087 + const offers = fixture.requests("/helper/udid-1/webrtc/offer");
1088 + expect(offers).toHaveLength(1);
1089 + const offer = fixture.bodyOf(offers[0]!);
1090 + expect(offer).toMatchObject({ type: "offer", sdp: "v=0 offer", codec: "h264" });
1091 + expect(offer.sessionId).toMatch(
1092 + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
1093 + );
1094 + expect(peer.remoteDescription).toEqual({ type: "answer", sdp: "v=0 answer" });
1095 +
1096 + peer.ontrack?.({ streams: [{}], track: {} });
1097 + fixture.frameCallbacks.shift()?.();
1098 + expect(fixture.presented).toEqual([[fixture.video, 402, 874]]);
1099 + expect(fixture.events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
1100 +
1101 + fixture.client.stop();
1102 + expect(peer.closed).toBe(true);
1103 + expect(fixture.video.remove).toHaveBeenCalled();
1104 + const closes = fixture.requests("/helper/udid-1/webrtc/close");
1105 + expect(closes.map(fixture.bodyOf)).toEqual([{ sessionId: offer.sessionId }]);
1106 + });
1107 +
1108 + it("reconnects a WebRTC feed that stops producing frames", async () => {
1109 + const fixture = webRtcFixture({ webCodecs: true });
1110 + fixture.client.start();
1111 + await fixture.settle();
1112 + const peer = fixture.peers[0]!;
1113 + peer.ontrack?.({ streams: [{}], track: {} });
1114 + fixture.frameCallbacks.shift()?.();
1115 + expect(fixture.events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
1116 + // Frames keep coming: nothing happens.
1117 + await vi.advanceTimersByTimeAsync(10_000);
1118 + fixture.frameCallbacks.shift()?.();
1119 + await vi.advanceTimersByTimeAsync(10_000);
1120 + expect(fixture.peers).toHaveLength(1);
1121 + expect(peer.closed).toBe(false);
1122 + // Then none for the first-frame deadline (checked every 5 s): the feed
1123 + // reconnects through the usual retry path.
1124 + await vi.advanceTimersByTimeAsync(15_000);
1125 + expect(peer.closed).toBe(true);
1126 + expect(fixture.events.onStatus).toHaveBeenLastCalledWith(
1127 + "connecting",
1128 + "Device stream stalled. Reconnecting…",
1129 + );
1130 + await vi.advanceTimersByTimeAsync(1_000);
1131 + await fixture.settle();
1132 + expect(fixture.peers).toHaveLength(2);
1133 + expect(fixture.peers[1]!.remoteDescription).toEqual({ type: "answer", sdp: "v=0 answer" });
1134 + fixture.client.stop();
1135 + });
1136 +
1137 + it("retries instead of offering when ICE gathering times out without a candidate", async () => {
1138 + const fixture = webRtcFixture({ webCodecs: true, iceGatheringState: "gathering" });
1139 + fixture.client.start();
1140 + await fixture.settle();
1141 + expect(fixture.peers).toHaveLength(1);
1142 + expect(fixture.requests("/helper/udid-1/webrtc/offer")).toHaveLength(0);
1143 + // The gathering deadline passes with no reflexive or relay candidate: no
1144 + // offer is sent, the attempt is retried with twice the deadline.
1145 + await vi.advanceTimersByTimeAsync(2_000);
1146 + expect(fixture.peers[0]!.closed).toBe(true);
1147 + expect(fixture.requests("/helper/udid-1/webrtc/offer")).toHaveLength(0);
1148 + expect(fixture.events.onStatus).toHaveBeenLastCalledWith(
1149 + "connecting",
1150 + "Device stream found no network candidates yet. Retrying…",
1151 + );
1152 + await vi.advanceTimersByTimeAsync(1_000);
1153 + await fixture.settle();
1154 + expect(fixture.peers).toHaveLength(2);
1155 + await vi.advanceTimersByTimeAsync(2_000);
1156 + expect(fixture.peers[1]!.closed).toBe(false);
1157 + // The second deadline, 4 s, is the bound: the offer goes as gathered.
1158 + await vi.advanceTimersByTimeAsync(3_000);
1159 + await fixture.settle();
1160 + expect(fixture.peers).toHaveLength(2);
1161 + expect(fixture.peers[1]!.closed).toBe(false);
1162 + expect(fixture.requests("/helper/udid-1/webrtc/offer")).toHaveLength(1);
1163 + fixture.client.stop();
1164 + });
1165 +
1166 + it("retries a WebRTC offer while another viewer is negotiating", async () => {
1167 + const fixture = webRtcFixture({ webCodecs: true, offerStatuses: [409] });
1168 + fixture.client.start();
1169 + await fixture.settle();
1170 + expect(fixture.peers).toHaveLength(1);
1171 + expect(fixture.peers[0]!.closed).toBe(true);
1172 + await vi.advanceTimersByTimeAsync(1_000);
1173 + await fixture.settle();
1174 + expect(fixture.peers).toHaveLength(2);
1175 + expect(fixture.peers[1]!.remoteDescription).toEqual({ type: "answer", sdp: "v=0 answer" });
1176 + fixture.client.stop();
1177 + });
1178 +
1179 + it("moves an MJPEG viewer back to the canvas for a WebRTC-only device", async () => {
1180 + const fixture = webRtcFixture({ webCodecs: false });
1181 + const listeners: Record<string, () => void> = {};
1182 + const image = {
1183 + naturalWidth: 0,
1184 + naturalHeight: 0,
1185 + src: "",
1186 + addEventListener: (type: string, listener: () => void) => {
1187 + listeners[type] = listener;
1188 + },
1189 + removeEventListener: vi.fn(),
1190 + removeAttribute: vi.fn(),
1191 + };
1192 + fixture.client.start();
1193 + fixture.client.setMjpegImage(image as unknown as HTMLImageElement);
1194 + expect(fixture.events.onMjpegFallback).toHaveBeenCalledOnce();
1195 + listeners.error?.();
1196 + await fixture.settle();
1197 + expect(fixture.events.onCanvasVideo).toHaveBeenCalledOnce();
1198 + expect(fixture.peers).toHaveLength(1);
1199 + expect(fixture.requests("/helper/udid-1/webrtc/offer")).toHaveLength(1);
1200 + fixture.client.stop();
1201 + });
1202 +});
packages/client-runtime/src/device/stream.ts
+280 -1
@@ -57,6 +57,33 @@ const controlReplySchema = Schema.Struct({
57 });
58 const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema);
59 const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema);
60 +const iceServerSchema = Schema.Struct({
61 + urls: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
62 + username: Schema.optionalKey(Schema.String),
63 + credential: Schema.optionalKey(Schema.String),
64 +});
65 +const webRtcSettingsSchema = Schema.Struct({
66 + transport: Schema.Literal("webrtc"),
67 + iceServers: Schema.optionalKey(Schema.Array(iceServerSchema)),
68 + iceTransportPolicy: Schema.optionalKey(Schema.Literals(["all", "relay"])),
69 +});
70 +// serve-sim reports its stream settings as `streamSettings` (EAS) or `stream`.
71 +const serveSimApiSchema = Schema.Struct({
72 + streamSettings: Schema.optionalKey(Schema.Unknown),
73 + stream: Schema.optionalKey(Schema.Unknown),
74 +});
75 +const decodeServeSimApi = Schema.decodeUnknownOption(serveSimApiSchema);
76 +const decodeWebRtcSettings = Schema.decodeUnknownOption(webRtcSettingsSchema);
77 +type WebRtcSettings = typeof webRtcSettingsSchema.Type;
78 +const decodeTransportLocked = Schema.decodeUnknownOption(
79 + Schema.Struct({
80 + error: Schema.Literal("stream_transport_locked"),
81 + transport: Schema.Literal("webrtc"),
82 + }),
83 +);
84 +const decodeWebRtcAnswer = Schema.decodeUnknownOption(
85 + Schema.Struct({ type: Schema.Literal("answer"), sdp: Schema.String }),
86 +);
87
88 export interface DuoPanelSinks {
89 readonly cover: DeviceFrameSink;
@@ -77,6 +104,11 @@ export interface DeviceStreamEvents {
104 * and attach it with `setMjpegImage` so the client can observe real frames.
105 */
106 readonly onMjpegFallback: (url: string) => void;
107 + /**
108 + * The stream moved from MJPEG back to painting the canvas, because the
109 + * device serves WebRTC only; the owner should show the canvas again.
110 + */
111 + readonly onCanvasVideo?: () => void;
112 readonly onInputConnected: (connected: boolean, detail?: string) => void;
113 }
114
@@ -94,6 +126,19 @@ export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appS
126
127 const RETRY_DELAY_MS = 1_000;
128 const FIRST_FRAME_TIMEOUT_MS = 15_000;
129 +// A WebRTC feed that shows no frame for this long is reconnected; checked
130 +// every WEBRTC_STALL_CHECK_MS, so a transient `disconnected` peer that
131 +// recovers is left alone.
132 +const WEBRTC_STALL_TIMEOUT_MS = FIRST_FRAME_TIMEOUT_MS;
133 +const WEBRTC_STALL_CHECK_MS = 5_000;
134 +const TRANSPORT_PROBE_TIMEOUT_MS = 5_000;
135 +// Offers carry every candidate: serve-sim does not accept trickled ICE.
136 +const ICE_GATHERING_TIMEOUT_MS = 2_000;
137 +// A gathering cut short without a reflexive or relay candidate is retried
138 +// with twice the deadline, up to this; then the offer goes as gathered. The
139 +// whole sequence stays inside the first-frame deadline, which retries do
140 +// not extend.
141 +const ICE_GATHERING_MAX_TIMEOUT_MS = 4_000;
142 const MJPEG_FRAME_CHECK_MS = 250;
143 const FRAME_DURATION_US = 16_667;
144 const SEMU_MAGIC = 0x53454d55;
@@ -115,6 +160,15 @@ const IOS_TAG_INPUT_ADMITTED = 0x83;
160 const encoder = new TextEncoder();
161 const decoder = new TextDecoder();
162
163 +/** serve-sim requires a v4 UUID; `crypto.randomUUID` is missing over plain-HTTP LAN origins. */
164 +function webRtcSessionId(): string {
165 + const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16));
166 + bytes[6] = (bytes[6]! & 0x0f) | 0x40;
167 + bytes[8] = (bytes[8]! & 0x3f) | 0x80;
168 + const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
169 + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
170 +}
171 +
172 const isWebCodecsSupported = (): boolean =>
173 typeof globalThis !== "undefined" &&
174 "VideoDecoder" in globalThis &&
@@ -367,6 +421,9 @@ export function createDeviceStreamClient(
421 let panelSinks: DuoPanelSinks | null = null;
422 let rotationCursor: DeviceScreenSize["orientation"] | null = null;
423 let pendingOrientation: { requestId: number } | null = null;
424 + let iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
425 + let peer: { connection: RTCPeerConnection; sessionId: string; video: HTMLVideoElement } | null =
426 + null;
427 const duoControl = createDuoControl({
428 send(request) {
429 if (
@@ -454,7 +511,11 @@ export function createDeviceStreamClient(
511 }
512 };
513 const error = () => {
457 - if (!released) fail("Could not receive the device stream. Reconnect to try again.");
514 + if (released) return;
515 + const failure = "Could not receive the device stream. Reconnect to try again.";
516 + // An <img> cannot read the 409 that a WebRTC-only serve-sim answers with.
517 + if (typeof RTCPeerConnection === "undefined") fail(failure);
518 + else void switchToWebRtc(failure, false);
519 };
520 image.addEventListener("load", check);
521 image.addEventListener("error", error);
@@ -646,6 +707,15 @@ export function createDeviceStreamClient(
707 return;
708 }
709 if (response.status === 401 || response.status === 403) return handleUnauthorized();
710 + if (response.status === 409 && !target.panelId) {
711 + const locked = decodeTransportLocked(await response.json().catch(() => null));
712 + if (Option.isSome(locked)) {
713 + if (!isCurrent()) return;
714 + controller = null;
715 + void switchToWebRtc("This browser cannot play this device's WebRTC stream.", true);
716 + return;
717 + }
718 + }
719 if (target.panelId && [400, 404, 405, 410].includes(response.status)) {
720 await response.body?.cancel();
721 setStatus("error", "This Device Hub does not provide fixed Duo display feeds.");
@@ -753,6 +823,213 @@ export function createDeviceStreamClient(
823 }
824 };
825
826 + /**
827 + * serve-sim can be locked to WebRTC (EAS Simulator always is), and then
828 + * refuses AVCC and MJPEG. An unreadable answer keeps the AVCC path.
829 + */
830 + const probeIosTransport = async (): Promise<WebRtcSettings | null> => {
831 + const probe = new AbortController();
832 + const timeout = setTimeout(() => probe.abort(), TRANSPORT_PROBE_TIMEOUT_MS);
833 + try {
834 + const response = await fetch(httpUrl(`/api?device=${device}`), {
835 + signal: probe.signal,
836 + credentials: access.credentials ? "include" : "same-origin",
837 + });
838 + if (!response.ok) return null;
839 + const api = decodeServeSimApi(await response.json());
840 + if (Option.isNone(api)) return null;
841 + const settings = decodeWebRtcSettings(api.value.streamSettings ?? api.value.stream);
842 + return Option.isSome(settings) ? settings.value : null;
843 + } catch {
844 + return null;
845 + } finally {
846 + clearTimeout(timeout);
847 + }
848 + };
849 +
850 + const closePeer = () => {
851 + const current = peer;
852 + if (!current) return;
853 + peer = null;
854 + current.connection.close();
855 + current.video.srcObject = null;
856 + current.video.remove();
857 + // Releases serve-sim's capture session; best effort because the page may be unloading.
858 + void fetch(httpUrl(`/helper/${device}/webrtc/close`), {
859 + method: "POST",
860 + keepalive: true,
861 + headers: { "content-type": "application/json" },
862 + body: JSON.stringify({ sessionId: current.sessionId }),
863 + credentials: access.credentials ? "include" : "same-origin",
864 + }).catch(() => {});
865 + };
866 +
867 + const readIosWebRtc = async (settings: WebRtcSettings) => {
868 + const lifecycle = generation;
869 + const feedGeneration = ++videoGeneration;
870 + closePeer();
871 + const connection = new RTCPeerConnection({
872 + ...(settings.iceServers
873 + ? {
874 + iceServers: settings.iceServers.map((server) => ({
875 + ...server,
876 + urls: typeof server.urls === "string" ? server.urls : [...server.urls],
877 + })),
878 + }
879 + : {}),
880 + iceTransportPolicy: settings.iceTransportPolicy ?? "all",
881 + });
882 + // Frames only reach requestVideoFrameCallback from a video that is in the
883 + // document; it stays invisible because the sink paints them.
884 + const video = document.createElement("video");
885 + video.muted = true;
886 + video.playsInline = true;
887 + video.setAttribute("aria-hidden", "true");
888 + video.style.cssText =
889 + "position:fixed;width:1px;height:1px;opacity:0;pointer-events:none;left:0;top:0";
890 + document.body.append(video);
891 + const current = { connection, sessionId: webRtcSessionId(), video };
892 + peer = current;
893 + const isCurrent = () =>
894 + !stopped &&
895 + generation === lifecycle &&
896 + videoGeneration === feedGeneration &&
897 + peer === current;
898 + // After the first frame nothing else bounds a feed that stops: the peer
899 + // can sit `disconnected` with the last frame on the canvas. A long gap
900 + // between frames reconnects through the same path as a failed connection.
901 + let checksWithoutFrame = 0;
902 + let watchdog: ReturnType<typeof setInterval> | null = null;
903 + const clearWatchdog = () => {
904 + if (watchdog !== null) clearInterval(watchdog);
905 + watchdog = null;
906 + };
907 + const retry = (detail: string) => {
908 + clearWatchdog();
909 + if (!isCurrent()) return;
910 + closePeer();
911 + connecting(detail);
912 + scheduleRetry("video", () => void readIosWebRtc(settings));
913 + };
914 + const presentFrame = () => {
915 + if (!isCurrent()) {
916 + clearWatchdog();
917 + return;
918 + }
919 + if (video.videoWidth > 0 && video.videoHeight > 0) {
920 + paint(video, video.videoWidth, video.videoHeight);
921 + checksWithoutFrame = 0;
922 + watchdog ??= setInterval(() => {
923 + if (!isCurrent()) return clearWatchdog();
924 + checksWithoutFrame++;
925 + if (checksWithoutFrame * WEBRTC_STALL_CHECK_MS >= WEBRTC_STALL_TIMEOUT_MS) {
926 + retry("Device stream stalled. Reconnecting…");
927 + }
928 + }, WEBRTC_STALL_CHECK_MS);
929 + }
930 + video.requestVideoFrameCallback(presentFrame);
931 + };
932 + const transceiver = connection.addTransceiver("video", { direction: "recvonly" });
933 + const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? [];
934 + if (codecs.length > 0) {
935 + // serve-sim encodes H.264; other codecs stay as fallbacks it may refuse.
936 + transceiver.setCodecPreferences([
937 + ...codecs.filter((codec) => codec.mimeType.toLowerCase() === "video/h264"),
938 + ...codecs.filter((codec) => codec.mimeType.toLowerCase() !== "video/h264"),
939 + ]);
940 + }
941 + connection.ontrack = (event) => {
942 + if (!isCurrent()) return;
943 + iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
944 + video.srcObject = event.streams[0] ?? new MediaStream([event.track]);
945 + void video.play().catch(() => {});
946 + video.requestVideoFrameCallback(presentFrame);
947 + };
948 + connection.onconnectionstatechange = () => {
949 + if (connection.connectionState === "failed")
950 + retry("Device stream connection failed. Reconnecting…");
951 + };
952 + try {
953 + await connection.setLocalDescription(await connection.createOffer());
954 + const gathered = await new Promise<boolean>((resolve) => {
955 + if (connection.iceGatheringState === "complete") return resolve(true);
956 + const timer = setTimeout(() => done(false), iceGatheringDeadlineMs);
957 + function done(complete: boolean) {
958 + clearTimeout(timer);
959 + connection.removeEventListener("icegatheringstatechange", changed);
960 + resolve(complete);
961 + }
962 + function changed() {
963 + if (connection.iceGatheringState === "complete") done(true);
964 + }
965 + connection.addEventListener("icegatheringstatechange", changed);
966 + });
967 + if (!isCurrent()) return;
968 + // serve-sim takes the offer's candidates as the whole set; none trickle
969 + // later. A simulator in the cloud needs a reflexive or relay candidate,
970 + // which STUN and TURN can take longer than the deadline to produce: an
971 + // offer cut short without one is retried with a longer deadline, up to
972 + // a bound, after which it goes as gathered rather than never.
973 + const sdp = connection.localDescription?.sdp ?? "";
974 + if (
975 + !gathered &&
976 + !/^a=candidate:.* typ (?:srflx|relay)\b/m.test(sdp) &&
977 + iceGatheringDeadlineMs < ICE_GATHERING_MAX_TIMEOUT_MS
978 + ) {
979 + iceGatheringDeadlineMs = Math.min(iceGatheringDeadlineMs * 2, ICE_GATHERING_MAX_TIMEOUT_MS);
980 + return retry("Device stream found no network candidates yet. Retrying…");
981 + }
982 + const response = await fetch(httpUrl(`/helper/${device}/webrtc/offer`), {
983 + method: "POST",
984 + headers: { "content-type": "application/json" },
985 + body: JSON.stringify({
986 + type: "offer",
987 + sdp: connection.localDescription?.sdp,
988 + sessionId: current.sessionId,
989 + codec: "h264",
990 + }),
991 + credentials: access.credentials ? "include" : "same-origin",
992 + });
993 + if (!isCurrent()) {
994 + await response.body?.cancel();
995 + return;
996 + }
997 + if (response.status === 401 || response.status === 403) return handleUnauthorized();
998 + // 409: another viewer is still negotiating with this simulator.
999 + if (!response.ok) return retry(`Device stream offer failed (${response.status}). Retrying…`);
1000 + const answer = decodeWebRtcAnswer(await response.json());
1001 + if (!isCurrent()) return;
1002 + if (Option.isNone(answer))
1003 + return retry("Device stream returned an invalid answer. Retrying…");
1004 + await connection.setRemoteDescription(answer.value);
1005 + } catch (cause) {
1006 + retry((cause as Error).message);
1007 + }
1008 + };
1009 +
1010 + /**
1011 + * Called once AVCC or MJPEG was refused. `locked` means serve-sim already
1012 + * said it streams WebRTC only, so a missing /api answer still switches.
1013 + */
1014 + const switchToWebRtc = async (failure: string, locked: boolean) => {
1015 + const session = generation;
1016 + if (typeof RTCPeerConnection === "undefined") return fail(failure);
1017 + const settings =
1018 + (await probeIosTransport()) ?? (locked ? { transport: "webrtc" as const } : null);
1019 + if (stopped || generation !== session) return;
1020 + if (!settings) return fail(failure);
1021 + if (mjpeg) {
1022 + mjpeg = false;
1023 + releaseImage?.();
1024 + releaseImage = null;
1025 + events.onCanvasVideo?.();
1026 + }
1027 + // MJPEG may have shown frames already: say the stream is connecting again
1028 + // rather than leave a blank canvas until the first WebRTC frame.
1029 + connecting("Switching to WebRTC.");
1030 + void readIosWebRtc(settings);
1031 + };
1032 +
1033 const startDuoVideo = (panels: DuoPanelSinks) => {
1034 for (const panel of panelClients) panel.stop();
1035 // Physical handoff elects a native surface. Fixed-panel encoders can keep
@@ -987,6 +1264,7 @@ export function createDeviceStreamClient(
1264 primeController = null;
1265 controller?.abort();
1266 controller = null;
1267 + closePeer();
1268 const discarded = socket;
1269 socket = null;
1270 discarded?.close();
@@ -1033,6 +1311,7 @@ export function createDeviceStreamClient(
1311 videoGeneration++;
1312 controller?.abort();
1313 controller = null;
1314 + closePeer();
1315 closeDecoder();
1316 const retry = retryTimers.get("video");
1317 if (retry) clearTimeout(retry);