The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 22:00 UTC
feat(device): stream iOS devices that serve WebRTC only (#21841)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 20, 2026 at 00:57 UTC
6dc63bd85bf6930191e4ee858fdc7d940ab5bd5d
7 files changed
+592
-2
apps/mobile/src/features/devices/device-stream.browser.ts
+4
@@ -140,6 +140,10 @@ export function start(configuration: DeviceStreamConfiguration) {
140
canvas.style.display = "none";
141
image.style.display = "block";
142
},
143
+ onCanvasVideo: () => {
144
+ image.style.display = "none";
145
+ canvas.style.display = "";
146
+ },
147
onUnauthorized: unauthorized,
148
onInputConnected: (connected) => {
149
inputConnected = connected;
apps/server/src/device/DeviceHubProxy.test.ts
+15
@@ -152,6 +152,21 @@ describe("device hub proxy", () => {
152
expect(headers[0]).not.toHaveProperty("x-forwarded-proto");
153
});
154
155
+ it("lets a viewer negotiate a WebRTC stream with read scope", async () => {
156
+ const { handler, requests, headers } = fixture([AuthOrchestrationReadScope]);
157
+ const offer = await handler(
158
+ new Request("http://t3.test/api/device-hub/vendor/serve-sim/helper/udid-1/webrtc/offer", {
159
+ method: "POST",
160
+ headers: { "content-type": "application/json" },
161
+ body: '{"type":"offer"}',
162
+ }),
163
+ );
164
+ expect(offer.status).toBe(200);
165
+ await offer.text();
166
+ expect(requests).toEqual(["http://hub.test/vendor/serve-sim/helper/udid-1/webrtc/offer"]);
167
+ expect(headers[0]?.["content-type"]).toBe("application/json");
168
+ });
169
+
170
it("never forwards the vendor shell endpoint", async () => {
171
const { handler, requests } = fixture([AuthOrchestrationOperateScope]);
172
expect(
apps/server/src/device/DeviceHubProxy.ts
+8
-1
@@ -28,6 +28,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
28
/^\/vendor\/serve-sim\/api\/event-log(\/events)?$/,
29
/^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
30
/^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
31
+ /^\/vendor\/serve-sim\/helper\/[^/]+\/webrtc\/(offer|close)$/,
32
/^\/vendor\/serve-sim\/appstate$/,
33
/^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/,
34
/^\/vendor\/serve-emu\/health$/,
@@ -35,6 +36,8 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
36
37
const MUTABLE_PATHS: ReadonlyArray<RegExp> = [
38
/^\/vendor\/serve-sim\/api\/screenshot$/,
39
+ // WebRTC signalling: opening and closing a view of the stream, so read scope.
40
+ /^\/vendor\/serve-sim\/helper\/[^/]+\/webrtc\/(offer|close)$/,
41
/^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/,
42
/^\/vendor\/serve-emu\/api\/fold$/,
43
];
@@ -120,7 +123,11 @@ const proxyHttp = Effect.fn("DeviceHubProxy.proxyHttp")(function* (
123
HttpClientRequest.setHeaders(forwardHeaders(request, hubOrigin)),
124
method === "GET" || method === "HEAD"
125
? (self) => self
123
- : HttpClientRequest.bodyStream(request.stream),
126
+ : // A bare stream body is sent as octet-stream; serve-sim refuses WebRTC
127
+ // offers that are not application/json.
128
+ HttpClientRequest.bodyStream(request.stream, {
129
+ contentType: request.headers["content-type"],
130
+ }),
131
);
132
const response = yield* httpClient.execute(upstreamRequest);
133
const headers: Record<string, string> = {};
apps/web/src/components/device/DeviceStreamView.tsx
+1
@@ -164,6 +164,7 @@ export function DeviceStreamView(props: {
164
setMjpegUrl(url);
165
setMjpegGeneration((generation) => generation + 1);
166
},
167
+ onCanvasVideo: () => setMjpegUrl(null),
168
onInputConnected: (connected, detail) => {
169
setInputState({ connected, ...(detail ? { detail } : {}) });
170
if (!connected) setShowRestartNotice(false);
docs/internals/devices.md
+7
@@ -91,3 +91,10 @@ Simulators encode H.264 High 5.1. Hardware decoders on some machines and all
91
headless browsers reject that profile, and WebCodecs is secure-context only, so
92
the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on
93
iOS. Android has no MJPEG; there the panel reports that it cannot decode.
94
+
95
+serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses
96
+AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so
97
+hubs that serve it pay nothing, and switches on that 409, or when its MJPEG
98
+image fails in a browser that has `RTCPeerConnection`. Offers are not trickled:
99
+serve-sim takes one offer with every candidate. The video element only feeds
100
+the same canvas sink, so the panel and the mobile WebView keep one surface.
packages/client-runtime/src/device/stream.test.ts
+277
@@ -923,3 +923,280 @@ it("reopens a stalled AVCC body without disconnecting input, then receives fresh
923
vi.unstubAllGlobals();
924
}
925
});
926
+
927
+describe("WebRTC-only iOS devices", () => {
928
+ afterEach(() => {
929
+ vi.useRealTimers();
930
+ vi.unstubAllGlobals();
931
+ });
932
+
933
+ function webRtcFixture(options: {
934
+ webCodecs: boolean;
935
+ offerStatuses?: number[];
936
+ /** The fake peer's ICE gathering state; `gathering` never completes. */
937
+ iceGatheringState?: "complete" | "gathering";
938
+ }) {
939
+ vi.useFakeTimers();
940
+ if (options.webCodecs) {
941
+ vi.stubGlobal("VideoDecoder", vi.fn());
942
+ vi.stubGlobal("EncodedVideoChunk", vi.fn());
943
+ }
944
+ class FakeSocket {
945
+ static OPEN = 1;
946
+ readyState = 1;
947
+ binaryType = "";
948
+ onmessage: unknown = null;
949
+ onclose: unknown = null;
950
+ onerror: unknown = null;
951
+ send = vi.fn();
952
+ close = vi.fn();
953
+ }
954
+ vi.stubGlobal("WebSocket", FakeSocket);
955
+ const peers: FakePeer[] = [];
956
+ class FakePeer {
957
+ iceGatheringState = options.iceGatheringState ?? "complete";
958
+ connectionState = "new";
959
+ localDescription: { type: string; sdp: string } | null = null;
960
+ remoteDescription: unknown = null;
961
+ ontrack: ((event: { streams: unknown[]; track: unknown }) => void) | null = null;
962
+ onconnectionstatechange: (() => void) | null = null;
963
+ closed = false;
964
+ readonly config: unknown;
965
+ readonly codecPreferences = vi.fn();
966
+ constructor(config: unknown) {
967
+ this.config = config;
968
+ peers.push(this);
969
+ }
970
+ addTransceiver() {
971
+ return { setCodecPreferences: this.codecPreferences };
972
+ }
973
+ async createOffer() {
974
+ return { type: "offer", sdp: "v=0 offer" };
975
+ }
976
+ async setLocalDescription(description: { type: string; sdp: string }) {
977
+ this.localDescription = description;
978
+ }
979
+ async setRemoteDescription(description: unknown) {
980
+ this.remoteDescription = description;
981
+ }
982
+ addEventListener() {}
983
+ removeEventListener() {}
984
+ close() {
985
+ this.closed = true;
986
+ }
987
+ }
988
+ vi.stubGlobal("RTCPeerConnection", FakePeer);
989
+ vi.stubGlobal("RTCRtpReceiver", {
990
+ getCapabilities: () => ({ codecs: [{ mimeType: "video/VP8" }, { mimeType: "video/H264" }] }),
991
+ });
992
+ const frameCallbacks: Array<() => void> = [];
993
+ const video = {
994
+ muted: false,
995
+ playsInline: false,
996
+ srcObject: null as unknown,
997
+ videoWidth: 402,
998
+ videoHeight: 874,
999
+ style: { cssText: "" },
1000
+ setAttribute: vi.fn(),
1001
+ remove: vi.fn(),
1002
+ play: vi.fn(() => Promise.resolve()),
1003
+ requestVideoFrameCallback: vi.fn((callback: () => void) => {
1004
+ frameCallbacks.push(callback);
1005
+ }),
1006
+ };
1007
+ vi.stubGlobal("document", { createElement: () => video, body: { append: vi.fn() } });
1008
+ const offerStatuses = [...(options.offerStatuses ?? [])];
1009
+ const fetch = vi.fn((url: string) => {
1010
+ const path = new URL(url).pathname;
1011
+ if (path.endsWith("/stream.avcc"))
1012
+ return Promise.resolve(
1013
+ Response.json({ error: "stream_transport_locked", transport: "webrtc" }, { status: 409 }),
1014
+ );
1015
+ if (path.endsWith("/vendor/serve-sim/api"))
1016
+ return Promise.resolve(
1017
+ Response.json({
1018
+ streamSettings: { transport: "webrtc", iceServers: [{ urls: ["stun:stun.test"] }] },
1019
+ }),
1020
+ );
1021
+ if (path.endsWith("/webrtc/offer")) {
1022
+ const status = offerStatuses.shift() ?? 200;
1023
+ return Promise.resolve(
1024
+ status === 200
1025
+ ? Response.json({ type: "answer", sdp: "v=0 answer" })
1026
+ : Response.json({ error: "webrtc_session_busy" }, { status }),
1027
+ );
1028
+ }
1029
+ if (path.endsWith("/webrtc/close"))
1030
+ return Promise.resolve(new Response(null, { status: 204 }));
1031
+ return Promise.resolve(new Response("refused", { status: 409 }));
1032
+ });
1033
+ vi.stubGlobal("fetch", fetch);
1034
+ const presented: Array<[unknown, number, number]> = [];
1035
+ const events = {
1036
+ onStatus: vi.fn(),
1037
+ onScreen: vi.fn(),
1038
+ onUnauthorized: vi.fn(),
1039
+ onMjpegFallback: vi.fn(),
1040
+ onCanvasVideo: vi.fn(),
1041
+ onInputConnected: vi.fn(),
1042
+ };
1043
+ const client = createDeviceStreamClient(
1044
+ {
1045
+ platform: "ios",
1046
+ deviceId: "udid-1",
1047
+ preferMjpeg: !options.webCodecs,
1048
+ access: {
1049
+ httpBase: "https://environment.test/api/device-hub",
1050
+ wsBase: "wss://environment.test/api/device-hub",
1051
+ credentials: false,
1052
+ query: { hostId: "eas" },
1053
+ },
1054
+ },
1055
+ {
1056
+ present: (source, width, height) => {
1057
+ presented.push([source, width, height]);
1058
+ return true;
1059
+ },
1060
+ },
1061
+ events,
1062
+ );
1063
+ const requests = (suffix: string) =>
1064
+ fetch.mock.calls.filter(([url]) => new URL(url).pathname.endsWith(suffix));
1065
+ const bodyOf = (call: unknown[]) =>
1066
+ JSON.parse(String((call[1] as RequestInit | undefined)?.body)) as Record<string, unknown>;
1067
+ const settle = async () => {
1068
+ for (let turn = 0; turn < 20; turn++) await vi.advanceTimersByTimeAsync(0);
1069
+ };
1070
+ return { client, peers, video, frameCallbacks, presented, events, requests, bodyOf, settle };
1071
+ }
1072
+
1073
+ it("switches to WebRTC when serve-sim refuses AVCC and releases the session on stop", async () => {
1074
+ const fixture = webRtcFixture({ webCodecs: true });
1075
+ fixture.client.start();
1076
+ await fixture.settle();
1077
+ expect(fixture.peers).toHaveLength(1);
1078
+ const peer = fixture.peers[0]!;
1079
+ expect(peer.config).toEqual({
1080
+ iceServers: [{ urls: ["stun:stun.test"] }],
1081
+ iceTransportPolicy: "all",
1082
+ });
1083
+ expect(peer.codecPreferences).toHaveBeenCalledWith([
1084
+ { mimeType: "video/H264" },
1085
+ { mimeType: "video/VP8" },
1086
+ ]);
1087
+ const offers = fixture.requests("/helper/udid-1/webrtc/offer");
1088
+ expect(offers).toHaveLength(1);
1089
+ const offer = fixture.bodyOf(offers[0]!);
1090
+ expect(offer).toMatchObject({ type: "offer", sdp: "v=0 offer", codec: "h264" });
1091
+ expect(offer.sessionId).toMatch(
1092
+ /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
1093
+ );
1094
+ expect(peer.remoteDescription).toEqual({ type: "answer", sdp: "v=0 answer" });
1095
+
1096
+ peer.ontrack?.({ streams: [{}], track: {} });
1097
+ fixture.frameCallbacks.shift()?.();
1098
+ expect(fixture.presented).toEqual([[fixture.video, 402, 874]]);
1099
+ expect(fixture.events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
1100
+
1101
+ fixture.client.stop();
1102
+ expect(peer.closed).toBe(true);
1103
+ expect(fixture.video.remove).toHaveBeenCalled();
1104
+ const closes = fixture.requests("/helper/udid-1/webrtc/close");
1105
+ expect(closes.map(fixture.bodyOf)).toEqual([{ sessionId: offer.sessionId }]);
1106
+ });
1107
+
1108
+ it("reconnects a WebRTC feed that stops producing frames", async () => {
1109
+ const fixture = webRtcFixture({ webCodecs: true });
1110
+ fixture.client.start();
1111
+ await fixture.settle();
1112
+ const peer = fixture.peers[0]!;
1113
+ peer.ontrack?.({ streams: [{}], track: {} });
1114
+ fixture.frameCallbacks.shift()?.();
1115
+ expect(fixture.events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
1116
+ // Frames keep coming: nothing happens.
1117
+ await vi.advanceTimersByTimeAsync(10_000);
1118
+ fixture.frameCallbacks.shift()?.();
1119
+ await vi.advanceTimersByTimeAsync(10_000);
1120
+ expect(fixture.peers).toHaveLength(1);
1121
+ expect(peer.closed).toBe(false);
1122
+ // Then none for the first-frame deadline (checked every 5 s): the feed
1123
+ // reconnects through the usual retry path.
1124
+ await vi.advanceTimersByTimeAsync(15_000);
1125
+ expect(peer.closed).toBe(true);
1126
+ expect(fixture.events.onStatus).toHaveBeenLastCalledWith(
1127
+ "connecting",
1128
+ "Device stream stalled. Reconnecting…",
1129
+ );
1130
+ await vi.advanceTimersByTimeAsync(1_000);
1131
+ await fixture.settle();
1132
+ expect(fixture.peers).toHaveLength(2);
1133
+ expect(fixture.peers[1]!.remoteDescription).toEqual({ type: "answer", sdp: "v=0 answer" });
1134
+ fixture.client.stop();
1135
+ });
1136
+
1137
+ it("retries instead of offering when ICE gathering times out without a candidate", async () => {
1138
+ const fixture = webRtcFixture({ webCodecs: true, iceGatheringState: "gathering" });
1139
+ fixture.client.start();
1140
+ await fixture.settle();
1141
+ expect(fixture.peers).toHaveLength(1);
1142
+ expect(fixture.requests("/helper/udid-1/webrtc/offer")).toHaveLength(0);
1143
+ // The gathering deadline passes with no reflexive or relay candidate: no
1144
+ // offer is sent, the attempt is retried with twice the deadline.
1145
+ await vi.advanceTimersByTimeAsync(2_000);
1146
+ expect(fixture.peers[0]!.closed).toBe(true);
1147
+ expect(fixture.requests("/helper/udid-1/webrtc/offer")).toHaveLength(0);
1148
+ expect(fixture.events.onStatus).toHaveBeenLastCalledWith(
1149
+ "connecting",
1150
+ "Device stream found no network candidates yet. Retrying…",
1151
+ );
1152
+ await vi.advanceTimersByTimeAsync(1_000);
1153
+ await fixture.settle();
1154
+ expect(fixture.peers).toHaveLength(2);
1155
+ await vi.advanceTimersByTimeAsync(2_000);
1156
+ expect(fixture.peers[1]!.closed).toBe(false);
1157
+ // The second deadline, 4 s, is the bound: the offer goes as gathered.
1158
+ await vi.advanceTimersByTimeAsync(3_000);
1159
+ await fixture.settle();
1160
+ expect(fixture.peers).toHaveLength(2);
1161
+ expect(fixture.peers[1]!.closed).toBe(false);
1162
+ expect(fixture.requests("/helper/udid-1/webrtc/offer")).toHaveLength(1);
1163
+ fixture.client.stop();
1164
+ });
1165
+
1166
+ it("retries a WebRTC offer while another viewer is negotiating", async () => {
1167
+ const fixture = webRtcFixture({ webCodecs: true, offerStatuses: [409] });
1168
+ fixture.client.start();
1169
+ await fixture.settle();
1170
+ expect(fixture.peers).toHaveLength(1);
1171
+ expect(fixture.peers[0]!.closed).toBe(true);
1172
+ await vi.advanceTimersByTimeAsync(1_000);
1173
+ await fixture.settle();
1174
+ expect(fixture.peers).toHaveLength(2);
1175
+ expect(fixture.peers[1]!.remoteDescription).toEqual({ type: "answer", sdp: "v=0 answer" });
1176
+ fixture.client.stop();
1177
+ });
1178
+
1179
+ it("moves an MJPEG viewer back to the canvas for a WebRTC-only device", async () => {
1180
+ const fixture = webRtcFixture({ webCodecs: false });
1181
+ const listeners: Record<string, () => void> = {};
1182
+ const image = {
1183
+ naturalWidth: 0,
1184
+ naturalHeight: 0,
1185
+ src: "",
1186
+ addEventListener: (type: string, listener: () => void) => {
1187
+ listeners[type] = listener;
1188
+ },
1189
+ removeEventListener: vi.fn(),
1190
+ removeAttribute: vi.fn(),
1191
+ };
1192
+ fixture.client.start();
1193
+ fixture.client.setMjpegImage(image as unknown as HTMLImageElement);
1194
+ expect(fixture.events.onMjpegFallback).toHaveBeenCalledOnce();
1195
+ listeners.error?.();
1196
+ await fixture.settle();
1197
+ expect(fixture.events.onCanvasVideo).toHaveBeenCalledOnce();
1198
+ expect(fixture.peers).toHaveLength(1);
1199
+ expect(fixture.requests("/helper/udid-1/webrtc/offer")).toHaveLength(1);
1200
+ fixture.client.stop();
1201
+ });
1202
+});
packages/client-runtime/src/device/stream.ts
+280
-1
@@ -57,6 +57,33 @@ const controlReplySchema = Schema.Struct({
57
});
58
const decodeScreenConfig = Schema.decodeUnknownOption(screenConfigSchema);
59
const decodeControlReply = Schema.decodeUnknownOption(controlReplySchema);
60
+const iceServerSchema = Schema.Struct({
61
+ urls: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
62
+ username: Schema.optionalKey(Schema.String),
63
+ credential: Schema.optionalKey(Schema.String),
64
+});
65
+const webRtcSettingsSchema = Schema.Struct({
66
+ transport: Schema.Literal("webrtc"),
67
+ iceServers: Schema.optionalKey(Schema.Array(iceServerSchema)),
68
+ iceTransportPolicy: Schema.optionalKey(Schema.Literals(["all", "relay"])),
69
+});
70
+// serve-sim reports its stream settings as `streamSettings` (EAS) or `stream`.
71
+const serveSimApiSchema = Schema.Struct({
72
+ streamSettings: Schema.optionalKey(Schema.Unknown),
73
+ stream: Schema.optionalKey(Schema.Unknown),
74
+});
75
+const decodeServeSimApi = Schema.decodeUnknownOption(serveSimApiSchema);
76
+const decodeWebRtcSettings = Schema.decodeUnknownOption(webRtcSettingsSchema);
77
+type WebRtcSettings = typeof webRtcSettingsSchema.Type;
78
+const decodeTransportLocked = Schema.decodeUnknownOption(
79
+ Schema.Struct({
80
+ error: Schema.Literal("stream_transport_locked"),
81
+ transport: Schema.Literal("webrtc"),
82
+ }),
83
+);
84
+const decodeWebRtcAnswer = Schema.decodeUnknownOption(
85
+ Schema.Struct({ type: Schema.Literal("answer"), sdp: Schema.String }),
86
+);
87
88
export interface DuoPanelSinks {
89
readonly cover: DeviceFrameSink;
@@ -77,6 +104,11 @@ export interface DeviceStreamEvents {
104
* and attach it with `setMjpegImage` so the client can observe real frames.
105
*/
106
readonly onMjpegFallback: (url: string) => void;
107
+ /**
108
+ * The stream moved from MJPEG back to painting the canvas, because the
109
+ * device serves WebRTC only; the owner should show the canvas again.
110
+ */
111
+ readonly onCanvasVideo?: () => void;
112
readonly onInputConnected: (connected: boolean, detail?: string) => void;
113
}
114
@@ -94,6 +126,19 @@ export type DeviceHardwareButton = "home" | "back" | "recents" | "power" | "appS
126
127
const RETRY_DELAY_MS = 1_000;
128
const FIRST_FRAME_TIMEOUT_MS = 15_000;
129
+// A WebRTC feed that shows no frame for this long is reconnected; checked
130
+// every WEBRTC_STALL_CHECK_MS, so a transient `disconnected` peer that
131
+// recovers is left alone.
132
+const WEBRTC_STALL_TIMEOUT_MS = FIRST_FRAME_TIMEOUT_MS;
133
+const WEBRTC_STALL_CHECK_MS = 5_000;
134
+const TRANSPORT_PROBE_TIMEOUT_MS = 5_000;
135
+// Offers carry every candidate: serve-sim does not accept trickled ICE.
136
+const ICE_GATHERING_TIMEOUT_MS = 2_000;
137
+// A gathering cut short without a reflexive or relay candidate is retried
138
+// with twice the deadline, up to this; then the offer goes as gathered. The
139
+// whole sequence stays inside the first-frame deadline, which retries do
140
+// not extend.
141
+const ICE_GATHERING_MAX_TIMEOUT_MS = 4_000;
142
const MJPEG_FRAME_CHECK_MS = 250;
143
const FRAME_DURATION_US = 16_667;
144
const SEMU_MAGIC = 0x53454d55;
@@ -115,6 +160,15 @@ const IOS_TAG_INPUT_ADMITTED = 0x83;
160
const encoder = new TextEncoder();
161
const decoder = new TextDecoder();
162
163
+/** serve-sim requires a v4 UUID; `crypto.randomUUID` is missing over plain-HTTP LAN origins. */
164
+function webRtcSessionId(): string {
165
+ const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16));
166
+ bytes[6] = (bytes[6]! & 0x0f) | 0x40;
167
+ bytes[8] = (bytes[8]! & 0x3f) | 0x80;
168
+ const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
169
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
170
+}
171
+
172
const isWebCodecsSupported = (): boolean =>
173
typeof globalThis !== "undefined" &&
174
"VideoDecoder" in globalThis &&
@@ -367,6 +421,9 @@ export function createDeviceStreamClient(
421
let panelSinks: DuoPanelSinks | null = null;
422
let rotationCursor: DeviceScreenSize["orientation"] | null = null;
423
let pendingOrientation: { requestId: number } | null = null;
424
+ let iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
425
+ let peer: { connection: RTCPeerConnection; sessionId: string; video: HTMLVideoElement } | null =
426
+ null;
427
const duoControl = createDuoControl({
428
send(request) {
429
if (
@@ -454,7 +511,11 @@ export function createDeviceStreamClient(
511
}
512
};
513
const error = () => {
457
- if (!released) fail("Could not receive the device stream. Reconnect to try again.");
514
+ if (released) return;
515
+ const failure = "Could not receive the device stream. Reconnect to try again.";
516
+ // An <img> cannot read the 409 that a WebRTC-only serve-sim answers with.
517
+ if (typeof RTCPeerConnection === "undefined") fail(failure);
518
+ else void switchToWebRtc(failure, false);
519
};
520
image.addEventListener("load", check);
521
image.addEventListener("error", error);
@@ -646,6 +707,15 @@ export function createDeviceStreamClient(
707
return;
708
}
709
if (response.status === 401 || response.status === 403) return handleUnauthorized();
710
+ if (response.status === 409 && !target.panelId) {
711
+ const locked = decodeTransportLocked(await response.json().catch(() => null));
712
+ if (Option.isSome(locked)) {
713
+ if (!isCurrent()) return;
714
+ controller = null;
715
+ void switchToWebRtc("This browser cannot play this device's WebRTC stream.", true);
716
+ return;
717
+ }
718
+ }
719
if (target.panelId && [400, 404, 405, 410].includes(response.status)) {
720
await response.body?.cancel();
721
setStatus("error", "This Device Hub does not provide fixed Duo display feeds.");
@@ -753,6 +823,213 @@ export function createDeviceStreamClient(
823
}
824
};
825
826
+ /**
827
+ * serve-sim can be locked to WebRTC (EAS Simulator always is), and then
828
+ * refuses AVCC and MJPEG. An unreadable answer keeps the AVCC path.
829
+ */
830
+ const probeIosTransport = async (): Promise<WebRtcSettings | null> => {
831
+ const probe = new AbortController();
832
+ const timeout = setTimeout(() => probe.abort(), TRANSPORT_PROBE_TIMEOUT_MS);
833
+ try {
834
+ const response = await fetch(httpUrl(`/api?device=${device}`), {
835
+ signal: probe.signal,
836
+ credentials: access.credentials ? "include" : "same-origin",
837
+ });
838
+ if (!response.ok) return null;
839
+ const api = decodeServeSimApi(await response.json());
840
+ if (Option.isNone(api)) return null;
841
+ const settings = decodeWebRtcSettings(api.value.streamSettings ?? api.value.stream);
842
+ return Option.isSome(settings) ? settings.value : null;
843
+ } catch {
844
+ return null;
845
+ } finally {
846
+ clearTimeout(timeout);
847
+ }
848
+ };
849
+
850
+ const closePeer = () => {
851
+ const current = peer;
852
+ if (!current) return;
853
+ peer = null;
854
+ current.connection.close();
855
+ current.video.srcObject = null;
856
+ current.video.remove();
857
+ // Releases serve-sim's capture session; best effort because the page may be unloading.
858
+ void fetch(httpUrl(`/helper/${device}/webrtc/close`), {
859
+ method: "POST",
860
+ keepalive: true,
861
+ headers: { "content-type": "application/json" },
862
+ body: JSON.stringify({ sessionId: current.sessionId }),
863
+ credentials: access.credentials ? "include" : "same-origin",
864
+ }).catch(() => {});
865
+ };
866
+
867
+ const readIosWebRtc = async (settings: WebRtcSettings) => {
868
+ const lifecycle = generation;
869
+ const feedGeneration = ++videoGeneration;
870
+ closePeer();
871
+ const connection = new RTCPeerConnection({
872
+ ...(settings.iceServers
873
+ ? {
874
+ iceServers: settings.iceServers.map((server) => ({
875
+ ...server,
876
+ urls: typeof server.urls === "string" ? server.urls : [...server.urls],
877
+ })),
878
+ }
879
+ : {}),
880
+ iceTransportPolicy: settings.iceTransportPolicy ?? "all",
881
+ });
882
+ // Frames only reach requestVideoFrameCallback from a video that is in the
883
+ // document; it stays invisible because the sink paints them.
884
+ const video = document.createElement("video");
885
+ video.muted = true;
886
+ video.playsInline = true;
887
+ video.setAttribute("aria-hidden", "true");
888
+ video.style.cssText =
889
+ "position:fixed;width:1px;height:1px;opacity:0;pointer-events:none;left:0;top:0";
890
+ document.body.append(video);
891
+ const current = { connection, sessionId: webRtcSessionId(), video };
892
+ peer = current;
893
+ const isCurrent = () =>
894
+ !stopped &&
895
+ generation === lifecycle &&
896
+ videoGeneration === feedGeneration &&
897
+ peer === current;
898
+ // After the first frame nothing else bounds a feed that stops: the peer
899
+ // can sit `disconnected` with the last frame on the canvas. A long gap
900
+ // between frames reconnects through the same path as a failed connection.
901
+ let checksWithoutFrame = 0;
902
+ let watchdog: ReturnType<typeof setInterval> | null = null;
903
+ const clearWatchdog = () => {
904
+ if (watchdog !== null) clearInterval(watchdog);
905
+ watchdog = null;
906
+ };
907
+ const retry = (detail: string) => {
908
+ clearWatchdog();
909
+ if (!isCurrent()) return;
910
+ closePeer();
911
+ connecting(detail);
912
+ scheduleRetry("video", () => void readIosWebRtc(settings));
913
+ };
914
+ const presentFrame = () => {
915
+ if (!isCurrent()) {
916
+ clearWatchdog();
917
+ return;
918
+ }
919
+ if (video.videoWidth > 0 && video.videoHeight > 0) {
920
+ paint(video, video.videoWidth, video.videoHeight);
921
+ checksWithoutFrame = 0;
922
+ watchdog ??= setInterval(() => {
923
+ if (!isCurrent()) return clearWatchdog();
924
+ checksWithoutFrame++;
925
+ if (checksWithoutFrame * WEBRTC_STALL_CHECK_MS >= WEBRTC_STALL_TIMEOUT_MS) {
926
+ retry("Device stream stalled. Reconnecting…");
927
+ }
928
+ }, WEBRTC_STALL_CHECK_MS);
929
+ }
930
+ video.requestVideoFrameCallback(presentFrame);
931
+ };
932
+ const transceiver = connection.addTransceiver("video", { direction: "recvonly" });
933
+ const codecs = RTCRtpReceiver.getCapabilities("video")?.codecs ?? [];
934
+ if (codecs.length > 0) {
935
+ // serve-sim encodes H.264; other codecs stay as fallbacks it may refuse.
936
+ transceiver.setCodecPreferences([
937
+ ...codecs.filter((codec) => codec.mimeType.toLowerCase() === "video/h264"),
938
+ ...codecs.filter((codec) => codec.mimeType.toLowerCase() !== "video/h264"),
939
+ ]);
940
+ }
941
+ connection.ontrack = (event) => {
942
+ if (!isCurrent()) return;
943
+ iceGatheringDeadlineMs = ICE_GATHERING_TIMEOUT_MS;
944
+ video.srcObject = event.streams[0] ?? new MediaStream([event.track]);
945
+ void video.play().catch(() => {});
946
+ video.requestVideoFrameCallback(presentFrame);
947
+ };
948
+ connection.onconnectionstatechange = () => {
949
+ if (connection.connectionState === "failed")
950
+ retry("Device stream connection failed. Reconnecting…");
951
+ };
952
+ try {
953
+ await connection.setLocalDescription(await connection.createOffer());
954
+ const gathered = await new Promise<boolean>((resolve) => {
955
+ if (connection.iceGatheringState === "complete") return resolve(true);
956
+ const timer = setTimeout(() => done(false), iceGatheringDeadlineMs);
957
+ function done(complete: boolean) {
958
+ clearTimeout(timer);
959
+ connection.removeEventListener("icegatheringstatechange", changed);
960
+ resolve(complete);
961
+ }
962
+ function changed() {
963
+ if (connection.iceGatheringState === "complete") done(true);
964
+ }
965
+ connection.addEventListener("icegatheringstatechange", changed);
966
+ });
967
+ if (!isCurrent()) return;
968
+ // serve-sim takes the offer's candidates as the whole set; none trickle
969
+ // later. A simulator in the cloud needs a reflexive or relay candidate,
970
+ // which STUN and TURN can take longer than the deadline to produce: an
971
+ // offer cut short without one is retried with a longer deadline, up to
972
+ // a bound, after which it goes as gathered rather than never.
973
+ const sdp = connection.localDescription?.sdp ?? "";
974
+ if (
975
+ !gathered &&
976
+ !/^a=candidate:.* typ (?:srflx|relay)\b/m.test(sdp) &&
977
+ iceGatheringDeadlineMs < ICE_GATHERING_MAX_TIMEOUT_MS
978
+ ) {
979
+ iceGatheringDeadlineMs = Math.min(iceGatheringDeadlineMs * 2, ICE_GATHERING_MAX_TIMEOUT_MS);
980
+ return retry("Device stream found no network candidates yet. Retrying…");
981
+ }
982
+ const response = await fetch(httpUrl(`/helper/${device}/webrtc/offer`), {
983
+ method: "POST",
984
+ headers: { "content-type": "application/json" },
985
+ body: JSON.stringify({
986
+ type: "offer",
987
+ sdp: connection.localDescription?.sdp,
988
+ sessionId: current.sessionId,
989
+ codec: "h264",
990
+ }),
991
+ credentials: access.credentials ? "include" : "same-origin",
992
+ });
993
+ if (!isCurrent()) {
994
+ await response.body?.cancel();
995
+ return;
996
+ }
997
+ if (response.status === 401 || response.status === 403) return handleUnauthorized();
998
+ // 409: another viewer is still negotiating with this simulator.
999
+ if (!response.ok) return retry(`Device stream offer failed (${response.status}). Retrying…`);
1000
+ const answer = decodeWebRtcAnswer(await response.json());
1001
+ if (!isCurrent()) return;
1002
+ if (Option.isNone(answer))
1003
+ return retry("Device stream returned an invalid answer. Retrying…");
1004
+ await connection.setRemoteDescription(answer.value);
1005
+ } catch (cause) {
1006
+ retry((cause as Error).message);
1007
+ }
1008
+ };
1009
+
1010
+ /**
1011
+ * Called once AVCC or MJPEG was refused. `locked` means serve-sim already
1012
+ * said it streams WebRTC only, so a missing /api answer still switches.
1013
+ */
1014
+ const switchToWebRtc = async (failure: string, locked: boolean) => {
1015
+ const session = generation;
1016
+ if (typeof RTCPeerConnection === "undefined") return fail(failure);
1017
+ const settings =
1018
+ (await probeIosTransport()) ?? (locked ? { transport: "webrtc" as const } : null);
1019
+ if (stopped || generation !== session) return;
1020
+ if (!settings) return fail(failure);
1021
+ if (mjpeg) {
1022
+ mjpeg = false;
1023
+ releaseImage?.();
1024
+ releaseImage = null;
1025
+ events.onCanvasVideo?.();
1026
+ }
1027
+ // MJPEG may have shown frames already: say the stream is connecting again
1028
+ // rather than leave a blank canvas until the first WebRTC frame.
1029
+ connecting("Switching to WebRTC.");
1030
+ void readIosWebRtc(settings);
1031
+ };
1032
+
1033
const startDuoVideo = (panels: DuoPanelSinks) => {
1034
for (const panel of panelClients) panel.stop();
1035
// Physical handoff elects a native surface. Fixed-panel encoders can keep
@@ -987,6 +1264,7 @@ export function createDeviceStreamClient(
1264
primeController = null;
1265
controller?.abort();
1266
controller = null;
1267
+ closePeer();
1268
const discarded = socket;
1269
socket = null;
1270
discarded?.close();
@@ -1033,6 +1311,7 @@ export function createDeviceStreamClient(
1311
videoGeneration++;
1312
controller?.abort();
1313
controller = null;
1314
+ closePeer();
1315
closeDecoder();
1316
const retry = retryTimers.get("video");
1317
if (retry) clearTimeout(retry);