The best way to control Claude Code, Codex and more from any computer or phone
fix(mobile): restore low-latency decoding for simulator streams (#22733)
Yash Singh committed
Oct 21, 2026 at 02:45 UTC
ca049dc2053806a35d55c38d5c4e7f7e9cf13659
5 files changed
+306
-5
docs/internals/devices.md
+8
-4
@@ -105,10 +105,14 @@ binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over
105
one WebSocket. [`deviceStream.ts`](../../apps/web/src/components/device/deviceStream.ts)
106
speaks both so one panel covers both platforms.
107
108
-Simulators encode H.264 High 5.1. Hardware decoders on some machines and all
109
-headless browsers reject that profile, and WebCodecs is secure-context only, so
110
-the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on
111
-iOS. Android has no MJPEG; there the panel reports that it cannot decode.
108
+Simulators encode H.264 High 5.1 without frame reordering, but the hub's fallback
109
+encoder can omit the SPS VUI that declares its reorder limits. Chromium's hardware
110
+decoder then retains frames for seconds despite `optimizeForLatency` and an empty
111
+`decodeQueueSize`. The iOS AVCC viewer restores the missing metadata before
112
+configuring the decoder so hardware decoding can remain enabled. WebCodecs is
113
+secure-context only, so the viewer probes `isConfigSupported` and falls back to
114
+MJPEG on iOS when decoding is unavailable. Android has no MJPEG; there the panel
115
+reports that it cannot decode.
116
117
serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses
118
AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so
packages/client-runtime/src/device/avcConfig.test.ts
new
+122
@@ -0,0 +1,122 @@
1
+import { describe, expect, it } from "vite-plus/test";
2
+
3
+import { simulatorAvcConfig } from "./avcConfig.ts";
4
+
5
+// Captured from the hub before and after its encoder falls back from low-latency rate control.
6
+const lowLatency = Uint8Array.from([
7
+ 1, 100, 0, 51, 255, 225, 0, 22, 39, 100, 0, 51, 172, 19, 20, 60, 4, 192, 20, 158, 106, 154, 129,
8
+ 1, 1, 3, 194, 1, 8, 248, 1, 0, 4, 40, 238, 60, 176,
9
+]);
10
+const fallback = Uint8Array.from([
11
+ 1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1, 0, 4, 40,
12
+ 238, 60, 176,
13
+]);
14
+const repaired = Uint8Array.from([
15
+ 1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208, 68, 35,
16
+ 80, 1, 0, 4, 40, 238, 60, 176,
17
+]);
18
+
19
+describe("simulator AVC reorder metadata", () => {
20
+ it("adds zero reordering and one reference frame to the captured fallback SPS", () => {
21
+ expect(simulatorAvcConfig(fallback)).toEqual(repaired);
22
+ expect(simulatorAvcConfig(repaired)).toBe(repaired);
23
+ });
24
+
25
+ it("preserves existing VUI and leaves the original codec description untouched", () => {
26
+ const original = fallback.slice();
27
+ simulatorAvcConfig(fallback);
28
+ expect(fallback).toEqual(original);
29
+ expect(simulatorAvcConfig(lowLatency)).toBe(lowLatency);
30
+ });
31
+
32
+ it("repairs multiple SPS entries while preserving PPS and profile extensions", () => {
33
+ const extensions = [0xfd, 0xf8, 0xf8, 0];
34
+ const description = Uint8Array.from([
35
+ ...fallback.subarray(0, 5),
36
+ 0xe2,
37
+ ...fallback.subarray(6, 20),
38
+ ...lowLatency.subarray(6, 30),
39
+ ...fallback.subarray(20),
40
+ ...extensions,
41
+ ]);
42
+ expect(simulatorAvcConfig(description)).toEqual(
43
+ Uint8Array.from([
44
+ ...description.subarray(0, 6),
45
+ ...repaired.subarray(6, 25),
46
+ ...lowLatency.subarray(6, 30),
47
+ ...fallback.subarray(20),
48
+ ...extensions,
49
+ ]),
50
+ );
51
+ });
52
+
53
+ it("passes through every truncated prefix of the captured configurations", () => {
54
+ for (const config of [fallback, lowLatency]) {
55
+ for (let length = 0; length < config.length; length++) {
56
+ const truncated = config.subarray(0, length);
57
+ expect(simulatorAvcConfig(truncated)).toBe(truncated);
58
+ }
59
+ }
60
+ });
61
+
62
+ it("does not modify unsupported profiles, scaling lists, or invalid parameter-set lengths", () => {
63
+ const unsupportedProfile = fallback.slice();
64
+ unsupportedProfile[9] = 110;
65
+ const scalingLists = fallback.slice();
66
+ scalingLists[12] = scalingLists[12]! | 1;
67
+ const invalidSpsLength = fallback.slice();
68
+ invalidSpsLength[6] = 0xff;
69
+ const invalidPpsLength = fallback.slice();
70
+ invalidPpsLength[21] = 0xff;
71
+ for (const config of [unsupportedProfile, scalingLists, invalidSpsLength, invalidPpsLength]) {
72
+ expect(simulatorAvcConfig(config)).toBe(config);
73
+ }
74
+ });
75
+
76
+ it("requires valid SPS trailing bits before appending VUI metadata", () => {
77
+ const invalidPadding = fallback.slice();
78
+ invalidPadding[19] = 0;
79
+ expect(simulatorAvcConfig(invalidPadding)).toBe(invalidPadding);
80
+ });
81
+
82
+ it("passes through invalid NAL headers and unterminated Exp-Golomb values", () => {
83
+ const forbiddenBit = fallback.slice();
84
+ forbiddenBit[8] = forbiddenBit[8]! | 0x80;
85
+ expect(simulatorAvcConfig(forbiddenBit)).toBe(forbiddenBit);
86
+ const unterminated = fallback.slice();
87
+ unterminated.fill(0, 12, 20);
88
+ expect(simulatorAvcConfig(unterminated)).toBe(unterminated);
89
+ const overlong = Uint8Array.from([
90
+ ...fallback.subarray(0, 6),
91
+ 0,
92
+ 13,
93
+ 39,
94
+ 100,
95
+ 0,
96
+ 51,
97
+ 0,
98
+ 0,
99
+ 3,
100
+ 0,
101
+ 0,
102
+ 3,
103
+ 0,
104
+ 0,
105
+ 0,
106
+ ...fallback.subarray(20),
107
+ ]);
108
+ expect(simulatorAvcConfig(overlong)).toBe(overlong);
109
+ });
110
+
111
+ it("bounds parsing of oversized configurations and SPS data", () => {
112
+ const oversizedConfig = new Uint8Array(64 * 1024 + 1);
113
+ oversizedConfig.set(fallback);
114
+ expect(simulatorAvcConfig(oversizedConfig)).toBe(oversizedConfig);
115
+ const oversizedSps = new Uint8Array(6 + 2 + 4097 + 7);
116
+ oversizedSps.set(fallback.subarray(0, 6));
117
+ oversizedSps.set([0x10, 1], 6);
118
+ oversizedSps.set(fallback.subarray(8, 20), 8);
119
+ oversizedSps.set(fallback.subarray(20), 8 + 4097);
120
+ expect(simulatorAvcConfig(oversizedSps)).toBe(oversizedSps);
121
+ });
122
+});
packages/client-runtime/src/device/avcConfig.ts
new
+141
@@ -0,0 +1,141 @@
1
+const MAX_CONFIG_BYTES = 64 * 1024;
2
+const MAX_SPS_BYTES = 4096;
3
+
4
+const unsignedGolomb = (value: number) => {
5
+ const bits = (value + 1).toString(2);
6
+ return "0".repeat(bits.length - 1) + bits;
7
+};
8
+
9
+function withReorderLimits(sps: Uint8Array): Uint8Array {
10
+ if (sps.length < 4 || sps.length > MAX_SPS_BYTES || (sps[0]! & 0x9f) !== 7) return sps;
11
+ const rbsp: number[] = [];
12
+ let zeros = 0;
13
+ for (let index = 1; index < sps.length; index++) {
14
+ const byte = sps[index]!;
15
+ if (zeros >= 2 && byte === 3) {
16
+ if (index + 1 >= sps.length || sps[index + 1]! > 3) return sps;
17
+ zeros = 0;
18
+ continue;
19
+ }
20
+ rbsp.push(byte);
21
+ zeros = byte === 0 ? zeros + 1 : 0;
22
+ }
23
+ const bits = rbsp.map((byte) => byte.toString(2).padStart(8, "0")).join("");
24
+ let position = 0;
25
+ const read = (count: number) => {
26
+ if (position + count > bits.length) throw new Error("Truncated H.264 SPS");
27
+ const value = Number.parseInt(bits.slice(position, position + count), 2);
28
+ position += count;
29
+ return value;
30
+ };
31
+ const readUnsignedGolomb = () => {
32
+ let leadingZeros = 0;
33
+ while (read(1) === 0) {
34
+ if (++leadingZeros > 31) throw new Error("Invalid H.264 Exp-Golomb value");
35
+ }
36
+ return 2 ** leadingZeros - 1 + (leadingZeros === 0 ? 0 : read(leadingZeros));
37
+ };
38
+
39
+ try {
40
+ // Only handle the hub's 8-bit 4:2:0 High profile without scaling lists.
41
+ // Unknown encoder configurations pass through rather than guessing their layout.
42
+ if (read(8) !== 100) return sps;
43
+ read(16);
44
+ if (readUnsignedGolomb() > 31 || readUnsignedGolomb() !== 1) return sps;
45
+ if (readUnsignedGolomb() !== 0 || readUnsignedGolomb() !== 0) return sps;
46
+ read(1);
47
+ if (read(1) !== 0) return sps;
48
+ if (readUnsignedGolomb() > 12) return sps;
49
+ const pictureOrderCountType = readUnsignedGolomb();
50
+ if (pictureOrderCountType === 0) {
51
+ if (readUnsignedGolomb() > 12) return sps;
52
+ } else if (pictureOrderCountType === 1) {
53
+ read(1);
54
+ readUnsignedGolomb();
55
+ readUnsignedGolomb();
56
+ const cycleLength = readUnsignedGolomb();
57
+ if (cycleLength > 255) return sps;
58
+ for (let index = 0; index < cycleLength; index++) readUnsignedGolomb();
59
+ } else if (pictureOrderCountType !== 2) return sps;
60
+ const maxReferenceFrames = readUnsignedGolomb();
61
+ if (maxReferenceFrames > 16) return sps;
62
+ read(1);
63
+ if (readUnsignedGolomb() > 65535 || readUnsignedGolomb() > 65535) return sps;
64
+ if (read(1) === 0) read(1);
65
+ read(1);
66
+ if (read(1) !== 0) {
67
+ for (let index = 0; index < 4; index++) readUnsignedGolomb();
68
+ }
69
+ const vuiPosition = position;
70
+ if (read(1) !== 0) return sps;
71
+ // With no VUI, only rbsp_stop_one_bit and byte padding may follow.
72
+ if (!/^10{0,7}$/.test(bits.slice(position))) return sps;
73
+
74
+ // serve-sim disables frame reordering. Its fallback encoder omits the VUI,
75
+ // causing Chromium to infer a large reorder queue despite optimizeForLatency.
76
+ // Preserve the SPS and add VUI defaults with max_num_reorder_frames = 0.
77
+ // The flag bits enable VUI, omit its eight optional sections, and enable
78
+ // bitstream restrictions and motion_vectors_over_pic_boundaries_flag.
79
+ const restricted =
80
+ bits.slice(0, vuiPosition) +
81
+ "10000000011" +
82
+ [2, 1, 16, 16, 0, maxReferenceFrames].map(unsignedGolomb).join("") +
83
+ "1";
84
+ const padded = restricted.padEnd(Math.ceil(restricted.length / 8) * 8, "0");
85
+ const output = [sps[0]!];
86
+ zeros = 0;
87
+ for (let index = 0; index < padded.length; index += 8) {
88
+ const byte = Number.parseInt(padded.slice(index, index + 8), 2);
89
+ if (zeros >= 2 && byte <= 3) {
90
+ output.push(3);
91
+ zeros = 0;
92
+ }
93
+ output.push(byte);
94
+ zeros = byte === 0 ? zeros + 1 : 0;
95
+ }
96
+ return Uint8Array.from(output);
97
+ } catch {
98
+ return sps;
99
+ }
100
+}
101
+
102
+/** Repair missing reorder metadata only for serve-sim's non-reordering AVCC feed. */
103
+export function simulatorAvcConfig(description: Uint8Array): Uint8Array {
104
+ if (description.length < 7 || description.length > MAX_CONFIG_BYTES || description[0] !== 1)
105
+ return description;
106
+ const parameterSets = description[5]! & 0x1f;
107
+ if (parameterSets === 0) return description;
108
+ const parts: Uint8Array[] = [description.subarray(0, 6)];
109
+ let position = 6;
110
+ let changed = false;
111
+ for (let index = 0; index < parameterSets; index++) {
112
+ if (position + 2 > description.length) return description;
113
+ const length = description[position]! * 256 + description[position + 1]!;
114
+ position += 2;
115
+ if (length === 0 || position + length > description.length) return description;
116
+ const sps = description.subarray(position, position + length);
117
+ const repaired = withReorderLimits(sps);
118
+ changed ||= repaired !== sps;
119
+ parts.push(Uint8Array.of(repaired.length >>> 8, repaired.length & 0xff), repaired);
120
+ position += length;
121
+ }
122
+ const suffix = position;
123
+ if (position >= description.length) return description;
124
+ const pictureParameterSets = description[position++]!;
125
+ for (let index = 0; index < pictureParameterSets; index++) {
126
+ if (position + 2 > description.length) return description;
127
+ const length = description[position]! * 256 + description[position + 1]!;
128
+ position += 2;
129
+ if (length === 0 || position + length > description.length) return description;
130
+ position += length;
131
+ }
132
+ if (!changed) return description;
133
+ parts.push(description.subarray(suffix));
134
+ const output = new Uint8Array(parts.reduce((length, part) => length + part.length, 0));
135
+ position = 0;
136
+ for (const part of parts) {
137
+ output.set(part, position);
138
+ position += part.length;
139
+ }
140
+ return output;
141
+}
packages/client-runtime/src/device/stream.test.ts
+30
@@ -323,6 +323,36 @@ describe("shared device stream readiness and recovery", () => {
323
vi.unstubAllGlobals();
324
});
325
326
+ it("repairs the hub's fallback configuration without disabling hardware decoding", async () => {
327
+ const { client, videoBodies, Decoder } = recoveryFixture("ios", false);
328
+ try {
329
+ client.start();
330
+ await vi.advanceTimersByTimeAsync(0);
331
+ videoBodies[0]!.enqueue(
332
+ Uint8Array.from(
333
+ envelope(
334
+ 1,
335
+ [
336
+ 1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1,
337
+ 0, 4, 40, 238, 60, 176,
338
+ ],
339
+ ),
340
+ ),
341
+ );
342
+ await vi.advanceTimersByTimeAsync(0);
343
+ expect(Decoder.isConfigSupported).toHaveBeenCalledExactlyOnceWith({
344
+ codec: "avc1.640033",
345
+ description: Uint8Array.from([
346
+ 1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208,
347
+ 68, 35, 80, 1, 0, 4, 40, 238, 60, 176,
348
+ ]),
349
+ optimizeForLatency: true,
350
+ });
351
+ } finally {
352
+ client.stop();
353
+ }
354
+ });
355
+
356
it("does not repaint a delayed JPEG seed over newer decoded video", async () => {
357
const { client, videoBodies, drawImage, decodedFrame } = recoveryFixture("ios", false);
358
let resolveBitmap!: (bitmap: ImageBitmap) => void;
packages/client-runtime/src/device/stream.ts
+5
-1
@@ -10,6 +10,7 @@ import {
10
import * as Schema from "effect/Schema";
11
import * as Option from "effect/Option";
12
import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
13
+import { simulatorAvcConfig } from "./avcConfig.ts";
14
import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
15
import type { DevicePlatform } from "@t3tools/contracts";
16
@@ -759,7 +760,10 @@ export function createDeviceStreamClient(
760
case "description": {
761
awaitingKeyframe = true;
762
const configured = await configureDecoder(
762
- { codec: avcCodecString(chunk.payload), description: chunk.payload },
763
+ {
764
+ codec: avcCodecString(chunk.payload),
765
+ description: simulatorAvcConfig(chunk.payload),
766
+ },
767
isCurrent,
768
);
769
if (!isCurrent()) return;