@devries-cameron20 / t3code-1 / commits / ca049dc205
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(mobile): restore low-latency decoding for simulator streams (#22733)

Yash Singh committed Oct 21, 2026 at 02:45 UTC ca049dc2053806a35d55c38d5c4e7f7e9cf13659
5 files changed +306 -5
docs/internals/devices.md
+8 -4
@@ -105,10 +105,14 @@ binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over
105 one WebSocket. [`deviceStream.ts`](../../apps/web/src/components/device/deviceStream.ts)
106 speaks both so one panel covers both platforms.
107
108 -Simulators encode H.264 High 5.1. Hardware decoders on some machines and all
109 -headless browsers reject that profile, and WebCodecs is secure-context only, so
110 -the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on
111 -iOS. Android has no MJPEG; there the panel reports that it cannot decode.
108 +Simulators encode H.264 High 5.1 without frame reordering, but the hub's fallback
109 +encoder can omit the SPS VUI that declares its reorder limits. Chromium's hardware
110 +decoder then retains frames for seconds despite `optimizeForLatency` and an empty
111 +`decodeQueueSize`. The iOS AVCC viewer restores the missing metadata before
112 +configuring the decoder so hardware decoding can remain enabled. WebCodecs is
113 +secure-context only, so the viewer probes `isConfigSupported` and falls back to
114 +MJPEG on iOS when decoding is unavailable. Android has no MJPEG; there the panel
115 +reports that it cannot decode.
116
117 serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses
118 AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so
packages/client-runtime/src/device/avcConfig.test.ts new
+122
@@ -0,0 +1,122 @@
1 +import { describe, expect, it } from "vite-plus/test";
2 +
3 +import { simulatorAvcConfig } from "./avcConfig.ts";
4 +
5 +// Captured from the hub before and after its encoder falls back from low-latency rate control.
6 +const lowLatency = Uint8Array.from([
7 + 1, 100, 0, 51, 255, 225, 0, 22, 39, 100, 0, 51, 172, 19, 20, 60, 4, 192, 20, 158, 106, 154, 129,
8 + 1, 1, 3, 194, 1, 8, 248, 1, 0, 4, 40, 238, 60, 176,
9 +]);
10 +const fallback = Uint8Array.from([
11 + 1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1, 0, 4, 40,
12 + 238, 60, 176,
13 +]);
14 +const repaired = Uint8Array.from([
15 + 1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208, 68, 35,
16 + 80, 1, 0, 4, 40, 238, 60, 176,
17 +]);
18 +
19 +describe("simulator AVC reorder metadata", () => {
20 + it("adds zero reordering and one reference frame to the captured fallback SPS", () => {
21 + expect(simulatorAvcConfig(fallback)).toEqual(repaired);
22 + expect(simulatorAvcConfig(repaired)).toBe(repaired);
23 + });
24 +
25 + it("preserves existing VUI and leaves the original codec description untouched", () => {
26 + const original = fallback.slice();
27 + simulatorAvcConfig(fallback);
28 + expect(fallback).toEqual(original);
29 + expect(simulatorAvcConfig(lowLatency)).toBe(lowLatency);
30 + });
31 +
32 + it("repairs multiple SPS entries while preserving PPS and profile extensions", () => {
33 + const extensions = [0xfd, 0xf8, 0xf8, 0];
34 + const description = Uint8Array.from([
35 + ...fallback.subarray(0, 5),
36 + 0xe2,
37 + ...fallback.subarray(6, 20),
38 + ...lowLatency.subarray(6, 30),
39 + ...fallback.subarray(20),
40 + ...extensions,
41 + ]);
42 + expect(simulatorAvcConfig(description)).toEqual(
43 + Uint8Array.from([
44 + ...description.subarray(0, 6),
45 + ...repaired.subarray(6, 25),
46 + ...lowLatency.subarray(6, 30),
47 + ...fallback.subarray(20),
48 + ...extensions,
49 + ]),
50 + );
51 + });
52 +
53 + it("passes through every truncated prefix of the captured configurations", () => {
54 + for (const config of [fallback, lowLatency]) {
55 + for (let length = 0; length < config.length; length++) {
56 + const truncated = config.subarray(0, length);
57 + expect(simulatorAvcConfig(truncated)).toBe(truncated);
58 + }
59 + }
60 + });
61 +
62 + it("does not modify unsupported profiles, scaling lists, or invalid parameter-set lengths", () => {
63 + const unsupportedProfile = fallback.slice();
64 + unsupportedProfile[9] = 110;
65 + const scalingLists = fallback.slice();
66 + scalingLists[12] = scalingLists[12]! | 1;
67 + const invalidSpsLength = fallback.slice();
68 + invalidSpsLength[6] = 0xff;
69 + const invalidPpsLength = fallback.slice();
70 + invalidPpsLength[21] = 0xff;
71 + for (const config of [unsupportedProfile, scalingLists, invalidSpsLength, invalidPpsLength]) {
72 + expect(simulatorAvcConfig(config)).toBe(config);
73 + }
74 + });
75 +
76 + it("requires valid SPS trailing bits before appending VUI metadata", () => {
77 + const invalidPadding = fallback.slice();
78 + invalidPadding[19] = 0;
79 + expect(simulatorAvcConfig(invalidPadding)).toBe(invalidPadding);
80 + });
81 +
82 + it("passes through invalid NAL headers and unterminated Exp-Golomb values", () => {
83 + const forbiddenBit = fallback.slice();
84 + forbiddenBit[8] = forbiddenBit[8]! | 0x80;
85 + expect(simulatorAvcConfig(forbiddenBit)).toBe(forbiddenBit);
86 + const unterminated = fallback.slice();
87 + unterminated.fill(0, 12, 20);
88 + expect(simulatorAvcConfig(unterminated)).toBe(unterminated);
89 + const overlong = Uint8Array.from([
90 + ...fallback.subarray(0, 6),
91 + 0,
92 + 13,
93 + 39,
94 + 100,
95 + 0,
96 + 51,
97 + 0,
98 + 0,
99 + 3,
100 + 0,
101 + 0,
102 + 3,
103 + 0,
104 + 0,
105 + 0,
106 + ...fallback.subarray(20),
107 + ]);
108 + expect(simulatorAvcConfig(overlong)).toBe(overlong);
109 + });
110 +
111 + it("bounds parsing of oversized configurations and SPS data", () => {
112 + const oversizedConfig = new Uint8Array(64 * 1024 + 1);
113 + oversizedConfig.set(fallback);
114 + expect(simulatorAvcConfig(oversizedConfig)).toBe(oversizedConfig);
115 + const oversizedSps = new Uint8Array(6 + 2 + 4097 + 7);
116 + oversizedSps.set(fallback.subarray(0, 6));
117 + oversizedSps.set([0x10, 1], 6);
118 + oversizedSps.set(fallback.subarray(8, 20), 8);
119 + oversizedSps.set(fallback.subarray(20), 8 + 4097);
120 + expect(simulatorAvcConfig(oversizedSps)).toBe(oversizedSps);
121 + });
122 +});
packages/client-runtime/src/device/avcConfig.ts new
+141
@@ -0,0 +1,141 @@
1 +const MAX_CONFIG_BYTES = 64 * 1024;
2 +const MAX_SPS_BYTES = 4096;
3 +
4 +const unsignedGolomb = (value: number) => {
5 + const bits = (value + 1).toString(2);
6 + return "0".repeat(bits.length - 1) + bits;
7 +};
8 +
9 +function withReorderLimits(sps: Uint8Array): Uint8Array {
10 + if (sps.length < 4 || sps.length > MAX_SPS_BYTES || (sps[0]! & 0x9f) !== 7) return sps;
11 + const rbsp: number[] = [];
12 + let zeros = 0;
13 + for (let index = 1; index < sps.length; index++) {
14 + const byte = sps[index]!;
15 + if (zeros >= 2 && byte === 3) {
16 + if (index + 1 >= sps.length || sps[index + 1]! > 3) return sps;
17 + zeros = 0;
18 + continue;
19 + }
20 + rbsp.push(byte);
21 + zeros = byte === 0 ? zeros + 1 : 0;
22 + }
23 + const bits = rbsp.map((byte) => byte.toString(2).padStart(8, "0")).join("");
24 + let position = 0;
25 + const read = (count: number) => {
26 + if (position + count > bits.length) throw new Error("Truncated H.264 SPS");
27 + const value = Number.parseInt(bits.slice(position, position + count), 2);
28 + position += count;
29 + return value;
30 + };
31 + const readUnsignedGolomb = () => {
32 + let leadingZeros = 0;
33 + while (read(1) === 0) {
34 + if (++leadingZeros > 31) throw new Error("Invalid H.264 Exp-Golomb value");
35 + }
36 + return 2 ** leadingZeros - 1 + (leadingZeros === 0 ? 0 : read(leadingZeros));
37 + };
38 +
39 + try {
40 + // Only handle the hub's 8-bit 4:2:0 High profile without scaling lists.
41 + // Unknown encoder configurations pass through rather than guessing their layout.
42 + if (read(8) !== 100) return sps;
43 + read(16);
44 + if (readUnsignedGolomb() > 31 || readUnsignedGolomb() !== 1) return sps;
45 + if (readUnsignedGolomb() !== 0 || readUnsignedGolomb() !== 0) return sps;
46 + read(1);
47 + if (read(1) !== 0) return sps;
48 + if (readUnsignedGolomb() > 12) return sps;
49 + const pictureOrderCountType = readUnsignedGolomb();
50 + if (pictureOrderCountType === 0) {
51 + if (readUnsignedGolomb() > 12) return sps;
52 + } else if (pictureOrderCountType === 1) {
53 + read(1);
54 + readUnsignedGolomb();
55 + readUnsignedGolomb();
56 + const cycleLength = readUnsignedGolomb();
57 + if (cycleLength > 255) return sps;
58 + for (let index = 0; index < cycleLength; index++) readUnsignedGolomb();
59 + } else if (pictureOrderCountType !== 2) return sps;
60 + const maxReferenceFrames = readUnsignedGolomb();
61 + if (maxReferenceFrames > 16) return sps;
62 + read(1);
63 + if (readUnsignedGolomb() > 65535 || readUnsignedGolomb() > 65535) return sps;
64 + if (read(1) === 0) read(1);
65 + read(1);
66 + if (read(1) !== 0) {
67 + for (let index = 0; index < 4; index++) readUnsignedGolomb();
68 + }
69 + const vuiPosition = position;
70 + if (read(1) !== 0) return sps;
71 + // With no VUI, only rbsp_stop_one_bit and byte padding may follow.
72 + if (!/^10{0,7}$/.test(bits.slice(position))) return sps;
73 +
74 + // serve-sim disables frame reordering. Its fallback encoder omits the VUI,
75 + // causing Chromium to infer a large reorder queue despite optimizeForLatency.
76 + // Preserve the SPS and add VUI defaults with max_num_reorder_frames = 0.
77 + // The flag bits enable VUI, omit its eight optional sections, and enable
78 + // bitstream restrictions and motion_vectors_over_pic_boundaries_flag.
79 + const restricted =
80 + bits.slice(0, vuiPosition) +
81 + "10000000011" +
82 + [2, 1, 16, 16, 0, maxReferenceFrames].map(unsignedGolomb).join("") +
83 + "1";
84 + const padded = restricted.padEnd(Math.ceil(restricted.length / 8) * 8, "0");
85 + const output = [sps[0]!];
86 + zeros = 0;
87 + for (let index = 0; index < padded.length; index += 8) {
88 + const byte = Number.parseInt(padded.slice(index, index + 8), 2);
89 + if (zeros >= 2 && byte <= 3) {
90 + output.push(3);
91 + zeros = 0;
92 + }
93 + output.push(byte);
94 + zeros = byte === 0 ? zeros + 1 : 0;
95 + }
96 + return Uint8Array.from(output);
97 + } catch {
98 + return sps;
99 + }
100 +}
101 +
102 +/** Repair missing reorder metadata only for serve-sim's non-reordering AVCC feed. */
103 +export function simulatorAvcConfig(description: Uint8Array): Uint8Array {
104 + if (description.length < 7 || description.length > MAX_CONFIG_BYTES || description[0] !== 1)
105 + return description;
106 + const parameterSets = description[5]! & 0x1f;
107 + if (parameterSets === 0) return description;
108 + const parts: Uint8Array[] = [description.subarray(0, 6)];
109 + let position = 6;
110 + let changed = false;
111 + for (let index = 0; index < parameterSets; index++) {
112 + if (position + 2 > description.length) return description;
113 + const length = description[position]! * 256 + description[position + 1]!;
114 + position += 2;
115 + if (length === 0 || position + length > description.length) return description;
116 + const sps = description.subarray(position, position + length);
117 + const repaired = withReorderLimits(sps);
118 + changed ||= repaired !== sps;
119 + parts.push(Uint8Array.of(repaired.length >>> 8, repaired.length & 0xff), repaired);
120 + position += length;
121 + }
122 + const suffix = position;
123 + if (position >= description.length) return description;
124 + const pictureParameterSets = description[position++]!;
125 + for (let index = 0; index < pictureParameterSets; index++) {
126 + if (position + 2 > description.length) return description;
127 + const length = description[position]! * 256 + description[position + 1]!;
128 + position += 2;
129 + if (length === 0 || position + length > description.length) return description;
130 + position += length;
131 + }
132 + if (!changed) return description;
133 + parts.push(description.subarray(suffix));
134 + const output = new Uint8Array(parts.reduce((length, part) => length + part.length, 0));
135 + position = 0;
136 + for (const part of parts) {
137 + output.set(part, position);
138 + position += part.length;
139 + }
140 + return output;
141 +}
packages/client-runtime/src/device/stream.test.ts
+30
@@ -323,6 +323,36 @@ describe("shared device stream readiness and recovery", () => {
323 vi.unstubAllGlobals();
324 });
325
326 + it("repairs the hub's fallback configuration without disabling hardware decoding", async () => {
327 + const { client, videoBodies, Decoder } = recoveryFixture("ios", false);
328 + try {
329 + client.start();
330 + await vi.advanceTimersByTimeAsync(0);
331 + videoBodies[0]!.enqueue(
332 + Uint8Array.from(
333 + envelope(
334 + 1,
335 + [
336 + 1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1,
337 + 0, 4, 40, 238, 60, 176,
338 + ],
339 + ),
340 + ),
341 + );
342 + await vi.advanceTimersByTimeAsync(0);
343 + expect(Decoder.isConfigSupported).toHaveBeenCalledExactlyOnceWith({
344 + codec: "avc1.640033",
345 + description: Uint8Array.from([
346 + 1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208,
347 + 68, 35, 80, 1, 0, 4, 40, 238, 60, 176,
348 + ]),
349 + optimizeForLatency: true,
350 + });
351 + } finally {
352 + client.stop();
353 + }
354 + });
355 +
356 it("does not repaint a delayed JPEG seed over newer decoded video", async () => {
357 const { client, videoBodies, drawImage, decodedFrame } = recoveryFixture("ios", false);
358 let resolveBitmap!: (bitmap: ImageBitmap) => void;
packages/client-runtime/src/device/stream.ts
+5 -1
@@ -10,6 +10,7 @@ import {
10 import * as Schema from "effect/Schema";
11 import * as Option from "effect/Option";
12 import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
13 +import { simulatorAvcConfig } from "./avcConfig.ts";
14 import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
15 import type { DevicePlatform } from "@t3tools/contracts";
16
@@ -759,7 +760,10 @@ export function createDeviceStreamClient(
760 case "description": {
761 awaitingKeyframe = true;
762 const configured = await configureDecoder(
762 - { codec: avcCodecString(chunk.payload), description: chunk.payload },
763 + {
764 + codec: avcCodecString(chunk.payload),
765 + description: simulatorAvcConfig(chunk.payload),
766 + },
767 isCurrent,
768 );
769 if (!isCurrent()) return;