@devries-cameron20 / t3code-1 / commits / 54c5f25249
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(device): use expo-device-hub 0.19.1 (#28346)

Yash Singh committed Oct 31, 2026 at 16:30 UTC 54c5f25249961a1aa0b61932329092eadbd9d6fc
6 files changed +3 -301
apps/server/src/device/deviceToolVersions.json
+1 -1
@@ -1,4 +1,4 @@
1 {
2 "agent-device": "0.21.31",
3 - "expo-device-hub": "0.19.0-canary-20261023-1357114"
3 + "expo-device-hub": "0.19.1"
4 }
docs/internals/devices.md
+1 -6
@@ -141,12 +141,7 @@ binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over
141 one WebSocket. [`DeviceStreamView.tsx`](../../apps/web/src/components/device/DeviceStreamView.tsx)
142 speaks both so one panel covers both platforms.
143
144 -Simulators encode H.264 High 5.1 without frame reordering, but the hub's fallback
145 -encoder can omit the SPS VUI that declares its reorder limits. Chromium's hardware
146 -decoder then retains frames for seconds despite `optimizeForLatency` and an empty
147 -`decodeQueueSize`. The iOS AVCC viewer restores the missing metadata before
148 -configuring the decoder so hardware decoding can remain enabled. WebCodecs is
149 -secure-context only, so the viewer probes `isConfigSupported` and falls back to
144 +WebCodecs is secure-context only, so the viewer probes `isConfigSupported` and falls back to
145 MJPEG on iOS when decoding is unavailable. Android's SEMU socket has no MJPEG;
146 there the panel reports that it cannot decode.
147
packages/client-runtime/src/device/avcConfig.test.ts deleted
-122
@@ -1,122 +0,0 @@
1 -import { describe, expect, it } from "vite-plus/test";
2 -
3 -import { simulatorAvcConfig } from "./avcConfig.ts";
4 -
5 -// Captured from the hub before and after its encoder falls back from low-latency rate control.
6 -const lowLatency = Uint8Array.from([
7 - 1, 100, 0, 51, 255, 225, 0, 22, 39, 100, 0, 51, 172, 19, 20, 60, 4, 192, 20, 158, 106, 154, 129,
8 - 1, 1, 3, 194, 1, 8, 248, 1, 0, 4, 40, 238, 60, 176,
9 -]);
10 -const fallback = Uint8Array.from([
11 - 1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1, 0, 4, 40,
12 - 238, 60, 176,
13 -]);
14 -const repaired = Uint8Array.from([
15 - 1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208, 68, 35,
16 - 80, 1, 0, 4, 40, 238, 60, 176,
17 -]);
18 -
19 -describe("simulator AVC reorder metadata", () => {
20 - it("adds zero reordering and one reference frame to the captured fallback SPS", () => {
21 - expect(simulatorAvcConfig(fallback)).toEqual(repaired);
22 - expect(simulatorAvcConfig(repaired)).toBe(repaired);
23 - });
24 -
25 - it("preserves existing VUI and leaves the original codec description untouched", () => {
26 - const original = fallback.slice();
27 - simulatorAvcConfig(fallback);
28 - expect(fallback).toEqual(original);
29 - expect(simulatorAvcConfig(lowLatency)).toBe(lowLatency);
30 - });
31 -
32 - it("repairs multiple SPS entries while preserving PPS and profile extensions", () => {
33 - const extensions = [0xfd, 0xf8, 0xf8, 0];
34 - const description = Uint8Array.from([
35 - ...fallback.subarray(0, 5),
36 - 0xe2,
37 - ...fallback.subarray(6, 20),
38 - ...lowLatency.subarray(6, 30),
39 - ...fallback.subarray(20),
40 - ...extensions,
41 - ]);
42 - expect(simulatorAvcConfig(description)).toEqual(
43 - Uint8Array.from([
44 - ...description.subarray(0, 6),
45 - ...repaired.subarray(6, 25),
46 - ...lowLatency.subarray(6, 30),
47 - ...fallback.subarray(20),
48 - ...extensions,
49 - ]),
50 - );
51 - });
52 -
53 - it("passes through every truncated prefix of the captured configurations", () => {
54 - for (const config of [fallback, lowLatency]) {
55 - for (let length = 0; length < config.length; length++) {
56 - const truncated = config.subarray(0, length);
57 - expect(simulatorAvcConfig(truncated)).toBe(truncated);
58 - }
59 - }
60 - });
61 -
62 - it("does not modify unsupported profiles, scaling lists, or invalid parameter-set lengths", () => {
63 - const unsupportedProfile = fallback.slice();
64 - unsupportedProfile[9] = 110;
65 - const scalingLists = fallback.slice();
66 - scalingLists[12] = scalingLists[12]! | 1;
67 - const invalidSpsLength = fallback.slice();
68 - invalidSpsLength[6] = 0xff;
69 - const invalidPpsLength = fallback.slice();
70 - invalidPpsLength[21] = 0xff;
71 - for (const config of [unsupportedProfile, scalingLists, invalidSpsLength, invalidPpsLength]) {
72 - expect(simulatorAvcConfig(config)).toBe(config);
73 - }
74 - });
75 -
76 - it("requires valid SPS trailing bits before appending VUI metadata", () => {
77 - const invalidPadding = fallback.slice();
78 - invalidPadding[19] = 0;
79 - expect(simulatorAvcConfig(invalidPadding)).toBe(invalidPadding);
80 - });
81 -
82 - it("passes through invalid NAL headers and unterminated Exp-Golomb values", () => {
83 - const forbiddenBit = fallback.slice();
84 - forbiddenBit[8] = forbiddenBit[8]! | 0x80;
85 - expect(simulatorAvcConfig(forbiddenBit)).toBe(forbiddenBit);
86 - const unterminated = fallback.slice();
87 - unterminated.fill(0, 12, 20);
88 - expect(simulatorAvcConfig(unterminated)).toBe(unterminated);
89 - const overlong = Uint8Array.from([
90 - ...fallback.subarray(0, 6),
91 - 0,
92 - 13,
93 - 39,
94 - 100,
95 - 0,
96 - 51,
97 - 0,
98 - 0,
99 - 3,
100 - 0,
101 - 0,
102 - 3,
103 - 0,
104 - 0,
105 - 0,
106 - ...fallback.subarray(20),
107 - ]);
108 - expect(simulatorAvcConfig(overlong)).toBe(overlong);
109 - });
110 -
111 - it("bounds parsing of oversized configurations and SPS data", () => {
112 - const oversizedConfig = new Uint8Array(64 * 1024 + 1);
113 - oversizedConfig.set(fallback);
114 - expect(simulatorAvcConfig(oversizedConfig)).toBe(oversizedConfig);
115 - const oversizedSps = new Uint8Array(6 + 2 + 4097 + 7);
116 - oversizedSps.set(fallback.subarray(0, 6));
117 - oversizedSps.set([0x10, 1], 6);
118 - oversizedSps.set(fallback.subarray(8, 20), 8);
119 - oversizedSps.set(fallback.subarray(20), 8 + 4097);
120 - expect(simulatorAvcConfig(oversizedSps)).toBe(oversizedSps);
121 - });
122 -});
packages/client-runtime/src/device/avcConfig.ts deleted
-140
@@ -1,140 +0,0 @@
1 -const MAX_CONFIG_BYTES = 64 * 1024;
2 -const MAX_SPS_BYTES = 4096;
3 -
4 -const unsignedGolomb = (value: number) => {
5 - const bits = (value + 1).toString(2);
6 - return "0".repeat(bits.length - 1) + bits;
7 -};
8 -
9 -function withReorderLimits(sps: Uint8Array): Uint8Array {
10 - if (sps.length < 4 || sps.length > MAX_SPS_BYTES || (sps[0]! & 0x9f) !== 7) return sps;
11 - const rbsp: number[] = [];
12 - let zeros = 0;
13 - for (let index = 1; index < sps.length; index++) {
14 - const byte = sps[index]!;
15 - if (zeros >= 2 && byte === 3) {
16 - if (index + 1 >= sps.length || sps[index + 1]! > 3) return sps;
17 - zeros = 0;
18 - continue;
19 - }
20 - rbsp.push(byte);
21 - zeros = byte === 0 ? zeros + 1 : 0;
22 - }
23 - const bits = rbsp.map((byte) => byte.toString(2).padStart(8, "0")).join("");
24 - let position = 0;
25 - const read = (count: number) => {
26 - if (position + count > bits.length) throw new Error("Truncated H.264 SPS");
27 - const value = Number.parseInt(bits.slice(position, position + count), 2);
28 - position += count;
29 - return value;
30 - };
31 - const readUnsignedGolomb = () => {
32 - let leadingZeros = 0;
33 - while (read(1) === 0) {
34 - if (++leadingZeros > 31) throw new Error("Invalid H.264 Exp-Golomb value");
35 - }
36 - return 2 ** leadingZeros - 1 + (leadingZeros === 0 ? 0 : read(leadingZeros));
37 - };
38 -
39 - try {
40 - // Only handle the hub's 8-bit 4:2:0 High profile without scaling lists.
41 - // Unknown encoder configurations pass through rather than guessing their layout.
42 - if (read(8) !== 100) return sps;
43 - read(16);
44 - if (readUnsignedGolomb() > 31 || readUnsignedGolomb() !== 1) return sps;
45 - if (readUnsignedGolomb() !== 0 || readUnsignedGolomb() !== 0) return sps;
46 - read(1);
47 - if (read(1) !== 0) return sps;
48 - if (readUnsignedGolomb() > 12) return sps;
49 - const pictureOrderCountType = readUnsignedGolomb();
50 - if (pictureOrderCountType === 0) {
51 - if (readUnsignedGolomb() > 12) return sps;
52 - } else if (pictureOrderCountType === 1) {
53 - read(1);
54 - readUnsignedGolomb();
55 - readUnsignedGolomb();
56 - const cycleLength = readUnsignedGolomb();
57 - if (cycleLength > 255) return sps;
58 - for (let index = 0; index < cycleLength; index++) readUnsignedGolomb();
59 - } else if (pictureOrderCountType !== 2) return sps;
60 - const maxReferenceFrames = readUnsignedGolomb();
61 - if (maxReferenceFrames > 16) return sps;
62 - read(1);
63 - if (readUnsignedGolomb() > 65535 || readUnsignedGolomb() > 65535) return sps;
64 - if (read(1) === 0) read(1);
65 - read(1);
66 - if (read(1) !== 0) {
67 - for (let index = 0; index < 4; index++) readUnsignedGolomb();
68 - }
69 - const vuiPosition = position;
70 - if (read(1) !== 0) return sps;
71 - // With no VUI, only rbsp_stop_one_bit and byte padding may follow.
72 - if (!/^10{0,7}$/.test(bits.slice(position))) return sps;
73 -
74 - // serve-sim disables reordering, but its fallback encoder omits VUI, so Chromium
75 - // infers a large queue despite optimizeForLatency. Add VUI with zero reorder frames.
76 - // Flags omit eight optional sections and enable bitstream restrictions
77 - // and motion_vectors_over_pic_boundaries_flag.
78 - const restricted =
79 - bits.slice(0, vuiPosition) +
80 - "10000000011" +
81 - [2, 1, 16, 16, 0, maxReferenceFrames].map(unsignedGolomb).join("") +
82 - "1";
83 - const padded = restricted.padEnd(Math.ceil(restricted.length / 8) * 8, "0");
84 - const output = [sps[0]!];
85 - zeros = 0;
86 - for (let index = 0; index < padded.length; index += 8) {
87 - const byte = Number.parseInt(padded.slice(index, index + 8), 2);
88 - if (zeros >= 2 && byte <= 3) {
89 - output.push(3);
90 - zeros = 0;
91 - }
92 - output.push(byte);
93 - zeros = byte === 0 ? zeros + 1 : 0;
94 - }
95 - return Uint8Array.from(output);
96 - } catch {
97 - return sps;
98 - }
99 -}
100 -
101 -/** Repair missing reorder metadata only for serve-sim's non-reordering AVCC feed. */
102 -export function simulatorAvcConfig(description: Uint8Array): Uint8Array {
103 - if (description.length < 7 || description.length > MAX_CONFIG_BYTES || description[0] !== 1)
104 - return description;
105 - const parameterSets = description[5]! & 0x1f;
106 - if (parameterSets === 0) return description;
107 - const parts: Uint8Array[] = [description.subarray(0, 6)];
108 - let position = 6;
109 - let changed = false;
110 - for (let index = 0; index < parameterSets; index++) {
111 - if (position + 2 > description.length) return description;
112 - const length = description[position]! * 256 + description[position + 1]!;
113 - position += 2;
114 - if (length === 0 || position + length > description.length) return description;
115 - const sps = description.subarray(position, position + length);
116 - const repaired = withReorderLimits(sps);
117 - changed ||= repaired !== sps;
118 - parts.push(Uint8Array.of(repaired.length >>> 8, repaired.length & 0xff), repaired);
119 - position += length;
120 - }
121 - const suffix = position;
122 - if (position >= description.length) return description;
123 - const pictureParameterSets = description[position++]!;
124 - for (let index = 0; index < pictureParameterSets; index++) {
125 - if (position + 2 > description.length) return description;
126 - const length = description[position]! * 256 + description[position + 1]!;
127 - position += 2;
128 - if (length === 0 || position + length > description.length) return description;
129 - position += length;
130 - }
131 - if (!changed) return description;
132 - parts.push(description.subarray(suffix));
133 - const output = new Uint8Array(parts.reduce((length, part) => length + part.length, 0));
134 - position = 0;
135 - for (const part of parts) {
136 - output.set(part, position);
137 - position += part.length;
138 - }
139 - return output;
140 -}
packages/client-runtime/src/device/stream.test.ts
-30
@@ -421,36 +421,6 @@ describe("shared device stream readiness and recovery", () => {
421 vi.unstubAllGlobals();
422 });
423
424 - it("repairs the hub's fallback configuration without disabling hardware decoding", async () => {
425 - const { client, videoBodies, Decoder } = recoveryFixture("ios", false);
426 - try {
427 - client.start();
428 - await vi.advanceTimersByTimeAsync(0);
429 - videoBodies[0]!.enqueue(
430 - Uint8Array.from(
431 - envelope(
432 - 1,
433 - [
434 - 1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1,
435 - 0, 4, 40, 238, 60, 176,
436 - ],
437 - ),
438 - ),
439 - );
440 - await vi.advanceTimersByTimeAsync(0);
441 - expect(Decoder.isConfigSupported).toHaveBeenCalledExactlyOnceWith({
442 - codec: "avc1.640033",
443 - description: Uint8Array.from([
444 - 1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208,
445 - 68, 35, 80, 1, 0, 4, 40, 238, 60, 176,
446 - ]),
447 - optimizeForLatency: true,
448 - });
449 - } finally {
450 - client.stop();
451 - }
452 - });
453 -
424 it("streams and drives a serve-sim-only Android device over serve-sim's AVCC and input socket", async () => {
425 const { client, events, fetch, sockets, videoBodies, decoders } = recoveryFixture(
426 "android",
packages/client-runtime/src/device/stream.ts
+1 -2
@@ -9,7 +9,6 @@ import {
9 import * as Schema from "effect/Schema";
10 import * as Option from "effect/Option";
11 import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
12 -import { simulatorAvcConfig } from "./avcConfig.ts";
12 import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
13 import type { DevicePlatform } from "@t3tools/contracts";
14
@@ -781,7 +780,7 @@ export function createDeviceStreamClient(
780 const configured = await configureDecoder(
781 {
782 codec: avcCodecString(chunk.payload),
784 - description: simulatorAvcConfig(chunk.payload),
783 + description: chunk.payload,
784 },
785 isCurrent,
786 );