@devries-cameron20 / t3code-1 / commits / 812c0a71c7
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

feat(devices): support Android EAS Simulator sessions (#26797)

Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Nathan Schroeder committed Oct 29, 2026 at 19:36 UTC 812c0a71c785c69c34a0111c3ce6976f64cd5bfe
12 files changed +405 -100
apps/server/src/device/DeviceHubProxy.test.ts
+26
@@ -169,6 +169,32 @@ describe("device hub proxy", () => {
169 expect(headers[0]?.["content-type"]).toBe("application/json");
170 });
171
172 + it("lets a viewer read Android stream capabilities and signal WebRTC with read scope", async () => {
173 + const { handler, requests } = fixture([AuthOrchestrationReadScope]);
174 + const hub = "http://t3.test/api/device-hub/vendor/serve-emu";
175 + const api = await handler(new Request(`${hub}/api?device=session-1`));
176 + expect(api.status).toBe(200);
177 + await api.text();
178 + for (const action of ["offer", "close"]) {
179 + const signal = await handler(
180 + new Request(`${hub}/webrtc/${action}?device=session-1`, {
181 + method: "POST",
182 + headers: { "content-type": "application/json" },
183 + body: '{"sessionId":"viewer"}',
184 + }),
185 + );
186 + expect(signal.status).toBe(200);
187 + await signal.text();
188 + }
189 + expect(requests).toEqual([
190 + "http://hub.test/vendor/serve-emu/api?device=session-1",
191 + "http://hub.test/vendor/serve-emu/webrtc/offer?device=session-1",
192 + "http://hub.test/vendor/serve-emu/webrtc/close?device=session-1",
193 + ]);
194 + expect((await handler(new Request(`${hub}/api`, { method: "POST" }))).status).toBe(405);
195 + expect((await handler(new Request(`${hub}/webrtc/offer`))).status).toBe(405);
196 + });
197 +
198 it("sends requests a host routes itself to its upstream with its credentials", async () => {
199 const { handler, requests, headers } = fixture(
200 [AuthOrchestrationReadScope],
apps/server/src/device/DeviceHubProxy.ts
+8 -4
@@ -27,14 +27,17 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
27 /^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
28 /^\/vendor\/serve-sim\/helper\/[^/]+\/webrtc\/(offer|close)$/,
29 /^\/vendor\/serve-sim\/appstate$/,
30 + /^\/vendor\/serve-emu\/api$/,
31 /^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/,
32 /^\/vendor\/serve-emu\/health$/,
33 ];
34
35 +// Paths that take methods besides GET; one missing from ALLOWED_PATHS takes no GET.
36 const MUTABLE_PATHS: ReadonlyArray<RegExp> = [
37 /^\/vendor\/serve-sim\/api\/screenshot$/,
38 // WebRTC signalling: opening and closing a view of the stream, so read scope.
39 /^\/vendor\/serve-sim\/helper\/[^/]+\/webrtc\/(offer|close)$/,
40 + /^\/vendor\/serve-emu\/webrtc\/(offer|close)$/,
41 /^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/,
42 /^\/vendor\/serve-emu\/api\/fold$/,
43 ];
@@ -178,14 +181,15 @@ const handler = Effect.gen(function* () {
181 }
182 const hubPath = url.value.pathname.slice(DeviceService.DEVICE_HUB_ROUTE_PREFIX.length) || "/";
183 const upgrade = isWebSocketUpgrade(request);
181 - const allowed = (upgrade ? ALLOWED_WS_PATHS : ALLOWED_PATHS).some((pattern) =>
182 - pattern.test(hubPath),
183 - );
184 + const matches = (paths: ReadonlyArray<RegExp>) => paths.some((pattern) => pattern.test(hubPath));
185 + const allowed = upgrade
186 + ? matches(ALLOWED_WS_PATHS)
187 + : matches(ALLOWED_PATHS) || matches(MUTABLE_PATHS);
188 if (!allowed) {
189 return HttpServerResponse.text("Not Found", { status: 404 });
190 }
191 const readOnly = request.method === "GET" || request.method === "HEAD";
188 - if (!upgrade && !readOnly && !MUTABLE_PATHS.some((pattern) => pattern.test(hubPath))) {
192 + if (!upgrade && !matches(readOnly ? ALLOWED_PATHS : MUTABLE_PATHS)) {
193 return HttpServerResponse.text("Method Not Allowed", { status: 405 });
194 }
195 const controlsDevice =
apps/server/src/device/EasDeviceHost.test.ts
+81 -6
@@ -62,6 +62,7 @@ const fakeExpo = (
62 const calls: Array<{ operation: string; variables: Record<string, unknown> }> = [];
63 let created = 0;
64 let createdName = "";
65 + const previews: string[] = [];
66 let previewFails = options.failPreview ?? false;
67 let createFails = options.failCreate ?? false;
68 let sessionGated = false;
@@ -69,6 +70,7 @@ const fakeExpo = (
70 Effect.gen(function* () {
71 const url = new URL(request.url);
72 if (url.origin === "https://preview.test") {
73 + previews.push(`${url.pathname}${url.search}`);
74 expect(request.headers.authorization).toBe("Bearer preview-token");
75 if (options.previewGate) {
76 yield* Deferred.succeed(options.previewGate.requested, undefined);
@@ -77,7 +79,17 @@ const fakeExpo = (
79 if (previewFails) {
80 return HttpClientResponse.fromWeb(request, new Response("bad gateway", { status: 502 }));
81 }
80 - return HttpClientResponse.fromWeb(request, Response.json({ device: "UDID-1" }));
82 + if (url.pathname.endsWith("/api/screenshot")) {
83 + return HttpClientResponse.fromWeb(request, new Response(Uint8Array.of(137, 80, 78, 71)));
84 + }
85 + return HttpClientResponse.fromWeb(
86 + request,
87 + Response.json(
88 + url.pathname === "/vendor/serve-emu/api"
89 + ? { serial: "emulator-5554" }
90 + : { device: "UDID-1" },
91 + ),
92 + );
93 }
94 const body =
95 request.body._tag === "Uint8Array"
@@ -195,7 +207,7 @@ const fakeExpo = (
207 const allowCreate = () => {
208 createFails = false;
209 };
198 - return { http, named, stopped, allowPreview, allowCreate };
210 + return { http, named, stopped, allowPreview, allowCreate, previews };
211 };
212
213 const layer = Layer.mergeAll(
@@ -221,7 +233,10 @@ describe("EAS Simulator host", () => {
233 const host = yield* makeHost(expo, config(), scope);
234 const ready = yield* host.ensureReady(() => Effect.void);
235 const catalog = (yield* ready.devices.list).devices;
224 - expect(catalog.map((device) => [device.id, device.booted])).toEqual([["eas-ios", false]]);
236 + expect(catalog.map((device) => [device.id, device.booted])).toEqual([
237 + ["eas-ios", false],
238 + ["eas-android", false],
239 + ]);
240
241 expect(yield* ready.devices.boot(catalog[0]!)).toBe("session-1");
242 expect(expo.named("T3CreateEasSimulatorSession")[0]?.variables.input).toMatchObject({
@@ -266,6 +281,55 @@ describe("EAS Simulator host", () => {
281 }).pipe(Effect.scoped, Effect.provide(layer)),
282 );
283
284 + it.effect("runs an Android session through serve-emu beside an iOS one", () =>
285 + Effect.gen(function* () {
286 + const expo = fakeExpo();
287 + const scope = yield* Scope.make();
288 + const host = yield* makeHost(expo, config(), scope);
289 + expect(yield* host.platformAvailability("android")).toEqual({
290 + platform: "android",
291 + available: true,
292 + });
293 + const ready = yield* host.ensureReady(() => Effect.void);
294 + const [ios, android] = (yield* ready.devices.list).devices;
295 + expect(yield* ready.devices.boot(ios!)).toBe("session-1");
296 + expect(yield* ready.devices.boot(android!)).toBe("session-2");
297 + expect(expo.named("T3CreateEasSimulatorSession")[1]?.variables.input).toMatchObject({
298 + platform: "ANDROID",
299 + });
300 + const running = (yield* ready.devices.list).devices.find(
301 + (device) => device.id === "session-2",
302 + );
303 + expect(running).toMatchObject({ platform: "android", booted: true });
304 +
305 + expect(yield* ready.devices.agentEndpoint!("session-2")).toMatchObject({
306 + agentDeviceId: "emulator-5554",
307 + });
308 + expect(Array.from(yield* ready.devices.screenshot(running!))).toEqual([137, 80, 78, 71]);
309 + expect(expo.previews.slice(-2)).toEqual([
310 + "/vendor/serve-emu/api",
311 + "/vendor/serve-emu/api/screenshot?device=emulator-5554",
312 + ]);
313 + expect(
314 + yield* ready.upstream!(
315 + "/vendor/serve-emu/webrtc/offer",
316 + new URLSearchParams("device=session-2&hostId=eas"),
317 + ),
318 + ).toEqual({
319 + url: "https://preview.test/vendor/serve-emu/webrtc/offer?device=emulator-5554&hostId=eas",
320 + headers: { authorization: "Bearer preview-token" },
321 + protocols: ["serve-emu.token.preview-token"],
322 + });
323 + // Each session answers only on its own platform's stream.
324 + expect(
325 + yield* ready.upstream!("/vendor/serve-emu/api", new URLSearchParams("device=session-1")),
326 + ).toBeNull();
327 + expect(
328 + yield* ready.upstream!("/vendor/serve-sim/api", new URLSearchParams("device=session-2")),
329 + ).toBeNull();
330 + }).pipe(Effect.scoped, Effect.provide(layer)),
331 + );
332 +
333 it.effect("refuses to start past its session limit", () =>
334 Effect.gen(function* () {
335 const expo = fakeExpo();
@@ -292,6 +356,7 @@ describe("EAS Simulator host", () => {
356 expect(yield* ready.devices.boot(catalogDevice!)).toBe("session-2");
357 expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([
358 "eas-ios",
359 + "eas-android",
360 "session-1",
361 "session-2",
362 ]);
@@ -306,6 +371,7 @@ describe("EAS Simulator host", () => {
371 expect(expo.stopped()).toEqual(["session-1"]);
372 expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([
373 "eas-ios",
374 + "eas-android",
375 "session-2",
376 ]);
377 yield* Scope.close(scope, Exit.void);
@@ -421,7 +487,10 @@ describe("EAS Simulator host", () => {
487 expect(refused.message).toContain("limited to 1 running session");
488 stopFails = false;
489 yield* ready.devices.list;
424 - expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual(["eas-ios"]);
490 + expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([
491 + "eas-ios",
492 + "eas-android",
493 + ]);
494 expect(expo.stopped().filter((id) => id === "session-1").length).toBeGreaterThanOrEqual(2);
495 }).pipe(Effect.scoped, Effect.provide(layer)),
496 );
@@ -438,7 +507,10 @@ describe("EAS Simulator host", () => {
507 expect((yield* ready.devices.list).devices.map((device) => device.id)).toContain("session-1");
508 // Expo ended it by itself: the stop keeps failing, but the status says so.
509 status = "STOPPED";
441 - expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual(["eas-ios"]);
510 + expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([
511 + "eas-ios",
512 + "eas-android",
513 + ]);
514 status = "IN_PROGRESS";
515 expect(yield* ready.devices.boot(catalogDevice!)).toBe("session-2");
516 }).pipe(Effect.scoped, Effect.provide(layer)),
@@ -488,7 +560,10 @@ describe("EAS Simulator host", () => {
560 expect(expo.stopped()).toEqual(["session-1"]);
561 const failure = yield* Fiber.join(booting).pipe(Effect.flip);
562 expect(failure.message).toContain("was stopped while the session started");
491 - expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual(["eas-ios"]);
563 + expect((yield* ready.devices.list).devices.map((device) => device.id)).toEqual([
564 + "eas-ios",
565 + "eas-android",
566 + ]);
567 }).pipe(Effect.scoped, Effect.provide(layer)),
568 );
569 });
apps/server/src/device/EasDeviceHost.ts
+79 -41
@@ -42,14 +42,48 @@ const STATUS_TTL_MS = 15_000;
42 // Expo's own CLI waits this long once a worker picks the session up.
43 const START_TIMEOUT = Duration.minutes(15);
44 const PREVIEW_TIMEOUT = Duration.seconds(20);
45 -const SERVE_SIM_PREFIX = "/vendor/serve-sim";
45 +const decodeServeSimApi = Schema.decodeUnknownEffect(
46 + Schema.Struct({ device: Schema.NonEmptyString }),
47 +);
48 +const decodeServeEmuApi = Schema.decodeUnknownEffect(
49 + Schema.Struct({ serial: Schema.NonEmptyString }),
50 +);
51 +/**
52 + * Each session's preview host serves serve-sim at its root and serve-emu under
53 + * its vendor prefix, and names its device by an id of its own that agent-device
54 + * also selects it by.
55 + */
56 +const PLATFORMS = {
57 + ios: {
58 + name: "iPhone (EAS Simulator)",
59 + hubPrefix: "/vendor/serve-sim",
60 + previewPrefix: "",
61 + protocol: "serve-sim",
62 + remoteDeviceId: (api: unknown) => decodeServeSimApi(api).pipe(Effect.map((api) => api.device)),
63 + },
64 + android: {
65 + name: "Android (EAS Simulator)",
66 + hubPrefix: "/vendor/serve-emu",
67 + previewPrefix: "/vendor/serve-emu",
68 + protocol: "serve-emu",
69 + remoteDeviceId: (api: unknown) => decodeServeEmuApi(api).pipe(Effect.map((api) => api.serial)),
70 + },
71 +} as const;
72
73 /** EAS picks the simulator model; one entry starts a session. */
74 const CATALOG: ReadonlyArray<DeviceHost.HostDevice> = [
75 {
76 id: "eas-ios",
77 platform: "ios",
52 - name: "iPhone (EAS Simulator)",
78 + name: PLATFORMS.ios.name,
79 + version: "Cloud",
80 + booted: false,
81 + physical: false,
82 + },
83 + {
84 + id: "eas-android",
85 + platform: "android",
86 + name: PLATFORMS.android.name,
87 version: "Cloud",
88 booted: false,
89 physical: false,
@@ -59,7 +93,8 @@ const CATALOG: ReadonlyArray<DeviceHost.HostDevice> = [
93 interface RunningSession {
94 readonly sessionId: string;
95 readonly name: string;
62 - readonly udid: string;
96 + readonly platform: DevicePlatform;
97 + readonly remoteDeviceId: string;
98 readonly preview: { readonly origin: string; readonly token: Redacted.Redacted<string> };
99 readonly agent: DeviceHost.AgentDaemonEndpoint;
100 /** Set once a stop has been asked for but Expo has not confirmed it. */
@@ -94,7 +129,6 @@ const RECORD_MAX_AGE_MS = 24 * 60 * 60_000;
129 const UNCERTAIN_CREATE_MS = 5 * 60_000;
130 // One failed request to Expo during a minutes-long start does not end the start.
131 const TRANSIENT_RETRY = { times: 3, schedule: Schedule.spaced("2 seconds") } as const;
97 -const decodePreviewApi = Schema.decodeUnknownEffect(Schema.Struct({ device: Schema.String }));
132
133 /** A malformed escape names no device, so the proxy answers 404 instead of failing. */
134 const decodePathSegment = (segment: string): string | null => {
@@ -116,7 +150,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
150 const hostId = config.id;
151 const ledgerPath = path.join(server.stateDir, "device", "eas", `${hostId}.json`);
152 // Keyed by Expo's session id, which is also the device id: every EAS
119 - // simulator reports the same UDID, so the UDID cannot tell sessions apart.
153 + // simulator reports the same remote device id, so that id cannot tell sessions apart.
154 const sessions = new Map<string, RunningSession>();
155 const checkedAt = new Map<string, number>();
156 const startLock = yield* Semaphore.make(1);
@@ -348,18 +382,21 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
382 ),
383 );
384
351 - const previewDevice = (origin: string, token: Redacted.Redacted<string>) =>
385 + const previewDevice = (
386 + origin: string,
387 + token: Redacted.Redacted<string>,
388 + platform: DevicePlatform,
389 + ) =>
390 http
391 .execute(
354 - HttpClientRequest.get(`${origin}/api`).pipe(
392 + HttpClientRequest.get(`${origin}${PLATFORMS[platform].previewPrefix}/api`).pipe(
393 HttpClientRequest.bearerToken(Redacted.value(token)),
394 ),
395 )
396 .pipe(
397 Effect.flatMap(HttpClientResponse.filterStatusOk),
398 Effect.flatMap((response) => response.json),
361 - Effect.flatMap(decodePreviewApi),
362 - Effect.map((api) => api.device),
399 + Effect.flatMap(PLATFORMS[platform].remoteDeviceId),
400 Effect.scoped,
401 Effect.timeout(PREVIEW_TIMEOUT),
402 Effect.retry(TRANSIENT_RETRY),
@@ -500,7 +537,9 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
537 ),
538 Effect.mapError((error) => unavailable(error.message, error)),
539 );
503 - return yield* restore(trackSession(expo, sessionId, name, startedIn)).pipe(
540 + return yield* restore(
541 + trackSession(expo, sessionId, name, device.platform, startedIn),
542 + ).pipe(
543 Effect.onError(() =>
544 stopSession(expo, sessionId, name).pipe(
545 // Still running and billed: owned, counted, and retried by listing.
@@ -520,6 +559,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
559 expo: ExpoApi,
560 sessionId: string,
561 name: string,
562 + platform: DevicePlatform,
563 startedIn: number,
564 ) {
565 const endpoints = yield* awaitRunning(expo, sessionId);
@@ -532,7 +572,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
572 });
573 }
574 const origin = new URL(endpoints.preview.url).origin;
535 - const udid = yield* previewDevice(origin, endpoints.preview.token).pipe(
575 + const remoteDeviceId = yield* previewDevice(origin, endpoints.preview.token, platform).pipe(
576 Effect.mapError(
577 (cause) =>
578 new DeviceBootError({
@@ -549,7 +589,8 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
589 sessions.set(sessionId, {
590 sessionId,
591 name,
552 - udid,
592 + platform,
593 + remoteDeviceId,
594 preview: { origin, token: endpoints.preview.token },
595 agent: {
596 baseUrl: endpoints.agentDevice.url,
@@ -650,8 +691,8 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
691 ...CATALOG,
692 ...[...sessions.values()].map((session): DeviceHost.HostDevice => ({
693 id: session.sessionId,
653 - platform: "ios",
654 - name: "iPhone (EAS Simulator)",
694 + platform: session.platform,
695 + name: PLATFORMS[session.platform].name,
696 version: "Cloud",
697 booted: true,
698 physical: false,
@@ -708,7 +749,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
749 http
750 .execute(
751 HttpClientRequest.post(
711 - `${session.preview.origin}/api/screenshot?device=${encodeURIComponent(session.udid)}`,
752 + `${session.preview.origin}${PLATFORMS[session.platform].previewPrefix}/api/screenshot?device=${encodeURIComponent(session.remoteDeviceId)}`,
753 ).pipe(HttpClientRequest.bearerToken(Redacted.value(session.preview.token))),
754 )
755 .pipe(
@@ -730,42 +771,45 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
771 agentEndpoint: (deviceId) =>
772 Option.match(sessionFor(deviceId), {
773 onNone: () => Effect.fail(unavailable("This EAS Simulator device is no longer running.")),
733 - // agent-device selects the simulator by its UDID, not the session id T3 shows.
734 - onSome: (session) => Effect.succeed({ ...session.agent, agentDeviceId: session.udid }),
774 + onSome: (session) =>
775 + Effect.succeed({ ...session.agent, agentDeviceId: session.remoteDeviceId }),
776 }),
777 endsDevices: true,
778 };
779
780 /**
740 - * serve-sim runs at the root of each session's preview host and wants its
741 - * token on every request. Clients name the device by its session id; serve-sim
742 - * knows it by its UDID, so the id is swapped for the UDID in the path and in
743 - * the `device` parameter.
781 + * Clients name the device by its session id; the preview host knows it by its
782 + * own id, so the session id is swapped for that in the path and in the
783 + * `device` parameter. A session answers only on its own platform's server.
784 */
785 const upstream: NonNullable<DeviceHost.DeviceHostReady["upstream"]> = (hubPath, search) =>
786 Effect.sync(() => {
747 - if (!hubPath.startsWith(`${SERVE_SIM_PREFIX}/`)) return null;
787 + const platform = (["ios", "android"] as const).find((candidate) =>
788 + hubPath.startsWith(`${PLATFORMS[candidate].hubPrefix}/`),
789 + );
790 + if (!platform) return null;
791 const helper = /^\/vendor\/serve-sim\/helper\/([^/]+)\//.exec(hubPath);
792 const deviceId = helper ? decodePathSegment(helper[1]!) : search.get("device");
793 if (helper && deviceId === null) return null;
794 const session =
752 - deviceId !== null && deviceId !== undefined
795 + deviceId !== null
796 ? sessions.get(deviceId)
797 : sessions.size === 1
798 ? [...sessions.values()][0]
799 : undefined;
757 - if (!session) return null;
800 + if (session?.platform !== platform) return null;
801 + const { hubPrefix, previewPrefix, protocol } = PLATFORMS[platform];
802 const path = helper
759 - ? `/helper/${encodeURIComponent(session.udid)}${hubPath.slice(helper[0].length - 1)}`
760 - : hubPath.slice(SERVE_SIM_PREFIX.length);
803 + ? `/helper/${encodeURIComponent(session.remoteDeviceId)}${hubPath.slice(helper[0].length - 1)}`
804 + : `${previewPrefix}${hubPath.slice(hubPrefix.length)}`;
805 const params = new URLSearchParams(search);
762 - if (params.has("device")) params.set("device", session.udid);
806 + if (params.has("device")) params.set("device", session.remoteDeviceId);
807 const query = params.size > 0 ? `?${params.toString()}` : "";
808 const token = Redacted.value(session.preview.token);
809 return {
810 url: `${session.preview.origin}${path}${query}`,
811 headers: { authorization: `Bearer ${token}` },
768 - protocols: [`serve-sim.token.${token}`],
812 + protocols: [`${protocol}.token.${token}`],
813 };
814 });
815
@@ -833,19 +877,13 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice
877 // Each server signs in on its own, so one with no Expo credential answers for itself. Reading
878 // the credential is local; whether Expo accepts it is only known once a session starts.
879 const platformAvailability = (platform: DevicePlatform) =>
836 - platform === "ios"
837 - ? resolveExpoCredential(config.accessToken).pipe(
838 - Effect.provide(services),
839 - Effect.as({ platform, available: true }),
840 - Effect.catch((error) =>
841 - Effect.succeed({ platform, available: false, reason: error.message }),
842 - ),
843 - )
844 - : Effect.succeed({
845 - platform,
846 - available: false,
847 - reason: "EAS Simulator Android emulators are not supported yet.",
848 - });
880 + resolveExpoCredential(config.accessToken).pipe(
881 + Effect.provide(services),
882 + Effect.as({ platform, available: true }),
883 + Effect.catch((error) =>
884 + Effect.succeed({ platform, available: false, reason: error.message }),
885 + ),
886 + );
887
888 const summary: Effect.Effect<DeviceHostSummary> = Effect.all([
889 platformAvailability("ios"),
apps/web/src/components/device/DeviceControlsRail.tsx
+10 -10
@@ -1,4 +1,4 @@
1 -import type { DevicePlatform } from "@t3tools/contracts";
1 +import type { DeviceHostSummary, DevicePlatform } from "@t3tools/contracts";
2 import {
3 Camera,
4 ChevronLeft,
@@ -36,6 +36,7 @@ import type { DeviceControls } from "./useDeviceControls";
36
37 export function DeviceControlsRail(props: {
38 platform: DevicePlatform;
39 + hostKind: DeviceHostSummary["kind"] | undefined;
40 handle: DeviceStreamHandle | null;
41 view: DeviceViewControls;
42 controls: DeviceControls;
@@ -46,13 +47,12 @@ export function DeviceControlsRail(props: {
47 onFloat: () => void;
48 onClose: () => void;
49 onPowerOff: () => void;
49 - /** The host takes input but runs no device commands (Simlock): no rotation, settings, tools, or screenshots. */
50 - inputOnly?: boolean;
50 }) {
51 const { view, handle, controls } = props;
52 const popupSide = "left";
53 const settings = controls.detail?.settings;
54 const inputDisabled = !handle?.inputConnected;
55 + const runsHostCommands = props.hostKind === "local" || props.hostKind === "ssh";
56 const nextAppearance = settings?.appearance === "dark" ? "light" : "dark";
57 return (
58 <aside
@@ -87,7 +87,7 @@ export function DeviceControlsRail(props: {
87 >
88 <Square />
89 </RailButton>
90 - {props.inputOnly ? null : (
90 + {runsHostCommands ? (
91 <Menu>
92 <MenuTrigger
93 render={
@@ -116,9 +116,9 @@ export function DeviceControlsRail(props: {
116 </MenuItem>
117 </MenuPopup>
118 </Menu>
119 - )}
119 + ) : null}
120 </>
121 - ) : props.inputOnly ? null : (
121 + ) : props.hostKind === "simlock" ? null : (
122 <RailButton
123 tooltipSide={popupSide}
124 label="Rotate device"
@@ -128,7 +128,7 @@ export function DeviceControlsRail(props: {
128 <RotateCcw />
129 </RailButton>
130 )}
131 - {props.inputOnly ? null : (
131 + {runsHostCommands ? (
132 <>
133 <RailDivider />
134 <RailButton
@@ -193,8 +193,8 @@ export function DeviceControlsRail(props: {
193 <SlidersHorizontal />
194 </RailButton>
195 </>
196 - )}
197 - {props.inputOnly ? null : (
196 + ) : null}
197 + {props.hostKind === "simlock" ? null : (
198 <RailButton
199 tooltipSide={popupSide}
200 label={props.screenshotPending ? "Capturing screenshot" : "Save screenshot"}
@@ -229,7 +229,7 @@ export function DeviceControlsRail(props: {
229 <MenuSeparator />
230 <MenuItem variant="destructive" onClick={props.onPowerOff}>
231 <Power />
232 - {props.inputOnly ? "Return device to the pool" : "Power off device"}
232 + {props.hostKind === "simlock" ? "Return device to the pool" : "Power off device"}
233 </MenuItem>
234 </MenuPopup>
235 </Menu>
apps/web/src/components/device/DevicePanel.tsx
+1 -1
@@ -238,7 +238,7 @@ export function DevicePanel(props: {
238 environmentId={environmentId}
239 device={activeDevice}
240 hostLabel={activeHost?.label ?? "Device host"}
241 - simlock={activeHost?.kind === "simlock"}
241 + hostKind={activeHost?.kind}
242 hostDiagnostics={
243 diagnostics.find(({ hostId }) => hostId === activeDevice.hostId)?.detail
244 }
apps/web/src/components/device/DeviceSetup.tsx
+1 -1
@@ -119,7 +119,7 @@ export function platformSetupStatus(state: DeviceServiceState, platform: DeviceP
119 ready: true,
120 message:
121 remoteHost?.kind === "eas"
122 - ? `${remoteHost.label} starts iOS Simulators in the cloud with EAS Simulator.`
122 + ? `${remoteHost.label} starts ${platform === "ios" ? "iOS Simulators" : "Android emulators"} in the cloud with EAS Simulator.`
123 : platform === "ios"
124 ? `Xcode and iOS Simulator are available${location}.`
125 : `The Android SDK and Emulator are available${location}.`,
apps/web/src/components/device/DeviceWorkspace.tsx
+11 -6
@@ -1,4 +1,4 @@
1 -import type { DeviceSummary, EnvironmentId } from "@t3tools/contracts";
1 +import type { DeviceHostSummary, DeviceSummary, EnvironmentId } from "@t3tools/contracts";
2 import { useRef, useState } from "react";
3 import {
4 captureDeviceScreenshot,
@@ -16,8 +16,7 @@ export function DeviceWorkspace(props: {
16 environmentId: EnvironmentId;
17 device: DeviceSummary;
18 hostLabel: string;
19 - /** Leased from a Simlock pool: serve-sim input, no host commands. */
20 - simlock?: boolean;
19 + hostKind: DeviceHostSummary["kind"] | undefined;
20 hostDiagnostics: string | undefined;
21 visible: boolean;
22 onFloat: () => void;
@@ -31,7 +30,13 @@ export function DeviceWorkspace(props: {
30 // A capture always finishes, even if the panel hides or closes meanwhile; the user asked for it.
31 const capturing = useRef(false);
32 const access = useDeviceHubAccess(props.environmentId, props.device.hostId);
34 - const controls = useDeviceControls({ ...props, access });
33 + // Settings and tools run commands on the device's own machine, which EAS and Simlock do not expose.
34 + const runsHostCommands = props.hostKind === "local" || props.hostKind === "ssh";
35 + const controls = useDeviceControls({
36 + ...props,
37 + access,
38 + visible: props.visible && runsHostCommands,
39 + });
40 const saveScreenshot = () => {
41 if (!access || capturing.current || !props.visible) return;
42 capturing.current = true;
@@ -82,12 +87,13 @@ export function DeviceWorkspace(props: {
87 deviceId={props.device.id}
88 hostId={props.device.hostId}
89 visible={props.visible}
85 - serveSimOnly={props.simlock === true}
90 + serveSimOnly={props.hostKind === "simlock"}
91 axOverlay={axOverlay}
92 allowPhoneView
93 renderControls={(view) => (
94 <DeviceControlsRail
95 platform={props.device.platform}
96 + hostKind={props.hostKind}
97 handle={view.handle}
98 view={view}
99 controls={controls}
@@ -98,7 +104,6 @@ export function DeviceWorkspace(props: {
104 onFloat={props.onFloat}
105 onClose={props.onClose}
106 onPowerOff={props.onPowerOff}
101 - inputOnly={props.simlock === true}
107 />
108 )}
109 />
docs/internals/devices.md
+10 -7
@@ -36,13 +36,13 @@ its sessions. Sessions it starts are recorded under the state directory and
36 stopped by the next server if this one dies; sessions other tools started on
37 the same account are never touched.
38
39 -Each session brings its own serve-sim preview host and agent-device daemon,
40 -with their own tokens. The proxy asks the host to route each request
41 -(`DeviceHostReady.upstream`) and adds the session token server-side, so
39 +Each session brings its own preview host (serve-sim, or serve-emu for Android)
40 +and agent-device daemon, with their own tokens. The proxy asks the host to
41 +route each request (`DeviceHostReady.upstream`) and adds the session token server-side, so
42 clients never hold it. Agent config files are written per device
43 (`DeviceOperations.agentEndpoint`), which is why `device_open` opens the
44 -device before targeting it. Device settings run `simctl` on the simulator's
45 -machine, which EAS does not expose, so they are unavailable there.
44 +device before targeting it. Device settings and tools run `simctl` or `adb` on
45 +the device's machine, which EAS does not expose, so they are unavailable there.
46
47 ## Simlock hosts lease through the agent-device front
48
@@ -147,8 +147,8 @@ decoder then retains frames for seconds despite `optimizeForLatency` and an empt
147 `decodeQueueSize`. The iOS AVCC viewer restores the missing metadata before
148 configuring the decoder so hardware decoding can remain enabled. WebCodecs is
149 secure-context only, so the viewer probes `isConfigSupported` and falls back to
150 -MJPEG on iOS when decoding is unavailable. Android has no MJPEG; there the panel
151 -reports that it cannot decode.
150 +MJPEG on iOS when decoding is unavailable. Android's SEMU socket has no MJPEG;
151 +there the panel reports that it cannot decode.
152
153 serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses
154 AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so
@@ -156,3 +156,6 @@ hubs that serve it pay nothing, and switches on that 409, or when its MJPEG
156 image fails in a browser that has `RTCPeerConnection`. Offers are not trickled:
157 serve-sim takes one offer with every candidate. The video element only feeds
158 the same canvas sink, so the panel and the mobile WebView keep one surface.
159 +Android asks serve-emu's `/api` before opening its video socket and streams
160 +WebRTC when it is advertised (EAS Simulator), keeping a separate `video=0`
161 +socket for input so input can reconnect without restarting video.
docs/user/devices.md
+1 -1
@@ -94,7 +94,7 @@ started from then on; your own Device panel is unaffected.
94 The device stream goes through the environment server, so it works over the
95 local network, Tailscale, and T3 Connect. Live video needs a secure page
96 (HTTPS or localhost); on a plain-HTTP remote origin iOS falls back to a slower
97 -still-image stream and Android cannot show video.
97 +still-image stream and Android cannot show video, except on EAS Simulator.
98
99 ## SSH device hosts
100
packages/client-runtime/src/device/stream.test.ts
+137 -1
@@ -75,7 +75,7 @@ describe("native device stream transport", () => {
75 }
76 }
77 vi.stubGlobal("WebSocket", FakeSocket);
78 - const fetch = vi.fn(() => Promise.resolve(new Response("frame")));
78 + const fetch = vi.fn(() => Promise.resolve(Response.json({})));
79 vi.stubGlobal("fetch", fetch);
80 const events = {
81 onStatus: vi.fn(),
@@ -251,6 +251,7 @@ function recoveryFixture(
251 const fetch = vi.fn((url: string, init: RequestInit) => {
252 const signal = init.signal!;
253 signals.push(signal);
254 + if (url.includes("/api?")) return Promise.resolve(Response.json({}));
255 if (!url.includes("stream.avcc")) return Promise.resolve(new Response("prime"));
256 return Promise.resolve(
257 new Response(
@@ -450,9 +451,11 @@ describe("shared device stream readiness and recovery", () => {
451 }),
452 );
453 client.start();
454 + await vi.advanceTimersByTimeAsync(0);
455 sockets[0]!.onmessage?.({ data: sps });
456 client.stop();
457 client.start();
458 + await vi.advanceTimersByTimeAsync(0);
459 resolveSupport({ supported: true });
460 await vi.advanceTimersByTimeAsync(0);
461 expect(decoders).toHaveLength(0);
@@ -491,3 +494,136 @@ describe("shared device stream readiness and recovery", () => {
494 expect(vi.getTimerCount()).toBe(0);
495 });
496 });
497 +
498 +describe("Android WebRTC device streams", () => {
499 + afterEach(() => {
500 + vi.useRealTimers();
501 + vi.unstubAllGlobals();
502 + });
503 +
504 + function setup() {
505 + vi.useFakeTimers();
506 + const sockets: Socket[] = [];
507 + class Socket {
508 + static OPEN = 1;
509 + readyState = 1;
510 + binaryType = "";
511 + onopen: (() => void) | null = null;
512 + onmessage: ((event: { data: string }) => void) | null = null;
513 + onclose: ((event: { code: number; reason: string }) => void) | null = null;
514 + send = vi.fn();
515 + close = vi.fn();
516 + readonly url: string;
517 + constructor(url: string) {
518 + this.url = url;
519 + sockets.push(this);
520 + }
521 + }
522 + const videos: Video[] = [];
523 + class Video {
524 + muted = false;
525 + playsInline = false;
526 + style = { cssText: "" };
527 + srcObject: unknown = null;
528 + videoWidth = 400;
529 + videoHeight = 800;
530 + setAttribute = vi.fn();
531 + remove = vi.fn();
532 + play = vi.fn(async () => {});
533 + callback: (() => void) | null = null;
534 + requestVideoFrameCallback(callback: () => void) {
535 + this.callback = callback;
536 + return 0;
537 + }
538 + constructor() {
539 + videos.push(this);
540 + }
541 + }
542 + const peers: Peer[] = [];
543 + class Peer {
544 + iceGatheringState = "complete";
545 + connectionState = "connected";
546 + localDescription = { type: "offer", sdp: "offer-sdp" };
547 + ontrack: ((event: { streams: unknown[]; track: unknown }) => void) | null = null;
548 + onconnectionstatechange: (() => void) | null = null;
549 + close = vi.fn();
550 + createOffer = vi.fn(async () => this.localDescription);
551 + setLocalDescription = vi.fn(async () => {});
552 + setRemoteDescription = vi.fn(async () => {});
553 + addTransceiver = vi.fn(() => ({ setCodecPreferences: vi.fn() }));
554 + addEventListener = vi.fn();
555 + removeEventListener = vi.fn();
556 + constructor() {
557 + peers.push(this);
558 + }
559 + }
560 + vi.stubGlobal("WebSocket", Socket);
561 + vi.stubGlobal("RTCPeerConnection", Peer);
562 + vi.stubGlobal("RTCRtpReceiver", { getCapabilities: () => ({ codecs: [] }) });
563 + vi.stubGlobal("document", { createElement: () => new Video(), body: { append: vi.fn() } });
564 + const fetch = vi.fn((url: string, _init?: RequestInit) =>
565 + Promise.resolve(
566 + url.includes("/api?")
567 + ? Response.json({ serial: "emulator-5554", stream: { transport: "webrtc" } })
568 + : Response.json({ type: "answer", sdp: "answer-sdp" }),
569 + ),
570 + );
571 + vi.stubGlobal("fetch", fetch);
572 + const events = {
573 + onStatus: vi.fn(),
574 + onScreen: vi.fn(),
575 + onUnauthorized: vi.fn(),
576 + onMjpegFallback: vi.fn(),
577 + onInputConnected: vi.fn(),
578 + };
579 + const present = vi.fn(() => true);
580 + const client = createDeviceStreamClient(
581 + {
582 + platform: "android",
583 + deviceId: "session-1",
584 + access: {
585 + httpBase: "https://test/api/device-hub",
586 + wsBase: "wss://test/api/device-hub",
587 + credentials: true,
588 + query: { hostId: "eas" },
589 + },
590 + },
591 + { present },
592 + events,
593 + );
594 + return { client, events, fetch, peers, videos, sockets, present };
595 + }
596 +
597 + it("streams over WebRTC when serve-emu advertises it, with input that reconnects on its own", async () => {
598 + const { client, events, fetch, peers, videos, sockets, present } = setup();
599 + client.start();
600 + await vi.advanceTimersByTimeAsync(0);
601 + expect(fetch.mock.calls.map(([url]) => url)).toEqual([
602 + "https://test/api/device-hub/vendor/serve-emu/api?device=session-1&hostId=eas",
603 + "https://test/api/device-hub/vendor/serve-emu/webrtc/offer?device=session-1&hostId=eas",
604 + ]);
605 + expect(peers[0]?.setRemoteDescription).toHaveBeenCalledWith({
606 + type: "answer",
607 + sdp: "answer-sdp",
608 + });
609 + peers[0]?.ontrack?.({ streams: [{}], track: {} });
610 + videos[0]?.callback?.();
611 + expect(present).toHaveBeenCalledWith(videos[0], 400, 800);
612 + expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
613 +
614 + expect(sockets.map((socket) => socket.url)).toEqual([
615 + "wss://test/api/device-hub/vendor/serve-emu/ws?device=session-1&video=0&hostId=eas",
616 + ]);
617 + sockets[0]!.onopen?.();
618 + client.pressButton("back");
619 + expect(sockets[0]!.send).toHaveBeenCalledWith(JSON.stringify({ type: "back" }));
620 + events.onStatus.mockClear();
621 + sockets[0]!.onclose?.({ code: 1001, reason: "going away" });
622 + await vi.advanceTimersByTimeAsync(1_000);
623 + expect(sockets[1]?.url).toContain("video=0");
624 + expect(peers).toHaveLength(1);
625 + expect(peers[0]?.close).not.toHaveBeenCalled();
626 + expect(events.onStatus).not.toHaveBeenCalled();
627 + client.stop();
628 + });
629 +});
packages/client-runtime/src/device/stream.ts
+40 -22
@@ -836,9 +836,10 @@ export function createDeviceStreamClient(
836
837 /**
838 * serve-sim can be locked to WebRTC (EAS Simulator always is), and then
839 - * refuses AVCC and MJPEG. An unreadable answer keeps the AVCC path.
839 + * refuses AVCC and MJPEG; serve-emu advertises WebRTC the same way. An
840 + * unreadable answer keeps the default path.
841 */
841 - const probeIosTransport = async (): Promise<WebRtcSettings | null> => {
842 + const probeWebRtc = async (): Promise<WebRtcSettings | null> => {
843 const probe = new AbortController();
844 const timeout = setTimeout(() => probe.abort(), TRANSPORT_PROBE_TIMEOUT_MS);
845 try {
@@ -858,6 +859,9 @@ export function createDeviceStreamClient(
859 }
860 };
861
862 + const webRtcUrl = (action: "offer" | "close") =>
863 + httpUrl(avcc ? `/helper/${device}/webrtc/${action}` : `/webrtc/${action}?device=${device}`);
864 +
865 const closePeer = () => {
866 const current = peer;
867 if (!current) return;
@@ -865,8 +869,8 @@ export function createDeviceStreamClient(
869 current.connection.close();
870 current.video.srcObject = null;
871 current.video.remove();
868 - // Releases serve-sim's capture session; best effort because the page may be unloading.
869 - void fetch(httpUrl(`/helper/${device}/webrtc/close`), {
872 + // Releases the capture session; best effort because the page may be unloading.
873 + void fetch(webRtcUrl("close"), {
874 method: "POST",
875 keepalive: true,
876 headers: { "content-type": "application/json" },
@@ -875,7 +879,7 @@ export function createDeviceStreamClient(
879 }).catch(() => {});
880 };
881
878 - const readIosWebRtc = async (settings: WebRtcSettings) => {
882 + const readWebRtc = async (settings: WebRtcSettings) => {
883 const lifecycle = generation;
884 const feedGeneration = ++videoGeneration;
885 closePeer();
@@ -920,7 +924,7 @@ export function createDeviceStreamClient(
924 if (!isCurrent()) return;
925 closePeer();
926 connecting(detail);
923 - scheduleRetry("video", () => void readIosWebRtc(settings));
927 + scheduleRetry("video", () => void readWebRtc(settings));
928 };
929 const presentFrame = () => {
930 if (!isCurrent()) {
@@ -988,7 +992,7 @@ export function createDeviceStreamClient(
992 iceGatheringDeadlineMs = Math.min(iceGatheringDeadlineMs * 2, ICE_GATHERING_MAX_TIMEOUT_MS);
993 return retry("Device stream found no network candidates yet. Retrying…");
994 }
991 - const response = await fetch(httpUrl(`/helper/${device}/webrtc/offer`), {
995 + const response = await fetch(webRtcUrl("offer"), {
996 method: "POST",
997 headers: { "content-type": "application/json" },
998 body: JSON.stringify({
@@ -1023,8 +1027,7 @@ export function createDeviceStreamClient(
1027 const switchToWebRtc = async (failure: string, locked: boolean) => {
1028 const session = generation;
1029 if (typeof RTCPeerConnection === "undefined") return fail(failure);
1026 - const settings =
1027 - (await probeIosTransport()) ?? (locked ? { transport: "webrtc" as const } : null);
1030 + const settings = (await probeWebRtc()) ?? (locked ? { transport: "webrtc" as const } : null);
1031 if (stopped || generation !== session) return;
1032 if (!settings) return fail(failure);
1033 if (mjpeg) {
@@ -1036,7 +1039,7 @@ export function createDeviceStreamClient(
1039 // MJPEG may have shown frames already: say the stream is connecting again
1040 // rather than leave a blank canvas until the first WebRTC frame.
1041 connecting("Switching to WebRTC.");
1039 - void readIosWebRtc(settings);
1042 + void readWebRtc(settings);
1043 };
1044
1045 const startDuoVideo = (panels: DuoPanelSinks) => {
@@ -1161,18 +1164,21 @@ export function createDeviceStreamClient(
1164 ws.onerror = () => ws.close();
1165 };
1166
1164 - const connectAndroid = () => {
1167 + /** `inputOnly` keeps a socket for input beside a WebRTC feed, so its reconnects leave video alone. */
1168 + const connectAndroid = (inputOnly = false) => {
1169 if (stopped) return;
1166 - const ws = new WebSocket(wsUrl(`/ws?device=${device}&frame-meta=1`));
1170 + const ws = new WebSocket(
1171 + wsUrl(`/ws?device=${device}&${inputOnly ? "video=0" : "frame-meta=1"}`),
1172 + );
1173 ws.binaryType = "arraybuffer";
1174 socket = ws;
1175 ws.onopen = () => {
1176 if (stopped || socket !== ws) return;
1171 - connecting();
1177 + if (!inputOnly) connecting();
1178 events.onInputConnected(true);
1179 };
1180 ws.onmessage = (event) => {
1175 - if (stopped || socket !== ws) return;
1181 + if (stopped || socket !== ws || inputOnly) return;
1182 if (typeof event.data === "string") {
1183 // The encoder restarts at a new size when the device rotates; the
1184 // next keyframe carries a fresh SPS, so the decoder is rebuilt from it.
@@ -1224,13 +1230,29 @@ export function createDeviceStreamClient(
1230 (event.code === 1006 && access.query.wsTicket)
1231 )
1232 return handleUnauthorized();
1227 - configuring = false;
1228 - connecting(event.reason || undefined);
1229 - scheduleRetry("input", connectAndroid);
1233 + if (!inputOnly) {
1234 + configuring = false;
1235 + connecting(event.reason || undefined);
1236 + }
1237 + // A full socket re-probes, so one failed /api read does not pin SEMU on a WebRTC host.
1238 + scheduleRetry("input", () => (inputOnly ? connectAndroid(true) : void startAndroid()));
1239 };
1240 ws.onerror = () => ws.close();
1241 };
1242
1243 + const startAndroid = async () => {
1244 + const session = generation;
1245 + const settings = await probeWebRtc();
1246 + if (stopped || generation !== session) return;
1247 + if (settings) {
1248 + if (typeof RTCPeerConnection === "undefined")
1249 + return fail("This browser cannot receive the device's WebRTC stream.");
1250 + if (!target.videoOnly) connectAndroid(true);
1251 + void readWebRtc(settings);
1252 + } else if (useWebCodecs) connectAndroid();
1253 + else fail("This browser cannot decode the Android stream (WebCodecs unavailable).");
1254 + };
1255 +
1256 const start = () => {
1257 if (!stopped) return;
1258 stopped = false;
@@ -1247,11 +1269,7 @@ export function createDeviceStreamClient(
1269 if (!target.videoOnly) void connectIosInput();
1270 if (useWebCodecs) void readIosVideo();
1271 else fallBackToMjpeg();
1250 - } else if (useWebCodecs) {
1251 - connectAndroid();
1252 - } else {
1253 - fail("This browser cannot decode the Android stream (WebCodecs unavailable).");
1254 - }
1272 + } else void startAndroid();
1273 };
1274
1275 const stop = () => {