@devries-cameron20 / t3code Mirror · read-only / commits / a3df6044a4
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 12:00 UTC

fix(web): preserve the active pointer in the flat Device view (#21430)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: GPT-6.1 Sol <noreply@openai.com> Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com>

Victor Vogel committed Oct 30, 2026 at 15:26 UTC a3df6044a43e21d1b182175346f77bb471ad7ede
2 files changed +233 -17
apps/web/src/components/device/DeviceStreamView.browser.test.tsx new
+208
@@ -0,0 +1,208 @@
1 +import { RegistryContext } from "@effect/atom-react";
2 +import { AtomRegistry } from "effect/reactivity";
3 +import "~/index.css";
4 +import { EnvironmentId } from "@t3tools/contracts";
5 +import { expect, it, vi } from "vite-plus/test";
6 +
7 +import { act, render, stubGlobal } from "../../../test/render";
8 +
9 +vi.mock(import("~/state/device"), async (original) => ({
10 + ...(await original()),
11 + useDeviceHubAccess: () => access,
12 +}));
13 +vi.mock(import("./DevicePhoneViewport"), () => ({
14 + DevicePhoneViewport: () => <></>,
15 +}));
16 +import { DeviceStreamView } from "./DeviceStreamView";
17 +
18 +const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true };
19 +function setup() {
20 + const registry = AtomRegistry.make();
21 + const touches: unknown[] = [];
22 + const image = vi.spyOn(HTMLImageElement.prototype, "src", "set").mockImplementation(() => {});
23 + // Select MJPEG without starting a WebCodecs decoder. Input uses the real stream client.
24 + const decoder = stubGlobal("VideoDecoder", undefined);
25 + Reflect.deleteProperty(globalThis, "VideoDecoder");
26 + const fetch = stubGlobal("fetch", () => Promise.resolve(new Response("prime")));
27 + // The client opens its input socket only after the MJPEG prime resolves.
28 + type InputSocket = {
29 + onmessage?: (event: { data: ArrayBuffer }) => void;
30 + };
31 + let open!: (socket: InputSocket) => void;
32 + const opened = new Promise<InputSocket>((resolve) => {
33 + open = resolve;
34 + });
35 + const socket = stubGlobal(
36 + "WebSocket",
37 + class {
38 + static OPEN = 1;
39 + readyState = 1;
40 + constructor() {
41 + open(this);
42 + }
43 + onmessage?: (event: { data: ArrayBuffer }) => void;
44 + send(message: string | Uint8Array) {
45 + if (typeof message !== "string" && message[0] === 0x03) {
46 + touches.push(JSON.parse(new TextDecoder().decode(message.subarray(1))));
47 + }
48 + }
49 + close() {}
50 + },
51 + );
52 + return {
53 + view: () => (
54 + <RegistryContext.Provider value={registry}>
55 + <div style={{ width: 400, height: 800 }}>{view()}</div>
56 + </RegistryContext.Provider>
57 + ),
58 + [Symbol.dispose]() {
59 + registry.dispose();
60 + image.mockRestore();
61 + socket[Symbol.dispose]();
62 + fetch[Symbol.dispose]();
63 + decoder[Symbol.dispose]();
64 + },
65 + touches,
66 + async configure(inputUnavailable = false) {
67 + const json = new TextEncoder().encode(
68 + JSON.stringify({ width: 400, height: 800, orientation: "portrait", inputUnavailable }),
69 + );
70 + const packet = new Uint8Array(1 + json.length);
71 + packet[0] = 0x82;
72 + packet.set(json, 1);
73 + const input = await opened;
74 + input.onmessage?.({ data: Uint8Array.of(0x83).buffer });
75 + input.onmessage?.({ data: packet.buffer });
76 + },
77 + };
78 +}
79 +
80 +function view() {
81 + return (
82 + <DeviceStreamView
83 + hostId="local"
84 + environmentId={EnvironmentId.make("test")}
85 + deviceId="test"
86 + platform="ios"
87 + visible
88 + allowPhoneView={false}
89 + renderControls={(controls) => (
90 + <output>
91 + {controls.handle?.inputConnected ? "Input connected" : "Input disconnected"}
92 + </output>
93 + )}
94 + />
95 + );
96 +}
97 +
98 +// router: false because the device view renders no links and never navigates.
99 +const options = { router: false } as const;
100 +
101 +function flatPointer(container: HTMLElement) {
102 + const surface = container.querySelector("canvas")!.parentElement!.parentElement!;
103 + // Synthetic IDs exercise concurrent touches independently of the browser's capture state.
104 + surface.setPointerCapture = () => {};
105 + const rect = new DOMRect(0, 0, 400, 800);
106 + surface.getBoundingClientRect = () => rect;
107 + return {
108 + send(type: string, pointerId: number, x = 0.25, y = 0.25) {
109 + surface.dispatchEvent(
110 + new PointerEvent(type, {
111 + bubbles: true,
112 + pointerId,
113 + pointerType: "touch",
114 + clientX: rect.left + rect.width * x,
115 + clientY: rect.top + rect.height * y,
116 + }),
117 + );
118 + },
119 + };
120 +}
121 +
122 +it("keeps the first flat pointer active when a second finger moves and lifts", async () => {
123 + using input = setup();
124 + const { configure, touches } = input;
125 + await using screen = await render(input.view(), options);
126 + await act(async () => configure());
127 + await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
128 + const { send } = flatPointer(screen.container);
129 + send("pointerdown", 1);
130 + send("pointerdown", 2, 0.75, 0.75);
131 + send("pointermove", 2, 0.875, 0.875);
132 + send("pointerup", 2);
133 + send("pointercancel", 2);
134 + send("pointermove", 1, 0.5, 0.5);
135 + send("pointerup", 1, 0.75, 0.75);
136 + send("pointermove", 1);
137 + expect(touches).toEqual([
138 + { type: "begin", x: 0.25, y: 0.25 },
139 + { type: "move", x: 0.5, y: 0.5 },
140 + { type: "end", x: 0.75, y: 0.75 },
141 + ]);
142 +});
143 +
144 +it("ignores a flat pointer that goes down before input connects", async () => {
145 + using input = setup();
146 + const { configure, touches } = input;
147 + await using screen = await render(input.view(), options);
148 + const { send } = flatPointer(screen.container);
149 + send("pointerdown", 1);
150 + await act(async () => configure());
151 + await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
152 + send("pointerdown", 2, 0.5, 0.5);
153 + send("pointerup", 2, 0.75, 0.75);
154 + expect(touches).toEqual([
155 + { type: "begin", x: 0.5, y: 0.5 },
156 + { type: "end", x: 0.75, y: 0.75 },
157 + ]);
158 +});
159 +
160 +it.each(["pointerup", "pointercancel", "lostpointercapture"])(
161 + "ends the owned flat pointer exactly once on %s, then accepts a new pointer",
162 + async (ending) => {
163 + using input = setup();
164 + const { configure, touches } = input;
165 + await using screen = await render(input.view(), options);
166 + await act(async () => configure());
167 + await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
168 + const { send } = flatPointer(screen.container);
169 + send("pointerdown", 7);
170 + send("pointermove", 7, 0.5, 0.5);
171 + send("lostpointercapture", 8, 0, 0);
172 + // Capture loss reports its own point; the end must still use the last move.
173 + const at = ending === "lostpointercapture" ? 0 : 0.5;
174 + send(ending, 7, at, at);
175 + send("lostpointercapture", 7, 0, 0);
176 + send("pointerup", 7);
177 + send("pointermove", 7);
178 + send("pointerdown", 9);
179 + send("pointerup", 9);
180 + expect(touches).toEqual([
181 + { type: "begin", x: 0.25, y: 0.25 },
182 + { type: "move", x: 0.5, y: 0.5 },
183 + { type: "end", x: 0.5, y: 0.5 },
184 + { type: "begin", x: 0.25, y: 0.25 },
185 + { type: "end", x: 0.25, y: 0.25 },
186 + ]);
187 + },
188 +);
189 +
190 +it("releases flat pointer ownership when input disconnects", async () => {
191 + using input = setup();
192 + await using screen = await render(input.view(), options);
193 + await act(() => input.configure());
194 + await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
195 + const { send } = flatPointer(screen.container);
196 + send("pointerdown", 1);
197 + await act(() => input.configure(true));
198 + await expect.element(screen.getByText("Input disconnected", { exact: true })).toBeVisible();
199 + await act(() => input.configure());
200 + await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
201 + send("pointerdown", 2, 0.5, 0.5);
202 + send("pointerup", 2, 0.75, 0.75);
203 + expect(input.touches).toEqual([
204 + { type: "begin", x: 0.25, y: 0.25 },
205 + { type: "begin", x: 0.5, y: 0.5 },
206 + { type: "end", x: 0.75, y: 0.75 },
207 + ]);
208 +});
apps/web/src/components/device/DeviceStreamView.tsx
+25 -17
@@ -1,7 +1,7 @@
1 import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
2 import { useAtomValue } from "@effect/atom-react";
3 import { AsyncResult, Atom } from "effect/reactivity";
4 -import { useId, useRef, useState, type ReactNode } from "react";
4 +import { useEffect, useId, useRef, useState, type ReactNode } from "react";
5
6 import { cn } from "~/lib/utils";
7 import { Button } from "~/components/ui/button";
@@ -201,7 +201,14 @@ export function DeviceStreamView(props: {
201 );
202 const axElements = AsyncResult.getOrElse(axResult, () => NO_AX_ELEMENTS);
203
204 - const pointerActive = useRef(false);
204 + const activePointer = useRef<{ id: number; x: number; y: number } | null>(null);
205 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- clears pointer ownership when its device input connection disconnects or is replaced
206 + useEffect(() => {
207 + if (!client || !inputState.connected) return;
208 + return () => {
209 + activePointer.current = null;
210 + };
211 + }, [client, inputState.connected]);
212 const normalizedPoint = (event: React.PointerEvent<HTMLElement>) => {
213 const rect = event.currentTarget.getBoundingClientRect();
214 const x = (event.clientX - rect.left) / rect.width;
@@ -209,6 +216,14 @@ export function DeviceStreamView(props: {
216 return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
217 };
218
219 + const endPointer = (event: React.PointerEvent<HTMLElement>) => {
220 + const pointer = activePointer.current;
221 + if (pointer?.id !== event.pointerId) return;
222 + activePointer.current = null;
223 + const { x, y } = event.type === "lostpointercapture" ? pointer : normalizedPoint(event);
224 + client?.sendTouch("end", x, y);
225 + };
226 +
227 const phoneUnavailableReason =
228 isDuo && !screen?.supportsHingeAngle
229 ? "iPhone Duo 3D requires Device Hub 0.11.0 or newer"
@@ -335,30 +350,23 @@ export function DeviceStreamView(props: {
350 className={cn("relative select-none", showPhone && "invisible pointer-events-none")}
351 style={{ width: frame.width, height: frame.height }}
352 onPointerDown={(event) => {
338 - if (event.button !== 0) return;
353 + if (event.button !== 0 || !inputState.connected || activePointer.current !== null)
354 + return;
355 event.currentTarget.setPointerCapture(event.pointerId);
356 (event.currentTarget.parentElement as HTMLElement | null)?.focus();
341 - pointerActive.current = true;
357 const { x, y } = normalizedPoint(event);
358 + activePointer.current = { id: event.pointerId, x, y };
359 client?.sendTouch("begin", x, y);
360 }}
361 onPointerMove={(event) => {
346 - if (!pointerActive.current) return;
362 + if (activePointer.current?.id !== event.pointerId) return;
363 const { x, y } = normalizedPoint(event);
364 + activePointer.current = { id: event.pointerId, x, y };
365 client?.sendTouch("move", x, y);
366 }}
350 - onPointerUp={(event) => {
351 - if (!pointerActive.current) return;
352 - pointerActive.current = false;
353 - const { x, y } = normalizedPoint(event);
354 - client?.sendTouch("end", x, y);
355 - }}
356 - onPointerCancel={(event) => {
357 - if (!pointerActive.current) return;
358 - pointerActive.current = false;
359 - const { x, y } = normalizedPoint(event);
360 - client?.sendTouch("end", x, y);
361 - }}
367 + onPointerUp={endPointer}
368 + onPointerCancel={endPointer}
369 + onLostPointerCapture={endPointer}
370 >
371 <div
372 ref={mountCanvas}