The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 12:00 UTC
fix(web): preserve the active pointer in the flat Device view (#21430)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: GPT-6.1 Sol <noreply@openai.com> Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com>
Victor Vogel committed
Oct 30, 2026 at 15:26 UTC
a3df6044a43e21d1b182175346f77bb471ad7ede
2 files changed
+233
-17
apps/web/src/components/device/DeviceStreamView.browser.test.tsx
new
+208
@@ -0,0 +1,208 @@
1
+import { RegistryContext } from "@effect/atom-react";
2
+import { AtomRegistry } from "effect/reactivity";
3
+import "~/index.css";
4
+import { EnvironmentId } from "@t3tools/contracts";
5
+import { expect, it, vi } from "vite-plus/test";
6
+
7
+import { act, render, stubGlobal } from "../../../test/render";
8
+
9
+vi.mock(import("~/state/device"), async (original) => ({
10
+ ...(await original()),
11
+ useDeviceHubAccess: () => access,
12
+}));
13
+vi.mock(import("./DevicePhoneViewport"), () => ({
14
+ DevicePhoneViewport: () => <></>,
15
+}));
16
+import { DeviceStreamView } from "./DeviceStreamView";
17
+
18
+const access = { httpBase: "http://test", wsBase: "ws://test", query: {}, credentials: true };
19
+function setup() {
20
+ const registry = AtomRegistry.make();
21
+ const touches: unknown[] = [];
22
+ const image = vi.spyOn(HTMLImageElement.prototype, "src", "set").mockImplementation(() => {});
23
+ // Select MJPEG without starting a WebCodecs decoder. Input uses the real stream client.
24
+ const decoder = stubGlobal("VideoDecoder", undefined);
25
+ Reflect.deleteProperty(globalThis, "VideoDecoder");
26
+ const fetch = stubGlobal("fetch", () => Promise.resolve(new Response("prime")));
27
+ // The client opens its input socket only after the MJPEG prime resolves.
28
+ type InputSocket = {
29
+ onmessage?: (event: { data: ArrayBuffer }) => void;
30
+ };
31
+ let open!: (socket: InputSocket) => void;
32
+ const opened = new Promise<InputSocket>((resolve) => {
33
+ open = resolve;
34
+ });
35
+ const socket = stubGlobal(
36
+ "WebSocket",
37
+ class {
38
+ static OPEN = 1;
39
+ readyState = 1;
40
+ constructor() {
41
+ open(this);
42
+ }
43
+ onmessage?: (event: { data: ArrayBuffer }) => void;
44
+ send(message: string | Uint8Array) {
45
+ if (typeof message !== "string" && message[0] === 0x03) {
46
+ touches.push(JSON.parse(new TextDecoder().decode(message.subarray(1))));
47
+ }
48
+ }
49
+ close() {}
50
+ },
51
+ );
52
+ return {
53
+ view: () => (
54
+ <RegistryContext.Provider value={registry}>
55
+ <div style={{ width: 400, height: 800 }}>{view()}</div>
56
+ </RegistryContext.Provider>
57
+ ),
58
+ [Symbol.dispose]() {
59
+ registry.dispose();
60
+ image.mockRestore();
61
+ socket[Symbol.dispose]();
62
+ fetch[Symbol.dispose]();
63
+ decoder[Symbol.dispose]();
64
+ },
65
+ touches,
66
+ async configure(inputUnavailable = false) {
67
+ const json = new TextEncoder().encode(
68
+ JSON.stringify({ width: 400, height: 800, orientation: "portrait", inputUnavailable }),
69
+ );
70
+ const packet = new Uint8Array(1 + json.length);
71
+ packet[0] = 0x82;
72
+ packet.set(json, 1);
73
+ const input = await opened;
74
+ input.onmessage?.({ data: Uint8Array.of(0x83).buffer });
75
+ input.onmessage?.({ data: packet.buffer });
76
+ },
77
+ };
78
+}
79
+
80
+function view() {
81
+ return (
82
+ <DeviceStreamView
83
+ hostId="local"
84
+ environmentId={EnvironmentId.make("test")}
85
+ deviceId="test"
86
+ platform="ios"
87
+ visible
88
+ allowPhoneView={false}
89
+ renderControls={(controls) => (
90
+ <output>
91
+ {controls.handle?.inputConnected ? "Input connected" : "Input disconnected"}
92
+ </output>
93
+ )}
94
+ />
95
+ );
96
+}
97
+
98
+// router: false because the device view renders no links and never navigates.
99
+const options = { router: false } as const;
100
+
101
+function flatPointer(container: HTMLElement) {
102
+ const surface = container.querySelector("canvas")!.parentElement!.parentElement!;
103
+ // Synthetic IDs exercise concurrent touches independently of the browser's capture state.
104
+ surface.setPointerCapture = () => {};
105
+ const rect = new DOMRect(0, 0, 400, 800);
106
+ surface.getBoundingClientRect = () => rect;
107
+ return {
108
+ send(type: string, pointerId: number, x = 0.25, y = 0.25) {
109
+ surface.dispatchEvent(
110
+ new PointerEvent(type, {
111
+ bubbles: true,
112
+ pointerId,
113
+ pointerType: "touch",
114
+ clientX: rect.left + rect.width * x,
115
+ clientY: rect.top + rect.height * y,
116
+ }),
117
+ );
118
+ },
119
+ };
120
+}
121
+
122
+it("keeps the first flat pointer active when a second finger moves and lifts", async () => {
123
+ using input = setup();
124
+ const { configure, touches } = input;
125
+ await using screen = await render(input.view(), options);
126
+ await act(async () => configure());
127
+ await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
128
+ const { send } = flatPointer(screen.container);
129
+ send("pointerdown", 1);
130
+ send("pointerdown", 2, 0.75, 0.75);
131
+ send("pointermove", 2, 0.875, 0.875);
132
+ send("pointerup", 2);
133
+ send("pointercancel", 2);
134
+ send("pointermove", 1, 0.5, 0.5);
135
+ send("pointerup", 1, 0.75, 0.75);
136
+ send("pointermove", 1);
137
+ expect(touches).toEqual([
138
+ { type: "begin", x: 0.25, y: 0.25 },
139
+ { type: "move", x: 0.5, y: 0.5 },
140
+ { type: "end", x: 0.75, y: 0.75 },
141
+ ]);
142
+});
143
+
144
+it("ignores a flat pointer that goes down before input connects", async () => {
145
+ using input = setup();
146
+ const { configure, touches } = input;
147
+ await using screen = await render(input.view(), options);
148
+ const { send } = flatPointer(screen.container);
149
+ send("pointerdown", 1);
150
+ await act(async () => configure());
151
+ await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
152
+ send("pointerdown", 2, 0.5, 0.5);
153
+ send("pointerup", 2, 0.75, 0.75);
154
+ expect(touches).toEqual([
155
+ { type: "begin", x: 0.5, y: 0.5 },
156
+ { type: "end", x: 0.75, y: 0.75 },
157
+ ]);
158
+});
159
+
160
+it.each(["pointerup", "pointercancel", "lostpointercapture"])(
161
+ "ends the owned flat pointer exactly once on %s, then accepts a new pointer",
162
+ async (ending) => {
163
+ using input = setup();
164
+ const { configure, touches } = input;
165
+ await using screen = await render(input.view(), options);
166
+ await act(async () => configure());
167
+ await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
168
+ const { send } = flatPointer(screen.container);
169
+ send("pointerdown", 7);
170
+ send("pointermove", 7, 0.5, 0.5);
171
+ send("lostpointercapture", 8, 0, 0);
172
+ // Capture loss reports its own point; the end must still use the last move.
173
+ const at = ending === "lostpointercapture" ? 0 : 0.5;
174
+ send(ending, 7, at, at);
175
+ send("lostpointercapture", 7, 0, 0);
176
+ send("pointerup", 7);
177
+ send("pointermove", 7);
178
+ send("pointerdown", 9);
179
+ send("pointerup", 9);
180
+ expect(touches).toEqual([
181
+ { type: "begin", x: 0.25, y: 0.25 },
182
+ { type: "move", x: 0.5, y: 0.5 },
183
+ { type: "end", x: 0.5, y: 0.5 },
184
+ { type: "begin", x: 0.25, y: 0.25 },
185
+ { type: "end", x: 0.25, y: 0.25 },
186
+ ]);
187
+ },
188
+);
189
+
190
+it("releases flat pointer ownership when input disconnects", async () => {
191
+ using input = setup();
192
+ await using screen = await render(input.view(), options);
193
+ await act(() => input.configure());
194
+ await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
195
+ const { send } = flatPointer(screen.container);
196
+ send("pointerdown", 1);
197
+ await act(() => input.configure(true));
198
+ await expect.element(screen.getByText("Input disconnected", { exact: true })).toBeVisible();
199
+ await act(() => input.configure());
200
+ await expect.element(screen.getByText("Input connected", { exact: true })).toBeVisible();
201
+ send("pointerdown", 2, 0.5, 0.5);
202
+ send("pointerup", 2, 0.75, 0.75);
203
+ expect(input.touches).toEqual([
204
+ { type: "begin", x: 0.25, y: 0.25 },
205
+ { type: "begin", x: 0.5, y: 0.5 },
206
+ { type: "end", x: 0.75, y: 0.75 },
207
+ ]);
208
+});
apps/web/src/components/device/DeviceStreamView.tsx
+25
-17
@@ -1,7 +1,7 @@
1
import type { DevicePlatform, EnvironmentId } from "@t3tools/contracts";
2
import { useAtomValue } from "@effect/atom-react";
3
import { AsyncResult, Atom } from "effect/reactivity";
4
-import { useId, useRef, useState, type ReactNode } from "react";
4
+import { useEffect, useId, useRef, useState, type ReactNode } from "react";
5
6
import { cn } from "~/lib/utils";
7
import { Button } from "~/components/ui/button";
@@ -201,7 +201,14 @@ export function DeviceStreamView(props: {
201
);
202
const axElements = AsyncResult.getOrElse(axResult, () => NO_AX_ELEMENTS);
203
204
- const pointerActive = useRef(false);
204
+ const activePointer = useRef<{ id: number; x: number; y: number } | null>(null);
205
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- clears pointer ownership when its device input connection disconnects or is replaced
206
+ useEffect(() => {
207
+ if (!client || !inputState.connected) return;
208
+ return () => {
209
+ activePointer.current = null;
210
+ };
211
+ }, [client, inputState.connected]);
212
const normalizedPoint = (event: React.PointerEvent<HTMLElement>) => {
213
const rect = event.currentTarget.getBoundingClientRect();
214
const x = (event.clientX - rect.left) / rect.width;
@@ -209,6 +216,14 @@ export function DeviceStreamView(props: {
216
return { x: Math.min(1, Math.max(0, x)), y: Math.min(1, Math.max(0, y)) };
217
};
218
219
+ const endPointer = (event: React.PointerEvent<HTMLElement>) => {
220
+ const pointer = activePointer.current;
221
+ if (pointer?.id !== event.pointerId) return;
222
+ activePointer.current = null;
223
+ const { x, y } = event.type === "lostpointercapture" ? pointer : normalizedPoint(event);
224
+ client?.sendTouch("end", x, y);
225
+ };
226
+
227
const phoneUnavailableReason =
228
isDuo && !screen?.supportsHingeAngle
229
? "iPhone Duo 3D requires Device Hub 0.11.0 or newer"
@@ -335,30 +350,23 @@ export function DeviceStreamView(props: {
350
className={cn("relative select-none", showPhone && "invisible pointer-events-none")}
351
style={{ width: frame.width, height: frame.height }}
352
onPointerDown={(event) => {
338
- if (event.button !== 0) return;
353
+ if (event.button !== 0 || !inputState.connected || activePointer.current !== null)
354
+ return;
355
event.currentTarget.setPointerCapture(event.pointerId);
356
(event.currentTarget.parentElement as HTMLElement | null)?.focus();
341
- pointerActive.current = true;
357
const { x, y } = normalizedPoint(event);
358
+ activePointer.current = { id: event.pointerId, x, y };
359
client?.sendTouch("begin", x, y);
360
}}
361
onPointerMove={(event) => {
346
- if (!pointerActive.current) return;
362
+ if (activePointer.current?.id !== event.pointerId) return;
363
const { x, y } = normalizedPoint(event);
364
+ activePointer.current = { id: event.pointerId, x, y };
365
client?.sendTouch("move", x, y);
366
}}
350
- onPointerUp={(event) => {
351
- if (!pointerActive.current) return;
352
- pointerActive.current = false;
353
- const { x, y } = normalizedPoint(event);
354
- client?.sendTouch("end", x, y);
355
- }}
356
- onPointerCancel={(event) => {
357
- if (!pointerActive.current) return;
358
- pointerActive.current = false;
359
- const { x, y } = normalizedPoint(event);
360
- client?.sendTouch("end", x, y);
361
- }}
367
+ onPointerUp={endPointer}
368
+ onPointerCancel={endPointer}
369
+ onLostPointerCapture={endPointer}
370
>
371
<div
372
ref={mountCanvas}