The best way to control Claude Code, Codex and more from any computer or phone

main
ts 501 lines 20.2 KB
Raw
1 import * as NodeNet from "node:net";
2
3 import {
4 connectReachabilityHelpUrl,
5 type EnvironmentConnectReachability,
6 formatConnectReachable,
7 } from "@t3tools/contracts";
8 import { readTailscaleExitNode, type TailscaleExitNode } from "@t3tools/tailscale";
9 import * as Clock from "effect/Clock";
10 import * as Context from "effect/Context";
11 import * as DateTime from "effect/DateTime";
12 import * as Duration from "effect/Duration";
13 import * as Effect from "effect/Effect";
14 import * as Layer from "effect/Layer";
15 import * as Option from "effect/Option";
16 import * as Ref from "effect/Ref";
17 import * as Scope from "effect/Scope";
18 import * as Stream from "effect/Stream";
19 import * as SynchronizedRef from "effect/SynchronizedRef";
20 import * as Schema from "effect/Schema";
21 import { HttpClient, HttpClientResponse } from "effect/http";
22 import { ChildProcessSpawner } from "effect/process";
23
24 import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
25 import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts";
26 import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
27
28 // cloudflared finds the edge through the SRV record `_v2-origintunneld._tcp.argotunnel.com`,
29 // which names these two hosts on port 7844. QUIC and HTTP/2 both use that port.
30 const TUNNEL_EDGE_HOSTS = ["region1.v2.argotunnel.com", "region2.v2.argotunnel.com"];
31 const TUNNEL_EDGE_PORT = 7844;
32 const EDGE_CONNECT_TIMEOUT = Duration.seconds(5);
33 const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10);
34 const ERROR_BODY_READ_TIMEOUT = Duration.seconds(2);
35 // An IP literal, so the answer does not come from this machine's DNS, over HTTPS because
36 // some networks intercept port 53.
37 const PUBLIC_DNS_OVER_HTTPS = "https://1.1.1.1/dns-query";
38 const PUBLIC_DNS_TIMEOUT = Duration.seconds(5);
39 // cloudflared registers several connections within a few seconds; they are one change.
40 const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3);
41 // Readiness changes miss failures and recoveries past cloudflared (DNS, origin, edge errors).
42 const RECHECK_INTERVAL = Duration.minutes(5);
43 // cloudflared can keep reporting ready connections the edge already dropped, so its
44 // own readiness watch never restarts it. A 530 can also be brief, so restart only once it
45 // lasts, and wait longer after each restart that did not help.
46 const DEAD_CONNECTOR_RESTART_AFTER = Duration.minutes(2);
47 const DEAD_CONNECTOR_RESTART_AFTER_MAX = Duration.hours(1);
48
49 export type EndpointProbe =
50 | { readonly kind: "status"; readonly status: number; readonly cloudflareError?: string }
51 | {
52 readonly kind: "dns_failed";
53 /** Whether public DNS has the name; absent when it could not be asked. */
54 readonly existsPublicly?: boolean;
55 }
56 | { readonly kind: "timeout" | "network_error" };
57
58 /** Everything one check learned. `null` means that step did not run. */
59 export interface ReachabilityFacts {
60 readonly managedTunnelConfigured: boolean;
61 readonly connector: ManagedEndpointRuntime.ConnectorFacts | null;
62 readonly endpoint: EndpointProbe | null;
63 readonly edgeReachable: boolean | null;
64 readonly exitNode: TailscaleExitNode | null;
65 }
66
67 type Unreachable = Extract<EnvironmentConnectReachability, { status: "unreachable" }>;
68
69 function exitNodeHint(exitNode: TailscaleExitNode | null): string | null {
70 if (!exitNode) return null;
71 const name = exitNode.name ? ` "${exitNode.name}"` : "";
72 return `Tailscale exit node${name} is on. Traffic to Cloudflare goes through it.`;
73 }
74
75 /** Turns what one check learned into the answer every surface shows. */
76 export function classifyReachability(
77 facts: ReachabilityFacts,
78 checkedAt: string,
79 ): EnvironmentConnectReachability {
80 const { connector, endpoint, exitNode } = facts;
81 const fail = (
82 cause: Unreachable["cause"],
83 reason: string,
84 fix: string,
85 hint: string | null = null,
86 ): Unreachable => ({ status: "unreachable", checkedAt, cause, reason, hint, fix });
87 const restart = "Restart T3 Code. If it stays, run `t3 connect` again.";
88
89 if (!facts.managedTunnelConfigured) {
90 // A running server reads the link `t3 connect` stores only at startup.
91 return fail(
92 "not_linked",
93 "The server has not set up a T3 Connect tunnel.",
94 "Run `t3 connect`, then restart T3 Code so it applies the link.",
95 );
96 }
97 if (connector === null) {
98 return fail(
99 "connector_not_running",
100 "The tunnel client (cloudflared) is not running.",
101 "Restart T3 Code. If it stays down, check the server log for relay client errors.",
102 );
103 }
104 const status = endpoint?.kind === "status" ? endpoint.status : null;
105 if (status === 200) {
106 return {
107 status: "reachable",
108 checkedAt,
109 readyConnections: connector.readyConnections,
110 protocol: connector.protocol,
111 };
112 }
113 // The edge test is TCP only. With QUIC connections up, a 502 from a working tunnel, or
114 // DNS already failing, a failed TCP test would hide the real cause.
115 if (
116 facts.edgeReachable === false &&
117 status !== 502 &&
118 !(connector.readyConnections && connector.readyConnections > 0) &&
119 endpoint?.kind !== "dns_failed"
120 ) {
121 return fail(
122 "edge_unreachable",
123 `Cloudflare's tunnel port (${TUNNEL_EDGE_PORT}) times out from this machine.`,
124 exitNode
125 ? `tailscale set --exit-node= (or allow outbound ${TUNNEL_EDGE_PORT} on that network)`
126 : `Allow outbound TCP and UDP port ${TUNNEL_EDGE_PORT}, or turn off the VPN or firewall that blocks it.`,
127 exitNodeHint(exitNode) ?? "A VPN, proxy, or firewall may block it.",
128 );
129 }
130 if (status === 502) {
131 return fail(
132 "origin_unreachable",
133 "The tunnel is up, but it cannot reach the T3 server on this machine.",
134 "Restart T3 Code so the tunnel uses the port the server listens on.",
135 );
136 }
137 if (endpoint === null || connector.readyConnections === 0) {
138 return fail(
139 "connector_not_ready",
140 "cloudflared has not connected to Cloudflare yet.",
141 "Wait a minute. If it stays, check the network and the server log.",
142 );
143 }
144 const cloudflareError = endpoint.kind === "status" ? endpoint.cloudflareError : undefined;
145 // Cloudflare answers 530 with error 1033 when the tunnel has no live connector. Other
146 // codes, such as 1016 for a broken DNS record, are not fixed by restarting cloudflared.
147 if (status === 530 && (cloudflareError === undefined || cloudflareError === "1033")) {
148 return fail(
149 "no_connector",
150 "Cloudflare has no live connection from this machine for the tunnel.",
151 restart,
152 );
153 }
154 if (status === 530 && cloudflareError === "1016") {
155 return fail(
156 "dns_failed",
157 "The T3 Connect address no longer points to this machine's tunnel (Cloudflare error 1016). T3 Connect may have removed this environment.",
158 "Link this environment again. Turn T3 Connect off and on in Settings > Sharing & access, or run `t3 connect link` and then restart T3 Code.",
159 );
160 }
161 if (status !== null) {
162 const code = cloudflareError === undefined ? "" : ` (Cloudflare error ${cloudflareError})`;
163 return fail(
164 "unexpected_response",
165 `The T3 Connect address answered with HTTP ${status}${code}.`,
166 restart,
167 );
168 }
169 if (endpoint.kind === "dns_failed") {
170 // A lookup made before the relay published the record leaves a cached "not found"
171 // for the zone's negative TTL (30 minutes).
172 if (endpoint.existsPublicly === true) {
173 return fail(
174 "dns_failed",
175 "The T3 Connect address resolves in public DNS, but this network's DNS still has an old cached answer that it does not exist.",
176 "Wait. The cached answer usually clears within about 30 minutes.",
177 exitNodeHint(exitNode),
178 );
179 }
180 if (endpoint.existsPublicly === false) {
181 return fail(
182 "dns_failed",
183 "The T3 Connect address is missing from public DNS. The relay did not publish it.",
184 restart,
185 );
186 }
187 return fail(
188 "dns_failed",
189 "The T3 Connect address does not resolve from this machine.",
190 "Check the DNS settings on this machine or its VPN.",
191 exitNodeHint(exitNode),
192 );
193 }
194 return fail(
195 "timeout",
196 "The T3 Connect address did not answer from this machine.",
197 "Check the network, VPN, or proxy on this machine.",
198 exitNodeHint(exitNode),
199 );
200 }
201
202 /** The loud diagnosis the CLI prints and the server logs. */
203 export function formatReachabilityFailure(result: Unreachable): string {
204 return [
205 "✕ T3 Connect is NOT reachable from other devices",
206 ` ${result.reason}`,
207 ...(result.hint ? [` ${result.hint}`] : []),
208 ` Fix: ${result.fix}`,
209 ` More info: ${connectReachabilityHelpUrl(result.cause)}`,
210 ].join("\n");
211 }
212
213 /** The code of a failed name lookup, or null. getaddrinfo reports more than the DNS codes, such as EBUSY. */
214 function dnsLookupErrorCode(error: unknown): string | null {
215 let current = error;
216 for (let depth = 0; depth < 6 && typeof current === "object" && current !== null; depth++) {
217 const code = "code" in current && typeof current.code === "string" ? current.code : "";
218 if (("syscall" in current && current.syscall === "getaddrinfo") || DNS_ERROR_CODES.has(code)) {
219 return code;
220 }
221 current = "cause" in current ? current.cause : undefined;
222 }
223 return null;
224 }
225
226 const DNS_ERROR_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "EAI_NONAME"]);
227 // Answers that the name does not exist, which a resolver may have cached. EAI_AGAIN means
228 // the resolver did not answer. glibc behind systemd-resolved can report NXDOMAIN as EBUSY.
229 const NOT_FOUND_CODES = new Set(["ENOTFOUND", "EAI_NONAME", "EBUSY"]);
230
231 const DnsJsonAnswer = Schema.Struct({ Status: Schema.Number });
232
233 const canOpenTcp = (host: string, port: number) =>
234 Effect.callback<boolean>((resume) => {
235 const socket = NodeNet.connect({ host, port });
236 const finish = (connected: boolean) => {
237 socket.destroy();
238 resume(Effect.succeed(connected));
239 };
240 socket.setTimeout(Duration.toMillis(EDGE_CONNECT_TIMEOUT), () => finish(false));
241 socket.once("connect", () => finish(true));
242 socket.once("error", () => finish(false));
243 return Effect.sync(() => socket.destroy());
244 });
245
246 export class ConnectReachability extends Context.Service<
247 ConnectReachability,
248 {
249 /** Runs a check now. The server also runs one when the tunnel's readiness changes. */
250 readonly check: Effect.Effect<EnvironmentConnectReachability>;
251 readonly latest: Effect.Effect<Option.Option<EnvironmentConnectReachability>>;
252 }
253 >()("t3/cloud/ConnectReachability") {}
254
255 const make = Effect.gen(function* () {
256 const runtime = yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime;
257 const secrets = yield* ServerSecretStore.ServerSecretStore;
258 const httpClient = yield* HttpClient.HttpClient;
259 // `order` ranks checks by start, so a slow older one cannot replace a newer result.
260 const latestRef = yield* Ref.make(
261 Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(),
262 );
263 const checkCounter = yield* Ref.make(0);
264 // Checks update this as they finish, even when a newer check already did. A dead streak
265 // or a restart backoff that began before a healthy check no longer counts.
266 const lastHealthyOrder = yield* Ref.make(0);
267 const deadConnectorRef = yield* SynchronizedRef.make<{
268 readonly streak: {
269 readonly pid: number;
270 readonly since: number;
271 readonly order: number;
272 } | null;
273 readonly restartAfterMillis: number;
274 readonly restartOrder: number;
275 }>({
276 streak: null,
277 restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
278 restartOrder: 0,
279 });
280 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
281 const scope = yield* Scope.Scope;
282 const readExitNode = readTailscaleExitNode.pipe(
283 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
284 Effect.orElseSucceed(() => null),
285 );
286
287 // DNS JSON status 0 is NOERROR (the name exists, even with no A record) and 3 is NXDOMAIN.
288 const existsInPublicDns = Effect.fn("ConnectReachability.existsInPublicDns")(
289 function* (hostname: string) {
290 const response = yield* httpClient.get(PUBLIC_DNS_OVER_HTTPS, {
291 urlParams: { name: hostname, type: "A" },
292 headers: { accept: "application/dns-json" },
293 });
294 const { Status } = yield* HttpClientResponse.schemaBodyJson(DnsJsonAnswer)(response);
295 return Status === 0 ? true : Status === 3 ? false : null;
296 },
297 Effect.scoped,
298 Effect.timeoutOption(PUBLIC_DNS_TIMEOUT),
299 Effect.map(Option.getOrNull),
300 Effect.orElseSucceed(() => null),
301 );
302
303 const probeEndpoint = Effect.fn("ConnectReachability.probeEndpoint")(
304 function* (hostname: string) {
305 const response = yield* httpClient
306 .get(`https://${hostname}/.well-known/t3/environment`, { acceptJson: true })
307 .pipe(Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT));
308 if (Option.isNone(response)) return { kind: "timeout" } satisfies EndpointProbe;
309 const { status } = response.value;
310 if (status !== 530) return { kind: "status", status } satisfies EndpointProbe;
311 // Cloudflare's error page is HTML unless the client accepts JSON, which carries
312 // `"error_code":1033`. Clients that send no User-Agent get `error code: 1033`.
313 const body = yield* response.value.text.pipe(
314 Effect.timeoutOption(ERROR_BODY_READ_TIMEOUT),
315 Effect.orElseSucceed(() => Option.none<string>()),
316 );
317 const code = /error[_ ]code"?:\s*(\d{4})/i.exec(Option.getOrElse(body, () => ""))?.[1];
318 return (
319 code === undefined
320 ? { kind: "status", status }
321 : { kind: "status", status, cloudflareError: code }
322 ) satisfies EndpointProbe;
323 },
324 Effect.scoped,
325 (effect, hostname) =>
326 effect.pipe(
327 Effect.catch((error) => {
328 const code = dnsLookupErrorCode(error);
329 if (code === null) return Effect.succeed<EndpointProbe>({ kind: "network_error" });
330 if (!NOT_FOUND_CODES.has(code))
331 return Effect.succeed<EndpointProbe>({ kind: "dns_failed" });
332 return existsInPublicDns(hostname).pipe(
333 Effect.map((existsPublicly): EndpointProbe =>
334 existsPublicly === null
335 ? { kind: "dns_failed" }
336 : { kind: "dns_failed", existsPublicly },
337 ),
338 );
339 }),
340 ),
341 );
342
343 const gatherFacts = Effect.gen(function* () {
344 const managedTunnelConfigured = Option.isSome(
345 yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG).pipe(Effect.orElseSucceed(Option.none)),
346 );
347 const connector = managedTunnelConfigured ? yield* runtime.inspectConnector : null;
348 const endpoint =
349 connector?.hostname && connector.readyConnections !== 0
350 ? yield* probeEndpoint(connector.hostname)
351 : null;
352 const facts: ReachabilityFacts = {
353 managedTunnelConfigured,
354 connector,
355 endpoint,
356 edgeReachable: null,
357 exitNode: null,
358 };
359 if (connector === null || (endpoint?.kind === "status" && endpoint.status === 200)) {
360 return facts;
361 }
362 const [edgeResults, exitNode] = yield* Effect.all(
363 [
364 Effect.forEach(TUNNEL_EDGE_HOSTS, (host) => canOpenTcp(host, TUNNEL_EDGE_PORT), {
365 concurrency: "unbounded",
366 }),
367 readExitNode,
368 ],
369 { concurrency: 2 },
370 );
371 return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode };
372 });
373
374 const isLatest = (order: number) =>
375 Ref.get(latestRef).pipe(
376 Effect.map((latest) => Option.isNone(latest) || latest.value.order <= order),
377 );
378
379 // Runs inside the check span; the runtime restart has its own. Returns the streak a dead
380 // connector just started and when to confirm it.
381 const restartDeadConnector = Effect.fnUntraced(function* (
382 facts: ReachabilityFacts,
383 result: EnvironmentConnectReachability,
384 order: number,
385 ) {
386 const now = yield* Clock.currentTimeMillis;
387 const { connector } = facts;
388 const dead =
389 result.status === "unreachable" &&
390 result.cause === "no_connector" &&
391 connector !== null &&
392 (connector.readyConnections ?? 0) > 0
393 ? connector
394 : null;
395 return yield* SynchronizedRef.modifyEffect(deadConnectorRef, (current) =>
396 Effect.gen(function* () {
397 const healthy = yield* Ref.get(lastHealthyOrder);
398 const state = {
399 streak: current.streak && current.streak.order > healthy ? current.streak : null,
400 restartAfterMillis:
401 current.restartOrder > healthy
402 ? current.restartAfterMillis
403 : Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
404 restartOrder: current.restartOrder,
405 };
406 if (!(yield* isLatest(order))) return [null, state] as const;
407 if (dead === null) return [null, { ...state, streak: null }] as const;
408 if (state.streak?.pid !== dead.pid) {
409 const started = { ...state, streak: { pid: dead.pid, since: now, order } };
410 return [{ order, after: Duration.millis(state.restartAfterMillis) }, started] as const;
411 }
412 if (now - state.streak.since < state.restartAfterMillis) return [null, state] as const;
413 // Checks can invalidate the streak while this request waits for the runtime's lock.
414 const streakOrder = state.streak.order;
415 const stillDead = Effect.gen(function* () {
416 if (!(yield* isLatest(order))) return false;
417 return (yield* Ref.get(lastHealthyOrder)) < streakOrder;
418 });
419 if (!(yield* runtime.restartConnector(dead.pid, stillDead))) {
420 return [null, state] as const;
421 }
422 const restarted = {
423 streak: null,
424 restartAfterMillis: Math.min(
425 state.restartAfterMillis * 2,
426 Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),
427 ),
428 restartOrder: order,
429 };
430 return [null, restarted] as const;
431 }),
432 );
433 });
434
435 const check: Effect.Effect<EnvironmentConnectReachability> = Effect.gen(function* () {
436 const order = yield* Ref.updateAndGet(checkCounter, (count) => count + 1);
437 const facts = yield* gatherFacts;
438 const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now));
439 if (result.status === "reachable") {
440 yield* Ref.update(lastHealthyOrder, (healthy) => Math.max(healthy, order));
441 }
442 const previous = yield* Ref.modify(latestRef, (current) =>
443 Option.isSome(current) && current.value.order > order
444 ? [null, current]
445 : [current, Option.some({ order, result })],
446 );
447 if (previous === null) return result;
448 const started = yield* restartDeadConnector(facts, result, order);
449 // Nothing else would look again until the next periodic check.
450 if (started !== null) {
451 const confirm = SynchronizedRef.get(deadConnectorRef).pipe(
452 Effect.flatMap(({ streak }) =>
453 streak?.order === started.order ? Effect.suspend(() => check) : Effect.void,
454 ),
455 );
456 yield* Effect.sleep(started.after).pipe(Effect.andThen(confirm), Effect.forkIn(scope));
457 }
458 const changed = Option.match(previous, {
459 onNone: () => true,
460 onSome: ({ result: last }) =>
461 last.status !== result.status ||
462 (last.status === "unreachable" &&
463 result.status === "unreachable" &&
464 last.cause !== result.cause),
465 });
466 if (!changed || (result.status === "unreachable" && result.cause === "not_linked")) {
467 return result;
468 }
469 if (result.status === "unreachable") {
470 // A connector that has not connected yet is usually still starting; the
471 // runtime restarts it, and an unreachable edge is reported as its own cause.
472 const log = result.cause === "connector_not_ready" ? Effect.logWarning : Effect.logError;
473 yield* log(`\n${formatReachabilityFailure(result)}`).pipe(
474 Effect.annotateLogs({ cause: result.cause }),
475 );
476 } else {
477 yield* Effect.logInfo(
478 `T3 Connect is reachable from other devices. ${formatConnectReachable(result)}`,
479 );
480 }
481 return result;
482 }).pipe(Effect.withSpan("ConnectReachability.check"));
483
484 yield* runtime.readinessChanged.pipe(
485 Stream.debounce(READINESS_CHANGE_DEBOUNCE),
486 Stream.runForEach(() => check),
487 Effect.forkScoped,
488 );
489 yield* Effect.sleep(RECHECK_INTERVAL).pipe(
490 Effect.andThen(check),
491 Effect.forever,
492 Effect.forkScoped,
493 );
494
495 return ConnectReachability.of({
496 check,
497 latest: Ref.get(latestRef).pipe(Effect.map(Option.map(({ result }) => result))),
498 });
499 });
500
501 export const layer = Layer.effect(ConnectReachability, make);