The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | // @effect-diagnostics nodeBuiltinImport:off - reads the user guide to check its anchors. |
| 2 | import * as NodeFS from "node:fs"; |
| 3 | import * as NodeNet from "node:net"; |
| 4 | |
| 5 | import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts"; |
| 6 | import { describe, expect, it } from "@effect/vitest"; |
| 7 | import * as Clock from "effect/Clock"; |
| 8 | import * as Deferred from "effect/Deferred"; |
| 9 | import * as Duration from "effect/Duration"; |
| 10 | import * as Effect from "effect/Effect"; |
| 11 | import * as Fiber from "effect/Fiber"; |
| 12 | import * as Layer from "effect/Layer"; |
| 13 | import * as Queue from "effect/Queue"; |
| 14 | import * as Stream from "effect/Stream"; |
| 15 | import * as TestClock from "effect/testing/TestClock"; |
| 16 | import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"; |
| 17 | import { ChildProcessSpawner } from "effect/process"; |
| 18 | import { vi } from "vite-plus/test"; |
| 19 | |
| 20 | import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; |
| 21 | import * as ConnectReachability from "./ConnectReachability.ts"; |
| 22 | import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts"; |
| 23 | import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; |
| 24 | |
| 25 | vi.mock("node:net", { spy: true }); |
| 26 | |
| 27 | const checkedAt = "2026-10-19T00:00:00.000Z"; |
| 28 | const connected = { |
| 29 | pid: 1, |
| 30 | readyConnections: 4, |
| 31 | hostname: "env.example.test", |
| 32 | origin: null, |
| 33 | protocol: "quic", |
| 34 | }; |
| 35 | const notConnected = { pid: 1, readyConnections: 0, hostname: null, origin: null, protocol: null }; |
| 36 | const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({ |
| 37 | managedTunnelConfigured: true, |
| 38 | connector: connected, |
| 39 | endpoint: null, |
| 40 | edgeReachable: null, |
| 41 | exitNode: null, |
| 42 | ...overrides, |
| 43 | }); |
| 44 | |
| 45 | describe("classifyReachability", () => { |
| 46 | it.each([ |
| 47 | [{ kind: "status", status: 502 }, "origin_unreachable"], |
| 48 | [{ kind: "status", status: 530 }, "no_connector"], |
| 49 | [{ kind: "status", status: 530, cloudflareError: "1033" }, "no_connector"], |
| 50 | [{ kind: "status", status: 530, cloudflareError: "1016" }, "dns_failed"], |
| 51 | [{ kind: "status", status: 530, cloudflareError: "1001" }, "unexpected_response"], |
| 52 | [{ kind: "network_error" }, "timeout"], |
| 53 | ] as const)("maps endpoint result %o to %s", (endpoint, cause) => { |
| 54 | expect(classifyReachability(facts({ endpoint, edgeReachable: true }), checkedAt)).toMatchObject( |
| 55 | { status: "unreachable", cause }, |
| 56 | ); |
| 57 | }); |
| 58 | |
| 59 | it("keeps the endpoint cause when the TCP edge test cannot explain it", () => { |
| 60 | // QUIC (UDP) can connect while TCP 7844 is blocked. |
| 61 | expect( |
| 62 | classifyReachability( |
| 63 | facts({ endpoint: { kind: "status", status: 502 }, edgeReachable: false }), |
| 64 | checkedAt, |
| 65 | ), |
| 66 | ).toMatchObject({ cause: "origin_unreachable" }); |
| 67 | expect( |
| 68 | classifyReachability( |
| 69 | facts({ |
| 70 | connector: { |
| 71 | pid: 1, |
| 72 | readyConnections: null, |
| 73 | hostname: "env.example.test", |
| 74 | origin: null, |
| 75 | protocol: null, |
| 76 | }, |
| 77 | endpoint: { kind: "status", status: 502 }, |
| 78 | edgeReachable: false, |
| 79 | }), |
| 80 | checkedAt, |
| 81 | ), |
| 82 | ).toMatchObject({ cause: "origin_unreachable" }); |
| 83 | // Failed DNS also fails the edge lookups. |
| 84 | expect( |
| 85 | classifyReachability( |
| 86 | facts({ |
| 87 | connector: { |
| 88 | pid: 1, |
| 89 | readyConnections: null, |
| 90 | hostname: "env.example.test", |
| 91 | origin: null, |
| 92 | protocol: null, |
| 93 | }, |
| 94 | endpoint: { kind: "dns_failed" }, |
| 95 | edgeReachable: false, |
| 96 | }), |
| 97 | checkedAt, |
| 98 | ), |
| 99 | ).toMatchObject({ cause: "dns_failed" }); |
| 100 | }); |
| 101 | |
| 102 | it("waits on a connector that has not connected while the edge answers", () => { |
| 103 | expect( |
| 104 | classifyReachability(facts({ connector: notConnected, edgeReachable: true }), checkedAt), |
| 105 | ).toMatchObject({ cause: "connector_not_ready" }); |
| 106 | }); |
| 107 | |
| 108 | it("does not point an unlinked server at a background service it may not have", () => { |
| 109 | // Desktop and `t3 serve` run the server without a background service. |
| 110 | const result = classifyReachability(facts({ managedTunnelConfigured: false }), checkedAt); |
| 111 | expect(result).toMatchObject({ cause: "not_linked" }); |
| 112 | expect(result.status === "unreachable" && result.fix).not.toContain("t3 service"); |
| 113 | }); |
| 114 | }); |
| 115 | |
| 116 | interface ServedResponse { |
| 117 | readonly status: number; |
| 118 | readonly body: string | ReadableStream<Uint8Array> | { readonly cloudflareError: number }; |
| 119 | } |
| 120 | |
| 121 | // What Cloudflare serves for its 530 page: HTML unless the client accepts JSON. |
| 122 | const cloudflareErrorPage = (request: HttpClientRequest.HttpClientRequest, code: number) => |
| 123 | request.headers.accept === "application/json" |
| 124 | ? JSON.stringify({ status: 530, error_code: code, error_name: "tunnel_error" }) |
| 125 | : `<!doctype html><h1><span>Error</span> <span>${code}</span></h1>`; |
| 126 | |
| 127 | type Served = Effect.Effect<ServedResponse, HttpClientError.HttpClientError>; |
| 128 | |
| 129 | const reachabilityLayer = ( |
| 130 | respond: Served | ((request: HttpClientRequest.HttpClientRequest) => Served), |
| 131 | restartConnector: (pid: number, stillDead: Effect.Effect<boolean>) => Effect.Effect<boolean>, |
| 132 | ) => |
| 133 | ConnectReachability.layer.pipe( |
| 134 | Layer.provide( |
| 135 | Layer.mergeAll( |
| 136 | Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({ |
| 137 | inspectConnector: Effect.succeed(connected), |
| 138 | readinessChanged: Stream.never, |
| 139 | restartConnector, |
| 140 | }), |
| 141 | Layer.mock(ServerSecretStore.ServerSecretStore)({ |
| 142 | get: () => Effect.succeedSome(new Uint8Array()), |
| 143 | }), |
| 144 | Layer.succeed( |
| 145 | HttpClient.HttpClient, |
| 146 | HttpClient.make((request) => |
| 147 | (Effect.isEffect(respond) ? respond : respond(request)).pipe( |
| 148 | Effect.map(({ status, body }) => |
| 149 | HttpClientResponse.fromWeb( |
| 150 | request, |
| 151 | new Response( |
| 152 | typeof body === "object" && "cloudflareError" in body |
| 153 | ? cloudflareErrorPage(request, body.cloudflareError) |
| 154 | : body, |
| 155 | { status }, |
| 156 | ), |
| 157 | ), |
| 158 | ), |
| 159 | ), |
| 160 | ), |
| 161 | ), |
| 162 | Layer.succeed( |
| 163 | ChildProcessSpawner.ChildProcessSpawner, |
| 164 | ChildProcessSpawner.make(() => Effect.die("no tailscale")), |
| 165 | ), |
| 166 | ), |
| 167 | ), |
| 168 | ); |
| 169 | |
| 170 | const mockEdgeConnect = () => |
| 171 | vi.spyOn(NodeNet, "connect").mockImplementation(() => { |
| 172 | const socket = new NodeNet.Socket(); |
| 173 | queueMicrotask(() => socket.emit("connect")); |
| 174 | return socket; |
| 175 | }); |
| 176 | |
| 177 | describe("ConnectReachability stale connector recovery", () => { |
| 178 | it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () => |
| 179 | Effect.gen(function* () { |
| 180 | using _connect = mockEdgeConnect(); |
| 181 | const restarts = yield* Queue.unbounded<number>(); |
| 182 | // Cloudflare answers 530 while cloudflared still reports four ready connections. |
| 183 | let status = 530; |
| 184 | let body = { cloudflareError: 1033 }; |
| 185 | const layer = reachabilityLayer( |
| 186 | Effect.sync(() => ({ status, body })), |
| 187 | (pid, stillDead) => |
| 188 | stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))), |
| 189 | ); |
| 190 | // The scheduled checks run in the layer's scope, so the assertions run inside it. |
| 191 | yield* Effect.gen(function* () { |
| 192 | const reachability = yield* ConnectReachability.ConnectReachability; |
| 193 | |
| 194 | // The first sighting schedules its own confirming check two minutes later. |
| 195 | expect(yield* reachability.check).toMatchObject({ cause: "no_connector" }); |
| 196 | yield* TestClock.adjust(Duration.minutes(1)); |
| 197 | expect(yield* Queue.size(restarts)).toBe(0); |
| 198 | yield* TestClock.adjust(Duration.minutes(1)); |
| 199 | expect(yield* Queue.take(restarts)).toBe(connected.pid); |
| 200 | |
| 201 | // A restart that did not help doubles the wait. |
| 202 | yield* reachability.check; |
| 203 | yield* TestClock.adjust(Duration.minutes(2)); |
| 204 | expect(yield* Queue.size(restarts)).toBe(0); |
| 205 | yield* TestClock.adjust(Duration.minutes(2)); |
| 206 | expect(yield* Queue.take(restarts)).toBe(connected.pid); |
| 207 | |
| 208 | // Reaching the endpoint resets the wait, and a 530 for a broken DNS record is left alone. |
| 209 | status = 200; |
| 210 | expect(yield* reachability.check).toMatchObject({ status: "reachable" }); |
| 211 | status = 530; |
| 212 | body = { cloudflareError: 1016 }; |
| 213 | expect(yield* reachability.check).toMatchObject({ cause: "dns_failed" }); |
| 214 | yield* TestClock.adjust(Duration.minutes(2)); |
| 215 | expect(yield* Queue.size(restarts)).toBe(0); |
| 216 | |
| 217 | body = { cloudflareError: 1033 }; |
| 218 | yield* reachability.check; |
| 219 | yield* TestClock.adjust(Duration.minutes(2)); |
| 220 | expect(yield* Queue.take(restarts)).toBe(connected.pid); |
| 221 | }).pipe(Effect.provide(layer)); |
| 222 | }), |
| 223 | ); |
| 224 | |
| 225 | it.effect("starts a new streak after a healthy check that a newer check overtook", () => |
| 226 | Effect.gen(function* () { |
| 227 | using _connect = mockEdgeConnect(); |
| 228 | const restartTimes = yield* Queue.unbounded<number>(); |
| 229 | const healthyRequested = yield* Deferred.make<void>(); |
| 230 | const healthyAnswer = yield* Deferred.make<void>(); |
| 231 | let status = 530; |
| 232 | const layer = reachabilityLayer( |
| 233 | Effect.gen(function* () { |
| 234 | const served = status; |
| 235 | if (served === 200) { |
| 236 | yield* Deferred.succeed(healthyRequested, undefined); |
| 237 | yield* Deferred.await(healthyAnswer); |
| 238 | } |
| 239 | return { status: served, body: { cloudflareError: 1033 } }; |
| 240 | }), |
| 241 | (_pid, stillDead) => |
| 242 | stillDead.pipe( |
| 243 | Effect.tap((dead) => |
| 244 | dead |
| 245 | ? Clock.currentTimeMillis.pipe( |
| 246 | Effect.flatMap((now) => Queue.offer(restartTimes, now)), |
| 247 | ) |
| 248 | : Effect.void, |
| 249 | ), |
| 250 | ), |
| 251 | ); |
| 252 | yield* Effect.gen(function* () { |
| 253 | const reachability = yield* ConnectReachability.ConnectReachability; |
| 254 | const startedAt = yield* Clock.currentTimeMillis; |
| 255 | yield* reachability.check; |
| 256 | yield* TestClock.adjust(Duration.minutes(1)); |
| 257 | |
| 258 | // The healthy answer arrives after a newer check already reported the tunnel dead. |
| 259 | status = 200; |
| 260 | const healthy = yield* Effect.forkChild(reachability.check); |
| 261 | yield* Deferred.await(healthyRequested); |
| 262 | status = 530; |
| 263 | yield* reachability.check; |
| 264 | yield* Deferred.succeed(healthyAnswer, undefined); |
| 265 | expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" }); |
| 266 | |
| 267 | // The grace period starts again at the next dead check, a minute after the first one. |
| 268 | yield* reachability.check; |
| 269 | yield* TestClock.adjust(Duration.minutes(2)); |
| 270 | expect((yield* Queue.take(restartTimes)) - startedAt).toBe( |
| 271 | Duration.toMillis(Duration.minutes(3)), |
| 272 | ); |
| 273 | }).pipe(Effect.provide(layer)); |
| 274 | }), |
| 275 | ); |
| 276 | |
| 277 | it.effect("rejects a restart when an overtaken healthy check finishes while it waits", () => |
| 278 | Effect.gen(function* () { |
| 279 | using _connect = mockEdgeConnect(); |
| 280 | const healthyRequested = yield* Deferred.make<void>(); |
| 281 | const healthyAnswer = yield* Deferred.make<void>(); |
| 282 | const restartRequested = yield* Deferred.make<void>(); |
| 283 | const releaseRestart = yield* Deferred.make<void>(); |
| 284 | const guardChecks = yield* Queue.unbounded<boolean>(); |
| 285 | let status = 530; |
| 286 | const layer = reachabilityLayer( |
| 287 | Effect.gen(function* () { |
| 288 | const served = status; |
| 289 | if (served === 200) { |
| 290 | yield* Deferred.succeed(healthyRequested, undefined); |
| 291 | yield* Deferred.await(healthyAnswer); |
| 292 | } |
| 293 | return { status: served, body: { cloudflareError: 1033 } }; |
| 294 | }), |
| 295 | (_pid, stillDead) => |
| 296 | Effect.gen(function* () { |
| 297 | yield* Deferred.succeed(restartRequested, undefined); |
| 298 | yield* Deferred.await(releaseRestart); |
| 299 | const accepted = yield* stillDead; |
| 300 | yield* Queue.offer(guardChecks, accepted); |
| 301 | return accepted; |
| 302 | }), |
| 303 | ); |
| 304 | yield* Effect.gen(function* () { |
| 305 | const reachability = yield* ConnectReachability.ConnectReachability; |
| 306 | yield* reachability.check; |
| 307 | yield* TestClock.adjust(Duration.seconds(119)); |
| 308 | |
| 309 | status = 200; |
| 310 | const healthy = yield* Effect.forkChild(reachability.check); |
| 311 | yield* Deferred.await(healthyRequested); |
| 312 | status = 530; |
| 313 | yield* TestClock.adjust(Duration.seconds(1)); |
| 314 | yield* Deferred.await(restartRequested); |
| 315 | |
| 316 | yield* Deferred.succeed(healthyAnswer, undefined); |
| 317 | expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" }); |
| 318 | yield* Deferred.succeed(releaseRestart, undefined); |
| 319 | expect(yield* Queue.take(guardChecks)).toBe(false); |
| 320 | |
| 321 | yield* reachability.check; |
| 322 | yield* TestClock.adjust(Duration.minutes(1)); |
| 323 | expect(yield* Queue.size(guardChecks)).toBe(0); |
| 324 | yield* TestClock.adjust(Duration.minutes(1)); |
| 325 | expect(yield* Queue.take(guardChecks)).toBe(true); |
| 326 | }).pipe(Effect.provide(layer)); |
| 327 | }), |
| 328 | ); |
| 329 | |
| 330 | it.effect("drops the confirming check of a streak that already ended", () => |
| 331 | Effect.gen(function* () { |
| 332 | using _connect = mockEdgeConnect(); |
| 333 | const restarts = yield* Queue.unbounded<number>(); |
| 334 | let status = 530; |
| 335 | let requests = 0; |
| 336 | const layer = reachabilityLayer( |
| 337 | Effect.sync(() => { |
| 338 | requests += 1; |
| 339 | return { status, body: { cloudflareError: 1033 } }; |
| 340 | }), |
| 341 | (pid, stillDead) => |
| 342 | stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))), |
| 343 | ); |
| 344 | yield* Effect.gen(function* () { |
| 345 | const reachability = yield* ConnectReachability.ConnectReachability; |
| 346 | yield* reachability.check; |
| 347 | yield* TestClock.adjust(Duration.minutes(1)); |
| 348 | status = 200; |
| 349 | yield* reachability.check; |
| 350 | status = 530; |
| 351 | yield* reachability.check; |
| 352 | |
| 353 | const before = requests; |
| 354 | yield* TestClock.adjust(Duration.minutes(1)); |
| 355 | expect(requests).toBe(before); |
| 356 | yield* TestClock.adjust(Duration.minutes(1)); |
| 357 | expect(yield* Queue.take(restarts)).toBe(connected.pid); |
| 358 | }).pipe(Effect.provide(layer)); |
| 359 | }), |
| 360 | ); |
| 361 | |
| 362 | it.effect("keeps a 530 whose error body never finishes", () => |
| 363 | Effect.gen(function* () { |
| 364 | using _connect = mockEdgeConnect(); |
| 365 | let markBodyRead = () => {}; |
| 366 | const bodyRead = new Promise<void>((resolve) => { |
| 367 | markBodyRead = resolve; |
| 368 | }); |
| 369 | const layer = reachabilityLayer( |
| 370 | Effect.sync(() => ({ |
| 371 | status: 530, |
| 372 | body: new ReadableStream({ |
| 373 | pull: () => { |
| 374 | markBodyRead(); |
| 375 | return new Promise<void>(() => {}); |
| 376 | }, |
| 377 | }), |
| 378 | })), |
| 379 | () => Effect.succeed(false), |
| 380 | ); |
| 381 | yield* Effect.gen(function* () { |
| 382 | const reachability = yield* ConnectReachability.ConnectReachability; |
| 383 | const check = yield* Effect.forkChild(reachability.check); |
| 384 | yield* Effect.promise(() => bodyRead); |
| 385 | yield* TestClock.adjust(Duration.seconds(2)); |
| 386 | expect(yield* Fiber.join(check)).toMatchObject({ cause: "no_connector" }); |
| 387 | }).pipe(Effect.provide(layer)); |
| 388 | }), |
| 389 | ); |
| 390 | }); |
| 391 | |
| 392 | describe("ConnectReachability DNS failures", () => { |
| 393 | // undici wraps getaddrinfo's error in `TypeError: fetch failed`. glibc behind |
| 394 | // systemd-resolved can report an NXDOMAIN as EBUSY rather than ENOTFOUND. |
| 395 | const lookupFailed = (code: string) => |
| 396 | Effect.fail( |
| 397 | new HttpClientError.HttpClientError({ |
| 398 | reason: new HttpClientError.TransportError({ |
| 399 | request: HttpClientRequest.get("https://env.example.test"), |
| 400 | cause: new TypeError("fetch failed", { |
| 401 | cause: Object.assign(new Error(`getaddrinfo ${code} env.example.test`), { |
| 402 | code, |
| 403 | syscall: "getaddrinfo", |
| 404 | hostname: "env.example.test", |
| 405 | }), |
| 406 | }), |
| 407 | }), |
| 408 | }), |
| 409 | ); |
| 410 | |
| 411 | it.effect("reports a getaddrinfo EBUSY as a DNS failure", () => |
| 412 | Effect.gen(function* () { |
| 413 | using _connect = mockEdgeConnect(); |
| 414 | const layer = reachabilityLayer(lookupFailed("EBUSY"), () => Effect.succeed(false)); |
| 415 | const result = yield* ConnectReachability.ConnectReachability.use( |
| 416 | (reachability) => reachability.check, |
| 417 | ).pipe(Effect.provide(layer)); |
| 418 | expect(result).toMatchObject({ status: "unreachable", cause: "dns_failed" }); |
| 419 | }), |
| 420 | ); |
| 421 | |
| 422 | // A local "not found" cached before the relay published the record looks like a missing |
| 423 | // record. Public DNS tells them apart. |
| 424 | it.effect("says whether the name exists in public DNS", () => |
| 425 | Effect.gen(function* () { |
| 426 | using _connect = mockEdgeConnect(); |
| 427 | for (const [lookupCode, dnsStatus, reason] of [ |
| 428 | ["ENOTFOUND", 0, "old cached answer"], |
| 429 | ["ENOTFOUND", 3, "missing from public DNS"], |
| 430 | // The local resolver did not answer, so nothing it cached is to blame. |
| 431 | ["EAI_AGAIN", 0, "does not resolve from this machine"], |
| 432 | ] as const) { |
| 433 | const layer = reachabilityLayer( |
| 434 | (request) => |
| 435 | request.url.startsWith("https://1.1.1.1/dns-query") |
| 436 | ? Effect.succeed({ status: 200, body: JSON.stringify({ Status: dnsStatus }) }) |
| 437 | : lookupFailed(lookupCode), |
| 438 | () => Effect.succeed(false), |
| 439 | ); |
| 440 | const result = yield* ConnectReachability.ConnectReachability.use( |
| 441 | (reachability) => reachability.check, |
| 442 | ).pipe(Effect.provide(layer)); |
| 443 | expect(result).toMatchObject({ |
| 444 | cause: "dns_failed", |
| 445 | reason: expect.stringContaining(reason), |
| 446 | }); |
| 447 | } |
| 448 | }), |
| 449 | ); |
| 450 | }); |
| 451 | |
| 452 | describe("T3 Connect troubleshooting links", () => { |
| 453 | it("each section names a heading in the user guide", () => { |
| 454 | const guide = NodeFS.readFileSync( |
| 455 | new URL("../../../../docs/user/t3-connect-troubleshooting.md", import.meta.url), |
| 456 | "utf8", |
| 457 | ); |
| 458 | // GitHub's heading anchors: lowercase, drop punctuation, spaces become hyphens. |
| 459 | const anchors = new Set( |
| 460 | [...guide.matchAll(/^#{2,3} (.+)$/gm)].map(([, heading]) => |
| 461 | heading! |
| 462 | .toLowerCase() |
| 463 | .replace(/[^\p{L}\p{N} _-]/gu, "") |
| 464 | .replace(/ /g, "-"), |
| 465 | ), |
| 466 | ); |
| 467 | for (const section of T3_CONNECT_TROUBLESHOOTING_SECTIONS) { |
| 468 | expect(anchors, section).toContain(section); |
| 469 | } |
| 470 | }); |
| 471 | }); |