@devries-cameron20 / t3code Mirror · read-only / commits / fcf50ffe38
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 16:00 UTC

fix(connect): T3 Connect recovers when its tunnel is routed to a dead local port (#24247)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Ishaan Kothari committed Nov 13, 2026 at 20:13 UTC fcf50ffe380e79bbb210a6741bd24782be6aa6f8
8 files changed +251 -7
apps/server/src/cloud/CloudLink.lifecycle.test.ts
+1
@@ -1194,6 +1194,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
1194 pid: 123,
1195 readyConnections: 0,
1196 hostname: "prod-abc.t3coderelay.com",
1197 + origin: null,
1198 protocol: null,
1199 },
1200 }),
apps/server/src/cloud/ConnectReachability.test.ts
+10 -2
@@ -25,8 +25,14 @@ import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
25 vi.mock("node:net", { spy: true });
26
27 const checkedAt = "2026-10-19T00:00:00.000Z";
28 -const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
29 -const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null };
28 +const connected = {
29 + pid: 1,
30 + readyConnections: 4,
31 + hostname: "env.example.test",
32 + origin: null,
33 + protocol: "quic",
34 +};
35 +const notConnected = { pid: 1, readyConnections: 0, hostname: null, origin: null, protocol: null };
36 const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({
37 managedTunnelConfigured: true,
38 connector: connected,
@@ -64,6 +70,7 @@ describe("classifyReachability", () => {
70 pid: 1,
71 readyConnections: null,
72 hostname: "env.example.test",
73 + origin: null,
74 protocol: null,
75 },
76 endpoint: { kind: "status", status: 502 },
@@ -80,6 +87,7 @@ describe("classifyReachability", () => {
87 pid: 1,
88 readyConnections: null,
89 hostname: "env.example.test",
90 + origin: null,
91 protocol: null,
92 },
93 endpoint: { kind: "dns_failed" },
apps/server/src/cloud/ManagedEndpointRuntime.test.ts
+63 -1
@@ -22,7 +22,7 @@ import * as Sink from "effect/Sink";
22 import * as Stream from "effect/Stream";
23 import * as TestClock from "effect/testing/TestClock";
24 import { ChildProcess, ChildProcessSpawner } from "effect/process";
25 -import { HttpClient } from "effect/http";
25 +import { HttpClient, HttpClientResponse } from "effect/http";
26 import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
27 import * as RelayClient from "@t3tools/shared/relayClient";
28
@@ -259,6 +259,68 @@ describe("CloudManagedEndpointRuntime", () => {
259 }),
260 );
261
262 + it.effect("reads the public hostname and its origin from cloudflared's remote config", () =>
263 + Effect.gen(function* () {
264 + let ingress: unknown = [
265 + { hostname: "env.example.t3.codes", service: "http://127.0.0.1:3773" },
266 + { service: "http_status:404" },
267 + ];
268 + const httpClient = HttpClient.make((request) =>
269 + Effect.succeed(
270 + HttpClientResponse.fromWeb(
271 + request,
272 + Response.json(
273 + request.url.endsWith("/config")
274 + ? { config: ingress === undefined ? {} : { ingress } }
275 + : { readyConnections: 4 },
276 + ),
277 + ),
278 + ),
279 + );
280 + const output = yield* Queue.unbounded<Uint8Array>();
281 + const spawner = ChildProcessSpawner.make(() =>
282 + Effect.gen(function* () {
283 + const handle = makeHandle({
284 + pid: 702,
285 + onKill: () => {},
286 + output: Stream.fromQueue(output),
287 + });
288 + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
289 + return handle;
290 + }),
291 + );
292 + const runtime = yield* buildCloudManagedEndpointRuntime(
293 + spawner,
294 + layerRelayClientAvailable,
295 + httpClient,
296 + );
297 + yield* runtime.applyConfig({
298 + providerKind: "cloudflare_tunnel",
299 + connectorToken: "connector-token",
300 + tunnelId: "tunnel-1",
301 + });
302 + yield* Queue.offer(
303 + output,
304 + new TextEncoder().encode(
305 + "2026-10-06T00:00:01Z INF Starting metrics server on 127.0.0.1:20241/metrics\n" +
306 + "2026-10-06T00:00:01Z INF Registered tunnel connection connIndex=0\n",
307 + ),
308 + );
309 + yield* Stream.runHead(runtime.tunnelConnected);
310 +
311 + expect(yield* runtime.inspectConnector).toMatchObject({
312 + readyConnections: 4,
313 + hostname: "env.example.t3.codes",
314 + origin: "http://127.0.0.1:3773",
315 + });
316 + // Before the edge pushes a config, the origin is unknown rather than wrong.
317 + for (const missing of [null, undefined]) {
318 + ingress = missing;
319 + expect(yield* runtime.inspectConnector).toMatchObject({ hostname: null, origin: null });
320 + }
321 + }),
322 + );
323 +
324 it.effect("logs a connection unregistration at info with the connector token redacted", () => {
325 const logs: Array<{ readonly level: string; readonly message: unknown }> = [];
326 const unregistrationLogged = Deferred.makeUnsafe<void>();
apps/server/src/cloud/ManagedEndpointRuntime.ts
+20 -3
@@ -60,6 +60,8 @@ export interface ConnectorFacts {
60 readonly readyConnections: number | null;
61 /** The public tunnel hostname from the remote config cloudflared receives once it connects. */
62 readonly hostname: string | null;
63 + /** The local origin that remote config routes the public hostname to. */
64 + readonly origin: string | null;
65 readonly protocol: string | null;
66 }
67
@@ -123,7 +125,14 @@ const ConnectorReadyBody = Schema.Struct({ readyConnections: Schema.Number });
125 const ConnectorConfigBody = Schema.Struct({
126 config: Schema.Struct({
127 ingress: Schema.optional(
126 - Schema.NullOr(Schema.Array(Schema.Struct({ hostname: Schema.optional(Schema.String) }))),
128 + Schema.NullOr(
129 + Schema.Array(
130 + Schema.Struct({
131 + hostname: Schema.optional(Schema.String),
132 + service: Schema.optional(Schema.String),
133 + }),
134 + ),
135 + ),
136 ),
137 }),
138 });
@@ -924,7 +933,13 @@ export const make = Effect.gen(function* () {
933 const pid = Number(active.child.pid);
934 const protocol = yield* Ref.get(active.protocol);
935 if (!(yield* Deferred.isDone(active.metricsAddress))) {
927 - return { pid, readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
936 + return {
937 + pid,
938 + readyConnections: null,
939 + hostname: null,
940 + origin: null,
941 + protocol,
942 + } satisfies ConnectorFacts;
943 }
944 const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`;
945 const [ready, config] = yield* Effect.all(
@@ -934,10 +949,12 @@ export const make = Effect.gen(function* () {
949 ],
950 { concurrency: 2 },
951 );
952 + const hostnameRule = config?.config.ingress?.find((rule) => rule.hostname);
953 return {
954 pid,
955 readyConnections: ready?.readyConnections ?? null,
940 - hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null,
956 + hostname: hostnameRule?.hostname ?? null,
957 + origin: hostnameRule?.service ?? null,
958 protocol,
959 } satisfies ConnectorFacts;
960 });
apps/server/src/cloud/managedTunnelStartup.test.ts
+79 -1
@@ -1,7 +1,12 @@
1 import { describe, expect, it } from "@effect/vitest";
2 import * as Effect from "effect/Effect";
3 +import * as Fiber from "effect/Fiber";
4 +import * as TestClock from "effect/testing/TestClock";
5
4 -import { retryManagedTunnelRegistration } from "./managedTunnelStartup.ts";
6 +import {
7 + retryManagedTunnelRegistration,
8 + watchManagedTunnelOrigin,
9 +} from "./managedTunnelStartup.ts";
10
11 describe("retryManagedTunnelRegistration", () => {
12 it.effect("does not fall back or retry when registration is permanently rejected", () =>
@@ -26,3 +31,76 @@ describe("retryManagedTunnelRegistration", () => {
31 }),
32 );
33 });
34 +
35 +describe("watchManagedTunnelOrigin", () => {
36 + const localOrigin = "http://127.0.0.1:3773";
37 +
38 + /** Runs the watcher for `minutes`, serving each check the next ingress in `ingress`. */
39 + const watch = (input: {
40 + readonly ingress: ReadonlyArray<string | null>;
41 + readonly minutes: number;
42 + readonly failRegistration?: boolean;
43 + }) =>
44 + Effect.gen(function* () {
45 + let checks = 0;
46 + const registeredAtMinute: Array<number> = [];
47 + let minute = 0;
48 + const fiber = yield* watchManagedTunnelOrigin({
49 + localOrigin,
50 + readIngressOrigin: Effect.sync(
51 + () => input.ingress[Math.min(checks++, input.ingress.length - 1)] ?? null,
52 + ),
53 + register: Effect.suspend(() => {
54 + registeredAtMinute.push(minute);
55 + return input.failRegistration ? Effect.fail("relay unavailable") : Effect.void;
56 + }),
57 + }).pipe(Effect.forkChild);
58 + while (minute < input.minutes) {
59 + minute += 1;
60 + yield* TestClock.adjust("1 minute");
61 + }
62 + yield* Fiber.interrupt(fiber);
63 + return { checks, registeredAtMinute };
64 + });
65 +
66 + it.effect("leaves a matching or unknown ingress alone", () =>
67 + Effect.gen(function* () {
68 + const result = yield* watch({
69 + ingress: [null, `${localOrigin}/`, localOrigin],
70 + minutes: 5,
71 + });
72 + expect(result.checks).toBe(5);
73 + expect(result.registeredAtMinute).toEqual([]);
74 + }),
75 + );
76 +
77 + it.effect("keeps checking after a failed registration", () =>
78 + Effect.gen(function* () {
79 + const result = yield* watch({
80 + ingress: ["http://127.0.0.1:46169", "http://127.0.0.1:46169", localOrigin],
81 + minutes: 8,
82 + failRegistration: true,
83 + });
84 + expect(result.registeredAtMinute).toEqual([1, 3]);
85 + // Minutes 1, 3, and 7; the match at 7 restores the one-minute interval.
86 + expect(result.checks).toBe(4);
87 + }),
88 + );
89 +
90 + it.effect("backs off while the relay keeps routing elsewhere", () =>
91 + Effect.gen(function* () {
92 + const result = yield* watch({ ingress: ["http://127.0.0.1:46169"], minutes: 20 });
93 + expect(result.registeredAtMinute).toEqual([1, 3, 7, 15]);
94 + }),
95 + );
96 +
97 + it.effect("keeps the backoff when one check cannot read the ingress", () =>
98 + Effect.gen(function* () {
99 + const result = yield* watch({
100 + ingress: ["http://127.0.0.1:46169", null, "http://127.0.0.1:46169"],
101 + minutes: 10,
102 + });
103 + expect(result.registeredAtMinute).toEqual([1, 5, 9]);
104 + }),
105 + );
106 +});
apps/server/src/cloud/managedTunnelStartup.ts
+54
@@ -1,4 +1,5 @@
1 import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2 +import * as Cause from "effect/Cause";
3 import * as Clock from "effect/Clock";
4 import * as Duration from "effect/Duration";
5 import * as Effect from "effect/Effect";
@@ -212,3 +213,56 @@ export const makeManagedTunnelRecovery = Effect.fn("ManagedTunnelRecovery.make")
213 ),
214 );
215 });
216 +
217 +const MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL = Duration.minutes(1);
218 +
219 +const urlOrigin = (url: string) => {
220 + try {
221 + return new URL(url).origin;
222 + } catch {
223 + return null;
224 + }
225 +};
226 +
227 +/**
228 + * Only this server runs the connector, so an ingress naming another origin is a dead
229 + * registration. Checks every minute and re-registers this origin on a mismatch,
230 + * doubling the wait (up to 30 minutes) while the relay keeps routing elsewhere.
231 + */
232 +export const watchManagedTunnelOrigin = Effect.fn("ManagedTunnelOrigin.watch")(function* <
233 + E,
234 + R,
235 +>(options: {
236 + readonly localOrigin: string;
237 + /** The service URL cloudflared routes the public hostname to, or `null` while unknown. */
238 + readonly readIngressOrigin: Effect.Effect<string | null>;
239 + readonly register: Effect.Effect<unknown, E, R>;
240 +}) {
241 + const localOrigin = urlOrigin(options.localOrigin);
242 + let wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL;
243 + while (true) {
244 + yield* Effect.sleep(wait);
245 + const ingress = yield* options.readIngressOrigin;
246 + // An unreadable ingress says nothing, so it keeps whatever backoff is running.
247 + if (ingress === null) continue;
248 + const ingressOrigin = urlOrigin(ingress);
249 + if (ingressOrigin !== null && ingressOrigin === localOrigin) {
250 + wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL;
251 + continue;
252 + }
253 + yield* Effect.logWarning(
254 + "T3 Connect routes this server's tunnel to another origin; registering this server again",
255 + { ingressOrigin: ingressOrigin ?? "unparseable", localOrigin },
256 + );
257 + yield* options.register.pipe(
258 + Effect.catchCause((cause) =>
259 + Cause.hasInterrupts(cause)
260 + ? Effect.interrupt
261 + : Effect.logWarning("Failed to register this server's T3 Connect origin again", {
262 + cause,
263 + }),
264 + ),
265 + );
266 + wait = Duration.min(Duration.times(wait, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN);
267 + }
268 +});
apps/server/src/guest/GuestShareService.test.ts
+1
@@ -83,6 +83,7 @@ const layerConnect = Layer.mock(CloudManagedEndpointRuntime)({
83 pid: 1,
84 readyConnections: 1,
85 hostname: value,
86 + origin: null,
87 protocol: "http2",
88 })),
89 });
apps/server/src/server.ts
+23
@@ -183,6 +183,7 @@ import {
183 managedTunnelStartupAction,
184 retryDesiredLinkReconcile,
185 retryManagedTunnelRegistration,
186 + watchManagedTunnelOrigin,
187 } from "./cloud/managedTunnelStartup.ts";
188 import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";
189 import * as CloudCliState from "./cloud/CliState.ts";
@@ -1152,6 +1153,28 @@ const layerMakeServer = Layer.unwrap(
1153 Stream.runForEach(recoverManagedTunnel),
1154 Effect.forkScoped,
1155 );
1156 + yield* watchManagedTunnelOrigin({
1157 + localOrigin,
1158 + readIngressOrigin: endpointRuntime.inspectConnector.pipe(
1159 + Effect.map((connector) => connector?.origin ?? null),
1160 + ),
1161 + // Like startup, a publish-only link never routes the tunnel to this host.
1162 + register: CloudCliState.readCliDesiredLinkMode.pipe(
1163 + Effect.flatMap((mode) =>
1164 + mode === "publish_only"
1165 + ? Effect.void
1166 + : cloudLink
1167 + .registerManagedTunnelRecovery(localOrigin)
1168 + .pipe(
1169 + Effect.flatMap((registration) =>
1170 + registration.status === "recovery_required"
1171 + ? endpointRuntime.requestRecovery(registration.config)
1172 + : Effect.void,
1173 + ),
1174 + ),
1175 + ),
1176 + ),
1177 + }).pipe(Effect.forkScoped);
1178 const wantsCliLink = hasCloudPublicConfig
1179 ? yield* CloudCliState.readCliDesiredCloudLink.pipe(
1180 Effect.catch((cause) =>