@devries-cameron20 / t3code-1 / commits / c192c439c1
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): restart T3 Connect's tunnel client when Cloudflare drops it (#24946)

Co-authored-by: PR Batch Tester <agent@local.test> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>

shivam committed Nov 5, 2026 at 06:58 UTC c192c439c15f85310b6f1732d83c562c19d5978a
6 files changed +531 -22
apps/server/src/cloud/CloudLink.lifecycle.test.ts
+1
@@ -206,6 +206,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
206 }),
207 recoveryRequests: Stream.empty,
208 requestRecovery: () => Effect.void,
209 + restartConnector: () => Effect.succeed(false),
210 tunnelConnected: Stream.empty,
211 inspectConnector: Effect.succeed(null),
212 readinessChanged: Stream.empty,
apps/server/src/cloud/ConnectReachability.test.ts
+308 -4
@@ -1,14 +1,32 @@
1 // @effect-diagnostics nodeBuiltinImport:off - reads the user guide to check its anchors.
2 import * as NodeFS from "node:fs";
3 +import * as NodeNet from "node:net";
4
5 import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts";
6 import { describe, expect, it } from "@effect/vitest";
7 +import * as Clock from "effect/Clock";
8 +import * as Deferred from "effect/Deferred";
9 +import * as Duration from "effect/Duration";
10 +import * as Effect from "effect/Effect";
11 +import * as Fiber from "effect/Fiber";
12 +import * as Layer from "effect/Layer";
13 +import * as Queue from "effect/Queue";
14 +import * as Stream from "effect/Stream";
15 +import * as TestClock from "effect/testing/TestClock";
16 +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
17 +import { ChildProcessSpawner } from "effect/process";
18 +import { vi } from "vite-plus/test";
19
20 +import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
21 +import * as ConnectReachability from "./ConnectReachability.ts";
22 import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";
23 +import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
24 +
25 +vi.mock("node:net", { spy: true });
26
27 const checkedAt = "2026-10-19T00:00:00.000Z";
10 -const connected = { readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
11 -const notConnected = { readyConnections: 0, hostname: null, protocol: null };
28 +const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
29 +const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null };
30 const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({
31 managedTunnelConfigured: true,
32 connector: connected,
@@ -22,6 +40,8 @@ describe("classifyReachability", () => {
40 it.each([
41 [{ kind: "status", status: 502 }, "origin_unreachable"],
42 [{ kind: "status", status: 530 }, "no_connector"],
43 + [{ kind: "status", status: 530, cloudflareError: "1033" }, "no_connector"],
44 + [{ kind: "status", status: 530, cloudflareError: "1016" }, "unexpected_response"],
45 [{ kind: "network_error" }, "timeout"],
46 ] as const)("maps endpoint result %o to %s", (endpoint, cause) => {
47 expect(classifyReachability(facts({ endpoint, edgeReachable: true }), checkedAt)).toMatchObject(
@@ -40,7 +60,12 @@ describe("classifyReachability", () => {
60 expect(
61 classifyReachability(
62 facts({
43 - connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
63 + connector: {
64 + pid: 1,
65 + readyConnections: null,
66 + hostname: "env.example.test",
67 + protocol: null,
68 + },
69 endpoint: { kind: "status", status: 502 },
70 edgeReachable: false,
71 }),
@@ -51,7 +76,12 @@ describe("classifyReachability", () => {
76 expect(
77 classifyReachability(
78 facts({
54 - connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
79 + connector: {
80 + pid: 1,
81 + readyConnections: null,
82 + hostname: "env.example.test",
83 + protocol: null,
84 + },
85 endpoint: { kind: "dns_failed" },
86 edgeReachable: false,
87 }),
@@ -74,6 +104,280 @@ describe("classifyReachability", () => {
104 });
105 });
106
107 +interface ServedResponse {
108 + readonly status: number;
109 + readonly body: string | ReadableStream<Uint8Array> | { readonly cloudflareError: number };
110 +}
111 +
112 +// What Cloudflare serves for its 530 page: HTML unless the client accepts JSON.
113 +const cloudflareErrorPage = (request: HttpClientRequest.HttpClientRequest, code: number) =>
114 + request.headers.accept === "application/json"
115 + ? JSON.stringify({ status: 530, error_code: code, error_name: "tunnel_error" })
116 + : `<!doctype html><h1><span>Error</span> <span>${code}</span></h1>`;
117 +
118 +const reachabilityLayer = (
119 + respond: Effect.Effect<ServedResponse>,
120 + restartConnector: (pid: number, stillDead: Effect.Effect<boolean>) => Effect.Effect<boolean>,
121 +) =>
122 + ConnectReachability.layer.pipe(
123 + Layer.provide(
124 + Layer.mergeAll(
125 + Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({
126 + inspectConnector: Effect.succeed(connected),
127 + readinessChanged: Stream.never,
128 + restartConnector,
129 + }),
130 + Layer.mock(ServerSecretStore.ServerSecretStore)({
131 + get: () => Effect.succeedSome(new Uint8Array()),
132 + }),
133 + Layer.succeed(
134 + HttpClient.HttpClient,
135 + HttpClient.make((request) =>
136 + respond.pipe(
137 + Effect.map(({ status, body }) =>
138 + HttpClientResponse.fromWeb(
139 + request,
140 + new Response(
141 + typeof body === "object" && "cloudflareError" in body
142 + ? cloudflareErrorPage(request, body.cloudflareError)
143 + : body,
144 + { status },
145 + ),
146 + ),
147 + ),
148 + ),
149 + ),
150 + ),
151 + Layer.succeed(
152 + ChildProcessSpawner.ChildProcessSpawner,
153 + ChildProcessSpawner.make(() => Effect.die("no tailscale")),
154 + ),
155 + ),
156 + ),
157 + );
158 +
159 +const mockEdgeConnect = () =>
160 + vi.spyOn(NodeNet, "connect").mockImplementation(() => {
161 + const socket = new NodeNet.Socket();
162 + queueMicrotask(() => socket.emit("connect"));
163 + return socket;
164 + });
165 +
166 +describe("ConnectReachability stale connector recovery", () => {
167 + it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>
168 + Effect.gen(function* () {
169 + using _connect = mockEdgeConnect();
170 + const restarts = yield* Queue.unbounded<number>();
171 + // Cloudflare answers 530 while cloudflared still reports four ready connections.
172 + let status = 530;
173 + let body = { cloudflareError: 1033 };
174 + const layer = reachabilityLayer(
175 + Effect.sync(() => ({ status, body })),
176 + (pid, stillDead) =>
177 + stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
178 + );
179 + // The scheduled checks run in the layer's scope, so the assertions run inside it.
180 + yield* Effect.gen(function* () {
181 + const reachability = yield* ConnectReachability.ConnectReachability;
182 +
183 + // The first sighting schedules its own confirming check two minutes later.
184 + expect(yield* reachability.check).toMatchObject({ cause: "no_connector" });
185 + yield* TestClock.adjust(Duration.minutes(1));
186 + expect(yield* Queue.size(restarts)).toBe(0);
187 + yield* TestClock.adjust(Duration.minutes(1));
188 + expect(yield* Queue.take(restarts)).toBe(connected.pid);
189 +
190 + // A restart that did not help doubles the wait.
191 + yield* reachability.check;
192 + yield* TestClock.adjust(Duration.minutes(2));
193 + expect(yield* Queue.size(restarts)).toBe(0);
194 + yield* TestClock.adjust(Duration.minutes(2));
195 + expect(yield* Queue.take(restarts)).toBe(connected.pid);
196 +
197 + // Reaching the endpoint resets the wait, and a 530 for a broken DNS record is left alone.
198 + status = 200;
199 + expect(yield* reachability.check).toMatchObject({ status: "reachable" });
200 + status = 530;
201 + body = { cloudflareError: 1016 };
202 + expect(yield* reachability.check).toMatchObject({ cause: "unexpected_response" });
203 + yield* TestClock.adjust(Duration.minutes(2));
204 + expect(yield* Queue.size(restarts)).toBe(0);
205 +
206 + body = { cloudflareError: 1033 };
207 + yield* reachability.check;
208 + yield* TestClock.adjust(Duration.minutes(2));
209 + expect(yield* Queue.take(restarts)).toBe(connected.pid);
210 + }).pipe(Effect.provide(layer));
211 + }),
212 + );
213 +
214 + it.effect("starts a new streak after a healthy check that a newer check overtook", () =>
215 + Effect.gen(function* () {
216 + using _connect = mockEdgeConnect();
217 + const restartTimes = yield* Queue.unbounded<number>();
218 + const healthyRequested = yield* Deferred.make<void>();
219 + const healthyAnswer = yield* Deferred.make<void>();
220 + let status = 530;
221 + const layer = reachabilityLayer(
222 + Effect.gen(function* () {
223 + const served = status;
224 + if (served === 200) {
225 + yield* Deferred.succeed(healthyRequested, undefined);
226 + yield* Deferred.await(healthyAnswer);
227 + }
228 + return { status: served, body: { cloudflareError: 1033 } };
229 + }),
230 + (_pid, stillDead) =>
231 + stillDead.pipe(
232 + Effect.tap((dead) =>
233 + dead
234 + ? Clock.currentTimeMillis.pipe(
235 + Effect.flatMap((now) => Queue.offer(restartTimes, now)),
236 + )
237 + : Effect.void,
238 + ),
239 + ),
240 + );
241 + yield* Effect.gen(function* () {
242 + const reachability = yield* ConnectReachability.ConnectReachability;
243 + const startedAt = yield* Clock.currentTimeMillis;
244 + yield* reachability.check;
245 + yield* TestClock.adjust(Duration.minutes(1));
246 +
247 + // The healthy answer arrives after a newer check already reported the tunnel dead.
248 + status = 200;
249 + const healthy = yield* Effect.forkChild(reachability.check);
250 + yield* Deferred.await(healthyRequested);
251 + status = 530;
252 + yield* reachability.check;
253 + yield* Deferred.succeed(healthyAnswer, undefined);
254 + expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
255 +
256 + // The grace period starts again at the next dead check, a minute after the first one.
257 + yield* reachability.check;
258 + yield* TestClock.adjust(Duration.minutes(2));
259 + expect((yield* Queue.take(restartTimes)) - startedAt).toBe(
260 + Duration.toMillis(Duration.minutes(3)),
261 + );
262 + }).pipe(Effect.provide(layer));
263 + }),
264 + );
265 +
266 + it.effect("rejects a restart when an overtaken healthy check finishes while it waits", () =>
267 + Effect.gen(function* () {
268 + using _connect = mockEdgeConnect();
269 + const healthyRequested = yield* Deferred.make<void>();
270 + const healthyAnswer = yield* Deferred.make<void>();
271 + const restartRequested = yield* Deferred.make<void>();
272 + const releaseRestart = yield* Deferred.make<void>();
273 + const guardChecks = yield* Queue.unbounded<boolean>();
274 + let status = 530;
275 + const layer = reachabilityLayer(
276 + Effect.gen(function* () {
277 + const served = status;
278 + if (served === 200) {
279 + yield* Deferred.succeed(healthyRequested, undefined);
280 + yield* Deferred.await(healthyAnswer);
281 + }
282 + return { status: served, body: { cloudflareError: 1033 } };
283 + }),
284 + (_pid, stillDead) =>
285 + Effect.gen(function* () {
286 + yield* Deferred.succeed(restartRequested, undefined);
287 + yield* Deferred.await(releaseRestart);
288 + const accepted = yield* stillDead;
289 + yield* Queue.offer(guardChecks, accepted);
290 + return accepted;
291 + }),
292 + );
293 + yield* Effect.gen(function* () {
294 + const reachability = yield* ConnectReachability.ConnectReachability;
295 + yield* reachability.check;
296 + yield* TestClock.adjust(Duration.seconds(119));
297 +
298 + status = 200;
299 + const healthy = yield* Effect.forkChild(reachability.check);
300 + yield* Deferred.await(healthyRequested);
301 + status = 530;
302 + yield* TestClock.adjust(Duration.seconds(1));
303 + yield* Deferred.await(restartRequested);
304 +
305 + yield* Deferred.succeed(healthyAnswer, undefined);
306 + expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
307 + yield* Deferred.succeed(releaseRestart, undefined);
308 + expect(yield* Queue.take(guardChecks)).toBe(false);
309 +
310 + yield* reachability.check;
311 + yield* TestClock.adjust(Duration.minutes(1));
312 + expect(yield* Queue.size(guardChecks)).toBe(0);
313 + yield* TestClock.adjust(Duration.minutes(1));
314 + expect(yield* Queue.take(guardChecks)).toBe(true);
315 + }).pipe(Effect.provide(layer));
316 + }),
317 + );
318 +
319 + it.effect("drops the confirming check of a streak that already ended", () =>
320 + Effect.gen(function* () {
321 + using _connect = mockEdgeConnect();
322 + const restarts = yield* Queue.unbounded<number>();
323 + let status = 530;
324 + let requests = 0;
325 + const layer = reachabilityLayer(
326 + Effect.sync(() => {
327 + requests += 1;
328 + return { status, body: { cloudflareError: 1033 } };
329 + }),
330 + (pid, stillDead) =>
331 + stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
332 + );
333 + yield* Effect.gen(function* () {
334 + const reachability = yield* ConnectReachability.ConnectReachability;
335 + yield* reachability.check;
336 + yield* TestClock.adjust(Duration.minutes(1));
337 + status = 200;
338 + yield* reachability.check;
339 + status = 530;
340 + yield* reachability.check;
341 +
342 + const before = requests;
343 + yield* TestClock.adjust(Duration.minutes(1));
344 + expect(requests).toBe(before);
345 + yield* TestClock.adjust(Duration.minutes(1));
346 + expect(yield* Queue.take(restarts)).toBe(connected.pid);
347 + }).pipe(Effect.provide(layer));
348 + }),
349 + );
350 +
351 + it.effect("keeps a 530 whose error body never finishes", () =>
352 + Effect.gen(function* () {
353 + using _connect = mockEdgeConnect();
354 + let markBodyRead = () => {};
355 + const bodyRead = new Promise<void>((resolve) => {
356 + markBodyRead = resolve;
357 + });
358 + const layer = reachabilityLayer(
359 + Effect.sync(() => ({
360 + status: 530,
361 + body: new ReadableStream({
362 + pull: () => {
363 + markBodyRead();
364 + return new Promise<void>(() => {});
365 + },
366 + }),
367 + })),
368 + () => Effect.succeed(false),
369 + );
370 + yield* Effect.gen(function* () {
371 + const reachability = yield* ConnectReachability.ConnectReachability;
372 + const check = yield* Effect.forkChild(reachability.check);
373 + yield* Effect.promise(() => bodyRead);
374 + yield* TestClock.adjust(Duration.seconds(2));
375 + expect(yield* Fiber.join(check)).toMatchObject({ cause: "no_connector" });
376 + }).pipe(Effect.provide(layer));
377 + }),
378 + );
379 +});
380 +
381 describe("T3 Connect troubleshooting links", () => {
382 it("each section names a heading in the user guide", () => {
383 const guide = NodeFS.readFileSync(
apps/server/src/cloud/ConnectReachability.ts
+136 -17
@@ -6,6 +6,7 @@ import {
6 formatConnectReachable,
7 } from "@t3tools/contracts";
8 import { readTailscaleExitNode, type TailscaleExitNode } from "@t3tools/tailscale";
9 +import * as Clock from "effect/Clock";
10 import * as Context from "effect/Context";
11 import * as DateTime from "effect/DateTime";
12 import * as Duration from "effect/Duration";
@@ -13,7 +14,9 @@ import * as Effect from "effect/Effect";
14 import * as Layer from "effect/Layer";
15 import * as Option from "effect/Option";
16 import * as Ref from "effect/Ref";
17 +import * as Scope from "effect/Scope";
18 import * as Stream from "effect/Stream";
19 +import * as SynchronizedRef from "effect/SynchronizedRef";
20 import { HttpClient } from "effect/http";
21 import { ChildProcessSpawner } from "effect/process";
22
@@ -27,13 +30,19 @@ const TUNNEL_EDGE_HOSTS = ["region1.v2.argotunnel.com", "region2.v2.argotunnel.c
30 const TUNNEL_EDGE_PORT = 7844;
31 const EDGE_CONNECT_TIMEOUT = Duration.seconds(5);
32 const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10);
33 +const ERROR_BODY_READ_TIMEOUT = Duration.seconds(2);
34 // cloudflared registers several connections within a few seconds; they are one change.
35 const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3);
36 // Readiness changes miss failures and recoveries past cloudflared (DNS, origin, edge errors).
37 const RECHECK_INTERVAL = Duration.minutes(5);
38 +// cloudflared can keep reporting ready connections the edge already dropped, so its
39 +// own readiness watch never restarts it. A 530 can also be brief, so restart only once it
40 +// lasts, and wait longer after each restart that did not help.
41 +const DEAD_CONNECTOR_RESTART_AFTER = Duration.minutes(2);
42 +const DEAD_CONNECTOR_RESTART_AFTER_MAX = Duration.hours(1);
43
44 export type EndpointProbe =
36 - | { readonly kind: "status"; readonly status: number }
45 + | { readonly kind: "status"; readonly status: number; readonly cloudflareError?: string }
46 | { readonly kind: "dns_failed" | "timeout" | "network_error" };
47
48 /** Everything one check learned. `null` means that step did not run. */
@@ -122,8 +131,10 @@ export function classifyReachability(
131 "Wait a minute. If it stays, check the network and the server log.",
132 );
133 }
125 - // Cloudflare answers 530 (error 1033) when the tunnel has no live connector.
126 - if (status === 530) {
134 + const cloudflareError = endpoint.kind === "status" ? endpoint.cloudflareError : undefined;
135 + // Cloudflare answers 530 with error 1033 when the tunnel has no live connector. Other
136 + // codes, such as 1016 for a broken DNS record, are not fixed by restarting cloudflared.
137 + if (status === 530 && (cloudflareError === undefined || cloudflareError === "1033")) {
138 return fail(
139 "no_connector",
140 "Cloudflare has no live connection from this machine for the tunnel.",
@@ -131,9 +142,10 @@ export function classifyReachability(
142 );
143 }
144 if (status !== null) {
145 + const code = cloudflareError === undefined ? "" : ` (Cloudflare error ${cloudflareError})`;
146 return fail(
147 "unexpected_response",
136 - `The T3 Connect address answered with HTTP ${status}.`,
148 + `The T3 Connect address answered with HTTP ${status}${code}.`,
149 restart,
150 );
151 }
@@ -207,24 +219,57 @@ const make = Effect.gen(function* () {
219 Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(),
220 );
221 const checkCounter = yield* Ref.make(0);
222 + // Checks update this as they finish, even when a newer check already did. A dead streak
223 + // or a restart backoff that began before a healthy check no longer counts.
224 + const lastHealthyOrder = yield* Ref.make(0);
225 + const deadConnectorRef = yield* SynchronizedRef.make<{
226 + readonly streak: {
227 + readonly pid: number;
228 + readonly since: number;
229 + readonly order: number;
230 + } | null;
231 + readonly restartAfterMillis: number;
232 + readonly restartOrder: number;
233 + }>({
234 + streak: null,
235 + restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
236 + restartOrder: 0,
237 + });
238 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
239 + const scope = yield* Scope.Scope;
240 const readExitNode = readTailscaleExitNode.pipe(
241 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
242 Effect.orElseSucceed(() => null),
243 );
244
216 - const probeEndpoint = (hostname: string) =>
217 - httpClient.get(`https://${hostname}/.well-known/t3/environment`).pipe(
218 - Effect.map((response): EndpointProbe => ({ kind: "status", status: response.status })),
219 - Effect.scoped,
220 - Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT),
221 - Effect.map(Option.getOrElse((): EndpointProbe => ({ kind: "timeout" }))),
222 - Effect.catch((error) =>
223 - Effect.succeed<EndpointProbe>({
224 - kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",
225 - }),
226 - ),
227 - );
245 + const probeEndpoint = Effect.fn("ConnectReachability.probeEndpoint")(
246 + function* (hostname: string) {
247 + const response = yield* httpClient
248 + .get(`https://${hostname}/.well-known/t3/environment`, { acceptJson: true })
249 + .pipe(Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT));
250 + if (Option.isNone(response)) return { kind: "timeout" } satisfies EndpointProbe;
251 + const { status } = response.value;
252 + if (status !== 530) return { kind: "status", status } satisfies EndpointProbe;
253 + // Cloudflare's error page is HTML unless the client accepts JSON, which carries
254 + // `"error_code":1033`. Clients that send no User-Agent get `error code: 1033`.
255 + const body = yield* response.value.text.pipe(
256 + Effect.timeoutOption(ERROR_BODY_READ_TIMEOUT),
257 + Effect.orElseSucceed(() => Option.none<string>()),
258 + );
259 + const code = /error[_ ]code"?:\s*(\d{4})/i.exec(Option.getOrElse(body, () => ""))?.[1];
260 + return (
261 + code === undefined
262 + ? { kind: "status", status }
263 + : { kind: "status", status, cloudflareError: code }
264 + ) satisfies EndpointProbe;
265 + },
266 + Effect.scoped,
267 + Effect.catch((error) =>
268 + Effect.succeed<EndpointProbe>({
269 + kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",
270 + }),
271 + ),
272 + );
273
274 const gatherFacts = Effect.gen(function* () {
275 const managedTunnelConfigured = Option.isSome(
@@ -257,16 +302,90 @@ const make = Effect.gen(function* () {
302 return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode };
303 });
304
260 - const check = Effect.gen(function* () {
305 + const isLatest = (order: number) =>
306 + Ref.get(latestRef).pipe(
307 + Effect.map((latest) => Option.isNone(latest) || latest.value.order <= order),
308 + );
309 +
310 + // Runs inside the check span; the runtime restart has its own. Returns the streak a dead
311 + // connector just started and when to confirm it.
312 + const restartDeadConnector = Effect.fnUntraced(function* (
313 + facts: ReachabilityFacts,
314 + result: EnvironmentConnectReachability,
315 + order: number,
316 + ) {
317 + const now = yield* Clock.currentTimeMillis;
318 + const { connector } = facts;
319 + const dead =
320 + result.status === "unreachable" &&
321 + result.cause === "no_connector" &&
322 + connector !== null &&
323 + (connector.readyConnections ?? 0) > 0
324 + ? connector
325 + : null;
326 + return yield* SynchronizedRef.modifyEffect(deadConnectorRef, (current) =>
327 + Effect.gen(function* () {
328 + const healthy = yield* Ref.get(lastHealthyOrder);
329 + const state = {
330 + streak: current.streak && current.streak.order > healthy ? current.streak : null,
331 + restartAfterMillis:
332 + current.restartOrder > healthy
333 + ? current.restartAfterMillis
334 + : Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
335 + restartOrder: current.restartOrder,
336 + };
337 + if (!(yield* isLatest(order))) return [null, state] as const;
338 + if (dead === null) return [null, { ...state, streak: null }] as const;
339 + if (state.streak?.pid !== dead.pid) {
340 + const started = { ...state, streak: { pid: dead.pid, since: now, order } };
341 + return [{ order, after: Duration.millis(state.restartAfterMillis) }, started] as const;
342 + }
343 + if (now - state.streak.since < state.restartAfterMillis) return [null, state] as const;
344 + // Checks can invalidate the streak while this request waits for the runtime's lock.
345 + const streakOrder = state.streak.order;
346 + const stillDead = Effect.gen(function* () {
347 + if (!(yield* isLatest(order))) return false;
348 + return (yield* Ref.get(lastHealthyOrder)) < streakOrder;
349 + });
350 + if (!(yield* runtime.restartConnector(dead.pid, stillDead))) {
351 + return [null, state] as const;
352 + }
353 + const restarted = {
354 + streak: null,
355 + restartAfterMillis: Math.min(
356 + state.restartAfterMillis * 2,
357 + Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),
358 + ),
359 + restartOrder: order,
360 + };
361 + return [null, restarted] as const;
362 + }),
363 + );
364 + });
365 +
366 + const check: Effect.Effect<EnvironmentConnectReachability> = Effect.gen(function* () {
367 const order = yield* Ref.updateAndGet(checkCounter, (count) => count + 1);
368 const facts = yield* gatherFacts;
369 const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now));
370 + if (result.status === "reachable") {
371 + yield* Ref.update(lastHealthyOrder, (healthy) => Math.max(healthy, order));
372 + }
373 const previous = yield* Ref.modify(latestRef, (current) =>
374 Option.isSome(current) && current.value.order > order
375 ? [null, current]
376 : [current, Option.some({ order, result })],
377 );
378 if (previous === null) return result;
379 + const started = yield* restartDeadConnector(facts, result, order);
380 + // Nothing else would look again until the next periodic check.
381 + if (started !== null) {
382 + const confirm = SynchronizedRef.get(deadConnectorRef).pipe(
383 + Effect.flatMap(({ streak }) =>
384 + streak?.order === started.order ? Effect.suspend(() => check) : Effect.void,
385 + ),
386 + );
387 + yield* Effect.sleep(started.after).pipe(Effect.andThen(confirm), Effect.forkIn(scope));
388 + }
389 const changed = Option.match(previous, {
390 onNone: () => true,
391 onSome: ({ result: last }) =>
apps/server/src/cloud/ManagedEndpointRuntime.test.ts
+55
@@ -413,6 +413,28 @@ describe("CloudManagedEndpointRuntime", () => {
413 return { spawner, spawned, killed, exits, started, emit, observe };
414 });
415
416 + it.effect("restarts only the inspected connector and supervises its replacement", () =>
417 + Effect.gen(function* () {
418 + const fixture = yield* makeTransportSpawner();
419 + const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
420 + expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(false);
421 + yield* runtime.applyConfig(tunnelConfig);
422 + yield* fixture.observe(runtime, 0);
423 + expect(yield* runtime.inspectConnector).toMatchObject({ pid: 800 });
424 + expect(yield* runtime.restartConnector(799, Effect.succeed(true))).toBe(false);
425 + // A newer check saw the tunnel recover while this one waited for the lock.
426 + expect(yield* runtime.restartConnector(800, Effect.succeed(false))).toBe(false);
427 + expect(fixture.killed).toEqual([]);
428 + expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(true);
429 + expect(yield* Queue.take(fixture.started)).toBe(0);
430 + expect(yield* Queue.take(fixture.started)).toBe(1);
431 + yield* fixture.observe(runtime, 1);
432 + expect(fixture.killed).toContain(800);
433 + expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(false);
434 + expect(yield* runtime.inspectConnector).toMatchObject({ pid: 801 });
435 + }),
436 + );
437 +
438 it.effect("preserves an explicit transport choice despite repeated QUIC stalls", () =>
439 Effect.gen(function* () {
440 const fixture = yield* makeTransportSpawner({ transport: "quic" });
@@ -557,6 +579,39 @@ describe("CloudManagedEndpointRuntime", () => {
579 }),
580 );
581
582 + it.effect("decides a restart only once it holds the reconcile lock", () =>
583 + Effect.gen(function* () {
584 + const checking = yield* Deferred.make<void>();
585 + const releaseCheck = yield* Deferred.make<void>();
586 + const fixture = yield* makeTransportSpawner({
587 + isRunning: Deferred.succeed(checking, undefined).pipe(
588 + Effect.andThen(Deferred.await(releaseCheck)),
589 + Effect.as(true),
590 + ),
591 + });
592 + const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
593 + yield* runtime.applyConfig(tunnelConfig);
594 + yield* fixture.observe(runtime, 0);
595 +
596 + // Hold reconciliation in a process probe so the restart waits for the lock.
597 + const probe = yield* runtime.applyConfig(tunnelConfig).pipe(Effect.forkChild);
598 + yield* Deferred.await(checking);
599 + let stillDead = true;
600 + const restart = yield* runtime
601 + .restartConnector(
602 + 800,
603 + Effect.sync(() => stillDead),
604 + )
605 + .pipe(Effect.forkChild({ startImmediately: true }));
606 + stillDead = false;
607 + yield* Deferred.succeed(releaseCheck, undefined);
608 + yield* Fiber.join(probe);
609 +
610 + expect(yield* Fiber.join(restart)).toBe(false);
611 + expect(fixture.killed).toEqual([]);
612 + }),
613 + );
614 +
615 it.effect("keeps redacted connector logs bounded even when rotation fails", () =>
616 Effect.gen(function* () {
617 const fs = yield* FileSystem.FileSystem;
apps/server/src/cloud/ManagedEndpointRuntime.ts
+30 -1
@@ -53,6 +53,7 @@ export type CloudManagedEndpointRuntimeStatus =
53
54 /** What the running cloudflared reports about itself. `null` fields are not known yet. */
55 export interface ConnectorFacts {
56 + readonly pid: number;
57 readonly readyConnections: number | null;
58 /** The public tunnel hostname from the remote config cloudflared receives once it connects. */
59 readonly hostname: string | null;
@@ -71,6 +72,14 @@ export class CloudManagedEndpointRuntime extends Context.Service<
72 readonly readinessChanged: Stream.Stream<void>;
73 readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
74 readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
75 + /**
76 + * Restarts the connector if it still runs as `pid` and `stillDead` holds once it has
77 + * the lock. Returns whether it sent the restart.
78 + */
79 + readonly restartConnector: (
80 + pid: number,
81 + stillDead: Effect.Effect<boolean>,
82 + ) => Effect.Effect<boolean>;
83 readonly tunnelConnected: Stream.Stream<void>;
84 readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
85 }
@@ -822,6 +831,23 @@ export const make = Effect.gen(function* () {
831 withRelayClientLog,
832 );
833
834 + const restartConnector = Effect.fn("CloudManagedEndpointRuntime.restartConnector")(function* (
835 + pid: number,
836 + stillDead: Effect.Effect<boolean>,
837 + ) {
838 + const active = yield* Ref.get(activeRef);
839 + if (!active || Number(active.child.pid) !== pid || !(yield* stillDead)) return false;
840 + yield* Effect.logWarning(
841 + "Cloudflare has no live connection from the relay client; restarting it",
842 + { pid, tunnelId: active.config.tunnelId, tunnelName: active.config.tunnelName },
843 + );
844 + // The supervisor starts the replacement with its usual crash-loop backoff.
845 + return yield* active.child.kill().pipe(
846 + Effect.as(true),
847 + Effect.orElseSucceed(() => false),
848 + );
849 + }, reconcileSemaphore.withPermits(1));
850 +
851 const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) =>
852 httpClient.get(url).pipe(
853 Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)),
@@ -837,9 +863,10 @@ export const make = Effect.gen(function* () {
863 if (!active || !(yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)))) {
864 return null;
865 }
866 + const pid = Number(active.child.pid);
867 const protocol = yield* Ref.get(active.protocol);
868 if (!(yield* Deferred.isDone(active.metricsAddress))) {
842 - return { readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
869 + return { pid, readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
870 }
871 const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`;
872 const [ready, config] = yield* Effect.all(
@@ -850,6 +877,7 @@ export const make = Effect.gen(function* () {
877 { concurrency: 2 },
878 );
879 return {
880 + pid,
881 readyConnections: ready?.readyConnections ?? null,
882 hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null,
883 protocol,
@@ -864,6 +892,7 @@ export const make = Effect.gen(function* () {
892 requestRecovery: Effect.fn("CloudManagedEndpointRuntime.requestRecovery")(function (config) {
893 return Queue.offer(recoveryRequests, config).pipe(Effect.asVoid);
894 }),
895 + restartConnector,
896 tunnelConnected: Stream.fromQueue(tunnelConnections),
897 withLinkStateLock: linkStateSemaphore.withPermits(1),
898 });
apps/server/src/guest/GuestShareService.test.ts
+1
@@ -80,6 +80,7 @@ const layerOrchestration = Layer.mergeAll(
80 const hostname = Ref.makeUnsafe<string | null>("env.t3.test");
81 const layerConnect = Layer.mock(CloudManagedEndpointRuntime)({
82 inspectConnector: Effect.map(Ref.get(hostname), (value) => ({
83 + pid: 1,
84 readyConnections: 1,
85 hostname: value,
86 protocol: "http2",