The best way to control Claude Code, Codex and more from any computer or phone
fix(server): restart T3 Connect's tunnel client when Cloudflare drops it (#24946)
Co-authored-by: PR Batch Tester <agent@local.test> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>
shivam committed
Nov 5, 2026 at 06:58 UTC
c192c439c15f85310b6f1732d83c562c19d5978a
6 files changed
+531
-22
apps/server/src/cloud/CloudLink.lifecycle.test.ts
+1
@@ -206,6 +206,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
206
}),
207
recoveryRequests: Stream.empty,
208
requestRecovery: () => Effect.void,
209
+ restartConnector: () => Effect.succeed(false),
210
tunnelConnected: Stream.empty,
211
inspectConnector: Effect.succeed(null),
212
readinessChanged: Stream.empty,
apps/server/src/cloud/ConnectReachability.test.ts
+308
-4
@@ -1,14 +1,32 @@
1
// @effect-diagnostics nodeBuiltinImport:off - reads the user guide to check its anchors.
2
import * as NodeFS from "node:fs";
3
+import * as NodeNet from "node:net";
4
5
import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts";
6
import { describe, expect, it } from "@effect/vitest";
7
+import * as Clock from "effect/Clock";
8
+import * as Deferred from "effect/Deferred";
9
+import * as Duration from "effect/Duration";
10
+import * as Effect from "effect/Effect";
11
+import * as Fiber from "effect/Fiber";
12
+import * as Layer from "effect/Layer";
13
+import * as Queue from "effect/Queue";
14
+import * as Stream from "effect/Stream";
15
+import * as TestClock from "effect/testing/TestClock";
16
+import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
17
+import { ChildProcessSpawner } from "effect/process";
18
+import { vi } from "vite-plus/test";
19
20
+import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
21
+import * as ConnectReachability from "./ConnectReachability.ts";
22
import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";
23
+import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
24
+
25
+vi.mock("node:net", { spy: true });
26
27
const checkedAt = "2026-10-19T00:00:00.000Z";
10
-const connected = { readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
11
-const notConnected = { readyConnections: 0, hostname: null, protocol: null };
28
+const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
29
+const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null };
30
const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({
31
managedTunnelConfigured: true,
32
connector: connected,
@@ -22,6 +40,8 @@ describe("classifyReachability", () => {
40
it.each([
41
[{ kind: "status", status: 502 }, "origin_unreachable"],
42
[{ kind: "status", status: 530 }, "no_connector"],
43
+ [{ kind: "status", status: 530, cloudflareError: "1033" }, "no_connector"],
44
+ [{ kind: "status", status: 530, cloudflareError: "1016" }, "unexpected_response"],
45
[{ kind: "network_error" }, "timeout"],
46
] as const)("maps endpoint result %o to %s", (endpoint, cause) => {
47
expect(classifyReachability(facts({ endpoint, edgeReachable: true }), checkedAt)).toMatchObject(
@@ -40,7 +60,12 @@ describe("classifyReachability", () => {
60
expect(
61
classifyReachability(
62
facts({
43
- connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
63
+ connector: {
64
+ pid: 1,
65
+ readyConnections: null,
66
+ hostname: "env.example.test",
67
+ protocol: null,
68
+ },
69
endpoint: { kind: "status", status: 502 },
70
edgeReachable: false,
71
}),
@@ -51,7 +76,12 @@ describe("classifyReachability", () => {
76
expect(
77
classifyReachability(
78
facts({
54
- connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
79
+ connector: {
80
+ pid: 1,
81
+ readyConnections: null,
82
+ hostname: "env.example.test",
83
+ protocol: null,
84
+ },
85
endpoint: { kind: "dns_failed" },
86
edgeReachable: false,
87
}),
@@ -74,6 +104,280 @@ describe("classifyReachability", () => {
104
});
105
});
106
107
+interface ServedResponse {
108
+ readonly status: number;
109
+ readonly body: string | ReadableStream<Uint8Array> | { readonly cloudflareError: number };
110
+}
111
+
112
+// What Cloudflare serves for its 530 page: HTML unless the client accepts JSON.
113
+const cloudflareErrorPage = (request: HttpClientRequest.HttpClientRequest, code: number) =>
114
+ request.headers.accept === "application/json"
115
+ ? JSON.stringify({ status: 530, error_code: code, error_name: "tunnel_error" })
116
+ : `<!doctype html><h1><span>Error</span> <span>${code}</span></h1>`;
117
+
118
+const reachabilityLayer = (
119
+ respond: Effect.Effect<ServedResponse>,
120
+ restartConnector: (pid: number, stillDead: Effect.Effect<boolean>) => Effect.Effect<boolean>,
121
+) =>
122
+ ConnectReachability.layer.pipe(
123
+ Layer.provide(
124
+ Layer.mergeAll(
125
+ Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({
126
+ inspectConnector: Effect.succeed(connected),
127
+ readinessChanged: Stream.never,
128
+ restartConnector,
129
+ }),
130
+ Layer.mock(ServerSecretStore.ServerSecretStore)({
131
+ get: () => Effect.succeedSome(new Uint8Array()),
132
+ }),
133
+ Layer.succeed(
134
+ HttpClient.HttpClient,
135
+ HttpClient.make((request) =>
136
+ respond.pipe(
137
+ Effect.map(({ status, body }) =>
138
+ HttpClientResponse.fromWeb(
139
+ request,
140
+ new Response(
141
+ typeof body === "object" && "cloudflareError" in body
142
+ ? cloudflareErrorPage(request, body.cloudflareError)
143
+ : body,
144
+ { status },
145
+ ),
146
+ ),
147
+ ),
148
+ ),
149
+ ),
150
+ ),
151
+ Layer.succeed(
152
+ ChildProcessSpawner.ChildProcessSpawner,
153
+ ChildProcessSpawner.make(() => Effect.die("no tailscale")),
154
+ ),
155
+ ),
156
+ ),
157
+ );
158
+
159
+const mockEdgeConnect = () =>
160
+ vi.spyOn(NodeNet, "connect").mockImplementation(() => {
161
+ const socket = new NodeNet.Socket();
162
+ queueMicrotask(() => socket.emit("connect"));
163
+ return socket;
164
+ });
165
+
166
+describe("ConnectReachability stale connector recovery", () => {
167
+ it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>
168
+ Effect.gen(function* () {
169
+ using _connect = mockEdgeConnect();
170
+ const restarts = yield* Queue.unbounded<number>();
171
+ // Cloudflare answers 530 while cloudflared still reports four ready connections.
172
+ let status = 530;
173
+ let body = { cloudflareError: 1033 };
174
+ const layer = reachabilityLayer(
175
+ Effect.sync(() => ({ status, body })),
176
+ (pid, stillDead) =>
177
+ stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
178
+ );
179
+ // The scheduled checks run in the layer's scope, so the assertions run inside it.
180
+ yield* Effect.gen(function* () {
181
+ const reachability = yield* ConnectReachability.ConnectReachability;
182
+
183
+ // The first sighting schedules its own confirming check two minutes later.
184
+ expect(yield* reachability.check).toMatchObject({ cause: "no_connector" });
185
+ yield* TestClock.adjust(Duration.minutes(1));
186
+ expect(yield* Queue.size(restarts)).toBe(0);
187
+ yield* TestClock.adjust(Duration.minutes(1));
188
+ expect(yield* Queue.take(restarts)).toBe(connected.pid);
189
+
190
+ // A restart that did not help doubles the wait.
191
+ yield* reachability.check;
192
+ yield* TestClock.adjust(Duration.minutes(2));
193
+ expect(yield* Queue.size(restarts)).toBe(0);
194
+ yield* TestClock.adjust(Duration.minutes(2));
195
+ expect(yield* Queue.take(restarts)).toBe(connected.pid);
196
+
197
+ // Reaching the endpoint resets the wait, and a 530 for a broken DNS record is left alone.
198
+ status = 200;
199
+ expect(yield* reachability.check).toMatchObject({ status: "reachable" });
200
+ status = 530;
201
+ body = { cloudflareError: 1016 };
202
+ expect(yield* reachability.check).toMatchObject({ cause: "unexpected_response" });
203
+ yield* TestClock.adjust(Duration.minutes(2));
204
+ expect(yield* Queue.size(restarts)).toBe(0);
205
+
206
+ body = { cloudflareError: 1033 };
207
+ yield* reachability.check;
208
+ yield* TestClock.adjust(Duration.minutes(2));
209
+ expect(yield* Queue.take(restarts)).toBe(connected.pid);
210
+ }).pipe(Effect.provide(layer));
211
+ }),
212
+ );
213
+
214
+ it.effect("starts a new streak after a healthy check that a newer check overtook", () =>
215
+ Effect.gen(function* () {
216
+ using _connect = mockEdgeConnect();
217
+ const restartTimes = yield* Queue.unbounded<number>();
218
+ const healthyRequested = yield* Deferred.make<void>();
219
+ const healthyAnswer = yield* Deferred.make<void>();
220
+ let status = 530;
221
+ const layer = reachabilityLayer(
222
+ Effect.gen(function* () {
223
+ const served = status;
224
+ if (served === 200) {
225
+ yield* Deferred.succeed(healthyRequested, undefined);
226
+ yield* Deferred.await(healthyAnswer);
227
+ }
228
+ return { status: served, body: { cloudflareError: 1033 } };
229
+ }),
230
+ (_pid, stillDead) =>
231
+ stillDead.pipe(
232
+ Effect.tap((dead) =>
233
+ dead
234
+ ? Clock.currentTimeMillis.pipe(
235
+ Effect.flatMap((now) => Queue.offer(restartTimes, now)),
236
+ )
237
+ : Effect.void,
238
+ ),
239
+ ),
240
+ );
241
+ yield* Effect.gen(function* () {
242
+ const reachability = yield* ConnectReachability.ConnectReachability;
243
+ const startedAt = yield* Clock.currentTimeMillis;
244
+ yield* reachability.check;
245
+ yield* TestClock.adjust(Duration.minutes(1));
246
+
247
+ // The healthy answer arrives after a newer check already reported the tunnel dead.
248
+ status = 200;
249
+ const healthy = yield* Effect.forkChild(reachability.check);
250
+ yield* Deferred.await(healthyRequested);
251
+ status = 530;
252
+ yield* reachability.check;
253
+ yield* Deferred.succeed(healthyAnswer, undefined);
254
+ expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
255
+
256
+ // The grace period starts again at the next dead check, a minute after the first one.
257
+ yield* reachability.check;
258
+ yield* TestClock.adjust(Duration.minutes(2));
259
+ expect((yield* Queue.take(restartTimes)) - startedAt).toBe(
260
+ Duration.toMillis(Duration.minutes(3)),
261
+ );
262
+ }).pipe(Effect.provide(layer));
263
+ }),
264
+ );
265
+
266
+ it.effect("rejects a restart when an overtaken healthy check finishes while it waits", () =>
267
+ Effect.gen(function* () {
268
+ using _connect = mockEdgeConnect();
269
+ const healthyRequested = yield* Deferred.make<void>();
270
+ const healthyAnswer = yield* Deferred.make<void>();
271
+ const restartRequested = yield* Deferred.make<void>();
272
+ const releaseRestart = yield* Deferred.make<void>();
273
+ const guardChecks = yield* Queue.unbounded<boolean>();
274
+ let status = 530;
275
+ const layer = reachabilityLayer(
276
+ Effect.gen(function* () {
277
+ const served = status;
278
+ if (served === 200) {
279
+ yield* Deferred.succeed(healthyRequested, undefined);
280
+ yield* Deferred.await(healthyAnswer);
281
+ }
282
+ return { status: served, body: { cloudflareError: 1033 } };
283
+ }),
284
+ (_pid, stillDead) =>
285
+ Effect.gen(function* () {
286
+ yield* Deferred.succeed(restartRequested, undefined);
287
+ yield* Deferred.await(releaseRestart);
288
+ const accepted = yield* stillDead;
289
+ yield* Queue.offer(guardChecks, accepted);
290
+ return accepted;
291
+ }),
292
+ );
293
+ yield* Effect.gen(function* () {
294
+ const reachability = yield* ConnectReachability.ConnectReachability;
295
+ yield* reachability.check;
296
+ yield* TestClock.adjust(Duration.seconds(119));
297
+
298
+ status = 200;
299
+ const healthy = yield* Effect.forkChild(reachability.check);
300
+ yield* Deferred.await(healthyRequested);
301
+ status = 530;
302
+ yield* TestClock.adjust(Duration.seconds(1));
303
+ yield* Deferred.await(restartRequested);
304
+
305
+ yield* Deferred.succeed(healthyAnswer, undefined);
306
+ expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
307
+ yield* Deferred.succeed(releaseRestart, undefined);
308
+ expect(yield* Queue.take(guardChecks)).toBe(false);
309
+
310
+ yield* reachability.check;
311
+ yield* TestClock.adjust(Duration.minutes(1));
312
+ expect(yield* Queue.size(guardChecks)).toBe(0);
313
+ yield* TestClock.adjust(Duration.minutes(1));
314
+ expect(yield* Queue.take(guardChecks)).toBe(true);
315
+ }).pipe(Effect.provide(layer));
316
+ }),
317
+ );
318
+
319
+ it.effect("drops the confirming check of a streak that already ended", () =>
320
+ Effect.gen(function* () {
321
+ using _connect = mockEdgeConnect();
322
+ const restarts = yield* Queue.unbounded<number>();
323
+ let status = 530;
324
+ let requests = 0;
325
+ const layer = reachabilityLayer(
326
+ Effect.sync(() => {
327
+ requests += 1;
328
+ return { status, body: { cloudflareError: 1033 } };
329
+ }),
330
+ (pid, stillDead) =>
331
+ stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
332
+ );
333
+ yield* Effect.gen(function* () {
334
+ const reachability = yield* ConnectReachability.ConnectReachability;
335
+ yield* reachability.check;
336
+ yield* TestClock.adjust(Duration.minutes(1));
337
+ status = 200;
338
+ yield* reachability.check;
339
+ status = 530;
340
+ yield* reachability.check;
341
+
342
+ const before = requests;
343
+ yield* TestClock.adjust(Duration.minutes(1));
344
+ expect(requests).toBe(before);
345
+ yield* TestClock.adjust(Duration.minutes(1));
346
+ expect(yield* Queue.take(restarts)).toBe(connected.pid);
347
+ }).pipe(Effect.provide(layer));
348
+ }),
349
+ );
350
+
351
+ it.effect("keeps a 530 whose error body never finishes", () =>
352
+ Effect.gen(function* () {
353
+ using _connect = mockEdgeConnect();
354
+ let markBodyRead = () => {};
355
+ const bodyRead = new Promise<void>((resolve) => {
356
+ markBodyRead = resolve;
357
+ });
358
+ const layer = reachabilityLayer(
359
+ Effect.sync(() => ({
360
+ status: 530,
361
+ body: new ReadableStream({
362
+ pull: () => {
363
+ markBodyRead();
364
+ return new Promise<void>(() => {});
365
+ },
366
+ }),
367
+ })),
368
+ () => Effect.succeed(false),
369
+ );
370
+ yield* Effect.gen(function* () {
371
+ const reachability = yield* ConnectReachability.ConnectReachability;
372
+ const check = yield* Effect.forkChild(reachability.check);
373
+ yield* Effect.promise(() => bodyRead);
374
+ yield* TestClock.adjust(Duration.seconds(2));
375
+ expect(yield* Fiber.join(check)).toMatchObject({ cause: "no_connector" });
376
+ }).pipe(Effect.provide(layer));
377
+ }),
378
+ );
379
+});
380
+
381
describe("T3 Connect troubleshooting links", () => {
382
it("each section names a heading in the user guide", () => {
383
const guide = NodeFS.readFileSync(
apps/server/src/cloud/ConnectReachability.ts
+136
-17
@@ -6,6 +6,7 @@ import {
6
formatConnectReachable,
7
} from "@t3tools/contracts";
8
import { readTailscaleExitNode, type TailscaleExitNode } from "@t3tools/tailscale";
9
+import * as Clock from "effect/Clock";
10
import * as Context from "effect/Context";
11
import * as DateTime from "effect/DateTime";
12
import * as Duration from "effect/Duration";
@@ -13,7 +14,9 @@ import * as Effect from "effect/Effect";
14
import * as Layer from "effect/Layer";
15
import * as Option from "effect/Option";
16
import * as Ref from "effect/Ref";
17
+import * as Scope from "effect/Scope";
18
import * as Stream from "effect/Stream";
19
+import * as SynchronizedRef from "effect/SynchronizedRef";
20
import { HttpClient } from "effect/http";
21
import { ChildProcessSpawner } from "effect/process";
22
@@ -27,13 +30,19 @@ const TUNNEL_EDGE_HOSTS = ["region1.v2.argotunnel.com", "region2.v2.argotunnel.c
30
const TUNNEL_EDGE_PORT = 7844;
31
const EDGE_CONNECT_TIMEOUT = Duration.seconds(5);
32
const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10);
33
+const ERROR_BODY_READ_TIMEOUT = Duration.seconds(2);
34
// cloudflared registers several connections within a few seconds; they are one change.
35
const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3);
36
// Readiness changes miss failures and recoveries past cloudflared (DNS, origin, edge errors).
37
const RECHECK_INTERVAL = Duration.minutes(5);
38
+// cloudflared can keep reporting ready connections the edge already dropped, so its
39
+// own readiness watch never restarts it. A 530 can also be brief, so restart only once it
40
+// lasts, and wait longer after each restart that did not help.
41
+const DEAD_CONNECTOR_RESTART_AFTER = Duration.minutes(2);
42
+const DEAD_CONNECTOR_RESTART_AFTER_MAX = Duration.hours(1);
43
44
export type EndpointProbe =
36
- | { readonly kind: "status"; readonly status: number }
45
+ | { readonly kind: "status"; readonly status: number; readonly cloudflareError?: string }
46
| { readonly kind: "dns_failed" | "timeout" | "network_error" };
47
48
/** Everything one check learned. `null` means that step did not run. */
@@ -122,8 +131,10 @@ export function classifyReachability(
131
"Wait a minute. If it stays, check the network and the server log.",
132
);
133
}
125
- // Cloudflare answers 530 (error 1033) when the tunnel has no live connector.
126
- if (status === 530) {
134
+ const cloudflareError = endpoint.kind === "status" ? endpoint.cloudflareError : undefined;
135
+ // Cloudflare answers 530 with error 1033 when the tunnel has no live connector. Other
136
+ // codes, such as 1016 for a broken DNS record, are not fixed by restarting cloudflared.
137
+ if (status === 530 && (cloudflareError === undefined || cloudflareError === "1033")) {
138
return fail(
139
"no_connector",
140
"Cloudflare has no live connection from this machine for the tunnel.",
@@ -131,9 +142,10 @@ export function classifyReachability(
142
);
143
}
144
if (status !== null) {
145
+ const code = cloudflareError === undefined ? "" : ` (Cloudflare error ${cloudflareError})`;
146
return fail(
147
"unexpected_response",
136
- `The T3 Connect address answered with HTTP ${status}.`,
148
+ `The T3 Connect address answered with HTTP ${status}${code}.`,
149
restart,
150
);
151
}
@@ -207,24 +219,57 @@ const make = Effect.gen(function* () {
219
Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(),
220
);
221
const checkCounter = yield* Ref.make(0);
222
+ // Checks update this as they finish, even when a newer check already did. A dead streak
223
+ // or a restart backoff that began before a healthy check no longer counts.
224
+ const lastHealthyOrder = yield* Ref.make(0);
225
+ const deadConnectorRef = yield* SynchronizedRef.make<{
226
+ readonly streak: {
227
+ readonly pid: number;
228
+ readonly since: number;
229
+ readonly order: number;
230
+ } | null;
231
+ readonly restartAfterMillis: number;
232
+ readonly restartOrder: number;
233
+ }>({
234
+ streak: null,
235
+ restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
236
+ restartOrder: 0,
237
+ });
238
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
239
+ const scope = yield* Scope.Scope;
240
const readExitNode = readTailscaleExitNode.pipe(
241
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
242
Effect.orElseSucceed(() => null),
243
);
244
216
- const probeEndpoint = (hostname: string) =>
217
- httpClient.get(`https://${hostname}/.well-known/t3/environment`).pipe(
218
- Effect.map((response): EndpointProbe => ({ kind: "status", status: response.status })),
219
- Effect.scoped,
220
- Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT),
221
- Effect.map(Option.getOrElse((): EndpointProbe => ({ kind: "timeout" }))),
222
- Effect.catch((error) =>
223
- Effect.succeed<EndpointProbe>({
224
- kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",
225
- }),
226
- ),
227
- );
245
+ const probeEndpoint = Effect.fn("ConnectReachability.probeEndpoint")(
246
+ function* (hostname: string) {
247
+ const response = yield* httpClient
248
+ .get(`https://${hostname}/.well-known/t3/environment`, { acceptJson: true })
249
+ .pipe(Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT));
250
+ if (Option.isNone(response)) return { kind: "timeout" } satisfies EndpointProbe;
251
+ const { status } = response.value;
252
+ if (status !== 530) return { kind: "status", status } satisfies EndpointProbe;
253
+ // Cloudflare's error page is HTML unless the client accepts JSON, which carries
254
+ // `"error_code":1033`. Clients that send no User-Agent get `error code: 1033`.
255
+ const body = yield* response.value.text.pipe(
256
+ Effect.timeoutOption(ERROR_BODY_READ_TIMEOUT),
257
+ Effect.orElseSucceed(() => Option.none<string>()),
258
+ );
259
+ const code = /error[_ ]code"?:\s*(\d{4})/i.exec(Option.getOrElse(body, () => ""))?.[1];
260
+ return (
261
+ code === undefined
262
+ ? { kind: "status", status }
263
+ : { kind: "status", status, cloudflareError: code }
264
+ ) satisfies EndpointProbe;
265
+ },
266
+ Effect.scoped,
267
+ Effect.catch((error) =>
268
+ Effect.succeed<EndpointProbe>({
269
+ kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",
270
+ }),
271
+ ),
272
+ );
273
274
const gatherFacts = Effect.gen(function* () {
275
const managedTunnelConfigured = Option.isSome(
@@ -257,16 +302,90 @@ const make = Effect.gen(function* () {
302
return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode };
303
});
304
260
- const check = Effect.gen(function* () {
305
+ const isLatest = (order: number) =>
306
+ Ref.get(latestRef).pipe(
307
+ Effect.map((latest) => Option.isNone(latest) || latest.value.order <= order),
308
+ );
309
+
310
+ // Runs inside the check span; the runtime restart has its own. Returns the streak a dead
311
+ // connector just started and when to confirm it.
312
+ const restartDeadConnector = Effect.fnUntraced(function* (
313
+ facts: ReachabilityFacts,
314
+ result: EnvironmentConnectReachability,
315
+ order: number,
316
+ ) {
317
+ const now = yield* Clock.currentTimeMillis;
318
+ const { connector } = facts;
319
+ const dead =
320
+ result.status === "unreachable" &&
321
+ result.cause === "no_connector" &&
322
+ connector !== null &&
323
+ (connector.readyConnections ?? 0) > 0
324
+ ? connector
325
+ : null;
326
+ return yield* SynchronizedRef.modifyEffect(deadConnectorRef, (current) =>
327
+ Effect.gen(function* () {
328
+ const healthy = yield* Ref.get(lastHealthyOrder);
329
+ const state = {
330
+ streak: current.streak && current.streak.order > healthy ? current.streak : null,
331
+ restartAfterMillis:
332
+ current.restartOrder > healthy
333
+ ? current.restartAfterMillis
334
+ : Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
335
+ restartOrder: current.restartOrder,
336
+ };
337
+ if (!(yield* isLatest(order))) return [null, state] as const;
338
+ if (dead === null) return [null, { ...state, streak: null }] as const;
339
+ if (state.streak?.pid !== dead.pid) {
340
+ const started = { ...state, streak: { pid: dead.pid, since: now, order } };
341
+ return [{ order, after: Duration.millis(state.restartAfterMillis) }, started] as const;
342
+ }
343
+ if (now - state.streak.since < state.restartAfterMillis) return [null, state] as const;
344
+ // Checks can invalidate the streak while this request waits for the runtime's lock.
345
+ const streakOrder = state.streak.order;
346
+ const stillDead = Effect.gen(function* () {
347
+ if (!(yield* isLatest(order))) return false;
348
+ return (yield* Ref.get(lastHealthyOrder)) < streakOrder;
349
+ });
350
+ if (!(yield* runtime.restartConnector(dead.pid, stillDead))) {
351
+ return [null, state] as const;
352
+ }
353
+ const restarted = {
354
+ streak: null,
355
+ restartAfterMillis: Math.min(
356
+ state.restartAfterMillis * 2,
357
+ Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),
358
+ ),
359
+ restartOrder: order,
360
+ };
361
+ return [null, restarted] as const;
362
+ }),
363
+ );
364
+ });
365
+
366
+ const check: Effect.Effect<EnvironmentConnectReachability> = Effect.gen(function* () {
367
const order = yield* Ref.updateAndGet(checkCounter, (count) => count + 1);
368
const facts = yield* gatherFacts;
369
const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now));
370
+ if (result.status === "reachable") {
371
+ yield* Ref.update(lastHealthyOrder, (healthy) => Math.max(healthy, order));
372
+ }
373
const previous = yield* Ref.modify(latestRef, (current) =>
374
Option.isSome(current) && current.value.order > order
375
? [null, current]
376
: [current, Option.some({ order, result })],
377
);
378
if (previous === null) return result;
379
+ const started = yield* restartDeadConnector(facts, result, order);
380
+ // Nothing else would look again until the next periodic check.
381
+ if (started !== null) {
382
+ const confirm = SynchronizedRef.get(deadConnectorRef).pipe(
383
+ Effect.flatMap(({ streak }) =>
384
+ streak?.order === started.order ? Effect.suspend(() => check) : Effect.void,
385
+ ),
386
+ );
387
+ yield* Effect.sleep(started.after).pipe(Effect.andThen(confirm), Effect.forkIn(scope));
388
+ }
389
const changed = Option.match(previous, {
390
onNone: () => true,
391
onSome: ({ result: last }) =>
apps/server/src/cloud/ManagedEndpointRuntime.test.ts
+55
@@ -413,6 +413,28 @@ describe("CloudManagedEndpointRuntime", () => {
413
return { spawner, spawned, killed, exits, started, emit, observe };
414
});
415
416
+ it.effect("restarts only the inspected connector and supervises its replacement", () =>
417
+ Effect.gen(function* () {
418
+ const fixture = yield* makeTransportSpawner();
419
+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
420
+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(false);
421
+ yield* runtime.applyConfig(tunnelConfig);
422
+ yield* fixture.observe(runtime, 0);
423
+ expect(yield* runtime.inspectConnector).toMatchObject({ pid: 800 });
424
+ expect(yield* runtime.restartConnector(799, Effect.succeed(true))).toBe(false);
425
+ // A newer check saw the tunnel recover while this one waited for the lock.
426
+ expect(yield* runtime.restartConnector(800, Effect.succeed(false))).toBe(false);
427
+ expect(fixture.killed).toEqual([]);
428
+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(true);
429
+ expect(yield* Queue.take(fixture.started)).toBe(0);
430
+ expect(yield* Queue.take(fixture.started)).toBe(1);
431
+ yield* fixture.observe(runtime, 1);
432
+ expect(fixture.killed).toContain(800);
433
+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(false);
434
+ expect(yield* runtime.inspectConnector).toMatchObject({ pid: 801 });
435
+ }),
436
+ );
437
+
438
it.effect("preserves an explicit transport choice despite repeated QUIC stalls", () =>
439
Effect.gen(function* () {
440
const fixture = yield* makeTransportSpawner({ transport: "quic" });
@@ -557,6 +579,39 @@ describe("CloudManagedEndpointRuntime", () => {
579
}),
580
);
581
582
+ it.effect("decides a restart only once it holds the reconcile lock", () =>
583
+ Effect.gen(function* () {
584
+ const checking = yield* Deferred.make<void>();
585
+ const releaseCheck = yield* Deferred.make<void>();
586
+ const fixture = yield* makeTransportSpawner({
587
+ isRunning: Deferred.succeed(checking, undefined).pipe(
588
+ Effect.andThen(Deferred.await(releaseCheck)),
589
+ Effect.as(true),
590
+ ),
591
+ });
592
+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
593
+ yield* runtime.applyConfig(tunnelConfig);
594
+ yield* fixture.observe(runtime, 0);
595
+
596
+ // Hold reconciliation in a process probe so the restart waits for the lock.
597
+ const probe = yield* runtime.applyConfig(tunnelConfig).pipe(Effect.forkChild);
598
+ yield* Deferred.await(checking);
599
+ let stillDead = true;
600
+ const restart = yield* runtime
601
+ .restartConnector(
602
+ 800,
603
+ Effect.sync(() => stillDead),
604
+ )
605
+ .pipe(Effect.forkChild({ startImmediately: true }));
606
+ stillDead = false;
607
+ yield* Deferred.succeed(releaseCheck, undefined);
608
+ yield* Fiber.join(probe);
609
+
610
+ expect(yield* Fiber.join(restart)).toBe(false);
611
+ expect(fixture.killed).toEqual([]);
612
+ }),
613
+ );
614
+
615
it.effect("keeps redacted connector logs bounded even when rotation fails", () =>
616
Effect.gen(function* () {
617
const fs = yield* FileSystem.FileSystem;
apps/server/src/cloud/ManagedEndpointRuntime.ts
+30
-1
@@ -53,6 +53,7 @@ export type CloudManagedEndpointRuntimeStatus =
53
54
/** What the running cloudflared reports about itself. `null` fields are not known yet. */
55
export interface ConnectorFacts {
56
+ readonly pid: number;
57
readonly readyConnections: number | null;
58
/** The public tunnel hostname from the remote config cloudflared receives once it connects. */
59
readonly hostname: string | null;
@@ -71,6 +72,14 @@ export class CloudManagedEndpointRuntime extends Context.Service<
72
readonly readinessChanged: Stream.Stream<void>;
73
readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
74
readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
75
+ /**
76
+ * Restarts the connector if it still runs as `pid` and `stillDead` holds once it has
77
+ * the lock. Returns whether it sent the restart.
78
+ */
79
+ readonly restartConnector: (
80
+ pid: number,
81
+ stillDead: Effect.Effect<boolean>,
82
+ ) => Effect.Effect<boolean>;
83
readonly tunnelConnected: Stream.Stream<void>;
84
readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
85
}
@@ -822,6 +831,23 @@ export const make = Effect.gen(function* () {
831
withRelayClientLog,
832
);
833
834
+ const restartConnector = Effect.fn("CloudManagedEndpointRuntime.restartConnector")(function* (
835
+ pid: number,
836
+ stillDead: Effect.Effect<boolean>,
837
+ ) {
838
+ const active = yield* Ref.get(activeRef);
839
+ if (!active || Number(active.child.pid) !== pid || !(yield* stillDead)) return false;
840
+ yield* Effect.logWarning(
841
+ "Cloudflare has no live connection from the relay client; restarting it",
842
+ { pid, tunnelId: active.config.tunnelId, tunnelName: active.config.tunnelName },
843
+ );
844
+ // The supervisor starts the replacement with its usual crash-loop backoff.
845
+ return yield* active.child.kill().pipe(
846
+ Effect.as(true),
847
+ Effect.orElseSucceed(() => false),
848
+ );
849
+ }, reconcileSemaphore.withPermits(1));
850
+
851
const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) =>
852
httpClient.get(url).pipe(
853
Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)),
@@ -837,9 +863,10 @@ export const make = Effect.gen(function* () {
863
if (!active || !(yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)))) {
864
return null;
865
}
866
+ const pid = Number(active.child.pid);
867
const protocol = yield* Ref.get(active.protocol);
868
if (!(yield* Deferred.isDone(active.metricsAddress))) {
842
- return { readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
869
+ return { pid, readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
870
}
871
const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`;
872
const [ready, config] = yield* Effect.all(
@@ -850,6 +877,7 @@ export const make = Effect.gen(function* () {
877
{ concurrency: 2 },
878
);
879
return {
880
+ pid,
881
readyConnections: ready?.readyConnections ?? null,
882
hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null,
883
protocol,
@@ -864,6 +892,7 @@ export const make = Effect.gen(function* () {
892
requestRecovery: Effect.fn("CloudManagedEndpointRuntime.requestRecovery")(function (config) {
893
return Queue.offer(recoveryRequests, config).pipe(Effect.asVoid);
894
}),
895
+ restartConnector,
896
tunnelConnected: Stream.fromQueue(tunnelConnections),
897
withLinkStateLock: linkStateSemaphore.withPermits(1),
898
});
apps/server/src/guest/GuestShareService.test.ts
+1
@@ -80,6 +80,7 @@ const layerOrchestration = Layer.mergeAll(
80
const hostname = Ref.makeUnsafe<string | null>("env.t3.test");
81
const layerConnect = Layer.mock(CloudManagedEndpointRuntime)({
82
inspectConnector: Effect.map(Ref.get(hostname), (value) => ({
83
+ pid: 1,
84
readyConnections: 1,
85
hostname: value,
86
protocol: "http2",