@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 987 lines 39.5 KB
Raw
1 import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2 import { RotatingFileSink, RotatingFileSinkError } from "@t3tools/shared/logging";
3 import * as HostProcess from "@t3tools/shared/HostProcess";
4 import * as RelayClient from "@t3tools/shared/relayClient";
5 import * as Clock from "effect/Clock";
6 import * as Config from "effect/Config";
7 import * as Context from "effect/Context";
8 import * as Deferred from "effect/Deferred";
9 import * as Duration from "effect/Duration";
10 import * as Effect from "effect/Effect";
11 import * as Exit from "effect/Exit";
12 import * as Layer from "effect/Layer";
13 import * as Logger from "effect/Logger";
14 import * as Option from "effect/Option";
15 import * as Path from "effect/Path";
16 import * as Queue from "effect/Queue";
17 import * as Ref from "effect/Ref";
18 import * as Result from "effect/Result";
19 import * as Schema from "effect/Schema";
20 import * as Semaphore from "effect/Semaphore";
21 import * as Scope from "effect/Scope";
22 import * as Stream from "effect/Stream";
23 import { HttpClient, HttpClientResponse } from "effect/http";
24 import * as ChildProcess from "effect/process/ChildProcess";
25 import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
26
27 import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
28 import * as ServerConfig from "../config.ts";
29 import * as ManagedLimits from "../policy/managedLimits.ts";
30 import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
31 import * as RelayClientLedger from "./RelayClientLedger.ts";
32
33 export type CloudManagedEndpointRuntimeStatus =
34 | {
35 readonly status: "disabled";
36 }
37 | {
38 readonly status: "failed";
39 readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"];
40 readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed";
41 readonly reason: string;
42 readonly tunnelId?: string;
43 readonly tunnelName?: string;
44 }
45 | {
46 readonly status: "running";
47 readonly providerKind: "cloudflare_tunnel";
48 readonly pid: number;
49 readonly tunnelId?: string;
50 readonly tunnelName?: string;
51 }
52 | {
53 readonly status: "unsupported";
54 readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"];
55 };
56
57 /** What the running cloudflared reports about itself. `null` fields are not known yet. */
58 export interface ConnectorFacts {
59 readonly pid: number;
60 readonly readyConnections: number | null;
61 /** The public tunnel hostname from the remote config cloudflared receives once it connects. */
62 readonly hostname: string | null;
63 /** The local origin that remote config routes the public hostname to. */
64 readonly origin: string | null;
65 readonly protocol: string | null;
66 }
67
68 export class CloudManagedEndpointRuntime extends Context.Service<
69 CloudManagedEndpointRuntime,
70 {
71 readonly applyConfig: (
72 config: RelayManagedEndpointRuntimeConfig | null,
73 ) => Effect.Effect<CloudManagedEndpointRuntimeStatus>;
74 /** `null` when no connector is running. */
75 readonly inspectConnector: Effect.Effect<ConnectorFacts | null>;
76 /** Emits when a connector registers a connection or its readiness changes. */
77 readonly readinessChanged: Stream.Stream<void>;
78 readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
79 readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
80 /**
81 * Restarts the connector if it still runs as `pid` and `stillDead` holds once it has
82 * the lock. Returns whether it sent the restart.
83 */
84 readonly restartConnector: (
85 pid: number,
86 stillDead: Effect.Effect<boolean>,
87 ) => Effect.Effect<boolean>;
88 readonly tunnelConnected: Stream.Stream<void>;
89 readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
90 }
91 >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {}
92
93 interface ActiveConnector {
94 readonly child: ChildProcessSpawner.ChildProcessHandle;
95 readonly executable: RelayClient.AvailableRelayClient;
96 readonly scope: Scope.Closeable;
97 readonly configKey: string;
98 readonly config: RelayManagedEndpointRuntimeConfig;
99 readonly startedAtMillis: number;
100 readonly registered: Deferred.Deferred<void>;
101 readonly metricsAddress: Deferred.Deferred<string>;
102 readonly protocol: Ref.Ref<string | null>;
103 }
104
105 // An instantly failing relay client otherwise respawns ~100 times/sec and exhausts V8 via tracing.
106 const isRotatingFileSinkError = Schema.is(RotatingFileSinkError);
107
108 const RELAY_RESTART_STABLE_UPTIME_MS = 30_000;
109 const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;
110 const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;
111 // Newly created tunnels can fail authorization briefly while Cloudflare propagates their token.
112 const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;
113 // Unknown rejection wording can leave a dead tunnel unconnected; periodically request recovery.
114 const CONNECTOR_REGISTRATION_TIMEOUT = Duration.minutes(3);
115 // cloudflared resolves edges only at startup and can keep dialing stale addresses after a network change.
116 const CONNECTOR_READINESS_INTERVAL = Duration.seconds(30);
117 const CONNECTOR_READINESS_REQUEST_TIMEOUT = Duration.seconds(5);
118 const CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART = 6;
119 const CONNECTOR_NOT_READY_CHECKS_MAX = 60;
120 const RELAY_CLIENT_METRICS_LINE = /\bStarting metrics server on (127\.0\.0\.1:\d+)\/metrics\b/u;
121 const RELAY_CLIENT_PROTOCOL = /\bprotocol=(\w+)/u;
122 // cloudflared's metrics server: `/ready` answers 200 or 503 with this body, and
123 // `/config` holds the ingress rules the edge pushed after the first connection.
124 const ConnectorReadyBody = Schema.Struct({ readyConnections: Schema.Number });
125 const ConnectorConfigBody = Schema.Struct({
126 config: Schema.Struct({
127 ingress: Schema.optional(
128 Schema.NullOr(
129 Schema.Array(
130 Schema.Struct({
131 hostname: Schema.optional(Schema.String),
132 service: Schema.optional(Schema.String),
133 }),
134 ),
135 ),
136 ),
137 }),
138 });
139 // Registration resets cloudflared's own fallback, even when QUIC stalls again
140 // immediately. Keep that evidence across registrations and connector restarts.
141 const QUIC_STALLS_BEFORE_HTTP2 = 30;
142 const QUIC_STALL_WINDOW_MS = 5 * 60_000;
143 // Retry QUIC on a later startup after transient failures or network changes; reuse never renews this hint.
144 const HTTP2_FALLBACK_TTL_MS = 24 * 60 * 60_000;
145 const HTTP2_FALLBACK_STATE = "cloud-endpoint-http2-fallback";
146 const Http2Fallback = Schema.fromJsonString(Schema.Struct({ fellBackAt: Schema.Number }));
147 const decodeHttp2Fallback = Schema.decodeUnknownOption(Http2Fallback);
148 const encodeHttp2Fallback = Schema.encodeSync(Http2Fallback);
149 // A failed background install of the pinned relay client is retried no sooner
150 // than this, so a crash-looping connector or an offline host does not redownload
151 // on every reconcile.
152 const RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS = 10 * 60_000;
153 export const RELAY_CLIENT_LOG_FILE = "relay-client.ndjson";
154
155 /** Preserve user-chosen PATH binaries and overrides; CLI downloads require consent. */
156 function needsPinnedRelayClient(executable: RelayClient.RelayClientStatus): boolean {
157 return (
158 executable.status === "missing" ||
159 (executable.status === "available" &&
160 executable.source === "managed" &&
161 !RelayClient.isPinnedManagedRelayClient(executable))
162 );
163 }
164
165 export function classifyRelayClientOutput(
166 line: string,
167 ): "connected" | "unregistered" | "warning" | "debug" {
168 if (/\bRegistered tunnel connection\b/iu.test(line)) {
169 return "connected";
170 }
171 // cloudflared logs a browser closing its request mid-stream as ERR; the tunnel itself is fine.
172 if (
173 /\bERR\s+(?:Request failed\s+)?error="Incoming request ended abruptly: context canceled"(?:\s|$)/u.test(
174 line,
175 )
176 ) {
177 return "debug";
178 }
179 // cloudflared uses zerolog FTL/PNC for fatal/panic; classify them above debug too.
180 if (/\b(?:ERR|WRN|FTL|PNC)\b/u.test(line)) {
181 return "warning";
182 }
183 return /\bINF\b.*\bUnregistered tunnel connection\b/u.test(line) ? "unregistered" : "debug";
184 }
185
186 /** Edge rejection wording varies; any Unauthorized registration error counts. */
187 export function isRejectedRelayClientTunnelOutput(line: string): boolean {
188 return (
189 /\bRegister tunnel error from server side\b/iu.test(line) &&
190 /error="(?:Unauthorized:[^"]*|Failed to get tunnel|Tunnel not found|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
191 line,
192 )
193 );
194 }
195
196 export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean {
197 if (typeof status !== "object" || status === null || !("status" in status)) {
198 return false;
199 }
200 if (status.status !== "failed" || !("failure" in status)) {
201 return false;
202 }
203 return status.failure === "not-installed" || status.failure === "spawn-failed";
204 }
205
206 function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string {
207 return JSON.stringify({
208 providerKind: config.providerKind,
209 connectorToken: config.connectorToken,
210 tunnelId: config.tunnelId ?? null,
211 tunnelName: config.tunnelName ?? null,
212 });
213 }
214
215 const stopConnector = (connector: ActiveConnector | null) =>
216 connector
217 ? Scope.close(connector.scope, Exit.void).pipe(
218 Effect.tap(() =>
219 Effect.logInfo("Relay client stopped", {
220 pid: Number(connector.child.pid),
221 }),
222 ),
223 Effect.ignore,
224 )
225 : Effect.void;
226
227 /** @public */
228 export const make = Effect.gen(function* () {
229 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
230 const relayClient = yield* RelayClient.RelayClient;
231 const relayClientLedger = yield* RelayClientLedger.RelayClientLedger;
232 const secrets = yield* ServerSecretStore.ServerSecretStore;
233 const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient);
234 // Without T3 Connect no connector ever runs, whoever asks for one.
235 const connectAllowed = ManagedLimits.allowsConnect(yield* ManagedPolicy.ManagedPolicy);
236 const activeRef = yield* Ref.make<ActiveConnector | null>(null);
237 const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig | null>(null);
238 const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1);
239 const tunnelConnections = yield* Queue.sliding<void>(1);
240 const readinessChanges = yield* Queue.sliding<void>(1);
241 const reconcileSemaphore = yield* Semaphore.make(1);
242 const restartDelayRef = yield* Ref.make(0);
243 const linkStateSemaphore = yield* Semaphore.make(1);
244 const runtimeScope = yield* Effect.scope;
245 const installInFlightRef = yield* Ref.make(false);
246 const lastInstallFailureAtRef = yield* Ref.make<number | null>(null);
247 const prunedRef = yield* Ref.make(false);
248 const retryLoopRunningRef = yield* Ref.make(false);
249 const notReadyChecksBeforeRestartRef = yield* Ref.make(CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
250 // The relay client shares the server's process group, so it exits on these signals before the finalizer runs.
251 let shutdownSignalReceived = false;
252 const onShutdownSignal = () => {
253 shutdownSignalReceived = true;
254 };
255 yield* Effect.acquireRelease(
256 Effect.sync(() => {
257 process.on("SIGINT", onShutdownSignal);
258 process.on("SIGTERM", onShutdownSignal);
259 }),
260 () =>
261 Effect.sync(() => {
262 process.off("SIGINT", onShutdownSignal);
263 process.off("SIGTERM", onShutdownSignal);
264 }),
265 );
266 // An explicit cloudflared transport in the environment is the user's choice.
267 const env = yield* HostProcess.Environment;
268 const transportChosenByUser = Option.isSome(
269 yield* Config.String("TUNNEL_TRANSPORT_PROTOCOL").pipe(Config.option),
270 );
271 const previousFallback = transportChosenByUser
272 ? Option.none()
273 : yield* secrets.get(HTTP2_FALLBACK_STATE).pipe(
274 Effect.map(Option.flatMap((bytes) => decodeHttp2Fallback(new TextDecoder().decode(bytes)))),
275 Effect.orElseSucceed(() => Option.none()),
276 );
277 const now = yield* Clock.currentTimeMillis;
278 const http2Ref = yield* Ref.make(
279 Option.isSome(previousFallback) &&
280 now >= previousFallback.value.fellBackAt &&
281 now - previousFallback.value.fellBackAt < HTTP2_FALLBACK_TTL_MS,
282 );
283 const quicStallsRef = yield* Ref.make<ReadonlyArray<number>>([]);
284 let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];
285
286 const { logsDir } = yield* ServerConfig.ServerConfig;
287 const path = yield* Path.Path;
288 const relayClientLogPath = path.join(logsDir, RELAY_CLIENT_LOG_FILE);
289 // Synchronous appends avoid a flush timer for this low-volume log.
290 const fallbackLoggers = yield* Effect.service(Logger.CurrentLoggers);
291 let writeFailureReported = false;
292 const relayClientLog = yield* Effect.try({
293 try: () =>
294 new RotatingFileSink({
295 filePath: relayClientLogPath,
296 maxBytes: 1024 * 1024,
297 maxFiles: 2,
298 throwOnError: true,
299 }),
300 catch: (cause) =>
301 isRotatingFileSinkError(cause)
302 ? cause
303 : new RotatingFileSinkError({
304 operation: "initialize",
305 filePath: relayClientLogPath,
306 cause,
307 }),
308 }).pipe(
309 Effect.map((sink) =>
310 Logger.make((options) => {
311 try {
312 const entry = Logger.formatJson.log(options);
313 const bytes = Buffer.byteLength(entry) + 1;
314 sink.write(
315 bytes <= 1024 * 1024
316 ? `${entry}\n`
317 : `${JSON.stringify({
318 timestamp: options.date.toISOString(),
319 level: options.logLevel.toUpperCase(),
320 message: ["Relay client log entry omitted: exceeds 1 MiB", { bytes }],
321 })}\n`,
322 );
323 } catch {
324 if (!writeFailureReported) {
325 writeFailureReported = true;
326 for (const logger of fallbackLoggers) {
327 logger.log({
328 ...options,
329 logLevel: "Warn",
330 message: ["Relay client log unavailable", { filePath: relayClientLogPath }],
331 });
332 }
333 }
334 }
335 }),
336 ),
337 Effect.catch((error) =>
338 Effect.logWarning("Relay client log unavailable", {
339 filePath: relayClientLogPath,
340 cause: error,
341 }).pipe(Effect.as(undefined)),
342 ),
343 );
344 // Forked connector fibers inherit this logger after the applyConfig span ends.
345 const withRelayClientLog = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
346 relayClientLog ? Effect.withLogger(effect, relayClientLog) : effect;
347
348 const stopActive = Effect.gen(function* () {
349 const active = yield* Ref.getAndSet(activeRef, null);
350 yield* stopConnector(active);
351 });
352
353 // Lives until connector exit and starts replacements; avoid nested lifetime spans.
354 const superviseConnector = Effect.fnUntraced(
355 function* (connector: ActiveConnector) {
356 const result = yield* Effect.result(connector.child.exitCode);
357 const activeAtExit = yield* Ref.get(activeRef);
358 if (
359 activeAtExit?.child.pid !== connector.child.pid ||
360 activeAtExit.configKey !== connector.configKey
361 ) {
362 return;
363 }
364 const uptimeMillis = (yield* Clock.currentTimeMillis) - connector.startedAtMillis;
365 // Back off outside the semaphore so config changes remain responsive; recheck config afterwards.
366 const restartDelayMillis = yield* Ref.modify(restartDelayRef, (current) => {
367 if (uptimeMillis >= RELAY_RESTART_STABLE_UPTIME_MS) {
368 return [0, 0];
369 }
370 return [
371 current,
372 current === 0
373 ? RELAY_RESTART_BACKOFF_BASE_MS
374 : Math.min(current * 2, RELAY_RESTART_BACKOFF_MAX_MS),
375 ];
376 });
377 if (restartDelayMillis > 0) {
378 yield* Effect.logWarning("Relay client is crash-looping; delaying restart", {
379 pid: Number(connector.child.pid),
380 uptimeMillis,
381 restartDelayMillis,
382 tunnelId: connector.config.tunnelId,
383 tunnelName: connector.config.tunnelName,
384 });
385 yield* Effect.sleep(Duration.millis(restartDelayMillis));
386 }
387 yield* reconcileSemaphore.withPermits(1)(
388 Effect.gen(function* () {
389 const active = yield* Ref.get(activeRef);
390 if (
391 active?.child.pid !== connector.child.pid ||
392 active.configKey !== connector.configKey
393 ) {
394 return;
395 }
396 yield* Ref.set(activeRef, null);
397 yield* stopConnector(connector);
398
399 const desiredConfig = yield* Ref.get(desiredConfigRef);
400 if (
401 shutdownSignalReceived ||
402 !desiredConfig ||
403 desiredConfig.providerKind !== "cloudflare_tunnel" ||
404 runtimeConfigKey(desiredConfig) !== connector.configKey
405 ) {
406 return;
407 }
408
409 yield* Effect.logWarning("Relay client exited; restarting", {
410 pid: Number(connector.child.pid),
411 ...(Result.isSuccess(result)
412 ? { exitCode: Number(result.success) }
413 : { cause: result.failure }),
414 tunnelId: connector.config.tunnelId,
415 tunnelName: connector.config.tunnelName,
416 });
417 yield* Queue.offer(recoveryRequests, connector.config);
418 yield* reconcileConfig(desiredConfig);
419 }),
420 );
421 },
422 Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),
423 );
424
425 // Keep the old connector until its replacement registers, preserving the relay connection.
426 const installPinnedRelayClient = Effect.gen(function* () {
427 if (yield* Ref.getAndSet(installInFlightRef, true)) return;
428 yield* Effect.logInfo("Installing the pinned relay client", {
429 version: RelayClient.CLOUDFLARED_VERSION,
430 });
431 yield* relayClient.install.pipe(
432 Effect.tap(() => Ref.set(lastInstallFailureAtRef, null)),
433 Effect.flatMap((installed) =>
434 reconcileSemaphore.withPermits(1)(
435 Effect.gen(function* () {
436 const desiredConfig = yield* Ref.get(desiredConfigRef);
437 if (!desiredConfig || desiredConfig.providerKind !== "cloudflare_tunnel") return;
438 const active = yield* Ref.get(activeRef);
439 // Self-updated binaries share the pinned path; compare versions to detect replacements.
440 if (
441 active?.executable.executablePath === installed.executablePath &&
442 active.executable.version === installed.version
443 ) {
444 return;
445 }
446 // A scanner can block a fresh binary's probe; retry instead of restarting onto the older binary.
447 const resolved = yield* relayClient.resolve;
448 if (
449 resolved.status !== "available" ||
450 resolved.executablePath !== installed.executablePath ||
451 resolved.version !== installed.version
452 ) {
453 return;
454 }
455 yield* Effect.logInfo("Relay client installed; moving the connector to it", {
456 version: installed.version,
457 previousVersion: active?.executable.version,
458 });
459 const previous = yield* Ref.getAndSet(activeRef, null);
460 const status = yield* reconcileConfig(desiredConfig);
461 const next = yield* Ref.get(activeRef);
462 if (!next) {
463 yield* Effect.logWarning("Relay client did not start after the update", status);
464 // The supervisor skips exits while detached; request recovery explicitly.
465 const previousRunning = previous
466 ? yield* previous.child.isRunning.pipe(Effect.orElseSucceed(() => false))
467 : false;
468 if (previous && previousRunning) {
469 yield* Ref.set(activeRef, previous);
470 } else {
471 yield* stopConnector(previous);
472 yield* Queue.offer(recoveryRequests, desiredConfig);
473 }
474 return;
475 }
476 if (previous) {
477 yield* Deferred.await(next.registered).pipe(
478 Effect.ensuring(stopConnector(previous)),
479 Effect.forkIn(next.scope),
480 );
481 }
482 }),
483 ),
484 ),
485 Effect.catchTags({
486 RelayClientInstallError: (error) =>
487 Clock.currentTimeMillis.pipe(
488 Effect.flatMap((failedAt) => Ref.set(lastInstallFailureAtRef, failedAt)),
489 Effect.andThen(
490 Effect.logWarning("Could not install the pinned relay client", {
491 reason: error.reason,
492 message: error.message,
493 cause: error.cause,
494 }),
495 ),
496 ),
497 }),
498 Effect.ensuring(Ref.set(installInFlightRef, false)),
499 );
500 });
501
502 const ensurePinnedRelayClient = Effect.gen(function* () {
503 const now = yield* Clock.currentTimeMillis;
504 const lastFailureAt = yield* Ref.get(lastInstallFailureAtRef);
505 if (lastFailureAt !== null && now - lastFailureAt < RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS) {
506 return;
507 }
508 yield* installPinnedRelayClient;
509 }).pipe(Effect.forkIn(runtimeScope), Effect.asVoid);
510
511 // Retry installs skipped by applyConfig without the failure gate.
512 const retryPinnedRelayClientInstall = Effect.gen(function* () {
513 while (true) {
514 yield* Effect.sleep(Duration.millis(RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS));
515 const desiredConfig = yield* Ref.get(desiredConfigRef);
516 if (!desiredConfig || desiredConfig.providerKind !== "cloudflare_tunnel") return;
517 // An installed pin still needs activation after a failed probe or while an older connector runs.
518 const active = yield* Ref.get(activeRef);
519 if (active && !needsPinnedRelayClient(active.executable)) return;
520 yield* installPinnedRelayClient;
521 }
522 }).pipe(Effect.ensuring(Ref.set(retryLoopRunningRef, false)));
523
524 const startPinnedRelayClientRetries = Effect.gen(function* () {
525 if (yield* Ref.getAndSet(retryLoopRunningRef, true)) return;
526 yield* Effect.forkIn(retryPinnedRelayClientInstall, runtimeScope);
527 });
528
529 // Registration retries can last for the connector lifetime; keep this loop untraced.
530 const watchConnectorRegistration = Effect.fnUntraced(
531 function* (connector: ActiveConnector) {
532 yield* Effect.sleep(CONNECTOR_REGISTRATION_TIMEOUT);
533 if (yield* Deferred.isDone(connector.registered)) return true;
534 yield* Effect.logWarning(
535 "Relay client has not registered a tunnel connection; requesting recovery",
536 {
537 pid: Number(connector.child.pid),
538 tunnelId: connector.config.tunnelId,
539 tunnelName: connector.config.tunnelName,
540 },
541 );
542 yield* Queue.offer(recoveryRequests, connector.config);
543 return false;
544 },
545 Effect.repeat({ until: (connected) => connected }),
546 );
547
548 // Only an explicit 503 counts: a failed request says nothing about the
549 // Readiness polling runs for the connector lifetime; keep this loop untraced.
550 const watchConnectorReadiness = Effect.fnUntraced(function* (connector: ActiveConnector) {
551 const readyUrl = `http://${yield* Deferred.await(connector.metricsAddress)}/ready`;
552 let notReadyChecks = 0;
553 let lastReady: boolean | null = null;
554 while (true) {
555 yield* Effect.sleep(CONNECTOR_READINESS_INTERVAL);
556 const status = yield* httpClient.get(readyUrl).pipe(
557 Effect.map((response) => response.status),
558 Effect.scoped,
559 Effect.withTracerEnabled(false),
560 Effect.timeoutOption(CONNECTOR_READINESS_REQUEST_TIMEOUT),
561 Effect.orElseSucceed(() => Option.none<number>()),
562 );
563 if (Option.isNone(status)) continue;
564 if (status.value === 200 || status.value === 503) {
565 const ready = status.value === 200;
566 if (ready !== lastReady) yield* Queue.offer(readinessChanges, undefined);
567 lastReady = ready;
568 }
569 if (status.value === 200) {
570 notReadyChecks = 0;
571 yield* Ref.set(notReadyChecksBeforeRestartRef, CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
572 continue;
573 }
574 if (status.value !== 503) continue;
575 notReadyChecks += 1;
576 const threshold = yield* Ref.get(notReadyChecksBeforeRestartRef);
577 if (notReadyChecks < threshold) continue;
578 notReadyChecks = 0;
579 yield* Ref.set(
580 notReadyChecksBeforeRestartRef,
581 Math.min(threshold * 2, CONNECTOR_NOT_READY_CHECKS_MAX),
582 );
583 yield* Effect.logWarning("Relay client has no tunnel connection; restarting it", {
584 pid: Number(connector.child.pid),
585 notReadyChecks: threshold,
586 tunnelId: connector.config.tunnelId,
587 tunnelName: connector.config.tunnelName,
588 });
589 yield* connector.child.kill().pipe(Effect.ignore);
590 }
591 });
592
593 const rememberHttp2Fallback = Effect.fnUntraced(function* () {
594 yield* Ref.set(http2Ref, true);
595 yield* secrets
596 .set(
597 HTTP2_FALLBACK_STATE,
598 new TextEncoder().encode(
599 encodeHttp2Fallback({ fellBackAt: yield* Clock.currentTimeMillis }),
600 ),
601 )
602 .pipe(
603 Effect.catch(() => Effect.logWarning("Could not persist the relay client HTTP/2 fallback")),
604 );
605 });
606
607 const switchToHttp2 = Effect.fnUntraced(function* (connector: ActiveConnector) {
608 const active = yield* Ref.get(activeRef);
609 const desiredConfig = yield* Ref.get(desiredConfigRef);
610 if ((yield* Ref.get(http2Ref)) || active !== connector || !desiredConfig) return;
611 yield* rememberHttp2Fallback();
612 yield* Effect.logWarning("Relay client keeps losing QUIC connections; switching to HTTP/2", {
613 pid: Number(connector.child.pid),
614 tunnelId: connector.config.tunnelId,
615 tunnelName: connector.config.tunnelName,
616 });
617 // Shutdown must not interrupt cleanup after active ownership is cleared.
618 yield* Effect.uninterruptible(stopActive);
619 const status = yield* reconcileConfig(desiredConfig);
620 // A failed spawn has no supervisor to request recovery.
621 if (status.status !== "running") {
622 yield* Queue.offer(recoveryRequests, desiredConfig);
623 }
624 }, reconcileSemaphore.withPermits(1));
625
626 const noteQuicStall = Effect.fnUntraced(function* (connector: ActiveConnector) {
627 if (transportChosenByUser || (yield* Ref.get(http2Ref))) return;
628 const now = yield* Clock.currentTimeMillis;
629 if ((yield* Ref.get(activeRef)) !== connector) return;
630 const stalls = yield* Ref.updateAndGet(quicStallsRef, (previous) => [
631 ...previous.filter((at) => now - at < QUIC_STALL_WINDOW_MS),
632 now,
633 ]);
634 if (stalls.length < QUIC_STALLS_BEFORE_HTTP2) return;
635 // Stopping the connector interrupts the output observer that called this.
636 yield* Effect.forkIn(switchToHttp2(connector), runtimeScope);
637 });
638
639 const observeConnectorOutput = (connector: ActiveConnector) => {
640 let rejectedRegistrations = 0;
641
642 return connector.child.all.pipe(
643 Stream.decodeText(),
644 Stream.splitLines,
645 Stream.map((line) => line.trim()),
646 Stream.filter((line) => line.length > 0),
647 Stream.runForEach((line) => {
648 const output = line.replaceAll(connector.config.connectorToken, "<redacted>");
649 const attributes = {
650 pid: Number(connector.child.pid),
651 tunnelId: connector.config.tunnelId,
652 tunnelName: connector.config.tunnelName,
653 output,
654 };
655 switch (classifyRelayClientOutput(line)) {
656 case "connected": {
657 rejectedRegistrations = 0;
658 const protocol = RELAY_CLIENT_PROTOCOL.exec(line)?.[1] ?? null;
659 return Deferred.succeed(connector.registered, undefined).pipe(
660 Effect.andThen(Ref.set(connector.protocol, protocol)),
661 // cloudflared started with `auto` fell back on its own; later starts skip QUIC too.
662 Effect.andThen(
663 protocol === "http2" && !transportChosenByUser
664 ? Ref.get(http2Ref).pipe(
665 Effect.flatMap((http2) => (http2 ? Effect.void : rememberHttp2Fallback())),
666 )
667 : Effect.void,
668 ),
669 Effect.andThen(
670 Effect.logInfo("Relay client tunnel connection registered", attributes),
671 ),
672 Effect.andThen(Queue.offer(tunnelConnections, undefined)),
673 Effect.andThen(Queue.offer(readinessChanges, undefined)),
674 Effect.andThen(
675 RelayClient.isPinnedManagedRelayClient(connector.executable)
676 ? Ref.getAndSet(prunedRef, true).pipe(
677 Effect.flatMap((pruned) =>
678 pruned ? Effect.void : relayClient.pruneManagedVersions,
679 ),
680 )
681 : Effect.void,
682 ),
683 Effect.asVoid,
684 );
685 }
686 case "unregistered":
687 return Effect.logInfo("Relay client tunnel connection unregistered", attributes);
688 case "warning":
689 if (/\btimeout: no recent network activity\b/u.test(line)) {
690 return noteQuicStall(connector).pipe(
691 Effect.andThen(
692 Effect.logWarning("Relay client reported a transport warning", attributes),
693 ),
694 );
695 }
696 if (isRejectedRelayClientTunnelOutput(line)) {
697 rejectedRegistrations += 1;
698 if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) {
699 rejectedRegistrations = 0;
700 return Effect.logWarning(
701 "Relay client tunnel was rejected; requesting recovery",
702 attributes,
703 ).pipe(
704 Effect.andThen(Queue.offer(recoveryRequests, connector.config)),
705 Effect.asVoid,
706 );
707 }
708 }
709 return Effect.logWarning("Relay client reported a transport warning", attributes);
710 case "debug": {
711 const metricsAddress = RELAY_CLIENT_METRICS_LINE.exec(line)?.[1];
712 return metricsAddress
713 ? Deferred.succeed(connector.metricsAddress, metricsAddress).pipe(
714 Effect.andThen(Effect.logDebug("Relay client output", attributes)),
715 )
716 : Effect.logDebug("Relay client output", attributes);
717 }
718 }
719 }),
720 Effect.catchCause((cause) =>
721 Effect.logWarning("Relay client output observer failed", {
722 cause,
723 pid: Number(connector.child.pid),
724 tunnelId: connector.config.tunnelId,
725 tunnelName: connector.config.tunnelName,
726 }),
727 ),
728 );
729 };
730
731 reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) {
732 if (!config || config.providerKind !== "cloudflare_tunnel") {
733 yield* stopActive;
734 return config
735 ? { status: "unsupported", providerKind: config.providerKind }
736 : { status: "disabled" };
737 }
738
739 const nextConfigKey = runtimeConfigKey(config);
740 const active = yield* Ref.get(activeRef);
741 if (active?.configKey === nextConfigKey) {
742 const isRunning = yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false));
743 if (isRunning) {
744 return {
745 status: "running",
746 providerKind: "cloudflare_tunnel",
747 pid: Number(active.child.pid),
748 ...(active.config.tunnelId ? { tunnelId: active.config.tunnelId } : {}),
749 ...(active.config.tunnelName ? { tunnelName: active.config.tunnelName } : {}),
750 } satisfies CloudManagedEndpointRuntimeStatus;
751 }
752 }
753
754 yield* stopActive;
755
756 const executable = yield* relayClient.resolve;
757 if (needsPinnedRelayClient(executable)) {
758 yield* ensurePinnedRelayClient;
759 yield* startPinnedRelayClientRetries;
760 }
761 if (executable.status !== "available") {
762 return {
763 status: "failed",
764 providerKind: "cloudflare_tunnel",
765 failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed",
766 reason:
767 executable.status === "unsupported"
768 ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.`
769 : "The relay client is not installed.",
770 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
771 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
772 } satisfies CloudManagedEndpointRuntimeStatus;
773 }
774
775 const http2 = yield* Ref.get(http2Ref);
776 const connectorScope = yield* Scope.make("sequential");
777 // Added before the spawn so it runs after the spawner's finalizer has stopped the client.
778 let forgetRelayClient: Effect.Effect<void> = Effect.void;
779 yield* Scope.addFinalizer(
780 connectorScope,
781 Effect.suspend(() => forgetRelayClient),
782 );
783 const child = yield* spawner
784 .spawn(
785 ChildProcess.make(
786 executable.executablePath,
787 [
788 "tunnel",
789 "--no-autoupdate",
790 "--loglevel",
791 "info",
792 "--output",
793 "default",
794 "--metrics",
795 "127.0.0.1:0",
796 ...(http2 ? ["--protocol", "http2"] : []),
797 "run",
798 ],
799 {
800 detached: false,
801 env: {
802 ...env,
803 TUNNEL_TOKEN: config.connectorToken,
804 },
805 shell: false,
806 stderr: "pipe",
807 stdout: "pipe",
808 },
809 ),
810 )
811 .pipe(
812 Effect.provideService(Scope.Scope, connectorScope),
813 Effect.tap((child) =>
814 Effect.logInfo("Relay client process started; waiting for tunnel connection", {
815 pid: Number(child.pid),
816 source: executable.source,
817 version: executable.version,
818 protocol: http2 ? "http2" : "auto",
819 tunnelId: config.tunnelId,
820 tunnelName: config.tunnelName,
821 }),
822 ),
823 Effect.catch((cause) =>
824 Effect.logWarning("Failed to start relay client", {
825 cause,
826 tunnelId: config.tunnelId,
827 tunnelName: config.tunnelName,
828 }).pipe(
829 Effect.andThen(Scope.close(connectorScope, Exit.void).pipe(Effect.ignore)),
830 Effect.as({
831 status: "failed",
832 providerKind: "cloudflare_tunnel",
833 failure: "spawn-failed",
834 reason: String(cause),
835 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
836 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
837 } satisfies CloudManagedEndpointRuntimeStatus),
838 ),
839 ),
840 );
841
842 if ("status" in child && child.status === "failed") {
843 return child;
844 }
845
846 if (!("status" in child)) {
847 // An interrupt before activeRef owns a supervised connector would leave its client open or unwatched.
848 yield* Effect.uninterruptibleMask((restore) =>
849 Effect.gen(function* () {
850 forgetRelayClient = yield* relayClientLedger.track(Number(child.pid));
851 const connector = {
852 child,
853 executable,
854 scope: connectorScope,
855 configKey: nextConfigKey,
856 config,
857 startedAtMillis: yield* Clock.currentTimeMillis,
858 registered: yield* Deferred.make<void>(),
859 metricsAddress: yield* Deferred.make<string>(),
860 protocol: yield* Ref.make<string | null>(null),
861 } satisfies ActiveConnector;
862 yield* Ref.set(activeRef, connector);
863 yield* Effect.forkIn(restore(observeConnectorOutput(connector)), connectorScope);
864 yield* Effect.forkIn(restore(superviseConnector(connector)), connectorScope);
865 yield* Effect.forkIn(restore(watchConnectorRegistration(connector)), connectorScope);
866 yield* Effect.forkIn(restore(watchConnectorReadiness(connector)), connectorScope);
867 }),
868 );
869 return {
870 status: "running",
871 providerKind: "cloudflare_tunnel",
872 pid: Number(child.pid),
873 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
874 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
875 } satisfies CloudManagedEndpointRuntimeStatus;
876 }
877
878 return {
879 status: "failed",
880 providerKind: "cloudflare_tunnel",
881 failure: "spawn-failed",
882 reason: "Relay client did not start.",
883 ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
884 ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
885 } satisfies CloudManagedEndpointRuntimeStatus;
886 });
887
888 const applyConfig = Effect.fn("CloudManagedEndpointRuntime.applyConfig")(
889 (requested: RelayManagedEndpointRuntimeConfig | null) =>
890 reconcileSemaphore.withPermits(1)(
891 Effect.gen(function* () {
892 const config = connectAllowed ? requested : null;
893 // Keep backoff when recovery returns the same tunnel/token, or every recovery respawns a crash loop.
894 const desired = yield* Ref.get(desiredConfigRef);
895 const unchanged =
896 desired !== null &&
897 config !== null &&
898 runtimeConfigKey(desired) === runtimeConfigKey(config);
899 if (!unchanged) {
900 yield* Ref.set(restartDelayRef, 0);
901 }
902 yield* Ref.set(desiredConfigRef, config);
903 return yield* reconcileConfig(config);
904 }),
905 ),
906 withRelayClientLog,
907 );
908
909 const restartConnector = Effect.fn("CloudManagedEndpointRuntime.restartConnector")(function* (
910 pid: number,
911 stillDead: Effect.Effect<boolean>,
912 ) {
913 const active = yield* Ref.get(activeRef);
914 if (!active || Number(active.child.pid) !== pid || !(yield* stillDead)) return false;
915 yield* Effect.logWarning(
916 "Cloudflare has no live connection from the relay client; restarting it",
917 { pid, tunnelId: active.config.tunnelId, tunnelName: active.config.tunnelName },
918 );
919 // The supervisor starts the replacement with its usual crash-loop backoff.
920 return yield* active.child.kill().pipe(
921 Effect.as(true),
922 Effect.orElseSucceed(() => false),
923 );
924 }, reconcileSemaphore.withPermits(1));
925
926 const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) =>
927 httpClient.get(url).pipe(
928 Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)),
929 Effect.scoped,
930 Effect.withTracerEnabled(false),
931 Effect.timeoutOption(CONNECTOR_READINESS_REQUEST_TIMEOUT),
932 Effect.map(Option.getOrNull),
933 Effect.orElseSucceed(() => null),
934 );
935
936 const inspectConnector = Effect.gen(function* () {
937 const active = yield* Ref.get(activeRef);
938 if (!active || !(yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)))) {
939 return null;
940 }
941 const pid = Number(active.child.pid);
942 const protocol = yield* Ref.get(active.protocol);
943 if (!(yield* Deferred.isDone(active.metricsAddress))) {
944 return {
945 pid,
946 readyConnections: null,
947 hostname: null,
948 origin: null,
949 protocol,
950 } satisfies ConnectorFacts;
951 }
952 const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`;
953 const [ready, config] = yield* Effect.all(
954 [
955 readMetrics(`${metricsUrl}/ready`, ConnectorReadyBody),
956 readMetrics(`${metricsUrl}/config`, ConnectorConfigBody),
957 ],
958 { concurrency: 2 },
959 );
960 const hostnameRule = config?.config.ingress?.find((rule) => rule.hostname);
961 return {
962 pid,
963 readyConnections: ready?.readyConnections ?? null,
964 hostname: hostnameRule?.hostname ?? null,
965 origin: hostnameRule?.service ?? null,
966 protocol,
967 } satisfies ConnectorFacts;
968 });
969
970 const runtime = CloudManagedEndpointRuntime.of({
971 applyConfig,
972 inspectConnector,
973 readinessChanged: Stream.fromQueue(readinessChanges),
974 recoveryRequests: Stream.fromQueue(recoveryRequests),
975 requestRecovery: Effect.fn("CloudManagedEndpointRuntime.requestRecovery")(function (config) {
976 return Queue.offer(recoveryRequests, config).pipe(Effect.asVoid);
977 }),
978 restartConnector,
979 tunnelConnected: Stream.fromQueue(tunnelConnections),
980 withLinkStateLock: linkStateSemaphore.withPermits(1),
981 });
982
983 yield* Effect.addFinalizer(() => runtime.applyConfig(null));
984 return runtime;
985 });
986
987 export const layer = Layer.effect(CloudManagedEndpointRuntime, make);