The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; |
| 2 | import { RotatingFileSink, RotatingFileSinkError } from "@t3tools/shared/logging"; |
| 3 | import * as HostProcess from "@t3tools/shared/HostProcess"; |
| 4 | import * as RelayClient from "@t3tools/shared/relayClient"; |
| 5 | import * as Clock from "effect/Clock"; |
| 6 | import * as Config from "effect/Config"; |
| 7 | import * as Context from "effect/Context"; |
| 8 | import * as Deferred from "effect/Deferred"; |
| 9 | import * as Duration from "effect/Duration"; |
| 10 | import * as Effect from "effect/Effect"; |
| 11 | import * as Exit from "effect/Exit"; |
| 12 | import * as Layer from "effect/Layer"; |
| 13 | import * as Logger from "effect/Logger"; |
| 14 | import * as Option from "effect/Option"; |
| 15 | import * as Path from "effect/Path"; |
| 16 | import * as Queue from "effect/Queue"; |
| 17 | import * as Ref from "effect/Ref"; |
| 18 | import * as Result from "effect/Result"; |
| 19 | import * as Schema from "effect/Schema"; |
| 20 | import * as Semaphore from "effect/Semaphore"; |
| 21 | import * as Scope from "effect/Scope"; |
| 22 | import * as Stream from "effect/Stream"; |
| 23 | import { HttpClient, HttpClientResponse } from "effect/http"; |
| 24 | import * as ChildProcess from "effect/process/ChildProcess"; |
| 25 | import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; |
| 26 | |
| 27 | import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; |
| 28 | import * as ServerConfig from "../config.ts"; |
| 29 | import * as ManagedLimits from "../policy/managedLimits.ts"; |
| 30 | import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; |
| 31 | import * as RelayClientLedger from "./RelayClientLedger.ts"; |
| 32 | |
| 33 | export type CloudManagedEndpointRuntimeStatus = |
| 34 | | { |
| 35 | readonly status: "disabled"; |
| 36 | } |
| 37 | | { |
| 38 | readonly status: "failed"; |
| 39 | readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"]; |
| 40 | readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed"; |
| 41 | readonly reason: string; |
| 42 | readonly tunnelId?: string; |
| 43 | readonly tunnelName?: string; |
| 44 | } |
| 45 | | { |
| 46 | readonly status: "running"; |
| 47 | readonly providerKind: "cloudflare_tunnel"; |
| 48 | readonly pid: number; |
| 49 | readonly tunnelId?: string; |
| 50 | readonly tunnelName?: string; |
| 51 | } |
| 52 | | { |
| 53 | readonly status: "unsupported"; |
| 54 | readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"]; |
| 55 | }; |
| 56 | |
| 57 | /** What the running cloudflared reports about itself. `null` fields are not known yet. */ |
| 58 | export interface ConnectorFacts { |
| 59 | readonly pid: number; |
| 60 | readonly readyConnections: number | null; |
| 61 | /** The public tunnel hostname from the remote config cloudflared receives once it connects. */ |
| 62 | readonly hostname: string | null; |
| 63 | /** The local origin that remote config routes the public hostname to. */ |
| 64 | readonly origin: string | null; |
| 65 | readonly protocol: string | null; |
| 66 | } |
| 67 | |
| 68 | export class CloudManagedEndpointRuntime extends Context.Service< |
| 69 | CloudManagedEndpointRuntime, |
| 70 | { |
| 71 | readonly applyConfig: ( |
| 72 | config: RelayManagedEndpointRuntimeConfig | null, |
| 73 | ) => Effect.Effect<CloudManagedEndpointRuntimeStatus>; |
| 74 | /** `null` when no connector is running. */ |
| 75 | readonly inspectConnector: Effect.Effect<ConnectorFacts | null>; |
| 76 | /** Emits when a connector registers a connection or its readiness changes. */ |
| 77 | readonly readinessChanged: Stream.Stream<void>; |
| 78 | readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>; |
| 79 | readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>; |
| 80 | /** |
| 81 | * Restarts the connector if it still runs as `pid` and `stillDead` holds once it has |
| 82 | * the lock. Returns whether it sent the restart. |
| 83 | */ |
| 84 | readonly restartConnector: ( |
| 85 | pid: number, |
| 86 | stillDead: Effect.Effect<boolean>, |
| 87 | ) => Effect.Effect<boolean>; |
| 88 | readonly tunnelConnected: Stream.Stream<void>; |
| 89 | readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>; |
| 90 | } |
| 91 | >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} |
| 92 | |
| 93 | interface ActiveConnector { |
| 94 | readonly child: ChildProcessSpawner.ChildProcessHandle; |
| 95 | readonly executable: RelayClient.AvailableRelayClient; |
| 96 | readonly scope: Scope.Closeable; |
| 97 | readonly configKey: string; |
| 98 | readonly config: RelayManagedEndpointRuntimeConfig; |
| 99 | readonly startedAtMillis: number; |
| 100 | readonly registered: Deferred.Deferred<void>; |
| 101 | readonly metricsAddress: Deferred.Deferred<string>; |
| 102 | readonly protocol: Ref.Ref<string | null>; |
| 103 | } |
| 104 | |
| 105 | // An instantly failing relay client otherwise respawns ~100 times/sec and exhausts V8 via tracing. |
| 106 | const isRotatingFileSinkError = Schema.is(RotatingFileSinkError); |
| 107 | |
| 108 | const RELAY_RESTART_STABLE_UPTIME_MS = 30_000; |
| 109 | const RELAY_RESTART_BACKOFF_BASE_MS = 1_000; |
| 110 | const RELAY_RESTART_BACKOFF_MAX_MS = 60_000; |
| 111 | // Newly created tunnels can fail authorization briefly while Cloudflare propagates their token. |
| 112 | const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4; |
| 113 | // Unknown rejection wording can leave a dead tunnel unconnected; periodically request recovery. |
| 114 | const CONNECTOR_REGISTRATION_TIMEOUT = Duration.minutes(3); |
| 115 | // cloudflared resolves edges only at startup and can keep dialing stale addresses after a network change. |
| 116 | const CONNECTOR_READINESS_INTERVAL = Duration.seconds(30); |
| 117 | const CONNECTOR_READINESS_REQUEST_TIMEOUT = Duration.seconds(5); |
| 118 | const CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART = 6; |
| 119 | const CONNECTOR_NOT_READY_CHECKS_MAX = 60; |
| 120 | const RELAY_CLIENT_METRICS_LINE = /\bStarting metrics server on (127\.0\.0\.1:\d+)\/metrics\b/u; |
| 121 | const RELAY_CLIENT_PROTOCOL = /\bprotocol=(\w+)/u; |
| 122 | // cloudflared's metrics server: `/ready` answers 200 or 503 with this body, and |
| 123 | // `/config` holds the ingress rules the edge pushed after the first connection. |
| 124 | const ConnectorReadyBody = Schema.Struct({ readyConnections: Schema.Number }); |
| 125 | const ConnectorConfigBody = Schema.Struct({ |
| 126 | config: Schema.Struct({ |
| 127 | ingress: Schema.optional( |
| 128 | Schema.NullOr( |
| 129 | Schema.Array( |
| 130 | Schema.Struct({ |
| 131 | hostname: Schema.optional(Schema.String), |
| 132 | service: Schema.optional(Schema.String), |
| 133 | }), |
| 134 | ), |
| 135 | ), |
| 136 | ), |
| 137 | }), |
| 138 | }); |
| 139 | // Registration resets cloudflared's own fallback, even when QUIC stalls again |
| 140 | // immediately. Keep that evidence across registrations and connector restarts. |
| 141 | const QUIC_STALLS_BEFORE_HTTP2 = 30; |
| 142 | const QUIC_STALL_WINDOW_MS = 5 * 60_000; |
| 143 | // Retry QUIC on a later startup after transient failures or network changes; reuse never renews this hint. |
| 144 | const HTTP2_FALLBACK_TTL_MS = 24 * 60 * 60_000; |
| 145 | const HTTP2_FALLBACK_STATE = "cloud-endpoint-http2-fallback"; |
| 146 | const Http2Fallback = Schema.fromJsonString(Schema.Struct({ fellBackAt: Schema.Number })); |
| 147 | const decodeHttp2Fallback = Schema.decodeUnknownOption(Http2Fallback); |
| 148 | const encodeHttp2Fallback = Schema.encodeSync(Http2Fallback); |
| 149 | // A failed background install of the pinned relay client is retried no sooner |
| 150 | // than this, so a crash-looping connector or an offline host does not redownload |
| 151 | // on every reconcile. |
| 152 | const RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS = 10 * 60_000; |
| 153 | export const RELAY_CLIENT_LOG_FILE = "relay-client.ndjson"; |
| 154 | |
| 155 | /** Preserve user-chosen PATH binaries and overrides; CLI downloads require consent. */ |
| 156 | function needsPinnedRelayClient(executable: RelayClient.RelayClientStatus): boolean { |
| 157 | return ( |
| 158 | executable.status === "missing" || |
| 159 | (executable.status === "available" && |
| 160 | executable.source === "managed" && |
| 161 | !RelayClient.isPinnedManagedRelayClient(executable)) |
| 162 | ); |
| 163 | } |
| 164 | |
| 165 | export function classifyRelayClientOutput( |
| 166 | line: string, |
| 167 | ): "connected" | "unregistered" | "warning" | "debug" { |
| 168 | if (/\bRegistered tunnel connection\b/iu.test(line)) { |
| 169 | return "connected"; |
| 170 | } |
| 171 | // cloudflared logs a browser closing its request mid-stream as ERR; the tunnel itself is fine. |
| 172 | if ( |
| 173 | /\bERR\s+(?:Request failed\s+)?error="Incoming request ended abruptly: context canceled"(?:\s|$)/u.test( |
| 174 | line, |
| 175 | ) |
| 176 | ) { |
| 177 | return "debug"; |
| 178 | } |
| 179 | // cloudflared uses zerolog FTL/PNC for fatal/panic; classify them above debug too. |
| 180 | if (/\b(?:ERR|WRN|FTL|PNC)\b/u.test(line)) { |
| 181 | return "warning"; |
| 182 | } |
| 183 | return /\bINF\b.*\bUnregistered tunnel connection\b/u.test(line) ? "unregistered" : "debug"; |
| 184 | } |
| 185 | |
| 186 | /** Edge rejection wording varies; any Unauthorized registration error counts. */ |
| 187 | export function isRejectedRelayClientTunnelOutput(line: string): boolean { |
| 188 | return ( |
| 189 | /\bRegister tunnel error from server side\b/iu.test(line) && |
| 190 | /error="(?:Unauthorized:[^"]*|Failed to get tunnel|Tunnel not found|Record for tunnel not found|Invalid tunnel secret)"/iu.test( |
| 191 | line, |
| 192 | ) |
| 193 | ); |
| 194 | } |
| 195 | |
| 196 | export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean { |
| 197 | if (typeof status !== "object" || status === null || !("status" in status)) { |
| 198 | return false; |
| 199 | } |
| 200 | if (status.status !== "failed" || !("failure" in status)) { |
| 201 | return false; |
| 202 | } |
| 203 | return status.failure === "not-installed" || status.failure === "spawn-failed"; |
| 204 | } |
| 205 | |
| 206 | function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string { |
| 207 | return JSON.stringify({ |
| 208 | providerKind: config.providerKind, |
| 209 | connectorToken: config.connectorToken, |
| 210 | tunnelId: config.tunnelId ?? null, |
| 211 | tunnelName: config.tunnelName ?? null, |
| 212 | }); |
| 213 | } |
| 214 | |
| 215 | const stopConnector = (connector: ActiveConnector | null) => |
| 216 | connector |
| 217 | ? Scope.close(connector.scope, Exit.void).pipe( |
| 218 | Effect.tap(() => |
| 219 | Effect.logInfo("Relay client stopped", { |
| 220 | pid: Number(connector.child.pid), |
| 221 | }), |
| 222 | ), |
| 223 | Effect.ignore, |
| 224 | ) |
| 225 | : Effect.void; |
| 226 | |
| 227 | /** @public */ |
| 228 | export const make = Effect.gen(function* () { |
| 229 | const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; |
| 230 | const relayClient = yield* RelayClient.RelayClient; |
| 231 | const relayClientLedger = yield* RelayClientLedger.RelayClientLedger; |
| 232 | const secrets = yield* ServerSecretStore.ServerSecretStore; |
| 233 | const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient); |
| 234 | // Without T3 Connect no connector ever runs, whoever asks for one. |
| 235 | const connectAllowed = ManagedLimits.allowsConnect(yield* ManagedPolicy.ManagedPolicy); |
| 236 | const activeRef = yield* Ref.make<ActiveConnector | null>(null); |
| 237 | const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig | null>(null); |
| 238 | const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1); |
| 239 | const tunnelConnections = yield* Queue.sliding<void>(1); |
| 240 | const readinessChanges = yield* Queue.sliding<void>(1); |
| 241 | const reconcileSemaphore = yield* Semaphore.make(1); |
| 242 | const restartDelayRef = yield* Ref.make(0); |
| 243 | const linkStateSemaphore = yield* Semaphore.make(1); |
| 244 | const runtimeScope = yield* Effect.scope; |
| 245 | const installInFlightRef = yield* Ref.make(false); |
| 246 | const lastInstallFailureAtRef = yield* Ref.make<number | null>(null); |
| 247 | const prunedRef = yield* Ref.make(false); |
| 248 | const retryLoopRunningRef = yield* Ref.make(false); |
| 249 | const notReadyChecksBeforeRestartRef = yield* Ref.make(CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART); |
| 250 | // The relay client shares the server's process group, so it exits on these signals before the finalizer runs. |
| 251 | let shutdownSignalReceived = false; |
| 252 | const onShutdownSignal = () => { |
| 253 | shutdownSignalReceived = true; |
| 254 | }; |
| 255 | yield* Effect.acquireRelease( |
| 256 | Effect.sync(() => { |
| 257 | process.on("SIGINT", onShutdownSignal); |
| 258 | process.on("SIGTERM", onShutdownSignal); |
| 259 | }), |
| 260 | () => |
| 261 | Effect.sync(() => { |
| 262 | process.off("SIGINT", onShutdownSignal); |
| 263 | process.off("SIGTERM", onShutdownSignal); |
| 264 | }), |
| 265 | ); |
| 266 | // An explicit cloudflared transport in the environment is the user's choice. |
| 267 | const env = yield* HostProcess.Environment; |
| 268 | const transportChosenByUser = Option.isSome( |
| 269 | yield* Config.String("TUNNEL_TRANSPORT_PROTOCOL").pipe(Config.option), |
| 270 | ); |
| 271 | const previousFallback = transportChosenByUser |
| 272 | ? Option.none() |
| 273 | : yield* secrets.get(HTTP2_FALLBACK_STATE).pipe( |
| 274 | Effect.map(Option.flatMap((bytes) => decodeHttp2Fallback(new TextDecoder().decode(bytes)))), |
| 275 | Effect.orElseSucceed(() => Option.none()), |
| 276 | ); |
| 277 | const now = yield* Clock.currentTimeMillis; |
| 278 | const http2Ref = yield* Ref.make( |
| 279 | Option.isSome(previousFallback) && |
| 280 | now >= previousFallback.value.fellBackAt && |
| 281 | now - previousFallback.value.fellBackAt < HTTP2_FALLBACK_TTL_MS, |
| 282 | ); |
| 283 | const quicStallsRef = yield* Ref.make<ReadonlyArray<number>>([]); |
| 284 | let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; |
| 285 | |
| 286 | const { logsDir } = yield* ServerConfig.ServerConfig; |
| 287 | const path = yield* Path.Path; |
| 288 | const relayClientLogPath = path.join(logsDir, RELAY_CLIENT_LOG_FILE); |
| 289 | // Synchronous appends avoid a flush timer for this low-volume log. |
| 290 | const fallbackLoggers = yield* Effect.service(Logger.CurrentLoggers); |
| 291 | let writeFailureReported = false; |
| 292 | const relayClientLog = yield* Effect.try({ |
| 293 | try: () => |
| 294 | new RotatingFileSink({ |
| 295 | filePath: relayClientLogPath, |
| 296 | maxBytes: 1024 * 1024, |
| 297 | maxFiles: 2, |
| 298 | throwOnError: true, |
| 299 | }), |
| 300 | catch: (cause) => |
| 301 | isRotatingFileSinkError(cause) |
| 302 | ? cause |
| 303 | : new RotatingFileSinkError({ |
| 304 | operation: "initialize", |
| 305 | filePath: relayClientLogPath, |
| 306 | cause, |
| 307 | }), |
| 308 | }).pipe( |
| 309 | Effect.map((sink) => |
| 310 | Logger.make((options) => { |
| 311 | try { |
| 312 | const entry = Logger.formatJson.log(options); |
| 313 | const bytes = Buffer.byteLength(entry) + 1; |
| 314 | sink.write( |
| 315 | bytes <= 1024 * 1024 |
| 316 | ? `${entry}\n` |
| 317 | : `${JSON.stringify({ |
| 318 | timestamp: options.date.toISOString(), |
| 319 | level: options.logLevel.toUpperCase(), |
| 320 | message: ["Relay client log entry omitted: exceeds 1 MiB", { bytes }], |
| 321 | })}\n`, |
| 322 | ); |
| 323 | } catch { |
| 324 | if (!writeFailureReported) { |
| 325 | writeFailureReported = true; |
| 326 | for (const logger of fallbackLoggers) { |
| 327 | logger.log({ |
| 328 | ...options, |
| 329 | logLevel: "Warn", |
| 330 | message: ["Relay client log unavailable", { filePath: relayClientLogPath }], |
| 331 | }); |
| 332 | } |
| 333 | } |
| 334 | } |
| 335 | }), |
| 336 | ), |
| 337 | Effect.catch((error) => |
| 338 | Effect.logWarning("Relay client log unavailable", { |
| 339 | filePath: relayClientLogPath, |
| 340 | cause: error, |
| 341 | }).pipe(Effect.as(undefined)), |
| 342 | ), |
| 343 | ); |
| 344 | // Forked connector fibers inherit this logger after the applyConfig span ends. |
| 345 | const withRelayClientLog = <A, E, R>(effect: Effect.Effect<A, E, R>) => |
| 346 | relayClientLog ? Effect.withLogger(effect, relayClientLog) : effect; |
| 347 | |
| 348 | const stopActive = Effect.gen(function* () { |
| 349 | const active = yield* Ref.getAndSet(activeRef, null); |
| 350 | yield* stopConnector(active); |
| 351 | }); |
| 352 | |
| 353 | // Lives until connector exit and starts replacements; avoid nested lifetime spans. |
| 354 | const superviseConnector = Effect.fnUntraced( |
| 355 | function* (connector: ActiveConnector) { |
| 356 | const result = yield* Effect.result(connector.child.exitCode); |
| 357 | const activeAtExit = yield* Ref.get(activeRef); |
| 358 | if ( |
| 359 | activeAtExit?.child.pid !== connector.child.pid || |
| 360 | activeAtExit.configKey !== connector.configKey |
| 361 | ) { |
| 362 | return; |
| 363 | } |
| 364 | const uptimeMillis = (yield* Clock.currentTimeMillis) - connector.startedAtMillis; |
| 365 | // Back off outside the semaphore so config changes remain responsive; recheck config afterwards. |
| 366 | const restartDelayMillis = yield* Ref.modify(restartDelayRef, (current) => { |
| 367 | if (uptimeMillis >= RELAY_RESTART_STABLE_UPTIME_MS) { |
| 368 | return [0, 0]; |
| 369 | } |
| 370 | return [ |
| 371 | current, |
| 372 | current === 0 |
| 373 | ? RELAY_RESTART_BACKOFF_BASE_MS |
| 374 | : Math.min(current * 2, RELAY_RESTART_BACKOFF_MAX_MS), |
| 375 | ]; |
| 376 | }); |
| 377 | if (restartDelayMillis > 0) { |
| 378 | yield* Effect.logWarning("Relay client is crash-looping; delaying restart", { |
| 379 | pid: Number(connector.child.pid), |
| 380 | uptimeMillis, |
| 381 | restartDelayMillis, |
| 382 | tunnelId: connector.config.tunnelId, |
| 383 | tunnelName: connector.config.tunnelName, |
| 384 | }); |
| 385 | yield* Effect.sleep(Duration.millis(restartDelayMillis)); |
| 386 | } |
| 387 | yield* reconcileSemaphore.withPermits(1)( |
| 388 | Effect.gen(function* () { |
| 389 | const active = yield* Ref.get(activeRef); |
| 390 | if ( |
| 391 | active?.child.pid !== connector.child.pid || |
| 392 | active.configKey !== connector.configKey |
| 393 | ) { |
| 394 | return; |
| 395 | } |
| 396 | yield* Ref.set(activeRef, null); |
| 397 | yield* stopConnector(connector); |
| 398 | |
| 399 | const desiredConfig = yield* Ref.get(desiredConfigRef); |
| 400 | if ( |
| 401 | shutdownSignalReceived || |
| 402 | !desiredConfig || |
| 403 | desiredConfig.providerKind !== "cloudflare_tunnel" || |
| 404 | runtimeConfigKey(desiredConfig) !== connector.configKey |
| 405 | ) { |
| 406 | return; |
| 407 | } |
| 408 | |
| 409 | yield* Effect.logWarning("Relay client exited; restarting", { |
| 410 | pid: Number(connector.child.pid), |
| 411 | ...(Result.isSuccess(result) |
| 412 | ? { exitCode: Number(result.success) } |
| 413 | : { cause: result.failure }), |
| 414 | tunnelId: connector.config.tunnelId, |
| 415 | tunnelName: connector.config.tunnelName, |
| 416 | }); |
| 417 | yield* Queue.offer(recoveryRequests, connector.config); |
| 418 | yield* reconcileConfig(desiredConfig); |
| 419 | }), |
| 420 | ); |
| 421 | }, |
| 422 | Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })), |
| 423 | ); |
| 424 | |
| 425 | // Keep the old connector until its replacement registers, preserving the relay connection. |
| 426 | const installPinnedRelayClient = Effect.gen(function* () { |
| 427 | if (yield* Ref.getAndSet(installInFlightRef, true)) return; |
| 428 | yield* Effect.logInfo("Installing the pinned relay client", { |
| 429 | version: RelayClient.CLOUDFLARED_VERSION, |
| 430 | }); |
| 431 | yield* relayClient.install.pipe( |
| 432 | Effect.tap(() => Ref.set(lastInstallFailureAtRef, null)), |
| 433 | Effect.flatMap((installed) => |
| 434 | reconcileSemaphore.withPermits(1)( |
| 435 | Effect.gen(function* () { |
| 436 | const desiredConfig = yield* Ref.get(desiredConfigRef); |
| 437 | if (!desiredConfig || desiredConfig.providerKind !== "cloudflare_tunnel") return; |
| 438 | const active = yield* Ref.get(activeRef); |
| 439 | // Self-updated binaries share the pinned path; compare versions to detect replacements. |
| 440 | if ( |
| 441 | active?.executable.executablePath === installed.executablePath && |
| 442 | active.executable.version === installed.version |
| 443 | ) { |
| 444 | return; |
| 445 | } |
| 446 | // A scanner can block a fresh binary's probe; retry instead of restarting onto the older binary. |
| 447 | const resolved = yield* relayClient.resolve; |
| 448 | if ( |
| 449 | resolved.status !== "available" || |
| 450 | resolved.executablePath !== installed.executablePath || |
| 451 | resolved.version !== installed.version |
| 452 | ) { |
| 453 | return; |
| 454 | } |
| 455 | yield* Effect.logInfo("Relay client installed; moving the connector to it", { |
| 456 | version: installed.version, |
| 457 | previousVersion: active?.executable.version, |
| 458 | }); |
| 459 | const previous = yield* Ref.getAndSet(activeRef, null); |
| 460 | const status = yield* reconcileConfig(desiredConfig); |
| 461 | const next = yield* Ref.get(activeRef); |
| 462 | if (!next) { |
| 463 | yield* Effect.logWarning("Relay client did not start after the update", status); |
| 464 | // The supervisor skips exits while detached; request recovery explicitly. |
| 465 | const previousRunning = previous |
| 466 | ? yield* previous.child.isRunning.pipe(Effect.orElseSucceed(() => false)) |
| 467 | : false; |
| 468 | if (previous && previousRunning) { |
| 469 | yield* Ref.set(activeRef, previous); |
| 470 | } else { |
| 471 | yield* stopConnector(previous); |
| 472 | yield* Queue.offer(recoveryRequests, desiredConfig); |
| 473 | } |
| 474 | return; |
| 475 | } |
| 476 | if (previous) { |
| 477 | yield* Deferred.await(next.registered).pipe( |
| 478 | Effect.ensuring(stopConnector(previous)), |
| 479 | Effect.forkIn(next.scope), |
| 480 | ); |
| 481 | } |
| 482 | }), |
| 483 | ), |
| 484 | ), |
| 485 | Effect.catchTags({ |
| 486 | RelayClientInstallError: (error) => |
| 487 | Clock.currentTimeMillis.pipe( |
| 488 | Effect.flatMap((failedAt) => Ref.set(lastInstallFailureAtRef, failedAt)), |
| 489 | Effect.andThen( |
| 490 | Effect.logWarning("Could not install the pinned relay client", { |
| 491 | reason: error.reason, |
| 492 | message: error.message, |
| 493 | cause: error.cause, |
| 494 | }), |
| 495 | ), |
| 496 | ), |
| 497 | }), |
| 498 | Effect.ensuring(Ref.set(installInFlightRef, false)), |
| 499 | ); |
| 500 | }); |
| 501 | |
| 502 | const ensurePinnedRelayClient = Effect.gen(function* () { |
| 503 | const now = yield* Clock.currentTimeMillis; |
| 504 | const lastFailureAt = yield* Ref.get(lastInstallFailureAtRef); |
| 505 | if (lastFailureAt !== null && now - lastFailureAt < RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS) { |
| 506 | return; |
| 507 | } |
| 508 | yield* installPinnedRelayClient; |
| 509 | }).pipe(Effect.forkIn(runtimeScope), Effect.asVoid); |
| 510 | |
| 511 | // Retry installs skipped by applyConfig without the failure gate. |
| 512 | const retryPinnedRelayClientInstall = Effect.gen(function* () { |
| 513 | while (true) { |
| 514 | yield* Effect.sleep(Duration.millis(RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS)); |
| 515 | const desiredConfig = yield* Ref.get(desiredConfigRef); |
| 516 | if (!desiredConfig || desiredConfig.providerKind !== "cloudflare_tunnel") return; |
| 517 | // An installed pin still needs activation after a failed probe or while an older connector runs. |
| 518 | const active = yield* Ref.get(activeRef); |
| 519 | if (active && !needsPinnedRelayClient(active.executable)) return; |
| 520 | yield* installPinnedRelayClient; |
| 521 | } |
| 522 | }).pipe(Effect.ensuring(Ref.set(retryLoopRunningRef, false))); |
| 523 | |
| 524 | const startPinnedRelayClientRetries = Effect.gen(function* () { |
| 525 | if (yield* Ref.getAndSet(retryLoopRunningRef, true)) return; |
| 526 | yield* Effect.forkIn(retryPinnedRelayClientInstall, runtimeScope); |
| 527 | }); |
| 528 | |
| 529 | // Registration retries can last for the connector lifetime; keep this loop untraced. |
| 530 | const watchConnectorRegistration = Effect.fnUntraced( |
| 531 | function* (connector: ActiveConnector) { |
| 532 | yield* Effect.sleep(CONNECTOR_REGISTRATION_TIMEOUT); |
| 533 | if (yield* Deferred.isDone(connector.registered)) return true; |
| 534 | yield* Effect.logWarning( |
| 535 | "Relay client has not registered a tunnel connection; requesting recovery", |
| 536 | { |
| 537 | pid: Number(connector.child.pid), |
| 538 | tunnelId: connector.config.tunnelId, |
| 539 | tunnelName: connector.config.tunnelName, |
| 540 | }, |
| 541 | ); |
| 542 | yield* Queue.offer(recoveryRequests, connector.config); |
| 543 | return false; |
| 544 | }, |
| 545 | Effect.repeat({ until: (connected) => connected }), |
| 546 | ); |
| 547 | |
| 548 | // Only an explicit 503 counts: a failed request says nothing about the |
| 549 | // Readiness polling runs for the connector lifetime; keep this loop untraced. |
| 550 | const watchConnectorReadiness = Effect.fnUntraced(function* (connector: ActiveConnector) { |
| 551 | const readyUrl = `http://${yield* Deferred.await(connector.metricsAddress)}/ready`; |
| 552 | let notReadyChecks = 0; |
| 553 | let lastReady: boolean | null = null; |
| 554 | while (true) { |
| 555 | yield* Effect.sleep(CONNECTOR_READINESS_INTERVAL); |
| 556 | const status = yield* httpClient.get(readyUrl).pipe( |
| 557 | Effect.map((response) => response.status), |
| 558 | Effect.scoped, |
| 559 | Effect.withTracerEnabled(false), |
| 560 | Effect.timeoutOption(CONNECTOR_READINESS_REQUEST_TIMEOUT), |
| 561 | Effect.orElseSucceed(() => Option.none<number>()), |
| 562 | ); |
| 563 | if (Option.isNone(status)) continue; |
| 564 | if (status.value === 200 || status.value === 503) { |
| 565 | const ready = status.value === 200; |
| 566 | if (ready !== lastReady) yield* Queue.offer(readinessChanges, undefined); |
| 567 | lastReady = ready; |
| 568 | } |
| 569 | if (status.value === 200) { |
| 570 | notReadyChecks = 0; |
| 571 | yield* Ref.set(notReadyChecksBeforeRestartRef, CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART); |
| 572 | continue; |
| 573 | } |
| 574 | if (status.value !== 503) continue; |
| 575 | notReadyChecks += 1; |
| 576 | const threshold = yield* Ref.get(notReadyChecksBeforeRestartRef); |
| 577 | if (notReadyChecks < threshold) continue; |
| 578 | notReadyChecks = 0; |
| 579 | yield* Ref.set( |
| 580 | notReadyChecksBeforeRestartRef, |
| 581 | Math.min(threshold * 2, CONNECTOR_NOT_READY_CHECKS_MAX), |
| 582 | ); |
| 583 | yield* Effect.logWarning("Relay client has no tunnel connection; restarting it", { |
| 584 | pid: Number(connector.child.pid), |
| 585 | notReadyChecks: threshold, |
| 586 | tunnelId: connector.config.tunnelId, |
| 587 | tunnelName: connector.config.tunnelName, |
| 588 | }); |
| 589 | yield* connector.child.kill().pipe(Effect.ignore); |
| 590 | } |
| 591 | }); |
| 592 | |
| 593 | const rememberHttp2Fallback = Effect.fnUntraced(function* () { |
| 594 | yield* Ref.set(http2Ref, true); |
| 595 | yield* secrets |
| 596 | .set( |
| 597 | HTTP2_FALLBACK_STATE, |
| 598 | new TextEncoder().encode( |
| 599 | encodeHttp2Fallback({ fellBackAt: yield* Clock.currentTimeMillis }), |
| 600 | ), |
| 601 | ) |
| 602 | .pipe( |
| 603 | Effect.catch(() => Effect.logWarning("Could not persist the relay client HTTP/2 fallback")), |
| 604 | ); |
| 605 | }); |
| 606 | |
| 607 | const switchToHttp2 = Effect.fnUntraced(function* (connector: ActiveConnector) { |
| 608 | const active = yield* Ref.get(activeRef); |
| 609 | const desiredConfig = yield* Ref.get(desiredConfigRef); |
| 610 | if ((yield* Ref.get(http2Ref)) || active !== connector || !desiredConfig) return; |
| 611 | yield* rememberHttp2Fallback(); |
| 612 | yield* Effect.logWarning("Relay client keeps losing QUIC connections; switching to HTTP/2", { |
| 613 | pid: Number(connector.child.pid), |
| 614 | tunnelId: connector.config.tunnelId, |
| 615 | tunnelName: connector.config.tunnelName, |
| 616 | }); |
| 617 | // Shutdown must not interrupt cleanup after active ownership is cleared. |
| 618 | yield* Effect.uninterruptible(stopActive); |
| 619 | const status = yield* reconcileConfig(desiredConfig); |
| 620 | // A failed spawn has no supervisor to request recovery. |
| 621 | if (status.status !== "running") { |
| 622 | yield* Queue.offer(recoveryRequests, desiredConfig); |
| 623 | } |
| 624 | }, reconcileSemaphore.withPermits(1)); |
| 625 | |
| 626 | const noteQuicStall = Effect.fnUntraced(function* (connector: ActiveConnector) { |
| 627 | if (transportChosenByUser || (yield* Ref.get(http2Ref))) return; |
| 628 | const now = yield* Clock.currentTimeMillis; |
| 629 | if ((yield* Ref.get(activeRef)) !== connector) return; |
| 630 | const stalls = yield* Ref.updateAndGet(quicStallsRef, (previous) => [ |
| 631 | ...previous.filter((at) => now - at < QUIC_STALL_WINDOW_MS), |
| 632 | now, |
| 633 | ]); |
| 634 | if (stalls.length < QUIC_STALLS_BEFORE_HTTP2) return; |
| 635 | // Stopping the connector interrupts the output observer that called this. |
| 636 | yield* Effect.forkIn(switchToHttp2(connector), runtimeScope); |
| 637 | }); |
| 638 | |
| 639 | const observeConnectorOutput = (connector: ActiveConnector) => { |
| 640 | let rejectedRegistrations = 0; |
| 641 | |
| 642 | return connector.child.all.pipe( |
| 643 | Stream.decodeText(), |
| 644 | Stream.splitLines, |
| 645 | Stream.map((line) => line.trim()), |
| 646 | Stream.filter((line) => line.length > 0), |
| 647 | Stream.runForEach((line) => { |
| 648 | const output = line.replaceAll(connector.config.connectorToken, "<redacted>"); |
| 649 | const attributes = { |
| 650 | pid: Number(connector.child.pid), |
| 651 | tunnelId: connector.config.tunnelId, |
| 652 | tunnelName: connector.config.tunnelName, |
| 653 | output, |
| 654 | }; |
| 655 | switch (classifyRelayClientOutput(line)) { |
| 656 | case "connected": { |
| 657 | rejectedRegistrations = 0; |
| 658 | const protocol = RELAY_CLIENT_PROTOCOL.exec(line)?.[1] ?? null; |
| 659 | return Deferred.succeed(connector.registered, undefined).pipe( |
| 660 | Effect.andThen(Ref.set(connector.protocol, protocol)), |
| 661 | // cloudflared started with `auto` fell back on its own; later starts skip QUIC too. |
| 662 | Effect.andThen( |
| 663 | protocol === "http2" && !transportChosenByUser |
| 664 | ? Ref.get(http2Ref).pipe( |
| 665 | Effect.flatMap((http2) => (http2 ? Effect.void : rememberHttp2Fallback())), |
| 666 | ) |
| 667 | : Effect.void, |
| 668 | ), |
| 669 | Effect.andThen( |
| 670 | Effect.logInfo("Relay client tunnel connection registered", attributes), |
| 671 | ), |
| 672 | Effect.andThen(Queue.offer(tunnelConnections, undefined)), |
| 673 | Effect.andThen(Queue.offer(readinessChanges, undefined)), |
| 674 | Effect.andThen( |
| 675 | RelayClient.isPinnedManagedRelayClient(connector.executable) |
| 676 | ? Ref.getAndSet(prunedRef, true).pipe( |
| 677 | Effect.flatMap((pruned) => |
| 678 | pruned ? Effect.void : relayClient.pruneManagedVersions, |
| 679 | ), |
| 680 | ) |
| 681 | : Effect.void, |
| 682 | ), |
| 683 | Effect.asVoid, |
| 684 | ); |
| 685 | } |
| 686 | case "unregistered": |
| 687 | return Effect.logInfo("Relay client tunnel connection unregistered", attributes); |
| 688 | case "warning": |
| 689 | if (/\btimeout: no recent network activity\b/u.test(line)) { |
| 690 | return noteQuicStall(connector).pipe( |
| 691 | Effect.andThen( |
| 692 | Effect.logWarning("Relay client reported a transport warning", attributes), |
| 693 | ), |
| 694 | ); |
| 695 | } |
| 696 | if (isRejectedRelayClientTunnelOutput(line)) { |
| 697 | rejectedRegistrations += 1; |
| 698 | if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) { |
| 699 | rejectedRegistrations = 0; |
| 700 | return Effect.logWarning( |
| 701 | "Relay client tunnel was rejected; requesting recovery", |
| 702 | attributes, |
| 703 | ).pipe( |
| 704 | Effect.andThen(Queue.offer(recoveryRequests, connector.config)), |
| 705 | Effect.asVoid, |
| 706 | ); |
| 707 | } |
| 708 | } |
| 709 | return Effect.logWarning("Relay client reported a transport warning", attributes); |
| 710 | case "debug": { |
| 711 | const metricsAddress = RELAY_CLIENT_METRICS_LINE.exec(line)?.[1]; |
| 712 | return metricsAddress |
| 713 | ? Deferred.succeed(connector.metricsAddress, metricsAddress).pipe( |
| 714 | Effect.andThen(Effect.logDebug("Relay client output", attributes)), |
| 715 | ) |
| 716 | : Effect.logDebug("Relay client output", attributes); |
| 717 | } |
| 718 | } |
| 719 | }), |
| 720 | Effect.catchCause((cause) => |
| 721 | Effect.logWarning("Relay client output observer failed", { |
| 722 | cause, |
| 723 | pid: Number(connector.child.pid), |
| 724 | tunnelId: connector.config.tunnelId, |
| 725 | tunnelName: connector.config.tunnelName, |
| 726 | }), |
| 727 | ), |
| 728 | ); |
| 729 | }; |
| 730 | |
| 731 | reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) { |
| 732 | if (!config || config.providerKind !== "cloudflare_tunnel") { |
| 733 | yield* stopActive; |
| 734 | return config |
| 735 | ? { status: "unsupported", providerKind: config.providerKind } |
| 736 | : { status: "disabled" }; |
| 737 | } |
| 738 | |
| 739 | const nextConfigKey = runtimeConfigKey(config); |
| 740 | const active = yield* Ref.get(activeRef); |
| 741 | if (active?.configKey === nextConfigKey) { |
| 742 | const isRunning = yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)); |
| 743 | if (isRunning) { |
| 744 | return { |
| 745 | status: "running", |
| 746 | providerKind: "cloudflare_tunnel", |
| 747 | pid: Number(active.child.pid), |
| 748 | ...(active.config.tunnelId ? { tunnelId: active.config.tunnelId } : {}), |
| 749 | ...(active.config.tunnelName ? { tunnelName: active.config.tunnelName } : {}), |
| 750 | } satisfies CloudManagedEndpointRuntimeStatus; |
| 751 | } |
| 752 | } |
| 753 | |
| 754 | yield* stopActive; |
| 755 | |
| 756 | const executable = yield* relayClient.resolve; |
| 757 | if (needsPinnedRelayClient(executable)) { |
| 758 | yield* ensurePinnedRelayClient; |
| 759 | yield* startPinnedRelayClientRetries; |
| 760 | } |
| 761 | if (executable.status !== "available") { |
| 762 | return { |
| 763 | status: "failed", |
| 764 | providerKind: "cloudflare_tunnel", |
| 765 | failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed", |
| 766 | reason: |
| 767 | executable.status === "unsupported" |
| 768 | ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.` |
| 769 | : "The relay client is not installed.", |
| 770 | ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), |
| 771 | ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), |
| 772 | } satisfies CloudManagedEndpointRuntimeStatus; |
| 773 | } |
| 774 | |
| 775 | const http2 = yield* Ref.get(http2Ref); |
| 776 | const connectorScope = yield* Scope.make("sequential"); |
| 777 | // Added before the spawn so it runs after the spawner's finalizer has stopped the client. |
| 778 | let forgetRelayClient: Effect.Effect<void> = Effect.void; |
| 779 | yield* Scope.addFinalizer( |
| 780 | connectorScope, |
| 781 | Effect.suspend(() => forgetRelayClient), |
| 782 | ); |
| 783 | const child = yield* spawner |
| 784 | .spawn( |
| 785 | ChildProcess.make( |
| 786 | executable.executablePath, |
| 787 | [ |
| 788 | "tunnel", |
| 789 | "--no-autoupdate", |
| 790 | "--loglevel", |
| 791 | "info", |
| 792 | "--output", |
| 793 | "default", |
| 794 | "--metrics", |
| 795 | "127.0.0.1:0", |
| 796 | ...(http2 ? ["--protocol", "http2"] : []), |
| 797 | "run", |
| 798 | ], |
| 799 | { |
| 800 | detached: false, |
| 801 | env: { |
| 802 | ...env, |
| 803 | TUNNEL_TOKEN: config.connectorToken, |
| 804 | }, |
| 805 | shell: false, |
| 806 | stderr: "pipe", |
| 807 | stdout: "pipe", |
| 808 | }, |
| 809 | ), |
| 810 | ) |
| 811 | .pipe( |
| 812 | Effect.provideService(Scope.Scope, connectorScope), |
| 813 | Effect.tap((child) => |
| 814 | Effect.logInfo("Relay client process started; waiting for tunnel connection", { |
| 815 | pid: Number(child.pid), |
| 816 | source: executable.source, |
| 817 | version: executable.version, |
| 818 | protocol: http2 ? "http2" : "auto", |
| 819 | tunnelId: config.tunnelId, |
| 820 | tunnelName: config.tunnelName, |
| 821 | }), |
| 822 | ), |
| 823 | Effect.catch((cause) => |
| 824 | Effect.logWarning("Failed to start relay client", { |
| 825 | cause, |
| 826 | tunnelId: config.tunnelId, |
| 827 | tunnelName: config.tunnelName, |
| 828 | }).pipe( |
| 829 | Effect.andThen(Scope.close(connectorScope, Exit.void).pipe(Effect.ignore)), |
| 830 | Effect.as({ |
| 831 | status: "failed", |
| 832 | providerKind: "cloudflare_tunnel", |
| 833 | failure: "spawn-failed", |
| 834 | reason: String(cause), |
| 835 | ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), |
| 836 | ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), |
| 837 | } satisfies CloudManagedEndpointRuntimeStatus), |
| 838 | ), |
| 839 | ), |
| 840 | ); |
| 841 | |
| 842 | if ("status" in child && child.status === "failed") { |
| 843 | return child; |
| 844 | } |
| 845 | |
| 846 | if (!("status" in child)) { |
| 847 | // An interrupt before activeRef owns a supervised connector would leave its client open or unwatched. |
| 848 | yield* Effect.uninterruptibleMask((restore) => |
| 849 | Effect.gen(function* () { |
| 850 | forgetRelayClient = yield* relayClientLedger.track(Number(child.pid)); |
| 851 | const connector = { |
| 852 | child, |
| 853 | executable, |
| 854 | scope: connectorScope, |
| 855 | configKey: nextConfigKey, |
| 856 | config, |
| 857 | startedAtMillis: yield* Clock.currentTimeMillis, |
| 858 | registered: yield* Deferred.make<void>(), |
| 859 | metricsAddress: yield* Deferred.make<string>(), |
| 860 | protocol: yield* Ref.make<string | null>(null), |
| 861 | } satisfies ActiveConnector; |
| 862 | yield* Ref.set(activeRef, connector); |
| 863 | yield* Effect.forkIn(restore(observeConnectorOutput(connector)), connectorScope); |
| 864 | yield* Effect.forkIn(restore(superviseConnector(connector)), connectorScope); |
| 865 | yield* Effect.forkIn(restore(watchConnectorRegistration(connector)), connectorScope); |
| 866 | yield* Effect.forkIn(restore(watchConnectorReadiness(connector)), connectorScope); |
| 867 | }), |
| 868 | ); |
| 869 | return { |
| 870 | status: "running", |
| 871 | providerKind: "cloudflare_tunnel", |
| 872 | pid: Number(child.pid), |
| 873 | ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), |
| 874 | ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), |
| 875 | } satisfies CloudManagedEndpointRuntimeStatus; |
| 876 | } |
| 877 | |
| 878 | return { |
| 879 | status: "failed", |
| 880 | providerKind: "cloudflare_tunnel", |
| 881 | failure: "spawn-failed", |
| 882 | reason: "Relay client did not start.", |
| 883 | ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), |
| 884 | ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), |
| 885 | } satisfies CloudManagedEndpointRuntimeStatus; |
| 886 | }); |
| 887 | |
| 888 | const applyConfig = Effect.fn("CloudManagedEndpointRuntime.applyConfig")( |
| 889 | (requested: RelayManagedEndpointRuntimeConfig | null) => |
| 890 | reconcileSemaphore.withPermits(1)( |
| 891 | Effect.gen(function* () { |
| 892 | const config = connectAllowed ? requested : null; |
| 893 | // Keep backoff when recovery returns the same tunnel/token, or every recovery respawns a crash loop. |
| 894 | const desired = yield* Ref.get(desiredConfigRef); |
| 895 | const unchanged = |
| 896 | desired !== null && |
| 897 | config !== null && |
| 898 | runtimeConfigKey(desired) === runtimeConfigKey(config); |
| 899 | if (!unchanged) { |
| 900 | yield* Ref.set(restartDelayRef, 0); |
| 901 | } |
| 902 | yield* Ref.set(desiredConfigRef, config); |
| 903 | return yield* reconcileConfig(config); |
| 904 | }), |
| 905 | ), |
| 906 | withRelayClientLog, |
| 907 | ); |
| 908 | |
| 909 | const restartConnector = Effect.fn("CloudManagedEndpointRuntime.restartConnector")(function* ( |
| 910 | pid: number, |
| 911 | stillDead: Effect.Effect<boolean>, |
| 912 | ) { |
| 913 | const active = yield* Ref.get(activeRef); |
| 914 | if (!active || Number(active.child.pid) !== pid || !(yield* stillDead)) return false; |
| 915 | yield* Effect.logWarning( |
| 916 | "Cloudflare has no live connection from the relay client; restarting it", |
| 917 | { pid, tunnelId: active.config.tunnelId, tunnelName: active.config.tunnelName }, |
| 918 | ); |
| 919 | // The supervisor starts the replacement with its usual crash-loop backoff. |
| 920 | return yield* active.child.kill().pipe( |
| 921 | Effect.as(true), |
| 922 | Effect.orElseSucceed(() => false), |
| 923 | ); |
| 924 | }, reconcileSemaphore.withPermits(1)); |
| 925 | |
| 926 | const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) => |
| 927 | httpClient.get(url).pipe( |
| 928 | Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), |
| 929 | Effect.scoped, |
| 930 | Effect.withTracerEnabled(false), |
| 931 | Effect.timeoutOption(CONNECTOR_READINESS_REQUEST_TIMEOUT), |
| 932 | Effect.map(Option.getOrNull), |
| 933 | Effect.orElseSucceed(() => null), |
| 934 | ); |
| 935 | |
| 936 | const inspectConnector = Effect.gen(function* () { |
| 937 | const active = yield* Ref.get(activeRef); |
| 938 | if (!active || !(yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)))) { |
| 939 | return null; |
| 940 | } |
| 941 | const pid = Number(active.child.pid); |
| 942 | const protocol = yield* Ref.get(active.protocol); |
| 943 | if (!(yield* Deferred.isDone(active.metricsAddress))) { |
| 944 | return { |
| 945 | pid, |
| 946 | readyConnections: null, |
| 947 | hostname: null, |
| 948 | origin: null, |
| 949 | protocol, |
| 950 | } satisfies ConnectorFacts; |
| 951 | } |
| 952 | const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`; |
| 953 | const [ready, config] = yield* Effect.all( |
| 954 | [ |
| 955 | readMetrics(`${metricsUrl}/ready`, ConnectorReadyBody), |
| 956 | readMetrics(`${metricsUrl}/config`, ConnectorConfigBody), |
| 957 | ], |
| 958 | { concurrency: 2 }, |
| 959 | ); |
| 960 | const hostnameRule = config?.config.ingress?.find((rule) => rule.hostname); |
| 961 | return { |
| 962 | pid, |
| 963 | readyConnections: ready?.readyConnections ?? null, |
| 964 | hostname: hostnameRule?.hostname ?? null, |
| 965 | origin: hostnameRule?.service ?? null, |
| 966 | protocol, |
| 967 | } satisfies ConnectorFacts; |
| 968 | }); |
| 969 | |
| 970 | const runtime = CloudManagedEndpointRuntime.of({ |
| 971 | applyConfig, |
| 972 | inspectConnector, |
| 973 | readinessChanged: Stream.fromQueue(readinessChanges), |
| 974 | recoveryRequests: Stream.fromQueue(recoveryRequests), |
| 975 | requestRecovery: Effect.fn("CloudManagedEndpointRuntime.requestRecovery")(function (config) { |
| 976 | return Queue.offer(recoveryRequests, config).pipe(Effect.asVoid); |
| 977 | }), |
| 978 | restartConnector, |
| 979 | tunnelConnected: Stream.fromQueue(tunnelConnections), |
| 980 | withLinkStateLock: linkStateSemaphore.withPermits(1), |
| 981 | }); |
| 982 | |
| 983 | yield* Effect.addFinalizer(() => runtime.applyConfig(null)); |
| 984 | return runtime; |
| 985 | }); |
| 986 | |
| 987 | export const layer = Layer.effect(CloudManagedEndpointRuntime, make); |