The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; |
| 2 | import * as Cause from "effect/Cause"; |
| 3 | import * as Clock from "effect/Clock"; |
| 4 | import * as Duration from "effect/Duration"; |
| 5 | import * as Effect from "effect/Effect"; |
| 6 | import * as Schedule from "effect/Schedule"; |
| 7 | import * as Semaphore from "effect/Semaphore"; |
| 8 | |
| 9 | import { isRelayUnavailable, relayRetryAfterMillis } from "./relayResponse.ts"; |
| 10 | |
| 11 | // Relinking revoked environments is the user's choice; startup takes no action. |
| 12 | export type ManagedTunnelRegistrationResult = |
| 13 | | { readonly status: "not_linked" | "ready" | "unavailable" | "superseded" | "revoked" } |
| 14 | | { |
| 15 | readonly status: "recovery_required"; |
| 16 | readonly config: RelayManagedEndpointRuntimeConfig; |
| 17 | }; |
| 18 | |
| 19 | export type ManagedTunnelStartupAction = |
| 20 | | { readonly action: "none" } |
| 21 | | { readonly action: "reconcile_link" } |
| 22 | | { |
| 23 | readonly action: "request_recovery"; |
| 24 | readonly config: RelayManagedEndpointRuntimeConfig; |
| 25 | }; |
| 26 | |
| 27 | export function managedTunnelStartupAction(input: { |
| 28 | readonly wantsCliLink: boolean; |
| 29 | readonly registration: ManagedTunnelRegistrationResult; |
| 30 | }): ManagedTunnelStartupAction { |
| 31 | if (input.registration.status === "recovery_required") { |
| 32 | return { |
| 33 | action: "request_recovery", |
| 34 | config: input.registration.config, |
| 35 | }; |
| 36 | } |
| 37 | if (input.wantsCliLink && input.registration.status === "not_linked") { |
| 38 | return { action: "reconcile_link" }; |
| 39 | } |
| 40 | return { action: "none" }; |
| 41 | } |
| 42 | |
| 43 | const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10); |
| 44 | |
| 45 | // Back off to minutes and honor Retry-After to avoid adding load to a failing relay. |
| 46 | // Retries must rerun the operation: HttpClient.retryTransient would reuse proofs that |
| 47 | // expire after 60 s or single-use link challenges. |
| 48 | const RELAY_RETRY_MAX_DELAY = Duration.minutes(5); |
| 49 | |
| 50 | const relayRetrySchedule = Schedule.exponential("1 second").pipe( |
| 51 | Schedule.jittered, |
| 52 | Schedule.modifyDelay(({ duration }) => |
| 53 | Effect.succeed(Duration.min(duration, RELAY_RETRY_MAX_DELAY)), |
| 54 | ), |
| 55 | Schedule.modifyDelay(({ duration, input }) => { |
| 56 | const retryAfterMillis = relayRetryAfterMillis(input); |
| 57 | return Effect.succeed( |
| 58 | retryAfterMillis === undefined |
| 59 | ? duration |
| 60 | : Duration.max(duration, Duration.millis(retryAfterMillis)), |
| 61 | ); |
| 62 | }), |
| 63 | ); |
| 64 | |
| 65 | const windowMillis = Duration.toMillis(MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW); |
| 66 | |
| 67 | /** |
| 68 | * Retries on the relay schedule. The window closes at the first sleep that |
| 69 | * would end past it: then `onWindowExhausted` runs, and retrying continues |
| 70 | * only while `whilePastWindow` holds. One schedule covers both sides, so the |
| 71 | * relay's Retry-After still holds across the handoff. |
| 72 | */ |
| 73 | const retryAcrossWindow = <A, E, R>( |
| 74 | effect: Effect.Effect<A, E, R>, |
| 75 | options: { |
| 76 | readonly while: (error: E) => boolean; |
| 77 | readonly whilePastWindow: (error: E) => boolean; |
| 78 | readonly onWindowExhausted: (error: E) => Effect.Effect<void, never, R>; |
| 79 | }, |
| 80 | ) => |
| 81 | Effect.suspend(() => { |
| 82 | let pastWindow = false; |
| 83 | return effect.pipe( |
| 84 | Effect.retry({ |
| 85 | while: (error) => (pastWindow ? options.whilePastWindow(error) : options.while(error)), |
| 86 | schedule: relayRetrySchedule.pipe( |
| 87 | Schedule.setInputType<E>(), |
| 88 | Schedule.modifyDelay(({ duration, elapsed, input }) => { |
| 89 | if (pastWindow || elapsed + Duration.toMillis(duration) < windowMillis) { |
| 90 | return Effect.succeed(duration); |
| 91 | } |
| 92 | pastWindow = true; |
| 93 | return options.whilePastWindow(input) |
| 94 | ? options.onWindowExhausted(input).pipe(Effect.as(duration)) |
| 95 | : Effect.fail(input); |
| 96 | }), |
| 97 | ), |
| 98 | }), |
| 99 | ); |
| 100 | }); |
| 101 | |
| 102 | export const retryManagedTunnelRegistration = <A, E, R>( |
| 103 | registration: Effect.Effect<A, E, R>, |
| 104 | isRetryable: (error: E) => boolean, |
| 105 | onRetryWindowExhausted?: Effect.Effect<void, never, R>, |
| 106 | ) => |
| 107 | retryAcrossWindow(registration, { |
| 108 | while: isRetryable, |
| 109 | whilePastWindow: onRetryWindowExhausted === undefined ? () => false : isRetryable, |
| 110 | onWindowExhausted: () => onRetryWindowExhausted ?? Effect.void, |
| 111 | }); |
| 112 | |
| 113 | /** |
| 114 | * Relinks a desired link at startup. Past the retry window it keeps going only |
| 115 | * while the relay itself is unavailable: a headless host has nobody to restart |
| 116 | * it once the relay recovers, but a local failure (cloudflared that cannot run) |
| 117 | * would otherwise relink, and provision a tunnel, forever. |
| 118 | */ |
| 119 | export const retryDesiredLinkReconcile = <A, E, R>( |
| 120 | reconcile: Effect.Effect<A, E, R>, |
| 121 | isRetryable: (error: E) => boolean, |
| 122 | onRetryWindowExhausted: (error: E) => Effect.Effect<void, never, R>, |
| 123 | ) => |
| 124 | retryAcrossWindow(reconcile, { |
| 125 | while: isRetryable, |
| 126 | whilePastWindow: isRelayUnavailable, |
| 127 | onWindowExhausted: onRetryWindowExhausted, |
| 128 | }); |
| 129 | |
| 130 | // Every managed host shares one relay, so a host stuck in a bad loop must not |
| 131 | // turn into a fleet-wide request storm. |
| 132 | const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2); |
| 133 | |
| 134 | // Retries inside one recovery, while the relay keeps failing, stay at least a |
| 135 | // cooldown apart. They do not grow the cooldown: only a new request does. |
| 136 | const managedTunnelRecoveryRetrySchedule = relayRetrySchedule.pipe( |
| 137 | Schedule.modifyDelay(({ duration }) => |
| 138 | Effect.succeed(Duration.max(duration, MANAGED_TUNNEL_RECOVERY_COOLDOWN)), |
| 139 | ), |
| 140 | ); |
| 141 | const MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN = Duration.minutes(30); |
| 142 | // A recovery that fixed the tunnel stops the runtime asking for another, so a |
| 143 | // request this long after the cooldown means the previous one worked. |
| 144 | const MANAGED_TUNNEL_RECOVERY_SETTLED_AFTER = Duration.minutes(10); |
| 145 | |
| 146 | /** |
| 147 | * Recovery can only replace the relay-side tunnel. When the connector keeps |
| 148 | * failing for local reasons (no network, a firewall), the runtime keeps asking |
| 149 | * and recovery cannot help, so each request that follows right after a |
| 150 | * recovery doubles the wait, up to 30 minutes. |
| 151 | */ |
| 152 | function nextManagedTunnelRecoveryCooldown(input: { |
| 153 | readonly current: Duration.Duration; |
| 154 | readonly sinceLastRecovery: Duration.Duration; |
| 155 | }): Duration.Duration { |
| 156 | const settled = Duration.sum(input.current, MANAGED_TUNNEL_RECOVERY_SETTLED_AFTER); |
| 157 | if (Duration.isGreaterThanOrEqualTo(input.sinceLastRecovery, settled)) { |
| 158 | return MANAGED_TUNNEL_RECOVERY_COOLDOWN; |
| 159 | } |
| 160 | return Duration.min(Duration.times(input.current, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN); |
| 161 | } |
| 162 | |
| 163 | // Existing hosts register on their first boot after an upgrade, and desktop |
| 164 | // auto-update delivers that boot to many hosts at once. Spread the first |
| 165 | // registration so the relay and Cloudflare see a ramp instead of a spike. |
| 166 | export const MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER = Duration.seconds(30); |
| 167 | |
| 168 | /** |
| 169 | * Runs one recovery per request, one at a time: waits out the current |
| 170 | * cooldown, then calls the relay, retrying it while it stays unavailable. |
| 171 | */ |
| 172 | export const makeManagedTunnelRecovery = Effect.fn("ManagedTunnelRecovery.make")(function* < |
| 173 | A, |
| 174 | E, |
| 175 | R, |
| 176 | >(options: { |
| 177 | readonly recover: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<A, E, R>; |
| 178 | readonly isRetryable: (error: E) => boolean; |
| 179 | }) { |
| 180 | const lock = yield* Semaphore.make(1); |
| 181 | let lastRecoveryAtMillis = Number.NEGATIVE_INFINITY; |
| 182 | let cooldown = MANAGED_TUNNEL_RECOVERY_COOLDOWN; |
| 183 | const waitForCooldown = Effect.gen(function* () { |
| 184 | const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis; |
| 185 | cooldown = nextManagedTunnelRecoveryCooldown({ |
| 186 | current: cooldown, |
| 187 | sinceLastRecovery: Duration.millis(elapsed), |
| 188 | }); |
| 189 | const wait = Duration.toMillis(cooldown) - elapsed; |
| 190 | if (wait > 0) yield* Effect.sleep(Duration.millis(wait)); |
| 191 | }); |
| 192 | // Measured from the end: a recovery that retried through a long relay |
| 193 | // outage must not look settled the moment it finishes. |
| 194 | const recordRecovery = Clock.currentTimeMillis.pipe( |
| 195 | Effect.tap((now) => |
| 196 | Effect.sync(() => { |
| 197 | lastRecoveryAtMillis = now; |
| 198 | }), |
| 199 | ), |
| 200 | ); |
| 201 | return (config: RelayManagedEndpointRuntimeConfig) => |
| 202 | lock.withPermits(1)( |
| 203 | waitForCooldown.pipe( |
| 204 | Effect.andThen( |
| 205 | options.recover(config).pipe( |
| 206 | Effect.retry({ |
| 207 | while: options.isRetryable, |
| 208 | schedule: managedTunnelRecoveryRetrySchedule, |
| 209 | }), |
| 210 | Effect.ensuring(recordRecovery), |
| 211 | ), |
| 212 | ), |
| 213 | ), |
| 214 | ); |
| 215 | }); |
| 216 | |
| 217 | const MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL = Duration.minutes(1); |
| 218 | |
| 219 | const urlOrigin = (url: string) => { |
| 220 | try { |
| 221 | return new URL(url).origin; |
| 222 | } catch { |
| 223 | return null; |
| 224 | } |
| 225 | }; |
| 226 | |
| 227 | /** |
| 228 | * Only this server runs the connector, so an ingress naming another origin is a dead |
| 229 | * registration. Checks every minute and re-registers this origin on a mismatch, |
| 230 | * doubling the wait (up to 30 minutes) while the relay keeps routing elsewhere. |
| 231 | */ |
| 232 | export const watchManagedTunnelOrigin = Effect.fn("ManagedTunnelOrigin.watch")(function* < |
| 233 | E, |
| 234 | R, |
| 235 | >(options: { |
| 236 | readonly localOrigin: string; |
| 237 | /** The service URL cloudflared routes the public hostname to, or `null` while unknown. */ |
| 238 | readonly readIngressOrigin: Effect.Effect<string | null>; |
| 239 | readonly register: Effect.Effect<unknown, E, R>; |
| 240 | }) { |
| 241 | const localOrigin = urlOrigin(options.localOrigin); |
| 242 | let wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL; |
| 243 | while (true) { |
| 244 | yield* Effect.sleep(wait); |
| 245 | const ingress = yield* options.readIngressOrigin; |
| 246 | // An unreadable ingress says nothing, so it keeps whatever backoff is running. |
| 247 | if (ingress === null) continue; |
| 248 | const ingressOrigin = urlOrigin(ingress); |
| 249 | if (ingressOrigin !== null && ingressOrigin === localOrigin) { |
| 250 | wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL; |
| 251 | continue; |
| 252 | } |
| 253 | yield* Effect.logWarning( |
| 254 | "T3 Connect routes this server's tunnel to another origin; registering this server again", |
| 255 | { ingressOrigin: ingressOrigin ?? "unparseable", localOrigin }, |
| 256 | ); |
| 257 | yield* options.register.pipe( |
| 258 | Effect.catchCause((cause) => |
| 259 | Cause.hasInterrupts(cause) |
| 260 | ? Effect.interrupt |
| 261 | : Effect.logWarning("Failed to register this server's T3 Connect origin again", { |
| 262 | cause, |
| 263 | }), |
| 264 | ), |
| 265 | ); |
| 266 | wait = Duration.min(Duration.times(wait, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN); |
| 267 | } |
| 268 | }); |