@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 268 lines 9.93 KB
Raw
1 import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2 import * as Cause from "effect/Cause";
3 import * as Clock from "effect/Clock";
4 import * as Duration from "effect/Duration";
5 import * as Effect from "effect/Effect";
6 import * as Schedule from "effect/Schedule";
7 import * as Semaphore from "effect/Semaphore";
8
9 import { isRelayUnavailable, relayRetryAfterMillis } from "./relayResponse.ts";
10
11 // Relinking revoked environments is the user's choice; startup takes no action.
12 export type ManagedTunnelRegistrationResult =
13 | { readonly status: "not_linked" | "ready" | "unavailable" | "superseded" | "revoked" }
14 | {
15 readonly status: "recovery_required";
16 readonly config: RelayManagedEndpointRuntimeConfig;
17 };
18
19 export type ManagedTunnelStartupAction =
20 | { readonly action: "none" }
21 | { readonly action: "reconcile_link" }
22 | {
23 readonly action: "request_recovery";
24 readonly config: RelayManagedEndpointRuntimeConfig;
25 };
26
27 export function managedTunnelStartupAction(input: {
28 readonly wantsCliLink: boolean;
29 readonly registration: ManagedTunnelRegistrationResult;
30 }): ManagedTunnelStartupAction {
31 if (input.registration.status === "recovery_required") {
32 return {
33 action: "request_recovery",
34 config: input.registration.config,
35 };
36 }
37 if (input.wantsCliLink && input.registration.status === "not_linked") {
38 return { action: "reconcile_link" };
39 }
40 return { action: "none" };
41 }
42
43 const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10);
44
45 // Back off to minutes and honor Retry-After to avoid adding load to a failing relay.
46 // Retries must rerun the operation: HttpClient.retryTransient would reuse proofs that
47 // expire after 60 s or single-use link challenges.
48 const RELAY_RETRY_MAX_DELAY = Duration.minutes(5);
49
50 const relayRetrySchedule = Schedule.exponential("1 second").pipe(
51 Schedule.jittered,
52 Schedule.modifyDelay(({ duration }) =>
53 Effect.succeed(Duration.min(duration, RELAY_RETRY_MAX_DELAY)),
54 ),
55 Schedule.modifyDelay(({ duration, input }) => {
56 const retryAfterMillis = relayRetryAfterMillis(input);
57 return Effect.succeed(
58 retryAfterMillis === undefined
59 ? duration
60 : Duration.max(duration, Duration.millis(retryAfterMillis)),
61 );
62 }),
63 );
64
65 const windowMillis = Duration.toMillis(MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW);
66
67 /**
68 * Retries on the relay schedule. The window closes at the first sleep that
69 * would end past it: then `onWindowExhausted` runs, and retrying continues
70 * only while `whilePastWindow` holds. One schedule covers both sides, so the
71 * relay's Retry-After still holds across the handoff.
72 */
73 const retryAcrossWindow = <A, E, R>(
74 effect: Effect.Effect<A, E, R>,
75 options: {
76 readonly while: (error: E) => boolean;
77 readonly whilePastWindow: (error: E) => boolean;
78 readonly onWindowExhausted: (error: E) => Effect.Effect<void, never, R>;
79 },
80 ) =>
81 Effect.suspend(() => {
82 let pastWindow = false;
83 return effect.pipe(
84 Effect.retry({
85 while: (error) => (pastWindow ? options.whilePastWindow(error) : options.while(error)),
86 schedule: relayRetrySchedule.pipe(
87 Schedule.setInputType<E>(),
88 Schedule.modifyDelay(({ duration, elapsed, input }) => {
89 if (pastWindow || elapsed + Duration.toMillis(duration) < windowMillis) {
90 return Effect.succeed(duration);
91 }
92 pastWindow = true;
93 return options.whilePastWindow(input)
94 ? options.onWindowExhausted(input).pipe(Effect.as(duration))
95 : Effect.fail(input);
96 }),
97 ),
98 }),
99 );
100 });
101
102 export const retryManagedTunnelRegistration = <A, E, R>(
103 registration: Effect.Effect<A, E, R>,
104 isRetryable: (error: E) => boolean,
105 onRetryWindowExhausted?: Effect.Effect<void, never, R>,
106 ) =>
107 retryAcrossWindow(registration, {
108 while: isRetryable,
109 whilePastWindow: onRetryWindowExhausted === undefined ? () => false : isRetryable,
110 onWindowExhausted: () => onRetryWindowExhausted ?? Effect.void,
111 });
112
113 /**
114 * Relinks a desired link at startup. Past the retry window it keeps going only
115 * while the relay itself is unavailable: a headless host has nobody to restart
116 * it once the relay recovers, but a local failure (cloudflared that cannot run)
117 * would otherwise relink, and provision a tunnel, forever.
118 */
119 export const retryDesiredLinkReconcile = <A, E, R>(
120 reconcile: Effect.Effect<A, E, R>,
121 isRetryable: (error: E) => boolean,
122 onRetryWindowExhausted: (error: E) => Effect.Effect<void, never, R>,
123 ) =>
124 retryAcrossWindow(reconcile, {
125 while: isRetryable,
126 whilePastWindow: isRelayUnavailable,
127 onWindowExhausted: onRetryWindowExhausted,
128 });
129
130 // Every managed host shares one relay, so a host stuck in a bad loop must not
131 // turn into a fleet-wide request storm.
132 const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2);
133
134 // Retries inside one recovery, while the relay keeps failing, stay at least a
135 // cooldown apart. They do not grow the cooldown: only a new request does.
136 const managedTunnelRecoveryRetrySchedule = relayRetrySchedule.pipe(
137 Schedule.modifyDelay(({ duration }) =>
138 Effect.succeed(Duration.max(duration, MANAGED_TUNNEL_RECOVERY_COOLDOWN)),
139 ),
140 );
141 const MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN = Duration.minutes(30);
142 // A recovery that fixed the tunnel stops the runtime asking for another, so a
143 // request this long after the cooldown means the previous one worked.
144 const MANAGED_TUNNEL_RECOVERY_SETTLED_AFTER = Duration.minutes(10);
145
146 /**
147 * Recovery can only replace the relay-side tunnel. When the connector keeps
148 * failing for local reasons (no network, a firewall), the runtime keeps asking
149 * and recovery cannot help, so each request that follows right after a
150 * recovery doubles the wait, up to 30 minutes.
151 */
152 function nextManagedTunnelRecoveryCooldown(input: {
153 readonly current: Duration.Duration;
154 readonly sinceLastRecovery: Duration.Duration;
155 }): Duration.Duration {
156 const settled = Duration.sum(input.current, MANAGED_TUNNEL_RECOVERY_SETTLED_AFTER);
157 if (Duration.isGreaterThanOrEqualTo(input.sinceLastRecovery, settled)) {
158 return MANAGED_TUNNEL_RECOVERY_COOLDOWN;
159 }
160 return Duration.min(Duration.times(input.current, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN);
161 }
162
163 // Existing hosts register on their first boot after an upgrade, and desktop
164 // auto-update delivers that boot to many hosts at once. Spread the first
165 // registration so the relay and Cloudflare see a ramp instead of a spike.
166 export const MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER = Duration.seconds(30);
167
168 /**
169 * Runs one recovery per request, one at a time: waits out the current
170 * cooldown, then calls the relay, retrying it while it stays unavailable.
171 */
172 export const makeManagedTunnelRecovery = Effect.fn("ManagedTunnelRecovery.make")(function* <
173 A,
174 E,
175 R,
176 >(options: {
177 readonly recover: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<A, E, R>;
178 readonly isRetryable: (error: E) => boolean;
179 }) {
180 const lock = yield* Semaphore.make(1);
181 let lastRecoveryAtMillis = Number.NEGATIVE_INFINITY;
182 let cooldown = MANAGED_TUNNEL_RECOVERY_COOLDOWN;
183 const waitForCooldown = Effect.gen(function* () {
184 const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis;
185 cooldown = nextManagedTunnelRecoveryCooldown({
186 current: cooldown,
187 sinceLastRecovery: Duration.millis(elapsed),
188 });
189 const wait = Duration.toMillis(cooldown) - elapsed;
190 if (wait > 0) yield* Effect.sleep(Duration.millis(wait));
191 });
192 // Measured from the end: a recovery that retried through a long relay
193 // outage must not look settled the moment it finishes.
194 const recordRecovery = Clock.currentTimeMillis.pipe(
195 Effect.tap((now) =>
196 Effect.sync(() => {
197 lastRecoveryAtMillis = now;
198 }),
199 ),
200 );
201 return (config: RelayManagedEndpointRuntimeConfig) =>
202 lock.withPermits(1)(
203 waitForCooldown.pipe(
204 Effect.andThen(
205 options.recover(config).pipe(
206 Effect.retry({
207 while: options.isRetryable,
208 schedule: managedTunnelRecoveryRetrySchedule,
209 }),
210 Effect.ensuring(recordRecovery),
211 ),
212 ),
213 ),
214 );
215 });
216
217 const MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL = Duration.minutes(1);
218
219 const urlOrigin = (url: string) => {
220 try {
221 return new URL(url).origin;
222 } catch {
223 return null;
224 }
225 };
226
227 /**
228 * Only this server runs the connector, so an ingress naming another origin is a dead
229 * registration. Checks every minute and re-registers this origin on a mismatch,
230 * doubling the wait (up to 30 minutes) while the relay keeps routing elsewhere.
231 */
232 export const watchManagedTunnelOrigin = Effect.fn("ManagedTunnelOrigin.watch")(function* <
233 E,
234 R,
235 >(options: {
236 readonly localOrigin: string;
237 /** The service URL cloudflared routes the public hostname to, or `null` while unknown. */
238 readonly readIngressOrigin: Effect.Effect<string | null>;
239 readonly register: Effect.Effect<unknown, E, R>;
240 }) {
241 const localOrigin = urlOrigin(options.localOrigin);
242 let wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL;
243 while (true) {
244 yield* Effect.sleep(wait);
245 const ingress = yield* options.readIngressOrigin;
246 // An unreadable ingress says nothing, so it keeps whatever backoff is running.
247 if (ingress === null) continue;
248 const ingressOrigin = urlOrigin(ingress);
249 if (ingressOrigin !== null && ingressOrigin === localOrigin) {
250 wait = MANAGED_TUNNEL_ORIGIN_CHECK_INTERVAL;
251 continue;
252 }
253 yield* Effect.logWarning(
254 "T3 Connect routes this server's tunnel to another origin; registering this server again",
255 { ingressOrigin: ingressOrigin ?? "unparseable", localOrigin },
256 );
257 yield* options.register.pipe(
258 Effect.catchCause((cause) =>
259 Cause.hasInterrupts(cause)
260 ? Effect.interrupt
261 : Effect.logWarning("Failed to register this server's T3 Connect origin again", {
262 cause,
263 }),
264 ),
265 );
266 wait = Duration.min(Duration.times(wait, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN);
267 }
268 });