@devries-cameron20 / t3code Mirror · read-only / commits / cd8a98657b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 23:15 UTC

fix(server): T3 Connect backs off from a failing relay instead of hammering it (#22108)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 19, 2026 at 21:22 UTC cd8a98657b997198e4f68f7d7aef40b695fedc4c
5 files changed +455 -71
apps/server/src/cloud/managedTunnelStartup.test.ts
+206
@@ -1,13 +1,21 @@
1 import { describe, expect, it } from "@effect/vitest";
2 +import * as Clock from "effect/Clock";
3 import * as Deferred from "effect/Deferred";
4 +import * as Duration from "effect/Duration";
5 import * as Effect from "effect/Effect";
6 import * as Fiber from "effect/Fiber";
7 import * as TestClock from "effect/testing/TestClock";
8
9 import {
10 + MANAGED_TUNNEL_RECOVERY_COOLDOWN,
11 + makeManagedTunnelRecovery,
12 managedTunnelStartupAction,
13 + nextManagedTunnelRecoveryCooldown,
14 + relayRetrySchedule,
15 + retryDesiredLinkReconcile,
16 retryManagedTunnelRegistration,
17 } from "./managedTunnelStartup.ts";
18 +import { RelayRequestError } from "./relayResponse.ts";
19
20 describe("managedTunnelStartupAction", () => {
21 const config = {
@@ -129,3 +137,201 @@ describe("retryManagedTunnelRegistration", () => {
137 }),
138 );
139 });
140 +
141 +describe("nextManagedTunnelRecoveryCooldown", () => {
142 + const minutes = (value: number) => Duration.minutes(value);
143 +
144 + it("backs off while recoveries keep failing to fix the tunnel, up to 30 minutes", () => {
145 + let cooldown = MANAGED_TUNNEL_RECOVERY_COOLDOWN;
146 + const seen: Array<number> = [];
147 + for (let attempt = 0; attempt < 6; attempt++) {
148 + cooldown = nextManagedTunnelRecoveryCooldown({
149 + current: cooldown,
150 + sinceLastRecovery: minutes(3),
151 + });
152 + seen.push(Duration.toMinutes(cooldown));
153 + }
154 + expect(seen).toEqual([4, 8, 16, 30, 30, 30]);
155 + });
156 +
157 + it("starts over once the previous recovery held", () => {
158 + expect(
159 + Duration.toMinutes(
160 + nextManagedTunnelRecoveryCooldown({ current: minutes(16), sinceLastRecovery: minutes(26) }),
161 + ),
162 + ).toBe(2);
163 + expect(
164 + Duration.toMinutes(
165 + nextManagedTunnelRecoveryCooldown({ current: minutes(16), sinceLastRecovery: minutes(20) }),
166 + ),
167 + ).toBe(30);
168 + });
169 +});
170 +
171 +describe("relayRetrySchedule", () => {
172 + it.effect("waits at least as long as the relay's Retry-After", () =>
173 + Effect.gen(function* () {
174 + let attempts = 0;
175 + const request = Effect.suspend(() => {
176 + attempts += 1;
177 + return attempts === 1
178 + ? Effect.fail(
179 + new RelayRequestError({
180 + rejection: "unavailable",
181 + description: "paused",
182 + retryAfterMillis: 120_000,
183 + }),
184 + )
185 + : Effect.succeed("done");
186 + });
187 + const fiber = yield* Effect.forkChild(
188 + Effect.retry(request, { schedule: relayRetrySchedule }),
189 + {
190 + startImmediately: true,
191 + },
192 + );
193 + yield* TestClock.adjust("90 seconds");
194 + expect(attempts).toBe(1);
195 + yield* TestClock.adjust("31 seconds");
196 + expect(yield* Fiber.join(fiber)).toBe("done");
197 + expect(attempts).toBe(2);
198 + }),
199 + );
200 +});
201 +
202 +describe("makeManagedTunnelRecovery", () => {
203 + const config = {
204 + providerKind: "cloudflare_tunnel" as const,
205 + connectorToken: "connector-token",
206 + tunnelId: "tunnel-1",
207 + };
208 +
209 + it.effect("retries an unavailable relay every cooldown, and cools down from when it ends", () =>
210 + Effect.gen(function* () {
211 + const attemptMinutes: Array<number> = [];
212 + const recover = makeManagedTunnelRecovery({
213 + recover: () =>
214 + Effect.gen(function* () {
215 + attemptMinutes.push((yield* Clock.currentTimeMillis) / 60_000);
216 + return attemptMinutes.length < 4
217 + ? yield* Effect.fail("relay unavailable" as const)
218 + : true;
219 + }),
220 + isRetryable: () => true,
221 + });
222 + const run = yield* recover;
223 + const fiber = yield* Effect.forkChild(run(config), { startImmediately: true });
224 + yield* TestClock.adjust("6 minutes");
225 + expect(yield* Fiber.join(fiber)).toBe(true);
226 + expect(attemptMinutes.map(Math.round)).toEqual([0, 2, 4, 6]);
227 +
228 + yield* TestClock.adjust("1 minute");
229 + const next = yield* Effect.forkChild(run(config), { startImmediately: true });
230 + yield* TestClock.adjust("10 minutes");
231 + yield* Fiber.join(next);
232 + expect(attemptMinutes.map(Math.round)).toEqual([0, 2, 4, 6, 10]);
233 + }),
234 + );
235 +});
236 +
237 +describe("retryManagedTunnelRegistration fallback", () => {
238 + it.effect("falls back by the end of the window and keeps honoring Retry-After", () =>
239 + Effect.gen(function* () {
240 + let fallbackAtMillis: number | undefined;
241 + const attemptMillis: Array<number> = [];
242 + const registration = Clock.currentTimeMillis.pipe(
243 + Effect.tap((now) => Effect.sync(() => attemptMillis.push(now))),
244 + Effect.andThen(
245 + Effect.fail(
246 + new RelayRequestError({
247 + rejection: "unavailable",
248 + description: "paused",
249 + retryAfterMillis: 900_000,
250 + }),
251 + ),
252 + ),
253 + );
254 + yield* Effect.forkChild(
255 + retryManagedTunnelRegistration(
256 + registration,
257 + () => true,
258 + Clock.currentTimeMillis.pipe(
259 + Effect.tap((now) =>
260 + Effect.sync(() => {
261 + fallbackAtMillis ??= now;
262 + }),
263 + ),
264 + Effect.asVoid,
265 + ),
266 + ),
267 + { startImmediately: true },
268 + );
269 + yield* TestClock.adjust("1 hour");
270 + expect(fallbackAtMillis).toBeDefined();
271 + expect(fallbackAtMillis).toBeLessThanOrEqual(10 * 60_000);
272 + expect(attemptMillis.length).toBeGreaterThan(2);
273 + for (let index = 1; index < attemptMillis.length; index++) {
274 + expect(attemptMillis[index]! - attemptMillis[index - 1]!).toBeGreaterThanOrEqual(900_000);
275 + }
276 + }),
277 + );
278 +});
279 +
280 +describe("retryDesiredLinkReconcile", () => {
281 + const relayUnavailable = new RelayRequestError({
282 + rejection: "unavailable",
283 + description: "relay unavailable",
284 + });
285 +
286 + it.effect("gives up on a local failure once the retry window ends", () =>
287 + Effect.gen(function* () {
288 + let attempts = 0;
289 + let windowLogs = 0;
290 + const fiber = yield* Effect.forkChild(
291 + Effect.flip(
292 + retryDesiredLinkReconcile(
293 + Effect.suspend(() => {
294 + attempts += 1;
295 + return Effect.fail("cloudflared cannot run" as const);
296 + }),
297 + () => true,
298 + () =>
299 + Effect.sync(() => {
300 + windowLogs += 1;
301 + }),
302 + ),
303 + ),
304 + { startImmediately: true },
305 + );
306 + yield* TestClock.adjust("2 hours");
307 + expect(yield* Fiber.join(fiber)).toBe("cloudflared cannot run");
308 + expect(attempts).toBeLessThan(20);
309 + expect(windowLogs).toBe(0);
310 + }),
311 + );
312 +
313 + it.effect("keeps relinking while the relay is down and succeeds when it returns", () =>
314 + Effect.gen(function* () {
315 + let windowLogs = 0;
316 + const reconcile = Effect.gen(function* () {
317 + return (yield* Clock.currentTimeMillis) < 30 * 60_000
318 + ? yield* Effect.fail(relayUnavailable)
319 + : "managed";
320 + });
321 + const fiber = yield* Effect.forkChild(
322 + retryDesiredLinkReconcile(
323 + reconcile,
324 + () => true,
325 + () =>
326 + Effect.sync(() => {
327 + windowLogs += 1;
328 + }),
329 + ),
330 + { startImmediately: true },
331 + );
332 + yield* TestClock.adjust("45 minutes");
333 + expect(yield* Fiber.join(fiber)).toBe("managed");
334 + expect(windowLogs).toBe(1);
335 + }),
336 + );
337 +});
apps/server/src/cloud/managedTunnelStartup.ts
+165 -24
@@ -1,7 +1,11 @@
1 import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2 +import * as Clock from "effect/Clock";
3 import * as Duration from "effect/Duration";
4 import * as Effect from "effect/Effect";
5 import * as Schedule from "effect/Schedule";
6 +import * as Semaphore from "effect/Semaphore";
7 +
8 +import { isRelayUnavailable, relayRetryAfterMillis } from "./relayResponse.ts";
9
10 // "revoked" leads to no action: relinking a deregistered environment is the
11 // user's call, so startup neither retries nor relinks it.
@@ -38,40 +42,177 @@ export function managedTunnelStartupAction(input: {
42
43 const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10);
44
45 +// Every host retries against the same relay, so while it is failing the
46 +// retries themselves are load. Backing off to minutes, and never sooner than
47 +// the relay's Retry-After, lets it recover instead of absorbing the storm.
48 +// Not HttpClient.retryTransient: it resends the identical request, and these
49 +// carry proofs that expire after 60 s or single-use link challenges, so each
50 +// retry has to re-run the whole operation.
51 +const RELAY_RETRY_MAX_DELAY = Duration.minutes(5);
52 +
53 +export const relayRetrySchedule = Schedule.exponential("1 second").pipe(
54 + Schedule.jittered,
55 + Schedule.modifyDelay(({ duration }) =>
56 + Effect.succeed(Duration.min(duration, RELAY_RETRY_MAX_DELAY)),
57 + ),
58 + Schedule.modifyDelay(({ duration, input }) => {
59 + const retryAfterMillis = relayRetryAfterMillis(input);
60 + return Effect.succeed(
61 + retryAfterMillis === undefined
62 + ? duration
63 + : Duration.max(duration, Duration.millis(retryAfterMillis)),
64 + );
65 + }),
66 +);
67 +
68 +const windowMillis = Duration.toMillis(MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW);
69 +
70 +/**
71 + * Retries on the relay schedule. The window closes at the first sleep that
72 + * would end past it: then `onWindowExhausted` runs, and retrying continues
73 + * only while `whilePastWindow` holds. One schedule covers both sides, so the
74 + * relay's Retry-After still holds across the handoff.
75 + */
76 +const retryAcrossWindow = <A, E, R>(
77 + effect: Effect.Effect<A, E, R>,
78 + options: {
79 + readonly while: (error: E) => boolean;
80 + readonly whilePastWindow: (error: E) => boolean;
81 + readonly onWindowExhausted: (error: E) => Effect.Effect<void, never, R>;
82 + },
83 +) =>
84 + Effect.suspend(() => {
85 + let pastWindow = false;
86 + return effect.pipe(
87 + Effect.retry({
88 + while: (error) => (pastWindow ? options.whilePastWindow(error) : options.while(error)),
89 + schedule: relayRetrySchedule.pipe(
90 + Schedule.setInputType<E>(),
91 + Schedule.modifyDelay(({ duration, elapsed, input }) => {
92 + if (pastWindow || elapsed + Duration.toMillis(duration) < windowMillis) {
93 + return Effect.succeed(duration);
94 + }
95 + pastWindow = true;
96 + return options.whilePastWindow(input)
97 + ? options.onWindowExhausted(input).pipe(Effect.as(duration))
98 + : Effect.fail(input);
99 + }),
100 + ),
101 + }),
102 + );
103 + });
104 +
105 export const retryManagedTunnelRegistration = <A, E, R>(
106 registration: Effect.Effect<A, E, R>,
107 isRetryable: (error: E) => boolean,
108 onRetryWindowExhausted?: Effect.Effect<void, never, R>,
45 -) => {
46 - const schedule = Schedule.exponential("1 second").pipe(
47 - Schedule.modifyDelay(({ duration }) =>
48 - Effect.succeed(Duration.min(duration, Duration.seconds(30))),
49 - ),
50 - Schedule.jittered,
51 - );
52 - const withinWindow = registration.pipe(
53 - Effect.retry({
54 - while: isRetryable,
55 - schedule: schedule.pipe(
56 - Schedule.upTo({ duration: MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW }),
57 - ),
58 - }),
59 - );
60 - if (onRetryWindowExhausted === undefined) return withinWindow;
61 - return withinWindow.pipe(
62 - Effect.catchIf(isRetryable, () =>
63 - onRetryWindowExhausted.pipe(
64 - Effect.andThen(registration.pipe(Effect.retry({ while: isRetryable, schedule }))),
65 - ),
66 - ),
67 - );
68 -};
109 +) =>
110 + retryAcrossWindow(registration, {
111 + while: isRetryable,
112 + whilePastWindow: onRetryWindowExhausted === undefined ? () => false : isRetryable,
113 + onWindowExhausted: () => onRetryWindowExhausted ?? Effect.void,
114 + });
115 +
116 +/**
117 + * Relinks a desired link at startup. Past the retry window it keeps going only
118 + * while the relay itself is unavailable: a headless host has nobody to restart
119 + * it once the relay recovers, but a local failure (cloudflared that cannot run)
120 + * would otherwise relink, and provision a tunnel, forever.
121 + */
122 +export const retryDesiredLinkReconcile = <A, E, R>(
123 + reconcile: Effect.Effect<A, E, R>,
124 + isRetryable: (error: E) => boolean,
125 + onRetryWindowExhausted: (error: E) => Effect.Effect<void, never, R>,
126 +) =>
127 + retryAcrossWindow(reconcile, {
128 + while: isRetryable,
129 + whilePastWindow: isRelayUnavailable,
130 + onWindowExhausted: onRetryWindowExhausted,
131 + });
132
133 // Every managed host shares one relay, so a host stuck in a bad loop must not
134 // turn into a fleet-wide request storm.
135 export const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2);
136
137 +// Retries inside one recovery, while the relay keeps failing, stay at least a
138 +// cooldown apart. They do not grow the cooldown: only a new request does.
139 +const managedTunnelRecoveryRetrySchedule = relayRetrySchedule.pipe(
140 + Schedule.modifyDelay(({ duration }) =>
141 + Effect.succeed(Duration.max(duration, MANAGED_TUNNEL_RECOVERY_COOLDOWN)),
142 + ),
143 +);
144 +const MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN = Duration.minutes(30);
145 +// A recovery that fixed the tunnel stops the runtime asking for another, so a
146 +// request this long after the cooldown means the previous one worked.
147 +const MANAGED_TUNNEL_RECOVERY_SETTLED_AFTER = Duration.minutes(10);
148 +
149 +/**
150 + * Recovery can only replace the relay-side tunnel. When the connector keeps
151 + * failing for local reasons (no network, a firewall), the runtime keeps asking
152 + * and recovery cannot help, so each request that follows right after a
153 + * recovery doubles the wait, up to 30 minutes.
154 + */
155 +export function nextManagedTunnelRecoveryCooldown(input: {
156 + readonly current: Duration.Duration;
157 + readonly sinceLastRecovery: Duration.Duration;
158 +}): Duration.Duration {
159 + const settled = Duration.sum(input.current, MANAGED_TUNNEL_RECOVERY_SETTLED_AFTER);
160 + if (Duration.isGreaterThanOrEqualTo(input.sinceLastRecovery, settled)) {
161 + return MANAGED_TUNNEL_RECOVERY_COOLDOWN;
162 + }
163 + return Duration.min(Duration.times(input.current, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN);
164 +}
165 +
166 // Existing hosts register on their first boot after an upgrade, and desktop
167 // auto-update delivers that boot to many hosts at once. Spread the first
168 // registration so the relay and Cloudflare see a ramp instead of a spike.
169 export const MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER = Duration.seconds(30);
170 +
171 +/**
172 + * Runs one recovery per request, one at a time: waits out the current
173 + * cooldown, then calls the relay, retrying it while it stays unavailable.
174 + */
175 +export const makeManagedTunnelRecovery = Effect.fn("ManagedTunnelRecovery.make")(function* <
176 + A,
177 + E,
178 + R,
179 +>(options: {
180 + readonly recover: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<A, E, R>;
181 + readonly isRetryable: (error: E) => boolean;
182 +}) {
183 + const lock = yield* Semaphore.make(1);
184 + let lastRecoveryAtMillis = Number.NEGATIVE_INFINITY;
185 + let cooldown = MANAGED_TUNNEL_RECOVERY_COOLDOWN;
186 + const waitForCooldown = Effect.gen(function* () {
187 + const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis;
188 + cooldown = nextManagedTunnelRecoveryCooldown({
189 + current: cooldown,
190 + sinceLastRecovery: Duration.millis(elapsed),
191 + });
192 + const wait = Duration.toMillis(cooldown) - elapsed;
193 + if (wait > 0) yield* Effect.sleep(Duration.millis(wait));
194 + });
195 + // Measured from the end: a recovery that retried through a long relay
196 + // outage must not look settled the moment it finishes.
197 + const recordRecovery = Clock.currentTimeMillis.pipe(
198 + Effect.tap((now) =>
199 + Effect.sync(() => {
200 + lastRecoveryAtMillis = now;
201 + }),
202 + ),
203 + );
204 + return (config: RelayManagedEndpointRuntimeConfig) =>
205 + lock.withPermits(1)(
206 + waitForCooldown.pipe(
207 + Effect.andThen(
208 + options.recover(config).pipe(
209 + Effect.retry({
210 + while: options.isRetryable,
211 + schedule: managedTunnelRecoveryRetrySchedule,
212 + }),
213 + Effect.ensuring(recordRecovery),
214 + ),
215 + ),
216 + ),
217 + );
218 +});
apps/server/src/cloud/relayResponse.test.ts
+22
@@ -6,6 +6,7 @@ import * as HttpClientError from "effect/http/HttpClientError";
6 import {
7 filterRelayResponse,
8 relayRequestError,
9 + relayRetryAfterMillis,
10 shouldRetryRelayRequest,
11 } from "./relayResponse.ts";
12
@@ -121,6 +122,27 @@ it.effect("reports an unrecognized access denial without printing its response b
122 }),
123 );
124
125 +it.effect("keeps the relay's Retry-After on a retryable failure", () =>
126 + Effect.gen(function* () {
127 + const seconds = yield* filterRelayResponse(
128 + response(503, "paused", { "retry-after": "30" }),
129 + ).pipe(Effect.flip);
130 + expect(seconds).toMatchObject({ rejection: "unavailable", retryAfterMillis: 30_000 });
131 + expect(relayRetryAfterMillis({ cause: { cause: seconds } })).toBe(30_000);
132 +
133 + // The test clock starts at the epoch, so the date is 90 seconds after it.
134 + const date = yield* filterRelayResponse(
135 + response(429, "slow down", { "retry-after": "Thu, 01 Jan 1970 00:01:30 GMT" }),
136 + ).pipe(Effect.flip);
137 + expect(date.retryAfterMillis).toBe(90_000);
138 +
139 + const malformed = yield* filterRelayResponse(
140 + response(503, "paused", { "retry-after": "soon" }),
141 + ).pipe(Effect.flip);
142 + expect(malformed.retryAfterMillis).toBeUndefined();
143 + }),
144 +);
145 +
146 it.effect.each([408, 429, 500, 502, 503, 504])(
147 "keeps transient HTTP %s failures retryable",
148 (status) =>
apps/server/src/cloud/relayResponse.ts
+36 -1
@@ -1,4 +1,5 @@
1 import { RelayProtectedError } from "@t3tools/contracts/relay";
2 +import * as Clock from "effect/Clock";
3 import * as Effect from "effect/Effect";
4 import * as Option from "effect/Option";
5 import * as Schema from "effect/Schema";
@@ -15,6 +16,8 @@ export class RelayRequestError extends Schema.TaggedError<RelayRequestError>()(
16 {
17 rejection: Schema.Literals(["unauthorized", "forbidden", "rejected", "unavailable"]),
18 description: Schema.String,
19 + /** How long the relay asked callers to wait (`Retry-After`) before retrying. */
20 + retryAfterMillis: Schema.optionalKey(Schema.Number),
21 },
22 ) {
23 override get message(): string {
@@ -36,6 +39,30 @@ export function relayRequestError(cause: unknown): RelayRequestError {
39 export const shouldRetryRelayRequest = (error: unknown): boolean =>
40 !isRelayRequestError(error) || error.rejection === "unavailable";
41
42 +/** The relay answered and asked to be retried later; nothing local is wrong. */
43 +export const isRelayUnavailable = (error: unknown): boolean =>
44 + isRelayRequestError(error) && error.rejection === "unavailable";
45 +
46 +// Retry-After is either delay-seconds or an HTTP date (RFC 9110 §10.2.3).
47 +const MAX_RELAY_RETRY_AFTER_MILLIS = 15 * 60 * 1000;
48 +function parseRetryAfterMillis(value: string | undefined, nowMillis: number): number | undefined {
49 + if (value === undefined) return undefined;
50 + const trimmed = value.trim();
51 + const millis = /^\d+$/.test(trimmed) ? Number(trimmed) * 1000 : Date.parse(trimmed) - nowMillis;
52 + if (!Number.isFinite(millis) || millis <= 0) return undefined;
53 + return Math.min(millis, MAX_RELAY_RETRY_AFTER_MILLIS);
54 +}
55 +
56 +/** The relay's requested wait, found anywhere in a failure's cause chain. */
57 +export function relayRetryAfterMillis(error: unknown): number | undefined {
58 + for (let current = error, depth = 0; depth < 8; depth++) {
59 + if (isRelayRequestError(current)) return current.retryAfterMillis;
60 + if (typeof current !== "object" || current === null || !("cause" in current)) return undefined;
61 + current = current.cause;
62 + }
63 + return undefined;
64 +}
65 +
66 function recoveryHint(error: RelayProtectedError): string {
67 switch (error._tag) {
68 case "RelayEnvironmentLinkLimitExceededError":
@@ -88,5 +115,13 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func
115 ) {
116 return yield* new RelayRequestError({ rejection: "rejected", description });
117 }
91 - return yield* new RelayRequestError({ rejection: "unavailable", description });
118 + const retryAfterMillis = parseRetryAfterMillis(
119 + response.headers["retry-after"],
120 + yield* Clock.currentTimeMillis,
121 + );
122 + return yield* new RelayRequestError({
123 + rejection: "unavailable",
124 + description,
125 + ...(retryAfterMillis === undefined ? {} : { retryAfterMillis }),
126 + });
127 });
apps/server/src/server.ts
+26 -46
@@ -1,8 +1,6 @@
1 import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers";
2 import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
3 -import * as Clock from "effect/Clock";
3 import * as Random from "effect/Random";
5 -import * as Semaphore from "effect/Semaphore";
4 import * as DatabaseStorage from "./persistence/DatabaseStorage.ts";
5 import * as StorageCleanup from "./storageCleanup.ts";
6 import * as PullRequestSyncReactor from "./orchestration-v2/PullRequestSyncReactor.ts";
@@ -22,7 +20,6 @@ import * as Effect from "effect/Effect";
20 import * as Option from "effect/Option";
21 import * as Layer from "effect/Layer";
22 import * as Stream from "effect/Stream";
25 -import * as Schedule from "effect/Schedule";
23 import { FetchHttpClient, HttpRouter, HttpServer } from "effect/http";
24 import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
25
@@ -148,8 +145,9 @@ import * as RelayTracing from "./cloud/relayTracing.ts";
145 import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";
146 import {
147 MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER,
151 - MANAGED_TUNNEL_RECOVERY_COOLDOWN,
148 + makeManagedTunnelRecovery,
149 managedTunnelStartupAction,
150 + retryDesiredLinkReconcile,
151 retryManagedTunnelRegistration,
152 } from "./cloud/managedTunnelStartup.ts";
153 import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";
@@ -855,42 +853,26 @@ const layerMakeServer = Layer.unwrap(
853 if (typeof address === "string" || !("port" in address)) return;
854 const localOrigin = `http://127.0.0.1:${address.port}`;
855 const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime;
858 - const recoveryLock = yield* Semaphore.make(1);
859 - let lastRecoveryAtMillis = 0;
856 + const runManagedTunnelRecovery = yield* makeManagedTunnelRecovery({
857 + recover: (config: RelayManagedEndpointRuntimeConfig) =>
858 + cloudLink.recoverManagedTunnel(localOrigin, config, {
859 + retryRuntimeFailures: true,
860 + }),
861 + isRetryable: (error) =>
862 + CloudLink.shouldRetryCloudLink(error) &&
863 + error._tag !== "CloudLinkEndpointUnavailableError",
864 + });
865 const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) =>
861 - recoveryLock.withPermits(1)(
862 - Effect.gen(function* () {
863 - const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis;
864 - const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsed;
865 - if (wait > 0) yield* Effect.sleep(Duration.millis(wait));
866 - lastRecoveryAtMillis = yield* Clock.currentTimeMillis;
867 - }).pipe(
868 - Effect.andThen(
869 - cloudLink.recoverManagedTunnel(localOrigin, config, {
870 - retryRuntimeFailures: true,
871 - }),
872 - ),
873 - Effect.retry({
874 - while: (error) =>
875 - CloudLink.shouldRetryCloudLink(error) &&
876 - error._tag !== "CloudLinkEndpointUnavailableError",
877 - schedule: Schedule.exponential("1 second").pipe(
878 - Schedule.modifyDelay(({ duration }) =>
879 - Effect.succeed(Duration.min(duration, Duration.seconds(30))),
880 - ),
881 - Schedule.jittered,
882 - ),
883 - }),
884 - Effect.tap((recovered) =>
885 - recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void,
886 - ),
887 - Effect.catchCause((cause) =>
888 - Cause.hasInterrupts(cause)
889 - ? Effect.interrupt
890 - : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", {
891 - cause,
892 - }),
893 - ),
866 + runManagedTunnelRecovery(config).pipe(
867 + Effect.tap((recovered) =>
868 + recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void,
869 + ),
870 + Effect.catchCause((cause) =>
871 + Cause.hasInterrupts(cause)
872 + ? Effect.interrupt
873 + : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", {
874 + cause,
875 + }),
876 ),
877 );
878 yield* endpointRuntime.recoveryRequests.pipe(
@@ -992,15 +974,13 @@ const layerMakeServer = Layer.unwrap(
974 const reconciledMode = yield* cloudLink
975 .reconcileDesiredLinkIfStillDesired(localOrigin)
976 .pipe(
995 - Effect.retry({
996 - while: CloudLink.shouldRetryCloudLink,
997 - schedule: Schedule.exponential("1 second").pipe(
998 - Schedule.modifyDelay(({ duration }) =>
999 - Effect.succeed(Duration.min(duration, Duration.seconds(30))),
977 + (reconcile) =>
978 + retryDesiredLinkReconcile(reconcile, CloudLink.shouldRetryCloudLink, (cause) =>
979 + Effect.logWarning(
980 + "T3 Connect relay is still unavailable; retrying the desired link in the background",
981 + { cause },
982 ),
1001 - Schedule.upTo({ duration: "10 minutes" }),
983 ),
1003 - }),
984 Effect.tap((mode) =>
985 mode === null
986 ? Effect.void