The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 23:15 UTC
fix(server): T3 Connect backs off from a failing relay instead of hammering it (#22108)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 19, 2026 at 21:22 UTC
cd8a98657b997198e4f68f7d7aef40b695fedc4c
5 files changed
+455
-71
apps/server/src/cloud/managedTunnelStartup.test.ts
+206
@@ -1,13 +1,21 @@
1
import { describe, expect, it } from "@effect/vitest";
2
+import * as Clock from "effect/Clock";
3
import * as Deferred from "effect/Deferred";
4
+import * as Duration from "effect/Duration";
5
import * as Effect from "effect/Effect";
6
import * as Fiber from "effect/Fiber";
7
import * as TestClock from "effect/testing/TestClock";
8
9
import {
10
+ MANAGED_TUNNEL_RECOVERY_COOLDOWN,
11
+ makeManagedTunnelRecovery,
12
managedTunnelStartupAction,
13
+ nextManagedTunnelRecoveryCooldown,
14
+ relayRetrySchedule,
15
+ retryDesiredLinkReconcile,
16
retryManagedTunnelRegistration,
17
} from "./managedTunnelStartup.ts";
18
+import { RelayRequestError } from "./relayResponse.ts";
19
20
describe("managedTunnelStartupAction", () => {
21
const config = {
@@ -129,3 +137,201 @@ describe("retryManagedTunnelRegistration", () => {
137
}),
138
);
139
});
140
+
141
+describe("nextManagedTunnelRecoveryCooldown", () => {
142
+ const minutes = (value: number) => Duration.minutes(value);
143
+
144
+ it("backs off while recoveries keep failing to fix the tunnel, up to 30 minutes", () => {
145
+ let cooldown = MANAGED_TUNNEL_RECOVERY_COOLDOWN;
146
+ const seen: Array<number> = [];
147
+ for (let attempt = 0; attempt < 6; attempt++) {
148
+ cooldown = nextManagedTunnelRecoveryCooldown({
149
+ current: cooldown,
150
+ sinceLastRecovery: minutes(3),
151
+ });
152
+ seen.push(Duration.toMinutes(cooldown));
153
+ }
154
+ expect(seen).toEqual([4, 8, 16, 30, 30, 30]);
155
+ });
156
+
157
+ it("starts over once the previous recovery held", () => {
158
+ expect(
159
+ Duration.toMinutes(
160
+ nextManagedTunnelRecoveryCooldown({ current: minutes(16), sinceLastRecovery: minutes(26) }),
161
+ ),
162
+ ).toBe(2);
163
+ expect(
164
+ Duration.toMinutes(
165
+ nextManagedTunnelRecoveryCooldown({ current: minutes(16), sinceLastRecovery: minutes(20) }),
166
+ ),
167
+ ).toBe(30);
168
+ });
169
+});
170
+
171
+describe("relayRetrySchedule", () => {
172
+ it.effect("waits at least as long as the relay's Retry-After", () =>
173
+ Effect.gen(function* () {
174
+ let attempts = 0;
175
+ const request = Effect.suspend(() => {
176
+ attempts += 1;
177
+ return attempts === 1
178
+ ? Effect.fail(
179
+ new RelayRequestError({
180
+ rejection: "unavailable",
181
+ description: "paused",
182
+ retryAfterMillis: 120_000,
183
+ }),
184
+ )
185
+ : Effect.succeed("done");
186
+ });
187
+ const fiber = yield* Effect.forkChild(
188
+ Effect.retry(request, { schedule: relayRetrySchedule }),
189
+ {
190
+ startImmediately: true,
191
+ },
192
+ );
193
+ yield* TestClock.adjust("90 seconds");
194
+ expect(attempts).toBe(1);
195
+ yield* TestClock.adjust("31 seconds");
196
+ expect(yield* Fiber.join(fiber)).toBe("done");
197
+ expect(attempts).toBe(2);
198
+ }),
199
+ );
200
+});
201
+
202
+describe("makeManagedTunnelRecovery", () => {
203
+ const config = {
204
+ providerKind: "cloudflare_tunnel" as const,
205
+ connectorToken: "connector-token",
206
+ tunnelId: "tunnel-1",
207
+ };
208
+
209
+ it.effect("retries an unavailable relay every cooldown, and cools down from when it ends", () =>
210
+ Effect.gen(function* () {
211
+ const attemptMinutes: Array<number> = [];
212
+ const recover = makeManagedTunnelRecovery({
213
+ recover: () =>
214
+ Effect.gen(function* () {
215
+ attemptMinutes.push((yield* Clock.currentTimeMillis) / 60_000);
216
+ return attemptMinutes.length < 4
217
+ ? yield* Effect.fail("relay unavailable" as const)
218
+ : true;
219
+ }),
220
+ isRetryable: () => true,
221
+ });
222
+ const run = yield* recover;
223
+ const fiber = yield* Effect.forkChild(run(config), { startImmediately: true });
224
+ yield* TestClock.adjust("6 minutes");
225
+ expect(yield* Fiber.join(fiber)).toBe(true);
226
+ expect(attemptMinutes.map(Math.round)).toEqual([0, 2, 4, 6]);
227
+
228
+ yield* TestClock.adjust("1 minute");
229
+ const next = yield* Effect.forkChild(run(config), { startImmediately: true });
230
+ yield* TestClock.adjust("10 minutes");
231
+ yield* Fiber.join(next);
232
+ expect(attemptMinutes.map(Math.round)).toEqual([0, 2, 4, 6, 10]);
233
+ }),
234
+ );
235
+});
236
+
237
+describe("retryManagedTunnelRegistration fallback", () => {
238
+ it.effect("falls back by the end of the window and keeps honoring Retry-After", () =>
239
+ Effect.gen(function* () {
240
+ let fallbackAtMillis: number | undefined;
241
+ const attemptMillis: Array<number> = [];
242
+ const registration = Clock.currentTimeMillis.pipe(
243
+ Effect.tap((now) => Effect.sync(() => attemptMillis.push(now))),
244
+ Effect.andThen(
245
+ Effect.fail(
246
+ new RelayRequestError({
247
+ rejection: "unavailable",
248
+ description: "paused",
249
+ retryAfterMillis: 900_000,
250
+ }),
251
+ ),
252
+ ),
253
+ );
254
+ yield* Effect.forkChild(
255
+ retryManagedTunnelRegistration(
256
+ registration,
257
+ () => true,
258
+ Clock.currentTimeMillis.pipe(
259
+ Effect.tap((now) =>
260
+ Effect.sync(() => {
261
+ fallbackAtMillis ??= now;
262
+ }),
263
+ ),
264
+ Effect.asVoid,
265
+ ),
266
+ ),
267
+ { startImmediately: true },
268
+ );
269
+ yield* TestClock.adjust("1 hour");
270
+ expect(fallbackAtMillis).toBeDefined();
271
+ expect(fallbackAtMillis).toBeLessThanOrEqual(10 * 60_000);
272
+ expect(attemptMillis.length).toBeGreaterThan(2);
273
+ for (let index = 1; index < attemptMillis.length; index++) {
274
+ expect(attemptMillis[index]! - attemptMillis[index - 1]!).toBeGreaterThanOrEqual(900_000);
275
+ }
276
+ }),
277
+ );
278
+});
279
+
280
+describe("retryDesiredLinkReconcile", () => {
281
+ const relayUnavailable = new RelayRequestError({
282
+ rejection: "unavailable",
283
+ description: "relay unavailable",
284
+ });
285
+
286
+ it.effect("gives up on a local failure once the retry window ends", () =>
287
+ Effect.gen(function* () {
288
+ let attempts = 0;
289
+ let windowLogs = 0;
290
+ const fiber = yield* Effect.forkChild(
291
+ Effect.flip(
292
+ retryDesiredLinkReconcile(
293
+ Effect.suspend(() => {
294
+ attempts += 1;
295
+ return Effect.fail("cloudflared cannot run" as const);
296
+ }),
297
+ () => true,
298
+ () =>
299
+ Effect.sync(() => {
300
+ windowLogs += 1;
301
+ }),
302
+ ),
303
+ ),
304
+ { startImmediately: true },
305
+ );
306
+ yield* TestClock.adjust("2 hours");
307
+ expect(yield* Fiber.join(fiber)).toBe("cloudflared cannot run");
308
+ expect(attempts).toBeLessThan(20);
309
+ expect(windowLogs).toBe(0);
310
+ }),
311
+ );
312
+
313
+ it.effect("keeps relinking while the relay is down and succeeds when it returns", () =>
314
+ Effect.gen(function* () {
315
+ let windowLogs = 0;
316
+ const reconcile = Effect.gen(function* () {
317
+ return (yield* Clock.currentTimeMillis) < 30 * 60_000
318
+ ? yield* Effect.fail(relayUnavailable)
319
+ : "managed";
320
+ });
321
+ const fiber = yield* Effect.forkChild(
322
+ retryDesiredLinkReconcile(
323
+ reconcile,
324
+ () => true,
325
+ () =>
326
+ Effect.sync(() => {
327
+ windowLogs += 1;
328
+ }),
329
+ ),
330
+ { startImmediately: true },
331
+ );
332
+ yield* TestClock.adjust("45 minutes");
333
+ expect(yield* Fiber.join(fiber)).toBe("managed");
334
+ expect(windowLogs).toBe(1);
335
+ }),
336
+ );
337
+});
apps/server/src/cloud/managedTunnelStartup.ts
+165
-24
@@ -1,7 +1,11 @@
1
import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2
+import * as Clock from "effect/Clock";
3
import * as Duration from "effect/Duration";
4
import * as Effect from "effect/Effect";
5
import * as Schedule from "effect/Schedule";
6
+import * as Semaphore from "effect/Semaphore";
7
+
8
+import { isRelayUnavailable, relayRetryAfterMillis } from "./relayResponse.ts";
9
10
// "revoked" leads to no action: relinking a deregistered environment is the
11
// user's call, so startup neither retries nor relinks it.
@@ -38,40 +42,177 @@ export function managedTunnelStartupAction(input: {
42
43
const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10);
44
45
+// Every host retries against the same relay, so while it is failing the
46
+// retries themselves are load. Backing off to minutes, and never sooner than
47
+// the relay's Retry-After, lets it recover instead of absorbing the storm.
48
+// Not HttpClient.retryTransient: it resends the identical request, and these
49
+// carry proofs that expire after 60 s or single-use link challenges, so each
50
+// retry has to re-run the whole operation.
51
+const RELAY_RETRY_MAX_DELAY = Duration.minutes(5);
52
+
53
+export const relayRetrySchedule = Schedule.exponential("1 second").pipe(
54
+ Schedule.jittered,
55
+ Schedule.modifyDelay(({ duration }) =>
56
+ Effect.succeed(Duration.min(duration, RELAY_RETRY_MAX_DELAY)),
57
+ ),
58
+ Schedule.modifyDelay(({ duration, input }) => {
59
+ const retryAfterMillis = relayRetryAfterMillis(input);
60
+ return Effect.succeed(
61
+ retryAfterMillis === undefined
62
+ ? duration
63
+ : Duration.max(duration, Duration.millis(retryAfterMillis)),
64
+ );
65
+ }),
66
+);
67
+
68
+const windowMillis = Duration.toMillis(MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW);
69
+
70
+/**
71
+ * Retries on the relay schedule. The window closes at the first sleep that
72
+ * would end past it: then `onWindowExhausted` runs, and retrying continues
73
+ * only while `whilePastWindow` holds. One schedule covers both sides, so the
74
+ * relay's Retry-After still holds across the handoff.
75
+ */
76
+const retryAcrossWindow = <A, E, R>(
77
+ effect: Effect.Effect<A, E, R>,
78
+ options: {
79
+ readonly while: (error: E) => boolean;
80
+ readonly whilePastWindow: (error: E) => boolean;
81
+ readonly onWindowExhausted: (error: E) => Effect.Effect<void, never, R>;
82
+ },
83
+) =>
84
+ Effect.suspend(() => {
85
+ let pastWindow = false;
86
+ return effect.pipe(
87
+ Effect.retry({
88
+ while: (error) => (pastWindow ? options.whilePastWindow(error) : options.while(error)),
89
+ schedule: relayRetrySchedule.pipe(
90
+ Schedule.setInputType<E>(),
91
+ Schedule.modifyDelay(({ duration, elapsed, input }) => {
92
+ if (pastWindow || elapsed + Duration.toMillis(duration) < windowMillis) {
93
+ return Effect.succeed(duration);
94
+ }
95
+ pastWindow = true;
96
+ return options.whilePastWindow(input)
97
+ ? options.onWindowExhausted(input).pipe(Effect.as(duration))
98
+ : Effect.fail(input);
99
+ }),
100
+ ),
101
+ }),
102
+ );
103
+ });
104
+
105
export const retryManagedTunnelRegistration = <A, E, R>(
106
registration: Effect.Effect<A, E, R>,
107
isRetryable: (error: E) => boolean,
108
onRetryWindowExhausted?: Effect.Effect<void, never, R>,
45
-) => {
46
- const schedule = Schedule.exponential("1 second").pipe(
47
- Schedule.modifyDelay(({ duration }) =>
48
- Effect.succeed(Duration.min(duration, Duration.seconds(30))),
49
- ),
50
- Schedule.jittered,
51
- );
52
- const withinWindow = registration.pipe(
53
- Effect.retry({
54
- while: isRetryable,
55
- schedule: schedule.pipe(
56
- Schedule.upTo({ duration: MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW }),
57
- ),
58
- }),
59
- );
60
- if (onRetryWindowExhausted === undefined) return withinWindow;
61
- return withinWindow.pipe(
62
- Effect.catchIf(isRetryable, () =>
63
- onRetryWindowExhausted.pipe(
64
- Effect.andThen(registration.pipe(Effect.retry({ while: isRetryable, schedule }))),
65
- ),
66
- ),
67
- );
68
-};
109
+) =>
110
+ retryAcrossWindow(registration, {
111
+ while: isRetryable,
112
+ whilePastWindow: onRetryWindowExhausted === undefined ? () => false : isRetryable,
113
+ onWindowExhausted: () => onRetryWindowExhausted ?? Effect.void,
114
+ });
115
+
116
+/**
117
+ * Relinks a desired link at startup. Past the retry window it keeps going only
118
+ * while the relay itself is unavailable: a headless host has nobody to restart
119
+ * it once the relay recovers, but a local failure (cloudflared that cannot run)
120
+ * would otherwise relink, and provision a tunnel, forever.
121
+ */
122
+export const retryDesiredLinkReconcile = <A, E, R>(
123
+ reconcile: Effect.Effect<A, E, R>,
124
+ isRetryable: (error: E) => boolean,
125
+ onRetryWindowExhausted: (error: E) => Effect.Effect<void, never, R>,
126
+) =>
127
+ retryAcrossWindow(reconcile, {
128
+ while: isRetryable,
129
+ whilePastWindow: isRelayUnavailable,
130
+ onWindowExhausted: onRetryWindowExhausted,
131
+ });
132
133
// Every managed host shares one relay, so a host stuck in a bad loop must not
134
// turn into a fleet-wide request storm.
135
export const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2);
136
137
+// Retries inside one recovery, while the relay keeps failing, stay at least a
138
+// cooldown apart. They do not grow the cooldown: only a new request does.
139
+const managedTunnelRecoveryRetrySchedule = relayRetrySchedule.pipe(
140
+ Schedule.modifyDelay(({ duration }) =>
141
+ Effect.succeed(Duration.max(duration, MANAGED_TUNNEL_RECOVERY_COOLDOWN)),
142
+ ),
143
+);
144
+const MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN = Duration.minutes(30);
145
+// A recovery that fixed the tunnel stops the runtime asking for another, so a
146
+// request this long after the cooldown means the previous one worked.
147
+const MANAGED_TUNNEL_RECOVERY_SETTLED_AFTER = Duration.minutes(10);
148
+
149
+/**
150
+ * Recovery can only replace the relay-side tunnel. When the connector keeps
151
+ * failing for local reasons (no network, a firewall), the runtime keeps asking
152
+ * and recovery cannot help, so each request that follows right after a
153
+ * recovery doubles the wait, up to 30 minutes.
154
+ */
155
+export function nextManagedTunnelRecoveryCooldown(input: {
156
+ readonly current: Duration.Duration;
157
+ readonly sinceLastRecovery: Duration.Duration;
158
+}): Duration.Duration {
159
+ const settled = Duration.sum(input.current, MANAGED_TUNNEL_RECOVERY_SETTLED_AFTER);
160
+ if (Duration.isGreaterThanOrEqualTo(input.sinceLastRecovery, settled)) {
161
+ return MANAGED_TUNNEL_RECOVERY_COOLDOWN;
162
+ }
163
+ return Duration.min(Duration.times(input.current, 2), MANAGED_TUNNEL_RECOVERY_MAX_COOLDOWN);
164
+}
165
+
166
// Existing hosts register on their first boot after an upgrade, and desktop
167
// auto-update delivers that boot to many hosts at once. Spread the first
168
// registration so the relay and Cloudflare see a ramp instead of a spike.
169
export const MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER = Duration.seconds(30);
170
+
171
+/**
172
+ * Runs one recovery per request, one at a time: waits out the current
173
+ * cooldown, then calls the relay, retrying it while it stays unavailable.
174
+ */
175
+export const makeManagedTunnelRecovery = Effect.fn("ManagedTunnelRecovery.make")(function* <
176
+ A,
177
+ E,
178
+ R,
179
+>(options: {
180
+ readonly recover: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<A, E, R>;
181
+ readonly isRetryable: (error: E) => boolean;
182
+}) {
183
+ const lock = yield* Semaphore.make(1);
184
+ let lastRecoveryAtMillis = Number.NEGATIVE_INFINITY;
185
+ let cooldown = MANAGED_TUNNEL_RECOVERY_COOLDOWN;
186
+ const waitForCooldown = Effect.gen(function* () {
187
+ const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis;
188
+ cooldown = nextManagedTunnelRecoveryCooldown({
189
+ current: cooldown,
190
+ sinceLastRecovery: Duration.millis(elapsed),
191
+ });
192
+ const wait = Duration.toMillis(cooldown) - elapsed;
193
+ if (wait > 0) yield* Effect.sleep(Duration.millis(wait));
194
+ });
195
+ // Measured from the end: a recovery that retried through a long relay
196
+ // outage must not look settled the moment it finishes.
197
+ const recordRecovery = Clock.currentTimeMillis.pipe(
198
+ Effect.tap((now) =>
199
+ Effect.sync(() => {
200
+ lastRecoveryAtMillis = now;
201
+ }),
202
+ ),
203
+ );
204
+ return (config: RelayManagedEndpointRuntimeConfig) =>
205
+ lock.withPermits(1)(
206
+ waitForCooldown.pipe(
207
+ Effect.andThen(
208
+ options.recover(config).pipe(
209
+ Effect.retry({
210
+ while: options.isRetryable,
211
+ schedule: managedTunnelRecoveryRetrySchedule,
212
+ }),
213
+ Effect.ensuring(recordRecovery),
214
+ ),
215
+ ),
216
+ ),
217
+ );
218
+});
apps/server/src/cloud/relayResponse.test.ts
+22
@@ -6,6 +6,7 @@ import * as HttpClientError from "effect/http/HttpClientError";
6
import {
7
filterRelayResponse,
8
relayRequestError,
9
+ relayRetryAfterMillis,
10
shouldRetryRelayRequest,
11
} from "./relayResponse.ts";
12
@@ -121,6 +122,27 @@ it.effect("reports an unrecognized access denial without printing its response b
122
}),
123
);
124
125
+it.effect("keeps the relay's Retry-After on a retryable failure", () =>
126
+ Effect.gen(function* () {
127
+ const seconds = yield* filterRelayResponse(
128
+ response(503, "paused", { "retry-after": "30" }),
129
+ ).pipe(Effect.flip);
130
+ expect(seconds).toMatchObject({ rejection: "unavailable", retryAfterMillis: 30_000 });
131
+ expect(relayRetryAfterMillis({ cause: { cause: seconds } })).toBe(30_000);
132
+
133
+ // The test clock starts at the epoch, so the date is 90 seconds after it.
134
+ const date = yield* filterRelayResponse(
135
+ response(429, "slow down", { "retry-after": "Thu, 01 Jan 1970 00:01:30 GMT" }),
136
+ ).pipe(Effect.flip);
137
+ expect(date.retryAfterMillis).toBe(90_000);
138
+
139
+ const malformed = yield* filterRelayResponse(
140
+ response(503, "paused", { "retry-after": "soon" }),
141
+ ).pipe(Effect.flip);
142
+ expect(malformed.retryAfterMillis).toBeUndefined();
143
+ }),
144
+);
145
+
146
it.effect.each([408, 429, 500, 502, 503, 504])(
147
"keeps transient HTTP %s failures retryable",
148
(status) =>
apps/server/src/cloud/relayResponse.ts
+36
-1
@@ -1,4 +1,5 @@
1
import { RelayProtectedError } from "@t3tools/contracts/relay";
2
+import * as Clock from "effect/Clock";
3
import * as Effect from "effect/Effect";
4
import * as Option from "effect/Option";
5
import * as Schema from "effect/Schema";
@@ -15,6 +16,8 @@ export class RelayRequestError extends Schema.TaggedError<RelayRequestError>()(
16
{
17
rejection: Schema.Literals(["unauthorized", "forbidden", "rejected", "unavailable"]),
18
description: Schema.String,
19
+ /** How long the relay asked callers to wait (`Retry-After`) before retrying. */
20
+ retryAfterMillis: Schema.optionalKey(Schema.Number),
21
},
22
) {
23
override get message(): string {
@@ -36,6 +39,30 @@ export function relayRequestError(cause: unknown): RelayRequestError {
39
export const shouldRetryRelayRequest = (error: unknown): boolean =>
40
!isRelayRequestError(error) || error.rejection === "unavailable";
41
42
+/** The relay answered and asked to be retried later; nothing local is wrong. */
43
+export const isRelayUnavailable = (error: unknown): boolean =>
44
+ isRelayRequestError(error) && error.rejection === "unavailable";
45
+
46
+// Retry-After is either delay-seconds or an HTTP date (RFC 9110 §10.2.3).
47
+const MAX_RELAY_RETRY_AFTER_MILLIS = 15 * 60 * 1000;
48
+function parseRetryAfterMillis(value: string | undefined, nowMillis: number): number | undefined {
49
+ if (value === undefined) return undefined;
50
+ const trimmed = value.trim();
51
+ const millis = /^\d+$/.test(trimmed) ? Number(trimmed) * 1000 : Date.parse(trimmed) - nowMillis;
52
+ if (!Number.isFinite(millis) || millis <= 0) return undefined;
53
+ return Math.min(millis, MAX_RELAY_RETRY_AFTER_MILLIS);
54
+}
55
+
56
+/** The relay's requested wait, found anywhere in a failure's cause chain. */
57
+export function relayRetryAfterMillis(error: unknown): number | undefined {
58
+ for (let current = error, depth = 0; depth < 8; depth++) {
59
+ if (isRelayRequestError(current)) return current.retryAfterMillis;
60
+ if (typeof current !== "object" || current === null || !("cause" in current)) return undefined;
61
+ current = current.cause;
62
+ }
63
+ return undefined;
64
+}
65
+
66
function recoveryHint(error: RelayProtectedError): string {
67
switch (error._tag) {
68
case "RelayEnvironmentLinkLimitExceededError":
@@ -88,5 +115,13 @@ export const filterRelayResponse = Effect.fn("cloud.filter_relay_response")(func
115
) {
116
return yield* new RelayRequestError({ rejection: "rejected", description });
117
}
91
- return yield* new RelayRequestError({ rejection: "unavailable", description });
118
+ const retryAfterMillis = parseRetryAfterMillis(
119
+ response.headers["retry-after"],
120
+ yield* Clock.currentTimeMillis,
121
+ );
122
+ return yield* new RelayRequestError({
123
+ rejection: "unavailable",
124
+ description,
125
+ ...(retryAfterMillis === undefined ? {} : { retryAfterMillis }),
126
+ });
127
});
apps/server/src/server.ts
+26
-46
@@ -1,8 +1,6 @@
1
import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers";
2
import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
3
-import * as Clock from "effect/Clock";
3
import * as Random from "effect/Random";
5
-import * as Semaphore from "effect/Semaphore";
4
import * as DatabaseStorage from "./persistence/DatabaseStorage.ts";
5
import * as StorageCleanup from "./storageCleanup.ts";
6
import * as PullRequestSyncReactor from "./orchestration-v2/PullRequestSyncReactor.ts";
@@ -22,7 +20,6 @@ import * as Effect from "effect/Effect";
20
import * as Option from "effect/Option";
21
import * as Layer from "effect/Layer";
22
import * as Stream from "effect/Stream";
25
-import * as Schedule from "effect/Schedule";
23
import { FetchHttpClient, HttpRouter, HttpServer } from "effect/http";
24
import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
25
@@ -148,8 +145,9 @@ import * as RelayTracing from "./cloud/relayTracing.ts";
145
import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts";
146
import {
147
MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER,
151
- MANAGED_TUNNEL_RECOVERY_COOLDOWN,
148
+ makeManagedTunnelRecovery,
149
managedTunnelStartupAction,
150
+ retryDesiredLinkReconcile,
151
retryManagedTunnelRegistration,
152
} from "./cloud/managedTunnelStartup.ts";
153
import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts";
@@ -855,42 +853,26 @@ const layerMakeServer = Layer.unwrap(
853
if (typeof address === "string" || !("port" in address)) return;
854
const localOrigin = `http://127.0.0.1:${address.port}`;
855
const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime;
858
- const recoveryLock = yield* Semaphore.make(1);
859
- let lastRecoveryAtMillis = 0;
856
+ const runManagedTunnelRecovery = yield* makeManagedTunnelRecovery({
857
+ recover: (config: RelayManagedEndpointRuntimeConfig) =>
858
+ cloudLink.recoverManagedTunnel(localOrigin, config, {
859
+ retryRuntimeFailures: true,
860
+ }),
861
+ isRetryable: (error) =>
862
+ CloudLink.shouldRetryCloudLink(error) &&
863
+ error._tag !== "CloudLinkEndpointUnavailableError",
864
+ });
865
const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) =>
861
- recoveryLock.withPermits(1)(
862
- Effect.gen(function* () {
863
- const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis;
864
- const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsed;
865
- if (wait > 0) yield* Effect.sleep(Duration.millis(wait));
866
- lastRecoveryAtMillis = yield* Clock.currentTimeMillis;
867
- }).pipe(
868
- Effect.andThen(
869
- cloudLink.recoverManagedTunnel(localOrigin, config, {
870
- retryRuntimeFailures: true,
871
- }),
872
- ),
873
- Effect.retry({
874
- while: (error) =>
875
- CloudLink.shouldRetryCloudLink(error) &&
876
- error._tag !== "CloudLinkEndpointUnavailableError",
877
- schedule: Schedule.exponential("1 second").pipe(
878
- Schedule.modifyDelay(({ duration }) =>
879
- Effect.succeed(Duration.min(duration, Duration.seconds(30))),
880
- ),
881
- Schedule.jittered,
882
- ),
883
- }),
884
- Effect.tap((recovered) =>
885
- recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void,
886
- ),
887
- Effect.catchCause((cause) =>
888
- Cause.hasInterrupts(cause)
889
- ? Effect.interrupt
890
- : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", {
891
- cause,
892
- }),
893
- ),
866
+ runManagedTunnelRecovery(config).pipe(
867
+ Effect.tap((recovered) =>
868
+ recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void,
869
+ ),
870
+ Effect.catchCause((cause) =>
871
+ Cause.hasInterrupts(cause)
872
+ ? Effect.interrupt
873
+ : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", {
874
+ cause,
875
+ }),
876
),
877
);
878
yield* endpointRuntime.recoveryRequests.pipe(
@@ -992,15 +974,13 @@ const layerMakeServer = Layer.unwrap(
974
const reconciledMode = yield* cloudLink
975
.reconcileDesiredLinkIfStillDesired(localOrigin)
976
.pipe(
995
- Effect.retry({
996
- while: CloudLink.shouldRetryCloudLink,
997
- schedule: Schedule.exponential("1 second").pipe(
998
- Schedule.modifyDelay(({ duration }) =>
999
- Effect.succeed(Duration.min(duration, Duration.seconds(30))),
977
+ (reconcile) =>
978
+ retryDesiredLinkReconcile(reconcile, CloudLink.shouldRetryCloudLink, (cause) =>
979
+ Effect.logWarning(
980
+ "T3 Connect relay is still unavailable; retrying the desired link in the background",
981
+ { cause },
982
),
1001
- Schedule.upTo({ duration: "10 minutes" }),
983
),
1003
- }),
984
Effect.tap((mode) =>
985
mode === null
986
? Effect.void