The best way to control Claude Code, Codex and more from any computer or phone
fix(relay): tunnel cleanup keeps up with new idle tunnels (#26135)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 27, 2026 at 19:13 UTC
9638e178351167eb41186472ef8a55edac6ed6cf
3 files changed
+13
-10
docs/operations/relay-observability.md
+2
-1
@@ -147,7 +147,8 @@ error messages. An expired APNs key or FCM service account fails every job.
147
148
**Cloudflare API rate limited / near its limit.** The warning fires on sustained 429s; the page fires
149
when they are frequent enough to stall provisioning. Compare the call volume on the dashboard with the
150
-account limit.
150
+account limit, which Cloudflare raised to 20,000 calls per 5 minutes in October 2026 (peak use then
151
+was about 2,100).
152
153
**Traffic dropped.** The Worker is down, a deploy broke it, or spans stopped reaching Axiom. Check
154
`https://relay.t3.codes/health` first.
infra/relay/src/environments/ManagedEndpointReaper.test.ts
+8
-6
@@ -677,7 +677,8 @@ describe("ManagedEndpointReaper", () => {
677
});
678
679
it.effect("rotates the deletion order so persistent failures do not starve later tunnels", () => {
680
- const entries = Array.from({ length: 200 }, (_, index) =>
680
+ const limit = ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT;
681
+ const entries = Array.from({ length: limit * 2 }, (_, index) =>
682
tunnel({
683
id: `tunnel-${index}`,
684
suffix: index.toString(16).padStart(16, "0"),
@@ -688,7 +689,7 @@ describe("ManagedEndpointReaper", () => {
689
const state = harness({
690
tunnels: entries,
691
allocations: recoverableOwners(entries),
691
- failDeleteWhen: (id) => Number(id.split("-")[1]) < 100,
692
+ failDeleteWhen: (id) => Number(id.split("-")[1]) < limit,
693
});
694
695
return Effect.gen(function* () {
@@ -697,14 +698,15 @@ describe("ManagedEndpointReaper", () => {
698
yield* reaper.sweep;
699
yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
700
yield* reaper.sweep;
700
- const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= 100);
701
+ const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= limit);
702
expect(later.length).toBeGreaterThan(0);
703
}).pipe(Effect.provide(state.layer));
704
});
705
706
it.effect("rotates the attempt budget across both statuses over consecutive sweeps", () => {
707
+ const limit = ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT;
708
const entries = (["down", "inactive"] as const).flatMap((status) =>
707
- Array.from({ length: 100 }, (_, index) =>
709
+ Array.from({ length: limit }, (_, index) =>
710
tunnel({
711
id: `${status}-${index}`,
712
suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`,
@@ -720,13 +722,13 @@ describe("ManagedEndpointReaper", () => {
722
yield* TestClock.setTime(NOW_MILLIS);
723
yield* reaper.sweep;
724
const firstStatus = state.deleted[0]!.split("-")[0];
723
- expect(state.deleted).toHaveLength(100);
725
+ expect(state.deleted).toHaveLength(limit);
726
expect(state.deleted.every((id) => id.startsWith(`${firstStatus}-`))).toBe(true);
727
728
state.deleted.length = 0;
729
yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
730
yield* reaper.sweep;
729
- expect(state.deleted).toHaveLength(100);
731
+ expect(state.deleted).toHaveLength(limit);
732
expect(state.deleted.every((id) => !id.startsWith(`${firstStatus}-`))).toBe(true);
733
}).pipe(Effect.provide(state.layer));
734
});
infra/relay/src/environments/ManagedEndpointReaper.ts
+3
-3
@@ -17,13 +17,13 @@ export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5;
17
// Never-connected tunnels may still be pairing; allow an hour.
18
export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
19
export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
20
-export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;
20
+export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 250;
21
export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
22
const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
23
// Hosts without recovery cannot replace deleted tunnels; allow a longer absence.
24
const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
25
-// Parallelize to fit the cron budget, but stay below Hyperdrive's shared 20-connection limit.
26
-export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4;
25
+// Each delete holds a Hyperdrive connection for ~2s; stay well below the 40 request handlers share.
26
+export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 8;
27
// Leave time under the cron's two-minute limit to finish deletions and record counters.
28
const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000;
29