@devries-cameron20 / t3code-1 / commits / 9638e17835
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(relay): tunnel cleanup keeps up with new idle tunnels (#26135)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 27, 2026 at 19:13 UTC 9638e178351167eb41186472ef8a55edac6ed6cf
3 files changed +13 -10
docs/operations/relay-observability.md
+2 -1
@@ -147,7 +147,8 @@ error messages. An expired APNs key or FCM service account fails every job.
147
148 **Cloudflare API rate limited / near its limit.** The warning fires on sustained 429s; the page fires
149 when they are frequent enough to stall provisioning. Compare the call volume on the dashboard with the
150 -account limit.
150 +account limit, which Cloudflare raised to 20,000 calls per 5 minutes in October 2026 (peak use then
151 +was about 2,100).
152
153 **Traffic dropped.** The Worker is down, a deploy broke it, or spans stopped reaching Axiom. Check
154 `https://relay.t3.codes/health` first.
infra/relay/src/environments/ManagedEndpointReaper.test.ts
+8 -6
@@ -677,7 +677,8 @@ describe("ManagedEndpointReaper", () => {
677 });
678
679 it.effect("rotates the deletion order so persistent failures do not starve later tunnels", () => {
680 - const entries = Array.from({ length: 200 }, (_, index) =>
680 + const limit = ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT;
681 + const entries = Array.from({ length: limit * 2 }, (_, index) =>
682 tunnel({
683 id: `tunnel-${index}`,
684 suffix: index.toString(16).padStart(16, "0"),
@@ -688,7 +689,7 @@ describe("ManagedEndpointReaper", () => {
689 const state = harness({
690 tunnels: entries,
691 allocations: recoverableOwners(entries),
691 - failDeleteWhen: (id) => Number(id.split("-")[1]) < 100,
692 + failDeleteWhen: (id) => Number(id.split("-")[1]) < limit,
693 });
694
695 return Effect.gen(function* () {
@@ -697,14 +698,15 @@ describe("ManagedEndpointReaper", () => {
698 yield* reaper.sweep;
699 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
700 yield* reaper.sweep;
700 - const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= 100);
701 + const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= limit);
702 expect(later.length).toBeGreaterThan(0);
703 }).pipe(Effect.provide(state.layer));
704 });
705
706 it.effect("rotates the attempt budget across both statuses over consecutive sweeps", () => {
707 + const limit = ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT;
708 const entries = (["down", "inactive"] as const).flatMap((status) =>
707 - Array.from({ length: 100 }, (_, index) =>
709 + Array.from({ length: limit }, (_, index) =>
710 tunnel({
711 id: `${status}-${index}`,
712 suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`,
@@ -720,13 +722,13 @@ describe("ManagedEndpointReaper", () => {
722 yield* TestClock.setTime(NOW_MILLIS);
723 yield* reaper.sweep;
724 const firstStatus = state.deleted[0]!.split("-")[0];
723 - expect(state.deleted).toHaveLength(100);
725 + expect(state.deleted).toHaveLength(limit);
726 expect(state.deleted.every((id) => id.startsWith(`${firstStatus}-`))).toBe(true);
727
728 state.deleted.length = 0;
729 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
730 yield* reaper.sweep;
729 - expect(state.deleted).toHaveLength(100);
731 + expect(state.deleted).toHaveLength(limit);
732 expect(state.deleted.every((id) => !id.startsWith(`${firstStatus}-`))).toBe(true);
733 }).pipe(Effect.provide(state.layer));
734 });
infra/relay/src/environments/ManagedEndpointReaper.ts
+3 -3
@@ -17,13 +17,13 @@ export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5;
17 // Never-connected tunnels may still be pairing; allow an hour.
18 export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
19 export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
20 -export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;
20 +export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 250;
21 export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
22 const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
23 // Hosts without recovery cannot replace deleted tunnels; allow a longer absence.
24 const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
25 -// Parallelize to fit the cron budget, but stay below Hyperdrive's shared 20-connection limit.
26 -export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4;
25 +// Each delete holds a Hyperdrive connection for ~2s; stay well below the 40 request handlers share.
26 +export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 8;
27 // Leave time under the cron's two-minute limit to finish deletions and record counters.
28 const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000;
29