The best way to control Claude Code, Codex and more from any computer or phone

main
ts 424 lines 14.9 KB
Raw
1 import * as Clock from "effect/Clock";
2 import * as Context from "effect/Context";
3 import * as DateTime from "effect/DateTime";
4 import * as Effect from "effect/Effect";
5 import * as Layer from "effect/Layer";
6 import * as Option from "effect/Option";
7
8 import * as RelayConfiguration from "../Config.ts";
9 import {
10 MANAGED_ENDPOINT_ZONE_OWNER_STAGE,
11 managedEndpointTunnelNamePrefix,
12 } from "../deploymentConfig.ts";
13 import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
14 import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
15
16 const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5;
17 // Never-connected tunnels may still be pairing; allow an hour.
18 const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
19 const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
20 // Down tunnels arrive at ~66k/day, about what 250 per sweep removed; the delete budget caps a sweep below this.
21 export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 500;
22 const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
23 // Cloudflare's filtered list over a large backlog can outlast the 8 s limit request paths use.
24 const MANAGED_ENDPOINT_SWEEP_LIST_TIMEOUT = "30 seconds";
25 const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
26 // Hosts without recovery cannot replace deleted tunnels; allow a longer absence.
27 const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
28 // Each delete holds a Hyperdrive connection for ~2s; stay well below the 40 request handlers share.
29 const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 8;
30 // Leave time under the cron's two-minute limit to finish deletions and record counters.
31 const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000;
32
33 export interface ManagedEndpointSweepResult {
34 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;
35 readonly legacyMode: RelayConfiguration.ManagedEndpointCleanupMode;
36 readonly listRequests: number;
37 readonly scanned: number;
38 readonly attempted: number;
39 readonly deleted: number;
40 readonly wouldDelete: number;
41 readonly wouldDeleteLegacy: number;
42 readonly deletedLegacy: number;
43 readonly skippedLegacy: number;
44 readonly legacyOver7Days: number;
45 readonly legacyOver30Days: number;
46 readonly legacyOver90Days: number;
47 readonly skippedOrphan: number;
48 readonly skippedReplaced: number;
49 readonly skippedUnrecorded: number;
50 readonly totalDown: number | null;
51 readonly totalInactive: number | null;
52 readonly failed: number;
53 readonly truncated: boolean;
54 }
55
56 export class ManagedEndpointReaper extends Context.Service<
57 ManagedEndpointReaper,
58 {
59 readonly sweep: Effect.Effect<
60 ManagedEndpointSweepResult,
61 | ManagedEndpointProvider.ManagedEndpointTunnelClientError
62 | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError
63 >;
64 }
65 >()("t3code-relay/environments/ManagedEndpointReaper") {}
66
67 function isExpiredManagedTunnel(input: {
68 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel;
69 readonly status: "down" | "inactive";
70 readonly prefix: string;
71 readonly cutoff: DateTime.Utc;
72 }): input is typeof input & {
73 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
74 readonly id: string;
75 readonly name: string;
76 };
77 } {
78 const { tunnel, status, prefix, cutoff } = input;
79 if (
80 typeof tunnel.id !== "string" ||
81 typeof tunnel.name !== "string" ||
82 tunnel.status !== status ||
83 !tunnel.name.startsWith(prefix) ||
84 !/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length))
85 ) {
86 return false;
87 }
88 const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt;
89 if (typeof inactiveAt !== "string") {
90 return false;
91 }
92 const timestamp = DateTime.make(inactiveAt);
93 return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds;
94 }
95
96 function inactiveDaysAt(
97 tunnel: ManagedEndpointProvider.ManagedEndpointTunnel,
98 status: "down" | "inactive",
99 now: DateTime.Utc,
100 ): number | null {
101 const since = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt;
102 if (typeof since !== "string") return null;
103 const timestamp = DateTime.make(since);
104 if (Option.isNone(timestamp)) return null;
105 return (now.epochMilliseconds - timestamp.value.epochMilliseconds) / 86_400_000;
106 }
107
108 function rotatedPages(input: {
109 readonly totalCount: number | undefined;
110 readonly slot: number;
111 readonly limit: number;
112 }): ReadonlyArray<number> {
113 if (input.limit <= 0) return [];
114 if (input.totalCount === undefined) {
115 return Array.from({ length: input.limit }, (_, index) => index + 2);
116 }
117 const laterPageCount = Math.max(
118 0,
119 Math.ceil(input.totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) - 1,
120 );
121 if (laterPageCount === 0) return [];
122 const count = Math.min(input.limit, laterPageCount);
123 const start = input.slot % laterPageCount;
124 return Array.from({ length: count }, (_, index) => 2 + ((start + index) % laterPageCount));
125 }
126
127 const emptyResult = (
128 mode: RelayConfiguration.ManagedEndpointCleanupMode,
129 legacyMode: RelayConfiguration.ManagedEndpointCleanupMode,
130 ): ManagedEndpointSweepResult => ({
131 mode,
132 legacyMode,
133 listRequests: 0,
134 scanned: 0,
135 attempted: 0,
136 deleted: 0,
137 wouldDelete: 0,
138 wouldDeleteLegacy: 0,
139 deletedLegacy: 0,
140 skippedLegacy: 0,
141 legacyOver7Days: 0,
142 legacyOver30Days: 0,
143 legacyOver90Days: 0,
144 skippedOrphan: 0,
145 skippedReplaced: 0,
146 skippedUnrecorded: 0,
147 totalDown: null,
148 totalInactive: null,
149 failed: 0,
150 truncated: false,
151 });
152
153 const make = Effect.gen(function* () {
154 const config = yield* RelayConfiguration.RelayConfiguration;
155 const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient;
156 const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
157 const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
158
159 const sweep = Effect.gen(function* () {
160 const mode = config.managedEndpointCleanupMode ?? "off";
161 const legacyMode = config.legacyManagedEndpointCleanupMode ?? "off";
162 const namespace = config.managedEndpointNamespace;
163 if ((mode === "off" && legacyMode === "off") || !namespace) {
164 return emptyResult(mode, legacyMode);
165 }
166 const sweepStartedAtMillis = yield* Clock.currentTimeMillis;
167 const legacyGraceMinutes =
168 namespace !== MANAGED_ENDPOINT_ZONE_OWNER_STAGE &&
169 config.legacyTunnelGraceMinutes !== undefined
170 ? config.legacyTunnelGraceMinutes
171 : MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS * 24 * 60;
172 const legacyCutoff = DateTime.subtract(yield* DateTime.now, { minutes: legacyGraceMinutes });
173
174 const now = yield* DateTime.now;
175 const cutoffFor = (status: "down" | "inactive") =>
176 DateTime.subtract(now, {
177 minutes:
178 status === "down"
179 ? MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES
180 : MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES,
181 });
182 const prefix = managedEndpointTunnelNamePrefix(namespace);
183 const slot = Math.floor(
184 now.epochMilliseconds / (MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES * 60 * 1_000),
185 );
186 let listRequests = 0;
187 let truncated = false;
188 const totals: Record<"down" | "inactive", number | null> = { down: null, inactive: null };
189 const expired: Array<{
190 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
191 readonly id: string;
192 readonly name: string;
193 };
194 readonly status: "down" | "inactive";
195 readonly cutoff: DateTime.Utc;
196 }> = [];
197
198 for (const status of ["down", "inactive"] as const) {
199 const cutoff = cutoffFor(status);
200 const cutoffIso = DateTime.formatIso(cutoff);
201 const listPage = (page: number) => {
202 listRequests += 1;
203 return tunnels.list(
204 {
205 isDeleted: false,
206 includePrefix: prefix,
207 status,
208 existedAt: cutoffIso,
209 ...(status === "down" ? { wasInactiveAt: cutoffIso } : {}),
210 page,
211 perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE,
212 },
213 { timeout: MANAGED_ENDPOINT_SWEEP_LIST_TIMEOUT },
214 );
215 };
216 const first = yield* listPage(1);
217 const totalCount =
218 typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined;
219 totals[status] = totalCount ?? null;
220 const pages = rotatedPages({
221 totalCount,
222 slot,
223 limit: Math.floor(MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT / 2) - 1,
224 });
225 const responses = [first, ...(yield* Effect.forEach(pages, listPage, { concurrency: 1 }))];
226 if (
227 totalCount !== undefined &&
228 Math.ceil(totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) > responses.length
229 ) {
230 truncated = true;
231 } else if (
232 totalCount === undefined &&
233 responses.at(-1)?.result.length === MANAGED_ENDPOINT_SWEEP_PAGE_SIZE
234 ) {
235 truncated = true;
236 }
237 for (const response of responses) {
238 expired.push(
239 ...response.result
240 .map((tunnel) => ({ tunnel, status, prefix, cutoff }))
241 .filter(isExpiredManagedTunnel)
242 .map(({ tunnel }) => ({ tunnel, status, cutoff })),
243 );
244 }
245 }
246
247 const collected = [...new Map(expired.map((entry) => [entry.tunnel.id, entry])).values()];
248 // Rotate the start offset so repeatedly failing candidates cannot starve later ones.
249 const offset =
250 collected.length === 0 ? 0 : (slot * MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) % collected.length;
251 const uniqueExpired = [...collected.slice(offset), ...collected.slice(0, offset)];
252 const recorded = yield* allocations.listByTunnelNames(
253 uniqueExpired.map(({ tunnel }) => tunnel.name),
254 );
255 const recordedByTunnelName = new Map(
256 recorded.map((allocation) => [allocation.tunnelName, allocation]),
257 );
258 let attempted = 0;
259 let deleted = 0;
260 let wouldDelete = 0;
261 const candidates: Array<{
262 readonly owner: ManagedEndpointAllocations.ManagedEndpointTunnelAllocation;
263 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
264 readonly id: string;
265 readonly name: string;
266 };
267 readonly status: "down" | "inactive";
268 readonly legacy: boolean;
269 readonly inactiveBefore: string;
270 }> = [];
271 let wouldDeleteLegacy = 0;
272 let deletedLegacy = 0;
273 let skippedLegacy = 0;
274 const legacyOverDays = new Map<number, number>(
275 MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]),
276 );
277 let skippedOrphan = 0;
278 let skippedReplaced = 0;
279 let skippedUnrecorded = 0;
280 let failed = 0;
281
282 for (const { tunnel, status, cutoff } of uniqueExpired) {
283 const allocation = recordedByTunnelName.get(tunnel.name);
284 if (
285 allocation !== undefined &&
286 allocation.tunnelId !== null &&
287 allocation.tunnelId !== tunnel.id
288 ) {
289 skippedReplaced += 1;
290 continue;
291 }
292 const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;
293 if (allocation !== undefined && owner === undefined) {
294 skippedUnrecorded += 1;
295 continue;
296 }
297 // Without an allocation claim, deletion can race relinking by name; leave orphans for manual sweeps.
298 if (owner === undefined) {
299 skippedOrphan += 1;
300 continue;
301 }
302 const legacy = !owner.recoveryEnabled;
303 if (legacy) {
304 skippedLegacy += 1;
305 const inactiveDays = inactiveDaysAt(tunnel, status, now);
306 for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) {
307 if (inactiveDays !== null && inactiveDays > days) {
308 legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1);
309 }
310 }
311 if (
312 legacyMode === "off" ||
313 !isExpiredManagedTunnel({ tunnel, status, prefix, cutoff: legacyCutoff })
314 ) {
315 continue;
316 }
317 wouldDeleteLegacy += 1;
318 if (legacyMode === "dry-run") continue;
319 } else {
320 if (mode === "off") continue;
321 wouldDelete += 1;
322 if (mode === "dry-run") continue;
323 }
324 if (candidates.length >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
325 truncated = true;
326 break;
327 }
328 candidates.push({
329 owner,
330 tunnel,
331 status,
332 legacy,
333 inactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff),
334 });
335 }
336
337 const deleteDeadline = sweepStartedAtMillis + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS;
338 let stopDeleting = false;
339 yield* Effect.forEach(
340 candidates,
341 (candidate) =>
342 Effect.gen(function* () {
343 if (stopDeleting || (yield* Clock.currentTimeMillis) >= deleteDeadline) {
344 stopDeleting = true;
345 truncated = true;
346 return;
347 }
348 attempted += 1;
349 const result = yield* provider
350 .release({
351 userId: candidate.owner.userId,
352 environmentId: candidate.owner.environmentId,
353 expectedTunnelId: candidate.tunnel.id,
354 expectedInactiveBefore: candidate.inactiveBefore,
355 expectedStatus: candidate.status,
356 ...(candidate.legacy ? { markReleased: true } : {}),
357 })
358 .pipe(Effect.result);
359 if (result._tag === "Failure") {
360 failed += 1;
361 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
362 tunnelId: candidate.tunnel.id,
363 tunnelName: candidate.tunnel.name,
364 legacy: candidate.legacy,
365 cause: result.failure,
366 });
367 if (ManagedEndpointProvider.isCloudflareRateLimited(result.failure)) {
368 stopDeleting = true;
369 truncated = true;
370 }
371 } else if (result.success) {
372 deleted += 1;
373 if (candidate.legacy) deletedLegacy += 1;
374 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
375 tunnelId: candidate.tunnel.id,
376 tunnelName: candidate.tunnel.name,
377 status: candidate.status,
378 legacy: candidate.legacy,
379 });
380 }
381 }),
382 { concurrency: MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY, discard: true },
383 );
384
385 return {
386 mode,
387 legacyMode,
388 listRequests,
389 scanned: uniqueExpired.length,
390 attempted,
391 deleted,
392 wouldDelete,
393 wouldDeleteLegacy,
394 deletedLegacy,
395 skippedLegacy,
396 legacyOver7Days: legacyOverDays.get(7) ?? 0,
397 legacyOver30Days: legacyOverDays.get(30) ?? 0,
398 legacyOver90Days: legacyOverDays.get(90) ?? 0,
399 skippedOrphan,
400 skippedReplaced,
401 skippedUnrecorded,
402 totalDown: totals.down,
403 totalInactive: totals.inactive,
404 failed,
405 truncated,
406 };
407 }).pipe(
408 Effect.tap((result) =>
409 Effect.annotateCurrentSpan(
410 Object.fromEntries(
411 Object.entries(result).map(([key, value]) => [
412 `relay.managed_endpoint_reaper.${key}`,
413 value,
414 ]),
415 ),
416 ),
417 ),
418 Effect.withSpan("relay.managed_endpoint_reaper.sweep"),
419 );
420
421 return ManagedEndpointReaper.of({ sweep });
422 });
423
424 export const layer = Layer.effect(ManagedEndpointReaper, make);