The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as Clock from "effect/Clock"; |
| 2 | import * as Context from "effect/Context"; |
| 3 | import * as DateTime from "effect/DateTime"; |
| 4 | import * as Effect from "effect/Effect"; |
| 5 | import * as Layer from "effect/Layer"; |
| 6 | import * as Option from "effect/Option"; |
| 7 | |
| 8 | import * as RelayConfiguration from "../Config.ts"; |
| 9 | import { |
| 10 | MANAGED_ENDPOINT_ZONE_OWNER_STAGE, |
| 11 | managedEndpointTunnelNamePrefix, |
| 12 | } from "../deploymentConfig.ts"; |
| 13 | import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; |
| 14 | import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; |
| 15 | |
| 16 | const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5; |
| 17 | // Never-connected tunnels may still be pairing; allow an hour. |
| 18 | const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60; |
| 19 | const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100; |
| 20 | // Down tunnels arrive at ~66k/day, about what 250 per sweep removed; the delete budget caps a sweep below this. |
| 21 | export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 500; |
| 22 | const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10; |
| 23 | // Cloudflare's filtered list over a large backlog can outlast the 8 s limit request paths use. |
| 24 | const MANAGED_ENDPOINT_SWEEP_LIST_TIMEOUT = "30 seconds"; |
| 25 | const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const; |
| 26 | // Hosts without recovery cannot replace deleted tunnels; allow a longer absence. |
| 27 | const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7; |
| 28 | // Each delete holds a Hyperdrive connection for ~2s; stay well below the 40 request handlers share. |
| 29 | const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 8; |
| 30 | // Leave time under the cron's two-minute limit to finish deletions and record counters. |
| 31 | const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000; |
| 32 | |
| 33 | export interface ManagedEndpointSweepResult { |
| 34 | readonly mode: RelayConfiguration.ManagedEndpointCleanupMode; |
| 35 | readonly legacyMode: RelayConfiguration.ManagedEndpointCleanupMode; |
| 36 | readonly listRequests: number; |
| 37 | readonly scanned: number; |
| 38 | readonly attempted: number; |
| 39 | readonly deleted: number; |
| 40 | readonly wouldDelete: number; |
| 41 | readonly wouldDeleteLegacy: number; |
| 42 | readonly deletedLegacy: number; |
| 43 | readonly skippedLegacy: number; |
| 44 | readonly legacyOver7Days: number; |
| 45 | readonly legacyOver30Days: number; |
| 46 | readonly legacyOver90Days: number; |
| 47 | readonly skippedOrphan: number; |
| 48 | readonly skippedReplaced: number; |
| 49 | readonly skippedUnrecorded: number; |
| 50 | readonly totalDown: number | null; |
| 51 | readonly totalInactive: number | null; |
| 52 | readonly failed: number; |
| 53 | readonly truncated: boolean; |
| 54 | } |
| 55 | |
| 56 | export class ManagedEndpointReaper extends Context.Service< |
| 57 | ManagedEndpointReaper, |
| 58 | { |
| 59 | readonly sweep: Effect.Effect< |
| 60 | ManagedEndpointSweepResult, |
| 61 | | ManagedEndpointProvider.ManagedEndpointTunnelClientError |
| 62 | | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError |
| 63 | >; |
| 64 | } |
| 65 | >()("t3code-relay/environments/ManagedEndpointReaper") {} |
| 66 | |
| 67 | function isExpiredManagedTunnel(input: { |
| 68 | readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel; |
| 69 | readonly status: "down" | "inactive"; |
| 70 | readonly prefix: string; |
| 71 | readonly cutoff: DateTime.Utc; |
| 72 | }): input is typeof input & { |
| 73 | readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { |
| 74 | readonly id: string; |
| 75 | readonly name: string; |
| 76 | }; |
| 77 | } { |
| 78 | const { tunnel, status, prefix, cutoff } = input; |
| 79 | if ( |
| 80 | typeof tunnel.id !== "string" || |
| 81 | typeof tunnel.name !== "string" || |
| 82 | tunnel.status !== status || |
| 83 | !tunnel.name.startsWith(prefix) || |
| 84 | !/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length)) |
| 85 | ) { |
| 86 | return false; |
| 87 | } |
| 88 | const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt; |
| 89 | if (typeof inactiveAt !== "string") { |
| 90 | return false; |
| 91 | } |
| 92 | const timestamp = DateTime.make(inactiveAt); |
| 93 | return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds; |
| 94 | } |
| 95 | |
| 96 | function inactiveDaysAt( |
| 97 | tunnel: ManagedEndpointProvider.ManagedEndpointTunnel, |
| 98 | status: "down" | "inactive", |
| 99 | now: DateTime.Utc, |
| 100 | ): number | null { |
| 101 | const since = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt; |
| 102 | if (typeof since !== "string") return null; |
| 103 | const timestamp = DateTime.make(since); |
| 104 | if (Option.isNone(timestamp)) return null; |
| 105 | return (now.epochMilliseconds - timestamp.value.epochMilliseconds) / 86_400_000; |
| 106 | } |
| 107 | |
| 108 | function rotatedPages(input: { |
| 109 | readonly totalCount: number | undefined; |
| 110 | readonly slot: number; |
| 111 | readonly limit: number; |
| 112 | }): ReadonlyArray<number> { |
| 113 | if (input.limit <= 0) return []; |
| 114 | if (input.totalCount === undefined) { |
| 115 | return Array.from({ length: input.limit }, (_, index) => index + 2); |
| 116 | } |
| 117 | const laterPageCount = Math.max( |
| 118 | 0, |
| 119 | Math.ceil(input.totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) - 1, |
| 120 | ); |
| 121 | if (laterPageCount === 0) return []; |
| 122 | const count = Math.min(input.limit, laterPageCount); |
| 123 | const start = input.slot % laterPageCount; |
| 124 | return Array.from({ length: count }, (_, index) => 2 + ((start + index) % laterPageCount)); |
| 125 | } |
| 126 | |
| 127 | const emptyResult = ( |
| 128 | mode: RelayConfiguration.ManagedEndpointCleanupMode, |
| 129 | legacyMode: RelayConfiguration.ManagedEndpointCleanupMode, |
| 130 | ): ManagedEndpointSweepResult => ({ |
| 131 | mode, |
| 132 | legacyMode, |
| 133 | listRequests: 0, |
| 134 | scanned: 0, |
| 135 | attempted: 0, |
| 136 | deleted: 0, |
| 137 | wouldDelete: 0, |
| 138 | wouldDeleteLegacy: 0, |
| 139 | deletedLegacy: 0, |
| 140 | skippedLegacy: 0, |
| 141 | legacyOver7Days: 0, |
| 142 | legacyOver30Days: 0, |
| 143 | legacyOver90Days: 0, |
| 144 | skippedOrphan: 0, |
| 145 | skippedReplaced: 0, |
| 146 | skippedUnrecorded: 0, |
| 147 | totalDown: null, |
| 148 | totalInactive: null, |
| 149 | failed: 0, |
| 150 | truncated: false, |
| 151 | }); |
| 152 | |
| 153 | const make = Effect.gen(function* () { |
| 154 | const config = yield* RelayConfiguration.RelayConfiguration; |
| 155 | const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient; |
| 156 | const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; |
| 157 | const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; |
| 158 | |
| 159 | const sweep = Effect.gen(function* () { |
| 160 | const mode = config.managedEndpointCleanupMode ?? "off"; |
| 161 | const legacyMode = config.legacyManagedEndpointCleanupMode ?? "off"; |
| 162 | const namespace = config.managedEndpointNamespace; |
| 163 | if ((mode === "off" && legacyMode === "off") || !namespace) { |
| 164 | return emptyResult(mode, legacyMode); |
| 165 | } |
| 166 | const sweepStartedAtMillis = yield* Clock.currentTimeMillis; |
| 167 | const legacyGraceMinutes = |
| 168 | namespace !== MANAGED_ENDPOINT_ZONE_OWNER_STAGE && |
| 169 | config.legacyTunnelGraceMinutes !== undefined |
| 170 | ? config.legacyTunnelGraceMinutes |
| 171 | : MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS * 24 * 60; |
| 172 | const legacyCutoff = DateTime.subtract(yield* DateTime.now, { minutes: legacyGraceMinutes }); |
| 173 | |
| 174 | const now = yield* DateTime.now; |
| 175 | const cutoffFor = (status: "down" | "inactive") => |
| 176 | DateTime.subtract(now, { |
| 177 | minutes: |
| 178 | status === "down" |
| 179 | ? MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES |
| 180 | : MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES, |
| 181 | }); |
| 182 | const prefix = managedEndpointTunnelNamePrefix(namespace); |
| 183 | const slot = Math.floor( |
| 184 | now.epochMilliseconds / (MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES * 60 * 1_000), |
| 185 | ); |
| 186 | let listRequests = 0; |
| 187 | let truncated = false; |
| 188 | const totals: Record<"down" | "inactive", number | null> = { down: null, inactive: null }; |
| 189 | const expired: Array<{ |
| 190 | readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { |
| 191 | readonly id: string; |
| 192 | readonly name: string; |
| 193 | }; |
| 194 | readonly status: "down" | "inactive"; |
| 195 | readonly cutoff: DateTime.Utc; |
| 196 | }> = []; |
| 197 | |
| 198 | for (const status of ["down", "inactive"] as const) { |
| 199 | const cutoff = cutoffFor(status); |
| 200 | const cutoffIso = DateTime.formatIso(cutoff); |
| 201 | const listPage = (page: number) => { |
| 202 | listRequests += 1; |
| 203 | return tunnels.list( |
| 204 | { |
| 205 | isDeleted: false, |
| 206 | includePrefix: prefix, |
| 207 | status, |
| 208 | existedAt: cutoffIso, |
| 209 | ...(status === "down" ? { wasInactiveAt: cutoffIso } : {}), |
| 210 | page, |
| 211 | perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE, |
| 212 | }, |
| 213 | { timeout: MANAGED_ENDPOINT_SWEEP_LIST_TIMEOUT }, |
| 214 | ); |
| 215 | }; |
| 216 | const first = yield* listPage(1); |
| 217 | const totalCount = |
| 218 | typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined; |
| 219 | totals[status] = totalCount ?? null; |
| 220 | const pages = rotatedPages({ |
| 221 | totalCount, |
| 222 | slot, |
| 223 | limit: Math.floor(MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT / 2) - 1, |
| 224 | }); |
| 225 | const responses = [first, ...(yield* Effect.forEach(pages, listPage, { concurrency: 1 }))]; |
| 226 | if ( |
| 227 | totalCount !== undefined && |
| 228 | Math.ceil(totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) > responses.length |
| 229 | ) { |
| 230 | truncated = true; |
| 231 | } else if ( |
| 232 | totalCount === undefined && |
| 233 | responses.at(-1)?.result.length === MANAGED_ENDPOINT_SWEEP_PAGE_SIZE |
| 234 | ) { |
| 235 | truncated = true; |
| 236 | } |
| 237 | for (const response of responses) { |
| 238 | expired.push( |
| 239 | ...response.result |
| 240 | .map((tunnel) => ({ tunnel, status, prefix, cutoff })) |
| 241 | .filter(isExpiredManagedTunnel) |
| 242 | .map(({ tunnel }) => ({ tunnel, status, cutoff })), |
| 243 | ); |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | const collected = [...new Map(expired.map((entry) => [entry.tunnel.id, entry])).values()]; |
| 248 | // Rotate the start offset so repeatedly failing candidates cannot starve later ones. |
| 249 | const offset = |
| 250 | collected.length === 0 ? 0 : (slot * MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) % collected.length; |
| 251 | const uniqueExpired = [...collected.slice(offset), ...collected.slice(0, offset)]; |
| 252 | const recorded = yield* allocations.listByTunnelNames( |
| 253 | uniqueExpired.map(({ tunnel }) => tunnel.name), |
| 254 | ); |
| 255 | const recordedByTunnelName = new Map( |
| 256 | recorded.map((allocation) => [allocation.tunnelName, allocation]), |
| 257 | ); |
| 258 | let attempted = 0; |
| 259 | let deleted = 0; |
| 260 | let wouldDelete = 0; |
| 261 | const candidates: Array<{ |
| 262 | readonly owner: ManagedEndpointAllocations.ManagedEndpointTunnelAllocation; |
| 263 | readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { |
| 264 | readonly id: string; |
| 265 | readonly name: string; |
| 266 | }; |
| 267 | readonly status: "down" | "inactive"; |
| 268 | readonly legacy: boolean; |
| 269 | readonly inactiveBefore: string; |
| 270 | }> = []; |
| 271 | let wouldDeleteLegacy = 0; |
| 272 | let deletedLegacy = 0; |
| 273 | let skippedLegacy = 0; |
| 274 | const legacyOverDays = new Map<number, number>( |
| 275 | MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]), |
| 276 | ); |
| 277 | let skippedOrphan = 0; |
| 278 | let skippedReplaced = 0; |
| 279 | let skippedUnrecorded = 0; |
| 280 | let failed = 0; |
| 281 | |
| 282 | for (const { tunnel, status, cutoff } of uniqueExpired) { |
| 283 | const allocation = recordedByTunnelName.get(tunnel.name); |
| 284 | if ( |
| 285 | allocation !== undefined && |
| 286 | allocation.tunnelId !== null && |
| 287 | allocation.tunnelId !== tunnel.id |
| 288 | ) { |
| 289 | skippedReplaced += 1; |
| 290 | continue; |
| 291 | } |
| 292 | const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined; |
| 293 | if (allocation !== undefined && owner === undefined) { |
| 294 | skippedUnrecorded += 1; |
| 295 | continue; |
| 296 | } |
| 297 | // Without an allocation claim, deletion can race relinking by name; leave orphans for manual sweeps. |
| 298 | if (owner === undefined) { |
| 299 | skippedOrphan += 1; |
| 300 | continue; |
| 301 | } |
| 302 | const legacy = !owner.recoveryEnabled; |
| 303 | if (legacy) { |
| 304 | skippedLegacy += 1; |
| 305 | const inactiveDays = inactiveDaysAt(tunnel, status, now); |
| 306 | for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) { |
| 307 | if (inactiveDays !== null && inactiveDays > days) { |
| 308 | legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1); |
| 309 | } |
| 310 | } |
| 311 | if ( |
| 312 | legacyMode === "off" || |
| 313 | !isExpiredManagedTunnel({ tunnel, status, prefix, cutoff: legacyCutoff }) |
| 314 | ) { |
| 315 | continue; |
| 316 | } |
| 317 | wouldDeleteLegacy += 1; |
| 318 | if (legacyMode === "dry-run") continue; |
| 319 | } else { |
| 320 | if (mode === "off") continue; |
| 321 | wouldDelete += 1; |
| 322 | if (mode === "dry-run") continue; |
| 323 | } |
| 324 | if (candidates.length >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) { |
| 325 | truncated = true; |
| 326 | break; |
| 327 | } |
| 328 | candidates.push({ |
| 329 | owner, |
| 330 | tunnel, |
| 331 | status, |
| 332 | legacy, |
| 333 | inactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff), |
| 334 | }); |
| 335 | } |
| 336 | |
| 337 | const deleteDeadline = sweepStartedAtMillis + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS; |
| 338 | let stopDeleting = false; |
| 339 | yield* Effect.forEach( |
| 340 | candidates, |
| 341 | (candidate) => |
| 342 | Effect.gen(function* () { |
| 343 | if (stopDeleting || (yield* Clock.currentTimeMillis) >= deleteDeadline) { |
| 344 | stopDeleting = true; |
| 345 | truncated = true; |
| 346 | return; |
| 347 | } |
| 348 | attempted += 1; |
| 349 | const result = yield* provider |
| 350 | .release({ |
| 351 | userId: candidate.owner.userId, |
| 352 | environmentId: candidate.owner.environmentId, |
| 353 | expectedTunnelId: candidate.tunnel.id, |
| 354 | expectedInactiveBefore: candidate.inactiveBefore, |
| 355 | expectedStatus: candidate.status, |
| 356 | ...(candidate.legacy ? { markReleased: true } : {}), |
| 357 | }) |
| 358 | .pipe(Effect.result); |
| 359 | if (result._tag === "Failure") { |
| 360 | failed += 1; |
| 361 | yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { |
| 362 | tunnelId: candidate.tunnel.id, |
| 363 | tunnelName: candidate.tunnel.name, |
| 364 | legacy: candidate.legacy, |
| 365 | cause: result.failure, |
| 366 | }); |
| 367 | if (ManagedEndpointProvider.isCloudflareRateLimited(result.failure)) { |
| 368 | stopDeleting = true; |
| 369 | truncated = true; |
| 370 | } |
| 371 | } else if (result.success) { |
| 372 | deleted += 1; |
| 373 | if (candidate.legacy) deletedLegacy += 1; |
| 374 | yield* Effect.logInfo("Deleted an inactive managed tunnel", { |
| 375 | tunnelId: candidate.tunnel.id, |
| 376 | tunnelName: candidate.tunnel.name, |
| 377 | status: candidate.status, |
| 378 | legacy: candidate.legacy, |
| 379 | }); |
| 380 | } |
| 381 | }), |
| 382 | { concurrency: MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY, discard: true }, |
| 383 | ); |
| 384 | |
| 385 | return { |
| 386 | mode, |
| 387 | legacyMode, |
| 388 | listRequests, |
| 389 | scanned: uniqueExpired.length, |
| 390 | attempted, |
| 391 | deleted, |
| 392 | wouldDelete, |
| 393 | wouldDeleteLegacy, |
| 394 | deletedLegacy, |
| 395 | skippedLegacy, |
| 396 | legacyOver7Days: legacyOverDays.get(7) ?? 0, |
| 397 | legacyOver30Days: legacyOverDays.get(30) ?? 0, |
| 398 | legacyOver90Days: legacyOverDays.get(90) ?? 0, |
| 399 | skippedOrphan, |
| 400 | skippedReplaced, |
| 401 | skippedUnrecorded, |
| 402 | totalDown: totals.down, |
| 403 | totalInactive: totals.inactive, |
| 404 | failed, |
| 405 | truncated, |
| 406 | }; |
| 407 | }).pipe( |
| 408 | Effect.tap((result) => |
| 409 | Effect.annotateCurrentSpan( |
| 410 | Object.fromEntries( |
| 411 | Object.entries(result).map(([key, value]) => [ |
| 412 | `relay.managed_endpoint_reaper.${key}`, |
| 413 | value, |
| 414 | ]), |
| 415 | ), |
| 416 | ), |
| 417 | ), |
| 418 | Effect.withSpan("relay.managed_endpoint_reaper.sweep"), |
| 419 | ); |
| 420 | |
| 421 | return ManagedEndpointReaper.of({ sweep }); |
| 422 | }); |
| 423 | |
| 424 | export const layer = Layer.effect(ManagedEndpointReaper, make); |