The best way to control Claude Code, Codex and more from any computer or phone

main
ts 165 lines 6.45 KB
Raw
1 import * as ec2 from "@distilled.cloud/aws/ec2";
2 import * as Effect from "effect/Effect";
3 import * as Result from "effect/Result";
4
5 /**
6 * Shared teardown accelerator for network interfaces that AWS releases
7 * *asynchronously* after their owning resource is deleted.
8 *
9 * The canonical case is a VPC-attached Lambda Function: `DeleteFunction`
10 * returns immediately, but the function's Hyperplane ENIs stay `in-use` in
11 * their subnets/security groups for several minutes, and Lambda's own reaper
12 * may not delete the detached ENI for up to ~20 minutes. Until the ENI is
13 * gone, `DeleteSubnet`/`DeleteSecurityGroup` fail with `DependencyViolation`
14 * — historically forcing users to run `destroy` twice, minutes apart.
15 *
16 * `retryWhileLingeringEnis` wraps a subnet/security-group delete call (or an
17 * internet-gateway detach, which `DependencyViolation`s while any ENI in the
18 * VPC still holds a mapped public address): on every `DependencyViolation`
19 * it observes the ENIs still occupying the resource, explicitly deletes any
20 * *detached* (status `available`) Lambda ENI instead of waiting for Lambda's
21 * reaper, and — while the blockers are exclusively Lambda ENIs that are
22 * provably on their way out — keeps waiting on an extended budget that
23 * covers AWS's worst-case release window. Any other lingering dependency
24 * keeps today's shorter budget.
25 */
26
27 /**
28 * ENIs that their owning service tears down asynchronously and that are safe
29 * for us to delete once detached. Only Lambda Hyperplane ENIs qualify today:
30 * every other managed type (NAT gateways, VPC endpoints, ELB, EFS mount
31 * targets, ECS task ENIs) is deleted by the API call that deletes its owner
32 * — and their owners are proper alchemy resources ordered ahead of the
33 * subnet/SG by the deletion graph.
34 *
35 * Matching note (observed live): while attached, a Hyperplane ENI reports
36 * `InterfaceType: "lambda"` — but the moment Lambda detaches it, the type
37 * flips to plain `"interface"` and only the description (`AWS Lambda VPC
38 * ENI-{functionName}`) still identifies it. Match both, like Terraform does.
39 */
40 const isReapableEni = (eni: ec2.NetworkInterface): boolean =>
41 eni.InterfaceType === "lambda" || (eni.Description?.startsWith("AWS Lambda VPC ENI") ?? false);
42
43 export interface LingeringEniScope {
44 /** ENI filter naming the resource being deleted or detached. */
45 readonly name: "subnet-id" | "group-id" | "vpc-id";
46 readonly value: string;
47 }
48
49 /**
50 * Observe ENIs still occupying the resource and delete any detached
51 * reapable ones. Best-effort by design: the reaper only accelerates the
52 * outer DependencyViolation retry, so an account that cannot describe or
53 * delete ENIs degrades to plain waiting instead of failing the destroy.
54 */
55 const reapLingeringEnis = Effect.fn(function* (
56 scope: LingeringEniScope,
57 session: { note: (note: string) => Effect.Effect<void> },
58 ) {
59 const described = yield* ec2
60 .describeNetworkInterfaces({
61 Filters: [{ Name: scope.name, Values: [scope.value] }],
62 })
63 .pipe(Effect.orElseSucceed(() => ({ NetworkInterfaces: [] })));
64
65 const lingering = (described.NetworkInterfaces ?? []).filter(isReapableEni);
66
67 let reaped = 0;
68 let pending = 0;
69 for (const eni of lingering) {
70 if (eni.Status !== "available" || eni.NetworkInterfaceId === undefined) {
71 pending += 1;
72 continue;
73 }
74 const outcome = yield* ec2
75 .deleteNetworkInterface({
76 NetworkInterfaceId: eni.NetworkInterfaceId,
77 })
78 .pipe(
79 Effect.as("deleted" as const),
80 // Already gone counts as progress; anything else (raced back to
81 // in-use, throttle, missing IAM permission) must not fail the
82 // destroy — reaping is purely an accelerator, so degrade to the
83 // outer retry's plain waiting.
84 Effect.catchTag("InvalidNetworkInterfaceID.NotFound", () =>
85 Effect.succeed("deleted" as const),
86 ),
87 Effect.orElseSucceed(() => "pending" as const),
88 );
89 if (outcome === "deleted") {
90 yield* session.note(`Deleted detached Lambda ENI ${eni.NetworkInterfaceId}`);
91 reaped += 1;
92 } else {
93 pending += 1;
94 }
95 }
96
97 return {
98 /** Reapable ENIs still occupying the resource after this pass. */
99 pendingReapable: pending,
100 /** Detached reapable ENIs actually deleted this pass. */
101 reaped,
102 };
103 });
104
105 /**
106 * Retry `deleteCall` while it fails with a dependency violation, reaping
107 * lingering ENIs between attempts.
108 *
109 * - Base budget (~12 min) matches the historical subnet schedule: fast
110 * exponential start, capped at 30-second steps.
111 * - While the observed blockers include reapable ENIs (attached or just
112 * reaped), the budget extends to ~25 min — Lambda's documented worst-case
113 * ENI release window — because those blockers are guaranteed to clear.
114 */
115 export const retryWhileLingeringEnis = <A, E extends { _tag: string }, R>(
116 deleteCall: Effect.Effect<A, E, R>,
117 options: {
118 scope: LingeringEniScope;
119 isDependencyViolation: (error: E) => boolean;
120 session: { note: (note: string) => Effect.Effect<void> };
121 },
122 ) =>
123 Effect.gen(function* () {
124 const BASE_BUDGET_MILLIS = 12 * 60 * 1000;
125 const EXTENDED_BUDGET_MILLIS = 25 * 60 * 1000;
126
127 let elapsedMillis = 0;
128 let sawReapable = false;
129
130 for (let attempt = 1; ; attempt++) {
131 const result = yield* Effect.result(deleteCall);
132 if (Result.isSuccess(result)) {
133 return result.success;
134 }
135 const error = result.failure;
136 if (!options.isDependencyViolation(error)) {
137 return yield* Effect.fail(error);
138 }
139
140 const { pendingReapable, reaped } = yield* reapLingeringEnis(options.scope, options.session);
141 if (reaped > 0) {
142 // An ENI just left — the next attempt has a real chance; skip the
143 // backoff sleep and try immediately.
144 continue;
145 }
146 sawReapable ||= pendingReapable > 0;
147
148 const budget = sawReapable ? EXTENDED_BUDGET_MILLIS : BASE_BUDGET_MILLIS;
149 if (elapsedMillis >= budget) {
150 return yield* Effect.fail(error);
151 }
152
153 yield* options.session.note(
154 pendingReapable > 0
155 ? `Waiting for AWS to release ${pendingReapable} lingering ENI(s)... (attempt ${attempt})`
156 : `Waiting for dependencies to clear... (attempt ${attempt})`,
157 );
158
159 // Fast exponential start capped at 30-second steps (the historical
160 // subnet schedule).
161 const delayMillis = Math.min(1000 * 1.5 ** (attempt - 1), 30_000);
162 yield* Effect.sleep(delayMillis);
163 elapsedMillis += delayMillis;
164 }
165 });