The best way to control Claude Code, Codex and more from any computer or phone

main
ts 587 lines 19.7 KB
Raw
1 import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
2 import * as ec2 from "@distilled.cloud/aws/ec2";
3 import * as ecr from "@distilled.cloud/aws/ecr";
4 import * as ecs from "@distilled.cloud/aws/ecs";
5 import * as elbv2 from "@distilled.cloud/aws/elastic-load-balancing-v2";
6 import * as iam from "@distilled.cloud/aws/iam";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import * as Stream from "effect/Stream";
10
11 /**
12 * Orphan reclaim for the Fargate E2E smoke test (`Task.smoke.test.ts`).
13 *
14 * The smoke test's scratch stack keeps its state IN MEMORY, so a hard-killed
15 * run (vitest hard timeout, OOM, SIGKILL) orphans everything it deployed:
16 * the `AWS.ECS.Task` resource's out-of-band IAM roles / ECR repository / log
17 * group / task-definition family, the `AWS.ECS.Service`'s ALB + target group,
18 * and the EC2 networking chain (VPC/IGW/subnets/route tables/security group).
19 * Physical names carry a random per-instance suffix, so a later run cannot
20 * reclaim them through the engine — this sweep finds them by their
21 * DETERMINISTIC name prefixes (mirroring `createPhysicalName`) and by the
22 * `alchemy::stack` ownership tag, and deletes them idempotently.
23 *
24 * Run it BEFORE the test (pre-clean recovers a previously killed run) and as
25 * a finalizer AFTER (a passing run leaves ZERO leftovers — including the
26 * INACTIVE task-definition revisions that `deregisterTaskDefinition` alone
27 * leaves behind).
28 */
29
30 /**
31 * The exact test title — it becomes the scratch stack name (see
32 * `sanitizeStackName` in `Test/Core.ts`), which prefixes every physical name
33 * this test creates. The smoke test imports this constant so the title and
34 * the sweep can never drift apart.
35 */
36 export const E2E_TEST_TITLE =
37 "deploys a real Fargate task that serves HTTP and runs a background loop";
38
39 /** Deterministic cluster name pinned by the smoke test's `clusterName` prop. */
40 export const E2E_CLUSTER_NAME = "alchemy-test-ecs-task-e2e";
41
42 // Mirror of Test/Core.ts `sanitizeStackName`.
43 const STACK_NAME = E2E_TEST_TITLE.replaceAll(/[^a-zA-Z0-9_]/g, "-").replace(/-+/g, "-");
44
45 const STAGE = "test";
46
47 /**
48 * Mirror of `createPhysicalName`'s deterministic portion: the full name is
49 * `${stack}-${id}-${stage}-${16-char-suffix}` truncated to `maxLength` by
50 * slicing the prefix to `maxLength - 16`. Everything before the random
51 * suffix is deterministic — that's what we prefix-match on.
52 */
53 const physicalNamePrefix = (
54 id: string,
55 { maxLength, lowercase = false }: { maxLength: number; lowercase?: boolean },
56 ) => {
57 const prefix = `${STACK_NAME}-${id}-${STAGE}-`;
58 const sliced = prefix.length + 16 > maxLength ? prefix.slice(0, maxLength - 16) : prefix;
59 return (lowercase ? sliced.toLowerCase() : sliced).replaceAll(
60 lowercase ? /[^a-z0-9-]/g : /[^a-zA-Z0-9-]/g,
61 "-",
62 );
63 };
64
65 // AWS.ECS.Task out-of-band names (logical id "EcsE2ETask").
66 const ROLE_PREFIX = physicalNamePrefix("EcsE2ETask-task-role", {
67 maxLength: 64,
68 });
69 const REPO_PREFIX = physicalNamePrefix("EcsE2ETask-repo", {
70 maxLength: 256,
71 lowercase: true,
72 });
73 const LOG_GROUP_PREFIX = physicalNamePrefix("EcsE2ETask-logs", {
74 maxLength: 512,
75 lowercase: true,
76 });
77 const FAMILY_PREFIX = physicalNamePrefix("EcsE2ETask", {
78 maxLength: 255,
79 lowercase: true,
80 });
81 // AWS.ECS.Service managed ingress (logical id "EcsE2EService"). ALB/TG names
82 // are capped at 32 chars, so only the first 16 chars are deterministic —
83 // broad, so matches are additionally verified against the `alchemy::stack`
84 // tag before deletion.
85 const ALB_PREFIX = physicalNamePrefix("EcsE2EService-alb", {
86 maxLength: 32,
87 lowercase: true,
88 });
89 const TG_PREFIX = physicalNamePrefix("EcsE2EService-tg", {
90 maxLength: 32,
91 lowercase: true,
92 });
93
94 /**
95 * Bounded wait for the dependency-drain errors every step of the teardown can
96 * hit while ECS task / ALB ENIs release (~55s budget per step, only paid when
97 * an orphan actually exists).
98 */
99 const drainSchedule = Schedule.spaced("5 seconds");
100 const DRAIN_ATTEMPTS = 11;
101
102 const reclaimEcsCluster = Effect.gen(function* () {
103 // Delete any services in the deterministic cluster (force detaches them
104 // from the ALB and stops their tasks).
105 const serviceArns = yield* ecs.listServices({ cluster: E2E_CLUSTER_NAME }).pipe(
106 Effect.map((r) => r.serviceArns ?? []),
107 Effect.catchTag("ClusterNotFoundException", () => Effect.succeed([] as string[])),
108 );
109 yield* Effect.forEach(serviceArns, (serviceArn) =>
110 ecs
111 .deleteService({
112 cluster: E2E_CLUSTER_NAME,
113 service: serviceArn,
114 force: true,
115 })
116 .pipe(
117 Effect.catchTag(
118 ["ServiceNotFoundException", "ClusterNotFoundException"],
119 () => Effect.void,
120 ),
121 ),
122 );
123
124 // Stop any straggler tasks so the cluster drains quickly.
125 const taskArns = yield* ecs.listTasks({ cluster: E2E_CLUSTER_NAME }).pipe(
126 Effect.map((r) => r.taskArns ?? []),
127 Effect.catchTag("ClusterNotFoundException", () => Effect.succeed([] as string[])),
128 );
129 yield* Effect.forEach(taskArns, (taskArn) =>
130 ecs
131 .stopTask({
132 cluster: E2E_CLUSTER_NAME,
133 task: taskArn,
134 reason: "Task.smoke.test.ts orphan reclaim",
135 })
136 .pipe(
137 Effect.catchTag(
138 ["ClusterNotFoundException", "InvalidParameterException"],
139 () => Effect.void,
140 ),
141 ),
142 );
143
144 yield* ecs.deleteCluster({ cluster: E2E_CLUSTER_NAME }).pipe(
145 Effect.retry({
146 while: (e) =>
147 e._tag === "ClusterContainsServicesException" ||
148 e._tag === "ClusterContainsTasksException" ||
149 e._tag === "UpdateInProgressException",
150 schedule: drainSchedule,
151 times: DRAIN_ATTEMPTS,
152 }),
153 Effect.catchTag("ClusterNotFoundException", () => Effect.void),
154 Effect.asVoid,
155 );
156 });
157
158 // True when the resource's ELBv2 tags brand it as owned by this test's stack.
159 const isOwnedElbResource = (arn: string) =>
160 elbv2
161 .describeTags({ ResourceArns: [arn] })
162 .pipe(
163 Effect.map((r) =>
164 (r.TagDescriptions?.[0]?.Tags ?? []).some(
165 (tag) => tag.Key === "alchemy::stack" && tag.Value === STACK_NAME,
166 ),
167 ),
168 );
169
170 const reclaimIngress = Effect.gen(function* () {
171 const loadBalancers = yield* elbv2.describeLoadBalancers.items({}).pipe(
172 Stream.runCollect,
173 Effect.map((chunk) => Array.from(chunk)),
174 );
175 for (const lb of loadBalancers) {
176 if (
177 !lb.LoadBalancerArn ||
178 !lb.LoadBalancerName?.startsWith(ALB_PREFIX) ||
179 !(yield* isOwnedElbResource(lb.LoadBalancerArn))
180 ) {
181 continue;
182 }
183 yield* elbv2.deleteLoadBalancer({ LoadBalancerArn: lb.LoadBalancerArn }).pipe(
184 Effect.catchTag("LoadBalancerNotFoundException", () => Effect.void),
185 Effect.asVoid,
186 );
187 }
188
189 const targetGroups = yield* elbv2.describeTargetGroups.items({}).pipe(
190 Stream.runCollect,
191 Effect.map((chunk) => Array.from(chunk)),
192 );
193 for (const tg of targetGroups) {
194 if (
195 !tg.TargetGroupArn ||
196 !tg.TargetGroupName?.startsWith(TG_PREFIX) ||
197 !(yield* isOwnedElbResource(tg.TargetGroupArn))
198 ) {
199 continue;
200 }
201 // A TG stays in-use until its ALB's deletion propagates.
202 yield* elbv2.deleteTargetGroup({ TargetGroupArn: tg.TargetGroupArn }).pipe(
203 Effect.retry({
204 while: (e) => e._tag === "ResourceInUseException",
205 schedule: drainSchedule,
206 times: DRAIN_ATTEMPTS,
207 }),
208 Effect.asVoid,
209 );
210 }
211 });
212
213 const reclaimNetworking = Effect.gen(function* () {
214 const vpcs = yield* ec2
215 .describeVpcs({
216 Filters: [{ Name: "tag:alchemy::stack", Values: [STACK_NAME] }],
217 })
218 .pipe(Effect.map((r) => r.Vpcs ?? []));
219
220 for (const vpc of vpcs) {
221 const vpcId = vpc.VpcId!;
222
223 // Route tables: disassociate the non-main associations, delete non-main
224 // tables.
225 const routeTables = yield* ec2
226 .describeRouteTables({
227 Filters: [{ Name: "vpc-id", Values: [vpcId] }],
228 })
229 .pipe(Effect.map((r) => r.RouteTables ?? []));
230 for (const routeTable of routeTables) {
231 const associations = routeTable.Associations ?? [];
232 for (const association of associations) {
233 if (!association.Main && association.RouteTableAssociationId) {
234 yield* ec2
235 .disassociateRouteTable({
236 AssociationId: association.RouteTableAssociationId,
237 })
238 .pipe(
239 Effect.catchTag("InvalidAssociationID.NotFound", () => Effect.void),
240 Effect.asVoid,
241 );
242 }
243 }
244 if (routeTable.RouteTableId && !associations.some((association) => association.Main)) {
245 yield* ec2.deleteRouteTable({ RouteTableId: routeTable.RouteTableId }).pipe(
246 Effect.retry({
247 while: (e) => e._tag === "DependencyViolation",
248 schedule: drainSchedule,
249 times: DRAIN_ATTEMPTS,
250 }),
251 Effect.catchTag("InvalidRouteTableID.NotFound", () => Effect.void),
252 Effect.asVoid,
253 );
254 }
255 }
256
257 // Internet gateways: detach, then delete.
258 const igws = yield* ec2
259 .describeInternetGateways({
260 Filters: [{ Name: "attachment.vpc-id", Values: [vpcId] }],
261 })
262 .pipe(Effect.map((r) => r.InternetGateways ?? []));
263 for (const igw of igws) {
264 if (!igw.InternetGatewayId) continue;
265 yield* ec2
266 .detachInternetGateway({
267 InternetGatewayId: igw.InternetGatewayId,
268 VpcId: vpcId,
269 })
270 .pipe(
271 Effect.retry({
272 while: (e) => e._tag === "DependencyViolation",
273 schedule: drainSchedule,
274 times: DRAIN_ATTEMPTS,
275 }),
276 Effect.catchTag(
277 ["Gateway.NotAttached", "InvalidInternetGatewayID.NotFound"],
278 () => Effect.void,
279 ),
280 Effect.asVoid,
281 );
282 yield* ec2.deleteInternetGateway({ InternetGatewayId: igw.InternetGatewayId }).pipe(
283 Effect.retry({
284 while: (e) => e._tag === "DependencyViolation",
285 schedule: drainSchedule,
286 times: DRAIN_ATTEMPTS,
287 }),
288 Effect.catchTag("InvalidInternetGatewayID.NotFound", () => Effect.void),
289 Effect.asVoid,
290 );
291 }
292
293 // Security groups (waits out lingering task/ALB ENIs).
294 const securityGroups = yield* ec2
295 .describeSecurityGroups({
296 Filters: [{ Name: "vpc-id", Values: [vpcId] }],
297 })
298 .pipe(Effect.map((r) => r.SecurityGroups ?? []));
299 for (const securityGroup of securityGroups) {
300 if (!securityGroup.GroupId || securityGroup.GroupName === "default") {
301 continue;
302 }
303 yield* ec2.deleteSecurityGroup({ GroupId: securityGroup.GroupId }).pipe(
304 Effect.retry({
305 while: (e) => e._tag === "DependencyViolation",
306 schedule: drainSchedule,
307 times: DRAIN_ATTEMPTS,
308 }),
309 Effect.catchTag("InvalidGroup.NotFound", () => Effect.void),
310 Effect.asVoid,
311 );
312 }
313
314 // Subnets.
315 const subnets = yield* ec2
316 .describeSubnets({ Filters: [{ Name: "vpc-id", Values: [vpcId] }] })
317 .pipe(Effect.map((r) => r.Subnets ?? []));
318 for (const subnet of subnets) {
319 if (!subnet.SubnetId) continue;
320 yield* ec2.deleteSubnet({ SubnetId: subnet.SubnetId }).pipe(
321 Effect.retry({
322 while: (e) => e._tag === "DependencyViolation",
323 schedule: drainSchedule,
324 times: DRAIN_ATTEMPTS,
325 }),
326 Effect.catchTag("InvalidSubnetID.NotFound", () => Effect.void),
327 Effect.asVoid,
328 );
329 }
330
331 yield* ec2.deleteVpc({ VpcId: vpcId }).pipe(
332 Effect.retry({
333 while: (e) => e._tag === "DependencyViolation",
334 schedule: drainSchedule,
335 times: DRAIN_ATTEMPTS,
336 }),
337 Effect.catchTag("InvalidVpcID.NotFound", () => Effect.void),
338 Effect.asVoid,
339 );
340 }
341
342 // The smoke fixture normally reuses the standing default VPC, so its
343 // test-owned security group is not nested under a test-owned VPC above.
344 // Select it by the exact stack tag and leave every shared/default group
345 // untouched.
346 const ownedSecurityGroups = yield* ec2
347 .describeSecurityGroups({
348 Filters: [{ Name: "tag:alchemy::stack", Values: [STACK_NAME] }],
349 })
350 .pipe(Effect.map((r) => r.SecurityGroups ?? []));
351 for (const securityGroup of ownedSecurityGroups) {
352 if (!securityGroup.GroupId || securityGroup.GroupName === "default") {
353 continue;
354 }
355 yield* ec2.deleteSecurityGroup({ GroupId: securityGroup.GroupId }).pipe(
356 Effect.retry({
357 while: (e) => e._tag === "DependencyViolation",
358 schedule: drainSchedule,
359 times: DRAIN_ATTEMPTS,
360 }),
361 Effect.catchTag("InvalidGroup.NotFound", () => Effect.void),
362 Effect.asVoid,
363 );
364 }
365 });
366
367 const reclaimEcrRepositories = Effect.gen(function* () {
368 const repositories = yield* ecr.describeRepositories.items({}).pipe(
369 Stream.runCollect,
370 Effect.map((chunk) => Array.from(chunk)),
371 );
372 for (const repository of repositories) {
373 if (!repository.repositoryName?.startsWith(REPO_PREFIX)) continue;
374 yield* ecr
375 .deleteRepository({
376 repositoryName: repository.repositoryName,
377 force: true,
378 })
379 .pipe(
380 Effect.catchTag("RepositoryNotFoundException", () => Effect.void),
381 Effect.asVoid,
382 );
383 }
384 });
385
386 const reclaimLogGroups = Effect.gen(function* () {
387 const logGroups = yield* logs
388 .describeLogGroups({ logGroupNamePrefix: LOG_GROUP_PREFIX })
389 .pipe(Effect.map((r) => r.logGroups ?? []));
390 for (const logGroup of logGroups) {
391 if (!logGroup.logGroupName) continue;
392 yield* logs.deleteLogGroup({ logGroupName: logGroup.logGroupName }).pipe(
393 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
394 Effect.asVoid,
395 );
396 }
397 });
398
399 const reclaimIamRoles = Effect.gen(function* () {
400 const roles = yield* iam.listRoles.items({}).pipe(
401 Stream.runCollect,
402 Effect.map((chunk) => Array.from(chunk)),
403 );
404 for (const role of roles) {
405 const roleName = role.RoleName;
406 if (!roleName?.startsWith(ROLE_PREFIX)) continue;
407
408 const inlinePolicies = yield* iam.listRolePolicies({ RoleName: roleName }).pipe(
409 Effect.map((r) => r.PolicyNames ?? []),
410 Effect.catchTag("NoSuchEntityException", () => Effect.succeed([] as string[])),
411 );
412 yield* Effect.forEach(inlinePolicies, (policyName) =>
413 iam
414 .deleteRolePolicy({ RoleName: roleName, PolicyName: policyName })
415 .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void)),
416 );
417
418 const attached = yield* iam.listAttachedRolePolicies({ RoleName: roleName }).pipe(
419 Effect.map((r) => r.AttachedPolicies ?? []),
420 Effect.catchTag("NoSuchEntityException", () => Effect.succeed([])),
421 );
422 yield* Effect.forEach(attached, (policy) =>
423 iam
424 .detachRolePolicy({ RoleName: roleName, PolicyArn: policy.PolicyArn! })
425 .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void)),
426 );
427
428 yield* iam.deleteRole({ RoleName: roleName }).pipe(
429 Effect.catchTag("NoSuchEntityException", () => Effect.void),
430 Effect.asVoid,
431 );
432 }
433 });
434
435 /**
436 * Every task-definition family this test ever created (each run mints a
437 * fresh random-suffixed family). NOTE: `listTaskDefinitionFamilies` is a
438 * true prefix match; `listTaskDefinitions`' `familyPrefix` only matches the
439 * EXACT family name (verified live), so families must be enumerated first.
440 */
441 const listE2EFamilies = ecs.listTaskDefinitionFamilies
442 .items({ familyPrefix: FAMILY_PREFIX, status: "ALL" })
443 .pipe(
444 Stream.runCollect,
445 Effect.map((chunk) => Array.from(chunk)),
446 );
447
448 const listFamilyArns = (family: string, status: "ACTIVE" | "INACTIVE") =>
449 ecs
450 .listTaskDefinitions({ familyPrefix: family, status })
451 .pipe(Effect.map((r) => r.taskDefinitionArns ?? []));
452
453 const reclaimTaskDefinitions = Effect.gen(function* () {
454 const families = yield* listE2EFamilies;
455 for (const family of families) {
456 const active = yield* listFamilyArns(family, "ACTIVE");
457 yield* Effect.forEach(active, (arn) =>
458 ecs
459 .deregisterTaskDefinition({ taskDefinition: arn })
460 .pipe(Effect.catchTag("ClientException", () => Effect.void)),
461 );
462
463 const inactive = yield* listFamilyArns(family, "INACTIVE");
464 for (let i = 0; i < inactive.length; i += 10) {
465 yield* ecs
466 .deleteTaskDefinitions({ taskDefinitions: inactive.slice(i, i + 10) })
467 .pipe(Effect.catchTag("ClientException", () => Effect.void));
468 }
469 }
470 });
471
472 /**
473 * Idempotent full sweep, ordered so dependencies drain before their
474 * dependents are deleted: ECS service/cluster → ALB/TG → EC2 networking →
475 * ECR → log groups → IAM roles → task-definition revisions.
476 */
477 export const reclaimTaskE2EOrphans = Effect.gen(function* () {
478 yield* reclaimEcsCluster;
479 yield* reclaimIngress;
480 yield* reclaimNetworking;
481 yield* reclaimEcrRepositories;
482 yield* reclaimLogGroups;
483 yield* reclaimIamRoles;
484 yield* reclaimTaskDefinitions;
485 });
486
487 /**
488 * Read-only clean-slate audit: every cloud resource the smoke test can
489 * possibly leave behind, as human-readable identifiers. The smoke test
490 * asserts this is EMPTY after teardown, proving a passing run leaves zero
491 * leftovers.
492 */
493 export const scanTaskE2EOrphans = Effect.gen(function* () {
494 const leftovers: string[] = [];
495
496 const clusters = yield* ecs
497 .describeClusters({ clusters: [E2E_CLUSTER_NAME] })
498 .pipe(Effect.map((r) => r.clusters ?? []));
499 for (const cluster of clusters) {
500 if (cluster.status === "ACTIVE") {
501 leftovers.push(`ecs cluster ${cluster.clusterName}`);
502 }
503 }
504
505 const loadBalancers = yield* elbv2.describeLoadBalancers.items({}).pipe(
506 Stream.runCollect,
507 Effect.map((chunk) => Array.from(chunk)),
508 );
509 for (const lb of loadBalancers) {
510 if (
511 lb.LoadBalancerArn &&
512 lb.LoadBalancerName?.startsWith(ALB_PREFIX) &&
513 (yield* isOwnedElbResource(lb.LoadBalancerArn))
514 ) {
515 leftovers.push(`load balancer ${lb.LoadBalancerName}`);
516 }
517 }
518 const targetGroups = yield* elbv2.describeTargetGroups.items({}).pipe(
519 Stream.runCollect,
520 Effect.map((chunk) => Array.from(chunk)),
521 );
522 for (const tg of targetGroups) {
523 if (
524 tg.TargetGroupArn &&
525 tg.TargetGroupName?.startsWith(TG_PREFIX) &&
526 (yield* isOwnedElbResource(tg.TargetGroupArn))
527 ) {
528 leftovers.push(`target group ${tg.TargetGroupName}`);
529 }
530 }
531
532 const vpcs = yield* ec2
533 .describeVpcs({
534 Filters: [{ Name: "tag:alchemy::stack", Values: [STACK_NAME] }],
535 })
536 .pipe(Effect.map((r) => r.Vpcs ?? []));
537 for (const vpc of vpcs) {
538 leftovers.push(`vpc ${vpc.VpcId}`);
539 }
540 const securityGroups = yield* ec2
541 .describeSecurityGroups({
542 Filters: [{ Name: "tag:alchemy::stack", Values: [STACK_NAME] }],
543 })
544 .pipe(Effect.map((r) => r.SecurityGroups ?? []));
545 for (const securityGroup of securityGroups) {
546 leftovers.push(`security group ${securityGroup.GroupId}`);
547 }
548
549 const repositories = yield* ecr.describeRepositories.items({}).pipe(
550 Stream.runCollect,
551 Effect.map((chunk) => Array.from(chunk)),
552 );
553 for (const repository of repositories) {
554 if (repository.repositoryName?.startsWith(REPO_PREFIX)) {
555 leftovers.push(`ecr repository ${repository.repositoryName}`);
556 }
557 }
558
559 const logGroups = yield* logs
560 .describeLogGroups({ logGroupNamePrefix: LOG_GROUP_PREFIX })
561 .pipe(Effect.map((r) => r.logGroups ?? []));
562 for (const logGroup of logGroups) {
563 leftovers.push(`log group ${logGroup.logGroupName}`);
564 }
565
566 const roles = yield* iam.listRoles.items({}).pipe(
567 Stream.runCollect,
568 Effect.map((chunk) => Array.from(chunk)),
569 );
570 for (const role of roles) {
571 if (role.RoleName?.startsWith(ROLE_PREFIX)) {
572 leftovers.push(`iam role ${role.RoleName}`);
573 }
574 }
575
576 const families = yield* listE2EFamilies;
577 for (const family of families) {
578 for (const status of ["ACTIVE", "INACTIVE"] as const) {
579 const arns = yield* listFamilyArns(family, status);
580 for (const arn of arns) {
581 leftovers.push(`task definition ${arn}`);
582 }
583 }
584 }
585
586 return leftovers;
587 });