@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 1,620 lines 58.5 KB
Raw
1 import { createHash, randomBytes, randomUUID } from "node:crypto";
2 /**
3 * Standing prerequisites for the Floci-gated AWS suites (EKS, AMP scraper,
4 * SageMaker HyperPod and endpoints, RDS proxy, ECS capacity providers, Flink,
5 * custom domains, API Gateway VPC links, SES, OAM, Control Tower). They are
6 * created through the emulated AWS APIs before `test:aws:floci` spawns the
7 * suite, exported as the env vars the suites read, and removed after the run.
8 *
9 * Every call uses the credentials and Region of
10 * `packages/aws/src/Local/FlociServices.ts` (access key `test`,
11 * us-east-1): Floci scopes state by access key and Region, and that is what
12 * the local providers and the test-body SDK clients use under
13 * `ALCHEMY_TEST_DEV=1`. The OAM sink lives in a second emulator account,
14 * selected by a 12-digit access key (Floci core/common/AccountResolver.java).
15 */
16 import * as Floci from "@alchemy.run/aws/Local/Floci";
17 import * as acm from "@distilled.cloud/aws/acm";
18 import * as autoscaling from "@distilled.cloud/aws/auto-scaling";
19 import * as controltower from "@distilled.cloud/aws/controltower";
20 import { Credentials } from "@distilled.cloud/aws/Credentials";
21 import * as EC2 from "@distilled.cloud/aws/ec2";
22 import * as eks from "@distilled.cloud/aws/eks";
23 import * as elbv2 from "@distilled.cloud/aws/elastic-load-balancing-v2";
24 import * as Endpoint from "@distilled.cloud/aws/Endpoint";
25 import * as iam from "@distilled.cloud/aws/iam";
26 import * as oam from "@distilled.cloud/aws/oam";
27 import { Region, type RegionName } from "@distilled.cloud/aws/Region";
28 import * as route53 from "@distilled.cloud/aws/route-53";
29 import * as s3 from "@distilled.cloud/aws/s3";
30 import * as sagemaker from "@distilled.cloud/aws/sagemaker";
31 import * as secretsmanager from "@distilled.cloud/aws/secrets-manager";
32 import * as ses from "@distilled.cloud/aws/ses";
33 import * as sesv2 from "@distilled.cloud/aws/sesv2";
34 import * as ssm from "@distilled.cloud/aws/ssm";
35 import * as Cause from "effect/Cause";
36 import * as Config from "effect/Config";
37 import * as Console from "effect/Console";
38 import * as Data from "effect/Data";
39 import * as Effect from "effect/Effect";
40 import * as Exit from "effect/Exit";
41 import { FetchHttpClient } from "effect/http";
42 import * as HttpClient from "effect/http/HttpClient";
43 import * as Layer from "effect/Layer";
44 import * as Path from "effect/Path";
45 import * as ChildProcess from "effect/process/ChildProcess";
46 import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
47 import * as Redacted from "effect/Redacted";
48 import * as Schedule from "effect/Schedule";
49 import * as Stream from "effect/Stream";
50
51 export interface FlociStandingFixtures {
52 readonly env: Record<string, string>;
53 /** Removes every provisioned fixture, newest first. */
54 readonly teardown: Effect.Effect<void, FixtureError, PlatformServices>;
55 }
56
57 export interface FlociStandingFixtureOptions {
58 /** Selected test files (relative or absolute). Omit to provision every group. */
59 readonly files?: ReadonlyArray<string>;
60 }
61
62 const NAME = "alchemy-floci-standing";
63 const FIXTURE_TAG_KEY = "alchemy-floci-fixture";
64 const FIXTURE_TAG_VALUE = "standing";
65 const VPC_CIDR = "10.250.0.0/16";
66 // Must match FLOCI_REGION / FLOCI_ACCOUNT_ID in
67 // packages/aws/src/Local/FlociServices.ts.
68 const REGION = "us-east-1" as RegionName;
69 const ACCOUNT_ID = "000000000000";
70 // Second emulator account for the cross-account OAM sink.
71 const OAM_SINK_ACCOUNT_ID = "111111111111";
72 const LANDING_ZONE_VERSION = "3.3";
73 // Legacy detective guardrail; Floci's control catalog registers it
74 // (services/controlcatalog/ControlCatalogService.java).
75 const CONTROL_ARN = `arn:aws:controltower:${REGION}::control/AWS-GR_ENCRYPTED_VOLUMES`;
76 // Floci accepts 3.0/4.0/5.0 for AWSControlTowerBaseline; 4.0 pairs with landing zone 3.3.
77 const BASELINE_VERSION = "4.0";
78 const FLINK_JAR_KEY = "flink/flink-examples-streaming-1.20.0-TopSpeedWindowing.jar";
79 const FLINK_JAR_URL =
80 "https://repo1.maven.org/maven2/org/apache/flink/flink-examples-streaming/1.20.0/flink-examples-streaming-1.20.0-TopSpeedWindowing.jar";
81 const FLINK_JAR_SHA1 = "a9466bf8f379dce881d06236c752b84374abc54a";
82 const DOMAIN_ZONE = `${NAME}.test`;
83 const SES_ZONE = `${NAME}.example.com`;
84 const SES_REDIRECT_DOMAIN = `track.${SES_ZONE}`;
85 const SES_FROM = `${NAME}@example.com`;
86 // Default BounceSender of test/AWS/SES/handler.ts `/send-bounce`.
87 const SES_BOUNCE_SENDER = "mailer-daemon@ses-bindings.alchemy-test.example.com";
88 const SES_CVE_TEMPLATE = `${NAME}-cve`;
89 const SES_CVE_RECIPIENT = `${NAME}-cve-recipient@example.com`;
90 const SAGEMAKER_SERVE_IMAGE = "alchemy-floci/sagemaker-serve:1";
91 const log = (message: string) => Console.log(`test:aws:floci: ${message}`);
92
93 type Group =
94 | "eksRole"
95 | "eksCluster"
96 | "amp"
97 | "hyperpod"
98 | "rdsProxy"
99 | "ecsAsg"
100 | "flink"
101 | "domain"
102 | "vpcLink"
103 | "ses"
104 | "oam"
105 | "sagemakerEndpoint"
106 | "controlTower";
107
108 const groupVars: Record<Group, ReadonlyArray<string>> = {
109 eksRole: ["AWS_TEST_EKS_ROLE_ARN", "AWS_TEST_EKS_SUBNET_IDS"],
110 eksCluster: [
111 "AWS_TEST_EKS_CLUSTER",
112 "AWS_TEST_EKS_PRIVATE_SUBNETS",
113 "AWS_TEST_EKS_NODE_ROLE_ARN",
114 "AWS_TEST_EKS_FARGATE_ROLE_ARN",
115 "AWS_TEST_EKS_POD_ROLE_ARN",
116 ],
117 amp: [
118 "AWS_TEST_AMP_SCRAPER",
119 "AWS_TEST_AMP_SCRAPER_CLUSTER_ARN",
120 "AWS_TEST_AMP_SCRAPER_SUBNET_IDS",
121 ],
122 hyperpod: ["AWS_TEST_SAGEMAKER_HYPERPOD_EKS_CLUSTER_ARN"],
123 rdsProxy: [
124 "AWS_TEST_RDS_DBPROXY",
125 "DBPROXY_SUBNET_IDS",
126 "DBPROXY_ROLE_ARN",
127 "DBPROXY_SECRET_ARN",
128 ],
129 ecsAsg: ["TEST_ASG_ARN"],
130 flink: ["AWS_TEST_FLINK_START", "AWS_TEST_FLINK_JAR_BUCKET_ARN", "AWS_TEST_FLINK_JAR_KEY"],
131 domain: [
132 "AWS_TEST_DOMAIN",
133 "AWS_TEST_APIGATEWAY_DOMAIN",
134 "AWS_TEST_ACM_CERTIFICATE_ARN",
135 "AWS_TEST_APIGATEWAY_DOMAIN_NAME",
136 "AWS_TEST_APIGATEWAY_CERT_ARN",
137 "AWS_TEST_APIGATEWAYV2_DOMAIN_NAME",
138 "AWS_TEST_APPSYNC_DOMAIN_NAME",
139 "AWS_TEST_APPSYNC_DOMAIN_CERT_ARN",
140 "AWS_TEST_HOSTED_ZONE",
141 ],
142 vpcLink: ["ALCHEMY_TEST_VPC_LINK_TARGET_ARN"],
143 ses: [
144 "AWS_TEST_SES_FROM",
145 "AWS_TEST_SES_BOUNCE_MESSAGE_ID",
146 "AWS_TEST_SES_CVE_TEMPLATE",
147 "AWS_TEST_SES_CVE_RECIPIENT",
148 "AWS_TEST_SES_REDIRECT_DOMAIN",
149 "AWS_TEST_SES_VDM",
150 ],
151 oam: ["AWS_TEST_OAM_SINK_ARN"],
152 sagemakerEndpoint: ["AWS_TEST_SAGEMAKER_ENDPOINT", "AWS_TEST_SAGEMAKER_IMAGE"],
153 controlTower: [
154 "AWS_TEST_CONTROLTOWER",
155 "AWS_TEST_CONTROLTOWER_CONTROL",
156 "AWS_TEST_CONTROLTOWER_BASELINE_VERSION",
157 ],
158 };
159
160 // Suites (under packages/integration-tests/test/AWS) gated on each group's vars.
161 const groupSuites: Record<Group, ReadonlyArray<string>> = {
162 eksRole: ["EKS/Cluster.test.ts", "EKS/ClusterBindings.test.ts"],
163 eksCluster: [
164 "EKS/AccessEntry.test.ts",
165 "EKS/Addon.test.ts",
166 "EKS/FargateProfile.test.ts",
167 "EKS/Nodegroup.test.ts",
168 "EKS/PodIdentityAssociation.test.ts",
169 ],
170 amp: ["AMP/Scraper.test.ts"],
171 hyperpod: ["SageMaker/ClusterSchedulerConfig.test.ts", "SageMaker/ComputeQuota.test.ts"],
172 rdsProxy: [
173 "RDS/DBProxy.test.ts",
174 "RDS/DBProxyEndpoint.test.ts",
175 "RDS/DBProxyTargetGroup.test.ts",
176 ],
177 ecsAsg: ["ECS/CapacityProvider.test.ts", "ECS/CapacityProvider.smoke.test.ts"],
178 flink: ["KinesisAnalyticsV2/ApplicationSnapshot.test.ts"],
179 domain: [
180 "ApiGateway/BasePathMapping.test.ts",
181 "ApiGateway/DomainName.test.ts",
182 "ApiGatewayV2/DomainName.test.ts",
183 "AppSync/DomainName.test.ts",
184 "ECS/ServiceDomain.test.ts",
185 "Website/RouterHostnameBinding.test.ts",
186 ],
187 vpcLink: ["ApiGateway/VpcLink.test.ts"],
188 ses: [
189 "SES/Bindings.test.ts",
190 "SES/ConfigurationSet.test.ts",
191 "SES/CustomVerificationEmailTemplate.test.ts",
192 ],
193 oam: ["OAM/Sink.test.ts"],
194 sagemakerEndpoint: ["SageMaker/Endpoint.test.ts", "SageMaker/EndpointBindings.test.ts"],
195 controlTower: ["ControlTower/ControlTower.test.ts"],
196 };
197
198 export class FixtureError extends Data.TaggedError("FlociFixtureError")<{
199 readonly message: string;
200 }> {}
201
202 const fixtureError = (message: string) => Effect.fail(new FixtureError({ message }));
203
204 const credentialsFor = (accessKeyId: string) =>
205 Layer.succeed(
206 Credentials,
207 Effect.succeed({
208 accessKeyId: Redacted.make(accessKeyId),
209 secretAccessKey: Redacted.make("test"),
210 sessionToken: undefined,
211 region: REGION,
212 }),
213 );
214
215 const fixtureLayer = Layer.mergeAll(
216 credentialsFor("test"),
217 Layer.succeed(Region, Effect.succeed(REGION)),
218 Endpoint.of(`http://localhost:${Floci.DEFAULT_FLOCI_PORT}`),
219 FetchHttpClient.layer,
220 );
221
222 /** Platform services the caller provides (e.g. `NodeServices.layer`). */
223 type PlatformServices = Path.Path | ChildProcessSpawner.ChildProcessSpawner;
224
225 type Services = Layer.Success<typeof fixtureLayer> | PlatformServices;
226
227 interface Cleanup {
228 readonly label: string;
229 readonly run: Effect.Effect<void, unknown, Services>;
230 }
231
232 /** Runs `effect` as the second emulator account (Floci keys accounts by access key). */
233 const asSinkAccount = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
234 effect.pipe(Effect.provide(credentialsFor(OAM_SINK_ACCOUNT_ID)));
235
236 const register = (ledger: Array<Cleanup>, cleanup: Cleanup) =>
237 Effect.sync(() => {
238 ledger.push(cleanup);
239 });
240
241 /** Runs cleanups newest-first, continuing past failures. */
242 const runCleanups = (ledger: Array<Cleanup>) =>
243 Effect.gen(function* () {
244 const failures: Array<string> = [];
245 for (const cleanup of ledger.splice(0).reverse()) {
246 const exit = yield* Effect.exit(cleanup.run);
247 if (Exit.isFailure(exit)) {
248 failures.push(`${cleanup.label}: ${Cause.pretty(exit.cause)}`);
249 } else {
250 yield* log(`removed ${cleanup.label}`);
251 }
252 }
253 if (failures.length > 0) {
254 return yield* fixtureError(`could not remove standing fixtures:\n${failures.join("\n")}`);
255 }
256 });
257
258 // ─── EC2 ─────────────────────────────────────────────────────────────────────
259
260 const nameTags = (name: string) => [
261 { Key: "Name", Value: name },
262 { Key: FIXTURE_TAG_KEY, Value: FIXTURE_TAG_VALUE },
263 ];
264
265 const deleteVpc = (vpcId: string) =>
266 EC2.deleteVpc({ VpcId: vpcId }).pipe(
267 Effect.retry({
268 while: (error) => error._tag === "DependencyViolation",
269 schedule: Schedule.spaced("5 seconds"),
270 times: 12,
271 }),
272 Effect.catchTag("InvalidVpcID.NotFound", () => Effect.void),
273 );
274
275 const deleteSubnet = (subnetId: string) =>
276 EC2.deleteSubnet({ SubnetId: subnetId }).pipe(
277 Effect.retry({
278 while: (error) => error._tag === "DependencyViolation",
279 schedule: Schedule.spaced("5 seconds"),
280 times: 12,
281 }),
282 Effect.catchTag("InvalidSubnetID.NotFound", () => Effect.void),
283 );
284
285 const ensureVpc = (ledger: Array<Cleanup>) =>
286 Effect.gen(function* () {
287 const observed = yield* EC2.describeVpcs({
288 Filters: [
289 { Name: "tag:Name", Values: [NAME] },
290 { Name: `tag:${FIXTURE_TAG_KEY}`, Values: [FIXTURE_TAG_VALUE] },
291 ],
292 });
293 let vpcId = observed.Vpcs?.find((vpc) => vpc.CidrBlock === VPC_CIDR)?.VpcId;
294 if (vpcId === undefined) {
295 const created = yield* EC2.createVpc({
296 CidrBlock: VPC_CIDR,
297 TagSpecifications: [{ ResourceType: "vpc", Tags: nameTags(NAME) }],
298 });
299 vpcId = created.Vpc?.VpcId;
300 if (vpcId === undefined) {
301 return yield* fixtureError("CreateVpc returned no VpcId");
302 }
303 }
304 const id = vpcId;
305 yield* register(ledger, { label: `VPC ${id}`, run: deleteVpc(id) });
306 return id;
307 });
308
309 const ensurePrivateSubnet = (ledger: Array<Cleanup>, vpcId: string, index: number, zone: string) =>
310 Effect.gen(function* () {
311 const name = `${NAME}-private-${index + 1}`;
312 const cidr = `10.250.${index + 1}.0/24`;
313 const observed = yield* EC2.describeSubnets({
314 Filters: [
315 { Name: "vpc-id", Values: [vpcId] },
316 { Name: "tag:Name", Values: [name] },
317 ],
318 });
319 const existing = observed.Subnets?.find(
320 (subnet) => subnet.VpcId === vpcId && subnet.SubnetId !== undefined,
321 );
322 if (
323 existing?.SubnetId !== undefined &&
324 existing.AvailabilityZone === zone &&
325 existing.CidrBlock === cidr
326 ) {
327 const id = existing.SubnetId;
328 yield* register(ledger, { label: `subnet ${id}`, run: deleteSubnet(id) });
329 return id;
330 }
331 if (existing?.SubnetId !== undefined) {
332 yield* deleteSubnet(existing.SubnetId);
333 }
334 const created = yield* EC2.createSubnet({
335 VpcId: vpcId,
336 CidrBlock: cidr,
337 AvailabilityZone: zone,
338 TagSpecifications: [{ ResourceType: "subnet", Tags: nameTags(name) }],
339 });
340 const id = created.Subnet?.SubnetId;
341 if (id === undefined) {
342 return yield* fixtureError("CreateSubnet returned no SubnetId");
343 }
344 yield* register(ledger, { label: `subnet ${id}`, run: deleteSubnet(id) });
345 return id;
346 });
347
348 interface Network {
349 readonly vpcId: string;
350 readonly subnetIds: ReadonlyArray<string>;
351 }
352
353 /** Two private subnets in two AZs; the VPC never gets an internet gateway. */
354 const ensureNetwork = (ledger: Array<Cleanup>) =>
355 Effect.gen(function* () {
356 const vpcId = yield* ensureVpc(ledger);
357 const zones = yield* EC2.describeAvailabilityZones({});
358 const names = (zones.AvailabilityZones ?? [])
359 .filter((zone) => (zone.State ?? "available") === "available")
360 .flatMap((zone) => (zone.ZoneName === undefined ? [] : [zone.ZoneName]))
361 .sort();
362 if (names.length < 2) {
363 return yield* fixtureError(
364 `${REGION} reports ${names.length} available zone(s); two are required`,
365 );
366 }
367 const subnetIds = yield* Effect.forEach([0, 1], (index) =>
368 ensurePrivateSubnet(ledger, vpcId, index, names[index]!),
369 );
370 const tables = yield* EC2.describeRouteTables({
371 Filters: [{ Name: "vpc-id", Values: [vpcId] }],
372 });
373 const publicRoute = (tables.RouteTables ?? [])
374 .filter((table) => table.VpcId === vpcId)
375 .flatMap((table) => table.Routes ?? [])
376 .find((route) => route.GatewayId?.startsWith("igw-"));
377 if (publicRoute !== undefined) {
378 return yield* fixtureError(
379 `VPC ${vpcId} routes ${publicRoute.DestinationCidrBlock} to ${publicRoute.GatewayId}; the standing subnets must stay private`,
380 );
381 }
382 const network: Network = { vpcId, subnetIds };
383 return network;
384 });
385
386 // ─── IAM ─────────────────────────────────────────────────────────────────────
387
388 const trustPolicy = (service: string, actions: ReadonlyArray<string>) =>
389 JSON.stringify({
390 Version: "2012-10-17",
391 Statement: [{ Effect: "Allow", Principal: { Service: service }, Action: actions }],
392 });
393
394 const deleteRole = (roleName: string) =>
395 Effect.gen(function* () {
396 const attached = yield* iam.listAttachedRolePolicies({
397 RoleName: roleName,
398 });
399 for (const policy of attached.AttachedPolicies ?? []) {
400 if (policy.PolicyArn === undefined) continue;
401 yield* iam
402 .detachRolePolicy({ RoleName: roleName, PolicyArn: policy.PolicyArn })
403 .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void));
404 }
405 yield* iam.deleteRole({ RoleName: roleName });
406 }).pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void));
407
408 const ensureRole = (
409 ledger: Array<Cleanup>,
410 suffix: string,
411 service: string,
412 actions: ReadonlyArray<string>,
413 managedPolicies: ReadonlyArray<string>,
414 ) =>
415 Effect.gen(function* () {
416 const roleName = `${NAME}-${suffix}`;
417 const document = trustPolicy(service, actions);
418 const getRole = iam
419 .getRole({ RoleName: roleName })
420 .pipe(Effect.map((response) => response.Role));
421 const existing = yield* getRole.pipe(
422 Effect.catchTag("NoSuchEntityException", () => Effect.undefined),
423 );
424 const role =
425 existing === undefined
426 ? yield* iam
427 .createRole({
428 RoleName: roleName,
429 AssumeRolePolicyDocument: document,
430 Tags: [{ Key: FIXTURE_TAG_KEY, Value: FIXTURE_TAG_VALUE }],
431 })
432 .pipe(
433 Effect.map((response) => response.Role),
434 Effect.catchTag("EntityAlreadyExistsException", () => getRole),
435 )
436 : existing;
437 yield* register(ledger, {
438 label: `IAM role ${roleName}`,
439 run: deleteRole(roleName),
440 });
441 if (existing !== undefined) {
442 yield* iam.updateAssumeRolePolicy({
443 RoleName: roleName,
444 PolicyDocument: document,
445 });
446 }
447 // Floci resolves managed policy ARNs against the published catalog
448 // (services/iam/AwsManagedPolicies.java), so a typo fails here.
449 for (const policy of managedPolicies) {
450 yield* iam.attachRolePolicy({
451 RoleName: roleName,
452 PolicyArn: `arn:aws:iam::aws:policy/${policy}`,
453 });
454 }
455 return role.Arn;
456 });
457
458 // ─── EKS ─────────────────────────────────────────────────────────────────────
459
460 const findEksCluster = eks.describeCluster({ name: NAME }).pipe(
461 Effect.map((response) => response.cluster),
462 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
463 );
464
465 const deleteEksCluster = Effect.gen(function* () {
466 yield* eks
467 .deleteCluster({ name: NAME })
468 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
469 const remaining = yield* findEksCluster.pipe(
470 Effect.repeat({
471 schedule: Schedule.spaced("5 seconds"),
472 until: (cluster): boolean => cluster === undefined,
473 times: 60,
474 }),
475 );
476 if (remaining !== undefined) {
477 return yield* fixtureError(`EKS cluster ${NAME} is still ${remaining.status} after deletion`);
478 }
479 });
480
481 const ensureEksCluster = (ledger: Array<Cleanup>, network: Network, roleArn: string) =>
482 Effect.gen(function* () {
483 const existing = yield* findEksCluster;
484 const reusable =
485 existing !== undefined &&
486 existing.status !== "FAILED" &&
487 existing.status !== "DELETING" &&
488 existing.roleArn === roleArn &&
489 existing.resourcesVpcConfig?.vpcId === network.vpcId &&
490 existing.accessConfig?.authenticationMode === "API_AND_CONFIG_MAP";
491 yield* register(ledger, {
492 label: `EKS cluster ${NAME}`,
493 run: deleteEksCluster,
494 });
495 if (existing !== undefined && !reusable) {
496 yield* log(`replacing EKS cluster ${NAME} (${existing.status})`);
497 yield* deleteEksCluster;
498 }
499 if (!reusable) {
500 yield* log(`creating EKS cluster ${NAME} (waits for ACTIVE)`);
501 yield* eks
502 .createCluster({
503 name: NAME,
504 roleArn,
505 resourcesVpcConfig: {
506 subnetIds: [...network.subnetIds],
507 endpointPublicAccess: true,
508 endpointPrivateAccess: true,
509 },
510 accessConfig: { authenticationMode: "API_AND_CONFIG_MAP" },
511 tags: { [FIXTURE_TAG_KEY]: FIXTURE_TAG_VALUE },
512 })
513 .pipe(Effect.catchTag("ResourceInUseException", () => Effect.void));
514 }
515 // Floci boots a k3s container and flips CREATING -> ACTIVE once its API
516 // server answers (services/eks/EksService.java startReadinessPoller).
517 const cluster = yield* findEksCluster.pipe(
518 Effect.repeat({
519 schedule: Schedule.spaced("5 seconds"),
520 until: (cluster): boolean =>
521 cluster?.status === undefined ||
522 cluster.status === "ACTIVE" ||
523 cluster.status === "FAILED",
524 times: 120,
525 }),
526 );
527 if (cluster?.status !== "ACTIVE" || cluster.arn === undefined) {
528 return yield* fixtureError(
529 `EKS cluster ${NAME} did not become ACTIVE within 10 minutes (status: ${cluster?.status ?? "missing"})`,
530 );
531 }
532 return {
533 name: NAME,
534 arn: cluster.arn,
535 securityGroupId: cluster.resourcesVpcConfig?.clusterSecurityGroupId,
536 created: !reusable,
537 };
538 });
539
540 // ─── SageMaker HyperPod ──────────────────────────────────────────────────────
541
542 // DescribeCluster has no typed not-found tag in distilled; observe by listing.
543 const findHyperPodCluster = sagemaker
544 .listClusters({ NameContains: NAME })
545 .pipe(
546 Effect.map((response) =>
547 response.ClusterSummaries.find((summary) => summary.ClusterName === NAME),
548 ),
549 );
550
551 const deleteHyperPodCluster = Effect.gen(function* () {
552 yield* sagemaker.deleteCluster({ ClusterName: NAME }).pipe(
553 Effect.retry({
554 while: (error) => error._tag === "ConflictException",
555 schedule: Schedule.spaced("3 seconds"),
556 times: 20,
557 }),
558 Effect.catchTag("ResourceNotFound", () => Effect.void),
559 );
560 const remaining = yield* findHyperPodCluster.pipe(
561 Effect.repeat({
562 schedule: Schedule.spaced("3 seconds"),
563 until: (cluster): boolean => cluster === undefined,
564 times: 40,
565 }),
566 );
567 if (remaining !== undefined) {
568 return yield* fixtureError(
569 `HyperPod cluster ${NAME} is still ${remaining.ClusterStatus} after deletion`,
570 );
571 }
572 });
573
574 const ensureHyperPodCluster = (
575 ledger: Array<Cleanup>,
576 eksCluster: {
577 readonly arn: string;
578 readonly securityGroupId: string | undefined;
579 readonly created: boolean;
580 },
581 subnetIds: ReadonlyArray<string>,
582 executionRoleArn: string,
583 ) =>
584 Effect.gen(function* () {
585 if (eksCluster.securityGroupId === undefined) {
586 return yield* fixtureError(`EKS cluster ${NAME} reports no cluster security group`);
587 }
588 const existing = yield* findHyperPodCluster;
589 yield* register(ledger, {
590 label: `SageMaker HyperPod cluster ${NAME}`,
591 run: deleteHyperPodCluster,
592 });
593 const reusable =
594 existing !== undefined &&
595 !eksCluster.created &&
596 !["Failed", "Deleting", "RollingBack"].includes(existing.ClusterStatus);
597 if (existing !== undefined && !reusable) {
598 yield* log(`replacing HyperPod cluster ${NAME} (${existing.ClusterStatus})`);
599 yield* deleteHyperPodCluster;
600 }
601 if (!reusable) {
602 // EKS-orchestrated groups need no lifecycle scripts; Floci validates the
603 // execution role and an ACTIVE API_AND_CONFIG_MAP EKS cluster
604 // (services/sagemaker/SageMakerHyperPodService.java).
605 yield* sagemaker
606 .createCluster({
607 ClusterName: NAME,
608 Orchestrator: { Eks: { ClusterArn: eksCluster.arn } },
609 InstanceGroups: [
610 {
611 InstanceGroupName: "standing",
612 InstanceType: "ml.t3.medium",
613 InstanceCount: 1,
614 ExecutionRole: executionRoleArn,
615 },
616 ],
617 VpcConfig: {
618 SecurityGroupIds: [eksCluster.securityGroupId],
619 Subnets: [...subnetIds],
620 },
621 NodeRecovery: "Automatic",
622 Tags: [{ Key: FIXTURE_TAG_KEY, Value: FIXTURE_TAG_VALUE }],
623 })
624 .pipe(Effect.catchTag("ResourceInUse", () => Effect.void));
625 }
626 const cluster = yield* findHyperPodCluster.pipe(
627 Effect.repeat({
628 schedule: Schedule.spaced("3 seconds"),
629 until: (cluster): boolean =>
630 cluster?.ClusterStatus !== "Creating" && cluster?.ClusterStatus !== "Updating",
631 times: 40,
632 }),
633 );
634 if (cluster?.ClusterStatus !== "InService") {
635 return yield* fixtureError(
636 `HyperPod cluster ${NAME} did not reach InService (status: ${cluster?.ClusterStatus ?? "missing"})`,
637 );
638 }
639 return cluster.ClusterArn;
640 });
641
642 // ─── RDS proxy ───────────────────────────────────────────────────────────────
643
644 const RDS_PROXY_SECRET = `${NAME}-rds-proxy`;
645
646 const ensureRdsProxySecret = (ledger: Array<Cleanup>) =>
647 Effect.gen(function* () {
648 const password = yield* Effect.sync(() => randomBytes(24).toString("base64url"));
649 const arn = yield* secretsmanager
650 .createSecret({
651 Name: RDS_PROXY_SECRET,
652 Description: "Standing DB proxy credentials for the Floci test run",
653 SecretString: JSON.stringify({ username: "alchemy", password }),
654 Tags: [{ Key: FIXTURE_TAG_KEY, Value: FIXTURE_TAG_VALUE }],
655 })
656 .pipe(
657 Effect.map((response) => response.ARN),
658 Effect.catchTag("ResourceExistsException", () =>
659 secretsmanager
660 .describeSecret({ SecretId: RDS_PROXY_SECRET })
661 .pipe(Effect.map((response) => response.ARN)),
662 ),
663 );
664 if (arn === undefined) {
665 return yield* fixtureError(`secret ${RDS_PROXY_SECRET} has no ARN`);
666 }
667 yield* register(ledger, {
668 label: `secret ${RDS_PROXY_SECRET}`,
669 run: secretsmanager
670 .deleteSecret({ SecretId: arn, ForceDeleteWithoutRecovery: true })
671 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
672 });
673 return arn;
674 });
675
676 // ─── Auto Scaling (ECS capacity providers) ───────────────────────────────────
677
678 const ECS_ASG = `${NAME}-ecs`;
679 const AL2023_AMI_PARAMETER =
680 "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64";
681
682 const findAutoScalingGroup = autoscaling
683 .describeAutoScalingGroups({ AutoScalingGroupNames: [ECS_ASG] })
684 .pipe(Effect.map((response) => response.AutoScalingGroups[0]));
685
686 const findLaunchConfiguration = autoscaling
687 .describeLaunchConfigurations({ LaunchConfigurationNames: [ECS_ASG] })
688 .pipe(Effect.map((response) => response.LaunchConfigurations[0]));
689
690 // Floci answers a missing group or launch configuration with an untyped
691 // ValidationError, so every delete observes first.
692 const deleteAutoScalingGroup = Effect.gen(function* () {
693 if ((yield* findAutoScalingGroup) !== undefined) {
694 yield* autoscaling
695 .deleteAutoScalingGroup({
696 AutoScalingGroupName: ECS_ASG,
697 ForceDelete: true,
698 })
699 .pipe(
700 Effect.retry({
701 while: (error) =>
702 error._tag === "ScalingActivityInProgressFault" ||
703 error._tag === "ResourceInUseFault" ||
704 error._tag === "ResourceContentionFault",
705 schedule: Schedule.spaced("5 seconds"),
706 times: 12,
707 }),
708 );
709 }
710 const remaining = yield* findAutoScalingGroup.pipe(
711 Effect.repeat({
712 schedule: Schedule.spaced("5 seconds"),
713 until: (group): boolean => group === undefined,
714 times: 24,
715 }),
716 );
717 if (remaining !== undefined) {
718 return yield* fixtureError(`Auto Scaling group ${ECS_ASG} still exists`);
719 }
720 });
721
722 const deleteLaunchConfiguration = Effect.gen(function* () {
723 if ((yield* findLaunchConfiguration) === undefined) return;
724 yield* autoscaling.deleteLaunchConfiguration({ LaunchConfigurationName: ECS_ASG }).pipe(
725 Effect.retry({
726 while: (error) =>
727 error._tag === "ResourceInUseFault" || error._tag === "ResourceContentionFault",
728 schedule: Schedule.spaced("5 seconds"),
729 times: 12,
730 }),
731 );
732 });
733
734 const ensureEcsAutoScalingGroup = (ledger: Array<Cleanup>, network: Network) =>
735 Effect.gen(function* () {
736 // Floci resolves the public AMI parameter from its image catalog
737 // (services/ssm/SsmService.java publicParameter).
738 const parameter = yield* ssm.getParameter({ Name: AL2023_AMI_PARAMETER });
739 const value = parameter.Parameter?.Value;
740 const imageId = Redacted.isRedacted(value) ? Redacted.value(value) : value;
741 if (imageId === undefined) {
742 return yield* fixtureError(`${AL2023_AMI_PARAMETER} has no value`);
743 }
744 yield* register(ledger, {
745 label: `launch configuration ${ECS_ASG}`,
746 run: deleteLaunchConfiguration,
747 });
748 if ((yield* findLaunchConfiguration) === undefined) {
749 yield* autoscaling
750 .createLaunchConfiguration({
751 LaunchConfigurationName: ECS_ASG,
752 ImageId: imageId,
753 InstanceType: "t3.micro",
754 })
755 .pipe(Effect.catchTag("AlreadyExistsFault", () => Effect.void));
756 }
757 yield* register(ledger, {
758 label: `Auto Scaling group ${ECS_ASG}`,
759 run: deleteAutoScalingGroup,
760 });
761 if ((yield* findAutoScalingGroup) === undefined) {
762 yield* autoscaling
763 .createAutoScalingGroup({
764 AutoScalingGroupName: ECS_ASG,
765 LaunchConfigurationName: ECS_ASG,
766 MinSize: 0,
767 MaxSize: 0,
768 DesiredCapacity: 0,
769 VPCZoneIdentifier: network.subnetIds.join(","),
770 Tags: [
771 {
772 Key: FIXTURE_TAG_KEY,
773 Value: FIXTURE_TAG_VALUE,
774 PropagateAtLaunch: true,
775 },
776 ],
777 })
778 .pipe(Effect.catchTag("AlreadyExistsFault", () => Effect.void));
779 }
780 const group = yield* findAutoScalingGroup;
781 if (group?.AutoScalingGroupARN === undefined) {
782 return yield* fixtureError(`Auto Scaling group ${ECS_ASG} has no ARN`);
783 }
784 return group.AutoScalingGroupARN;
785 });
786
787 // ─── Network Load Balancer (API Gateway VPC link) ────────────────────────────
788
789 const VPC_LINK_NLB = `${NAME}-nlb`;
790
791 const findLoadBalancer = elbv2.describeLoadBalancers({ Names: [VPC_LINK_NLB] }).pipe(
792 Effect.map((response) => response.LoadBalancers?.[0]),
793 Effect.catchTag("LoadBalancerNotFoundException", () => Effect.undefined),
794 );
795
796 const deleteLoadBalancer = Effect.gen(function* () {
797 const existing = yield* findLoadBalancer;
798 if (existing?.LoadBalancerArn === undefined) return;
799 yield* elbv2.deleteLoadBalancer({ LoadBalancerArn: existing.LoadBalancerArn }).pipe(
800 Effect.retry({
801 while: (error) => error._tag === "ResourceInUseException",
802 schedule: Schedule.spaced("5 seconds"),
803 times: 12,
804 }),
805 Effect.catchTag("LoadBalancerNotFoundException", () => Effect.void),
806 );
807 const remaining = yield* findLoadBalancer.pipe(
808 Effect.repeat({
809 schedule: Schedule.spaced("3 seconds"),
810 until: (balancer): boolean => balancer === undefined,
811 times: 20,
812 }),
813 );
814 if (remaining !== undefined) {
815 return yield* fixtureError(`load balancer ${VPC_LINK_NLB} still exists`);
816 }
817 });
818
819 const ensureVpcLinkLoadBalancer = (ledger: Array<Cleanup>, network: Network) =>
820 Effect.gen(function* () {
821 yield* register(ledger, {
822 label: `load balancer ${VPC_LINK_NLB}`,
823 run: deleteLoadBalancer,
824 });
825 const existing = yield* findLoadBalancer;
826 const balancer =
827 existing ??
828 (yield* elbv2
829 .createLoadBalancer({
830 Name: VPC_LINK_NLB,
831 Type: "network",
832 Scheme: "internal",
833 Subnets: [...network.subnetIds],
834 Tags: [{ Key: FIXTURE_TAG_KEY, Value: FIXTURE_TAG_VALUE }],
835 })
836 .pipe(
837 Effect.map((response) => response.LoadBalancers?.[0]),
838 Effect.catchTag("DuplicateLoadBalancerNameException", () => findLoadBalancer),
839 ));
840 if (balancer?.LoadBalancerArn === undefined) {
841 return yield* fixtureError(`load balancer ${VPC_LINK_NLB} has no ARN`);
842 }
843 if (balancer.Type !== "network" || balancer.VpcId !== network.vpcId) {
844 return yield* fixtureError(
845 `load balancer ${VPC_LINK_NLB} exists as ${balancer.Type} in ${balancer.VpcId}; delete it and rerun`,
846 );
847 }
848 return balancer.LoadBalancerArn;
849 });
850
851 // ─── S3 (Flink application code) ─────────────────────────────────────────────
852
853 const FLINK_BUCKET = `${NAME}-flink`;
854
855 const deleteBucket = (bucket: string) =>
856 Effect.gen(function* () {
857 // Bounded pagination: the fixture bucket holds a single object.
858 for (let page = 0; page < 10; page++) {
859 const listed = yield* s3.listObjectsV2({ Bucket: bucket });
860 const objects = (listed.Contents ?? []).flatMap((object) =>
861 object.Key ? [{ Key: object.Key }] : [],
862 );
863 if (objects.length === 0) break;
864 yield* s3.deleteObjects({
865 Bucket: bucket,
866 Delete: { Objects: objects, Quiet: true },
867 });
868 if (!listed.IsTruncated) break;
869 }
870 yield* s3.deleteBucket({ Bucket: bucket });
871 }).pipe(Effect.catchTag("NoSuchBucket", () => Effect.void));
872
873 const downloadFlinkJar = Effect.gen(function* () {
874 const client = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk);
875 const response = yield* client
876 .get(FLINK_JAR_URL)
877 .pipe(Effect.retry({ schedule: Schedule.exponential("1 second"), times: 3 }));
878 const bytes = new Uint8Array(yield* response.arrayBuffer);
879 const sha1 = yield* Effect.sync(() => createHash("sha1").update(bytes).digest("hex"));
880 if (sha1 !== FLINK_JAR_SHA1) {
881 return yield* fixtureError(`${FLINK_JAR_URL} has sha1 ${sha1}, expected ${FLINK_JAR_SHA1}`);
882 }
883 return bytes;
884 }).pipe(
885 Effect.catchTags({
886 HttpClientError: (error) =>
887 fixtureError(`could not download ${FLINK_JAR_URL}: ${error.message}`),
888 }),
889 );
890
891 const ensureFlinkJar = (ledger: Array<Cleanup>) =>
892 Effect.gen(function* () {
893 const jar = yield* downloadFlinkJar;
894 const exists = yield* s3.headBucket({ Bucket: FLINK_BUCKET }).pipe(
895 Effect.as(true),
896 Effect.catchTag(["NotFound", "NoSuchBucket"], () => Effect.succeed(false)),
897 );
898 yield* register(ledger, {
899 label: `S3 bucket ${FLINK_BUCKET}`,
900 run: deleteBucket(FLINK_BUCKET),
901 });
902 if (!exists) {
903 yield* s3
904 .createBucket({ Bucket: FLINK_BUCKET })
905 .pipe(Effect.catchTag("BucketAlreadyOwnedByYou", () => Effect.void));
906 }
907 yield* s3.putObject({
908 Bucket: FLINK_BUCKET,
909 Key: FLINK_JAR_KEY,
910 Body: jar,
911 ContentType: "application/java-archive",
912 });
913 return `arn:aws:s3:::${FLINK_BUCKET}`;
914 });
915
916 // ─── Route 53 + ACM ──────────────────────────────────────────────────────────
917
918 const trimDot = (name: string) => name.replace(/\.+$/, "");
919 const zoneIdOf = (id: string) => id.replace(/^\/hostedzone\//, "");
920
921 const findPublicZone = (name: string) =>
922 route53
923 .listHostedZonesByName({ DNSName: name })
924 .pipe(
925 Effect.map((response) =>
926 (response.HostedZones ?? []).find(
927 (zone) => trimDot(zone.Name) === name && !zone.Config?.PrivateZone,
928 ),
929 ),
930 );
931
932 const listRecordSets = (zoneId: string) =>
933 Effect.gen(function* () {
934 const records: Array<route53.ResourceRecordSet> = [];
935 let start: Pick<
936 route53.ListResourceRecordSetsRequest,
937 "StartRecordName" | "StartRecordType" | "StartRecordIdentifier"
938 > = {};
939 for (let page = 0; page < 50; page++) {
940 const response = yield* route53.listResourceRecordSets({
941 HostedZoneId: zoneId,
942 ...start,
943 });
944 records.push(...(response.ResourceRecordSets ?? []));
945 if (!response.IsTruncated) break;
946 start = {
947 StartRecordName: response.NextRecordName,
948 StartRecordType: response.NextRecordType,
949 StartRecordIdentifier: response.NextRecordIdentifier,
950 };
951 }
952 return records;
953 });
954
955 /** Deletes every record except the apex SOA/NS, then the zone. */
956 const deleteZone = (zoneId: string, name: string) =>
957 Effect.gen(function* () {
958 const doomed = (yield* listRecordSets(zoneId)).filter(
959 (record) =>
960 !(trimDot(record.Name) === name && (record.Type === "SOA" || record.Type === "NS")),
961 );
962 if (doomed.length > 0) {
963 yield* route53.changeResourceRecordSets({
964 HostedZoneId: zoneId,
965 ChangeBatch: {
966 Changes: doomed.map((record) => ({
967 Action: "DELETE",
968 ResourceRecordSet: record,
969 })),
970 },
971 });
972 }
973 yield* route53.deleteHostedZone({ Id: zoneId });
974 }).pipe(Effect.catchTag("NoSuchHostedZone", () => Effect.void));
975
976 const ensurePublicZone = (ledger: Array<Cleanup>, name: string) =>
977 Effect.gen(function* () {
978 const existing = yield* findPublicZone(name);
979 let zoneId = existing === undefined ? undefined : zoneIdOf(existing.Id);
980 if (zoneId === undefined) {
981 const reference = yield* Effect.sync(() => randomUUID());
982 const created = yield* route53.createHostedZone({
983 Name: name,
984 CallerReference: `${NAME}-${reference}`,
985 HostedZoneConfig: {
986 Comment: "Standing fixture zone for the Floci test run",
987 PrivateZone: false,
988 },
989 });
990 zoneId = zoneIdOf(created.HostedZone.Id);
991 }
992 const id = zoneId;
993 yield* register(ledger, {
994 label: `hosted zone ${name}`,
995 run: deleteZone(id, name),
996 });
997 return id;
998 });
999
1000 const upsertCnames = (
1001 zoneId: string,
1002 records: ReadonlyArray<{ readonly name: string; readonly value: string }>,
1003 ) =>
1004 route53.changeResourceRecordSets({
1005 HostedZoneId: zoneId,
1006 ChangeBatch: {
1007 Changes: records.map((record) => ({
1008 Action: "UPSERT",
1009 ResourceRecordSet: {
1010 Name: record.name,
1011 Type: "CNAME",
1012 TTL: 300,
1013 ResourceRecords: [{ Value: record.value }],
1014 },
1015 })),
1016 },
1017 });
1018
1019 const describeCertificate = (arn: string) =>
1020 acm
1021 .describeCertificate({ CertificateArn: arn })
1022 .pipe(Effect.map((response) => response.Certificate));
1023
1024 const ensureZoneCertificate = (ledger: Array<Cleanup>, zoneId: string, zone: string) =>
1025 Effect.gen(function* () {
1026 const wildcard = `*.${zone}`;
1027 const summaries = yield* acm.listCertificates.pages({}).pipe(
1028 Stream.runCollect,
1029 Effect.map((pages) => Array.from(pages).flatMap((page) => page.CertificateSummaryList ?? [])),
1030 );
1031 const existing = summaries.find(
1032 (summary) =>
1033 summary.DomainName === zone &&
1034 (summary.SubjectAlternativeNameSummaries ?? []).includes(wildcard) &&
1035 (summary.Status === "ISSUED" || summary.Status === "PENDING_VALIDATION"),
1036 );
1037 const arn =
1038 existing?.CertificateArn ??
1039 (yield* acm
1040 .requestCertificate({
1041 DomainName: zone,
1042 SubjectAlternativeNames: [wildcard],
1043 ValidationMethod: "DNS",
1044 Tags: [{ Key: FIXTURE_TAG_KEY, Value: FIXTURE_TAG_VALUE }],
1045 })
1046 .pipe(Effect.map((response) => response.CertificateArn)));
1047 if (arn === undefined) {
1048 return yield* fixtureError(`RequestCertificate for ${zone} returned no ARN`);
1049 }
1050 const certificateArn = arn;
1051 yield* register(ledger, {
1052 label: `ACM certificate ${certificateArn}`,
1053 run: acm.deleteCertificate({ CertificateArn: certificateArn }).pipe(
1054 Effect.retry({
1055 while: (error) => error._tag === "ResourceInUseException",
1056 schedule: Schedule.spaced("5 seconds"),
1057 times: 12,
1058 }),
1059 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
1060 ),
1061 });
1062 // The zone and its wildcard can share one validation record.
1063 const pending = yield* describeCertificate(certificateArn).pipe(
1064 Effect.repeat({
1065 schedule: Schedule.spaced("2 seconds"),
1066 until: (certificate): boolean =>
1067 certificate?.Status === "ISSUED" ||
1068 (certificate?.DomainValidationOptions ?? []).every(
1069 (option) => option.ResourceRecord !== undefined,
1070 ),
1071 times: 15,
1072 }),
1073 );
1074 const records = new Map<string, { name: string; value: string }>();
1075 for (const option of pending?.DomainValidationOptions ?? []) {
1076 const record = option.ResourceRecord;
1077 if (record === undefined || record.Type !== "CNAME") continue;
1078 records.set(`${record.Name}|${record.Value}`, {
1079 name: record.Name,
1080 value: record.Value,
1081 });
1082 }
1083 if (pending?.Status !== "ISSUED") {
1084 if (records.size === 0) {
1085 return yield* fixtureError(
1086 `certificate ${certificateArn} exposes no DNS validation records`,
1087 );
1088 }
1089 yield* upsertCnames(zoneId, [...records.values()]);
1090 }
1091 // Floci settles DNS validation when the certificate is read
1092 // (services/acm/AcmService.java settleValidation).
1093 const issued = yield* describeCertificate(certificateArn).pipe(
1094 Effect.repeat({
1095 schedule: Schedule.spaced("3 seconds"),
1096 until: (certificate): boolean => certificate?.Status !== "PENDING_VALIDATION",
1097 times: 40,
1098 }),
1099 );
1100 if (issued?.Status !== "ISSUED") {
1101 return yield* fixtureError(
1102 `certificate ${certificateArn} is ${issued?.Status ?? "missing"}, not ISSUED`,
1103 );
1104 }
1105 return certificateArn;
1106 });
1107
1108 // ─── SES ─────────────────────────────────────────────────────────────────────
1109
1110 // v1 DeleteIdentity is idempotent for missing identities.
1111 const deleteSesIdentity = (identity: string) => ses.deleteIdentity({ Identity: identity });
1112
1113 /** A v1-verified email address (Floci verifies without the email round trip). */
1114 const ensureVerifiedAddress = (ledger: Array<Cleanup>, address: string) =>
1115 Effect.gen(function* () {
1116 yield* register(ledger, {
1117 label: `SES identity ${address}`,
1118 run: deleteSesIdentity(address),
1119 });
1120 yield* ses.verifyEmailIdentity({ EmailAddress: address });
1121 });
1122
1123 const ensureDkimVerifiedDomain = (ledger: Array<Cleanup>, zoneId: string, domain: string) =>
1124 Effect.gen(function* () {
1125 yield* register(ledger, {
1126 label: `SES identity ${domain}`,
1127 run: deleteSesIdentity(domain),
1128 });
1129 const getIdentity = sesv2.getEmailIdentity({ EmailIdentity: domain });
1130 const identity = yield* sesv2
1131 .createEmailIdentity({
1132 EmailIdentity: domain,
1133 Tags: [{ Key: FIXTURE_TAG_KEY, Value: FIXTURE_TAG_VALUE }],
1134 })
1135 .pipe(Effect.catchTag("AlreadyExistsException", () => getIdentity));
1136 const tokens = identity.DkimAttributes?.Tokens ?? [];
1137 if (tokens.length === 0) {
1138 return yield* fixtureError(`SES identity ${domain} has no DKIM tokens`);
1139 }
1140 yield* upsertCnames(
1141 zoneId,
1142 tokens.map((token) => ({
1143 name: `${token}._domainkey.${domain}`,
1144 value: `${token}.dkim.amazonses.com`,
1145 })),
1146 );
1147 // Floci re-checks the CNAMEs on read with a 5s lookup cache
1148 // (services/ses/SesIdentityService.java hasAllExpectedDkimRecords).
1149 const verified = yield* getIdentity.pipe(
1150 Effect.repeat({
1151 schedule: Schedule.spaced("3 seconds"),
1152 until: (observed): boolean => observed.VerifiedForSendingStatus === true,
1153 times: 20,
1154 }),
1155 );
1156 if (verified.VerifiedForSendingStatus !== true) {
1157 return yield* fixtureError(
1158 `SES identity ${domain} is not verified for sending (DKIM ${verified.DkimAttributes?.Status})`,
1159 );
1160 }
1161 });
1162
1163 const ensureCustomVerificationTemplate = (ledger: Array<Cleanup>) =>
1164 Effect.gen(function* () {
1165 yield* register(ledger, {
1166 label: `SES custom verification template ${SES_CVE_TEMPLATE}`,
1167 run: sesv2
1168 .deleteCustomVerificationEmailTemplate({
1169 TemplateName: SES_CVE_TEMPLATE,
1170 })
1171 .pipe(Effect.catchTag("NotFoundException", () => Effect.void)),
1172 });
1173 // The gated send registers the recipient as a pending identity.
1174 yield* register(ledger, {
1175 label: `SES identity ${SES_CVE_RECIPIENT}`,
1176 run: deleteSesIdentity(SES_CVE_RECIPIENT),
1177 });
1178 yield* sesv2
1179 .createCustomVerificationEmailTemplate({
1180 TemplateName: SES_CVE_TEMPLATE,
1181 FromEmailAddress: SES_FROM,
1182 TemplateSubject: "Confirm your email address",
1183 TemplateContent:
1184 "<html><body><p>Confirm your address for the Alchemy Floci tests.</p></body></html>",
1185 SuccessRedirectionURL: "https://example.com/verified",
1186 FailureRedirectionURL: "https://example.com/verify-failed",
1187 })
1188 .pipe(Effect.catchTag("AlreadyExistsException", () => Effect.void));
1189 });
1190
1191 const ensureVdmEnabled = (ledger: Array<Cleanup>) =>
1192 Effect.gen(function* () {
1193 const account = yield* sesv2.getAccount({});
1194 const previous: sesv2.VdmAttributes = account.VdmAttributes ?? {
1195 VdmEnabled: "DISABLED",
1196 };
1197 yield* register(ledger, {
1198 label: `SES VDM restore (${previous.VdmEnabled})`,
1199 run: sesv2.putAccountVdmAttributes({ VdmAttributes: previous }),
1200 });
1201 yield* sesv2.putAccountVdmAttributes({
1202 VdmAttributes: { ...previous, VdmEnabled: "ENABLED" },
1203 });
1204 });
1205
1206 const ensureSesFixtures = (ledger: Array<Cleanup>) =>
1207 Effect.gen(function* () {
1208 yield* ensureVerifiedAddress(ledger, SES_FROM);
1209 // The bounce sender must be verified for the gated SendBounce success
1210 // path; a fabricated message id is still rejected with MessageRejected.
1211 yield* ensureVerifiedAddress(ledger, SES_BOUNCE_SENDER);
1212 // Floci bounces any message it recorded (services/ses/SesService.java sendBounce).
1213 const sent = yield* sesv2.sendEmail({
1214 FromEmailAddress: SES_FROM,
1215 Destination: { ToAddresses: ["success@simulator.amazonses.com"] },
1216 Content: {
1217 Simple: {
1218 Subject: { Data: "Alchemy Floci standing bounce source" },
1219 Body: { Text: { Data: "Original message for the SendBounce test." } },
1220 },
1221 },
1222 });
1223 if (sent.MessageId === undefined) {
1224 return yield* fixtureError("SendEmail returned no MessageId");
1225 }
1226 yield* ensureCustomVerificationTemplate(ledger);
1227 const zoneId = yield* ensurePublicZone(ledger, SES_ZONE);
1228 yield* ensureDkimVerifiedDomain(ledger, zoneId, SES_REDIRECT_DOMAIN);
1229 yield* ensureVdmEnabled(ledger);
1230 return { bounceMessageId: sent.MessageId };
1231 });
1232
1233 // ─── OAM (second account) ────────────────────────────────────────────────────
1234
1235 const OAM_RESOURCE_TYPES = ["AWS::CloudWatch::Metric", "AWS::Logs::LogGroup", "AWS::XRay::Trace"];
1236
1237 const findSink = asSinkAccount(
1238 oam
1239 .listSinks({})
1240 .pipe(
1241 Effect.map(
1242 (response) => response.Items.find((sink) => sink.Name === NAME) ?? response.Items[0],
1243 ),
1244 ),
1245 );
1246
1247 const ensureOamSink = (ledger: Array<Cleanup>) =>
1248 Effect.gen(function* () {
1249 const existing = yield* findSink;
1250 const arn =
1251 existing?.Arn ??
1252 (yield* asSinkAccount(
1253 oam
1254 .createSink({
1255 Name: NAME,
1256 Tags: { [FIXTURE_TAG_KEY]: FIXTURE_TAG_VALUE },
1257 })
1258 .pipe(
1259 Effect.map((response) => response.Arn),
1260 // Floci allows one sink per account and Region.
1261 Effect.catchTag("ConflictException", () =>
1262 findSink.pipe(Effect.map((sink) => sink?.Arn)),
1263 ),
1264 ),
1265 ));
1266 if (arn === undefined) {
1267 return yield* fixtureError(`OAM sink ${NAME} has no ARN`);
1268 }
1269 const sinkArn = arn;
1270 yield* register(ledger, {
1271 label: `OAM sink ${sinkArn}`,
1272 run: asSinkAccount(
1273 oam.deleteSink({ Identifier: sinkArn }).pipe(
1274 // Links from the test account must be gone first.
1275 Effect.retry({
1276 while: (error) => error._tag === "ConflictException",
1277 schedule: Schedule.spaced("3 seconds"),
1278 times: 20,
1279 }),
1280 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
1281 ),
1282 ),
1283 });
1284 yield* asSinkAccount(
1285 oam.putSinkPolicy({
1286 SinkIdentifier: sinkArn,
1287 Policy: JSON.stringify({
1288 Version: "2012-10-17",
1289 Statement: [
1290 {
1291 Effect: "Allow",
1292 Principal: { AWS: [ACCOUNT_ID] },
1293 Action: ["oam:CreateLink", "oam:UpdateLink"],
1294 Resource: "*",
1295 Condition: {
1296 "ForAllValues:StringEquals": {
1297 "oam:ResourceTypes": OAM_RESOURCE_TYPES,
1298 },
1299 },
1300 },
1301 ],
1302 }),
1303 }),
1304 );
1305 return sinkArn;
1306 });
1307
1308 // ─── SageMaker serving image ─────────────────────────────────────────────────
1309
1310 /** Builds the local serving image Floci runs with `serve` (no registry push). */
1311 const buildSageMakerServeImage = Effect.gen(function* () {
1312 const path = yield* Path.Path;
1313 const context = yield* path.fromFileUrl(
1314 new URL("./floci-fixtures-assets/sagemaker-serve/", import.meta.url),
1315 );
1316 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1317 const result = yield* spawner
1318 .spawn(
1319 ChildProcess.make("docker", ["build", "--tag", SAGEMAKER_SERVE_IMAGE, context], {
1320 stdin: "ignore",
1321 stdout: "pipe",
1322 stderr: "pipe",
1323 }),
1324 )
1325 .pipe(
1326 Effect.flatMap((child) =>
1327 Effect.all(
1328 {
1329 exitCode: child.exitCode,
1330 stdout: child.stdout.pipe(Stream.decodeText, Stream.mkString),
1331 stderr: child.stderr.pipe(Stream.decodeText, Stream.mkString),
1332 },
1333 { concurrency: "unbounded" },
1334 ),
1335 ),
1336 Effect.scoped,
1337 );
1338 if (result.exitCode !== 0) {
1339 return yield* fixtureError(
1340 `docker build of ${SAGEMAKER_SERVE_IMAGE} exited ${result.exitCode}: ${result.stderr.trim().slice(-2000)}`,
1341 );
1342 }
1343 return SAGEMAKER_SERVE_IMAGE;
1344 }).pipe(
1345 Effect.catchTags({
1346 PlatformError: (error) =>
1347 fixtureError(`could not run docker build for ${SAGEMAKER_SERVE_IMAGE}: ${error.message}`),
1348 }),
1349 );
1350
1351 // ─── Control Tower ───────────────────────────────────────────────────────────
1352
1353 /** One landing zone per Region; Floci's CreateLandingZone needs no Organization. */
1354 const ensureLandingZone = (ledger: Array<Cleanup>) =>
1355 Effect.gen(function* () {
1356 // Floci keeps at most one landing zone per Region
1357 // (services/controltower/ControlTowerService.java createLandingZone).
1358 const listed = yield* controltower.listLandingZones({});
1359 let landingZoneArn = listed.landingZones[0]?.arn;
1360 if (landingZoneArn === undefined) {
1361 const created = yield* controltower
1362 .createLandingZone({
1363 version: LANDING_ZONE_VERSION,
1364 manifest: {
1365 governedRegions: [REGION],
1366 organizationStructure: { security: { name: "Security" } },
1367 centralizedLogging: {
1368 accountId: ACCOUNT_ID,
1369 configurations: {
1370 loggingBucket: { retentionDays: 365 },
1371 accessLoggingBucket: { retentionDays: 3650 },
1372 },
1373 enabled: true,
1374 },
1375 securityRoles: { accountId: ACCOUNT_ID },
1376 accessManagement: { enabled: false },
1377 },
1378 tags: { [FIXTURE_TAG_KEY]: FIXTURE_TAG_VALUE },
1379 })
1380 .pipe(
1381 Effect.map((response) => response.arn),
1382 Effect.catchTag("ConflictException", () =>
1383 controltower
1384 .listLandingZones({})
1385 .pipe(Effect.map((response) => response.landingZones[0]?.arn)),
1386 ),
1387 );
1388 if (created === undefined) {
1389 return yield* fixtureError("CreateLandingZone returned no ARN");
1390 }
1391 const arn = created;
1392 yield* register(ledger, {
1393 label: `landing zone ${arn}`,
1394 run: controltower
1395 .deleteLandingZone({ landingZoneIdentifier: arn })
1396 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
1397 });
1398 landingZoneArn = arn;
1399 } else {
1400 yield* log(`reusing landing zone ${landingZoneArn}`);
1401 }
1402 const landingZone = yield* controltower
1403 .getLandingZone({ landingZoneIdentifier: landingZoneArn })
1404 .pipe(
1405 Effect.map((response) => response.landingZone),
1406 Effect.repeat({
1407 schedule: Schedule.spaced("5 seconds"),
1408 until: (zone): boolean => zone.status !== "PROCESSING",
1409 times: 24,
1410 }),
1411 );
1412 if (landingZone.status !== "ACTIVE") {
1413 return yield* fixtureError(
1414 `landing zone ${landingZoneArn} is ${landingZone.status ?? "without status"}`,
1415 );
1416 }
1417 });
1418
1419 // ─── Orchestration ───────────────────────────────────────────────────────────
1420
1421 const matchesSuite = (file: string, suite: string) =>
1422 file.replaceAll("\\", "/").endsWith(`AWS/${suite}`);
1423
1424 const allVarsSet = (group: Group) =>
1425 Effect.forEach(groupVars[group], (name) => Config.String(name).pipe(Config.withDefault(""))).pipe(
1426 Effect.map((values) => values.every((value) => value !== "")),
1427 );
1428
1429 const selectGroups = (files: ReadonlyArray<string> | undefined) =>
1430 Effect.gen(function* () {
1431 const selected = new Set<Group>();
1432 for (const group of Object.keys(groupVars) as Array<Group>) {
1433 const wanted =
1434 files === undefined ||
1435 files.some((file) => groupSuites[group].some((suite) => matchesSuite(file, suite)));
1436 if (!wanted) continue;
1437 if (yield* allVarsSet(group)) {
1438 yield* log(`fixtures: ${group} uses the caller's env vars`);
1439 continue;
1440 }
1441 selected.add(group);
1442 }
1443 return selected;
1444 });
1445
1446 const provision = (external: boolean, groups: ReadonlySet<Group>, ledger: Array<Cleanup>) =>
1447 Effect.gen(function* () {
1448 yield* Floci.ensureFloci({ port: Floci.DEFAULT_FLOCI_PORT, external });
1449 yield* log(`fixtures: provisioning ${[...groups].join(", ")}`);
1450 const env: Record<string, string> = {};
1451 // Shared by every group that needs subnets; cleanups register once.
1452 const network = yield* Effect.cached(ensureNetwork(ledger));
1453 const needsEksCluster = groups.has("eksCluster") || groups.has("amp") || groups.has("hyperpod");
1454
1455 if (needsEksCluster || groups.has("eksRole")) {
1456 const { subnetIds } = yield* network;
1457 const subnets = subnetIds.join(",");
1458 const clusterRoleArn = yield* ensureRole(
1459 ledger,
1460 "eks-cluster",
1461 "eks.amazonaws.com",
1462 ["sts:AssumeRole", "sts:TagSession"],
1463 ["AmazonEKSClusterPolicy"],
1464 );
1465 if (groups.has("eksRole")) {
1466 env.AWS_TEST_EKS_ROLE_ARN = clusterRoleArn;
1467 env.AWS_TEST_EKS_SUBNET_IDS = subnets;
1468 }
1469
1470 if (needsEksCluster) {
1471 const cluster = yield* ensureEksCluster(ledger, yield* network, clusterRoleArn);
1472 if (groups.has("eksCluster")) {
1473 env.AWS_TEST_EKS_NODE_ROLE_ARN = yield* ensureRole(
1474 ledger,
1475 "eks-node",
1476 "ec2.amazonaws.com",
1477 ["sts:AssumeRole"],
1478 [
1479 "AmazonEKSWorkerNodePolicy",
1480 "AmazonEKS_CNI_Policy",
1481 "AmazonEC2ContainerRegistryReadOnly",
1482 ],
1483 );
1484 env.AWS_TEST_EKS_FARGATE_ROLE_ARN = yield* ensureRole(
1485 ledger,
1486 "eks-fargate",
1487 "eks-fargate-pods.amazonaws.com",
1488 ["sts:AssumeRole"],
1489 ["AmazonEKSFargatePodExecutionRolePolicy"],
1490 );
1491 env.AWS_TEST_EKS_POD_ROLE_ARN = yield* ensureRole(
1492 ledger,
1493 "eks-pod",
1494 "pods.eks.amazonaws.com",
1495 ["sts:AssumeRole", "sts:TagSession"],
1496 [],
1497 );
1498 env.AWS_TEST_EKS_CLUSTER = cluster.name;
1499 env.AWS_TEST_EKS_PRIVATE_SUBNETS = subnets;
1500 }
1501 if (groups.has("amp")) {
1502 env.AWS_TEST_AMP_SCRAPER = "1";
1503 env.AWS_TEST_AMP_SCRAPER_CLUSTER_ARN = cluster.arn;
1504 env.AWS_TEST_AMP_SCRAPER_SUBNET_IDS = subnets;
1505 }
1506 if (groups.has("hyperpod")) {
1507 const executionRoleArn = yield* ensureRole(
1508 ledger,
1509 "hyperpod",
1510 "sagemaker.amazonaws.com",
1511 ["sts:AssumeRole"],
1512 ["AmazonSageMakerClusterInstanceRolePolicy"],
1513 );
1514 env.AWS_TEST_SAGEMAKER_HYPERPOD_EKS_CLUSTER_ARN = yield* ensureHyperPodCluster(
1515 ledger,
1516 cluster,
1517 subnetIds,
1518 executionRoleArn,
1519 );
1520 }
1521 }
1522 }
1523
1524 if (groups.has("rdsProxy")) {
1525 const { subnetIds } = yield* network;
1526 const roleArn = yield* ensureRole(
1527 ledger,
1528 "rds-proxy",
1529 "rds.amazonaws.com",
1530 ["sts:AssumeRole"],
1531 ["SecretsManagerReadWrite"],
1532 );
1533 const secretArn = yield* ensureRdsProxySecret(ledger);
1534 env.AWS_TEST_RDS_DBPROXY = "1";
1535 env.DBPROXY_SUBNET_IDS = subnetIds.join(",");
1536 env.DBPROXY_ROLE_ARN = roleArn;
1537 env.DBPROXY_SECRET_ARN = secretArn;
1538 }
1539
1540 if (groups.has("ecsAsg")) {
1541 env.TEST_ASG_ARN = yield* ensureEcsAutoScalingGroup(ledger, yield* network);
1542 }
1543
1544 if (groups.has("vpcLink")) {
1545 env.ALCHEMY_TEST_VPC_LINK_TARGET_ARN = yield* ensureVpcLinkLoadBalancer(
1546 ledger,
1547 yield* network,
1548 );
1549 }
1550
1551 if (groups.has("flink")) {
1552 env.AWS_TEST_FLINK_START = "1";
1553 env.AWS_TEST_FLINK_JAR_BUCKET_ARN = yield* ensureFlinkJar(ledger);
1554 env.AWS_TEST_FLINK_JAR_KEY = FLINK_JAR_KEY;
1555 }
1556
1557 if (groups.has("domain")) {
1558 const zoneId = yield* ensurePublicZone(ledger, DOMAIN_ZONE);
1559 const certificateArn = yield* ensureZoneCertificate(ledger, zoneId, DOMAIN_ZONE);
1560 env.AWS_TEST_DOMAIN = DOMAIN_ZONE;
1561 env.AWS_TEST_HOSTED_ZONE = DOMAIN_ZONE;
1562 env.AWS_TEST_APIGATEWAY_DOMAIN = `apigw-bpm.${DOMAIN_ZONE}`;
1563 env.AWS_TEST_ACM_CERTIFICATE_ARN = certificateArn;
1564 env.AWS_TEST_APIGATEWAY_DOMAIN_NAME = `apigw.${DOMAIN_ZONE}`;
1565 env.AWS_TEST_APIGATEWAY_CERT_ARN = certificateArn;
1566 env.AWS_TEST_APIGATEWAYV2_DOMAIN_NAME = `apigw2.${DOMAIN_ZONE}`;
1567 env.AWS_TEST_APPSYNC_DOMAIN_NAME = `appsync.${DOMAIN_ZONE}`;
1568 env.AWS_TEST_APPSYNC_DOMAIN_CERT_ARN = certificateArn;
1569 }
1570
1571 if (groups.has("ses")) {
1572 const { bounceMessageId } = yield* ensureSesFixtures(ledger);
1573 env.AWS_TEST_SES_FROM = SES_FROM;
1574 env.AWS_TEST_SES_BOUNCE_MESSAGE_ID = bounceMessageId;
1575 env.AWS_TEST_SES_CVE_TEMPLATE = SES_CVE_TEMPLATE;
1576 env.AWS_TEST_SES_CVE_RECIPIENT = SES_CVE_RECIPIENT;
1577 env.AWS_TEST_SES_REDIRECT_DOMAIN = SES_REDIRECT_DOMAIN;
1578 env.AWS_TEST_SES_VDM = "1";
1579 }
1580
1581 if (groups.has("oam")) {
1582 env.AWS_TEST_OAM_SINK_ARN = yield* ensureOamSink(ledger);
1583 }
1584
1585 if (groups.has("sagemakerEndpoint")) {
1586 env.AWS_TEST_SAGEMAKER_ENDPOINT = "1";
1587 env.AWS_TEST_SAGEMAKER_IMAGE = yield* buildSageMakerServeImage;
1588 }
1589
1590 if (groups.has("controlTower")) {
1591 yield* ensureLandingZone(ledger);
1592 env.AWS_TEST_CONTROLTOWER = "1";
1593 env.AWS_TEST_CONTROLTOWER_CONTROL = CONTROL_ARN;
1594 env.AWS_TEST_CONTROLTOWER_BASELINE_VERSION = BASELINE_VERSION;
1595 }
1596 return env;
1597 });
1598
1599 /**
1600 * Provisions the standing fixtures the selected suites need. On failure or
1601 * interruption, everything already created is removed before the effect fails.
1602 */
1603 export const provisionFlociStandingFixtures = Effect.fn(function* (
1604 external: boolean,
1605 options: FlociStandingFixtureOptions = {},
1606 ) {
1607 const ledger: Array<Cleanup> = [];
1608 const groups = yield* selectGroups(options.files);
1609 const teardown: FlociStandingFixtures["teardown"] = Effect.suspend(() =>
1610 ledger.length === 0 ? Effect.void : runCleanups(ledger),
1611 ).pipe(Effect.provide(fixtureLayer));
1612 if (groups.size === 0) return { env: {}, teardown } satisfies FlociStandingFixtures;
1613 const env = yield* provision(external, groups, ledger).pipe(
1614 Effect.onError(() =>
1615 runCleanups(ledger).pipe(Effect.catch((error) => Console.error(String(error.message)))),
1616 ),
1617 Effect.provide(fixtureLayer),
1618 );
1619 return { env, teardown } satisfies FlociStandingFixtures;
1620 });