The best way to control Claude Code, Codex and more from any computer or phone

main
ts 200 lines 8.14 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { Cluster } from "@alchemy.run/aws/EMR/Cluster";
3 import { expect } from "@alchemy.run/test";
4 import * as Test from "@alchemy.run/test/Alchemy";
5 import * as EC2 from "@distilled.cloud/aws/ec2";
6 import * as emr from "@distilled.cloud/aws/emr";
7 import * as Output from "alchemy/Output";
8 import * as Duration from "effect/Duration";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import { getDefaultVpc } from "../DefaultVpc.ts";
12
13 const { test } = Test.make({ providers: AWS.providers() });
14
15 // Ungated typed-error probes: prove the distilled emr error union carries the
16 // not-found tags this provider's read/reconcile/delete paths depend on.
17 // DescribeCluster surfaces a missing cluster as InvalidRequestException
18 // ("Cluster id 'j-…' is not valid.") → synthetic ClusterNotFound, and
19 // TerminateJobFlows as ValidationException ("Specified job flow ID not
20 // valid.") → synthetic JobFlowNotFound.
21 test.provider(
22 "describeCluster on a nonexistent id fails with ClusterNotFound",
23 () =>
24 Effect.gen(function* () {
25 const error = yield* Effect.flip(emr.describeCluster({ ClusterId: "j-1K48XAOQ4XHCB" }));
26 expect(error._tag).toBe("ClusterNotFound");
27 }),
28 { tags: ["provider:aws", "provider:aws:emr", "live"] },
29 );
30
31 test.provider(
32 "terminateJobFlows on a nonexistent id fails with JobFlowNotFound",
33 () =>
34 Effect.gen(function* () {
35 const error = yield* Effect.flip(emr.terminateJobFlows({ JobFlowIds: ["j-1K48XAOQ4XHCB"] }));
36 expect(error._tag).toBe("JobFlowNotFound");
37 }),
38 { tags: ["provider:aws", "provider:aws:emr", "live"] },
39 );
40
41 // Resolve a subnet of the account's default VPC (public subnet — EMR
42 // clusters launch fine there and the factory never creates NAT gateways).
43 const resolveSubnet = Effect.gen(function* () {
44 const vpc = yield* getDefaultVpc;
45 const subnets = yield* EC2.describeSubnets({
46 Filters: [
47 { Name: "vpc-id", Values: [vpc.vpcId] },
48 { Name: "default-for-az", Values: ["true"] },
49 ],
50 });
51 const subnetId = (subnets.Subnets ?? []).flatMap((s) => (s.SubnetId ? [s.SubnetId] : []))[0];
52 return { subnetId };
53 });
54
55 // A provisioned EMR cluster takes ~10-15 minutes to reach WAITING, bills per
56 // instance-hour for 3 m5.xlarge-class instances while it exists, and takes
57 // ~5-10 more minutes to terminate. The full lifecycle is skipped under
58 // `--fast` and always destroys what it created.
59 test.provider.skipIf(!!process.env.FAST)(
60 "create Spark cluster, verify WAITING, update in place, destroy, verify terminating",
61 (stack) =>
62 Effect.gen(function* () {
63 yield* stack.destroy();
64
65 const { subnetId } = yield* resolveSubnet;
66 expect(subnetId).toBeDefined();
67
68 const deploy = (props: { stepConcurrencyLevel?: number; idleTimeoutSeconds?: number }) =>
69 stack.deploy(
70 Effect.gen(function* () {
71 const logs = yield* AWS.S3.Bucket("EmrLogs", { forceDestroy: true });
72 // EMR service role (legacy managed policy — the v2 policy
73 // requires tag-scoped resources).
74 const serviceRole = yield* AWS.IAM.Role("EmrServiceRole", {
75 assumeRolePolicyDocument: {
76 Version: "2012-10-17",
77 Statement: [
78 {
79 Effect: "Allow",
80 Principal: { Service: ["elasticmapreduce.amazonaws.com"] },
81 Action: ["sts:AssumeRole"],
82 },
83 ],
84 },
85 managedPolicyArns: [
86 "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole",
87 ],
88 });
89 // EC2 instance role + profile (the job-flow role).
90 const ec2Role = yield* AWS.IAM.Role("EmrEc2Role", {
91 assumeRolePolicyDocument: {
92 Version: "2012-10-17",
93 Statement: [
94 {
95 Effect: "Allow",
96 Principal: { Service: ["ec2.amazonaws.com"] },
97 Action: ["sts:AssumeRole"],
98 },
99 ],
100 },
101 managedPolicyArns: [
102 "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role",
103 ],
104 });
105 const instanceProfile = yield* AWS.IAM.InstanceProfile("EmrEc2Profile", {
106 roleName: ec2Role.roleName,
107 });
108 const cluster = yield* Cluster("Spark", {
109 releaseLabel: "emr-7.5.0",
110 applications: ["Spark", "Hadoop"],
111 serviceRole: serviceRole.roleName,
112 jobFlowRole: instanceProfile.instanceProfileName,
113 logUri: Output.interpolate`s3://${logs.bucketName}/logs/`,
114 instances: {
115 masterInstanceType: "m5.xlarge",
116 coreInstanceType: "m5.xlarge",
117 coreInstanceCount: 1,
118 ec2SubnetId: subnetId,
119 keepJobFlowAliveWhenNoSteps: true,
120 },
121 stepConcurrencyLevel: props.stepConcurrencyLevel,
122 autoTerminationPolicy: props.idleTimeoutSeconds
123 ? { idleTimeout: Duration.seconds(props.idleTimeoutSeconds) }
124 : undefined,
125 tags: { fixture: "emr-cluster" },
126 });
127 return { cluster };
128 }),
129 );
130
131 // Create — reconcile waits (bounded) for WAITING/RUNNING.
132 const { cluster } = yield* deploy({ idleTimeoutSeconds: 3600 });
133 expect(cluster.clusterId).toMatch(/^j-/);
134 expect(cluster.clusterArn).toContain(":cluster/");
135 expect(["WAITING", "RUNNING"]).toContain(cluster.state);
136
137 // Out-of-band verification via distilled.
138 const created = yield* emr.describeCluster({ ClusterId: cluster.clusterId });
139 expect(created.Cluster?.ReleaseLabel).toBe("emr-7.5.0");
140 expect((created.Cluster?.Applications ?? []).map((a) => a.Name).sort()).toEqual([
141 "Hadoop",
142 "Spark",
143 ]);
144 const tags = Object.fromEntries((created.Cluster?.Tags ?? []).map((t) => [t.Key, t.Value]));
145 expect(tags.fixture).toBe("emr-cluster");
146 const policy = yield* emr.getAutoTerminationPolicy({ ClusterId: cluster.clusterId });
147 expect(policy.AutoTerminationPolicy?.IdleTimeout).toBe(3600);
148
149 // Update in place — step concurrency and auto-termination policy sync
150 // without replacement.
151 const { cluster: updated } = yield* deploy({
152 stepConcurrencyLevel: 4,
153 idleTimeoutSeconds: 7200,
154 });
155 expect(updated.clusterId).toBe(cluster.clusterId);
156 const afterUpdate = yield* emr.describeCluster({ ClusterId: cluster.clusterId });
157 expect(afterUpdate.Cluster?.StepConcurrencyLevel).toBe(4);
158 const updatedPolicy = yield* emr.getAutoTerminationPolicy({ ClusterId: cluster.clusterId });
159 expect(updatedPolicy.AutoTerminationPolicy?.IdleTimeout).toBe(7200);
160
161 // Destroy — the provider initiates termination (irreversible) and
162 // waits for the cluster to leave the active states.
163 yield* stack.destroy();
164 yield* assertClusterTerminating(cluster.clusterId);
165 }),
166 // create (~10-15 min) + update + termination initiation, one test.
167 {
168 tags: [
169 "provider:aws",
170 "provider:aws:ec2",
171 "provider:aws:emr",
172 "provider:aws:iam",
173 "provider:aws:s3",
174 "live",
175 ],
176 timeout: 2_700_000,
177 },
178 );
179
180 // Termination is verified as INITIATED (TERMINATING, irreversible) or fully
181 // TERMINATED; full instance teardown takes ~5-10 more minutes server-side.
182 const assertClusterTerminating = (clusterId: string) =>
183 Effect.gen(function* () {
184 const state = yield* emr.describeCluster({ ClusterId: clusterId }).pipe(
185 Effect.map((r) => r.Cluster?.Status?.State ?? "gone"),
186 Effect.catchTag("ClusterNotFound", () => Effect.succeed("gone" as const)),
187 );
188 if (
189 state !== "gone" &&
190 state !== "TERMINATING" &&
191 state !== "TERMINATED" &&
192 state !== "TERMINATED_WITH_ERRORS"
193 ) {
194 return yield* Effect.fail(
195 new Error(`EMR cluster '${clusterId}' still active (state: ${state})`),
196 );
197 }
198 }).pipe(
199 Effect.retry({ schedule: Schedule.max([Schedule.fixed("10 seconds"), Schedule.recurs(20)]) }),
200 );