The best way to control Claude Code, Codex and more from any computer or phone

main
ts 446 lines 17 KB
Raw
1 import * as emr from "@distilled.cloud/aws/emr-serverless";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
8 import { toWireMinutes } from "alchemy/Util/Duration";
9 import type * as Duration from "effect/Duration";
10 import * as Effect from "effect/Effect";
11 import * as Option from "effect/Option";
12 import * as Stream from "effect/Stream";
13 import type { Providers } from "../Providers.ts";
14 import { awaitApplicationCreated, awaitApplicationStopped } from "./internal.ts";
15
16 /**
17 * The engine an EMR Serverless application runs. Changing the type replaces
18 * the application.
19 */
20 export type ApplicationType = "SPARK" | "HIVE";
21
22 /**
23 * Auto-stop behavior for an EMR Serverless {@link Application}.
24 */
25 export interface AutoStopConfiguration {
26 /**
27 * Whether the application stops automatically after being idle.
28 * @default true
29 */
30 enabled?: boolean;
31 /**
32 * Idle time after which the application stops automatically — e.g.
33 * `"5 minutes"` or `Duration.minutes(5)`. Sent to AWS as whole minutes.
34 * @default "15 minutes"
35 */
36 idleTimeout?: Duration.Input;
37 }
38
39 /** Convert the alchemy-facing auto-stop config to the wire shape (minutes). */
40 const toWireAutoStop = (config: AutoStopConfiguration): emr.AutoStopConfig => ({
41 enabled: config.enabled,
42 idleTimeoutMinutes: toWireMinutes(config.idleTimeout),
43 });
44
45 export interface ApplicationProps {
46 /**
47 * Name of the application (1-64 characters: letters, digits, `.` `_` `/`
48 * `#` `-`). Changing the name replaces the application.
49 * @default a generated physical name
50 */
51 applicationName?: string;
52 /**
53 * The engine type. Changing the type replaces the application.
54 * @default "SPARK"
55 */
56 type?: ApplicationType;
57 /**
58 * The Amazon EMR release associated with the application, e.g.
59 * `emr-7.9.0`. Updatable in place (the application must be in a `CREATED`
60 * or `STOPPED` state; the provider stops a `STARTED` application before
61 * applying the update).
62 */
63 releaseLabel: string;
64 /**
65 * The CPU architecture of the application.
66 * @default "X86_64"
67 */
68 architecture?: "X86_64" | "ARM64";
69 /**
70 * Pre-initialized capacity kept warm per worker type (e.g. `Driver` and
71 * `Executor` for Spark; `HiveDriver` and `TezTask` for Hive). Warm workers
72 * bill while the application is started.
73 */
74 initialCapacity?: Record<string, emr.InitialCapacityConfig>;
75 /**
76 * The maximum aggregate vCPU, memory and disk the application may scale to.
77 */
78 maximumCapacity?: emr.MaximumAllowedResources;
79 /**
80 * Whether the application starts automatically on job submission.
81 * @default enabled
82 */
83 autoStartConfiguration?: emr.AutoStartConfig;
84 /**
85 * Whether (and after how long idle) the application stops
86 * automatically.
87 * @default enabled after 15 idle minutes
88 */
89 autoStopConfiguration?: AutoStopConfiguration;
90 /**
91 * VPC connectivity (subnet + security group IDs) for jobs that must reach
92 * resources in a VPC. Omit for the default non-VPC connectivity.
93 */
94 networkConfiguration?: emr.NetworkConfiguration;
95 /**
96 * Tags to apply to the application. Merged with the internal Alchemy tags.
97 */
98 tags?: Record<string, string>;
99 }
100
101 export interface Application extends Resource<
102 "AWS.EMRServerless.Application",
103 ApplicationProps,
104 {
105 /** The ID of the application. */
106 applicationId: string;
107 /** The name of the application. */
108 applicationName: string;
109 /** The ARN of the application. */
110 applicationArn: string;
111 /** The application type (`SPARK` or `HIVE`). */
112 type: string;
113 /** The EMR release label the application runs (e.g. `emr-7.5.0`). */
114 releaseLabel: string;
115 /** The application state (e.g. `CREATED`, `STARTED`, `STOPPED`). */
116 state?: string;
117 },
118 {},
119 Providers
120 > {}
121
122 /**
123 * An Amazon EMR Serverless application — a serverless Spark or Hive
124 * environment that automatically provisions and scales workers per job,
125 * with no cluster to manage. An application in the `CREATED` or `STOPPED`
126 * state costs nothing; billing only occurs for workers while the application
127 * is started (including any pre-initialized `initialCapacity`).
128 *
129 * ### Creating Applications
130 * **Example:** Spark Application
131 * ```typescript
132 * import * as AWS from "@alchemy.run/aws";
133 *
134 * const app = yield* AWS.EMRServerless.Application("Spark", {
135 * releaseLabel: "emr-7.9.0",
136 * });
137 * // app.applicationId is passed to StartJobRun
138 * ```
139 *
140 * **Example:** Hive Application with Auto-Stop Tuning
141 * ```typescript
142 * const app = yield* AWS.EMRServerless.Application("Hive", {
143 * type: "HIVE",
144 * releaseLabel: "emr-7.9.0",
145 * autoStartConfiguration: { enabled: true },
146 * autoStopConfiguration: { enabled: true, idleTimeout: "5 minutes" },
147 * });
148 * ```
149 *
150 * ### Capacity
151 * **Example:** Pre-Initialized Capacity for Low-Latency Jobs
152 * ```typescript
153 * const app = yield* AWS.EMRServerless.Application("Warm", {
154 * releaseLabel: "emr-7.9.0",
155 * initialCapacity: {
156 * Driver: {
157 * workerCount: 1,
158 * workerConfiguration: { cpu: "2 vCPU", memory: "4 GB" },
159 * },
160 * Executor: {
161 * workerCount: 2,
162 * workerConfiguration: { cpu: "2 vCPU", memory: "4 GB" },
163 * },
164 * },
165 * maximumCapacity: { cpu: "16 vCPU", memory: "64 GB" },
166 * });
167 * ```
168 *
169 * ### Networking
170 * **Example:** VPC-Connected Application
171 * ```typescript
172 * const app = yield* AWS.EMRServerless.Application("InVpc", {
173 * releaseLabel: "emr-7.9.0",
174 * networkConfiguration: {
175 * subnetIds: [subnet.subnetId],
176 * securityGroupIds: [securityGroup.securityGroupId],
177 * },
178 * });
179 * ```
180 *
181 * @resource
182 */
183 export const Application = Resource<Application>("AWS.EMRServerless.Application");
184
185 /** Normalize a possibly-undefined structural prop for drift comparison. */
186 const canonical = (value: unknown): string => (value === undefined ? "" : JSON.stringify(value));
187
188 /**
189 * Derive a deterministic create token from the resource's instance ID so a
190 * retried create after a crashed reconcile never double-provisions.
191 */
192 const createToken = (instanceId: string): string =>
193 instanceId.replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 64) || "alchemy";
194
195 export const ApplicationProvider = () =>
196 Provider.effect(
197 Application,
198 Effect.gen(function* () {
199 const createName = Effect.fn(function* (
200 id: string,
201 props: { applicationName?: string | undefined },
202 ) {
203 return props.applicationName ?? (yield* createPhysicalName({ id, maxLength: 64 }));
204 });
205
206 const toAttributes = (application: emr.Application) => ({
207 applicationId: application.applicationId,
208 applicationName: application.name ?? application.applicationId,
209 applicationArn: application.arn,
210 type: application.type,
211 releaseLabel: application.releaseLabel,
212 state: application.state,
213 });
214
215 const observeById = Effect.fn(function* (applicationId: string) {
216 return yield* emr.getApplication({ applicationId }).pipe(
217 Effect.map((response) => response.application),
218 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
219 );
220 });
221
222 // Applications can only be fetched by id; when the id is unknown (state
223 // persistence failed before the first reconcile completed), find the
224 // live (non-terminated) application carrying the derived physical name.
225 const observeByName = Effect.fn(function* (name: string) {
226 const summary = yield* emr.listApplications.items({}).pipe(
227 Stream.filter((s) => s.name === name && s.state !== "TERMINATED"),
228 Stream.runHead,
229 Effect.map(Option.getOrUndefined),
230 );
231 if (summary === undefined) {
232 return undefined;
233 }
234 return yield* observeById(summary.id);
235 });
236
237 const observe = Effect.fn(function* (applicationId: string | undefined, name: string) {
238 const byId = applicationId !== undefined ? yield* observeById(applicationId) : undefined;
239 if (byId !== undefined && byId.state !== "TERMINATED") {
240 return byId;
241 }
242 return yield* observeByName(name);
243 });
244
245 const syncTags = Effect.fn(function* (
246 application: emr.Application,
247 desired: Record<string, string>,
248 ) {
249 // Diff against OBSERVED cloud tags (adoption may bring foreign tags).
250 const observed = Object.fromEntries(
251 Object.entries(application.tags ?? {}).filter(
252 (entry): entry is [string, string] => entry[1] !== undefined,
253 ),
254 );
255 const { upsert, removed } = diffTags(observed, desired);
256 if (upsert.length > 0) {
257 yield* emr.tagResource({
258 resourceArn: application.arn,
259 tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
260 });
261 }
262 if (removed.length > 0) {
263 yield* emr.untagResource({
264 resourceArn: application.arn,
265 tagKeys: removed,
266 });
267 }
268 });
269
270 return Application.Provider.of({
271 stables: ["applicationId", "applicationName", "applicationArn", "type"],
272
273 list: () =>
274 Effect.gen(function* () {
275 const pages = yield* emr.listApplications.pages({}).pipe(Stream.runCollect);
276 return Array.from(pages)
277 .flatMap((page) => page.applications)
278 .filter((s) => s.state !== "TERMINATED")
279 .map((s) => ({
280 applicationId: s.id,
281 applicationName: s.name ?? s.id,
282 applicationArn: s.arn,
283 type: s.type,
284 releaseLabel: s.releaseLabel,
285 state: s.state,
286 }));
287 }),
288
289 read: Effect.fn(function* ({ id, olds, output }) {
290 const name = output?.applicationName ?? (yield* createName(id, olds ?? {}));
291 const application = yield* observe(output?.applicationId, name);
292 if (application === undefined) {
293 return undefined;
294 }
295 const attrs = toAttributes(application);
296 return (yield* hasAlchemyTags(id, application.tags)) ? attrs : Unowned(attrs);
297 }),
298
299 diff: Effect.fn(function* ({ id, news, olds }) {
300 if (!isResolved(news)) return undefined;
301 const oldName = yield* createName(id, olds);
302 const newName = yield* createName(id, news);
303 if (oldName !== newName) {
304 return { action: "replace" } as const;
305 }
306 if ((olds.type ?? "SPARK") !== (news.type ?? "SPARK")) {
307 return { action: "replace" } as const;
308 }
309 // releaseLabel/capacity/autoStart/autoStop/network/tags → update
310 }),
311
312 reconcile: Effect.fn(function* ({ id, news, output, session, instanceId }) {
313 const name = output?.applicationName ?? (yield* createName(id, news));
314 const type = news.type ?? "SPARK";
315 const internalTags = yield* createInternalTags(id);
316 const desiredTags = { ...news.tags, ...internalTags };
317 const desiredAutoStop =
318 news.autoStopConfiguration !== undefined
319 ? toWireAutoStop(news.autoStopConfiguration)
320 : undefined;
321
322 // 1. OBSERVE — cloud state is authoritative; output is an id cache
323 let application = yield* observe(output?.applicationId, name);
324
325 // 2. ENSURE — create if missing; ~1 min async until CREATED
326 if (application === undefined || application.state === "TERMINATED") {
327 const created = yield* emr
328 .createApplication({
329 clientToken: createToken(instanceId),
330 name,
331 type,
332 releaseLabel: news.releaseLabel,
333 architecture: news.architecture,
334 initialCapacity: news.initialCapacity,
335 maximumCapacity: news.maximumCapacity,
336 autoStartConfiguration: news.autoStartConfiguration,
337 autoStopConfiguration: desiredAutoStop,
338 networkConfiguration: news.networkConfiguration,
339 tags: desiredTags,
340 })
341 .pipe(
342 // a concurrent reconciler already created it — observe instead
343 Effect.catchTag("ConflictException", () => Effect.undefined),
344 );
345 const applicationId =
346 created?.applicationId ?? (yield* observeByName(name))?.applicationId;
347 if (applicationId === undefined) {
348 return yield* emr.ResourceNotFoundException.make({
349 message: `application ${name} not visible after create`,
350 });
351 }
352 yield* session.note(`creating application ${name} (async)...`);
353 application = yield* awaitApplicationCreated(applicationId);
354 } else if (application.state === "CREATING") {
355 application = yield* awaitApplicationCreated(application.applicationId);
356 }
357
358 // 3. SYNC — diff each mutable aspect (observed vs desired), apply a
359 // single updateApplication carrying only the drifted fields.
360 const drift = {
361 releaseLabel:
362 news.releaseLabel !== application.releaseLabel ? news.releaseLabel : undefined,
363 architecture:
364 news.architecture !== undefined && news.architecture !== application.architecture
365 ? news.architecture
366 : undefined,
367 initialCapacity:
368 news.initialCapacity !== undefined &&
369 canonical(news.initialCapacity) !== canonical(application.initialCapacity)
370 ? news.initialCapacity
371 : undefined,
372 maximumCapacity:
373 news.maximumCapacity !== undefined &&
374 canonical(news.maximumCapacity) !== canonical(application.maximumCapacity)
375 ? news.maximumCapacity
376 : undefined,
377 autoStartConfiguration:
378 news.autoStartConfiguration !== undefined &&
379 canonical(news.autoStartConfiguration) !==
380 canonical(application.autoStartConfiguration)
381 ? news.autoStartConfiguration
382 : undefined,
383 autoStopConfiguration:
384 desiredAutoStop !== undefined &&
385 canonical(desiredAutoStop) !== canonical(application.autoStopConfiguration)
386 ? desiredAutoStop
387 : undefined,
388 networkConfiguration:
389 news.networkConfiguration !== undefined &&
390 canonical(news.networkConfiguration) !== canonical(application.networkConfiguration)
391 ? news.networkConfiguration
392 : undefined,
393 };
394 if (Object.values(drift).some((v) => v !== undefined)) {
395 // Updates require a CREATED or STOPPED state — stop a started
396 // application first (autoStart re-starts it on job submission).
397 if (application.state === "STARTED" || application.state === "STARTING") {
398 yield* emr.stopApplication({
399 applicationId: application.applicationId,
400 });
401 }
402 yield* awaitApplicationStopped(application.applicationId);
403 // Fresh token per update: distinct desired states must never be
404 // deduplicated against each other.
405 const updateToken = yield* Effect.sync(() => crypto.randomUUID());
406 const updated = yield* emr.updateApplication({
407 applicationId: application.applicationId,
408 clientToken: updateToken,
409 ...drift,
410 });
411 application = updated.application;
412 }
413
414 // 3b. SYNC TAGS — diff against observed cloud tags
415 yield* syncTags(application, desiredTags);
416
417 yield* session.note(application.applicationId);
418 return toAttributes(application);
419 }),
420
421 delete: Effect.fn(function* ({ output }) {
422 const application = yield* emr
423 .getApplication({ applicationId: output.applicationId })
424 .pipe(
425 Effect.map((response) => response.application),
426 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
427 );
428 if (application === undefined || application.state === "TERMINATED") {
429 return;
430 }
431 // Deletion requires a CREATED or STOPPED state.
432 if (application.state === "STARTED" || application.state === "STARTING") {
433 yield* emr.stopApplication({
434 applicationId: application.applicationId,
435 });
436 }
437 yield* awaitApplicationStopped(application.applicationId).pipe(
438 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
439 );
440 yield* emr
441 .deleteApplication({ applicationId: output.applicationId })
442 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
443 }),
444 });
445 }),
446 );