The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as emr from "@distilled.cloud/aws/emr-serverless"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import { createPhysicalName } from "alchemy/PhysicalName"; |
| 5 | import * as Provider from "alchemy/Provider"; |
| 6 | import { Resource } from "alchemy/Resource"; |
| 7 | import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags"; |
| 8 | import { toWireMinutes } from "alchemy/Util/Duration"; |
| 9 | import type * as Duration from "effect/Duration"; |
| 10 | import * as Effect from "effect/Effect"; |
| 11 | import * as Option from "effect/Option"; |
| 12 | import * as Stream from "effect/Stream"; |
| 13 | import type { Providers } from "../Providers.ts"; |
| 14 | import { awaitApplicationCreated, awaitApplicationStopped } from "./internal.ts"; |
| 15 | |
| 16 | /** |
| 17 | * The engine an EMR Serverless application runs. Changing the type replaces |
| 18 | * the application. |
| 19 | */ |
| 20 | export type ApplicationType = "SPARK" | "HIVE"; |
| 21 | |
| 22 | /** |
| 23 | * Auto-stop behavior for an EMR Serverless {@link Application}. |
| 24 | */ |
| 25 | export interface AutoStopConfiguration { |
| 26 | /** |
| 27 | * Whether the application stops automatically after being idle. |
| 28 | * @default true |
| 29 | */ |
| 30 | enabled?: boolean; |
| 31 | /** |
| 32 | * Idle time after which the application stops automatically — e.g. |
| 33 | * `"5 minutes"` or `Duration.minutes(5)`. Sent to AWS as whole minutes. |
| 34 | * @default "15 minutes" |
| 35 | */ |
| 36 | idleTimeout?: Duration.Input; |
| 37 | } |
| 38 | |
| 39 | /** Convert the alchemy-facing auto-stop config to the wire shape (minutes). */ |
| 40 | const toWireAutoStop = (config: AutoStopConfiguration): emr.AutoStopConfig => ({ |
| 41 | enabled: config.enabled, |
| 42 | idleTimeoutMinutes: toWireMinutes(config.idleTimeout), |
| 43 | }); |
| 44 | |
| 45 | export interface ApplicationProps { |
| 46 | /** |
| 47 | * Name of the application (1-64 characters: letters, digits, `.` `_` `/` |
| 48 | * `#` `-`). Changing the name replaces the application. |
| 49 | * @default a generated physical name |
| 50 | */ |
| 51 | applicationName?: string; |
| 52 | /** |
| 53 | * The engine type. Changing the type replaces the application. |
| 54 | * @default "SPARK" |
| 55 | */ |
| 56 | type?: ApplicationType; |
| 57 | /** |
| 58 | * The Amazon EMR release associated with the application, e.g. |
| 59 | * `emr-7.9.0`. Updatable in place (the application must be in a `CREATED` |
| 60 | * or `STOPPED` state; the provider stops a `STARTED` application before |
| 61 | * applying the update). |
| 62 | */ |
| 63 | releaseLabel: string; |
| 64 | /** |
| 65 | * The CPU architecture of the application. |
| 66 | * @default "X86_64" |
| 67 | */ |
| 68 | architecture?: "X86_64" | "ARM64"; |
| 69 | /** |
| 70 | * Pre-initialized capacity kept warm per worker type (e.g. `Driver` and |
| 71 | * `Executor` for Spark; `HiveDriver` and `TezTask` for Hive). Warm workers |
| 72 | * bill while the application is started. |
| 73 | */ |
| 74 | initialCapacity?: Record<string, emr.InitialCapacityConfig>; |
| 75 | /** |
| 76 | * The maximum aggregate vCPU, memory and disk the application may scale to. |
| 77 | */ |
| 78 | maximumCapacity?: emr.MaximumAllowedResources; |
| 79 | /** |
| 80 | * Whether the application starts automatically on job submission. |
| 81 | * @default enabled |
| 82 | */ |
| 83 | autoStartConfiguration?: emr.AutoStartConfig; |
| 84 | /** |
| 85 | * Whether (and after how long idle) the application stops |
| 86 | * automatically. |
| 87 | * @default enabled after 15 idle minutes |
| 88 | */ |
| 89 | autoStopConfiguration?: AutoStopConfiguration; |
| 90 | /** |
| 91 | * VPC connectivity (subnet + security group IDs) for jobs that must reach |
| 92 | * resources in a VPC. Omit for the default non-VPC connectivity. |
| 93 | */ |
| 94 | networkConfiguration?: emr.NetworkConfiguration; |
| 95 | /** |
| 96 | * Tags to apply to the application. Merged with the internal Alchemy tags. |
| 97 | */ |
| 98 | tags?: Record<string, string>; |
| 99 | } |
| 100 | |
| 101 | export interface Application extends Resource< |
| 102 | "AWS.EMRServerless.Application", |
| 103 | ApplicationProps, |
| 104 | { |
| 105 | /** The ID of the application. */ |
| 106 | applicationId: string; |
| 107 | /** The name of the application. */ |
| 108 | applicationName: string; |
| 109 | /** The ARN of the application. */ |
| 110 | applicationArn: string; |
| 111 | /** The application type (`SPARK` or `HIVE`). */ |
| 112 | type: string; |
| 113 | /** The EMR release label the application runs (e.g. `emr-7.5.0`). */ |
| 114 | releaseLabel: string; |
| 115 | /** The application state (e.g. `CREATED`, `STARTED`, `STOPPED`). */ |
| 116 | state?: string; |
| 117 | }, |
| 118 | {}, |
| 119 | Providers |
| 120 | > {} |
| 121 | |
| 122 | /** |
| 123 | * An Amazon EMR Serverless application — a serverless Spark or Hive |
| 124 | * environment that automatically provisions and scales workers per job, |
| 125 | * with no cluster to manage. An application in the `CREATED` or `STOPPED` |
| 126 | * state costs nothing; billing only occurs for workers while the application |
| 127 | * is started (including any pre-initialized `initialCapacity`). |
| 128 | * |
| 129 | * ### Creating Applications |
| 130 | * **Example:** Spark Application |
| 131 | * ```typescript |
| 132 | * import * as AWS from "@alchemy.run/aws"; |
| 133 | * |
| 134 | * const app = yield* AWS.EMRServerless.Application("Spark", { |
| 135 | * releaseLabel: "emr-7.9.0", |
| 136 | * }); |
| 137 | * // app.applicationId is passed to StartJobRun |
| 138 | * ``` |
| 139 | * |
| 140 | * **Example:** Hive Application with Auto-Stop Tuning |
| 141 | * ```typescript |
| 142 | * const app = yield* AWS.EMRServerless.Application("Hive", { |
| 143 | * type: "HIVE", |
| 144 | * releaseLabel: "emr-7.9.0", |
| 145 | * autoStartConfiguration: { enabled: true }, |
| 146 | * autoStopConfiguration: { enabled: true, idleTimeout: "5 minutes" }, |
| 147 | * }); |
| 148 | * ``` |
| 149 | * |
| 150 | * ### Capacity |
| 151 | * **Example:** Pre-Initialized Capacity for Low-Latency Jobs |
| 152 | * ```typescript |
| 153 | * const app = yield* AWS.EMRServerless.Application("Warm", { |
| 154 | * releaseLabel: "emr-7.9.0", |
| 155 | * initialCapacity: { |
| 156 | * Driver: { |
| 157 | * workerCount: 1, |
| 158 | * workerConfiguration: { cpu: "2 vCPU", memory: "4 GB" }, |
| 159 | * }, |
| 160 | * Executor: { |
| 161 | * workerCount: 2, |
| 162 | * workerConfiguration: { cpu: "2 vCPU", memory: "4 GB" }, |
| 163 | * }, |
| 164 | * }, |
| 165 | * maximumCapacity: { cpu: "16 vCPU", memory: "64 GB" }, |
| 166 | * }); |
| 167 | * ``` |
| 168 | * |
| 169 | * ### Networking |
| 170 | * **Example:** VPC-Connected Application |
| 171 | * ```typescript |
| 172 | * const app = yield* AWS.EMRServerless.Application("InVpc", { |
| 173 | * releaseLabel: "emr-7.9.0", |
| 174 | * networkConfiguration: { |
| 175 | * subnetIds: [subnet.subnetId], |
| 176 | * securityGroupIds: [securityGroup.securityGroupId], |
| 177 | * }, |
| 178 | * }); |
| 179 | * ``` |
| 180 | * |
| 181 | * @resource |
| 182 | */ |
| 183 | export const Application = Resource<Application>("AWS.EMRServerless.Application"); |
| 184 | |
| 185 | /** Normalize a possibly-undefined structural prop for drift comparison. */ |
| 186 | const canonical = (value: unknown): string => (value === undefined ? "" : JSON.stringify(value)); |
| 187 | |
| 188 | /** |
| 189 | * Derive a deterministic create token from the resource's instance ID so a |
| 190 | * retried create after a crashed reconcile never double-provisions. |
| 191 | */ |
| 192 | const createToken = (instanceId: string): string => |
| 193 | instanceId.replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 64) || "alchemy"; |
| 194 | |
| 195 | export const ApplicationProvider = () => |
| 196 | Provider.effect( |
| 197 | Application, |
| 198 | Effect.gen(function* () { |
| 199 | const createName = Effect.fn(function* ( |
| 200 | id: string, |
| 201 | props: { applicationName?: string | undefined }, |
| 202 | ) { |
| 203 | return props.applicationName ?? (yield* createPhysicalName({ id, maxLength: 64 })); |
| 204 | }); |
| 205 | |
| 206 | const toAttributes = (application: emr.Application) => ({ |
| 207 | applicationId: application.applicationId, |
| 208 | applicationName: application.name ?? application.applicationId, |
| 209 | applicationArn: application.arn, |
| 210 | type: application.type, |
| 211 | releaseLabel: application.releaseLabel, |
| 212 | state: application.state, |
| 213 | }); |
| 214 | |
| 215 | const observeById = Effect.fn(function* (applicationId: string) { |
| 216 | return yield* emr.getApplication({ applicationId }).pipe( |
| 217 | Effect.map((response) => response.application), |
| 218 | Effect.catchTag("ResourceNotFoundException", () => Effect.undefined), |
| 219 | ); |
| 220 | }); |
| 221 | |
| 222 | // Applications can only be fetched by id; when the id is unknown (state |
| 223 | // persistence failed before the first reconcile completed), find the |
| 224 | // live (non-terminated) application carrying the derived physical name. |
| 225 | const observeByName = Effect.fn(function* (name: string) { |
| 226 | const summary = yield* emr.listApplications.items({}).pipe( |
| 227 | Stream.filter((s) => s.name === name && s.state !== "TERMINATED"), |
| 228 | Stream.runHead, |
| 229 | Effect.map(Option.getOrUndefined), |
| 230 | ); |
| 231 | if (summary === undefined) { |
| 232 | return undefined; |
| 233 | } |
| 234 | return yield* observeById(summary.id); |
| 235 | }); |
| 236 | |
| 237 | const observe = Effect.fn(function* (applicationId: string | undefined, name: string) { |
| 238 | const byId = applicationId !== undefined ? yield* observeById(applicationId) : undefined; |
| 239 | if (byId !== undefined && byId.state !== "TERMINATED") { |
| 240 | return byId; |
| 241 | } |
| 242 | return yield* observeByName(name); |
| 243 | }); |
| 244 | |
| 245 | const syncTags = Effect.fn(function* ( |
| 246 | application: emr.Application, |
| 247 | desired: Record<string, string>, |
| 248 | ) { |
| 249 | // Diff against OBSERVED cloud tags (adoption may bring foreign tags). |
| 250 | const observed = Object.fromEntries( |
| 251 | Object.entries(application.tags ?? {}).filter( |
| 252 | (entry): entry is [string, string] => entry[1] !== undefined, |
| 253 | ), |
| 254 | ); |
| 255 | const { upsert, removed } = diffTags(observed, desired); |
| 256 | if (upsert.length > 0) { |
| 257 | yield* emr.tagResource({ |
| 258 | resourceArn: application.arn, |
| 259 | tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])), |
| 260 | }); |
| 261 | } |
| 262 | if (removed.length > 0) { |
| 263 | yield* emr.untagResource({ |
| 264 | resourceArn: application.arn, |
| 265 | tagKeys: removed, |
| 266 | }); |
| 267 | } |
| 268 | }); |
| 269 | |
| 270 | return Application.Provider.of({ |
| 271 | stables: ["applicationId", "applicationName", "applicationArn", "type"], |
| 272 | |
| 273 | list: () => |
| 274 | Effect.gen(function* () { |
| 275 | const pages = yield* emr.listApplications.pages({}).pipe(Stream.runCollect); |
| 276 | return Array.from(pages) |
| 277 | .flatMap((page) => page.applications) |
| 278 | .filter((s) => s.state !== "TERMINATED") |
| 279 | .map((s) => ({ |
| 280 | applicationId: s.id, |
| 281 | applicationName: s.name ?? s.id, |
| 282 | applicationArn: s.arn, |
| 283 | type: s.type, |
| 284 | releaseLabel: s.releaseLabel, |
| 285 | state: s.state, |
| 286 | })); |
| 287 | }), |
| 288 | |
| 289 | read: Effect.fn(function* ({ id, olds, output }) { |
| 290 | const name = output?.applicationName ?? (yield* createName(id, olds ?? {})); |
| 291 | const application = yield* observe(output?.applicationId, name); |
| 292 | if (application === undefined) { |
| 293 | return undefined; |
| 294 | } |
| 295 | const attrs = toAttributes(application); |
| 296 | return (yield* hasAlchemyTags(id, application.tags)) ? attrs : Unowned(attrs); |
| 297 | }), |
| 298 | |
| 299 | diff: Effect.fn(function* ({ id, news, olds }) { |
| 300 | if (!isResolved(news)) return undefined; |
| 301 | const oldName = yield* createName(id, olds); |
| 302 | const newName = yield* createName(id, news); |
| 303 | if (oldName !== newName) { |
| 304 | return { action: "replace" } as const; |
| 305 | } |
| 306 | if ((olds.type ?? "SPARK") !== (news.type ?? "SPARK")) { |
| 307 | return { action: "replace" } as const; |
| 308 | } |
| 309 | // releaseLabel/capacity/autoStart/autoStop/network/tags → update |
| 310 | }), |
| 311 | |
| 312 | reconcile: Effect.fn(function* ({ id, news, output, session, instanceId }) { |
| 313 | const name = output?.applicationName ?? (yield* createName(id, news)); |
| 314 | const type = news.type ?? "SPARK"; |
| 315 | const internalTags = yield* createInternalTags(id); |
| 316 | const desiredTags = { ...news.tags, ...internalTags }; |
| 317 | const desiredAutoStop = |
| 318 | news.autoStopConfiguration !== undefined |
| 319 | ? toWireAutoStop(news.autoStopConfiguration) |
| 320 | : undefined; |
| 321 | |
| 322 | // 1. OBSERVE — cloud state is authoritative; output is an id cache |
| 323 | let application = yield* observe(output?.applicationId, name); |
| 324 | |
| 325 | // 2. ENSURE — create if missing; ~1 min async until CREATED |
| 326 | if (application === undefined || application.state === "TERMINATED") { |
| 327 | const created = yield* emr |
| 328 | .createApplication({ |
| 329 | clientToken: createToken(instanceId), |
| 330 | name, |
| 331 | type, |
| 332 | releaseLabel: news.releaseLabel, |
| 333 | architecture: news.architecture, |
| 334 | initialCapacity: news.initialCapacity, |
| 335 | maximumCapacity: news.maximumCapacity, |
| 336 | autoStartConfiguration: news.autoStartConfiguration, |
| 337 | autoStopConfiguration: desiredAutoStop, |
| 338 | networkConfiguration: news.networkConfiguration, |
| 339 | tags: desiredTags, |
| 340 | }) |
| 341 | .pipe( |
| 342 | // a concurrent reconciler already created it — observe instead |
| 343 | Effect.catchTag("ConflictException", () => Effect.undefined), |
| 344 | ); |
| 345 | const applicationId = |
| 346 | created?.applicationId ?? (yield* observeByName(name))?.applicationId; |
| 347 | if (applicationId === undefined) { |
| 348 | return yield* emr.ResourceNotFoundException.make({ |
| 349 | message: `application ${name} not visible after create`, |
| 350 | }); |
| 351 | } |
| 352 | yield* session.note(`creating application ${name} (async)...`); |
| 353 | application = yield* awaitApplicationCreated(applicationId); |
| 354 | } else if (application.state === "CREATING") { |
| 355 | application = yield* awaitApplicationCreated(application.applicationId); |
| 356 | } |
| 357 | |
| 358 | // 3. SYNC — diff each mutable aspect (observed vs desired), apply a |
| 359 | // single updateApplication carrying only the drifted fields. |
| 360 | const drift = { |
| 361 | releaseLabel: |
| 362 | news.releaseLabel !== application.releaseLabel ? news.releaseLabel : undefined, |
| 363 | architecture: |
| 364 | news.architecture !== undefined && news.architecture !== application.architecture |
| 365 | ? news.architecture |
| 366 | : undefined, |
| 367 | initialCapacity: |
| 368 | news.initialCapacity !== undefined && |
| 369 | canonical(news.initialCapacity) !== canonical(application.initialCapacity) |
| 370 | ? news.initialCapacity |
| 371 | : undefined, |
| 372 | maximumCapacity: |
| 373 | news.maximumCapacity !== undefined && |
| 374 | canonical(news.maximumCapacity) !== canonical(application.maximumCapacity) |
| 375 | ? news.maximumCapacity |
| 376 | : undefined, |
| 377 | autoStartConfiguration: |
| 378 | news.autoStartConfiguration !== undefined && |
| 379 | canonical(news.autoStartConfiguration) !== |
| 380 | canonical(application.autoStartConfiguration) |
| 381 | ? news.autoStartConfiguration |
| 382 | : undefined, |
| 383 | autoStopConfiguration: |
| 384 | desiredAutoStop !== undefined && |
| 385 | canonical(desiredAutoStop) !== canonical(application.autoStopConfiguration) |
| 386 | ? desiredAutoStop |
| 387 | : undefined, |
| 388 | networkConfiguration: |
| 389 | news.networkConfiguration !== undefined && |
| 390 | canonical(news.networkConfiguration) !== canonical(application.networkConfiguration) |
| 391 | ? news.networkConfiguration |
| 392 | : undefined, |
| 393 | }; |
| 394 | if (Object.values(drift).some((v) => v !== undefined)) { |
| 395 | // Updates require a CREATED or STOPPED state — stop a started |
| 396 | // application first (autoStart re-starts it on job submission). |
| 397 | if (application.state === "STARTED" || application.state === "STARTING") { |
| 398 | yield* emr.stopApplication({ |
| 399 | applicationId: application.applicationId, |
| 400 | }); |
| 401 | } |
| 402 | yield* awaitApplicationStopped(application.applicationId); |
| 403 | // Fresh token per update: distinct desired states must never be |
| 404 | // deduplicated against each other. |
| 405 | const updateToken = yield* Effect.sync(() => crypto.randomUUID()); |
| 406 | const updated = yield* emr.updateApplication({ |
| 407 | applicationId: application.applicationId, |
| 408 | clientToken: updateToken, |
| 409 | ...drift, |
| 410 | }); |
| 411 | application = updated.application; |
| 412 | } |
| 413 | |
| 414 | // 3b. SYNC TAGS — diff against observed cloud tags |
| 415 | yield* syncTags(application, desiredTags); |
| 416 | |
| 417 | yield* session.note(application.applicationId); |
| 418 | return toAttributes(application); |
| 419 | }), |
| 420 | |
| 421 | delete: Effect.fn(function* ({ output }) { |
| 422 | const application = yield* emr |
| 423 | .getApplication({ applicationId: output.applicationId }) |
| 424 | .pipe( |
| 425 | Effect.map((response) => response.application), |
| 426 | Effect.catchTag("ResourceNotFoundException", () => Effect.undefined), |
| 427 | ); |
| 428 | if (application === undefined || application.state === "TERMINATED") { |
| 429 | return; |
| 430 | } |
| 431 | // Deletion requires a CREATED or STOPPED state. |
| 432 | if (application.state === "STARTED" || application.state === "STARTING") { |
| 433 | yield* emr.stopApplication({ |
| 434 | applicationId: application.applicationId, |
| 435 | }); |
| 436 | } |
| 437 | yield* awaitApplicationStopped(application.applicationId).pipe( |
| 438 | Effect.catchTag("ResourceNotFoundException", () => Effect.undefined), |
| 439 | ); |
| 440 | yield* emr |
| 441 | .deleteApplication({ applicationId: output.applicationId }) |
| 442 | .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)); |
| 443 | }), |
| 444 | }); |
| 445 | }), |
| 446 | ); |