@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 428 lines 14.6 KB
Raw
1 import * as emrc from "@distilled.cloud/aws/emr-containers";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Option from "effect/Option";
11 import * as Schedule from "effect/Schedule";
12 import * as Stream from "effect/Stream";
13 import type { Providers } from "../Providers.ts";
14
15 export interface JobTemplateSparkSubmitJobDriver {
16 /**
17 * The entry point of the Spark application — an S3 URI to a jar or Python
18 * script (e.g. `s3://my-bucket/scripts/etl.py`).
19 */
20 entryPoint: string;
21 /**
22 * Arguments passed to the entry point.
23 */
24 entryPointArguments?: string[];
25 /**
26 * `spark-submit` parameters (e.g. `--conf spark.executor.instances=2`).
27 */
28 sparkSubmitParameters?: string;
29 }
30
31 export interface JobTemplateSparkSqlJobDriver {
32 /**
33 * The S3 URI of the SQL file to execute.
34 */
35 entryPoint?: string;
36 /**
37 * Spark SQL parameters.
38 */
39 sparkSqlParameters?: string;
40 }
41
42 export interface JobTemplateJobDriver {
43 /**
44 * The Spark submit job driver.
45 */
46 sparkSubmitJobDriver?: JobTemplateSparkSubmitJobDriver;
47 /**
48 * The Spark SQL job driver.
49 */
50 sparkSqlJobDriver?: JobTemplateSparkSqlJobDriver;
51 }
52
53 export interface JobTemplateConfiguration {
54 /**
55 * The classification of the configuration (e.g. `spark-defaults`).
56 */
57 classification: string;
58 /**
59 * Configuration properties for the classification.
60 */
61 properties?: Record<string, string>;
62 /**
63 * Nested configurations.
64 */
65 configurations?: JobTemplateConfiguration[];
66 }
67
68 export interface JobTemplateMonitoringConfiguration {
69 /**
70 * Whether the persistent application UI (Spark history server) is enabled.
71 */
72 persistentAppUI?: "ENABLED" | "DISABLED" | (string & {});
73 /**
74 * CloudWatch log delivery for the job's containers.
75 */
76 cloudWatchMonitoringConfiguration?: {
77 /** The CloudWatch log group to deliver logs to. */
78 logGroupName?: string;
79 /** A prefix for the log stream names. */
80 logStreamNamePrefix?: string;
81 };
82 /**
83 * S3 log delivery for the job's containers.
84 */
85 s3MonitoringConfiguration?: {
86 /** The S3 URI to deliver logs to. */
87 logUri?: string;
88 };
89 }
90
91 export interface JobTemplateConfigurationOverrides {
92 /**
93 * Application configurations (Spark/Hadoop classification properties).
94 */
95 applicationConfiguration?: JobTemplateConfiguration[];
96 /**
97 * Monitoring (log delivery) configuration.
98 */
99 monitoringConfiguration?: JobTemplateMonitoringConfiguration;
100 }
101
102 export interface JobTemplateParameterConfiguration {
103 /**
104 * The data type of the template parameter.
105 * @default "STRING"
106 */
107 type?: "NUMBER" | "STRING" | (string & {});
108 /**
109 * The default value used when `StartJobRun` omits the parameter.
110 */
111 defaultValue?: string;
112 }
113
114 export interface JobTemplateDataProps {
115 /**
116 * The IAM execution role ARN the job runs as. May reference template
117 * parameters (e.g. `${ExecutionRoleArn}`).
118 */
119 executionRoleArn: string;
120 /**
121 * The EMR release label (e.g. `emr-7.5.0-latest`).
122 */
123 releaseLabel: string;
124 /**
125 * Configuration overrides applied to jobs started from the template.
126 */
127 configurationOverrides?: JobTemplateConfigurationOverrides;
128 /**
129 * The job driver (Spark submit or Spark SQL) for jobs started from the
130 * template.
131 */
132 jobDriver: JobTemplateJobDriver;
133 /**
134 * Declares the `${placeholders}` used in the template and their types /
135 * default values.
136 */
137 parameterConfiguration?: Record<string, JobTemplateParameterConfiguration>;
138 /**
139 * Tags applied to the *job runs* started from this template (distinct from
140 * the template's own `tags`).
141 */
142 jobTags?: Record<string, string>;
143 }
144
145 export interface JobTemplateProps {
146 /**
147 * Name of the job template (1-64 characters). Changing the name replaces
148 * the job template.
149 * @default a generated physical name
150 */
151 jobTemplateName?: string;
152 /**
153 * The StartJobRun values the template stores. Job templates are immutable —
154 * any change replaces the template.
155 */
156 jobTemplateData: JobTemplateDataProps;
157 /**
158 * The KMS key ARN used to encrypt the template. Changing it replaces the
159 * job template.
160 */
161 kmsKeyArn?: string;
162 /**
163 * Tags to apply to the job template. Merged with the internal Alchemy
164 * tags. Tags are set at creation only — the EMR containers `TagResource`
165 * API rejects job template ARNs (typed `InvalidResourceArn`), so changing
166 * tags replaces the template.
167 */
168 tags?: Record<string, string>;
169 }
170
171 export interface JobTemplate extends Resource<
172 "AWS.EMRContainers.JobTemplate",
173 JobTemplateProps,
174 {
175 /** The ID of the job template. */
176 jobTemplateId: string;
177 /** The name of the job template. */
178 jobTemplateName: string;
179 /** The ARN of the job template. */
180 jobTemplateArn: string;
181 },
182 {},
183 Providers
184 > {}
185
186 export class JobTemplateConsistencyError extends Data.TaggedError("JobTemplateConsistencyError")<{
187 readonly jobTemplateId: string;
188 readonly jobTemplateName: string;
189 readonly operation: "create" | "delete";
190 readonly message: string;
191 }> {}
192
193 /**
194 * An Amazon EMR on EKS job template — a stored set of `StartJobRun` values
195 * (execution role, release label, job driver, configuration) that can be
196 * referenced by ID when starting job runs, optionally with `${placeholder}`
197 * parameters filled in per run.
198 *
199 * Job templates are account-level and fully immutable: any change (including
200 * tags, which the tagging API does not support post-create for templates)
201 * replaces the template. They pair with the
202 * {@link StartJobRun | AWS.EMRContainers.StartJobRun} binding — a Lambda can
203 * start a templated Spark job with just the template ID and parameter values.
204 *
205 * ### Creating Job Templates
206 * **Example:** A Spark Job Template
207 * ```typescript
208 * import * as AWS from "@alchemy.run/aws";
209 *
210 * const template = yield* AWS.EMRContainers.JobTemplate("EtlTemplate", {
211 * jobTemplateData: {
212 * executionRoleArn: jobRole.roleArn,
213 * releaseLabel: "emr-7.5.0-latest",
214 * jobDriver: {
215 * sparkSubmitJobDriver: {
216 * entryPoint: "s3://my-bucket/scripts/etl.py",
217 * sparkSubmitParameters: "--conf spark.executor.instances=2",
218 * },
219 * },
220 * },
221 * });
222 * ```
223 *
224 * **Example:** Parameterized Template
225 * ```typescript
226 * const template = yield* AWS.EMRContainers.JobTemplate("Parameterized", {
227 * jobTemplateData: {
228 * executionRoleArn: jobRole.roleArn,
229 * releaseLabel: "emr-7.5.0-latest",
230 * jobDriver: {
231 * sparkSubmitJobDriver: { entryPoint: "${EntryPoint}" },
232 * },
233 * parameterConfiguration: {
234 * EntryPoint: { type: "STRING" },
235 * },
236 * },
237 * });
238 * // StartJobRun with jobTemplateId + jobTemplateParameters: { EntryPoint: "s3://..." }
239 * ```
240 *
241 * @resource
242 */
243 export const JobTemplate = Resource<JobTemplate>("AWS.EMRContainers.JobTemplate");
244
245 /** Convert declared props to the distilled wire shape. */
246 const toJobTemplateData = (data: JobTemplateDataProps): emrc.JobTemplateData => ({
247 executionRoleArn: data.executionRoleArn,
248 releaseLabel: data.releaseLabel,
249 configurationOverrides: data.configurationOverrides,
250 jobDriver: data.jobDriver,
251 parameterConfiguration: data.parameterConfiguration,
252 jobTags: data.jobTags,
253 });
254
255 export const JobTemplateProvider = () =>
256 Provider.effect(
257 JobTemplate,
258 Effect.gen(function* () {
259 const createName = Effect.fn(function* (
260 id: string,
261 props: { jobTemplateName?: string | undefined },
262 ) {
263 return props.jobTemplateName ?? (yield* createPhysicalName({ id, maxLength: 64 }));
264 });
265
266 const toAttributes = (jt: emrc.JobTemplate) => ({
267 jobTemplateId: jt.id!,
268 jobTemplateName: jt.name!,
269 jobTemplateArn: jt.arn!,
270 });
271
272 const observeById = Effect.fn(function* (id: string) {
273 return yield* emrc.describeJobTemplate({ id }).pipe(
274 Effect.map((response) => response.jobTemplate),
275 Effect.catchTag(
276 ["ResourceNotFoundException", "ValidationException"],
277 // a malformed/unknown id is "not present", same as not-found
278 () => Effect.undefined,
279 ),
280 );
281 });
282
283 const observeByName = Effect.fn(function* (name: string) {
284 return yield* emrc.listJobTemplates.items({}).pipe(
285 Stream.filter((jt) => jt.name === name),
286 Stream.runHead,
287 Effect.map(Option.getOrUndefined),
288 );
289 });
290
291 const observeListedById = Effect.fn(function* (id: string) {
292 return yield* emrc.listJobTemplates.items({}).pipe(
293 Stream.filter((jt) => jt.id === id),
294 Stream.runHead,
295 Effect.map(Option.getOrUndefined),
296 );
297 });
298
299 const awaitListVisibility = (id: string) =>
300 Effect.repeat(observeListedById(id), {
301 schedule: Schedule.spaced("3 seconds"),
302 until: (jt) => jt !== undefined,
303 times: 10,
304 });
305
306 const awaitAbsent = (id: string) =>
307 Effect.repeat(Effect.all([observeById(id), observeListedById(id)] as const), {
308 schedule: Schedule.spaced("3 seconds"),
309 until: ([described, listed]) => described === undefined && listed === undefined,
310 times: 10,
311 });
312
313 const observe = Effect.fn(function* (id: string | undefined, name: string) {
314 const byId = id !== undefined ? yield* observeById(id) : undefined;
315 return byId ?? (yield* observeByName(name));
316 });
317
318 return JobTemplate.Provider.of({
319 stables: ["jobTemplateId", "jobTemplateName", "jobTemplateArn"],
320
321 list: () =>
322 Effect.gen(function* () {
323 const pages = yield* emrc.listJobTemplates.pages({}).pipe(Stream.runCollect);
324 return Array.from(pages)
325 .flatMap((page) => page.templates ?? [])
326 .filter((jt) => jt.id !== undefined && jt.name !== undefined && jt.arn !== undefined)
327 .map(toAttributes);
328 }),
329
330 read: Effect.fn(function* ({ id, olds, output }) {
331 const name = output?.jobTemplateName ?? (yield* createName(id, olds ?? {}));
332 const jt = yield* observe(output?.jobTemplateId, name);
333 if (jt?.id === undefined || jt.arn === undefined) {
334 return undefined;
335 }
336 const attrs = toAttributes(jt);
337 return (yield* hasAlchemyTags(id, jt.tags)) ? attrs : Unowned(attrs);
338 }),
339
340 diff: Effect.fn(function* ({ id, news, olds }) {
341 if (!isResolved(news)) return undefined;
342 const oldName = yield* createName(id, olds);
343 const newName = yield* createName(id, news);
344 const { upsert, removed } = diffTags(olds.tags ?? {}, news.tags ?? {});
345 if (
346 oldName !== newName ||
347 JSON.stringify(toJobTemplateData(olds.jobTemplateData)) !==
348 JSON.stringify(toJobTemplateData(news.jobTemplateData)) ||
349 olds.kmsKeyArn !== news.kmsKeyArn ||
350 upsert.length > 0 ||
351 removed.length > 0
352 ) {
353 // job templates are fully immutable — even tags: the tagging API
354 // rejects job template ARNs (typed InvalidResourceArn), so tags
355 // are set at creation only and any change replaces
356 return { action: "replace" } as const;
357 }
358 }),
359
360 reconcile: Effect.fn(function* ({ id, news, output, session, instanceId }) {
361 const name = output?.jobTemplateName ?? (yield* createName(id, news));
362 const internalTags = yield* createInternalTags(id);
363 const desiredTags = { ...news.tags, ...internalTags };
364
365 // 1. OBSERVE — cloud state is authoritative; output is an id cache
366 let jt = yield* observe(output?.jobTemplateId, name);
367
368 // 2. ENSURE — create if missing (synchronous, available immediately)
369 if (jt === undefined) {
370 const created = yield* emrc.createJobTemplate({
371 // deterministic per instance: a retried create after a crashed
372 // reconcile never double-provisions
373 clientToken: instanceId.replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 64) || "alchemy",
374 name,
375 jobTemplateData: toJobTemplateData(news.jobTemplateData),
376 kmsKeyArn: news.kmsKeyArn,
377 tags: desiredTags,
378 });
379 jt =
380 created.id !== undefined
381 ? yield* emrc
382 .describeJobTemplate({ id: created.id })
383 .pipe(Effect.map((r) => r.jobTemplate))
384 : yield* observeByName(name);
385 if (jt?.id === undefined) {
386 return yield* emrc.ResourceNotFoundException.make({
387 message: `job template ${name} not visible after create`,
388 });
389 }
390 const listed = yield* awaitListVisibility(jt.id);
391 if (listed?.id !== jt.id) {
392 return yield* new JobTemplateConsistencyError({
393 jobTemplateId: jt.id,
394 jobTemplateName: name,
395 operation: "create",
396 message: `job template ${name} (${jt.id}) was not visible in ListJobTemplates after 30 seconds`,
397 });
398 }
399 }
400
401 // No tag sync: EMR containers' TagResource rejects job template
402 // ARNs (typed InvalidResourceArn) — tags exist at creation only,
403 // and diff() replaces the template on any tag change.
404
405 yield* session.note(jt.id!);
406 return toAttributes(jt);
407 }),
408
409 delete: Effect.fn(function* ({ output }) {
410 // deleteJobTemplate's typed union has no not-found tag — the API
411 // reports missing/already-deleted ids as validation errors, which
412 // makes the swallow below the idempotency guarantee.
413 yield* emrc
414 .deleteJobTemplate({ id: output.jobTemplateId })
415 .pipe(Effect.catchTag("ValidationException", () => Effect.void));
416 const [described, listed] = yield* awaitAbsent(output.jobTemplateId);
417 if (described !== undefined || listed !== undefined) {
418 return yield* new JobTemplateConsistencyError({
419 jobTemplateId: output.jobTemplateId,
420 jobTemplateName: output.jobTemplateName,
421 operation: "delete",
422 message: `job template ${output.jobTemplateName} (${output.jobTemplateId}) remained visible after delete for 30 seconds`,
423 });
424 }
425 }),
426 });
427 }),
428 );