The best way to control Claude Code, Codex and more from any computer or phone

main
ts 344 lines 11 KB
Raw
1 import * as glue from "@distilled.cloud/aws/glue";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
8 import { toWireMinutes } from "alchemy/Util/Duration";
9 import type * as Duration from "effect/Duration";
10 import * as Effect from "effect/Effect";
11 import * as Stream from "effect/Stream";
12 import { AWSEnvironment } from "../Environment.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 fetchObservedTags,
16 jobArn,
17 reapGlueLogStreams,
18 retryWhileRoleNotAssumable,
19 syncTags,
20 } from "./internal.ts";
21
22 /** Shared, service-managed log groups Glue writes job-run streams to. */
23 const GLUE_JOB_LOG_GROUPS = [
24 "/aws-glue/python-jobs/output",
25 "/aws-glue/python-jobs/error",
26 "/aws-glue/jobs/output",
27 "/aws-glue/jobs/error",
28 "/aws-glue/jobs/logs-v2",
29 ];
30
31 export interface JobCommand {
32 /**
33 * The command name: `glueetl` (Spark), `pythonshell` (Python shell), or
34 * `gluestreaming` (Spark streaming).
35 */
36 name: "glueetl" | "pythonshell" | "gluestreaming" | (string & {});
37 /** The S3 path to the job script, e.g. `s3://my-bucket/scripts/job.py`. */
38 scriptLocation?: string;
39 /** For `pythonshell`, the Python version (`3` or `3.9`). */
40 pythonVersion?: string;
41 /** The Ray runtime version (for Ray jobs). */
42 runtime?: string;
43 }
44
45 export interface JobProps {
46 /**
47 * Name of the job. If omitted, a unique name is generated. Changing the
48 * name replaces the job.
49 * @default a generated physical name
50 */
51 jobName?: string;
52 /**
53 * The IAM role (ARN or name) the job assumes to run.
54 */
55 role: string;
56 /**
57 * The job's execution command (script location + runtime).
58 */
59 command: JobCommand;
60 /**
61 * A description of the job.
62 */
63 description?: string;
64 /**
65 * Default arguments passed to the script (keys are `--`-prefixed), e.g.
66 * `{ "--TempDir": "s3://.../tmp/", "--job-language": "python" }`.
67 */
68 defaultArguments?: Record<string, string>;
69 /**
70 * Arguments that cannot be overridden at run time.
71 */
72 nonOverridableArguments?: Record<string, string>;
73 /**
74 * Glue connection names the job uses.
75 */
76 connections?: string[];
77 /**
78 * Max retries before the job run is considered failed.
79 */
80 maxRetries?: number;
81 /**
82 * Job timeout, e.g. `"1 hour"` or `Duration.minutes(60)`. Rounded to
83 * whole minutes on the wire (the Glue API unit).
84 */
85 timeout?: Duration.Input;
86 /**
87 * Max Glue data processing units (DPUs) for `pythonshell` (0.0625 or 1).
88 * Mutually exclusive with `numberOfWorkers`/`workerType`.
89 */
90 maxCapacity?: number;
91 /**
92 * The Glue version (e.g. `4.0`, `3.0`).
93 */
94 glueVersion?: string;
95 /**
96 * The number of workers of `workerType` allocated (Spark jobs).
97 */
98 numberOfWorkers?: number;
99 /**
100 * The worker type for Spark jobs: `Standard`, `G.1X`, `G.2X`, `G.025X`,
101 * `G.4X`, `G.8X`, `Z.2X`.
102 */
103 workerType?: string;
104 /**
105 * Execution class: `STANDARD` or `FLEX`.
106 */
107 executionClass?: "STANDARD" | "FLEX";
108 /**
109 * Concurrency configuration.
110 */
111 executionProperty?: {
112 /** Maximum number of concurrent runs allowed. */
113 maxConcurrentRuns?: number;
114 };
115 /**
116 * Tags to apply to the job. Merged with internal Alchemy tags.
117 */
118 tags?: Record<string, string>;
119 }
120
121 export interface Job extends Resource<
122 "AWS.Glue.Job",
123 JobProps,
124 {
125 /** The name of the job. */
126 jobName: string;
127 /** The ARN of the job. */
128 jobArn: string;
129 /** The IAM role the job assumes to run. */
130 role: string;
131 },
132 {},
133 Providers
134 > {}
135
136 /**
137 * An AWS Glue job — a Spark (`glueetl`), Python shell (`pythonshell`), or
138 * streaming ETL job definition (script in S3 + IAM role + arguments). The
139 * definition lifecycle is instant and free; job *runs* are billed and are
140 * started via `startJobRun`.
141 * ### Creating Jobs
142 * **Example:** Python Shell Job
143 * ```typescript
144 * import * as AWS from "@alchemy.run/aws";
145 *
146 * const job = yield* AWS.Glue.Job("Etl", {
147 * role: jobRole.roleArn,
148 * command: {
149 * name: "pythonshell",
150 * pythonVersion: "3.9",
151 * scriptLocation: "s3://my-bucket/scripts/etl.py",
152 * },
153 * maxCapacity: 0.0625,
154 * glueVersion: "3.0",
155 * defaultArguments: { "--job-language": "python" },
156 * });
157 * ```
158 *
159 * **Example:** Spark ETL Job
160 * ```typescript
161 * const job = yield* AWS.Glue.Job("SparkEtl", {
162 * role: jobRole.roleArn,
163 * command: {
164 * name: "glueetl",
165 * scriptLocation: "s3://my-bucket/scripts/spark.py",
166 * },
167 * glueVersion: "4.0",
168 * workerType: "G.1X",
169 * numberOfWorkers: 2,
170 * timeout: "1 hour",
171 * });
172 * ```
173 *
174 * ### Running Jobs
175 * **Example:** Start a Job Run from a Lambda
176 * ```typescript
177 * // init
178 * const startJobRun = yield* AWS.Glue.StartJobRun(job);
179 *
180 * // runtime
181 * const { JobRunId } = yield* startJobRun({});
182 * ```
183 *
184 * @resource
185 */
186 export const Job = Resource<Job>("AWS.Glue.Job");
187
188 export const JobProvider = () =>
189 Provider.effect(
190 Job,
191 Effect.gen(function* () {
192 const createName = Effect.fn(function* (id: string, props: { jobName?: string | undefined }) {
193 return props.jobName ?? (yield* createPhysicalName({ id, maxLength: 255 }));
194 });
195
196 const observe = Effect.fn(function* (name: string) {
197 return yield* glue.getJob({ JobName: name }).pipe(
198 Effect.map((r) => r.Job),
199 Effect.catchTag("EntityNotFoundException", () => Effect.undefined),
200 );
201 });
202
203 const buildCommand = (command: JobCommand) => ({
204 Name: command.name,
205 ScriptLocation: command.scriptLocation,
206 PythonVersion: command.pythonVersion,
207 Runtime: command.runtime,
208 });
209
210 const buildDefinition = (props: JobProps) => ({
211 Role: props.role,
212 Command: buildCommand(props.command),
213 Description: props.description,
214 DefaultArguments: props.defaultArguments,
215 NonOverridableArguments: props.nonOverridableArguments,
216 Connections:
217 props.connections !== undefined ? { Connections: props.connections } : undefined,
218 MaxRetries: props.maxRetries,
219 Timeout: toWireMinutes(props.timeout),
220 MaxCapacity: props.maxCapacity,
221 GlueVersion: props.glueVersion,
222 NumberOfWorkers: props.numberOfWorkers,
223 WorkerType: props.workerType,
224 ExecutionClass: props.executionClass,
225 ExecutionProperty: props.executionProperty
226 ? { MaxConcurrentRuns: props.executionProperty.maxConcurrentRuns }
227 : undefined,
228 });
229
230 return Job.Provider.of({
231 stables: ["jobName", "jobArn"],
232
233 list: () =>
234 Effect.gen(function* () {
235 const { accountId, region } = yield* AWSEnvironment.current;
236 const pages = yield* glue.getJobs.pages({}).pipe(Stream.runCollect);
237 return Array.from(pages)
238 .flatMap((page) => page.Jobs ?? [])
239 .filter((j) => j.Name !== undefined)
240 .map((j) => ({
241 jobName: j.Name!,
242 jobArn: jobArn(region, accountId, j.Name!),
243 role: j.Role ?? "",
244 }));
245 }),
246
247 read: Effect.fn(function* ({ id, olds, output }) {
248 const { accountId, region } = yield* AWSEnvironment.current;
249 const name = output?.jobName ?? (yield* createName(id, olds ?? {}));
250 const job = yield* observe(name);
251 if (job?.Name === undefined) return undefined;
252 const arn = jobArn(region, accountId, job.Name);
253 const attrs = {
254 jobName: job.Name,
255 jobArn: arn,
256 role: job.Role ?? "",
257 };
258 const tags = yield* fetchObservedTags(arn);
259 return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
260 }),
261
262 diff: Effect.fn(function* ({ id, news, olds }) {
263 if (!isResolved(news)) return undefined;
264 const oldName = yield* createName(id, olds);
265 const newName = yield* createName(id, news);
266 if (oldName !== newName) return { action: "replace" } as const;
267 // everything else is UpdateJob-able
268 }),
269
270 reconcile: Effect.fn(function* ({ id, news, output, session }) {
271 const { accountId, region } = yield* AWSEnvironment.current;
272 const name = output?.jobName ?? (yield* createName(id, news));
273 const internalTags = yield* createInternalTags(id);
274 const desiredTags = { ...news.tags, ...internalTags };
275 const arn = jobArn(region, accountId, name);
276
277 // 1. OBSERVE
278 let job = yield* observe(name);
279
280 // 2. ENSURE / 3. SYNC
281 if (job === undefined) {
282 yield* retryWhileRoleNotAssumable(
283 glue.createJob({
284 Name: name,
285 ...buildDefinition(news),
286 Tags: desiredTags,
287 }),
288 ).pipe(Effect.catchTag("AlreadyExistsException", () => Effect.void));
289 } else {
290 // UpdateJob replaces the full JobUpdate (Name is not part of it).
291 yield* retryWhileRoleNotAssumable(
292 glue.updateJob({
293 JobName: name,
294 JobUpdate: buildDefinition(news),
295 }),
296 );
297 }
298
299 // 3b. SYNC TAGS
300 const observedTags = yield* fetchObservedTags(arn);
301 yield* syncTags(arn, observedTags, desiredTags);
302
303 job = yield* observe(name);
304
305 yield* session.note(name);
306 return {
307 jobName: name,
308 jobArn: arn,
309 role: job?.Role ?? news.role,
310 };
311 }),
312
313 delete: Effect.fn(function* ({ output }) {
314 // Glue writes each run's logs to streams prefixed by the run id in
315 // SHARED, service-managed groups (`/aws-glue/python-jobs/output`,
316 // `/aws-glue/jobs/logs-v2`, …). deleteJob leaves them behind and
317 // drops the run history, so enumerate our runs and reap their
318 // streams first (never the shared groups themselves).
319 const job = yield* observe(output.jobName);
320 const runIds = yield* glue.getJobRuns.items({ JobName: output.jobName }).pipe(
321 Stream.map((run) => run.Id),
322 Stream.filter((runId): runId is string => runId !== undefined),
323 Stream.runCollect,
324 Effect.map((chunk) => Array.from(chunk)),
325 Effect.catchTag("EntityNotFoundException", () => Effect.succeed([] as string[])),
326 );
327 if (runIds.length > 0) {
328 const continuousLogGroup = job?.DefaultArguments?.["--continuous-log-logGroup"];
329 yield* reapGlueLogStreams(
330 [
331 ...GLUE_JOB_LOG_GROUPS,
332 ...(continuousLogGroup === undefined ? [] : [continuousLogGroup]),
333 ],
334 runIds,
335 );
336 }
337
338 // DeleteJob is idempotent — the API returns success (not an error)
339 // when the job definition is already gone.
340 yield* glue.deleteJob({ JobName: output.jobName });
341 }),
342 });
343 }),
344 );