@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 547 lines 20.7 KB
Raw
1 import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
2 import * as mwaa from "@distilled.cloud/aws/mwaa-serverless";
3 import { Unowned } from "alchemy/AdoptPolicy";
4 import { isResolved } from "alchemy/Diff";
5 import { createPhysicalName } from "alchemy/PhysicalName";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import type { Providers } from "../Providers.ts";
13
14 export interface WorkflowDefinitionS3Location {
15 /**
16 * Name of the S3 bucket that holds the workflow definition YAML file.
17 */
18 bucket: string;
19 /**
20 * Key of the workflow definition YAML object within the bucket.
21 */
22 objectKey: string;
23 /**
24 * Specific S3 object version of the definition to use. Omit to use the
25 * latest version.
26 */
27 versionId?: string;
28 }
29
30 export interface WorkflowEncryptionConfiguration {
31 /**
32 * How workflow data is encrypted — `"AWS_MANAGED_KEY"` (the default) or
33 * `"CUSTOMER_MANAGED_KEY"`.
34 *
35 * Encryption cannot be changed in place — changing it replaces the
36 * workflow.
37 * @default "AWS_MANAGED_KEY"
38 */
39 type: mwaa.EncryptionType;
40 /**
41 * ID or ARN of the customer managed KMS key to encrypt workflow data
42 * with. Required when `type` is `"CUSTOMER_MANAGED_KEY"`.
43 */
44 kmsKeyId?: string;
45 }
46
47 export interface WorkflowLoggingConfiguration {
48 /**
49 * Name of the CloudWatch log group that receives task logs for the
50 * workflow's runs.
51 */
52 logGroupName: string;
53 }
54
55 export interface WorkflowNetworkConfiguration {
56 /**
57 * Security group IDs applied to tasks that access resources inside your
58 * VPC.
59 */
60 securityGroupIds?: string[];
61 /**
62 * Subnet IDs the workflow's tasks run in when accessing resources inside
63 * your VPC.
64 */
65 subnetIds?: string[];
66 }
67
68 export interface WorkflowProps {
69 /**
70 * Name of the workflow. If omitted, a deterministic physical name is
71 * generated from the app, stage, and logical ID.
72 *
73 * Changing the name replaces the workflow.
74 * @default ${app}-${stage}-${id}
75 */
76 name?: string;
77 /**
78 * S3 location of the workflow definition — a YAML file that defines the
79 * DAG structure using supported AWS operators.
80 *
81 * Updating the definition creates a new workflow version and disables
82 * scheduling on all previous versions.
83 */
84 definitionS3Location: WorkflowDefinitionS3Location;
85 /**
86 * ARN of the IAM execution role that Amazon MWAA Serverless assumes to
87 * run the workflow's tasks. The role's trust policy must allow the
88 * `airflow-serverless.amazonaws.com` service principal to assume it.
89 */
90 roleArn: string;
91 /**
92 * Human-readable description of the workflow.
93 */
94 description?: string;
95 /**
96 * Encryption configuration for workflow data. Omit to use an AWS managed
97 * key. Changing encryption replaces the workflow (it cannot be updated
98 * in place).
99 * @default AWS managed key
100 */
101 encryptionConfiguration?: WorkflowEncryptionConfiguration;
102 /**
103 * CloudWatch logging configuration for the workflow's task logs.
104 */
105 loggingConfiguration?: WorkflowLoggingConfiguration;
106 /**
107 * Version of the workflow definition engine.
108 * @default 1
109 */
110 engineVersion?: mwaa.EngineVersion;
111 /**
112 * Network configuration for tasks that access resources inside your VPC.
113 */
114 networkConfiguration?: WorkflowNetworkConfiguration;
115 /**
116 * How workflow runs are triggered (for example on a schedule defined in
117 * the workflow definition, or only on demand).
118 */
119 triggerMode?: string;
120 /**
121 * Tags to apply to the workflow. Merged with internal Alchemy tags.
122 */
123 tags?: Record<string, string>;
124 }
125
126 export interface Workflow extends Resource<
127 "AWS.MWAAServerless.Workflow",
128 WorkflowProps,
129 {
130 /** Name of the workflow. */
131 name: string;
132 /** ARN of the workflow. */
133 workflowArn: string;
134 /** Latest workflow version (a new version is published on each update). */
135 workflowVersion: string | undefined;
136 /** Current lifecycle status of the workflow. */
137 workflowStatus: mwaa.WorkflowStatus | undefined;
138 /** ARN of the IAM role the workflow's tasks assume. */
139 roleArn: string | undefined;
140 /** How runs are triggered (e.g. scheduled or on-demand). */
141 triggerMode: string | undefined;
142 },
143 never,
144 Providers
145 > {}
146
147 /**
148 * An Amazon Managed Workflows for Apache Airflow **Serverless** workflow —
149 * a serverless Airflow DAG defined by a YAML file in S3 and executed by
150 * AWS-managed, multi-tenant Airflow infrastructure without provisioning an
151 * environment.
152 *
153 * Each update to the definition or configuration creates a new workflow
154 * version; MWAA Serverless keeps only the latest version actively
155 * scheduled.
156 * ### Creating a Workflow
157 * **Example:** Basic Workflow
158 * ```typescript
159 * import * as MWAAServerless from "@alchemy.run/aws/MWAAServerless";
160 * import * as IAM from "@alchemy.run/aws/IAM";
161 *
162 * const role = yield* IAM.Role("WorkflowRole", {
163 * assumeRolePolicyDocument: {
164 * Version: "2012-10-17",
165 * Statement: [{
166 * Effect: "Allow",
167 * Principal: { Service: "airflow-serverless.amazonaws.com" },
168 * Action: ["sts:AssumeRole"],
169 * }],
170 * },
171 * });
172 *
173 * const workflow = yield* MWAAServerless.Workflow("Etl", {
174 * definitionS3Location: {
175 * bucket: "my-dag-bucket",
176 * objectKey: "workflows/etl.yaml",
177 * },
178 * roleArn: role.roleArn,
179 * });
180 * ```
181 *
182 * **Example:** Workflow with Logging and Tags
183 * ```typescript
184 * const workflow = yield* MWAAServerless.Workflow("Etl", {
185 * definitionS3Location: {
186 * bucket: "my-dag-bucket",
187 * objectKey: "workflows/etl.yaml",
188 * },
189 * roleArn: role.roleArn,
190 * description: "nightly ETL",
191 * loggingConfiguration: { logGroupName: "/mwaa-serverless/etl" },
192 * tags: { team: "data" },
193 * });
194 * ```
195 *
196 * @resource
197 */
198 export const Workflow = Resource<Workflow>("AWS.MWAAServerless.Workflow");
199
200 /**
201 * Bounded retry for `createWorkflow` while a freshly created IAM execution
202 * role propagates — the service validates that it can assume the role at
203 * create time, which surfaces as a ValidationException/AccessDeniedException
204 * mentioning the role for the first seconds of the role's life.
205 *
206 * Explicitly annotated so the conditional `Retry.Return` type never leaks
207 * into declaration emit (it would widen `AWS.providers()` for consumers).
208 */
209 const retryWhileRolePropagates = <A, R>(
210 self: Effect.Effect<A, mwaa.CreateWorkflowError, R>,
211 ): Effect.Effect<A, mwaa.CreateWorkflowError, R> =>
212 Effect.retry(self, {
213 while: (e) =>
214 (e._tag === "ValidationException" || e._tag === "AccessDeniedException") &&
215 /role/i.test(e.message ?? ""),
216 schedule: Schedule.max([Schedule.fixed("3 seconds"), Schedule.recurs(8)]),
217 });
218
219 export const WorkflowProvider = () =>
220 Provider.effect(
221 Workflow,
222 Effect.gen(function* () {
223 const createName = Effect.fn(function* (id: string, props: Pick<WorkflowProps, "name">) {
224 return props.name ?? (yield* createPhysicalName({ id, maxLength: 64 }));
225 });
226
227 const observeByArn = (arn: string) =>
228 mwaa
229 .getWorkflow({ WorkflowArn: arn })
230 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
231
232 const findArnByName = (name: string) =>
233 mwaa.listWorkflows.items({}).pipe(
234 Stream.runCollect,
235 Effect.map((items) => Array.from(items).find((w) => w.Name === name)?.WorkflowArn),
236 );
237
238 const observe = Effect.fn(function* (name: string, arnHint: string | undefined) {
239 if (arnHint !== undefined) {
240 const found = yield* observeByArn(arnHint);
241 if (found !== undefined) return found;
242 }
243 const arn = yield* findArnByName(name);
244 return arn === undefined ? undefined : yield* observeByArn(arn);
245 });
246
247 const observedTags = (arn: string) =>
248 mwaa.listTagsForResource({ ResourceArn: arn }).pipe(
249 Effect.map((r) => {
250 const tags: Record<string, string> = {};
251 for (const [key, value] of Object.entries(r.Tags ?? {})) {
252 if (value !== undefined) tags[key] = value;
253 }
254 return tags;
255 }),
256 Effect.catchTag("ResourceNotFoundException", () =>
257 Effect.succeed({} as Record<string, string>),
258 ),
259 );
260
261 const toDefinitionS3Location = (
262 location: WorkflowDefinitionS3Location,
263 ): mwaa.DefinitionS3Location => ({
264 Bucket: location.bucket,
265 ObjectKey: location.objectKey,
266 VersionId: location.versionId,
267 });
268
269 const sameDefinition = (
270 live: mwaa.DefinitionS3Location | undefined,
271 desired: WorkflowDefinitionS3Location,
272 ) =>
273 live !== undefined &&
274 live.Bucket === desired.bucket &&
275 live.ObjectKey === desired.objectKey &&
276 (desired.versionId === undefined || live.VersionId === desired.versionId);
277
278 const sameNetwork = (
279 live: mwaa.NetworkConfiguration | undefined,
280 desired: WorkflowNetworkConfiguration,
281 ) =>
282 JSON.stringify({
283 securityGroupIds: live?.SecurityGroupIds ?? [],
284 subnetIds: live?.SubnetIds ?? [],
285 }) ===
286 JSON.stringify({
287 securityGroupIds: desired.securityGroupIds ?? [],
288 subnetIds: desired.subnetIds ?? [],
289 });
290
291 const toAttributes = (live: mwaa.GetWorkflowResponse, name: string) => ({
292 name: live.Name ?? name,
293 workflowArn: live.WorkflowArn,
294 workflowVersion: live.WorkflowVersion,
295 workflowStatus: live.WorkflowStatus,
296 roleArn: live.RoleArn,
297 triggerMode: live.TriggerMode,
298 });
299
300 return Workflow.Provider.of({
301 stables: ["name", "workflowArn"],
302 list: () =>
303 Effect.gen(function* () {
304 const items = yield* mwaa.listWorkflows.items({}).pipe(Stream.runCollect);
305 const workflows: {
306 name: string;
307 workflowArn: string;
308 workflowVersion: string | undefined;
309 workflowStatus: mwaa.WorkflowStatus | undefined;
310 roleArn: string | undefined;
311 triggerMode: string | undefined;
312 }[] = [];
313 for (const item of Array.from(items)) {
314 if (item.WorkflowArn === undefined || item.Name === undefined) {
315 continue;
316 }
317 // tolerate delete races between list and get
318 const live = yield* observeByArn(item.WorkflowArn);
319 if (live !== undefined && live.Name !== undefined) {
320 workflows.push(toAttributes(live, live.Name));
321 }
322 }
323 return workflows;
324 }),
325 read: Effect.fn(function* ({ id, olds, output }) {
326 const name = output?.name ?? (yield* createName(id, olds ?? {}));
327 const live = yield* observe(name, output?.workflowArn);
328 if (live === undefined || live.Name === undefined) {
329 return undefined;
330 }
331 const attrs = toAttributes(live, name);
332 const tags = yield* observedTags(live.WorkflowArn);
333 return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
334 }),
335 diff: Effect.fn(function* ({ id, news, olds }) {
336 if (!isResolved(news)) return undefined;
337 const oldName = yield* createName(id, olds ?? {});
338 const newName = yield* createName(id, news ?? {});
339 if (oldName !== newName) {
340 return { action: "replace" } as const;
341 }
342 // Encryption cannot be updated in place — UpdateWorkflow does not
343 // accept EncryptionConfiguration.
344 const oldEncryption = olds?.encryptionConfiguration;
345 const newEncryption = news.encryptionConfiguration;
346 if (
347 (oldEncryption?.type ?? "AWS_MANAGED_KEY") !==
348 (newEncryption?.type ?? "AWS_MANAGED_KEY") ||
349 oldEncryption?.kmsKeyId !== newEncryption?.kmsKeyId
350 ) {
351 return { action: "replace" } as const;
352 }
353 // fall through: engine default update logic for mutable fields
354 }),
355 reconcile: Effect.fn(function* ({ id, news, output, session }) {
356 const name = output?.name ?? (yield* createName(id, news));
357 const internalTags = yield* createInternalTags(id);
358 const desiredTags = { ...news.tags, ...internalTags };
359
360 // 1. OBSERVE — cloud state is authoritative; output.workflowArn is
361 // only a cache of the identifier.
362 let live = yield* observe(name, output?.workflowArn);
363
364 // 2. ENSURE — create when missing. A ConflictException means a
365 // concurrent create (or a previous run's create whose state
366 // persistence failed) raced us: converge on the workflow with
367 // OUR name if one now exists, otherwise propagate.
368 if (live === undefined) {
369 live = yield* mwaa
370 .createWorkflow({
371 Name: name,
372 DefinitionS3Location: toDefinitionS3Location(news.definitionS3Location),
373 RoleArn: news.roleArn,
374 Description: news.description,
375 EncryptionConfiguration:
376 news.encryptionConfiguration === undefined
377 ? undefined
378 : {
379 Type: news.encryptionConfiguration.type,
380 KmsKeyId: news.encryptionConfiguration.kmsKeyId,
381 },
382 LoggingConfiguration:
383 news.loggingConfiguration === undefined
384 ? undefined
385 : {
386 LogGroupName: news.loggingConfiguration.logGroupName,
387 },
388 EngineVersion: news.engineVersion,
389 NetworkConfiguration:
390 news.networkConfiguration === undefined
391 ? undefined
392 : {
393 SecurityGroupIds: news.networkConfiguration.securityGroupIds,
394 SubnetIds: news.networkConfiguration.subnetIds,
395 },
396 TriggerMode: news.triggerMode,
397 Tags: desiredTags,
398 })
399 .pipe(
400 retryWhileRolePropagates,
401 Effect.flatMap((created) => observeByArn(created.WorkflowArn)),
402 Effect.catchTag("ConflictException", (error) =>
403 observe(name, undefined).pipe(
404 Effect.flatMap((existing) =>
405 existing === undefined ? Effect.fail(error) : Effect.succeed(existing),
406 ),
407 ),
408 ),
409 );
410 }
411
412 const arn = live?.WorkflowArn ?? output?.workflowArn;
413
414 // 3. SYNC — diff each OBSERVED mutable aspect against the desired
415 // state; UpdateWorkflow requires the definition and role, so
416 // issue one full update when any managed aspect drifted (each
417 // update creates a new workflow version — skip on no-op).
418 if (live !== undefined && arn !== undefined) {
419 const drifted =
420 !sameDefinition(live.DefinitionS3Location, news.definitionS3Location) ||
421 live.RoleArn !== news.roleArn ||
422 (news.description !== undefined && live.Description !== news.description) ||
423 (news.loggingConfiguration !== undefined &&
424 live.LoggingConfiguration?.LogGroupName !==
425 news.loggingConfiguration.logGroupName) ||
426 (news.engineVersion !== undefined && live.EngineVersion !== news.engineVersion) ||
427 (news.networkConfiguration !== undefined &&
428 !sameNetwork(live.NetworkConfiguration, news.networkConfiguration)) ||
429 (news.triggerMode !== undefined && live.TriggerMode !== news.triggerMode);
430 if (drifted) {
431 yield* mwaa.updateWorkflow({
432 WorkflowArn: arn,
433 DefinitionS3Location: toDefinitionS3Location(news.definitionS3Location),
434 RoleArn: news.roleArn,
435 Description: news.description,
436 LoggingConfiguration:
437 news.loggingConfiguration === undefined
438 ? undefined
439 : {
440 LogGroupName: news.loggingConfiguration.logGroupName,
441 },
442 EngineVersion: news.engineVersion,
443 NetworkConfiguration:
444 news.networkConfiguration === undefined
445 ? undefined
446 : {
447 SecurityGroupIds: news.networkConfiguration.securityGroupIds,
448 SubnetIds: news.networkConfiguration.subnetIds,
449 },
450 TriggerMode: news.triggerMode,
451 });
452 live = (yield* observeByArn(arn)) ?? live;
453 }
454 }
455
456 // 3b. SYNC TAGS — diff against OBSERVED cloud tags so adoption
457 // converges (create-time tags only apply on first create).
458 if (arn !== undefined) {
459 const currentTags = yield* observedTags(arn);
460 const { upsert, removed } = diffTags(currentTags, desiredTags);
461 if (upsert.length > 0) {
462 yield* mwaa.tagResource({
463 ResourceArn: arn,
464 Tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
465 });
466 }
467 if (removed.length > 0) {
468 yield* mwaa.untagResource({
469 ResourceArn: arn,
470 TagKeys: removed,
471 });
472 }
473 }
474
475 yield* session.note(name);
476 return {
477 name: live?.Name ?? name,
478 workflowArn: arn!,
479 workflowVersion: live?.WorkflowVersion,
480 workflowStatus: live?.WorkflowStatus,
481 roleArn: live?.RoleArn ?? news.roleArn,
482 triggerMode: live?.TriggerMode ?? news.triggerMode,
483 };
484 }),
485 delete: Effect.fn(function* ({ output }) {
486 // Omitting WorkflowVersion deletes the workflow and all versions.
487 yield* mwaa
488 .deleteWorkflow({ WorkflowArn: output.workflowArn })
489 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
490
491 // MWAA Serverless auto-creates a per-workflow log group named
492 // `/aws/mwaa-serverless/{name}-{id}/` (the ARN's resource id,
493 // which includes the service-assigned 10-char suffix, plus a
494 // trailing slash) at CREATE time, and deleteWorkflow does NOT
495 // remove it — without this reap every deleted workflow (including
496 // the old workflow of a replacement) leaks an orphaned log group.
497 // A group with no streams or provably-quiescent ingestion (last
498 // ingestion > 2 minutes ago) has no pending flush and deletes in
499 // a single call, so routine deletes of never-run workflows stay
500 // fast. A group with recent ingestion — or one that does not
501 // exist yet — is re-reaped on a short bounded schedule
502 // (t=0s / 20s / 40s) to catch a late log flush from the
503 // workflow's final runs; each attempt is idempotent.
504 const workflowId = output.workflowArn.split(":workflow/")[1];
505 if (workflowId !== undefined) {
506 const logGroupName = `/aws/mwaa-serverless/${workflowId}/`;
507 const reapLogGroup = logs
508 .deleteLogGroup({ logGroupName })
509 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
510 const observed = yield* logs
511 .describeLogStreams({
512 logGroupName,
513 orderBy: "LastEventTime",
514 descending: true,
515 limit: 1,
516 })
517 .pipe(
518 Effect.map((r) => ({
519 exists: true,
520 lastIngestion: r.logStreams?.[0]?.lastIngestionTime,
521 })),
522 Effect.catchTag("ResourceNotFoundException", () =>
523 Effect.succeed({
524 exists: false,
525 lastIngestion: undefined as number | undefined,
526 }),
527 ),
528 );
529 const now = yield* Effect.sync(() => Date.now());
530 const quiescent =
531 observed.exists &&
532 (observed.lastIngestion === undefined || now - observed.lastIngestion > 120_000);
533 if (quiescent) {
534 yield* reapLogGroup;
535 } else {
536 yield* reapLogGroup.pipe(
537 Effect.repeat({
538 schedule: Schedule.spaced("20 seconds"),
539 times: 2,
540 }),
541 );
542 }
543 }
544 }),
545 });
546 }),
547 );