@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 612 lines 21 KB
Raw
1 import * as databrew from "@distilled.cloud/aws/databrew";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
8 import { toWireMinutes } from "alchemy/Util/Duration";
9 import * as Data from "effect/Data";
10 import type * as Duration from "effect/Duration";
11 import * as Effect from "effect/Effect";
12 import * as Option from "effect/Option";
13 import * as Schedule from "effect/Schedule";
14 import * as Stream from "effect/Stream";
15 import { AWSEnvironment } from "../Environment.ts";
16 import type { Providers } from "../Providers.ts";
17 import { buildS3Location, type S3Location } from "./Dataset.ts";
18 import {
19 cleanMap,
20 databrewArn,
21 fetchObservedTags,
22 retryWhileConflict,
23 retryWhileRoleNotAssumable,
24 syncTags,
25 } from "./internal.ts";
26
27 /** An S3 output target for a recipe job. */
28 export interface JobOutput {
29 /** The S3 location the transformed data is written to. */
30 location: S3Location;
31 /** The output file format. @default "CSV" */
32 format?:
33 | "CSV"
34 | "JSON"
35 | "PARQUET"
36 | "GLUEPARQUET"
37 | "AVRO"
38 | "ORC"
39 | "XML"
40 | "TABLEAUHYPER"
41 | (string & {});
42 /** Compression applied to the output files. */
43 compressionFormat?:
44 | "GZIP"
45 | "LZ4"
46 | "SNAPPY"
47 | "BZIP2"
48 | "DEFLATE"
49 | "LZO"
50 | "BROTLI"
51 | "ZSTD"
52 | "ZLIB"
53 | (string & {});
54 /** Columns to partition the output by. */
55 partitionColumns?: string[];
56 /** Overwrite previous output files on each run. @default false */
57 overwrite?: boolean;
58 /** Format-specific output options. */
59 formatOptions?: {
60 /** CSV output options. */
61 csv?: {
62 /** The single-character field delimiter. @default "," */
63 delimiter?: string;
64 };
65 };
66 /** Maximum number of output files. */
67 maxOutputFiles?: number;
68 }
69
70 /** A pointer to the recipe (and version) a recipe job applies. */
71 export interface JobRecipeReference {
72 /** The recipe name. */
73 name: string;
74 /**
75 * The recipe version to run.
76 * @default "LATEST_PUBLISHED"
77 */
78 recipeVersion?: string;
79 }
80
81 /** How much of the dataset a profile job analyzes. */
82 export interface JobSample {
83 /** `FULL_DATASET` or `CUSTOM_ROWS`. @default "CUSTOM_ROWS" */
84 mode?: "FULL_DATASET" | "CUSTOM_ROWS" | (string & {});
85 /** Row count when mode is `CUSTOM_ROWS`. @default 20000 */
86 size?: number;
87 }
88
89 /** Selects columns by exact name or regex. */
90 export interface ColumnSelector {
91 /** A regular expression matching column names. */
92 regex?: string;
93 /** An exact column name. */
94 name?: string;
95 }
96
97 /** Which statistics a profile job computes. */
98 export interface StatisticsConfiguration {
99 /** Statistics to include (default: all supported). */
100 includedStatistics?: string[];
101 /** Per-statistic parameter overrides. */
102 overrides?: {
103 /** The statistic name. */
104 statistic: string;
105 /** Statistic parameters. */
106 parameters: Record<string, string>;
107 }[];
108 }
109
110 /** Fine-grained configuration for a profile job. */
111 export interface ProfileJobConfiguration {
112 /** Dataset-level statistics configuration. */
113 datasetStatisticsConfiguration?: StatisticsConfiguration;
114 /** Restrict profiling to these columns. */
115 profileColumns?: ColumnSelector[];
116 /** Column-level statistics configurations. */
117 columnStatisticsConfigurations?: {
118 /** The columns the configuration applies to (default: all). */
119 selectors?: ColumnSelector[];
120 /** The statistics to compute for those columns. */
121 statistics: StatisticsConfiguration;
122 }[];
123 /** PII entity detection configuration. */
124 entityDetectorConfiguration?: {
125 /** Entity types to detect, e.g. `["USA_SSN", "EMAIL"]`. */
126 entityTypes: string[];
127 /** Statistics allowed on detected-entity columns. */
128 allowedStatistics?: {
129 /** The allowed statistic names. */
130 statistics: string[];
131 }[];
132 };
133 }
134
135 /** Attaches a ruleset to a profile job for data-quality validation. */
136 export interface ValidationConfiguration {
137 /** The ARN of the ruleset to validate against. */
138 rulesetArn: string;
139 /** The validation mode. @default "CHECK_ALL" */
140 validationMode?: "CHECK_ALL" | (string & {});
141 }
142
143 export interface JobProps {
144 /**
145 * Name of the job. If omitted, a unique name is generated. Changing the
146 * name replaces the job.
147 * @default a generated physical name
148 */
149 jobName?: string;
150 /**
151 * The job type: `PROFILE` analyzes a dataset and writes a data profile;
152 * `RECIPE` applies a recipe's transformations and writes the output.
153 * Changing the type replaces the job.
154 */
155 type: "PROFILE" | "RECIPE";
156 /**
157 * The dataset the job reads. Required for `PROFILE` jobs; for `RECIPE`
158 * jobs provide either `datasetName` + `recipeReference` or `projectName`.
159 * Changing it replaces the job.
160 */
161 datasetName?: string;
162 /**
163 * The IAM role ARN DataBrew assumes to run the job (read the input,
164 * write the output).
165 */
166 role: string;
167 /**
168 * S3 location the profile results are written to (PROFILE jobs only).
169 */
170 outputLocation?: S3Location;
171 /**
172 * Fine-grained profiling configuration (PROFILE jobs only).
173 */
174 configuration?: ProfileJobConfiguration;
175 /**
176 * Rulesets to validate the data against (PROFILE jobs only).
177 */
178 validationConfigurations?: ValidationConfiguration[];
179 /**
180 * How much of the dataset to profile (PROFILE jobs only).
181 * @default 20,000 rows
182 */
183 jobSample?: JobSample;
184 /**
185 * S3 outputs the transformed data is written to (RECIPE jobs only).
186 */
187 outputs?: JobOutput[];
188 /**
189 * The recipe to apply (RECIPE jobs only). Defaults to the recipe's latest
190 * *published* version. Changing it replaces the job.
191 */
192 recipeReference?: JobRecipeReference;
193 /**
194 * Derive dataset + recipe from an existing DataBrew project instead of
195 * `datasetName`/`recipeReference` (RECIPE jobs only). Changing it
196 * replaces the job.
197 */
198 projectName?: string;
199 /**
200 * The encryption mode for job output: `SSE-KMS` or `SSE-S3`.
201 */
202 encryptionMode?: "SSE-KMS" | "SSE-S3";
203 /**
204 * The KMS key ARN when `encryptionMode` is `SSE-KMS`.
205 */
206 encryptionKeyArn?: string;
207 /**
208 * Enable CloudWatch logging for the job.
209 * @default "ENABLE"
210 */
211 logSubscription?: "ENABLE" | "DISABLE";
212 /**
213 * Maximum number of compute nodes the job can consume.
214 * @default 5
215 */
216 maxCapacity?: number;
217 /**
218 * Maximum retries after a job run fails.
219 * @default 0
220 */
221 maxRetries?: number;
222 /**
223 * Job timeout, e.g. `"90 minutes"` or `Duration.hours(2)`. DataBrew
224 * measures the timeout in whole minutes.
225 * @default "48 hours"
226 */
227 timeout?: Duration.Input;
228 /**
229 * Tags to apply to the job. Merged with internal Alchemy tags.
230 */
231 tags?: Record<string, string>;
232 }
233
234 export interface Job extends Resource<
235 "AWS.DataBrew.Job",
236 JobProps,
237 {
238 /** Name of the job. */
239 jobName: string;
240 /** ARN of the job. */
241 jobArn: string;
242 /** Type of the job (`PROFILE` or `RECIPE`). */
243 type: string;
244 },
245 {},
246 Providers
247 > {}
248
249 /**
250 * An AWS Glue DataBrew job definition — either a `PROFILE` job that analyzes
251 * a dataset and writes a data-quality profile to S3, or a `RECIPE` job that
252 * applies a published recipe's transformations and writes the result to S3.
253 * The definition is free and instant; job *runs* are billed per node-hour
254 * and are started with `StartJobRun`.
255 * ### Profile Jobs
256 * **Example:** Profile a Dataset
257 * ```typescript
258 * import * as AWS from "@alchemy.run/aws";
259 *
260 * const profile = yield* AWS.DataBrew.Job("Profile", {
261 * type: "PROFILE",
262 * datasetName: dataset.datasetName,
263 * role: role.roleArn,
264 * outputLocation: { bucket: bucket.bucketName, key: "profiles/" },
265 * jobSample: { mode: "CUSTOM_ROWS", size: 1000 },
266 * });
267 * ```
268 *
269 * ### Recipe Jobs
270 * **Example:** Transform with a Published Recipe
271 * ```typescript
272 * const transform = yield* AWS.DataBrew.Job("Transform", {
273 * type: "RECIPE",
274 * datasetName: dataset.datasetName,
275 * recipeReference: { name: recipe.recipeName },
276 * role: role.roleArn,
277 * outputs: [
278 * {
279 * location: { bucket: bucket.bucketName, key: "curated/" },
280 * format: "CSV",
281 * overwrite: true,
282 * },
283 * ],
284 * });
285 * ```
286 *
287 * @resource
288 */
289 export const Job = Resource<Job>("AWS.DataBrew.Job");
290
291 /** The job's props don't satisfy the requirements of its `type`. */
292 export class DataBrewJobConfigError extends Data.TaggedError("DataBrewJobConfigError")<{
293 message: string;
294 }> {}
295
296 const validate = (props: JobProps) => {
297 if (props.type === "PROFILE") {
298 if (props.datasetName === undefined || props.outputLocation === undefined) {
299 return Effect.fail(
300 new DataBrewJobConfigError({
301 message: "PROFILE jobs require `datasetName` and `outputLocation`.",
302 }),
303 );
304 }
305 if (props.outputs !== undefined || props.recipeReference !== undefined) {
306 return Effect.fail(
307 new DataBrewJobConfigError({
308 message: "`outputs`/`recipeReference` only apply to RECIPE jobs.",
309 }),
310 );
311 }
312 } else {
313 const viaProject = props.projectName !== undefined;
314 const viaRecipe = props.datasetName !== undefined && props.recipeReference !== undefined;
315 if (!viaProject && !viaRecipe) {
316 return Effect.fail(
317 new DataBrewJobConfigError({
318 message: "RECIPE jobs require either `projectName` or `datasetName` + `recipeReference`.",
319 }),
320 );
321 }
322 if (props.outputs === undefined || props.outputs.length === 0) {
323 return Effect.fail(
324 new DataBrewJobConfigError({
325 message: "RECIPE jobs require at least one entry in `outputs`.",
326 }),
327 );
328 }
329 if (
330 props.outputLocation !== undefined ||
331 props.configuration !== undefined ||
332 props.jobSample !== undefined ||
333 props.validationConfigurations !== undefined
334 ) {
335 return Effect.fail(
336 new DataBrewJobConfigError({
337 message:
338 "`outputLocation`/`configuration`/`jobSample`/`validationConfigurations` only apply to PROFILE jobs.",
339 }),
340 );
341 }
342 }
343 return Effect.void;
344 };
345
346 const buildOutputs = (outputs: JobOutput[] | undefined) =>
347 outputs?.map((output) => ({
348 Location: buildS3Location(output.location),
349 Format: output.format,
350 CompressionFormat: output.compressionFormat,
351 PartitionColumns: output.partitionColumns,
352 Overwrite: output.overwrite,
353 FormatOptions: output.formatOptions
354 ? {
355 Csv: output.formatOptions.csv
356 ? { Delimiter: output.formatOptions.csv.delimiter }
357 : undefined,
358 }
359 : undefined,
360 MaxOutputFiles: output.maxOutputFiles,
361 }));
362
363 const buildStatisticsConfiguration = (config: StatisticsConfiguration) => ({
364 IncludedStatistics: config.includedStatistics,
365 Overrides: config.overrides?.map((o) => ({
366 Statistic: o.statistic,
367 Parameters: o.parameters,
368 })),
369 });
370
371 const buildColumnSelectors = (selectors: ColumnSelector[] | undefined) =>
372 selectors?.map((s) => ({ Regex: s.regex, Name: s.name }));
373
374 const buildConfiguration = (config: ProfileJobConfiguration | undefined) =>
375 config
376 ? {
377 DatasetStatisticsConfiguration: config.datasetStatisticsConfiguration
378 ? buildStatisticsConfiguration(config.datasetStatisticsConfiguration)
379 : undefined,
380 ProfileColumns: buildColumnSelectors(config.profileColumns),
381 ColumnStatisticsConfigurations: config.columnStatisticsConfigurations?.map((c) => ({
382 Selectors: buildColumnSelectors(c.selectors),
383 Statistics: buildStatisticsConfiguration(c.statistics),
384 })),
385 EntityDetectorConfiguration: config.entityDetectorConfiguration
386 ? {
387 EntityTypes: config.entityDetectorConfiguration.entityTypes,
388 AllowedStatistics: config.entityDetectorConfiguration.allowedStatistics?.map((a) => ({
389 Statistics: a.statistics,
390 })),
391 }
392 : undefined,
393 }
394 : undefined;
395
396 const buildValidationConfigurations = (configs: ValidationConfiguration[] | undefined) =>
397 configs?.map((c) => ({
398 RulesetArn: c.rulesetArn,
399 ValidationMode: c.validationMode,
400 }));
401
402 const buildJobSample = (sample: JobSample | undefined) =>
403 sample ? { Mode: sample.mode, Size: sample.size } : undefined;
404
405 const buildCommon = (props: JobProps) => ({
406 EncryptionKeyArn: props.encryptionKeyArn,
407 EncryptionMode: props.encryptionMode,
408 LogSubscription: props.logSubscription,
409 MaxCapacity: props.maxCapacity,
410 MaxRetries: props.maxRetries,
411 RoleArn: props.role,
412 Timeout: toWireMinutes(props.timeout),
413 });
414
415 export const JobProvider = () =>
416 Provider.effect(
417 Job,
418 Effect.gen(function* () {
419 const createName = Effect.fn(function* (id: string, props: { jobName?: string | undefined }) {
420 return props.jobName ?? (yield* createPhysicalName({ id, maxLength: 240 }));
421 });
422
423 const observe = Effect.fn(function* (name: string) {
424 return yield* databrew
425 .describeJob({ Name: name })
426 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
427 });
428
429 return Job.Provider.of({
430 stables: ["jobName", "jobArn"],
431
432 list: () =>
433 Effect.gen(function* () {
434 const { accountId, region } = yield* AWSEnvironment.current;
435 const pages = yield* databrew.listJobs.pages({}).pipe(Stream.runCollect);
436 return Array.from(pages)
437 .flatMap((page) => page.Jobs ?? [])
438 .map((j) => ({
439 jobName: j.Name,
440 jobArn: j.ResourceArn ?? databrewArn(region, accountId, "job", j.Name),
441 type: j.Type ?? "RECIPE",
442 }));
443 }),
444
445 read: Effect.fn(function* ({ id, olds, output }) {
446 const { accountId, region } = yield* AWSEnvironment.current;
447 const name = output?.jobName ?? (yield* createName(id, olds ?? {}));
448 const job = yield* observe(name);
449 if (job === undefined) return undefined;
450 const arn = job.ResourceArn ?? databrewArn(region, accountId, "job", name);
451 const attrs = {
452 jobName: name,
453 jobArn: arn,
454 type: job.Type ?? "RECIPE",
455 };
456 const tags = cleanMap(job.Tags);
457 return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
458 }),
459
460 diff: Effect.fn(function* ({ id, news, olds }) {
461 if (!isResolved(news)) return undefined;
462 const oldName = yield* createName(id, olds);
463 const newName = yield* createName(id, news);
464 if (oldName !== newName) return { action: "replace" } as const;
465 // The Update APIs accept neither the job type nor the input
466 // associations — those are create-only.
467 if (olds.type !== news.type) return { action: "replace" } as const;
468 if (olds.datasetName !== news.datasetName) {
469 return { action: "replace" } as const;
470 }
471 if (olds.projectName !== news.projectName) {
472 return { action: "replace" } as const;
473 }
474 if (
475 olds.recipeReference?.name !== news.recipeReference?.name ||
476 (olds.recipeReference?.recipeVersion ?? "LATEST_PUBLISHED") !==
477 (news.recipeReference?.recipeVersion ?? "LATEST_PUBLISHED")
478 ) {
479 return { action: "replace" } as const;
480 }
481 }),
482
483 reconcile: Effect.fn(function* ({ id, news, output, session }) {
484 const { accountId, region } = yield* AWSEnvironment.current;
485 yield* validate(news);
486 const name = output?.jobName ?? (yield* createName(id, news));
487 const internalTags = yield* createInternalTags(id);
488 const desiredTags = { ...news.tags, ...internalTags };
489
490 // 1. OBSERVE
491 const job = yield* observe(name);
492
493 // 2. ENSURE / 3. SYNC — Update{Profile,Recipe}Job is a full PUT
494 if (job === undefined) {
495 if (news.type === "PROFILE") {
496 yield* retryWhileRoleNotAssumable(
497 databrew.createProfileJob({
498 Name: name,
499 DatasetName: news.datasetName!,
500 OutputLocation: buildS3Location(news.outputLocation!),
501 Configuration: buildConfiguration(news.configuration),
502 ValidationConfigurations: buildValidationConfigurations(
503 news.validationConfigurations,
504 ),
505 JobSample: buildJobSample(news.jobSample),
506 Tags: desiredTags,
507 ...buildCommon(news),
508 }),
509 ).pipe(Effect.catchTag("ConflictException", () => Effect.void));
510 } else {
511 yield* retryWhileRoleNotAssumable(
512 databrew.createRecipeJob({
513 Name: name,
514 DatasetName: news.datasetName,
515 ProjectName: news.projectName,
516 RecipeReference: news.recipeReference
517 ? {
518 Name: news.recipeReference.name,
519 // the API rejects a null version ("version null is
520 // invalid") — apply the documented default explicitly
521 RecipeVersion: news.recipeReference.recipeVersion ?? "LATEST_PUBLISHED",
522 }
523 : undefined,
524 Outputs: buildOutputs(news.outputs),
525 Tags: desiredTags,
526 ...buildCommon(news),
527 }),
528 ).pipe(Effect.catchTag("ConflictException", () => Effect.void));
529 }
530 } else if (news.type === "PROFILE") {
531 yield* retryWhileRoleNotAssumable(
532 databrew.updateProfileJob({
533 Name: name,
534 OutputLocation: buildS3Location(news.outputLocation!),
535 Configuration: buildConfiguration(news.configuration),
536 ValidationConfigurations: buildValidationConfigurations(
537 news.validationConfigurations,
538 ),
539 JobSample: buildJobSample(news.jobSample),
540 ...buildCommon(news),
541 }),
542 );
543 } else {
544 yield* retryWhileRoleNotAssumable(
545 databrew.updateRecipeJob({
546 Name: name,
547 Outputs: buildOutputs(news.outputs),
548 ...buildCommon(news),
549 }),
550 );
551 }
552
553 const arn = job?.ResourceArn ?? databrewArn(region, accountId, "job", name);
554
555 // 3b. SYNC TAGS against observed cloud tags
556 const observedTags = yield* fetchObservedTags(arn);
557 yield* syncTags(arn, observedTags, desiredTags);
558
559 yield* session.note(name);
560 return { jobName: name, jobArn: arn, type: news.type };
561 }),
562
563 delete: Effect.fn(function* ({ output }) {
564 // A run in STARTING/RUNNING/STOPPING keeps the job's dataset and
565 // recipe associated — downstream Dataset/Recipe deletes then fail
566 // with ConflictException ("is used in job …") long after DeleteJob
567 // itself succeeds. Stop in-flight runs and wait (bounded) for every
568 // run to reach a terminal state before deleting the job.
569 const runs = yield* databrew.listJobRuns.items({ Name: output.jobName }).pipe(
570 Stream.runCollect,
571 Effect.map((chunk) => Array.from(chunk)),
572 Effect.catchTag("ResourceNotFoundException", () => Effect.succeed([])),
573 );
574 const isActive = (state: string | undefined): boolean =>
575 state === "STARTING" || state === "RUNNING" || state === "STOPPING";
576 yield* Effect.forEach(
577 runs.filter(
578 (run) =>
579 run.RunId !== undefined && (run.State === "STARTING" || run.State === "RUNNING"),
580 ),
581 (run) =>
582 databrew.stopJobRun({ Name: output.jobName, RunId: run.RunId! }).pipe(
583 // Already stopping/stopped (ValidationException) or gone.
584 Effect.catchTag(
585 ["ResourceNotFoundException", "ValidationException"],
586 () => Effect.undefined,
587 ),
588 ),
589 );
590 if (runs.some((run) => isActive(run.State))) {
591 yield* databrew.listJobRuns.items({ Name: output.jobName }).pipe(
592 // Settled = no active run observed; stop paginating at the
593 // first active run instead of draining every page.
594 Stream.filter((run) => isActive(run.State)),
595 Stream.runHead,
596 Effect.map(Option.isNone),
597 Effect.catchTag("ResourceNotFoundException", () => Effect.succeed(true)),
598 Effect.repeat({
599 schedule: Schedule.spaced("5 seconds"),
600 until: (settled): boolean => settled,
601 times: 18,
602 }),
603 );
604 }
605 // ConflictException while a job run is still in flight.
606 yield* retryWhileConflict(databrew.deleteJob({ Name: output.jobName })).pipe(
607 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
608 );
609 }),
610 });
611 }),
612 );