@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 414 lines 14.1 KB
Raw
1 import * as databrew from "@distilled.cloud/aws/databrew";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
8 import * as Effect from "effect/Effect";
9 import * as Stream from "effect/Stream";
10 import { AWSEnvironment } from "../Environment.ts";
11 import type { Providers } from "../Providers.ts";
12 import { databrewArn, fetchObservedTags, retryWhileConflict, syncTags } from "./internal.ts";
13
14 /** An Amazon S3 location (bucket + optional key prefix). */
15 export interface S3Location {
16 /** The S3 bucket name. */
17 bucket: string;
18 /** The object key (or key prefix) within the bucket. */
19 key?: string;
20 /** The account ID of the bucket owner (for cross-account buckets). */
21 bucketOwner?: string;
22 }
23
24 /** Where DataBrew reads the dataset's data from. Exactly one definition. */
25 export interface DatasetInput {
26 /** Read directly from Amazon S3. */
27 s3InputDefinition?: S3Location;
28 /** Read from an AWS Glue Data Catalog table. */
29 dataCatalogInputDefinition?: {
30 /** The Data Catalog ID (defaults to the caller's account). */
31 catalogId?: string;
32 /** The Glue database name. */
33 databaseName: string;
34 /** The Glue table name. */
35 tableName: string;
36 /** S3 temp directory for intermediate results. */
37 tempDirectory?: S3Location;
38 };
39 /** Read from a JDBC database through a Glue connection. */
40 databaseInputDefinition?: {
41 /** The Glue connection name. */
42 glueConnectionName: string;
43 /** The database table to read. Mutually exclusive with `queryString`. */
44 databaseTableName?: string;
45 /** S3 temp directory for intermediate results. */
46 tempDirectory?: S3Location;
47 /** A custom SQL query to select the data. */
48 queryString?: string;
49 };
50 }
51
52 /** Format-specific parsing options. */
53 export interface DatasetFormatOptions {
54 /** JSON parsing options. */
55 json?: {
56 /** Whether the file contains multi-line JSON records. */
57 multiLine?: boolean;
58 };
59 /** Excel parsing options. */
60 excel?: {
61 /** Sheet names to include (max 1). */
62 sheetNames?: string[];
63 /** Zero-based sheet indexes to include (max 1). */
64 sheetIndexes?: number[];
65 /** Whether the first row is a header. @default true */
66 headerRow?: boolean;
67 };
68 /** CSV parsing options. */
69 csv?: {
70 /** The single-character field delimiter. @default "," */
71 delimiter?: string;
72 /** Whether the first row is a header. @default true */
73 headerRow?: boolean;
74 };
75 }
76
77 /** A filter expression with substitution variables, e.g. `relative_before :dateParam`. */
78 export interface FilterExpression {
79 /** The expression, e.g. `ends_with :suffix`. */
80 expression: string;
81 /** Variable substitutions keyed by `:name`. */
82 valuesMap: Record<string, string>;
83 }
84
85 /** Options that shape how S3 path parameters select matching files. */
86 export interface DatasetPathOptions {
87 /** Only include files last modified within this condition. */
88 lastModifiedDateCondition?: FilterExpression;
89 /** Cap the number of matched files. */
90 filesLimit?: {
91 /** Maximum number of files to include. */
92 maxFiles: number;
93 /** Ordering attribute. @default "LAST_MODIFIED_DATE" */
94 orderedBy?: "LAST_MODIFIED_DATE" | (string & {});
95 /** Sort direction. @default "DESCENDING" */
96 order?: "DESCENDING" | "ASCENDING" | (string & {});
97 };
98 /** Path parameter definitions keyed by parameter name. */
99 parameters?: Record<
100 string,
101 {
102 /** The parameter name (must match the key). */
103 name: string;
104 /** The parameter type. */
105 type: "Datetime" | "Number" | "String" | (string & {});
106 /** Datetime parsing options (required for `Datetime`). */
107 datetimeOptions?: {
108 /** The datetime format, e.g. `yyyy-MM-dd`. */
109 format: string;
110 /** Timezone offset, e.g. `Z` or `+02:00`. */
111 timezoneOffset?: string;
112 /** Locale code for month/day names. */
113 localeCode?: string;
114 };
115 /** Whether to add a column holding the parameter value. */
116 createColumn?: boolean;
117 /** Filter which parameter values match. */
118 filter?: FilterExpression;
119 }
120 >;
121 }
122
123 export interface DatasetProps {
124 /**
125 * Name of the dataset. If omitted, a unique name is generated. Changing
126 * the name replaces the dataset.
127 * @default a generated physical name
128 */
129 datasetName?: string;
130 /**
131 * The file format of the source data: `CSV`, `JSON`, `PARQUET`, `EXCEL`,
132 * or `ORC`.
133 */
134 format?: "CSV" | "JSON" | "PARQUET" | "EXCEL" | "ORC" | (string & {});
135 /**
136 * Format-specific parsing options (CSV delimiter, JSON multi-line, Excel
137 * sheets).
138 */
139 formatOptions?: DatasetFormatOptions;
140 /**
141 * Where the data lives — S3, a Glue Data Catalog table, or a JDBC
142 * database.
143 */
144 input: DatasetInput;
145 /**
146 * Path options for dynamic S3 datasets (parameterized paths, file limits).
147 */
148 pathOptions?: DatasetPathOptions;
149 /**
150 * Tags to apply to the dataset. Merged with internal Alchemy tags.
151 */
152 tags?: Record<string, string>;
153 }
154
155 export interface Dataset extends Resource<
156 "AWS.DataBrew.Dataset",
157 DatasetProps,
158 {
159 /** Name of the dataset. */
160 datasetName: string;
161 /** ARN of the dataset. */
162 datasetArn: string;
163 },
164 {},
165 Providers
166 > {}
167
168 /**
169 * An AWS Glue DataBrew dataset — a pointer to source data (S3 file/prefix,
170 * Glue Data Catalog table, or JDBC query) plus parsing options. The dataset
171 * definition itself stores no data and is free; it is consumed by DataBrew
172 * projects and jobs.
173 * ### Creating Datasets
174 * **Example:** CSV Dataset from S3
175 * ```typescript
176 * import * as AWS from "@alchemy.run/aws";
177 *
178 * const dataset = yield* AWS.DataBrew.Dataset("Sales", {
179 * format: "CSV",
180 * formatOptions: { csv: { delimiter: ",", headerRow: true } },
181 * input: {
182 * s3InputDefinition: {
183 * bucket: bucket.bucketName,
184 * key: "raw/sales.csv",
185 * },
186 * },
187 * });
188 * ```
189 *
190 * **Example:** JSON Dataset
191 * ```typescript
192 * const dataset = yield* AWS.DataBrew.Dataset("Events", {
193 * format: "JSON",
194 * formatOptions: { json: { multiLine: false } },
195 * input: {
196 * s3InputDefinition: { bucket: bucket.bucketName, key: "events/" },
197 * },
198 * });
199 * ```
200 *
201 * ### Glue Data Catalog
202 * **Example:** Dataset from a Catalog Table
203 * ```typescript
204 * const dataset = yield* AWS.DataBrew.Dataset("Curated", {
205 * input: {
206 * dataCatalogInputDefinition: {
207 * databaseName: glueDatabase.databaseName,
208 * tableName: "curated_sales",
209 * },
210 * },
211 * });
212 * ```
213 *
214 * @resource
215 */
216 export const Dataset = Resource<Dataset>("AWS.DataBrew.Dataset");
217
218 export const buildS3Location = (location: S3Location) => ({
219 Bucket: location.bucket,
220 Key: location.key,
221 BucketOwner: location.bucketOwner,
222 });
223
224 const buildFilterExpression = (filter: FilterExpression) => ({
225 Expression: filter.expression,
226 ValuesMap: filter.valuesMap,
227 });
228
229 const buildInput = (input: DatasetInput) => ({
230 S3InputDefinition: input.s3InputDefinition ? buildS3Location(input.s3InputDefinition) : undefined,
231 DataCatalogInputDefinition: input.dataCatalogInputDefinition
232 ? {
233 CatalogId: input.dataCatalogInputDefinition.catalogId,
234 DatabaseName: input.dataCatalogInputDefinition.databaseName,
235 TableName: input.dataCatalogInputDefinition.tableName,
236 TempDirectory: input.dataCatalogInputDefinition.tempDirectory
237 ? buildS3Location(input.dataCatalogInputDefinition.tempDirectory)
238 : undefined,
239 }
240 : undefined,
241 DatabaseInputDefinition: input.databaseInputDefinition
242 ? {
243 GlueConnectionName: input.databaseInputDefinition.glueConnectionName,
244 DatabaseTableName: input.databaseInputDefinition.databaseTableName,
245 TempDirectory: input.databaseInputDefinition.tempDirectory
246 ? buildS3Location(input.databaseInputDefinition.tempDirectory)
247 : undefined,
248 QueryString: input.databaseInputDefinition.queryString,
249 }
250 : undefined,
251 });
252
253 const buildFormatOptions = (options: DatasetFormatOptions | undefined) =>
254 options
255 ? {
256 Json: options.json ? { MultiLine: options.json.multiLine } : undefined,
257 Excel: options.excel
258 ? {
259 SheetNames: options.excel.sheetNames,
260 SheetIndexes: options.excel.sheetIndexes,
261 HeaderRow: options.excel.headerRow,
262 }
263 : undefined,
264 Csv: options.csv
265 ? {
266 Delimiter: options.csv.delimiter,
267 HeaderRow: options.csv.headerRow,
268 }
269 : undefined,
270 }
271 : undefined;
272
273 const buildPathOptions = (options: DatasetPathOptions | undefined) =>
274 options
275 ? {
276 LastModifiedDateCondition: options.lastModifiedDateCondition
277 ? buildFilterExpression(options.lastModifiedDateCondition)
278 : undefined,
279 FilesLimit: options.filesLimit
280 ? {
281 MaxFiles: options.filesLimit.maxFiles,
282 OrderedBy: options.filesLimit.orderedBy,
283 Order: options.filesLimit.order,
284 }
285 : undefined,
286 Parameters: options.parameters
287 ? Object.fromEntries(
288 Object.entries(options.parameters).map(([key, param]) => [
289 key,
290 {
291 Name: param.name,
292 Type: param.type,
293 DatetimeOptions: param.datetimeOptions
294 ? {
295 Format: param.datetimeOptions.format,
296 TimezoneOffset: param.datetimeOptions.timezoneOffset,
297 LocaleCode: param.datetimeOptions.localeCode,
298 }
299 : undefined,
300 CreateColumn: param.createColumn,
301 Filter: param.filter ? buildFilterExpression(param.filter) : undefined,
302 },
303 ]),
304 )
305 : undefined,
306 }
307 : undefined;
308
309 export const DatasetProvider = () =>
310 Provider.effect(
311 Dataset,
312 Effect.gen(function* () {
313 const createName = Effect.fn(function* (
314 id: string,
315 props: { datasetName?: string | undefined },
316 ) {
317 return props.datasetName ?? (yield* createPhysicalName({ id, maxLength: 255 }));
318 });
319
320 const observe = Effect.fn(function* (name: string) {
321 return yield* databrew
322 .describeDataset({ Name: name })
323 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
324 });
325
326 const buildDefinition = (props: DatasetProps) => ({
327 Format: props.format,
328 FormatOptions: buildFormatOptions(props.formatOptions),
329 Input: buildInput(props.input),
330 PathOptions: buildPathOptions(props.pathOptions),
331 });
332
333 return Dataset.Provider.of({
334 stables: ["datasetName", "datasetArn"],
335
336 list: () =>
337 Effect.gen(function* () {
338 const { accountId, region } = yield* AWSEnvironment.current;
339 const pages = yield* databrew.listDatasets.pages({}).pipe(Stream.runCollect);
340 return Array.from(pages)
341 .flatMap((page) => page.Datasets ?? [])
342 .map((d) => ({
343 datasetName: d.Name,
344 datasetArn: d.ResourceArn ?? databrewArn(region, accountId, "dataset", d.Name),
345 }));
346 }),
347
348 read: Effect.fn(function* ({ id, olds, output }) {
349 const { accountId, region } = yield* AWSEnvironment.current;
350 const name = output?.datasetName ?? (yield* createName(id, olds ?? {}));
351 const dataset = yield* observe(name);
352 if (dataset === undefined) return undefined;
353 const arn = dataset.ResourceArn ?? databrewArn(region, accountId, "dataset", name);
354 const attrs = { datasetName: name, datasetArn: arn };
355 const tags = yield* fetchObservedTags(arn);
356 return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
357 }),
358
359 diff: Effect.fn(function* ({ id, news, olds }) {
360 if (!isResolved(news)) return undefined;
361 const oldName = yield* createName(id, olds);
362 const newName = yield* createName(id, news);
363 if (oldName !== newName) return { action: "replace" } as const;
364 // everything else is UpdateDataset-able
365 }),
366
367 reconcile: Effect.fn(function* ({ id, news, output, session }) {
368 const { accountId, region } = yield* AWSEnvironment.current;
369 const name = output?.datasetName ?? (yield* createName(id, news));
370 const internalTags = yield* createInternalTags(id);
371 const desiredTags = { ...news.tags, ...internalTags };
372
373 // 1. OBSERVE
374 const dataset = yield* observe(name);
375
376 // 2. ENSURE / 3. SYNC — UpdateDataset is a full PUT of the definition
377 if (dataset === undefined) {
378 yield* databrew
379 .createDataset({
380 Name: name,
381 ...buildDefinition(news),
382 Tags: desiredTags,
383 })
384 .pipe(
385 // creation race — another reconcile won; fall through to sync
386 Effect.catchTag("ConflictException", () => Effect.void),
387 );
388 } else {
389 yield* databrew.updateDataset({
390 Name: name,
391 ...buildDefinition(news),
392 });
393 }
394
395 const arn = dataset?.ResourceArn ?? databrewArn(region, accountId, "dataset", name);
396
397 // 3b. SYNC TAGS against observed cloud tags
398 const observedTags = yield* fetchObservedTags(arn);
399 yield* syncTags(arn, observedTags, desiredTags);
400
401 yield* session.note(name);
402 return { datasetName: name, datasetArn: arn };
403 }),
404
405 delete: Effect.fn(function* ({ output }) {
406 // A ConflictException surfaces briefly after an associated project
407 // or job is deleted (eventual consistency) — retry bounded.
408 yield* retryWhileConflict(databrew.deleteDataset({ Name: output.datasetName })).pipe(
409 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
410 );
411 }),
412 });
413 }),
414 );