The best way to control Claude Code, Codex and more from any computer or phone

main
ts 385 lines 13.2 KB
Raw
1 import * as glue from "@distilled.cloud/aws/glue";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
8 import * as Effect from "effect/Effect";
9 import * as Stream from "effect/Stream";
10 import { AWSEnvironment } from "../Environment.ts";
11 import type { Providers } from "../Providers.ts";
12 import { cleanMap, retryWhileConcurrentModification, tableArn } from "./internal.ts";
13
14 export interface GlueColumn {
15 /** Column name. */
16 name: string;
17 /** Column data type (Hive/Glue type string, e.g. `string`, `bigint`). */
18 type?: string;
19 /** Free-text comment. */
20 comment?: string;
21 /** Free-form key/value properties for the column. */
22 parameters?: Record<string, string>;
23 }
24
25 export interface GlueSerDeInfo {
26 /** Name of the SerDe. */
27 name?: string;
28 /**
29 * The serialization library class, e.g.
30 * `org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe`.
31 */
32 serializationLibrary?: string;
33 /** SerDe configuration properties. */
34 parameters?: Record<string, string>;
35 }
36
37 export interface GlueStorageDescriptor {
38 /** Columns of the table (schema). */
39 columns?: GlueColumn[];
40 /** Physical location of the data (usually an S3 path). */
41 location?: string;
42 /**
43 * The input format class, e.g.
44 * `org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat`.
45 */
46 inputFormat?: string;
47 /**
48 * The output format class, e.g.
49 * `org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat`.
50 */
51 outputFormat?: string;
52 /** Whether the data is compressed. */
53 compressed?: boolean;
54 /** Number of buckets used to bucket the table (if `bucketColumns` is set). */
55 numberOfBuckets?: number;
56 /** Serialization/deserialization metadata. */
57 serdeInfo?: GlueSerDeInfo;
58 /** Columns the table is bucketed by. */
59 bucketColumns?: string[];
60 /** Free-form storage descriptor properties. */
61 parameters?: Record<string, string>;
62 /** Whether the table stores data in subdirectories. */
63 storedAsSubDirectories?: boolean;
64 }
65
66 export interface TableProps {
67 /**
68 * Name of the database the table belongs to. Changing it replaces the
69 * table.
70 */
71 databaseName: string;
72 /**
73 * Name of the table. Glue lowercases table names. If omitted, a unique
74 * lowercase name is generated. Changing the name replaces the table.
75 * @default a generated lowercase physical name
76 */
77 tableName?: string;
78 /**
79 * A description of the table.
80 */
81 description?: string;
82 /**
83 * The table owner.
84 */
85 owner?: string;
86 /**
87 * The type of the table. Common values are `EXTERNAL_TABLE`,
88 * `GOVERNED`, and `VIRTUAL_VIEW`.
89 * @default "EXTERNAL_TABLE"
90 */
91 tableType?: string;
92 /**
93 * Retention time (in days) for the table.
94 */
95 retention?: number;
96 /**
97 * The physical storage descriptor: columns (schema), data location, input/
98 * output formats, and SerDe. This is what Athena reads to query the data.
99 */
100 storageDescriptor?: GlueStorageDescriptor;
101 /**
102 * Partition keys. Athena prunes partitions on these columns. Note: changing
103 * partition keys after creation requires `Force`; Alchemy sends the desired
104 * set on every update.
105 */
106 partitionKeys?: GlueColumn[];
107 /**
108 * Free-form key/value properties stored on the table. Alchemy adds its own
109 * `alchemy::*` ownership markers here (Glue tables are not ARN-taggable) —
110 * user keys are preserved.
111 */
112 parameters?: Record<string, string>;
113 /**
114 * The AWS account ID of the Data Catalog. Changing it replaces the table.
115 * @default the caller's account (the default Data Catalog)
116 */
117 catalogId?: string;
118 }
119
120 export interface Table extends Resource<
121 "AWS.Glue.Table",
122 TableProps,
123 {
124 /** The (lowercase) name of the table. */
125 tableName: string;
126 /** The name of the database the table belongs to. */
127 databaseName: string;
128 /** The ARN of the table. */
129 tableArn: string;
130 /** The AWS account ID of the Data Catalog the table lives in. */
131 catalogId: string;
132 },
133 {},
134 Providers
135 > {}
136
137 /**
138 * An AWS Glue Data Catalog table — a schema (columns), storage location, and
139 * SerDe over data in S3 (or another store). This is the unit Athena, Redshift
140 * Spectrum, and EMR query; it is the analytics foundation of a Glue database.
141 * ### Creating Tables
142 * **Example:** Parquet Table over S3
143 * ```typescript
144 * import * as AWS from "@alchemy.run/aws";
145 *
146 * const database = yield* AWS.Glue.Database("Analytics", {
147 * databaseName: "analytics",
148 * });
149 *
150 * const events = yield* AWS.Glue.Table("Events", {
151 * databaseName: database.databaseName,
152 * tableName: "events",
153 * tableType: "EXTERNAL_TABLE",
154 * storageDescriptor: {
155 * location: "s3://my-data-lake/events/",
156 * inputFormat:
157 * "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
158 * outputFormat:
159 * "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
160 * serdeInfo: {
161 * serializationLibrary:
162 * "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
163 * },
164 * columns: [
165 * { name: "id", type: "string" },
166 * { name: "amount", type: "double" },
167 * ],
168 * },
169 * partitionKeys: [{ name: "dt", type: "string" }],
170 * parameters: { classification: "parquet" },
171 * });
172 * ```
173 *
174 * @resource
175 */
176 export const Table = Resource<Table>("AWS.Glue.Table");
177
178 const toColumn = (column: GlueColumn) => ({
179 Name: column.name,
180 Type: column.type,
181 Comment: column.comment,
182 Parameters: column.parameters,
183 });
184
185 export const TableProvider = () =>
186 Provider.effect(
187 Table,
188 Effect.gen(function* () {
189 const createName = Effect.fn(function* (
190 id: string,
191 props: { tableName?: string | undefined },
192 ) {
193 return (
194 props.tableName ?? (yield* createPhysicalName({ id, maxLength: 255, lowercase: true }))
195 );
196 });
197
198 const observe = Effect.fn(function* (
199 databaseName: string,
200 name: string,
201 catalogId: string | undefined,
202 ) {
203 return yield* glue
204 .getTable({
205 DatabaseName: databaseName,
206 Name: name,
207 CatalogId: catalogId,
208 })
209 .pipe(
210 Effect.map((r) => r.Table),
211 Effect.catchTag("EntityNotFoundException", () => Effect.undefined),
212 );
213 });
214
215 const buildTableInput = (name: string, props: TableProps) => {
216 const sd = props.storageDescriptor;
217 return {
218 Name: name,
219 Description: props.description,
220 Owner: props.owner,
221 Retention: props.retention,
222 TableType: props.tableType ?? "EXTERNAL_TABLE",
223 StorageDescriptor: sd
224 ? {
225 Columns: sd.columns?.map(toColumn),
226 Location: sd.location,
227 InputFormat: sd.inputFormat,
228 OutputFormat: sd.outputFormat,
229 Compressed: sd.compressed,
230 NumberOfBuckets: sd.numberOfBuckets,
231 SerdeInfo: sd.serdeInfo
232 ? {
233 Name: sd.serdeInfo.name,
234 SerializationLibrary: sd.serdeInfo.serializationLibrary,
235 Parameters: sd.serdeInfo.parameters,
236 }
237 : undefined,
238 BucketColumns: sd.bucketColumns,
239 Parameters: sd.parameters,
240 StoredAsSubDirectories: sd.storedAsSubDirectories,
241 }
242 : undefined,
243 PartitionKeys: props.partitionKeys?.map(toColumn),
244 };
245 };
246
247 return Table.Provider.of({
248 stables: ["tableName", "databaseName", "tableArn", "catalogId"],
249
250 list: () =>
251 Effect.gen(function* () {
252 const { accountId, region } = yield* AWSEnvironment.current;
253 const dbPages = yield* glue.getDatabases.pages({}).pipe(Stream.runCollect);
254 const databases = Array.from(dbPages).flatMap((page) => page.DatabaseList ?? []);
255 const nested = yield* Effect.forEach(
256 databases,
257 (db) =>
258 Effect.gen(function* () {
259 const tablePages = yield* glue.getTables
260 .pages({ DatabaseName: db.Name, CatalogId: db.CatalogId })
261 .pipe(
262 Stream.runCollect,
263 Effect.map((chunk) => Array.from(chunk)),
264 Effect.catchTag("EntityNotFoundException", () =>
265 Effect.succeed([] as glue.GetTablesResponse[]),
266 ),
267 );
268 return tablePages
269 .flatMap((page) => page.TableList ?? [])
270 .map((table) => ({
271 tableName: table.Name,
272 databaseName: table.DatabaseName ?? db.Name,
273 tableArn: tableArn(
274 region,
275 table.CatalogId ?? db.CatalogId ?? accountId,
276 table.DatabaseName ?? db.Name,
277 table.Name,
278 ),
279 catalogId: table.CatalogId ?? db.CatalogId ?? accountId,
280 }));
281 }),
282 { concurrency: 5 },
283 );
284 return nested.flat();
285 }),
286
287 read: Effect.fn(function* ({ id, olds, output }) {
288 const { accountId, region } = yield* AWSEnvironment.current;
289 const databaseName = output?.databaseName ?? olds?.databaseName;
290 if (databaseName === undefined) return undefined;
291 const catalogId = output?.catalogId ?? olds?.catalogId ?? accountId;
292 const name = output?.tableName ?? (yield* createName(id, olds ?? {}));
293 const table = yield* observe(databaseName, name, catalogId);
294 if (table === undefined) return undefined;
295 const attrs = {
296 tableName: table.Name,
297 databaseName: table.DatabaseName ?? databaseName,
298 tableArn: tableArn(
299 region,
300 table.CatalogId ?? catalogId,
301 table.DatabaseName ?? databaseName,
302 table.Name,
303 ),
304 catalogId: table.CatalogId ?? catalogId,
305 };
306 return (yield* hasAlchemyTags(id, cleanMap(table.Parameters))) ? attrs : Unowned(attrs);
307 }),
308
309 diff: Effect.fn(function* ({ id, news, olds }) {
310 if (!isResolved(news)) return undefined;
311 const oldName = yield* createName(id, olds);
312 const newName = yield* createName(id, news);
313 if (oldName !== newName) return { action: "replace" } as const;
314 if (olds.databaseName !== news.databaseName) {
315 return { action: "replace" } as const;
316 }
317 if ((olds.catalogId ?? undefined) !== (news.catalogId ?? undefined)) {
318 return { action: "replace" } as const;
319 }
320 // schema / storage / partition keys / parameters → update
321 }),
322
323 reconcile: Effect.fn(function* ({ id, news, output, session }) {
324 const { accountId, region } = yield* AWSEnvironment.current;
325 const catalogId = news.catalogId ?? output?.catalogId ?? accountId;
326 const name = output?.tableName ?? (yield* createName(id, news));
327 const internalTags = yield* createInternalTags(id);
328 const tableInput = {
329 ...buildTableInput(name, news),
330 Parameters: { ...news.parameters, ...internalTags },
331 };
332
333 // 1. OBSERVE
334 let table = yield* observe(news.databaseName, name, catalogId);
335
336 // 2. ENSURE / 3. SYNC
337 if (table === undefined) {
338 yield* glue
339 .createTable({
340 CatalogId: catalogId,
341 DatabaseName: news.databaseName,
342 TableInput: tableInput,
343 })
344 .pipe(
345 Effect.catchTag("AlreadyExistsException", () => Effect.void),
346 retryWhileConcurrentModification,
347 );
348 } else {
349 // UpdateTable replaces the full TableInput. Force lets partition
350 // keys change without a version conflict.
351 yield* glue
352 .updateTable({
353 CatalogId: catalogId,
354 DatabaseName: news.databaseName,
355 TableInput: tableInput,
356 Force: true,
357 })
358 .pipe(retryWhileConcurrentModification);
359 }
360 table = yield* observe(news.databaseName, name, catalogId);
361
362 yield* session.note(name);
363 return {
364 tableName: name,
365 databaseName: news.databaseName,
366 tableArn: tableArn(region, table?.CatalogId ?? catalogId, news.databaseName, name),
367 catalogId: table?.CatalogId ?? catalogId,
368 };
369 }),
370
371 delete: Effect.fn(function* ({ output }) {
372 yield* glue
373 .deleteTable({
374 CatalogId: output.catalogId,
375 DatabaseName: output.databaseName,
376 Name: output.tableName,
377 })
378 .pipe(
379 retryWhileConcurrentModification,
380 Effect.catchTag("EntityNotFoundException", () => Effect.void),
381 );
382 }),
383 });
384 }),
385 );