@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 425 lines 14.1 KB
Raw
1 import * as glue from "@distilled.cloud/aws/glue";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { AWSEnvironment } from "../Environment.ts";
12 import type { Providers } from "../Providers.ts";
13 import {
14 crawlerArn,
15 fetchObservedTags,
16 reapGlueLogStreams,
17 retryCrawlerDelete,
18 retryWhileCrawlerTargetNotReady,
19 retryWhileCrawlerRunning,
20 retryWhileRoleNotAssumable,
21 syncTags,
22 } from "./internal.ts";
23
24 /** Shared, service-managed log group Glue writes every crawl to. */
25 const GLUE_CRAWLER_LOG_GROUP = "/aws-glue/crawlers";
26
27 export interface CrawlerS3Target {
28 /** S3 path to crawl, e.g. `s3://my-bucket/data/`. */
29 path: string;
30 /** Glob patterns to exclude. */
31 exclusions?: string[];
32 /** Name of a Glue connection to use (for VPC-scoped S3). */
33 connectionName?: string;
34 /** Sample size (files per leaf) to speed up crawling. */
35 sampleSize?: number;
36 /** SQS queue ARN for event-based (incremental) crawling. */
37 eventQueueArn?: string;
38 /** Dead-letter SQS queue ARN for event-based crawling. */
39 dlqEventQueueArn?: string;
40 }
41
42 export interface CrawlerJdbcTarget {
43 /** Name of the Glue connection to the JDBC source. */
44 connectionName?: string;
45 /** The path of the JDBC target, e.g. `database/schema/%`. */
46 path?: string;
47 /** Glob patterns to exclude. */
48 exclusions?: string[];
49 }
50
51 export interface CrawlerDynamoDbTarget {
52 /** The DynamoDB table name to crawl. */
53 path?: string;
54 /** Whether to scan all records (vs a sample). */
55 scanAll?: boolean;
56 /** Fraction of the table's read capacity to consume (0.1–1.5). */
57 scanRate?: number;
58 }
59
60 export interface CrawlerCatalogTarget {
61 /** The database of the catalog tables to recrawl. */
62 databaseName: string;
63 /** The tables in the database to recrawl. */
64 tables: string[];
65 /** Name of a Glue connection. */
66 connectionName?: string;
67 }
68
69 export interface CrawlerTargets {
70 /** S3 data-store targets. */
71 s3Targets?: CrawlerS3Target[];
72 /** JDBC data-store targets. */
73 jdbcTargets?: CrawlerJdbcTarget[];
74 /** DynamoDB targets. */
75 dynamoDbTargets?: CrawlerDynamoDbTarget[];
76 /** Glue Data Catalog targets (recrawl existing tables). */
77 catalogTargets?: CrawlerCatalogTarget[];
78 }
79
80 export interface CrawlerProps {
81 /**
82 * Name of the crawler. If omitted, a unique name is generated. Changing the
83 * name replaces the crawler.
84 * @default a generated physical name
85 */
86 crawlerName?: string;
87 /**
88 * The IAM role (ARN or name) the crawler assumes to access data stores and
89 * write to the Data Catalog.
90 */
91 role: string;
92 /**
93 * The Glue database where the crawler writes discovered tables.
94 */
95 databaseName?: string;
96 /**
97 * A description of the crawler.
98 */
99 description?: string;
100 /**
101 * The data stores to crawl.
102 */
103 targets: CrawlerTargets;
104 /**
105 * A `cron(...)` schedule expression. Omit for on-demand crawling.
106 * @example "cron(0 12 * * ? *)"
107 */
108 schedule?: string;
109 /**
110 * A prefix prepended to the names of tables the crawler creates.
111 */
112 tablePrefix?: string;
113 /**
114 * Custom classifier names, in priority order.
115 */
116 classifiers?: string[];
117 /**
118 * How the crawler handles schema changes and deleted objects.
119 */
120 schemaChangePolicy?: {
121 /** What to do when a schema changes: `LOG` or `UPDATE_IN_DATABASE`. */
122 updateBehavior?: "LOG" | "UPDATE_IN_DATABASE";
123 /**
124 * What to do with deleted objects: `LOG`, `DELETE_FROM_DATABASE`, or
125 * `DEPRECATE_IN_DATABASE`.
126 */
127 deleteBehavior?: "LOG" | "DELETE_FROM_DATABASE" | "DEPRECATE_IN_DATABASE";
128 };
129 /**
130 * Incremental-crawl behavior.
131 */
132 recrawlPolicy?: {
133 /**
134 * `CRAWL_EVERYTHING`, `CRAWL_NEW_FOLDERS_ONLY`, or `CRAWL_EVENT_MODE`.
135 */
136 recrawlBehavior?: "CRAWL_EVERYTHING" | "CRAWL_NEW_FOLDERS_ONLY" | "CRAWL_EVENT_MODE";
137 };
138 /**
139 * Crawler configuration JSON string (grouping/partitions behavior).
140 */
141 configuration?: string;
142 /**
143 * Tags to apply to the crawler. Merged with internal Alchemy tags.
144 */
145 tags?: Record<string, string>;
146 }
147
148 export interface Crawler extends Resource<
149 "AWS.Glue.Crawler",
150 CrawlerProps,
151 {
152 /** The name of the crawler. */
153 crawlerName: string;
154 /** The ARN of the crawler. */
155 crawlerArn: string;
156 /** The IAM role the crawler assumes. */
157 role: string;
158 /** The Glue database the crawler writes discovered tables to. */
159 databaseName: string | undefined;
160 /** The crawler state: `READY`, `RUNNING`, or `STOPPING`. */
161 state: string | undefined;
162 },
163 {},
164 Providers
165 > {}
166
167 /**
168 * An AWS Glue crawler — connects to an S3 (or JDBC/DynamoDB/catalog) data
169 * store, infers schemas, and populates the Glue Data Catalog with tables.
170 * Runs are asynchronous: create the crawler, then invoke `startCrawler` (or
171 * attach a schedule).
172 * ### Creating Crawlers
173 * **Example:** S3 Crawler
174 * ```typescript
175 * import * as AWS from "@alchemy.run/aws";
176 *
177 * const database = yield* AWS.Glue.Database("Analytics", {
178 * databaseName: "analytics",
179 * });
180 *
181 * const crawler = yield* AWS.Glue.Crawler("EventsCrawler", {
182 * role: crawlerRole.roleArn,
183 * databaseName: database.databaseName,
184 * targets: {
185 * s3Targets: [{ path: "s3://my-data-lake/events/" }],
186 * },
187 * });
188 * ```
189 *
190 * **Example:** Scheduled Crawler with Schema Policy
191 * ```typescript
192 * const crawler = yield* AWS.Glue.Crawler("EventsCrawler", {
193 * role: crawlerRole.roleArn,
194 * databaseName: database.databaseName,
195 * targets: { s3Targets: [{ path: "s3://my-data-lake/events/" }] },
196 * schedule: "cron(0 12 * * ? *)",
197 * tablePrefix: "raw_",
198 * schemaChangePolicy: {
199 * updateBehavior: "UPDATE_IN_DATABASE",
200 * deleteBehavior: "DEPRECATE_IN_DATABASE",
201 * },
202 * });
203 * ```
204 *
205 * @resource
206 */
207 export const Crawler = Resource<Crawler>("AWS.Glue.Crawler");
208
209 const toTargets = (targets: CrawlerTargets) => ({
210 S3Targets: targets.s3Targets?.map((t) => ({
211 Path: t.path,
212 Exclusions: t.exclusions,
213 ConnectionName: t.connectionName,
214 SampleSize: t.sampleSize,
215 EventQueueArn: t.eventQueueArn,
216 DlqEventQueueArn: t.dlqEventQueueArn,
217 })),
218 JdbcTargets: targets.jdbcTargets?.map((t) => ({
219 ConnectionName: t.connectionName,
220 Path: t.path,
221 Exclusions: t.exclusions,
222 })),
223 DynamoDBTargets: targets.dynamoDbTargets?.map((t) => ({
224 Path: t.path,
225 scanAll: t.scanAll,
226 scanRate: t.scanRate,
227 })),
228 CatalogTargets: targets.catalogTargets?.map((t) => ({
229 DatabaseName: t.databaseName,
230 Tables: t.tables,
231 ConnectionName: t.connectionName,
232 })),
233 });
234
235 export const CrawlerProvider = () =>
236 Provider.effect(
237 Crawler,
238 Effect.gen(function* () {
239 const createName = Effect.fn(function* (
240 id: string,
241 props: { crawlerName?: string | undefined },
242 ) {
243 return props.crawlerName ?? (yield* createPhysicalName({ id, maxLength: 255 }));
244 });
245
246 const observe = Effect.fn(function* (name: string) {
247 return yield* glue.getCrawler({ Name: name }).pipe(
248 Effect.map((r) => r.Crawler),
249 Effect.catchTag("EntityNotFoundException", () => Effect.undefined),
250 );
251 });
252
253 return Crawler.Provider.of({
254 stables: ["crawlerName", "crawlerArn"],
255
256 list: () =>
257 Effect.gen(function* () {
258 const { accountId, region } = yield* AWSEnvironment.current;
259 const pages = yield* glue.getCrawlers.pages({}).pipe(Stream.runCollect);
260 return Array.from(pages)
261 .flatMap((page) => page.Crawlers ?? [])
262 .filter((c) => c.Name !== undefined)
263 .map((c) => ({
264 crawlerName: c.Name!,
265 crawlerArn: crawlerArn(region, accountId, c.Name!),
266 role: c.Role ?? "",
267 databaseName: c.DatabaseName,
268 state: c.State,
269 }));
270 }),
271
272 read: Effect.fn(function* ({ id, olds, output }) {
273 const { accountId, region } = yield* AWSEnvironment.current;
274 const name = output?.crawlerName ?? (yield* createName(id, olds ?? {}));
275 const crawler = yield* observe(name);
276 if (crawler?.Name === undefined) return undefined;
277 const arn = crawlerArn(region, accountId, crawler.Name);
278 const attrs = {
279 crawlerName: crawler.Name,
280 crawlerArn: arn,
281 role: crawler.Role ?? "",
282 databaseName: crawler.DatabaseName,
283 state: crawler.State,
284 };
285 const tags = yield* fetchObservedTags(arn);
286 return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
287 }),
288
289 diff: Effect.fn(function* ({ id, news, olds }) {
290 if (!isResolved(news)) return undefined;
291 const oldName = yield* createName(id, olds);
292 const newName = yield* createName(id, news);
293 if (oldName !== newName) return { action: "replace" } as const;
294 // role / targets / schedule / policies / description → update
295 }),
296
297 reconcile: Effect.fn(function* ({ id, news, output, session }) {
298 const { accountId, region } = yield* AWSEnvironment.current;
299 const name = output?.crawlerName ?? (yield* createName(id, news));
300 const internalTags = yield* createInternalTags(id);
301 const desiredTags = { ...news.tags, ...internalTags };
302 const arn = crawlerArn(region, accountId, name);
303
304 const common = {
305 Role: news.role,
306 DatabaseName: news.databaseName,
307 Description: news.description,
308 Targets: toTargets(news.targets),
309 Schedule: news.schedule,
310 Classifiers: news.classifiers,
311 TablePrefix: news.tablePrefix,
312 SchemaChangePolicy: news.schemaChangePolicy
313 ? {
314 UpdateBehavior: news.schemaChangePolicy.updateBehavior,
315 DeleteBehavior: news.schemaChangePolicy.deleteBehavior,
316 }
317 : undefined,
318 RecrawlPolicy: news.recrawlPolicy
319 ? { RecrawlBehavior: news.recrawlPolicy.recrawlBehavior }
320 : undefined,
321 Configuration: news.configuration,
322 };
323
324 // 1. OBSERVE
325 let crawler = yield* observe(name);
326
327 // 2. ENSURE / 3. SYNC
328 if (crawler === undefined) {
329 yield* glue
330 .createCrawler({
331 Name: name,
332 ...common,
333 Tags: desiredTags,
334 })
335 .pipe(
336 retryWhileRoleNotAssumable,
337 retryWhileCrawlerTargetNotReady,
338 Effect.catchTag("AlreadyExistsException", () => Effect.void),
339 );
340 } else {
341 // updateCrawler fails with CrawlerRunningException mid-crawl and
342 // with GlueRoleNotAssumable during IAM propagation.
343 yield* glue
344 .updateCrawler({ Name: name, ...common })
345 .pipe(
346 retryWhileRoleNotAssumable,
347 retryWhileCrawlerTargetNotReady,
348 retryWhileCrawlerRunning,
349 );
350 }
351
352 // 3b. SYNC TAGS
353 const observedTags = yield* fetchObservedTags(arn);
354 yield* syncTags(arn, observedTags, desiredTags);
355
356 crawler = yield* observe(name);
357
358 yield* session.note(name);
359 return {
360 crawlerName: name,
361 crawlerArn: arn,
362 role: crawler?.Role ?? news.role,
363 databaseName: crawler?.DatabaseName ?? news.databaseName,
364 state: crawler?.State,
365 };
366 }),
367
368 delete: Effect.fn(function* ({ output }) {
369 const name = output.crawlerName;
370 // Crawler teardown first; the early returns only skip the
371 // crawler steps so the log reap below still runs on a retried
372 // delete whose crawler is already gone.
373 // oxlint-disable-next-line effecttsgo/nested-effect-gen-yield -- scopes the early returns
374 yield* Effect.gen(function* () {
375 let crawler = yield* observe(name);
376 if (crawler === undefined) return;
377
378 if (crawler.State === "RUNNING") {
379 yield* glue
380 .stopCrawler({ Name: name })
381 .pipe(
382 Effect.catchTag(
383 [
384 "CrawlerNotRunningException",
385 "CrawlerStoppingException",
386 "EntityNotFoundException",
387 ],
388 () => Effect.void,
389 ),
390 );
391 }
392
393 // Stop is asynchronous. Wait for READY (deletable) or absence;
394 // interrupted deletes can resume from RUNNING/STOPPING safely.
395 crawler = yield* Effect.repeat(observe(name), {
396 schedule: Schedule.fixed("2 seconds"),
397 until: (current) => current === undefined || current.State === "READY",
398 times: 15,
399 });
400 if (crawler === undefined) return;
401
402 yield* retryCrawlerDelete(glue.deleteCrawler({ Name: name })).pipe(
403 Effect.catchTag("EntityNotFoundException", () => Effect.void),
404 );
405
406 const remaining = yield* Effect.repeat(observe(name), {
407 schedule: Schedule.fixed("2 seconds"),
408 until: (current) => current === undefined,
409 times: 15,
410 });
411 if (remaining !== undefined) {
412 return yield* glue.OperationTimeoutException.make({
413 message: `crawler ${name} remained visible after delete for 30 seconds`,
414 });
415 }
416 });
417
418 // Glue logs every crawl to the stream `<crawlerName>` in the shared,
419 // service-managed `/aws-glue/crawlers` group. deleteCrawler leaves
420 // it behind, so reap our stream (never the shared group).
421 yield* reapGlueLogStreams([GLUE_CRAWLER_LOG_GROUP], [name], { exact: true });
422 }),
423 });
424 }),
425 );