@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 166 lines 6.27 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { Crawler, Database } from "@alchemy.run/aws/Glue";
3 import { Role } from "@alchemy.run/aws/IAM";
4 import { Bucket } from "@alchemy.run/aws/S3";
5 import { expect } from "@alchemy.run/test";
6 import * as Test from "@alchemy.run/test/Alchemy";
7 import * as glue from "@distilled.cloud/aws/glue";
8 import * as s3 from "@distilled.cloud/aws/s3";
9 import * as Output from "alchemy/Output";
10 import * as Effect from "effect/Effect";
11 import * as Schedule from "effect/Schedule";
12
13 const { test } = Test.make({ providers: AWS.providers() });
14
15 const getCrawler = (name: string) =>
16 glue.getCrawler({ Name: name }).pipe(
17 Effect.map((r) => r.Crawler),
18 Effect.catchTag("EntityNotFoundException", () => Effect.succeed(undefined)),
19 );
20
21 const crawlerRole = () =>
22 Role("GlueCrawlerRole", {
23 assumeRolePolicyDocument: {
24 Version: "2012-10-17",
25 Statement: [
26 {
27 Effect: "Allow",
28 Principal: { Service: "glue.amazonaws.com" },
29 Action: ["sts:AssumeRole"],
30 },
31 ],
32 },
33 managedPolicyArns: [
34 "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole",
35 "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess",
36 ],
37 });
38
39 test.provider(
40 "create, update, delete Glue crawler definition",
41 (stack) =>
42 Effect.gen(function* () {
43 yield* stack.destroy();
44
45 const created = yield* stack.deploy(
46 Effect.gen(function* () {
47 const database = yield* Database("CrawlerDb", {});
48 const bucket = yield* Bucket("CrawlerBucket", { forceDestroy: true });
49 const role = yield* crawlerRole();
50 const crawler = yield* Crawler("EventsCrawler", {
51 role: role.roleArn,
52 databaseName: database.databaseName,
53 targets: { s3Targets: [{ path: Output.interpolate`s3://${bucket.bucketName}/data/` }] },
54 tablePrefix: "raw_",
55 tags: { Environment: "test" },
56 });
57 return { database, bucket, crawler };
58 }),
59 );
60
61 expect(created.crawler.crawlerName).toBeDefined();
62 expect(created.crawler.crawlerArn).toContain(`:crawler/${created.crawler.crawlerName}`);
63
64 // out-of-band verification
65 const observed = yield* getCrawler(created.crawler.crawlerName);
66 expect(observed?.Name).toEqual(created.crawler.crawlerName);
67 expect(observed?.DatabaseName).toEqual(created.database.databaseName);
68 expect(observed?.TablePrefix).toEqual("raw_");
69 expect(observed?.Targets?.S3Targets?.[0]?.Path).toEqual(
70 `s3://${created.bucket.bucketName}/data/`,
71 );
72
73 // tags (crawlers ARE ARN-taggable)
74 const tags = yield* glue.getTags({ ResourceArn: created.crawler.crawlerArn });
75 expect(tags.Tags?.["alchemy::id"]).toBeDefined();
76 expect(tags.Tags?.Environment).toEqual("test");
77
78 // update: description + schedule
79 yield* stack.deploy(
80 Effect.gen(function* () {
81 const database = yield* Database("CrawlerDb", {});
82 const bucket = yield* Bucket("CrawlerBucket", { forceDestroy: true });
83 const role = yield* crawlerRole();
84 const crawler = yield* Crawler("EventsCrawler", {
85 role: role.roleArn,
86 databaseName: database.databaseName,
87 description: "crawls the events prefix",
88 targets: { s3Targets: [{ path: Output.interpolate`s3://${bucket.bucketName}/data/` }] },
89 tablePrefix: "raw_",
90 schedule: "cron(0 12 * * ? *)",
91 schemaChangePolicy: {
92 updateBehavior: "UPDATE_IN_DATABASE",
93 deleteBehavior: "DEPRECATE_IN_DATABASE",
94 },
95 tags: { Environment: "test" },
96 });
97 return { crawler };
98 }),
99 );
100
101 const reobserved = yield* getCrawler(created.crawler.crawlerName);
102 expect(reobserved?.Description).toEqual("crawls the events prefix");
103 expect(reobserved?.Schedule?.ScheduleExpression).toEqual("cron(0 12 * * ? *)");
104 expect(reobserved?.SchemaChangePolicy?.UpdateBehavior).toEqual("UPDATE_IN_DATABASE");
105
106 yield* stack.destroy();
107 const gone = yield* getCrawler(created.crawler.crawlerName);
108 expect(gone).toBeUndefined();
109 }),
110 { tags: ["provider:aws", "provider:aws:glue", "provider:aws:iam", "provider:aws:s3", "live"] },
111 );
112
113 // A live crawl takes ~1-4 minutes end to end, but under concurrent load it
114 // has not reached SUCCEEDED within 5 minutes (2026-10-09) — skipped under
115 // `--fast`.
116 test.provider.skipIf(!!process.env.FAST)(
117 "run a live crawl that populates the catalog",
118 (stack) =>
119 Effect.gen(function* () {
120 yield* stack.destroy();
121
122 const deployed = yield* stack.deploy(
123 Effect.gen(function* () {
124 const database = yield* Database("LiveCrawlDb", {});
125 const bucket = yield* Bucket("LiveCrawlBucket", { forceDestroy: true });
126 const role = yield* crawlerRole();
127 const crawler = yield* Crawler("LiveCrawler", {
128 role: role.roleArn,
129 databaseName: database.databaseName,
130 targets: { s3Targets: [{ path: Output.interpolate`s3://${bucket.bucketName}/data/` }] },
131 });
132 return { database, bucket, crawler };
133 }),
134 );
135
136 // seed a CSV object so the crawler infers a table
137 yield* s3.putObject({
138 Bucket: deployed.bucket.bucketName,
139 Key: "data/events/part-0.csv",
140 Body: new TextEncoder().encode("id,amount\n1,10.5\n2,20.0\n"),
141 ContentType: "text/csv",
142 });
143
144 yield* glue.startCrawler({ Name: deployed.crawler.crawlerName });
145
146 // poll until the crawler returns to READY with a terminal last-crawl
147 const finalState = yield* getCrawler(deployed.crawler.crawlerName).pipe(
148 Effect.repeat({
149 schedule: Schedule.spaced("15 seconds"),
150 until: (c) => c?.State === "READY" && c?.LastCrawl?.Status !== undefined,
151 times: 20,
152 }),
153 );
154 expect(finalState?.LastCrawl?.Status).toEqual("SUCCEEDED");
155
156 // a table should now exist in the catalog
157 const tables = yield* glue.getTables({ DatabaseName: deployed.database.databaseName });
158 expect(tables.TableList?.length ?? 0).toBeGreaterThan(0);
159
160 yield* stack.destroy();
161 }),
162 {
163 tags: ["provider:aws", "provider:aws:glue", "provider:aws:iam", "provider:aws:s3", "live"],
164 timeout: 360_000,
165 },
166 );