@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 217 lines 7.22 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { Dataset, Job, Recipe } from "@alchemy.run/aws/DataBrew";
3 import { Role } from "@alchemy.run/aws/IAM";
4 import { Bucket } from "@alchemy.run/aws/S3";
5 import { expect } from "@alchemy.run/test";
6 import * as Test from "@alchemy.run/test/Alchemy";
7 import * as databrew from "@distilled.cloud/aws/databrew";
8 import * as s3 from "@distilled.cloud/aws/s3";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11
12 const { test } = Test.make({ providers: AWS.providers() });
13
14 const getJob = (name: string) =>
15 databrew
16 .describeJob({ Name: name })
17 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.succeed(undefined)));
18
19 const brewRole = () =>
20 Role("DataBrewJobRole", {
21 assumeRolePolicyDocument: {
22 Version: "2012-10-17",
23 Statement: [
24 {
25 Effect: "Allow",
26 Principal: { Service: "databrew.amazonaws.com" },
27 Action: ["sts:AssumeRole"],
28 },
29 ],
30 },
31 managedPolicyArns: [
32 "arn:aws:iam::aws:policy/service-role/AwsGlueDataBrewServiceRole",
33 "arn:aws:iam::aws:policy/AmazonS3FullAccess",
34 ],
35 });
36
37 // Shared foundation: bucket + role + dataset + published recipe.
38 const foundation = Effect.gen(function* () {
39 const bucket = yield* Bucket("JobBucket", { forceDestroy: true });
40 const role = yield* brewRole();
41 const dataset = yield* Dataset("Source", {
42 format: "CSV",
43 formatOptions: { csv: { delimiter: ",", headerRow: true } },
44 input: {
45 s3InputDefinition: { bucket: bucket.bucketName, key: "raw/data.csv" },
46 },
47 });
48 const recipe = yield* Recipe("Transform", {
49 publish: true,
50 steps: [
51 {
52 action: {
53 operation: "UPPER_CASE",
54 parameters: { sourceColumn: "name" },
55 },
56 },
57 ],
58 });
59 return { bucket, role, dataset, recipe };
60 });
61
62 // CreateProfileJob/CreateRecipeJob validate the dataset's source object with
63 // the job role, so the CSV is seeded between the foundation and job deploys.
64 const seedSource = (bucketName: string) =>
65 s3.putObject({
66 Bucket: bucketName,
67 Key: "raw/data.csv",
68 Body: new TextEncoder().encode("id,name\n1,alice\n2,bob\n"),
69 ContentType: "text/csv",
70 });
71
72 const withJobs = (maxCapacity: number) =>
73 Effect.gen(function* () {
74 const base = yield* foundation;
75 const profileJob = yield* Job("ProfileJob", {
76 type: "PROFILE",
77 datasetName: base.dataset.datasetName,
78 role: base.role.roleArn,
79 outputLocation: { bucket: base.bucket.bucketName, key: "profiles/" },
80 jobSample: { mode: "CUSTOM_ROWS", size: 100 },
81 maxCapacity,
82 timeout: "1 hour",
83 tags: { Environment: "test" },
84 });
85 const recipeJob = yield* Job("RecipeJob", {
86 type: "RECIPE",
87 datasetName: base.dataset.datasetName,
88 recipeReference: { name: base.recipe.recipeName },
89 role: base.role.roleArn,
90 outputs: [
91 {
92 location: { bucket: base.bucket.bucketName, key: "curated/" },
93 format: "CSV",
94 overwrite: true,
95 },
96 ],
97 maxCapacity,
98 tags: { Environment: "test" },
99 });
100 return { ...base, profileJob, recipeJob };
101 });
102
103 test.provider(
104 "create, update, delete profile and recipe job definitions",
105 (stack) =>
106 Effect.gen(function* () {
107 yield* stack.destroy();
108
109 // stage 1: infrastructure the jobs depend on
110 const base = yield* stack.deploy(foundation);
111 yield* seedSource(base.bucket.bucketName);
112
113 // stage 2: the job definitions
114 const created = yield* stack.deploy(withJobs(2));
115
116 expect(created.profileJob.jobArn).toContain(`:job/${created.profileJob.jobName}`);
117 expect(created.profileJob.type).toEqual("PROFILE");
118 expect(created.recipeJob.type).toEqual("RECIPE");
119
120 // out-of-band verification
121 const profile = yield* getJob(created.profileJob.jobName);
122 expect(profile?.Type).toEqual("PROFILE");
123 expect(profile?.DatasetName).toEqual(created.dataset.datasetName);
124 expect(profile?.JobSample?.Size).toEqual(100);
125 expect(profile?.MaxCapacity).toEqual(2);
126 // Duration.Input "1 hour" lands on the wire as 60 (minutes)
127 expect(profile?.Timeout).toEqual(60);
128 expect(profile?.Tags?.["alchemy::id"]).toBeDefined();
129
130 const recipeJob = yield* getJob(created.recipeJob.jobName);
131 expect(recipeJob?.Type).toEqual("RECIPE");
132 expect(recipeJob?.RecipeReference?.Name).toEqual(created.recipe.recipeName);
133 expect(recipeJob?.Outputs?.[0]?.Location.Bucket).toEqual(created.bucket.bucketName);
134 expect(recipeJob?.Outputs?.[0]?.Overwrite).toEqual(true);
135
136 // update: bump capacity on both jobs
137 yield* stack.deploy(withJobs(3));
138 const reProfile = yield* getJob(created.profileJob.jobName);
139 expect(reProfile?.MaxCapacity).toEqual(3);
140 const reRecipeJob = yield* getJob(created.recipeJob.jobName);
141 expect(reRecipeJob?.MaxCapacity).toEqual(3);
142
143 yield* stack.destroy();
144 expect(yield* getJob(created.profileJob.jobName)).toBeUndefined();
145 expect(yield* getJob(created.recipeJob.jobName)).toBeUndefined();
146 }),
147 {
148 tags: ["provider:aws", "provider:aws:databrew", "provider:aws:iam", "provider:aws:s3", "live"],
149 timeout: 180_000,
150 },
151 );
152
153 // A live DataBrew job run spins up managed Spark capacity (billed per
154 // node-hour, takes several minutes) — skipped under `--fast`.
155 test.provider.skipIf(!!process.env.FAST)(
156 "run a live recipe job to success",
157 (stack) =>
158 Effect.gen(function* () {
159 yield* stack.destroy();
160
161 const base = yield* stack.deploy(foundation);
162 yield* seedSource(base.bucket.bucketName);
163
164 const created = yield* stack.deploy(
165 Effect.gen(function* () {
166 const infra = yield* foundation;
167 const job = yield* Job("RunJob", {
168 type: "RECIPE",
169 datasetName: infra.dataset.datasetName,
170 recipeReference: { name: infra.recipe.recipeName },
171 role: infra.role.roleArn,
172 outputs: [
173 {
174 location: { bucket: infra.bucket.bucketName, key: "curated/" },
175 format: "CSV",
176 overwrite: true,
177 },
178 ],
179 maxCapacity: 2,
180 timeout: "20 minutes",
181 });
182 return { ...infra, job };
183 }),
184 );
185
186 const { RunId } = yield* databrew.startJobRun({
187 Name: created.job.jobName,
188 });
189
190 const run = yield* databrew.describeJobRun({ Name: created.job.jobName, RunId }).pipe(
191 Effect.repeat({
192 schedule: Schedule.spaced("10 seconds"),
193 until: (r) =>
194 r.State === "SUCCEEDED" ||
195 r.State === "FAILED" ||
196 r.State === "TIMEOUT" ||
197 r.State === "STOPPED",
198 times: 60,
199 }),
200 );
201 expect(run.State).toEqual("SUCCEEDED");
202
203 // the transformed output landed in S3
204 const listed = yield* s3.listObjectsV2({
205 Bucket: created.bucket.bucketName,
206 Prefix: "curated/",
207 });
208 expect((listed.KeyCount ?? 0) > 0).toBe(true);
209
210 yield* stack.destroy();
211 expect(yield* getJob(created.job.jobName)).toBeUndefined();
212 }),
213 {
214 tags: ["provider:aws", "provider:aws:databrew", "provider:aws:iam", "provider:aws:s3", "live"],
215 timeout: 900_000,
216 },
217 );