The best way to control Claude Code, Codex and more from any computer or phone

main
ts 275 lines 9.91 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import {
3 IdMappingWorkflow,
4 IdNamespace,
5 MatchingWorkflow,
6 SchemaMapping,
7 } from "@alchemy.run/aws/EntityResolution";
8 import { toTagRecord } from "@alchemy.run/aws/EntityResolution/internal";
9 import { Database, Table } from "@alchemy.run/aws/Glue";
10 import { Role } from "@alchemy.run/aws/IAM";
11 import { Bucket } from "@alchemy.run/aws/S3";
12 import { expect } from "@alchemy.run/test";
13 import * as Test from "@alchemy.run/test/Alchemy";
14 import * as entityresolution from "@distilled.cloud/aws/entityresolution";
15 import * as s3 from "@distilled.cloud/aws/s3";
16 import * as Output from "alchemy/Output";
17 import * as Effect from "effect/Effect";
18 import * as Schedule from "effect/Schedule";
19
20 const { test } = Test.make({ providers: AWS.providers() });
21
22 // Typed-error probe: proves the SDK decodes Entity Resolution errors as
23 // typed tags. A missing workflow surfaces as ResourceNotFoundException.
24 test.provider(
25 "getMatchingWorkflow on a nonexistent name fails with a typed error",
26 () =>
27 Effect.gen(function* () {
28 const error = yield* Effect.flip(
29 entityresolution.getMatchingWorkflow({ workflowName: "does-not-exist-alchemy-probe" }),
30 );
31 expect(error._tag).toBe("ResourceNotFoundException");
32 }),
33 { tags: ["provider:aws", "provider:aws:entityresolution", "live"] },
34 );
35
36 const CUSTOMER_FIELDS: entityresolution.SchemaInputAttribute[] = [
37 { fieldName: "id", type: "UNIQUE_ID" },
38 { fieldName: "email", type: "EMAIL_ADDRESS", matchKey: "email" },
39 { fieldName: "name", type: "NAME", matchKey: "name" },
40 ];
41
42 /**
43 * The role Entity Resolution assumes to read the Glue input table (and its
44 * underlying S3 data) and write matched output to S3.
45 */
46 const workflowRole = () =>
47 Role("EntityResolutionRole", {
48 assumeRolePolicyDocument: {
49 Version: "2012-10-17",
50 Statement: [
51 {
52 Effect: "Allow",
53 Principal: { Service: "entityresolution.amazonaws.com" },
54 Action: ["sts:AssumeRole"],
55 },
56 ],
57 },
58 inlinePolicies: {
59 workflow: {
60 Version: "2012-10-17",
61 Statement: [
62 {
63 Effect: "Allow",
64 Action: [
65 "glue:GetDatabase",
66 "glue:GetTable",
67 "glue:GetPartition",
68 "glue:GetPartitions",
69 "glue:GetSchema",
70 "glue:GetSchemaVersion",
71 ],
72 Resource: ["*"],
73 },
74 {
75 Effect: "Allow",
76 Action: ["s3:GetObject", "s3:ListBucket", "s3:PutObject"],
77 Resource: ["*"],
78 },
79 ],
80 },
81 },
82 });
83
84 /** The full stack under test: bucket → Glue table → schema → workflow. */
85 const workflowStack = (variant: "v1" | "v2") =>
86 Effect.gen(function* () {
87 const bucket = yield* Bucket("ErBucket", { forceDestroy: true });
88 const role = yield* workflowRole();
89 const database = yield* Database("ErDb", {});
90 const table = yield* Table("Customers", {
91 databaseName: database.databaseName,
92 storageDescriptor: {
93 location: Output.interpolate`s3://${bucket.bucketName}/input/`,
94 inputFormat: "org.apache.hadoop.mapred.TextInputFormat",
95 outputFormat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
96 serdeInfo: {
97 serializationLibrary: "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
98 parameters: { "field.delim": ",", "skip.header.line.count": "1" },
99 },
100 columns: [
101 { name: "id", type: "string" },
102 { name: "email", type: "string" },
103 { name: "name", type: "string" },
104 ],
105 },
106 parameters: { classification: "csv" },
107 });
108 const schema = yield* SchemaMapping("CustomersSchema", {
109 description: "Customer records schema",
110 mappedInputFields: CUSTOMER_FIELDS,
111 tags: { Environment: "test" },
112 });
113 const workflow = yield* MatchingWorkflow("Dedupe", {
114 description: variant === "v1" ? "Match customers by email" : "Match by email or name",
115 inputSourceConfig: [{ inputSourceARN: table.tableArn, schemaName: schema.schemaName }],
116 outputSourceConfig: [
117 {
118 outputS3Path: Output.interpolate`s3://${bucket.bucketName}/matches/`,
119 output: [{ name: "id" }, { name: "email" }, { name: "name" }],
120 },
121 ],
122 resolutionTechniques: {
123 resolutionType: "RULE_MATCHING",
124 ruleBasedProperties: {
125 rules:
126 variant === "v1"
127 ? [{ ruleName: "ByEmail", matchingKeys: ["email"] }]
128 : [
129 { ruleName: "ByEmail", matchingKeys: ["email"] },
130 { ruleName: "ByName", matchingKeys: ["name"] },
131 ],
132 attributeMatchingModel: "ONE_TO_ONE",
133 },
134 },
135 roleArn: role.roleArn,
136 tags: { Environment: "test" },
137 });
138 return { bucket, role, database, table, schema, workflow };
139 });
140
141 // Schema mappings and matching workflow DEFINITIONS are cheap and instant —
142 // the live lifecycle runs ungated with out-of-band verification. The
143 // matching RUN (StartMatchingJob) takes many minutes and is gated below.
144 test.provider(
145 "create, update, destroy a schema mapping and matching workflow",
146 (stack) =>
147 Effect.gen(function* () {
148 yield* stack.destroy();
149
150 const created = yield* stack.deploy(workflowStack("v1"));
151
152 expect(created.schema.schemaArn).toContain(":schemamapping/");
153 expect(created.workflow.workflowArn).toContain(":matchingworkflow/");
154
155 // Out-of-band verification via distilled.
156 const observedSchema = yield* entityresolution.getSchemaMapping({
157 schemaName: created.schema.schemaName,
158 });
159 expect(observedSchema.mappedInputFields).toEqual(CUSTOMER_FIELDS);
160 // Entity Resolution Get* ops do not return tags — verify via
161 // listTagsForResource.
162 const schemaTags = yield* entityresolution.listTagsForResource({
163 resourceArn: created.schema.schemaArn,
164 });
165 expect(toTagRecord(schemaTags.tags)["alchemy::id"]).toBe("CustomersSchema");
166 expect(toTagRecord(schemaTags.tags).Environment).toBe("test");
167
168 const observedWorkflow = yield* entityresolution.getMatchingWorkflow({
169 workflowName: created.workflow.workflowName,
170 });
171 expect(observedWorkflow.description).toBe("Match customers by email");
172 expect(observedWorkflow.roleArn).toBe(created.role.roleArn);
173 expect(observedWorkflow.inputSourceConfig[0]?.schemaName).toBe(created.schema.schemaName);
174 expect(
175 observedWorkflow.resolutionTechniques.ruleBasedProperties?.rules.map((r) => r.ruleName),
176 ).toEqual(["ByEmail"]);
177 const workflowTags = yield* entityresolution.listTagsForResource({
178 resourceArn: created.workflow.workflowArn,
179 });
180 expect(toTagRecord(workflowTags.tags)["alchemy::id"]).toBe("Dedupe");
181
182 // Update in place: workflow description + an extra matching rule.
183 // (The schema mapping is left untouched — it is immutable while the
184 // workflow references it.)
185 const updated = yield* stack.deploy(workflowStack("v2"));
186 expect(updated.workflow.workflowArn).toBe(created.workflow.workflowArn);
187 expect(updated.schema.schemaArn).toBe(created.schema.schemaArn);
188
189 const reobserved = yield* entityresolution.getMatchingWorkflow({
190 workflowName: created.workflow.workflowName,
191 });
192 expect(reobserved.description).toBe("Match by email or name");
193 expect(
194 reobserved.resolutionTechniques.ruleBasedProperties?.rules.map((r) => r.ruleName),
195 ).toEqual(["ByEmail", "ByName"]);
196
197 // Destroy and verify deletion out-of-band.
198 yield* stack.destroy();
199 const workflowError = yield* Effect.flip(
200 entityresolution.getMatchingWorkflow({ workflowName: created.workflow.workflowName }),
201 );
202 expect(workflowError._tag).toBe("ResourceNotFoundException");
203 const schemaError = yield* Effect.flip(
204 entityresolution.getSchemaMapping({ schemaName: created.schema.schemaName }),
205 );
206 expect(schemaError._tag).toBe("ResourceNotFoundException");
207 }),
208 {
209 tags: [
210 "provider:aws",
211 "provider:aws:entityresolution",
212 "provider:aws:glue",
213 "provider:aws:iam",
214 "provider:aws:s3",
215 "live",
216 ],
217 timeout: 300_000,
218 },
219 );
220
221 // The matching RUN processes the full input through the Entity Resolution
222 // service and takes several minutes even for a 4-row input — gated so an
223 // explicitly-opted-in run can exercise it end-to-end.
224 test.provider.skipIf(!process.env.AWS_TEST_ENTITYRESOLUTION_RUN)(
225 "run a rule-based matching job end-to-end (gated)",
226 (stack) =>
227 Effect.gen(function* () {
228 yield* stack.destroy();
229
230 const created = yield* stack.deploy(workflowStack("v1"));
231
232 // Seed input records — two obvious duplicates by email.
233 yield* s3.putObject({
234 Bucket: created.bucket.bucketName,
235 Key: "input/customers.csv",
236 Body: new TextEncoder().encode(
237 "id,email,name\n" +
238 "1,jane@example.com,Jane Doe\n" +
239 "2,jane@example.com,Jane D\n" +
240 "3,bob@example.com,Bob Roe\n" +
241 "4,alice@example.com,Alice Poe\n",
242 ),
243 ContentType: "text/csv",
244 });
245
246 const { jobId } = yield* entityresolution.startMatchingJob({
247 workflowName: created.workflow.workflowName,
248 });
249
250 const job = yield* entityresolution
251 .getMatchingJob({ workflowName: created.workflow.workflowName, jobId })
252 .pipe(
253 Effect.repeat({
254 schedule: Schedule.spaced("15 seconds"),
255 until: (j) => j.status === "SUCCEEDED" || j.status === "FAILED",
256 times: 36,
257 }),
258 );
259 expect(job.status).toBe("SUCCEEDED");
260 expect(job.metrics?.inputRecords).toBe(4);
261
262 yield* stack.destroy();
263 }),
264 {
265 tags: [
266 "provider:aws",
267 "provider:aws:entityresolution",
268 "provider:aws:glue",
269 "provider:aws:iam",
270 "provider:aws:s3",
271 "live",
272 ],
273 timeout: 900_000,
274 },
275 );