The best way to control Claude Code, Codex and more from any computer or phone

main
ts 408 lines 15.2 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { describe, expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import * as Core from "@alchemy.run/test/Core";
5 import * as frauddetector from "@distilled.cloud/aws/frauddetector";
6 import * as Effect from "effect/Effect";
7 import * as HttpClient from "effect/http/HttpClient";
8 import * as HttpClientRequest from "effect/http/HttpClientRequest";
9 import * as Schedule from "effect/Schedule";
10 import FraudDetectorTestFunctionLive, {
11 FRAUD_EMAIL,
12 FraudDetectorTestFunction,
13 REVIEW_OUTCOME,
14 SEED_BLOCKED_IP,
15 } from "./fixtures/handler";
16
17 const testOptions = { providers: AWS.providers() };
18 const { test, beforeAll, afterAll } = Test.make(testOptions);
19
20 // The standing test account has no frauddetector:* IAM grant, so every live
21 // call surfaces AccessDeniedException. The full rule-based E2E is gated behind
22 // AWS_TEST_FRAUDDETECTOR=1 and runs unchanged in an entitled account; an
23 // ungated typed-error probe proves credentials reach the service and the SDK
24 // decodes a typed error either way.
25 const RUN_LIVE = !!process.env.AWS_TEST_FRAUDDETECTOR;
26
27 // Ungated typed-error probe: proves credentials reach Fraud Detector and the
28 // SDK decodes a typed error. In an entitled account the missing detector
29 // surfaces as ResourceNotFoundException; an ungranted account surfaces
30 // AccessDeniedException. Either way the boundary is a typed tag.
31 test.provider(
32 "getDetectors on a nonexistent id fails with a typed error",
33 () =>
34 Effect.gen(function* () {
35 const error = yield* Effect.flip(
36 frauddetector.getDetectors({ detectorId: "does_not_exist_detector" }),
37 );
38 expect(["ResourceNotFoundException", "AccessDeniedException"].includes(error._tag)).toBe(
39 true,
40 );
41 }),
42 { tags: ["provider:aws", "provider:aws:frauddetector", "live"] },
43 );
44
45 // Ungated typed-error probe for the event data plane: getEvent on a
46 // nonexistent event type decodes to a typed tag (ResourceNotFoundException in
47 // an entitled account, AccessDeniedException in an ungranted one).
48 test.provider(
49 "getEvent on a nonexistent event type fails with a typed error",
50 () =>
51 Effect.gen(function* () {
52 const error = yield* Effect.flip(
53 frauddetector.getEvent({
54 eventId: "does-not-exist",
55 eventTypeName: "does_not_exist_event_type",
56 }),
57 );
58 expect(["ResourceNotFoundException", "AccessDeniedException"].includes(error._tag)).toBe(
59 true,
60 );
61 }),
62 { tags: ["provider:aws", "provider:aws:frauddetector", "live"] },
63 );
64
65 // Ungated typed-error probe for the list data plane: updateList on a
66 // nonexistent list decodes to a typed tag (ResourceNotFoundException in an
67 // entitled account, AccessDeniedException in an ungranted one).
68 test.provider(
69 "updateList on a nonexistent list fails with a typed error",
70 () =>
71 Effect.gen(function* () {
72 const error = yield* Effect.flip(
73 frauddetector.updateList({
74 name: "does_not_exist_list",
75 elements: ["203.0.113.1"],
76 updateMode: "REPLACE",
77 }),
78 );
79 expect(["ResourceNotFoundException", "AccessDeniedException"].includes(error._tag)).toBe(
80 true,
81 );
82 }),
83 { tags: ["provider:aws", "provider:aws:frauddetector", "live"] },
84 );
85
86 const sharedStack = Core.scratchStack(testOptions, "FraudDetectorPrediction");
87
88 let baseUrl: string;
89
90 describe(
91 "FraudDetector GetEventPrediction (E2E)",
92 { tags: ["provider:aws", "provider:aws:frauddetector", "provider:aws:lambda", "live"] },
93 () => {
94 beforeAll(
95 Effect.gen(function* () {
96 if (!RUN_LIVE) return;
97 yield* Effect.logInfo("FraudDetector E2E setup: destroying previous run");
98 yield* sharedStack.destroy();
99
100 yield* Effect.logInfo(
101 "FraudDetector E2E setup: deploying entity type + variables + outcome + event type + detector + active version + Lambda",
102 );
103 const { functionUrl } = yield* sharedStack.deploy(
104 Effect.gen(function* () {
105 return yield* FraudDetectorTestFunction;
106 }).pipe(Effect.provide(FraudDetectorTestFunctionLive)),
107 );
108
109 expect(functionUrl).toBeTruthy();
110 baseUrl = functionUrl!.replace(/\/+$/, "");
111
112 // Readiness probe — fresh function URLs take seconds to serve 200s.
113 yield* HttpClient.get(`${baseUrl}/health`).pipe(
114 Effect.flatMap((response) =>
115 response.status === 200
116 ? Effect.succeed(response)
117 : Effect.fail(new Error(`Function not ready: ${response.status}`)),
118 ),
119 Effect.retry({
120 schedule: Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(60)]),
121 }),
122 );
123 }),
124 { timeout: 300_000 },
125 );
126 afterAll(
127 Effect.gen(function* () {
128 if (!RUN_LIVE) return;
129 yield* sharedStack.destroy();
130 }),
131 { timeout: 300_000 },
132 );
133
134 test.provider.skipIf(!RUN_LIVE)(
135 "a fraud email is scored and returns the review outcome",
136 () =>
137 Effect.gen(function* () {
138 // The detector version may take a moment to become ACTIVE after
139 // deploy; retry through transient 5xx while it settles.
140 const response = yield* HttpClient.execute(
141 HttpClientRequest.post(`${baseUrl}/predict`).pipe(
142 HttpClientRequest.bodyJsonUnsafe({ email: FRAUD_EMAIL }),
143 ),
144 ).pipe(
145 Effect.flatMap((res) =>
146 res.status === 200
147 ? Effect.succeed(res)
148 : Effect.fail(new Error(`predict failed: ${res.status}`)),
149 ),
150 Effect.retry({
151 schedule: Schedule.max([Schedule.exponential("2 seconds"), Schedule.recurs(8)]),
152 }),
153 );
154 const body = (yield* response.json) as { outcomes: string[] };
155 expect(body.outcomes).toContain(REVIEW_OUTCOME);
156 }),
157 { timeout: 180_000 },
158 );
159
160 test.provider.skipIf(!RUN_LIVE)(
161 "a benign email matches no rule and returns no outcome",
162 () =>
163 Effect.gen(function* () {
164 const response = yield* HttpClient.execute(
165 HttpClientRequest.post(`${baseUrl}/predict`).pipe(
166 HttpClientRequest.bodyJsonUnsafe({ email: "legit@example.com" }),
167 ),
168 ).pipe(
169 Effect.flatMap((res) =>
170 res.status === 200
171 ? Effect.succeed(res)
172 : Effect.fail(new Error(`predict failed: ${res.status}`)),
173 ),
174 Effect.retry({
175 schedule: Schedule.max([Schedule.exponential("2 seconds"), Schedule.recurs(8)]),
176 }),
177 );
178 const body = (yield* response.json) as { outcomes: string[] };
179 expect(body.outcomes).not.toContain(REVIEW_OUTCOME);
180 }),
181 { timeout: 180_000 },
182 );
183
184 test.provider.skipIf(!RUN_LIVE)(
185 "a stored event can be sent, read, labeled, and deleted",
186 () =>
187 Effect.gen(function* () {
188 const eventId = "alchemy-e2e-stored-event-1";
189 const client = yield* HttpClient.HttpClient;
190
191 const post = (path: string, body: unknown) =>
192 client
193 .execute(
194 HttpClientRequest.post(`${baseUrl}${path}`).pipe(
195 HttpClientRequest.bodyJsonUnsafe(body),
196 ),
197 )
198 .pipe(
199 Effect.flatMap((res) =>
200 res.status === 200
201 ? Effect.succeed(res)
202 : Effect.fail(new Error(`${path} failed: ${res.status}`)),
203 ),
204 Effect.retry({
205 schedule: Schedule.max([Schedule.exponential("2 seconds"), Schedule.recurs(6)]),
206 }),
207 );
208
209 const readEvent = client
210 .get(`${baseUrl}/event?id=${eventId}`)
211 .pipe(
212 Effect.flatMap((res) =>
213 res.status === 200
214 ? Effect.flatMap(res.json, (body) =>
215 Effect.succeed(body as { found: boolean; currentLabel?: string }),
216 )
217 : Effect.fail(new Error(`get event failed: ${res.status}`)),
218 ),
219 );
220
221 // Ingest, then poll until the stored event is readable (ingestion is
222 // eventually consistent).
223 yield* post("/event", { eventId, email: "legit@example.com" });
224 const stored = yield* readEvent.pipe(
225 Effect.retry({
226 schedule: Schedule.max([Schedule.spaced("3 seconds"), Schedule.recurs(10)]),
227 while: (e): boolean => e instanceof Error,
228 }),
229 Effect.repeat({
230 schedule: Schedule.spaced("3 seconds"),
231 until: (body): boolean => body.found,
232 times: 10,
233 }),
234 );
235 expect(stored.found).toBe(true);
236
237 // Label the stored event, then observe the label on read-back.
238 yield* post("/event/label", { eventId });
239 const labeled = yield* readEvent.pipe(
240 Effect.repeat({
241 schedule: Schedule.spaced("3 seconds"),
242 until: (body): boolean => body.currentLabel !== undefined,
243 times: 10,
244 }),
245 );
246 expect(labeled.currentLabel).toBeTruthy();
247
248 // Delete the stored event, then observe it gone.
249 const deleted = yield* client
250 .del(`${baseUrl}/event?id=${eventId}`)
251 .pipe(Effect.flatMap((res) => Effect.succeed(res.status)));
252 expect(deleted).toBe(200);
253 const gone = yield* readEvent.pipe(
254 Effect.repeat({
255 schedule: Schedule.spaced("3 seconds"),
256 until: (body): boolean => !body.found,
257 times: 10,
258 }),
259 );
260 expect(gone.found).toBe(false);
261 }),
262 { timeout: 180_000 },
263 );
264
265 test.provider.skipIf(!RUN_LIVE)(
266 "the deny-list is readable and appendable at runtime",
267 () =>
268 Effect.gen(function* () {
269 const client = yield* HttpClient.HttpClient;
270 const readList = client
271 .get(`${baseUrl}/list`)
272 .pipe(
273 Effect.flatMap((res) =>
274 res.status === 200
275 ? Effect.flatMap(res.json, (body) =>
276 Effect.succeed(body as { elements: string[] }),
277 )
278 : Effect.fail(new Error(`get list failed: ${res.status}`)),
279 ),
280 );
281
282 // The seeded element from the List resource is readable.
283 const seeded = yield* readList.pipe(
284 Effect.retry({
285 schedule: Schedule.max([Schedule.exponential("2 seconds"), Schedule.recurs(6)]),
286 }),
287 );
288 expect(seeded.elements).toContain(SEED_BLOCKED_IP);
289
290 // Append a new element at runtime, then observe it on read-back.
291 const appended = "198.51.100.77";
292 const post = yield* client.execute(
293 HttpClientRequest.post(`${baseUrl}/list/append`).pipe(
294 HttpClientRequest.bodyJsonUnsafe({ element: appended }),
295 ),
296 );
297 expect(post.status).toBe(200);
298 const after = yield* readList.pipe(
299 Effect.repeat({
300 schedule: Schedule.spaced("3 seconds"),
301 until: (body): boolean => body.elements.includes(appended),
302 times: 10,
303 }),
304 );
305 expect(after.elements).toContain(appended);
306 expect(after.elements).toContain(SEED_BLOCKED_IP);
307 }),
308 { timeout: 180_000 },
309 );
310
311 test.provider.skipIf(!RUN_LIVE)(
312 "past predictions can be listed and audited",
313 () =>
314 Effect.gen(function* () {
315 const client = yield* HttpClient.HttpClient;
316 const eventId = "alchemy-e2e-audited-prediction-1";
317
318 // Make a prediction with a known event id.
319 yield* client
320 .execute(
321 HttpClientRequest.post(`${baseUrl}/predict`).pipe(
322 HttpClientRequest.bodyJsonUnsafe({ email: FRAUD_EMAIL, eventId }),
323 ),
324 )
325 .pipe(
326 Effect.flatMap((res) =>
327 res.status === 200
328 ? Effect.succeed(res)
329 : Effect.fail(new Error(`predict failed: ${res.status}`)),
330 ),
331 Effect.retry({
332 schedule: Schedule.max([Schedule.exponential("2 seconds"), Schedule.recurs(6)]),
333 }),
334 );
335
336 // The prediction shows up in ListEventPredictions eventually.
337 const listed = yield* client.get(`${baseUrl}/predictions?eventId=${eventId}`).pipe(
338 Effect.flatMap((res) =>
339 res.status === 200
340 ? Effect.flatMap(res.json, (body) =>
341 Effect.succeed(body as { summaries: unknown[] }),
342 )
343 : Effect.fail(new Error(`list predictions failed: ${res.status}`)),
344 ),
345 Effect.repeat({
346 schedule: Schedule.spaced("5 seconds"),
347 until: (body): boolean => body.summaries.length > 0,
348 times: 10,
349 }),
350 );
351 expect(listed.summaries.length).toBeGreaterThan(0);
352
353 // Its full evaluation metadata is auditable.
354 const audit = yield* client
355 .get(`${baseUrl}/prediction-metadata?eventId=${eventId}`)
356 .pipe(
357 Effect.flatMap((res) =>
358 res.status === 200
359 ? Effect.flatMap(res.json, (body) =>
360 Effect.succeed(
361 body as { found: boolean; outcomes: string[]; ruleCount: number },
362 ),
363 )
364 : Effect.fail(new Error(`audit failed: ${res.status}`)),
365 ),
366 );
367 expect(audit.found).toBe(true);
368 expect(audit.outcomes).toContain(REVIEW_OUTCOME);
369 expect(audit.ruleCount).toBeGreaterThan(0);
370 }),
371 { timeout: 180_000 },
372 );
373
374 // Runs LAST — the purge deletes all stored events for the event type.
375 test.provider.skipIf(!RUN_LIVE)(
376 "stored events can be bulk purged by event type",
377 () =>
378 Effect.gen(function* () {
379 const client = yield* HttpClient.HttpClient;
380
381 const purged = yield* client
382 .post(`${baseUrl}/events/purge`)
383 .pipe(
384 Effect.flatMap((res) =>
385 res.status === 200
386 ? Effect.flatMap(res.json, (body) => Effect.succeed(body as { status?: string }))
387 : Effect.fail(new Error(`purge failed: ${res.status}`)),
388 ),
389 );
390 expect(purged.status).toBeTruthy();
391
392 // The companion status call reports the async deletion job.
393 const status = yield* client.get(`${baseUrl}/events/purge-status`).pipe(
394 Effect.flatMap((res) =>
395 res.status === 200
396 ? Effect.flatMap(res.json, (body) => Effect.succeed(body as { status?: string }))
397 : Effect.fail(new Error(`purge status failed: ${res.status}`)),
398 ),
399 Effect.retry({
400 schedule: Schedule.max([Schedule.exponential("2 seconds"), Schedule.recurs(6)]),
401 }),
402 );
403 expect(status.status).toBeTruthy();
404 }),
405 { timeout: 180_000 },
406 );
407 },
408 );