@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 417 lines 15.8 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { describe, expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import * as Core from "@alchemy.run/test/Core";
5 import * as batch from "@distilled.cloud/aws/batch";
6 import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
7 import * as eventbridge from "@distilled.cloud/aws/eventbridge";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as HttpClient from "effect/http/HttpClient";
11 import * as HttpClientRequest from "effect/http/HttpClientRequest";
12 import * as Schedule from "effect/Schedule";
13 import * as Stream from "effect/Stream";
14 import BatchTestFunctionLive, { BatchTestFunction } from "./handler";
15
16 const testOptions = { providers: AWS.providers() };
17 const { test, beforeAll, afterAll } = Test.make(testOptions);
18 const sharedStack = Core.scratchStack(testOptions, "BatchBindings");
19
20 let baseUrl: string;
21 const submittedJobIds = new Set<string>();
22
23 // AWS Batch auto-creates the shared, account-level `/aws/batch/job` log group
24 // the first time a job writes logs — it is NOT deleted with the compute
25 // environment / job queue / job definition. The JobDefinition provider reaps
26 // the family's log *streams* on delete, but the group itself lingers, and a
27 // stream flushed by a job finishing while the stack is mid-destroy can land
28 // after that reap. So on teardown: delete every stream this suite caused
29 // (family names are prefixed `{stackName}-`), then delete the group itself
30 // only when zero streams remain (nothing else is using it). The Batch service
31 // recreates the group on the next job, so deleting the empty group is safe.
32 // Idempotent: every delete tolerates ResourceNotFoundException; any other
33 // error is a defect so this is a valid `Effect.ensuring` finalizer.
34 const reapBatchJobLogGroup = Core.withProviders(
35 Effect.gen(function* () {
36 const logGroupName = "/aws/batch/job";
37 const streams = yield* logs.describeLogStreams.pages({ logGroupName }).pipe(
38 Stream.runCollect,
39 Effect.map((pages) => Array.from(pages).flatMap((page) => page.logStreams ?? [])),
40 Effect.catchTag("ResourceNotFoundException", () => Effect.succeed(undefined)),
41 );
42 if (streams === undefined) return; // group never created / already reaped
43
44 const isTestOwned = (name: string | undefined): boolean =>
45 name !== undefined && name.startsWith(`${sharedStack.name}-`);
46 yield* Effect.forEach(
47 streams.filter((s) => isTestOwned(s.logStreamName)),
48 (s) =>
49 logs
50 .deleteLogStream({ logGroupName, logStreamName: s.logStreamName! })
51 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
52 { concurrency: 4 },
53 );
54
55 // Shared group: only remove it when no foreign streams remain.
56 if (streams.some((s) => !isTestOwned(s.logStreamName))) return;
57 yield* logs
58 .deleteLogGroup({ logGroupName })
59 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
60 }).pipe(Effect.orDie),
61 testOptions,
62 sharedStack.name,
63 );
64
65 class TransientUpstream extends Data.TaggedError("TransientUpstream")<{
66 readonly status: number;
67 readonly body: string;
68 }> {}
69
70 // Fresh Lambda role + Batch permissions propagate eventually — the first
71 // submits can 500 with AccessDenied under the handler's `Effect.orDie`.
72 // Retry 5xx only; a genuine 4xx fails immediately.
73 const send = (request: HttpClientRequest.HttpClientRequest) =>
74 HttpClient.execute(request).pipe(
75 Effect.flatMap((response) =>
76 response.status >= 500
77 ? response.text.pipe(
78 Effect.flatMap((body) =>
79 Effect.fail(new TransientUpstream({ status: response.status, body })),
80 ),
81 )
82 : Effect.succeed(response),
83 ),
84 Effect.retry({
85 while: (e) => e._tag === "TransientUpstream",
86 schedule: Schedule.max([Schedule.exponential("1 second"), Schedule.recurs(8)]),
87 }),
88 );
89
90 const submit = (jobName: string) =>
91 Effect.gen(function* () {
92 const response = yield* send(
93 HttpClientRequest.post(`${baseUrl}/submit`).pipe(
94 HttpClientRequest.bodyJsonUnsafe({ jobName }),
95 ),
96 );
97 expect(response.status).toBe(200);
98 const submitted = (yield* response.json) as { jobId: string; jobName: string; jobArn?: string };
99 submittedJobIds.add(submitted.jobId);
100 return submitted;
101 });
102
103 /** Out-of-band job status via distilled. */
104 const jobStatus = (jobId: string) =>
105 batch.describeJobs({ jobs: [jobId] }).pipe(Effect.map((res) => res.jobs?.[0]?.status));
106
107 const terminalJobStatuses = new Set(["SUCCEEDED", "FAILED"]);
108
109 /**
110 * Release Fargate tasks before deleting the queue and compute environment.
111 *
112 * Several binding tests intentionally stop once a submitted job is visible;
113 * leaving those jobs active makes Batch drain its managed ENIs only after
114 * stack teardown has already started. Terminating every non-terminal job here
115 * moves that asynchronous wait to the front of teardown, before the VPC graph
116 * competes for the suite's hard timeout.
117 */
118 const drainSubmittedJobs = Core.withProviders(
119 Effect.gen(function* () {
120 const jobIds = [...submittedJobIds];
121 if (jobIds.length === 0) return;
122
123 const describeSubmitted = batch
124 .describeJobs({ jobs: jobIds })
125 .pipe(Effect.map((response) => response.jobs ?? []));
126 const jobs = yield* describeSubmitted;
127 yield* Effect.forEach(
128 jobs.filter((job) => !terminalJobStatuses.has(job.status ?? "")),
129 (job) => batch.terminateJob({ jobId: job.jobId!, reason: "Batch binding test teardown" }),
130 { concurrency: 4, discard: true },
131 );
132
133 yield* describeSubmitted.pipe(
134 Effect.repeat({
135 schedule: Schedule.spaced("2 seconds"),
136 until: (observed) => observed.every((job) => terminalJobStatuses.has(job.status ?? "")),
137 times: 10,
138 }),
139 );
140 submittedJobIds.clear();
141 }).pipe(Effect.orDie),
142 testOptions,
143 sharedStack.name,
144 );
145
146 describe(
147 "Batch Bindings",
148 {
149 tags: [
150 "provider:aws",
151 "provider:aws:batch",
152 "provider:aws:ec2",
153 "provider:aws:iam",
154 "provider:aws:lambda",
155 "live",
156 ],
157 },
158 () => {
159 beforeAll(
160 Effect.gen(function* () {
161 yield* Effect.logInfo("Batch test setup: destroying previous resources");
162 yield* sharedStack.destroy();
163
164 yield* Effect.logInfo(
165 "Batch test setup: deploying CE -> queue -> job definition -> Lambda",
166 );
167 const { functionUrl } = yield* sharedStack.deploy(
168 Effect.gen(function* () {
169 return yield* BatchTestFunction;
170 }).pipe(Effect.provide(BatchTestFunctionLive)),
171 );
172
173 expect(functionUrl).toBeTruthy();
174 baseUrl = functionUrl!.replace(/\/+$/, "");
175
176 // Readiness probe — fresh function URLs take seconds (sometimes over a
177 // minute) to serve 200s.
178 yield* HttpClient.get(`${baseUrl}/status?jobId=readiness-probe`).pipe(
179 Effect.flatMap((response) =>
180 response.status === 200
181 ? Effect.succeed(response)
182 : Effect.fail(new Error(`Function not ready: ${response.status}`)),
183 ),
184 Effect.retry({
185 schedule: Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(75)]),
186 }),
187 );
188 }),
189 { timeout: 300_000 },
190 );
191 afterAll(
192 drainSubmittedJobs.pipe(
193 Effect.andThen(sharedStack.destroy()),
194 Effect.ensuring(reapBatchJobLogGroup),
195 ),
196 { timeout: 240_000 },
197 );
198
199 describe("SubmitJob", () => {
200 test.provider(
201 "submits a job that leaves the queue head (RUNNABLE or beyond)",
202 () =>
203 Effect.gen(function* () {
204 const { jobId, jobName } = yield* submit("alchemy-e2e-echo");
205 expect(jobName).toBe("alchemy-e2e-echo");
206 expect(jobId).toBeTruthy();
207
208 // Out-of-band: the job progresses past SUBMITTED/PENDING within
209 // ~2 minutes on a healthy Fargate CE (bounded poll).
210 const status = yield* jobStatus(jobId).pipe(
211 Effect.repeat({
212 schedule: Schedule.spaced("5 seconds"),
213 until: (s) =>
214 s === "RUNNABLE" ||
215 s === "STARTING" ||
216 s === "RUNNING" ||
217 s === "SUCCEEDED" ||
218 s === "FAILED",
219 times: 24,
220 }),
221 );
222 expect(["RUNNABLE", "STARTING", "RUNNING", "SUCCEEDED"]).toContain(status);
223 }),
224 { timeout: 240_000 },
225 );
226
227 // The full RUNNABLE→STARTING→SUCCEEDED round-trip takes 1-3 minutes of
228 // Fargate provisioning (~50s when capacity is free) but has been observed
229 // stuck in RUNNABLE for >7 minutes under concurrent Batch load
230 // (2026-10-09), so it stays skipped under `--fast`.
231 test.provider.skipIf(!!process.env.FAST)(
232 "submitted echo job runs to SUCCEEDED",
233 () =>
234 Effect.gen(function* () {
235 const { jobId } = yield* submit("alchemy-e2e-echo-succeed");
236 const status = yield* jobStatus(jobId).pipe(
237 Effect.repeat({
238 schedule: Schedule.spaced("10 seconds"),
239 until: (s) => s === "SUCCEEDED" || s === "FAILED",
240 times: 24,
241 }),
242 );
243 expect(status).toBe("SUCCEEDED");
244 }),
245 { timeout: 300_000 },
246 );
247 });
248
249 describe("DescribeJobs", () => {
250 test.provider(
251 "describes a submitted job from the runtime",
252 () =>
253 Effect.gen(function* () {
254 const { jobId } = yield* submit("alchemy-e2e-describe");
255
256 const response = yield* send(HttpClientRequest.get(`${baseUrl}/status?jobId=${jobId}`));
257 expect(response.status).toBe(200);
258 const body = (yield* response.json) as { status?: string; jobQueue?: string };
259 expect(body.status).toBeTruthy();
260 expect(body.jobQueue).toContain("job-queue/");
261 }),
262 { timeout: 120_000 },
263 );
264 });
265
266 describe("CancelJob", () => {
267 test.provider(
268 "cancels a queued job from the runtime",
269 () =>
270 Effect.gen(function* () {
271 const { jobId } = yield* submit("alchemy-e2e-cancel");
272
273 // Cancel immediately — the job is still SUBMITTED/PENDING/RUNNABLE
274 // (Fargate placement takes ~a minute), so cancellation applies.
275 const response = yield* send(
276 HttpClientRequest.post(`${baseUrl}/cancel`).pipe(
277 HttpClientRequest.bodyJsonUnsafe({ jobId, reason: "alchemy e2e cancel test" }),
278 ),
279 );
280 expect(response.status).toBe(200);
281 expect((yield* response.json) as object).toEqual({ cancelled: true });
282
283 // Out-of-band: a cancelled-before-starting job lands in FAILED.
284 // (SUCCEEDED only if the cancel raced past STARTING — accepted to
285 // keep the test deterministic, but FAILED is the expected path.)
286 const status = yield* jobStatus(jobId).pipe(
287 Effect.repeat({
288 schedule: Schedule.spaced("5 seconds"),
289 until: (s): boolean => s === "FAILED" || s === "SUCCEEDED",
290 times: 36,
291 }),
292 );
293 expect(["FAILED", "SUCCEEDED"]).toContain(status);
294 }),
295 { timeout: 240_000 },
296 );
297 });
298
299 describe("ListJobs", () => {
300 test.provider(
301 "lists a submitted job in the bound queue from the runtime",
302 () =>
303 Effect.gen(function* () {
304 const { jobId } = yield* submit("alchemy-e2e-list");
305
306 // The job moves through SUBMITTED→PENDING→RUNNABLE→STARTING→…;
307 // poll the runtime ListJobs across the non-terminal statuses until
308 // the submitted job shows up.
309 const findJob = Effect.gen(function* () {
310 for (const jobStatus of [
311 "SUBMITTED",
312 "PENDING",
313 "RUNNABLE",
314 "STARTING",
315 "RUNNING",
316 "SUCCEEDED",
317 ]) {
318 const response = yield* send(
319 HttpClientRequest.get(`${baseUrl}/jobs?jobStatus=${jobStatus}`),
320 );
321 expect(response.status).toBe(200);
322 const body = (yield* response.json) as {
323 jobs: { jobId: string; jobName: string }[];
324 };
325 const match = body.jobs.find((job) => job.jobId === jobId);
326 if (match) return match;
327 }
328 return undefined;
329 });
330 const found = yield* findJob.pipe(
331 Effect.repeat({
332 schedule: Schedule.spaced("5 seconds"),
333 until: (match): boolean => match !== undefined,
334 times: 24,
335 }),
336 );
337 expect(found?.jobId).toBe(jobId);
338 expect(found?.jobName).toBe("alchemy-e2e-list");
339 }),
340 { timeout: 240_000 },
341 );
342 });
343
344 describe("GetJobQueueSnapshot", () => {
345 test.provider(
346 "snapshots the front of the bound queue from the runtime",
347 () =>
348 Effect.gen(function* () {
349 // The queue-ARN-scoped IAM grant + injected `jobQueue` selector:
350 // a 200 with a jobs array proves the call path end to end (the
351 // snapshot only surfaces RUNNABLE jobs, so membership is racy and
352 // not asserted).
353 const response = yield* send(HttpClientRequest.get(`${baseUrl}/snapshot`));
354 expect(response.status).toBe(200);
355 const body = (yield* response.json) as { jobs: unknown };
356 expect(Array.isArray(body.jobs)).toBe(true);
357 }),
358 { timeout: 120_000 },
359 );
360 });
361
362 describe("consumeJobEvents", { tags: ["provider:aws:eventbridge"] }, () => {
363 test.provider(
364 "created the EventBridge rule for batch job state changes",
365 () =>
366 Effect.gen(function* () {
367 // The rule's physical name embeds the fixture's logical id
368 // (`BatchTestFunction-BatchJobEvents`); find it on the default bus
369 // with bounded manual pagination.
370 let rule: eventbridge.Rule | undefined;
371 let nextToken: string | undefined;
372 for (let page = 0; page < 10 && !rule; page++) {
373 const result = yield* eventbridge.listRules({ NextToken: nextToken });
374 rule = (result.Rules ?? []).find((candidate) =>
375 candidate.Name?.includes("BatchJobEvents"),
376 );
377 nextToken = result.NextToken;
378 if (!nextToken) break;
379 }
380 expect(rule).toBeDefined();
381 expect(rule?.EventPattern).toContain("aws.batch");
382 expect(rule?.EventPattern).toContain("Batch Job State Change");
383 }),
384 { timeout: 60_000 },
385 );
386 });
387
388 describe("TerminateJob", () => {
389 test.provider(
390 "terminates a submitted job from the runtime",
391 () =>
392 Effect.gen(function* () {
393 const { jobId } = yield* submit("alchemy-e2e-terminate");
394
395 const response = yield* send(
396 HttpClientRequest.post(`${baseUrl}/terminate`).pipe(
397 HttpClientRequest.bodyJsonUnsafe({ jobId, reason: "alchemy e2e terminate test" }),
398 ),
399 );
400 expect(response.status).toBe(200);
401 expect((yield* response.json) as object).toEqual({ terminated: true });
402
403 // Out-of-band: a job terminated before running lands in FAILED.
404 const status = yield* jobStatus(jobId).pipe(
405 Effect.repeat({
406 schedule: Schedule.spaced("5 seconds"),
407 until: (s) => s === "FAILED" || s === "SUCCEEDED",
408 times: 24,
409 }),
410 );
411 expect(status).toBe("FAILED");
412 }),
413 { timeout: 240_000 },
414 );
415 });
416 },
417 );