@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 411 lines 15.2 KB
Raw
1 import { mkdtemp, rm, writeFile } from "node:fs/promises";
2 import { tmpdir } from "node:os";
3 import { dirname, resolve } from "node:path";
4 import { fileURLToPath, pathToFileURL } from "node:url";
5 import { expect, it } from "@alchemy.run/test";
6
7 const here = dirname(fileURLToPath(import.meta.url));
8 const cli = resolve(here, "../bin/alchemy-test.ts");
9 const api = JSON.stringify(pathToFileURL(resolve(here, "../src/index.ts")).href);
10
11 const run = async (root: string, plan: unknown, args: string[] = []) => {
12 const child = Bun.spawn(
13 [
14 process.execPath,
15 cli,
16 root,
17 "--retry",
18 "0",
19 "--timeout",
20 "2000",
21 "--concurrency",
22 "1",
23 "--plan",
24 typeof plan === "string" ? plan : JSON.stringify(plan),
25 ...args,
26 ],
27 {
28 cwd: root,
29 stdout: "pipe",
30 stderr: "pipe",
31 env: { ...process.env, NO_COLOR: "1" },
32 },
33 );
34 const timer = setTimeout(() => child.kill(), 15_000);
35 try {
36 const [code, stdout, stderr] = await Promise.all([
37 child.exited,
38 new Response(child.stdout).text(),
39 new Response(child.stderr).text(),
40 ]);
41 return { code, output: stdout + stderr };
42 } finally {
43 clearTimeout(timer);
44 child.kill();
45 }
46 };
47
48 it(
49 "plans sequence phases, overlap branches, honor file concurrency, and claim tests once",
50 async () => {
51 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-"));
52 try {
53 await writeFile(
54 resolve(root, "state.ts"),
55 `
56 export const local = new Set();
57 export const cleaned = new Set();
58 export const live = new Set();
59 export const calls = new Set();
60 const releases = {};
61 const arrivals = {};
62 export async function barrier(name) {
63 const promise = new Promise(resolve => (releases[name] ??= []).push(resolve));
64 arrivals[name] = (arrivals[name] ?? 0) + 1;
65 if (arrivals[name] === 2) for (const release of releases[name]) release();
66 await promise;
67 }
68 `,
69 );
70 for (const name of ["a", "b"])
71 await writeFile(
72 resolve(root, `local-${name}.test.ts`),
73 `
74 import { it, expect, afterAll } from ${api};
75 import {local, cleaned, barrier} from './state.ts';
76 it('local ${name}', async () => { await barrier('local'); local.add('${name}'); }, {tags:['local']});
77 afterAll(() => { cleaned.add('${name}'); });
78 `,
79 );
80 for (const provider of ["aws", "cloudflare"])
81 await writeFile(
82 resolve(root, `${provider}.test.ts`),
83 `
84 import { it, expect } from ${api};
85 import {local, cleaned, live, calls, barrier} from './state.ts';
86 it('${provider}', async () => {
87 expect(local.size).toBe(2); expect(cleaned.size).toBe(2);
88 expect(calls.has('${provider}')).toBe(false); calls.add('${provider}');
89 await barrier('providers'); live.add('${provider}');
90 }, {tags:['live','provider:${provider}']});
91 `,
92 );
93 await writeFile(
94 resolve(root, "shared.test.ts"),
95 `
96 import { it, expect, beforeAll, afterAll } from ${api};
97 import {local, live} from './state.ts';
98 let setups = 0, early = false, late = false;
99 beforeAll(() => { setups++; expect(setups).toBe(1); });
100 afterAll(() => { expect(early).toBe(true); expect(late).toBe(true); });
101 it('early', () => { early = true; }, {tags:['local']});
102 it('remaining live', () => { expect(early).toBe(true); expect(live.size).toBe(2); late = true; }, {tags:['live']});
103 it.skip('selected skipped', () => { throw new Error('skip ignored'); }, {tags:['live']});
104 it('unmatched', () => { throw new Error('unmatched ran'); }, {tags:['manual']});
105 it('opt in', () => { throw new Error('opt-in bypassed'); }, {tags:['live'], optInTags:['paid']});
106 `,
107 );
108 const result = await run(root, [
109 { tags: ["local"], concurrency: 3 },
110 [
111 { tags: ["provider:aws", "live"], concurrency: 1 },
112 { tags: ["provider:cloudflare", "live"], concurrency: 1 },
113 ],
114 { tags: ["live"] },
115 ]);
116 expect(result.output).toContain("6 passed");
117 expect(result.output).toContain("1 skipped");
118 expect(result.output).toContain("9 found");
119 expect(result.output).toContain("7 selected by plan");
120 expect(result.output).toContain("2 excluded by plan");
121 expect(result.code).toBe(0);
122 } finally {
123 await rm(root, { recursive: true, force: true });
124 }
125 },
126 { tags: ["unit", "local"] },
127 );
128
129 it(
130 "plan branches sharing a file serialize setup and preserve it until final teardown",
131 async () => {
132 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-shared-"));
133 try {
134 await writeFile(
135 resolve(root, "shared.test.ts"),
136 `
137 import { it, expect, beforeAll, afterAll } from ${api};
138 let ready = false, setup = 0, done = 0, active = false;
139 beforeAll(() => { expect(++setup).toBe(1); ready = true; });
140 afterAll(() => { expect(done).toBe(2); ready = false; });
141 for (const tag of ['aws','cf']) it(tag, async () => {
142 expect(ready).toBe(true); expect(active).toBe(false); active = true;
143 await new Promise(resolve => setTimeout(resolve, 20));
144 active = false; done++;
145 }, {tags:[tag]});
146 `,
147 );
148 const result = await run(root, [[{ tags: ["aws"] }, { tags: ["cf"] }], { tags: [] }]);
149 expect(result.output).toContain("2 passed");
150 expect(result.output).toContain("0 excluded by plan");
151 expect(result.code).toBe(0);
152 } finally {
153 await rm(root, { recursive: true, force: true });
154 }
155 },
156 { tags: ["unit", "local"] },
157 );
158
159 it(
160 "plans do not rerun failures and still run later phases and all teardown hooks",
161 async () => {
162 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-fail-"));
163 try {
164 await writeFile(
165 resolve(root, "failure.test.ts"),
166 `
167 import { it, expect, afterAll } from ${api};
168 let attempts = 0, later = false;
169 it('failure', () => { attempts++; throw new Error('expected failure'); }, {tags:['early','live']});
170 it('later', () => { expect(attempts).toBe(1); later = true; }, {tags:['live']});
171 afterAll(() => { expect(later).toBe(true); throw new Error('teardown sentinel'); });
172 afterAll(() => console.log('cleanup sentinel'));
173 `,
174 );
175 const result = await run(root, [{ tags: ["early"] }, { tags: ["live"] }]);
176 expect(result.code).not.toBe(0);
177 expect(result.output).toContain("2 failed");
178 expect(result.output).toContain("1 passed");
179 expect(result.output).toContain("teardown sentinel");
180 // The runner writes hook output to its persistent log even on failure.
181 const { readdir, readFile } = await import("node:fs/promises");
182 const dir = resolve(root, ".alchemy/log/test");
183 const logs = await readdir(dir);
184 expect(await readFile(resolve(dir, logs[0]!), "utf8")).toContain("cleanup sentinel");
185 } finally {
186 await rm(root, { recursive: true, force: true });
187 }
188 },
189 { tags: ["unit", "local"] },
190 );
191
192 it(
193 "validates plan JSON and tag expressions before importing tests",
194 async () => {
195 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-invalid-"));
196 try {
197 await writeFile(resolve(root, "bad.test.ts"), `throw new Error('IMPORTED_SENTINEL');`);
198 for (const plan of [
199 "{",
200 [],
201 [{}],
202 [{ tags: ["unit"], concurrency: 0 }],
203 [{ tags: ["unit"], concurrency: 1.5 }],
204 [{ tags: ["unit"], concurency: 2 }],
205 [[]],
206 [[[{ tags: [] }]]],
207 [{ tags: ["unit &&"] }],
208 ]) {
209 const result = await run(root, plan);
210 expect(result.code).not.toBe(0);
211 expect(result.output).toContain("Invalid --plan");
212 expect(result.output).not.toContain("IMPORTED_SENTINEL");
213 }
214 } finally {
215 await rm(root, { recursive: true, force: true });
216 }
217 },
218 { tags: ["unit", "local"] },
219 );
220
221 it(
222 "plans compose with global tags, names and explicit opt-ins",
223 async () => {
224 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-filters-"));
225 try {
226 await writeFile(
227 resolve(root, "filters.test.ts"),
228 `
229 import { it } from ${api};
230 it('chosen', () => {}, {tags:['live','provider:aws'], optInTags:['paid']});
231 it('other name', () => { throw new Error('name bypassed'); }, {tags:['live','provider:aws'], optInTags:['paid']});
232 it('chosen cf', () => { throw new Error('global tags bypassed'); }, {tags:['live','provider:cloudflare'], optInTags:['paid']});
233 `,
234 );
235 const result = await run(
236 root,
237 [{ tags: ["paid", "live"] }],
238 ["--tags", "provider:aws", "-t", "^.* > chosen$"],
239 );
240 expect(result.output).toContain("1 passed");
241 expect(result.output).toContain("1 found");
242 expect(result.code).toBe(0);
243 } finally {
244 await rm(root, { recursive: true, force: true });
245 }
246 },
247 { tags: ["unit", "local"] },
248 );
249
250 it(
251 "failed shared setup is not retried by later phases and still tears down once",
252 async () => {
253 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-setup-"));
254 try {
255 await writeFile(
256 resolve(root, "setup.test.ts"),
257 `
258 import { it, beforeAll, afterAll, expect } from ${api};
259 let setups = 0;
260 beforeAll(() => { setups++; throw new Error('setup sentinel'); });
261 afterAll(() => { expect(setups).toBe(1); });
262 it('first', () => { throw new Error('body ran'); }, {tags:['first']});
263 it('last', () => { throw new Error('body ran'); }, {tags:['last']});
264 `,
265 );
266 const result = await run(root, [{ tags: ["first"] }, { tags: ["last"] }]);
267 expect(result.code).not.toBe(0);
268 expect(result.output).toContain("2 failed");
269 expect(result.output).toContain("setup sentinel");
270 expect(result.output).not.toContain("body ran");
271 expect(result.output).not.toContain("afterAll hook failed");
272 } finally {
273 await rm(root, { recursive: true, force: true });
274 }
275 },
276 { tags: ["unit", "local"] },
277 );
278
279 it(
280 "plans honor only and wildcard phases leave opt-in tests for an explicit later phase",
281 async () => {
282 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-only-"));
283 try {
284 await writeFile(
285 resolve(root, "only.test.ts"),
286 `
287 import { describe, it, expect, beforeAll } from ${api};
288 let ready = false;
289 describe.only('selected suite', () => {
290 it('normal', () => { ready = true; }, {tags:['live']});
291 it('opt-in', () => { expect(ready).toBe(true); }, {tags:['live'], optInTags:['paid']});
292 });
293 describe('excluded suite', () => {
294 beforeAll(() => { throw new Error('excluded setup'); });
295 it('excluded', () => { throw new Error('only bypassed'); });
296 });
297 `,
298 );
299 const result = await run(root, [{ tags: ["*"] }, { tags: ["paid"] }]);
300 expect(result.code).toBe(0);
301 expect(result.output).toContain("2 passed");
302 expect(result.output).toContain("Plan phase 1/2: 1 tests");
303 expect(result.output).toContain("Plan phase 2/2: 1 tests");
304 expect(result.output).toContain("2 found");
305 expect(result.output).toContain("0 excluded by plan");
306 } finally {
307 await rm(root, { recursive: true, force: true });
308 }
309 },
310 { tags: ["unit", "local"] },
311 );
312
313 it(
314 "unmatched plans do not run hooks but still report import errors",
315 async () => {
316 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-empty-"));
317 try {
318 await writeFile(
319 resolve(root, "good.test.ts"),
320 `
321 import { it, beforeAll, afterAll } from ${api};
322 beforeAll(() => { throw new Error('unmatched setup'); });
323 afterAll(() => { throw new Error('unmatched teardown'); });
324 it('unmatched', () => { throw new Error('unmatched body'); }, {tags:['live']});
325 `,
326 );
327 await writeFile(resolve(root, "bad.test.ts"), `throw new Error('import sentinel');`);
328 const result = await run(root, [{ tags: ["local"] }, { tags: ["unit"] }]);
329 expect(result.code).not.toBe(0);
330 expect(result.output).toContain("1 failed");
331 expect(result.output).toContain("import sentinel");
332 expect(result.output).toContain("1 excluded by plan");
333 expect(result.output).not.toContain("unmatched setup");
334 expect(result.output).not.toContain("unmatched teardown");
335 } finally {
336 await rm(root, { recursive: true, force: true });
337 }
338 },
339 { tags: ["unit", "local"] },
340 );
341
342 it(
343 "dry-run prints first-match assignments and concurrency without executing bodies or hooks",
344 async () => {
345 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-dry-"));
346 try {
347 await writeFile(
348 resolve(root, "dry.test.ts"),
349 `
350 import { it, beforeAll, afterAll } from ${api};
351 beforeAll(() => { throw new Error('DRY_SETUP_RAN'); });
352 afterAll(() => { throw new Error('DRY_TEARDOWN_RAN'); });
353 it('local aws', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['local','live','provider:aws']});
354 it('aws', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['live','provider:aws']});
355 it.skip('cf', () => {}, {tags:['live','provider:cloudflare']});
356 it('left over', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['manual']});
357 `,
358 );
359 const result = await run(
360 root,
361 [
362 { tags: ["local"], concurrency: 64 },
363 [
364 { tags: ["live", "provider:aws"], concurrency: 2 },
365 { tags: ["live", "provider:cloudflare"], concurrency: "unbounded" },
366 ],
367 { tags: ["live"] },
368 ],
369 ["--dry-run"],
370 );
371 expect(result.code).toBe(0);
372 expect(result.output).toContain("Dry run — no tests or hooks executed");
373 expect(result.output).toContain("Phase 2 (parallel)");
374 expect(result.output).toContain('tags=["local"], concurrency=64 — 1 tests in 1 files');
375 expect(result.output).toContain(
376 'tags=["live","provider:aws"], concurrency=2 — 1 tests in 1 files',
377 );
378 expect(result.output).toContain(
379 "concurrency=unbounded — 1 tests in 1 files (1 skipped/todo)",
380 );
381 expect(result.output).toContain('tags=["live"], concurrency=1 — 0 tests in 0 files');
382 expect(result.output).toContain("4 found");
383 expect(result.output).toContain("3 selected by plan");
384 expect(result.output).toContain("1 excluded by plan");
385 expect(result.output).not.toContain("DRY_SETUP_RAN");
386 expect(result.output).not.toContain("DRY_TEARDOWN_RAN");
387 expect(result.output).not.toContain("DRY_BODY_RAN");
388 expect(result.output).not.toContain("running 3 tests");
389 } finally {
390 await rm(root, { recursive: true, force: true });
391 }
392 },
393 { tags: ["unit", "local"] },
394 );
395
396 it(
397 "dry-run reports collection failures",
398 async () => {
399 const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-dry-error-"));
400 try {
401 await writeFile(resolve(root, "bad.test.ts"), `throw new Error('dry import sentinel');`);
402 const result = await run(root, [{ tags: [] }], ["--dry-run"]);
403 expect(result.code).not.toBe(0);
404 expect(result.output).toContain("dry import sentinel");
405 expect(result.output).toContain("1 failed");
406 } finally {
407 await rm(root, { recursive: true, force: true });
408 }
409 },
410 { tags: ["unit", "local"] },
411 );