The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import { mkdtemp, rm, writeFile } from "node:fs/promises"; |
| 2 | import { tmpdir } from "node:os"; |
| 3 | import { dirname, resolve } from "node:path"; |
| 4 | import { fileURLToPath, pathToFileURL } from "node:url"; |
| 5 | import { expect, it } from "@alchemy.run/test"; |
| 6 | |
| 7 | const here = dirname(fileURLToPath(import.meta.url)); |
| 8 | const cli = resolve(here, "../bin/alchemy-test.ts"); |
| 9 | const api = JSON.stringify(pathToFileURL(resolve(here, "../src/index.ts")).href); |
| 10 | |
| 11 | const run = async (root: string, plan: unknown, args: string[] = []) => { |
| 12 | const child = Bun.spawn( |
| 13 | [ |
| 14 | process.execPath, |
| 15 | cli, |
| 16 | root, |
| 17 | "--retry", |
| 18 | "0", |
| 19 | "--timeout", |
| 20 | "2000", |
| 21 | "--concurrency", |
| 22 | "1", |
| 23 | "--plan", |
| 24 | typeof plan === "string" ? plan : JSON.stringify(plan), |
| 25 | ...args, |
| 26 | ], |
| 27 | { |
| 28 | cwd: root, |
| 29 | stdout: "pipe", |
| 30 | stderr: "pipe", |
| 31 | env: { ...process.env, NO_COLOR: "1" }, |
| 32 | }, |
| 33 | ); |
| 34 | const timer = setTimeout(() => child.kill(), 15_000); |
| 35 | try { |
| 36 | const [code, stdout, stderr] = await Promise.all([ |
| 37 | child.exited, |
| 38 | new Response(child.stdout).text(), |
| 39 | new Response(child.stderr).text(), |
| 40 | ]); |
| 41 | return { code, output: stdout + stderr }; |
| 42 | } finally { |
| 43 | clearTimeout(timer); |
| 44 | child.kill(); |
| 45 | } |
| 46 | }; |
| 47 | |
| 48 | it( |
| 49 | "plans sequence phases, overlap branches, honor file concurrency, and claim tests once", |
| 50 | async () => { |
| 51 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-")); |
| 52 | try { |
| 53 | await writeFile( |
| 54 | resolve(root, "state.ts"), |
| 55 | ` |
| 56 | export const local = new Set(); |
| 57 | export const cleaned = new Set(); |
| 58 | export const live = new Set(); |
| 59 | export const calls = new Set(); |
| 60 | const releases = {}; |
| 61 | const arrivals = {}; |
| 62 | export async function barrier(name) { |
| 63 | const promise = new Promise(resolve => (releases[name] ??= []).push(resolve)); |
| 64 | arrivals[name] = (arrivals[name] ?? 0) + 1; |
| 65 | if (arrivals[name] === 2) for (const release of releases[name]) release(); |
| 66 | await promise; |
| 67 | } |
| 68 | `, |
| 69 | ); |
| 70 | for (const name of ["a", "b"]) |
| 71 | await writeFile( |
| 72 | resolve(root, `local-${name}.test.ts`), |
| 73 | ` |
| 74 | import { it, expect, afterAll } from ${api}; |
| 75 | import {local, cleaned, barrier} from './state.ts'; |
| 76 | it('local ${name}', async () => { await barrier('local'); local.add('${name}'); }, {tags:['local']}); |
| 77 | afterAll(() => { cleaned.add('${name}'); }); |
| 78 | `, |
| 79 | ); |
| 80 | for (const provider of ["aws", "cloudflare"]) |
| 81 | await writeFile( |
| 82 | resolve(root, `${provider}.test.ts`), |
| 83 | ` |
| 84 | import { it, expect } from ${api}; |
| 85 | import {local, cleaned, live, calls, barrier} from './state.ts'; |
| 86 | it('${provider}', async () => { |
| 87 | expect(local.size).toBe(2); expect(cleaned.size).toBe(2); |
| 88 | expect(calls.has('${provider}')).toBe(false); calls.add('${provider}'); |
| 89 | await barrier('providers'); live.add('${provider}'); |
| 90 | }, {tags:['live','provider:${provider}']}); |
| 91 | `, |
| 92 | ); |
| 93 | await writeFile( |
| 94 | resolve(root, "shared.test.ts"), |
| 95 | ` |
| 96 | import { it, expect, beforeAll, afterAll } from ${api}; |
| 97 | import {local, live} from './state.ts'; |
| 98 | let setups = 0, early = false, late = false; |
| 99 | beforeAll(() => { setups++; expect(setups).toBe(1); }); |
| 100 | afterAll(() => { expect(early).toBe(true); expect(late).toBe(true); }); |
| 101 | it('early', () => { early = true; }, {tags:['local']}); |
| 102 | it('remaining live', () => { expect(early).toBe(true); expect(live.size).toBe(2); late = true; }, {tags:['live']}); |
| 103 | it.skip('selected skipped', () => { throw new Error('skip ignored'); }, {tags:['live']}); |
| 104 | it('unmatched', () => { throw new Error('unmatched ran'); }, {tags:['manual']}); |
| 105 | it('opt in', () => { throw new Error('opt-in bypassed'); }, {tags:['live'], optInTags:['paid']}); |
| 106 | `, |
| 107 | ); |
| 108 | const result = await run(root, [ |
| 109 | { tags: ["local"], concurrency: 3 }, |
| 110 | [ |
| 111 | { tags: ["provider:aws", "live"], concurrency: 1 }, |
| 112 | { tags: ["provider:cloudflare", "live"], concurrency: 1 }, |
| 113 | ], |
| 114 | { tags: ["live"] }, |
| 115 | ]); |
| 116 | expect(result.output).toContain("6 passed"); |
| 117 | expect(result.output).toContain("1 skipped"); |
| 118 | expect(result.output).toContain("9 found"); |
| 119 | expect(result.output).toContain("7 selected by plan"); |
| 120 | expect(result.output).toContain("2 excluded by plan"); |
| 121 | expect(result.code).toBe(0); |
| 122 | } finally { |
| 123 | await rm(root, { recursive: true, force: true }); |
| 124 | } |
| 125 | }, |
| 126 | { tags: ["unit", "local"] }, |
| 127 | ); |
| 128 | |
| 129 | it( |
| 130 | "plan branches sharing a file serialize setup and preserve it until final teardown", |
| 131 | async () => { |
| 132 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-shared-")); |
| 133 | try { |
| 134 | await writeFile( |
| 135 | resolve(root, "shared.test.ts"), |
| 136 | ` |
| 137 | import { it, expect, beforeAll, afterAll } from ${api}; |
| 138 | let ready = false, setup = 0, done = 0, active = false; |
| 139 | beforeAll(() => { expect(++setup).toBe(1); ready = true; }); |
| 140 | afterAll(() => { expect(done).toBe(2); ready = false; }); |
| 141 | for (const tag of ['aws','cf']) it(tag, async () => { |
| 142 | expect(ready).toBe(true); expect(active).toBe(false); active = true; |
| 143 | await new Promise(resolve => setTimeout(resolve, 20)); |
| 144 | active = false; done++; |
| 145 | }, {tags:[tag]}); |
| 146 | `, |
| 147 | ); |
| 148 | const result = await run(root, [[{ tags: ["aws"] }, { tags: ["cf"] }], { tags: [] }]); |
| 149 | expect(result.output).toContain("2 passed"); |
| 150 | expect(result.output).toContain("0 excluded by plan"); |
| 151 | expect(result.code).toBe(0); |
| 152 | } finally { |
| 153 | await rm(root, { recursive: true, force: true }); |
| 154 | } |
| 155 | }, |
| 156 | { tags: ["unit", "local"] }, |
| 157 | ); |
| 158 | |
| 159 | it( |
| 160 | "plans do not rerun failures and still run later phases and all teardown hooks", |
| 161 | async () => { |
| 162 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-fail-")); |
| 163 | try { |
| 164 | await writeFile( |
| 165 | resolve(root, "failure.test.ts"), |
| 166 | ` |
| 167 | import { it, expect, afterAll } from ${api}; |
| 168 | let attempts = 0, later = false; |
| 169 | it('failure', () => { attempts++; throw new Error('expected failure'); }, {tags:['early','live']}); |
| 170 | it('later', () => { expect(attempts).toBe(1); later = true; }, {tags:['live']}); |
| 171 | afterAll(() => { expect(later).toBe(true); throw new Error('teardown sentinel'); }); |
| 172 | afterAll(() => console.log('cleanup sentinel')); |
| 173 | `, |
| 174 | ); |
| 175 | const result = await run(root, [{ tags: ["early"] }, { tags: ["live"] }]); |
| 176 | expect(result.code).not.toBe(0); |
| 177 | expect(result.output).toContain("2 failed"); |
| 178 | expect(result.output).toContain("1 passed"); |
| 179 | expect(result.output).toContain("teardown sentinel"); |
| 180 | // The runner writes hook output to its persistent log even on failure. |
| 181 | const { readdir, readFile } = await import("node:fs/promises"); |
| 182 | const dir = resolve(root, ".alchemy/log/test"); |
| 183 | const logs = await readdir(dir); |
| 184 | expect(await readFile(resolve(dir, logs[0]!), "utf8")).toContain("cleanup sentinel"); |
| 185 | } finally { |
| 186 | await rm(root, { recursive: true, force: true }); |
| 187 | } |
| 188 | }, |
| 189 | { tags: ["unit", "local"] }, |
| 190 | ); |
| 191 | |
| 192 | it( |
| 193 | "validates plan JSON and tag expressions before importing tests", |
| 194 | async () => { |
| 195 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-invalid-")); |
| 196 | try { |
| 197 | await writeFile(resolve(root, "bad.test.ts"), `throw new Error('IMPORTED_SENTINEL');`); |
| 198 | for (const plan of [ |
| 199 | "{", |
| 200 | [], |
| 201 | [{}], |
| 202 | [{ tags: ["unit"], concurrency: 0 }], |
| 203 | [{ tags: ["unit"], concurrency: 1.5 }], |
| 204 | [{ tags: ["unit"], concurency: 2 }], |
| 205 | [[]], |
| 206 | [[[{ tags: [] }]]], |
| 207 | [{ tags: ["unit &&"] }], |
| 208 | ]) { |
| 209 | const result = await run(root, plan); |
| 210 | expect(result.code).not.toBe(0); |
| 211 | expect(result.output).toContain("Invalid --plan"); |
| 212 | expect(result.output).not.toContain("IMPORTED_SENTINEL"); |
| 213 | } |
| 214 | } finally { |
| 215 | await rm(root, { recursive: true, force: true }); |
| 216 | } |
| 217 | }, |
| 218 | { tags: ["unit", "local"] }, |
| 219 | ); |
| 220 | |
| 221 | it( |
| 222 | "plans compose with global tags, names and explicit opt-ins", |
| 223 | async () => { |
| 224 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-filters-")); |
| 225 | try { |
| 226 | await writeFile( |
| 227 | resolve(root, "filters.test.ts"), |
| 228 | ` |
| 229 | import { it } from ${api}; |
| 230 | it('chosen', () => {}, {tags:['live','provider:aws'], optInTags:['paid']}); |
| 231 | it('other name', () => { throw new Error('name bypassed'); }, {tags:['live','provider:aws'], optInTags:['paid']}); |
| 232 | it('chosen cf', () => { throw new Error('global tags bypassed'); }, {tags:['live','provider:cloudflare'], optInTags:['paid']}); |
| 233 | `, |
| 234 | ); |
| 235 | const result = await run( |
| 236 | root, |
| 237 | [{ tags: ["paid", "live"] }], |
| 238 | ["--tags", "provider:aws", "-t", "^.* > chosen$"], |
| 239 | ); |
| 240 | expect(result.output).toContain("1 passed"); |
| 241 | expect(result.output).toContain("1 found"); |
| 242 | expect(result.code).toBe(0); |
| 243 | } finally { |
| 244 | await rm(root, { recursive: true, force: true }); |
| 245 | } |
| 246 | }, |
| 247 | { tags: ["unit", "local"] }, |
| 248 | ); |
| 249 | |
| 250 | it( |
| 251 | "failed shared setup is not retried by later phases and still tears down once", |
| 252 | async () => { |
| 253 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-setup-")); |
| 254 | try { |
| 255 | await writeFile( |
| 256 | resolve(root, "setup.test.ts"), |
| 257 | ` |
| 258 | import { it, beforeAll, afterAll, expect } from ${api}; |
| 259 | let setups = 0; |
| 260 | beforeAll(() => { setups++; throw new Error('setup sentinel'); }); |
| 261 | afterAll(() => { expect(setups).toBe(1); }); |
| 262 | it('first', () => { throw new Error('body ran'); }, {tags:['first']}); |
| 263 | it('last', () => { throw new Error('body ran'); }, {tags:['last']}); |
| 264 | `, |
| 265 | ); |
| 266 | const result = await run(root, [{ tags: ["first"] }, { tags: ["last"] }]); |
| 267 | expect(result.code).not.toBe(0); |
| 268 | expect(result.output).toContain("2 failed"); |
| 269 | expect(result.output).toContain("setup sentinel"); |
| 270 | expect(result.output).not.toContain("body ran"); |
| 271 | expect(result.output).not.toContain("afterAll hook failed"); |
| 272 | } finally { |
| 273 | await rm(root, { recursive: true, force: true }); |
| 274 | } |
| 275 | }, |
| 276 | { tags: ["unit", "local"] }, |
| 277 | ); |
| 278 | |
| 279 | it( |
| 280 | "plans honor only and wildcard phases leave opt-in tests for an explicit later phase", |
| 281 | async () => { |
| 282 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-only-")); |
| 283 | try { |
| 284 | await writeFile( |
| 285 | resolve(root, "only.test.ts"), |
| 286 | ` |
| 287 | import { describe, it, expect, beforeAll } from ${api}; |
| 288 | let ready = false; |
| 289 | describe.only('selected suite', () => { |
| 290 | it('normal', () => { ready = true; }, {tags:['live']}); |
| 291 | it('opt-in', () => { expect(ready).toBe(true); }, {tags:['live'], optInTags:['paid']}); |
| 292 | }); |
| 293 | describe('excluded suite', () => { |
| 294 | beforeAll(() => { throw new Error('excluded setup'); }); |
| 295 | it('excluded', () => { throw new Error('only bypassed'); }); |
| 296 | }); |
| 297 | `, |
| 298 | ); |
| 299 | const result = await run(root, [{ tags: ["*"] }, { tags: ["paid"] }]); |
| 300 | expect(result.code).toBe(0); |
| 301 | expect(result.output).toContain("2 passed"); |
| 302 | expect(result.output).toContain("Plan phase 1/2: 1 tests"); |
| 303 | expect(result.output).toContain("Plan phase 2/2: 1 tests"); |
| 304 | expect(result.output).toContain("2 found"); |
| 305 | expect(result.output).toContain("0 excluded by plan"); |
| 306 | } finally { |
| 307 | await rm(root, { recursive: true, force: true }); |
| 308 | } |
| 309 | }, |
| 310 | { tags: ["unit", "local"] }, |
| 311 | ); |
| 312 | |
| 313 | it( |
| 314 | "unmatched plans do not run hooks but still report import errors", |
| 315 | async () => { |
| 316 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-empty-")); |
| 317 | try { |
| 318 | await writeFile( |
| 319 | resolve(root, "good.test.ts"), |
| 320 | ` |
| 321 | import { it, beforeAll, afterAll } from ${api}; |
| 322 | beforeAll(() => { throw new Error('unmatched setup'); }); |
| 323 | afterAll(() => { throw new Error('unmatched teardown'); }); |
| 324 | it('unmatched', () => { throw new Error('unmatched body'); }, {tags:['live']}); |
| 325 | `, |
| 326 | ); |
| 327 | await writeFile(resolve(root, "bad.test.ts"), `throw new Error('import sentinel');`); |
| 328 | const result = await run(root, [{ tags: ["local"] }, { tags: ["unit"] }]); |
| 329 | expect(result.code).not.toBe(0); |
| 330 | expect(result.output).toContain("1 failed"); |
| 331 | expect(result.output).toContain("import sentinel"); |
| 332 | expect(result.output).toContain("1 excluded by plan"); |
| 333 | expect(result.output).not.toContain("unmatched setup"); |
| 334 | expect(result.output).not.toContain("unmatched teardown"); |
| 335 | } finally { |
| 336 | await rm(root, { recursive: true, force: true }); |
| 337 | } |
| 338 | }, |
| 339 | { tags: ["unit", "local"] }, |
| 340 | ); |
| 341 | |
| 342 | it( |
| 343 | "dry-run prints first-match assignments and concurrency without executing bodies or hooks", |
| 344 | async () => { |
| 345 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-dry-")); |
| 346 | try { |
| 347 | await writeFile( |
| 348 | resolve(root, "dry.test.ts"), |
| 349 | ` |
| 350 | import { it, beforeAll, afterAll } from ${api}; |
| 351 | beforeAll(() => { throw new Error('DRY_SETUP_RAN'); }); |
| 352 | afterAll(() => { throw new Error('DRY_TEARDOWN_RAN'); }); |
| 353 | it('local aws', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['local','live','provider:aws']}); |
| 354 | it('aws', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['live','provider:aws']}); |
| 355 | it.skip('cf', () => {}, {tags:['live','provider:cloudflare']}); |
| 356 | it('left over', () => { throw new Error('DRY_BODY_RAN'); }, {tags:['manual']}); |
| 357 | `, |
| 358 | ); |
| 359 | const result = await run( |
| 360 | root, |
| 361 | [ |
| 362 | { tags: ["local"], concurrency: 64 }, |
| 363 | [ |
| 364 | { tags: ["live", "provider:aws"], concurrency: 2 }, |
| 365 | { tags: ["live", "provider:cloudflare"], concurrency: "unbounded" }, |
| 366 | ], |
| 367 | { tags: ["live"] }, |
| 368 | ], |
| 369 | ["--dry-run"], |
| 370 | ); |
| 371 | expect(result.code).toBe(0); |
| 372 | expect(result.output).toContain("Dry run — no tests or hooks executed"); |
| 373 | expect(result.output).toContain("Phase 2 (parallel)"); |
| 374 | expect(result.output).toContain('tags=["local"], concurrency=64 — 1 tests in 1 files'); |
| 375 | expect(result.output).toContain( |
| 376 | 'tags=["live","provider:aws"], concurrency=2 — 1 tests in 1 files', |
| 377 | ); |
| 378 | expect(result.output).toContain( |
| 379 | "concurrency=unbounded — 1 tests in 1 files (1 skipped/todo)", |
| 380 | ); |
| 381 | expect(result.output).toContain('tags=["live"], concurrency=1 — 0 tests in 0 files'); |
| 382 | expect(result.output).toContain("4 found"); |
| 383 | expect(result.output).toContain("3 selected by plan"); |
| 384 | expect(result.output).toContain("1 excluded by plan"); |
| 385 | expect(result.output).not.toContain("DRY_SETUP_RAN"); |
| 386 | expect(result.output).not.toContain("DRY_TEARDOWN_RAN"); |
| 387 | expect(result.output).not.toContain("DRY_BODY_RAN"); |
| 388 | expect(result.output).not.toContain("running 3 tests"); |
| 389 | } finally { |
| 390 | await rm(root, { recursive: true, force: true }); |
| 391 | } |
| 392 | }, |
| 393 | { tags: ["unit", "local"] }, |
| 394 | ); |
| 395 | |
| 396 | it( |
| 397 | "dry-run reports collection failures", |
| 398 | async () => { |
| 399 | const root = await mkdtemp(resolve(tmpdir(), "alchemy-plan-dry-error-")); |
| 400 | try { |
| 401 | await writeFile(resolve(root, "bad.test.ts"), `throw new Error('dry import sentinel');`); |
| 402 | const result = await run(root, [{ tags: [] }], ["--dry-run"]); |
| 403 | expect(result.code).not.toBe(0); |
| 404 | expect(result.output).toContain("dry import sentinel"); |
| 405 | expect(result.output).toContain("1 failed"); |
| 406 | } finally { |
| 407 | await rm(root, { recursive: true, force: true }); |
| 408 | } |
| 409 | }, |
| 410 | { tags: ["unit", "local"] }, |
| 411 | ); |