@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 818 lines 24.4 KB
Raw
1 import { describe, beforeAll, afterAll, it } from 'bun:test';
2 import { $ } from 'bun';
3 import { appendFile, writeFile, mkdtemp, readFile, rm } from 'node:fs/promises';
4 import { resolve, join } from 'node:path';
5 import { tmpdir } from 'node:os';
6 import type {
7 DefineTestsContext,
8 E2EConfig,
9 E2ETestResult,
10 E2ETestResultWithSetup,
11 E2ETestResultWithFiles,
12 NodeVersionMeta,
13 CliVersionMeta,
14 NonEmptyString,
15 RunFixtureOptions,
16 RunCmdOptions,
17 RuntimeVersions,
18 RuntimeKind,
19 } from './types';
20 import {
21 getRepoRoot,
22 resolveNodeVersionMetaList,
23 resolveCliVersionMetaList,
24 } from './config';
25 import type { ExecResult } from './image-lifecycle';
26 import {
27 ensureNodeImage,
28 runNodeContainer,
29 ensureCliImage,
30 runCliContainer,
31 } from './image-lifecycle';
32 import { WELL_KNOWN_ENV_VARS } from './const';
33 import { createVolume, generateVolumeName, initializeVolumeOwnership, removeVolume } from './volume';
34
35 // ============================================================================
36 // DEBUG.log Manager - Structured logging for e2e test output
37 // ============================================================================
38
39 /**
40 * Result of a single phase execution.
41 */
42 interface PhaseResult {
43 phase: 'setup' | 'fixture';
44 phaseIndex: number;
45 totalPhases: number;
46 command: string;
47 containerName: string;
48 durationMs: number;
49 exitCode: number;
50 stdout: string;
51 stderr: string;
52 }
53
54 /**
55 * Aggregated results for a runtime version.
56 */
57 interface VersionResult {
58 runtime: RuntimeKind;
59 version: string;
60 status: 'pass' | 'fail' | 'skipped';
61 reason?: string;
62 imageTag?: string;
63 phases: PhaseResult[];
64 totalDurationMs?: number;
65 }
66
67 /**
68 * Configuration for DebugLogManager.
69 */
70 interface DebugLogManagerOptions {
71 logPath: string;
72 suiteName: string;
73 testFilePath: string;
74 runtimeVersions: string[];
75 }
76
77 /**
78 * Manages structured DEBUG.log output for e2e tests.
79 * Clears the file at initialization, groups phases by runtime version,
80 * and writes a summary at the end.
81 */
82 class DebugLogManager {
83 private versionResults: VersionResult[] = [];
84 private startTime: Date;
85 private initialized = false;
86
87 constructor(private options: DebugLogManagerOptions) {
88 this.startTime = new Date();
89 }
90
91 /**
92 * Initialize the log file - clears it and writes the header.
93 * Must be called before any version sections are written.
94 */
95 async initialize(): Promise<void> {
96 if (this.initialized) return;
97 const header = this.formatHeader();
98 await writeFile(this.options.logPath, header, 'utf-8');
99 this.initialized = true;
100 }
101
102 /**
103 * Write a version section to the log file.
104 * Call this after all phases for a version have completed.
105 */
106 async writeVersionSection(result: VersionResult): Promise<void> {
107 // Ensure initialized (lazy init if needed)
108 if (!this.initialized) {
109 await this.initialize();
110 }
111 this.versionResults.push(result);
112 const section = this.formatVersionSection(result);
113 await appendFile(this.options.logPath, section, 'utf-8');
114 }
115
116 /**
117 * Write the summary section to the log file.
118 * Call this after all versions have completed.
119 */
120 async writeSummary(): Promise<void> {
121 const summary = this.formatSummary();
122 await appendFile(this.options.logPath, summary, 'utf-8');
123 }
124
125 private formatHeader(): string {
126 const divider = '='.repeat(80);
127 return [
128 divider,
129 `E2E Test: ${this.options.suiteName}`,
130 `Started: ${this.startTime.toISOString()}`,
131 `Test file: ${this.options.testFilePath}`,
132 `Runtime versions: ${this.options.runtimeVersions.join(', ')}`,
133 divider,
134 '',
135 ].join('\n');
136 }
137
138 private formatVersionSection(result: VersionResult): string {
139 const dividerHeavy = '#'.repeat(80);
140 const runtimeLabel =
141 result.runtime === 'node' ? 'Node.js' : 'CLI';
142 const lines: string[] = [
143 '',
144 dividerHeavy,
145 `### ${runtimeLabel} ${result.version}${result.status === 'skipped' ? ' (SKIPPED)' : ''}`,
146 dividerHeavy,
147 ];
148
149 if (result.status === 'skipped') {
150 lines.push(`Reason: ${result.reason || 'No reason provided'}`);
151 } else {
152 if (result.imageTag) {
153 lines.push(`Image: ${result.imageTag}`);
154 }
155 lines.push(''); // blank line before phases
156
157 for (const phase of result.phases) {
158 lines.push(...this.formatPhase(phase));
159 }
160 }
161
162 lines.push('');
163 return lines.join('\n');
164 }
165
166 private formatPhase(phase: PhaseResult): string[] {
167 const exitStatus = phase.exitCode === 0 ? 'success' : 'failure';
168 const durationSec = (phase.durationMs / 1000).toFixed(2);
169
170 return [
171 `--- Phase ${phase.phaseIndex}/${phase.totalPhases}: ${phase.phase} ---`,
172 `Container: ${phase.containerName}`,
173 `Command: ${phase.command}`,
174 `Duration: ${durationSec}s`,
175 `Exit Code: ${phase.exitCode} (${exitStatus})`,
176 '',
177 '[stdout]',
178 phase.stdout?.trim() || '(empty)',
179 '',
180 '[stderr]',
181 phase.stderr?.trim() || '(empty)',
182 '',
183 ];
184 }
185
186 private formatSummary(): string {
187 const divider = '='.repeat(80);
188 const endTime = new Date();
189 const totalDurationMs = endTime.getTime() - this.startTime.getTime();
190 const totalDurationSec = (totalDurationMs / 1000).toFixed(2);
191
192 const lines: string[] = [
193 divider,
194 'Summary',
195 divider,
196 ];
197
198 for (const result of this.versionResults) {
199 const runtimeLabel =
200 result.runtime === 'node' ? 'Node.js' : 'CLI';
201 if (result.status === 'skipped') {
202 lines.push(`${runtimeLabel} ${result.version}: SKIPPED`);
203 } else {
204 const status = result.status.toUpperCase();
205 const phaseCount = result.phases.length;
206 const versionDurationSec = result.totalDurationMs
207 ? (result.totalDurationMs / 1000).toFixed(2)
208 : '?';
209 lines.push(`${runtimeLabel} ${result.version}: ${status} (${phaseCount} phases, ${versionDurationSec}s total)`);
210 }
211 }
212
213 lines.push('');
214 lines.push(`Finished: ${endTime.toISOString()}`);
215 lines.push(`Total duration: ${totalDurationSec}s`);
216 lines.push(divider);
217 lines.push('');
218
219 return lines.join('\n');
220 }
221 }
222
223 // ============================================================================
224 // Container environment utilities
225 // ============================================================================
226
227
228 /**
229 * Builds environment variables to pass to the container.
230 * Merges auto-passthrough vars with explicitly provided env vars.
231 */
232 function buildContainerEnv(
233 explicitEnv?: Record<string, string | undefined>
234 ): Record<string, string | undefined> {
235 const env: Record<string, string | undefined> = {};
236
237 for (const varName of WELL_KNOWN_ENV_VARS) {
238 if (process.env[varName]) {
239 env[varName] = process.env[varName];
240 }
241 }
242
243 if (explicitEnv) {
244 Object.assign(env, explicitEnv);
245 }
246
247 return env;
248 }
249
250 /**
251 * Validates that env vars passed to E2EConfig don't have undefined values.
252 * When a test passes `process.env.SOME_VAR` and it's not set, we want to
253 * fail fast with a clear error rather than silently passing undefined to Docker.
254 *
255 * @param env - Environment variables from E2EConfig
256 * @param suiteName - Test suite name for error context
257 * @throws Error if any env vars have undefined values
258 */
259 function validateRequiredEnvVars(
260 env: Record<string, string | undefined> | undefined,
261 suiteName: string
262 ): void {
263 if (!env) return;
264
265 const missingVars = Object.entries(env)
266 .filter(([, value]) => value === undefined)
267 .map(([key]) => key);
268
269 if (missingVars.length > 0) {
270 throw new Error(
271 `[${suiteName}] Missing required environment variables: ${missingVars.join(', ')}\n` +
272 `Set these variables before running the tests, or remove them from E2EConfig.env if not required.`
273 );
274 }
275 }
276
277 // ============================================================================
278 // Command execution helpers
279 // ============================================================================
280
281 type RunCmdInput = string | RunCmdOptions;
282
283 function normalizeRunCmdInput(input: RunCmdInput): { command: string; files?: string[] } {
284 if (typeof input === 'string') {
285 return { command: input };
286 }
287 return { command: input.command, files: input.files };
288 }
289
290 function toContainerCwd(cwd: string): string {
291 return cwd.startsWith('/') ? cwd : `/app/${cwd}`;
292 }
293
294 async function dockerCp(source: string, destination: string): Promise<void> {
295 const result = await $`docker cp ${source} ${destination}`.nothrow();
296 if (result.exitCode !== 0) {
297 const details = (result.stderr || result.stdout).toString().trim();
298 throw new Error(`docker cp failed: ${details || 'unknown error'}`);
299 }
300 }
301
302 async function dockerRm(containerName: string): Promise<void> {
303 await $`docker rm ${containerName}`.nothrow();
304 }
305
306 async function captureFilesFromContainer(options: {
307 containerName: string;
308 containerCwd: string;
309 files: string[];
310 }): Promise<Record<string, string>> {
311 const { containerName, containerCwd, files } = options;
312 const tempDir = await mkdtemp(join(tmpdir(), 'composio-e2e-files-'));
313 const outputs: Record<string, string> = {};
314
315 try {
316 for (const file of files) {
317 const containerPath = file.startsWith('/') ? file : `${containerCwd}/${file}`;
318 const safeName = file.replace(/^\/+/, '').replace(/[\\/]/g, '__');
319 const hostPath = join(tempDir, safeName);
320 await dockerCp(`${containerName}:${containerPath}`, hostPath);
321 outputs[file] = await readFile(hostPath, 'utf-8');
322 }
323 } finally {
324 await rm(tempDir, { recursive: true, force: true });
325 }
326
327 return outputs;
328 }
329
330 /**
331 * Internal configuration for runE2E.
332 */
333 interface RunE2EInternalConfig extends E2EConfig {
334 cwd: string;
335 suiteName: string;
336 }
337
338 /**
339 * Context for tracking phase results within a version.
340 */
341 interface VersionExecutionContext {
342 phases: PhaseResult[];
343 versionStartTime: number;
344 imageTag: string;
345 }
346
347 // ============================================================================
348 // Shared Docker Executor Factories
349 // ============================================================================
350
351 /**
352 * A function that runs a command inside a Docker container.
353 * All runtime-specific container runners (Node, CLI) share this signature
354 * for the subset of options used by `createRunCmd`.
355 */
356 type ContainerRunner = (options: {
357 imageTag: string;
358 cmd: string;
359 cwd: string;
360 env: Record<string, string | undefined>;
361 name: string;
362 remove?: boolean;
363 volumes?: Array<{ volume: string; target: string; readonly: boolean }>;
364 }) => Promise<ExecResult>;
365
366 /**
367 * Creates a `runCmd` function that executes a command in a Docker container,
368 * tracks phase results, and optionally captures files from the container.
369 */
370 function createRunCmd(options: {
371 runContainer: ContainerRunner;
372 containerPrefix: string;
373 effectiveCwd: string;
374 containerEnv: Record<string, string | undefined>;
375 imageTag: string;
376 context: VersionExecutionContext;
377 }): (input: RunCmdInput) => Promise<E2ETestResult | E2ETestResultWithFiles> {
378 const { runContainer, containerPrefix, effectiveCwd, containerEnv, imageTag, context } = options;
379
380 return async (input: RunCmdInput) => {
381 const { command, files } = normalizeRunCmdInput(input);
382 const shouldCapture = Array.isArray(files) && files.length > 0;
383 const containerName = `${containerPrefix}-${Date.now()}`;
384 const startTime = Date.now();
385
386 const result = await runContainer({
387 imageTag,
388 cmd: command,
389 cwd: effectiveCwd,
390 env: containerEnv,
391 name: containerName,
392 remove: !shouldCapture,
393 });
394
395 const durationMs = Date.now() - startTime;
396
397 context.phases.push({
398 phase: 'fixture',
399 phaseIndex: context.phases.length + 1,
400 totalPhases: context.phases.length + 1,
401 command,
402 containerName,
403 durationMs,
404 exitCode: result.exitCode,
405 stdout: result.stdout,
406 stderr: result.stderr,
407 });
408
409 if (shouldCapture) {
410 const containerCwd = toContainerCwd(effectiveCwd);
411 try {
412 const captured = await captureFilesFromContainer({
413 containerName,
414 containerCwd,
415 files: files as string[],
416 });
417 return { ...result, files: captured };
418 } finally {
419 await dockerRm(containerName);
420 }
421 }
422
423 return result;
424 };
425 }
426
427 /**
428 * Creates a `finalizeVersionLog` function that writes the version's
429 * phase results to the debug log.
430 */
431 function createFinalizeVersionLog(options: {
432 runtime: RuntimeKind;
433 version: string;
434 context: VersionExecutionContext;
435 logManager: DebugLogManager;
436 }): () => Promise<void> {
437 const { runtime, version, context, logManager } = options;
438
439 return async () => {
440 const totalDurationMs = Date.now() - context.versionStartTime;
441 const allPassed = context.phases.every(p => p.exitCode === 0);
442 const status: 'pass' | 'fail' = allPassed ? 'pass' : 'fail';
443
444 await logManager.writeVersionSection({
445 runtime,
446 version,
447 status,
448 imageTag: context.imageTag,
449 phases: context.phases,
450 totalDurationMs,
451 });
452 };
453 }
454
455 // ============================================================================
456 // Node.js Docker Executors
457 // ============================================================================
458
459 /**
460 * Creates Docker execution utilities for a specific Node.js version.
461 */
462 function createNodeDockerExecutors(
463 config: RunE2EInternalConfig,
464 nodeVersion: string,
465 imageTag: string,
466 repoRoot: string,
467 logManager: DebugLogManager
468 ) {
469 const { cwd, suiteName, env, usesFixtures } = config;
470 const containerEnv = buildContainerEnv(env);
471 const effectiveCwd = usesFixtures ? `${cwd}/fixtures` : cwd;
472
473 const context: VersionExecutionContext = {
474 phases: [],
475 versionStartTime: Date.now(),
476 imageTag,
477 };
478
479 const containerPrefix = `e2e-${suiteName}-node-${nodeVersion.replace(/\./g, '-')}`;
480
481 const runCmd = createRunCmd({
482 runContainer: runNodeContainer,
483 containerPrefix,
484 effectiveCwd,
485 containerEnv,
486 imageTag,
487 context,
488 });
489
490 /**
491 * Unified fixture runner with optional setup phase.
492 *
493 * Without setup: Runs `node <filename>` directly (no Docker volumes).
494 * With setup: Creates a Docker volume, runs setup command with volume mounted
495 * read-write, then runs the fixture with volume mounted read-only.
496 */
497 function runFixture<const F extends string>(options: { filename: NonEmptyString<F> }): Promise<E2ETestResult>;
498 function runFixture<const F extends string, const S extends string>(options: { filename: NonEmptyString<F>; setup: NonEmptyString<S> }): Promise<E2ETestResultWithSetup>;
499 async function runFixture(options: RunFixtureOptions): Promise<E2ETestResult | E2ETestResultWithSetup> {
500 const { filename, setup } = options;
501
502 if (!setup) {
503 return runCmd(`node ${filename}`);
504 }
505
506 const volumeName = generateVolumeName(suiteName, nodeVersion);
507 const containerBaseName = `${containerPrefix}-${Date.now()}`;
508 const volumeTarget = `${effectiveCwd.startsWith('/') ? effectiveCwd : `/app/${effectiveCwd}`}/node_modules`;
509
510 try {
511 await createVolume(volumeName);
512 await initializeVolumeOwnership(volumeName, imageTag);
513
514 const setupContainerName = `${containerBaseName}-setup`;
515 const setupStartTime = Date.now();
516 const setupResult = await runNodeContainer({
517 imageTag,
518 cmd: setup,
519 cwd: effectiveCwd,
520 env: containerEnv,
521 name: setupContainerName,
522 volumes: [{ volume: volumeName, target: volumeTarget, readonly: false }],
523 });
524 const setupDurationMs = Date.now() - setupStartTime;
525
526 context.phases.push({
527 phase: 'setup',
528 phaseIndex: 1,
529 totalPhases: 2,
530 command: setup,
531 containerName: setupContainerName,
532 durationMs: setupDurationMs,
533 exitCode: setupResult.exitCode,
534 stdout: setupResult.stdout,
535 stderr: setupResult.stderr,
536 });
537
538 const fixtureContainerName = `${containerBaseName}-fixture`;
539 const fixtureStartTime = Date.now();
540 const fixtureResult = await runNodeContainer({
541 imageTag,
542 cmd: `node ${filename}`,
543 cwd: effectiveCwd,
544 env: containerEnv,
545 name: fixtureContainerName,
546 volumes: [{ volume: volumeName, target: volumeTarget, readonly: true }],
547 });
548 const fixtureDurationMs = Date.now() - fixtureStartTime;
549
550 context.phases.push({
551 phase: 'fixture',
552 phaseIndex: 2,
553 totalPhases: 2,
554 command: `node ${filename}`,
555 containerName: fixtureContainerName,
556 durationMs: fixtureDurationMs,
557 exitCode: fixtureResult.exitCode,
558 stdout: fixtureResult.stdout,
559 stderr: fixtureResult.stderr,
560 });
561
562 return { ...fixtureResult, setup: setupResult };
563 } finally {
564 await removeVolume(volumeName);
565 }
566 }
567
568 const finalizeVersionLog = createFinalizeVersionLog({
569 runtime: 'node',
570 version: nodeVersion,
571 context,
572 logManager,
573 });
574
575 return { runCmd, runFixture, finalizeVersionLog, context };
576 }
577
578 // ============================================================================
579 // CLI Docker Executors
580 // ============================================================================
581
582 /**
583 * Creates Docker execution utilities for a specific CLI version.
584 */
585 function createCliDockerExecutors(
586 config: RunE2EInternalConfig,
587 cliVersion: string,
588 imageTag: string,
589 repoRoot: string,
590 logManager: DebugLogManager
591 ) {
592 const { cwd, suiteName, env, usesFixtures } = config;
593 const containerEnv = buildContainerEnv(env);
594 const effectiveCwd = usesFixtures ? `${cwd}/fixtures` : cwd;
595
596 const context: VersionExecutionContext = {
597 phases: [],
598 versionStartTime: Date.now(),
599 imageTag,
600 };
601
602 const runCmd = createRunCmd({
603 runContainer: runCliContainer,
604 containerPrefix: `e2e-${suiteName}-cli-${cliVersion.replace(/\./g, '-')}`,
605 effectiveCwd,
606 containerEnv,
607 imageTag,
608 context,
609 });
610
611 const runFixture = ((_: RunFixtureOptions) => {
612 throw new Error('runFixture is not supported for CLI runtime tests');
613 }) as DefineTestsContext['runFixture'];
614
615 const finalizeVersionLog = createFinalizeVersionLog({
616 runtime: 'cli',
617 version: cliVersion,
618 context,
619 logManager,
620 });
621
622 return { runCmd, runFixture, finalizeVersionLog, context };
623 }
624
625 // ============================================================================
626 // Version normalization and resolution
627 // ============================================================================
628
629 /**
630 * Normalize config to RuntimeVersions format.
631 */
632 function normalizeVersionConfig(config: E2EConfig): RuntimeVersions {
633 if (config.versions) {
634 return config.versions;
635 }
636 // Default: use current version for Node.js (CLI is opt-in)
637 return { node: ['current'] };
638 }
639
640 /**
641 * Render a Node.js version meta for display in test names.
642 */
643 function renderNodeVersionMeta({ kind, value }: NodeVersionMeta): string {
644 switch (kind) {
645 case 'current':
646 return `Node.js ${value} [current]`;
647 case 'overridden':
648 return `Node.js ${value} [overridden]`;
649 case 'static':
650 return `Node.js ${value}`;
651 }
652 }
653
654 /**
655 * Render a CLI version meta for display in test names.
656 */
657 function renderCliVersionMeta({ kind, value }: CliVersionMeta): string {
658 switch (kind) {
659 case 'current':
660 return `CLI ${value} [current]`;
661 case 'overridden':
662 return `CLI ${value} [overridden]`;
663 case 'static':
664 return `CLI ${value}`;
665 }
666 }
667
668 // ============================================================================
669 // Main test runner
670 // ============================================================================
671
672 /**
673 * Timeout for the beforeAll hook that ensures a runtime Docker image exists.
674 *
675 * CI pre-builds these images in a dedicated workflow step, so this hook
676 * normally only runs `docker image inspect`. Locally, however, a cold
677 * `docker build` (apt, toolchain bootstrap, full pnpm install inside the
678 * image) can take well over ten minutes on a slow network, so the ceiling
679 * must accommodate a from-scratch build.
680 */
681 const IMAGE_ENSURE_TIMEOUT_MS = 1_200_000;
682
683 /**
684 * Runs e2e tests using bun:test.
685 * Creates a describe block per runtime version and passes test utilities to defineTests.
686 *
687 * Supports Node.js and CLI runtimes.
688 * In CI mode, versions not matching their respective env vars are skipped.
689 */
690 export function runE2E(config: RunE2EInternalConfig): void {
691 const { cwd, suiteName, defineTests, env } = config;
692
693 // Validate env vars early, before any Docker operations
694 validateRequiredEnvVars(env, suiteName);
695
696 const versions = normalizeVersionConfig(config);
697 const repoRoot = getRepoRoot();
698
699 // Resolve versions for each runtime
700 const nodeVersionMetaList = versions.node ? resolveNodeVersionMetaList(versions.node) : [];
701 const cliVersionMetaList = versions.cli ? resolveCliVersionMetaList(versions.cli) : [];
702
703 // Build runtime version labels for logging
704 const allRuntimeVersions = [
705 ...nodeVersionMetaList.map(v => `Node.js ${v.value}`),
706 ...cliVersionMetaList.map(v => `CLI ${v.value}`),
707 ];
708
709 // Create the debug log manager for structured output
710 const logPath = resolve(repoRoot, cwd, 'DEBUG.log');
711 const logManager = new DebugLogManager({
712 logPath,
713 suiteName,
714 testFilePath: `${cwd}/e2e.test.ts`,
715 runtimeVersions: allRuntimeVersions,
716 });
717
718 // Wrap all tests in a top-level describe to handle initialization and summary
719 describe(suiteName, () => {
720 // Initialize log file before any tests run
721 beforeAll(async () => {
722 await logManager.initialize();
723 });
724
725 // =========== Node.js Tests ===========
726 for (const nodeVersionMeta of nodeVersionMetaList) {
727 const testName = renderNodeVersionMeta(nodeVersionMeta);
728
729 // Skip this version in CI if not selected
730 if (nodeVersionMeta.skip.value) {
731 describe(`${testName} - skipped: ${nodeVersionMeta.skip.reason}`, () => {
732 beforeAll(async () => {
733 await logManager.writeVersionSection({
734 runtime: 'node',
735 version: nodeVersionMeta.value,
736 status: 'skipped',
737 reason: nodeVersionMeta.skip.reason || 'Skipped in CI',
738 phases: [],
739 });
740 });
741
742 it.skip('this version should not run in this CI matrix job', () => {});
743 });
744 continue;
745 }
746
747 describe(testName, () => {
748 let executors: ReturnType<typeof createNodeDockerExecutors>;
749
750 beforeAll(async () => {
751 const imageTag = await ensureNodeImage(nodeVersionMeta.value, { repoRoot });
752 executors = createNodeDockerExecutors(config, nodeVersionMeta.value, imageTag, repoRoot, logManager);
753 }, IMAGE_ENSURE_TIMEOUT_MS);
754
755 defineTests({
756 runtime: 'node',
757 runCmd: ((cmd: RunCmdInput) => executors.runCmd(cmd)) as DefineTestsContext['runCmd'],
758 runFixture: ((opts: RunFixtureOptions) => executors.runFixture(opts)) as DefineTestsContext['runFixture'],
759 });
760
761 afterAll(async () => {
762 if (executors) {
763 await executors.finalizeVersionLog();
764 }
765 });
766 });
767 }
768
769 // =========== CLI Tests ===========
770 for (const cliVersionMeta of cliVersionMetaList) {
771 const testName = renderCliVersionMeta(cliVersionMeta);
772
773 // Skip this version in CI if not selected
774 if (cliVersionMeta.skip.value) {
775 describe(`${testName} - skipped: ${cliVersionMeta.skip.reason}`, () => {
776 beforeAll(async () => {
777 await logManager.writeVersionSection({
778 runtime: 'cli',
779 version: cliVersionMeta.value,
780 status: 'skipped',
781 reason: cliVersionMeta.skip.reason || 'Skipped in CI',
782 phases: [],
783 });
784 });
785
786 it('this version should not run in this CI job', () => {});
787 });
788 continue;
789 }
790
791 describe(testName, () => {
792 let executors: ReturnType<typeof createCliDockerExecutors>;
793
794 beforeAll(async () => {
795 const imageTag = await ensureCliImage(cliVersionMeta.value, { repoRoot });
796 executors = createCliDockerExecutors(config, cliVersionMeta.value, imageTag, repoRoot, logManager);
797 }, IMAGE_ENSURE_TIMEOUT_MS);
798
799 defineTests({
800 runtime: 'cli',
801 runCmd: ((cmd: RunCmdInput) => executors.runCmd(cmd)) as DefineTestsContext['runCmd'],
802 runFixture: ((opts: RunFixtureOptions) => executors.runFixture(opts)) as DefineTestsContext['runFixture'],
803 });
804
805 afterAll(async () => {
806 if (executors) {
807 await executors.finalizeVersionLog();
808 }
809 });
810 });
811 }
812
813 // Write summary after all versions complete
814 afterAll(async () => {
815 await logManager.writeSummary();
816 });
817 });
818 }