@devries-cameron20 / composio Mirror · read-only / commits / 5519752ba
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 22, 08:15 UTC

feat(ts/e2e): add CLI runtime support and `composio version` e2e test

Extend the e2e test infrastructure to support CLI runtime tests alongside Node.js and Deno. Add a Dockerfile.cli that builds the composio binary and runs it in a scratch Debian container. Include the first CLI e2e test for `composio version` with file capture support.

jkomyno committed Feb 18, 2026 at 15:20 UTC 5519752ba4b6f9c031844f0364490b3b2799c1cf
19 files changed +972 -42
.github/workflows/ts.test-e2e.yml
+38
@@ -92,6 +92,44 @@ jobs:
92 - name: Run Deno E2E Tests
93 run: pnpm run test:e2e:deno
94
95 + # CLI E2E tests, running in scratch Docker to isolate environment
96 + test-e2e-cli-docker:
97 + name: E2E Test CLI (scratch)
98 + runs-on: ubuntu-24.04
99 + permissions: read-all
100 + env:
101 + CI: 1
102 + COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
103 + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
104 +
105 + steps:
106 + - name: Checkout Code
107 + uses: actions/checkout@v6
108 +
109 + - name: Setup Node.js, pnpm, Bun
110 + uses: ./.github/actions/setup-node-pnpm-bun
111 +
112 + - name: Install Dependencies
113 + run: pnpm install --frozen-lockfile
114 +
115 + - name: Build
116 + run: pnpm run build:packages
117 +
118 + - name: Pre-build Docker image for CLI e2e tests
119 + run: |
120 + CLI_VERSION=$(node -p "require('./ts/packages/cli/package.json').version")
121 + echo "COMPOSIO_E2E_CLI_VERSION=${CLI_VERSION}" >> "$GITHUB_ENV"
122 + docker build -f ts/e2e-tests/_utils/Dockerfile.cli \
123 + --build-arg CLI_VERSION="${CLI_VERSION}" \
124 + --label composio.e2e=true \
125 + --label composio.runtime=cli \
126 + --label "composio.cli_version=${CLI_VERSION}" \
127 + -t "composio-e2e-cli:${CLI_VERSION}" \
128 + .
129 +
130 + - name: Run CLI E2E Tests
131 + run: pnpm run test:e2e:cli
132 +
133 # Cloudflare Workerd E2E tests
134 test-e2e-cloudflare:
135 name: E2E Test Cloudflare Workerd
package.json
+1
@@ -16,6 +16,7 @@
16 "test:e2e": "turbo test:e2e --filter='@e2e-tests/*'",
17 "test:e2e:node": "turbo test:e2e:node --filter='@e2e-tests/node-*'",
18 "test:e2e:deno": "turbo test:e2e:deno --filter='@e2e-tests/deno-*'",
19 + "test:e2e:cli": "turbo test:e2e:cli --filter='@e2e-tests/cli-*'",
20 "test:e2e:cloudflare": "turbo test:e2e:cloudflare --filter='@e2e-tests/cf-*'",
21 "test:ui": "vitest --ui",
22 "typecheck": "turbo typecheck --filter=./ts/packages/**",
pnpm-lock.yaml
+6
@@ -173,6 +173,12 @@ importers:
173 specifier: 'catalog:'
174 version: 5.9.3
175
176 + ts/e2e-tests/cli/version:
177 + devDependencies:
178 + '@e2e-tests/utils':
179 + specifier: workspace:*
180 + version: link:../../_utils
181 +
182 ts/e2e-tests/runtimes/cloudflare/cf-workers-basic:
183 dependencies:
184 '@composio/core':
pnpm-workspace.yaml
+1
@@ -1,5 +1,6 @@
1 packages:
2 - ts/e2e-tests/_utils
3 + - ts/e2e-tests/cli/*
4 - ts/e2e-tests/runtimes/node/*
5 - ts/e2e-tests/runtimes/deno/*
6 - ts/e2e-tests/runtimes/cloudflare/*
ts/e2e-tests/README.md
+54 -3
@@ -1,6 +1,6 @@
1 # E2E Tests
2
3 -End-to-end tests for `@composio/core` across different JavaScript runtimes.
3 +End-to-end tests for `@composio/core` and the CLI across different runtimes.
4
5 ## Directory Structure
6
@@ -9,8 +9,9 @@ ts/e2e-tests/
9 ├── _utils/ # Shared test infrastructure
10 │ ├── Dockerfile.node # Docker image for Node.js tests
11 │ ├── Dockerfile.deno # Docker image for Deno tests
12 +│ ├── Dockerfile.cli # Docker image for CLI tests (scratch)
13 │ ├── scripts/ # Docker build and cleanup scripts
13 -│ │ ├── docker-build.ts # Pre-build images for all Node/Deno versions
14 +│ │ ├── docker-build.ts # Pre-build images for all Node/Deno/CLI versions
15 │ │ └── docker-clean.ts # Remove all e2e Docker images
16 │ ├── src/ # TypeScript runner utilities
17 │ │ ├── config.ts # Configuration utilities
@@ -23,7 +24,7 @@ ts/e2e-tests/
24 │ │ ├── types.ts # TypeScript type definitions
25 │ │ └── volume.ts # Docker volume management
26 │ └── README.md # Utils documentation
26 -└── runtimes/
27 +├── runtimes/
28 ├── node/ # Node.js runtime tests
29 │ ├── cjs-basic/ # CommonJS compatibility tests
30 │ ├── esm-basic/ # ESM compatibility tests
@@ -39,6 +40,8 @@ ts/e2e-tests/
40 ├── cf-workers-basic/ # Basic Cloudflare Workers tests
41 ├── cf-workers-files/ # Cloudflare Workers file handling tests
42 └── cf-workers-tool-router-ai/ # Cloudflare Workers AI SDK tool router tests
43 +└── cli/ # CLI runtime tests (scratch)
44 + └── version/ # composio version command tests
45 ```
46
47 ## Running Tests
@@ -83,6 +86,18 @@ COMPOSIO_E2E_DENO_VERSION=2.6.7 pnpm test:e2e:deno
86 pnpm test:e2e:cloudflare
87 ```
88
89 +### CLI Tests Only
90 +
91 +```bash
92 +pnpm test:e2e:cli
93 +```
94 +
95 +To run with a specific CLI version:
96 +
97 +```bash
98 +COMPOSIO_E2E_CLI_VERSION=0.1.24 pnpm test:e2e:cli
99 +```
100 +
101 ## Adding New Tests
102
103 ### Node.js Runtime Tests
@@ -208,6 +223,42 @@ e2e(import.meta.url, {
223 3. Add `test:e2e` and `test:e2e:cloudflare` scripts
224 4. Configure vitest with `@cloudflare/vitest-pool-workers`
225
226 +### CLI Runtime Tests
227 +
228 +1. Create a new directory under `cli/` (e.g., `cli/my-test`)
229 +2. Add a `package.json` with name `@e2e-tests/cli-my-test`
230 +3. Add `test:e2e` and `test:e2e:cli` scripts
231 +4. Create an `e2e.test.ts` file with inline configuration:
232 +
233 +```typescript
234 +import { e2e, sanitizeOutput, type E2ETestResult } from '@e2e-tests/utils';
235 +import { TIMEOUTS } from '@e2e-tests/utils/const';
236 +import { describe, it, expect, beforeAll } from 'bun:test';
237 +
238 +e2e(import.meta.url, {
239 + versions: {
240 + cli: ['current'],
241 + },
242 + defineTests: ({ runCmd }) => {
243 + let result: E2ETestResult;
244 +
245 + beforeAll(async () => {
246 + result = await runCmd('composio version');
247 + }, TIMEOUTS.FIXTURE);
248 +
249 + describe('output', () => {
250 + it('exits successfully', () => {
251 + expect(result.exitCode).toBe(0);
252 + });
253 +
254 + it('stdout matches snapshot', () => {
255 + expect(sanitizeOutput(result.stdout)).toMatchSnapshot();
256 + });
257 + });
258 + },
259 +});
260 +```
261 +
262 ## Debugging
263
264 Each test suite generates an ephemereal `DEBUG.log` file in its directory with structured output:
ts/e2e-tests/_utils/Dockerfile.cli new
+79
@@ -0,0 +1,79 @@
1 +# syntax=docker/dockerfile:1
2 +# Reusable Dockerfile for CLI e2e tests (debian runtime)
3 +# Usage: docker build -f Dockerfile.cli --build-arg CLI_VERSION=0.1.24 -t test-name .
4 +
5 +ARG NODE_VERSION=20.19.0
6 +ARG BUN_VERSION=1.3.8
7 +ARG CLI_VERSION=0.0.0
8 +
9 +FROM oven/bun:${BUN_VERSION} AS bun
10 +
11 +FROM node:${NODE_VERSION}-slim AS builder
12 +
13 +ENV PNPM_HOME="/pnpm"
14 +ENV PATH="$PNPM_HOME:/usr/local/bin:$PATH"
15 +
16 +WORKDIR /app
17 +
18 +# Bring in bun binary from the official image (no curl)
19 +COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun
20 +
21 +# Install corepack (pnpm) without curl/git
22 +RUN corepack enable
23 +
24 +# Copy lockfile first for optimal layer caching
25 +COPY pnpm-lock.yaml ./
26 +
27 +# Copy workspace configuration files
28 +COPY package.json \
29 + pnpm-workspace.yaml \
30 + .pnpmfile.cjs \
31 + tsconfig.base.json \
32 + tsdown.config.base.ts \
33 + .bun-version \
34 + ./
35 +
36 +# Copy pre-install scripts (rarely change)
37 +COPY ts/scripts/pre-install.sh ./ts/scripts/pre-install.sh
38 +COPY ts/scripts/pre-install/ ./ts/scripts/pre-install/
39 +
40 +# Copy all ts/packages (pre-built)
41 +COPY ts/packages/ ./ts/packages/
42 +
43 +# Install dependencies using pnpm
44 +# Note: We intentionally don't use --mount=type=cache for the pnpm store here
45 +# because it causes native binding issues across builds
46 +RUN pnpm install --frozen-lockfile
47 +
48 +# Build composio CLI binary
49 +WORKDIR /app/ts/packages/cli
50 +RUN bun build ./src/bin.ts \
51 + --env DEBUG_OVERRIDE_* \
52 + --compile \
53 + --production \
54 + --outfile /out/composio
55 +
56 +# Copy CLI e2e tests last (changes frequently)
57 +WORKDIR /app
58 +COPY ts/e2e-tests/cli/ /app/ts/e2e-tests/cli/
59 +
60 +# Pre-create Composio CLI cache directory to work around Bun issue #7967
61 +RUN mkdir -p /tmp/.composio
62 +
63 +FROM debian:bookworm-slim
64 +
65 +ENV PATH="/usr/local/bin:/bin"
66 +ENV HOME="/tmp"
67 +
68 +WORKDIR /app
69 +
70 +# CLI binary
71 +COPY --from=builder /out/composio /usr/local/bin/composio
72 +
73 +# CLI e2e tests
74 +COPY --from=builder /app/ts/e2e-tests/cli/ /app/ts/e2e-tests/cli/
75 +
76 +# CLI cache directory
77 +COPY --from=builder /tmp/.composio /tmp/.composio
78 +
79 +CMD ["/bin/sh"]
ts/e2e-tests/_utils/README.md
+34 -4
@@ -1,6 +1,6 @@
1 # E2E Test Utilities
2
3 -Shared infrastructure for running `@composio/core` end-to-end tests in isolated Docker environments.
3 +Shared infrastructure for running `@composio/core` and CLI end-to-end tests in isolated Docker environments.
4
5 ## What's Here
6
@@ -10,6 +10,7 @@ Shared infrastructure for running `@composio/core` end-to-end tests in isolated
10 | `scripts/` | Docker build and cleanup scripts |
11 | `Dockerfile.node` | Multi-stage Dockerfile for Node.js test environments |
12 | `Dockerfile.deno` | Dockerfile for Deno test environments |
13 +| `Dockerfile.cli` | Scratch Dockerfile for CLI test environments |
14
15 ## API
16
@@ -26,6 +27,7 @@ e2e(import.meta.url, {
27 versions: {
28 node: ['20.18.0', '20.19.0', '22.12.0'], // optional, defaults to .nvmrc
29 deno: ['2.6.7'], // optional, defaults to .dvmrc
30 + cli: ['current'], // optional, defaults to CLI package.json version
31 },
32 env: { MY_VAR: 'value' }, // optional env vars
33 defineTests: ({ runtime, runCmd, runFixture }) => {
@@ -61,6 +63,7 @@ Configuration object passed to `e2e()`:
63 | -------- | -------------------------------- | ------------------------------------------------ |
64 | `node` | `readonly NodeVersionFromUser[]` | Node.js versions. Defaults to `.nvmrc` |
65 | `deno` | `readonly DenoVersionFromUser[]` | Deno versions. Defaults to `.dvmrc` |
66 +| `cli` | `readonly CliVersionFromUser[]` | CLI versions. Defaults to CLI package.json |
67
68 ### `DefineTestsContext`
69
@@ -68,7 +71,7 @@ The context passed to the `defineTests` callback:
71
72 | Property | Type/Signature | Description |
73 | ------------ | ---------------------------------------------------------------- | ---------------------------------------------- |
71 -| `runtime` | `'node' \| 'deno'` | Current runtime being tested |
74 +| `runtime` | `'node' \| 'deno' \| 'cli'` | Current runtime being tested |
75 | `runCmd` | `(command: string) => Promise<E2ETestResult>` | Run arbitrary command in Docker container |
76 | `runFixture` | `(options: RunFixtureOptions) => Promise<E2ETestResult \| E2ETestResultWithSetup>` | Run fixture with optional setup phase |
77
@@ -111,6 +114,21 @@ interface E2ETestResult {
114 }
115 ```
116
117 +### `runCmd` with File Capture
118 +
119 +When you need to assert on files created inside the container, pass a `files` array:
120 +
121 +```typescript
122 +const result = await runCmd({
123 + command: 'composio version > out.txt',
124 + files: ['out.txt'],
125 +});
126 +
127 +expect(result.files['out.txt']).toBe('0.1.24');
128 +```
129 +
130 +The files are copied out of the container after execution and returned as a map in the result.
131 +
132 ### `E2ETestResultWithSetup`
133
134 Extended result when `runFixture` is called with a `setup` option:
@@ -190,6 +208,18 @@ The following versions are pre-defined in `const.ts`:
208 - `2.6.7`
209 - `current` (resolves to `.dvmrc` version)
210
211 +### CLI Version Resolution
212 +
213 +CLI versions to test are resolved in this order:
214 +
215 +1. **`COMPOSIO_E2E_CLI_VERSION` env var** (highest priority): Use `[env_value]`
216 +2. **`config.versions.cli`**: Use the provided array
217 +3. **Default**: Use version from `ts/packages/cli/package.json`
218 +
219 +### Well-Known CLI Versions
220 +
221 +- `current` (resolves to CLI package.json version)
222 +
223 ## Environment Variable Validation
224
225 Environment variables passed to `E2EConfig.env` are validated at test startup. If any variable has an `undefined` value, the test fails fast with a clear error message:
@@ -231,10 +261,10 @@ e2e(import.meta.url, {
261 ## Scripts
262
263 ```bash
234 -# Pre-build Docker images for all well-known Node and Deno versions
264 +# Pre-build Docker images for all well-known Node, Deno, and CLI versions
265 pnpm docker:build
266
237 -# Remove all e2e Docker images (both Node.js and Deno)
267 +# Remove all e2e Docker images (Node.js, Deno, and CLI)
268 pnpm docker:clean
269 ```
270
ts/e2e-tests/_utils/scripts/docker-build.ts
+75 -5
@@ -1,9 +1,9 @@
1 #!/usr/bin/env bun
2 /**
3 - * Builds Docker images for Node.js and Deno e2e tests.
3 + * Builds Docker images for Node.js, Deno, and CLI e2e tests.
4 *
5 - * In CI mode (COMPOSIO_E2E_NODE_VERSION or COMPOSIO_E2E_DENO_VERSION set):
6 - * Only builds the image for the specified version.
5 + * In CI mode (COMPOSIO_E2E_NODE_VERSION or COMPOSIO_E2E_DENO_VERSION or COMPOSIO_E2E_CLI_VERSION set):
6 + * Only builds the image for the specified version(s).
7 *
8 * In local mode:
9 * Builds images for all versions in WELL_KNOWN_NODE_VERSIONS and WELL_KNOWN_DENO_VERSIONS.
@@ -38,6 +38,15 @@ function getDvmrcVersion(repoRoot: string): string {
38 }
39 }
40
41 +function getCliPackageVersion(repoRoot: string): string {
42 + const pkg = readFileSync(resolve(repoRoot, 'ts/packages/cli/package.json'), 'utf-8');
43 + const parsed = JSON.parse(pkg) as { version?: string };
44 + if (!parsed.version) {
45 + throw new Error('Missing version in ts/packages/cli/package.json');
46 + }
47 + return String(parsed.version).trim();
48 +}
49 +
50 // ============================================================================
51 // Node.js image building
52 // ============================================================================
@@ -118,6 +127,46 @@ async function buildDenoImage(denoVersion: string, repoRoot: string): Promise<bo
127 return true;
128 }
129
130 +// ============================================================================
131 +// CLI image building
132 +// ============================================================================
133 +
134 +function imageTagForCliVersion(cliVersion: string): string {
135 + return `composio-e2e-cli:${cliVersion}`;
136 +}
137 +
138 +function defaultCliLabels(cliVersion: string): Record<string, string> {
139 + return {
140 + 'composio.e2e': 'true',
141 + 'composio.runtime': 'cli',
142 + 'composio.cli_version': cliVersion,
143 + };
144 +}
145 +
146 +async function buildCliImage(cliVersion: string, repoRoot: string): Promise<boolean> {
147 + const dockerfilePath = resolve(repoRoot, 'ts/e2e-tests/_utils/Dockerfile.cli');
148 + const imageTag = imageTagForCliVersion(cliVersion);
149 + const labels = defaultCliLabels(cliVersion);
150 + const labelArgs = Object.entries(labels)
151 + .map(([k, v]) => `--label=${k}=${v}`)
152 + .join(' ');
153 +
154 + console.log(`\nBuilding image for CLI ${cliVersion}...`);
155 +
156 + const result = await $`docker build -f ${dockerfilePath} --build-arg CLI_VERSION=${cliVersion} ${{ raw: labelArgs }} -t ${imageTag} ${repoRoot}`
157 + .cwd(repoRoot)
158 + .nothrow();
159 +
160 + if (result.exitCode !== 0) {
161 + console.error(` Failed to build ${imageTag}:`);
162 + console.error(result.stderr.toString() || result.stdout.toString());
163 + return false;
164 + }
165 +
166 + console.log(` Built ${imageTag}`);
167 + return true;
168 +}
169 +
170 // ============================================================================
171 // Main
172 // ============================================================================
@@ -126,17 +175,20 @@ async function main() {
175 const repoRoot = getRepoRoot();
176 const currentNodeVersion = getNvmrcVersion(repoRoot);
177 const currentDenoVersion = getDvmrcVersion(repoRoot);
178 + const currentCliVersion = getCliPackageVersion(repoRoot);
179 const envNodeVersion = Bun.env.COMPOSIO_E2E_NODE_VERSION;
180 const envDenoVersion = Bun.env.COMPOSIO_E2E_DENO_VERSION;
181 + const envCliVersion = Bun.env.COMPOSIO_E2E_CLI_VERSION;
182
183 const nodeVersions = new Set<string>();
184 const denoVersions = new Set<string>();
185 + const cliVersions = new Set<string>();
186
187 // Determine which Node.js versions to build
188 if (envNodeVersion) {
189 nodeVersions.add(envNodeVersion);
190 console.log(`Building Docker image for CI matrix Node.js version: ${envNodeVersion}`);
139 - } else if (!envDenoVersion) {
191 + } else if (!envDenoVersion && !envCliVersion) {
192 // Build all Node.js versions only if no specific runtime was requested
193 for (const version of WELL_KNOWN_NODE_VERSIONS) {
194 if (version === 'current') {
@@ -158,7 +210,7 @@ async function main() {
210 if (envDenoVersion) {
211 denoVersions.add(envDenoVersion);
212 console.log(`Building Docker image for CI matrix Deno version: ${envDenoVersion}`);
161 - } else if (!envNodeVersion) {
213 + } else if (!envNodeVersion && !envCliVersion) {
214 // Build all Deno versions only if no specific runtime was requested
215 for (const version of WELL_KNOWN_DENO_VERSIONS) {
216 if (version === 'current') {
@@ -176,6 +228,15 @@ async function main() {
228 }
229 }
230
231 + // Determine which CLI versions to build
232 + if (envCliVersion) {
233 + cliVersions.add(envCliVersion);
234 + console.log(`Building Docker image for CI matrix CLI version: ${envCliVersion}`);
235 + } else if (!envNodeVersion && !envDenoVersion) {
236 + cliVersions.add(currentCliVersion);
237 + console.log(`Building Docker image for CLI version: ${currentCliVersion}`);
238 + }
239 +
240 let failed = 0;
241 let total = 0;
242
@@ -197,6 +258,15 @@ async function main() {
258 }
259 }
260
261 + // Build CLI images
262 + for (const version of cliVersions) {
263 + total++;
264 + const success = await buildCliImage(version, repoRoot);
265 + if (!success) {
266 + failed++;
267 + }
268 + }
269 +
270 console.log('\n---');
271 console.log(`Built ${total - failed}/${total} images.`);
272
ts/e2e-tests/_utils/scripts/docker-clean.ts
+8 -4
@@ -1,13 +1,14 @@
1 #!/usr/bin/env bun
2 /**
3 - * Removes all Docker images used by Node.js and Deno e2e tests.
3 + * Removes all Docker images used by Node.js, Deno, and CLI e2e tests.
4 * Images are identified by the label `composio.e2e=true`.
5 */
6
7 import { $ } from 'bun';
8
9 -async function cleanImages(runtime: 'node' | 'deno'): Promise<number> {
10 - console.log(`Finding e2e ${runtime === 'node' ? 'Node.js' : 'Deno'} Docker images...`);
9 +async function cleanImages(runtime: 'node' | 'deno' | 'cli'): Promise<number> {
10 + const runtimeLabel = runtime === 'node' ? 'Node.js' : runtime === 'deno' ? 'Deno' : 'CLI';
11 + console.log(`Finding e2e ${runtimeLabel} Docker images...`);
12
13 // Find all images with the e2e labels
14 const result = await $`docker images --filter label=composio.e2e=true --filter label=composio.runtime=${runtime} --format {{.Repository}}:{{.Tag}}`
@@ -26,7 +27,7 @@ async function cleanImages(runtime: 'node' | 'deno'): Promise<number> {
27 .filter((img) => img.length > 0 && !img.includes('<none>'));
28
29 if (images.length === 0) {
29 - console.log(`No e2e ${runtime === 'node' ? 'Node.js' : 'Deno'} Docker images found.`);
30 + console.log(`No e2e ${runtimeLabel} Docker images found.`);
31 return 0;
32 }
33
@@ -59,6 +60,9 @@ async function main() {
60 // Clean Deno images
61 totalFailures += await cleanImages('deno');
62
63 + // Clean CLI images
64 + totalFailures += await cleanImages('cli');
65 +
66 if (totalFailures > 0) {
67 console.error(`\nFailed to remove ${totalFailures} image(s).`);
68 process.exit(1);
ts/e2e-tests/_utils/src/config.ts
+86 -1
@@ -2,13 +2,23 @@ import { dirname, resolve } from 'node:path';
2 import { fileURLToPath } from 'node:url';
3 import { readFileSync } from 'node:fs';
4 import process from 'node:process';
5 -import type { NodeVersionMeta, NodeVersionFromUser, DenoVersionMeta, DenoVersionFromUser, SkipInCI, NonEmptyArray } from './types';
5 +import type {
6 + NodeVersionMeta,
7 + NodeVersionFromUser,
8 + DenoVersionMeta,
9 + DenoVersionFromUser,
10 + CliVersionMeta,
11 + CliVersionFromUser,
12 + SkipInCI,
13 + NonEmptyArray,
14 +} from './types';
15
16 declare module 'bun' {
17 interface Env {
18 CI?: string;
19 COMPOSIO_E2E_NODE_VERSION?: string;
20 COMPOSIO_E2E_DENO_VERSION?: string;
21 + COMPOSIO_E2E_CLI_VERSION?: string;
22 }
23 }
24
@@ -123,6 +133,23 @@ export function getDvmrcVersion(): string {
133 }
134 }
135
136 +/**
137 + * Read the CLI version from ts/packages/cli/package.json.
138 + * Used to determine the version for 'current' tests.
139 + */
140 +export function getCliPackageVersion(): string {
141 + try {
142 + const pkg = readFileSync(resolve(getRepoRoot(), 'ts/packages/cli/package.json'), 'utf-8');
143 + const parsed = JSON.parse(pkg) as { version?: string };
144 + if (!parsed.version) {
145 + throw new Error('Missing version field');
146 + }
147 + return String(parsed.version).trim();
148 + } catch (err) {
149 + throw new Error(`Failed to read CLI version from ts/packages/cli/package.json: ${(err as Error).message}`);
150 + }
151 +}
152 +
153 /**
154 * Compute skip state for a Deno version in CI mode.
155 */
@@ -144,6 +171,27 @@ function computeSkipForDenoVersion(
171 return { value: false };
172 }
173
174 +/**
175 + * Compute skip state for a CLI version in CI mode.
176 + */
177 +function computeSkipForCliVersion(
178 + envVersion: string | undefined,
179 + versionValue: string
180 +): SkipInCI {
181 + if (!isCI() || !envVersion) {
182 + return { value: false };
183 + }
184 +
185 + if (versionValue !== envVersion) {
186 + return {
187 + value: true,
188 + reason: `CLI ${versionValue} not selected (running ${envVersion})`,
189 + };
190 + }
191 +
192 + return { value: false };
193 +}
194 +
195 /**
196 * Resolve the Deno versions to test against, with CI skip state.
197 *
@@ -185,3 +233,40 @@ export function resolveDenoVersionMetaList(
233 return resolvedVersions as NonEmptyArray<DenoVersionMeta>;
234 }
235
236 +/**
237 + * Resolve the CLI versions to test against, with CI skip state.
238 + *
239 + * The `'current'` version always resolves to the value in ts/packages/cli/package.json.
240 + *
241 + * In CI mode (CI env var set + COMPOSIO_E2E_CLI_VERSION set):
242 + * - Versions not matching COMPOSIO_E2E_CLI_VERSION are marked to skip
243 + *
244 + * In local mode:
245 + * - COMPOSIO_E2E_CLI_VERSION overrides everything (single version, no skip)
246 + * - Otherwise returns all configured versions (no skip)
247 + */
248 +export function resolveCliVersionMetaList(
249 + configCliVersions?: readonly CliVersionFromUser[]
250 +): NonEmptyArray<CliVersionMeta> {
251 + const envVersion = Bun.env.COMPOSIO_E2E_CLI_VERSION;
252 + const currentVersion = getCliPackageVersion();
253 +
254 + // Local mode with env override: single version, no skip
255 + if (!isCI() && envVersion) {
256 + return [{ kind: 'overridden', value: envVersion, skip: { value: false } }];
257 + }
258 +
259 + // No config provided: use package.json version
260 + if (configCliVersions === undefined || configCliVersions.length === 0) {
261 + return [{ kind: 'current', value: currentVersion, skip: { value: false } }];
262 + }
263 +
264 + const resolvedVersions = configCliVersions.map((v): CliVersionMeta => {
265 + if (v === 'current') {
266 + return { kind: 'current', value: currentVersion, skip: computeSkipForCliVersion(envVersion, currentVersion) };
267 + }
268 + return { kind: 'static', value: v, skip: computeSkipForCliVersion(envVersion, v) };
269 + });
270 +
271 + return resolvedVersions as NonEmptyArray<CliVersionMeta>;
272 +}
ts/e2e-tests/_utils/src/const.ts
+6
@@ -15,6 +15,12 @@ export const WELL_KNOWN_NODE_VERSIONS = ['20.18.0', '20.19.0', '22.12.0', 'curre
15 */
16 export const WELL_KNOWN_DENO_VERSIONS = ['2.6.7', 'current'] as const;
17
18 +/**
19 + * CLI versions that are well-known to the CI strategy.
20 + * `current` refers to the version in ts/packages/cli/package.json.
21 + */
22 +export const WELL_KNOWN_CLI_VERSIONS = ['current'] as const;
23 +
24 export const TIMEOUTS = {
25 DEFAULT: 5_000,
26 FIXTURE: 120_000,
ts/e2e-tests/_utils/src/image-lifecycle.ts
+193 -4
@@ -53,6 +53,8 @@ export interface RunNodeContainerOptions {
53 env?: Record<string, string | undefined>;
54 labels?: Record<string, string>;
55 name?: string;
56 + /** Whether to auto-remove the container after exit (default: true) */
57 + remove?: boolean;
58 }
59
60 /**
@@ -106,6 +108,20 @@ function defaultDenoLabels(denoVersion?: string): Record<string, string> {
108 return labels;
109 }
110
111 +/**
112 + * Creates default Docker labels for CLI e2e images.
113 + */
114 +function defaultCliLabels(cliVersion?: string): Record<string, string> {
115 + const labels: Record<string, string> = {
116 + 'composio.e2e': 'true',
117 + 'composio.runtime': 'cli',
118 + };
119 + if (cliVersion) {
120 + labels['composio.cli_version'] = cliVersion;
121 + }
122 + return labels;
123 +}
124 +
125 /**
126 * Converts labels object to Docker CLI arguments array.
127 */
@@ -224,7 +240,7 @@ export async function ensureNodeImage(
240 * Runs a command in a Node Docker container.
241 */
242 export async function runNodeContainer(options: RunNodeContainerOptions): Promise<ExecResult> {
227 - const { imageTag, cmd, cwd, mounts, volumes, env, labels, name } = options;
243 + const { imageTag, cmd, cwd, mounts, volumes, env, labels, name, remove } = options;
244
245 // Validate imageTag is provided, as it determines which Docker image to run.
246 if (!imageTag || typeof imageTag !== 'string') {
@@ -233,7 +249,10 @@ export async function runNodeContainer(options: RunNodeContainerOptions): Promis
249
250 const nodeVersion = parseNodeVersionFromImageTag(imageTag);
251
236 - const dockerArgs = ['run', '--rm'];
252 + const dockerArgs = ['run'];
253 + if (remove !== false) {
254 + dockerArgs.push('--rm');
255 + }
256
257 dockerArgs.push(...labelsToArgs({ ...defaultNodeLabels(nodeVersion), ...(labels ?? {}) }));
258
@@ -317,6 +336,8 @@ export interface RunDenoContainerOptions {
336 env?: Record<string, string | undefined>;
337 labels?: Record<string, string>;
338 name?: string;
339 + /** Whether to auto-remove the container after exit (default: true) */
340 + remove?: boolean;
341 }
342
343 /**
@@ -398,7 +419,7 @@ export async function ensureDenoImage(
419 * Runs a command in a Deno Docker container.
420 */
421 export async function runDenoContainer(options: RunDenoContainerOptions): Promise<ExecResult> {
401 - const { imageTag, cmd, cwd, mounts, volumes, env, labels, name } = options;
422 + const { imageTag, cmd, cwd, mounts, volumes, env, labels, name, remove } = options;
423
424 // Validate imageTag is provided, as it determines which Docker image to run.
425 if (!imageTag || typeof imageTag !== 'string') {
@@ -407,7 +428,10 @@ export async function runDenoContainer(options: RunDenoContainerOptions): Promis
428
429 const denoVersion = parseDenoVersionFromImageTag(imageTag);
430
410 - const dockerArgs = ['run', '--rm'];
431 + const dockerArgs = ['run'];
432 + if (remove !== false) {
433 + dockerArgs.push('--rm');
434 + }
435
436 dockerArgs.push(...labelsToArgs({ ...defaultDenoLabels(denoVersion), ...(labels ?? {}) }));
437
@@ -474,3 +498,168 @@ export async function runDenoContainer(options: RunDenoContainerOptions): Promis
498
499 return exec('docker', dockerArgs);
500 }
501 +
502 +// ============================================================================
503 +// CLI Docker utilities
504 +// ============================================================================
505 +
506 +/**
507 + * Options for running a CLI container.
508 + */
509 +export interface RunCliContainerOptions {
510 + imageTag: string;
511 + cmd: string | string[];
512 + cwd?: string;
513 + mounts?: DockerMount[];
514 + /** Named volume mounts (volumes must be created beforehand with `docker volume create`) */
515 + volumes?: DockerVolumeMount[];
516 + env?: Record<string, string | undefined>;
517 + labels?: Record<string, string>;
518 + name?: string;
519 + /** Whether to auto-remove the container after exit (default: true) */
520 + remove?: boolean;
521 +}
522 +
523 +/**
524 + * Options for ensureCliImage.
525 + */
526 +export interface EnsureCliImageOptions {
527 + repoRoot?: string;
528 + dockerfilePath?: string;
529 +}
530 +
531 +/**
532 + * Creates image tag for a given CLI version.
533 + */
534 +function imageTagForCliVersion(cliVersion: string): string {
535 + return `composio-e2e-cli:${cliVersion}`;
536 +}
537 +
538 +/**
539 + * Parses CLI version from an image tag.
540 + */
541 +function parseCliVersionFromImageTag(imageTag: string): string | undefined {
542 + const match = /^composio-e2e-cli:(.+)$/.exec(imageTag);
543 + return match?.[1];
544 +}
545 +
546 +/**
547 + * Ensures a Docker image exists for the given CLI version, building it if necessary.
548 + */
549 +export async function ensureCliImage(
550 + cliVersion: string,
551 + options: EnsureCliImageOptions = {}
552 +): Promise<string> {
553 + if (!cliVersion || typeof cliVersion !== 'string') {
554 + throw new Error(`ensureCliImage(${cliVersion}): cliVersion must be a non-empty string`);
555 + }
556 +
557 + const repoRoot = options.repoRoot ?? getRepoRoot();
558 + const dockerfilePath = options.dockerfilePath ?? resolve(repoRoot, 'ts/e2e-tests/_utils/Dockerfile.cli');
559 + const imageTag = imageTagForCliVersion(cliVersion);
560 +
561 + const inspect = await exec('docker', ['image', 'inspect', imageTag], { cwd: repoRoot });
562 + if (inspect.exitCode === 0) {
563 + return imageTag;
564 + }
565 +
566 + const buildArgs = [
567 + 'build',
568 + '-f',
569 + dockerfilePath,
570 + '--build-arg',
571 + `CLI_VERSION=${cliVersion}`,
572 + ...labelsToArgs(defaultCliLabels(cliVersion)),
573 + '-t',
574 + imageTag,
575 + repoRoot,
576 + ];
577 +
578 + const built = await exec('docker', buildArgs, { cwd: repoRoot });
579 + if (built.exitCode !== 0) {
580 + const output = built.stderr || built.stdout;
581 + if (output.includes('already exists')) {
582 + const recheck = await exec('docker', ['image', 'inspect', imageTag], { cwd: repoRoot });
583 + if (recheck.exitCode === 0) {
584 + return imageTag;
585 + }
586 + }
587 +
588 + const err = new Error(`Failed to build Docker image ${imageTag}`);
589 + (err as Error & { cause: Error }).cause = new Error(output);
590 + throw err;
591 + }
592 +
593 + return imageTag;
594 +}
595 +
596 +/**
597 + * Runs a command in a CLI Docker container.
598 + */
599 +export async function runCliContainer(options: RunCliContainerOptions): Promise<ExecResult> {
600 + const { imageTag, cmd, cwd, mounts, volumes, env, labels, name, remove } = options;
601 +
602 + if (!imageTag || typeof imageTag !== 'string') {
603 + throw new Error('runCliContainer({ imageTag, ... }): imageTag must be a non-empty string');
604 + }
605 +
606 + const cliVersion = parseCliVersionFromImageTag(imageTag);
607 +
608 + const dockerArgs = ['run'];
609 + if (remove !== false) {
610 + dockerArgs.push('--rm');
611 + }
612 +
613 + dockerArgs.push(...labelsToArgs({ ...defaultCliLabels(cliVersion), ...(labels ?? {}) }));
614 +
615 + if (name) {
616 + dockerArgs.push('--name', name);
617 + }
618 +
619 + if (cwd) {
620 + const containerCwd = cwd.startsWith('/') ? cwd : `/app/${cwd}`;
621 + dockerArgs.push('--workdir', containerCwd);
622 + }
623 +
624 + if (env) {
625 + for (const [k, v] of Object.entries(env)) {
626 + if (v === undefined) continue;
627 + dockerArgs.push('-e', `${k}=${v}`);
628 + }
629 + }
630 +
631 + if (mounts) {
632 + for (const m of mounts) {
633 + if (!m?.source || !m?.target) {
634 + throw new Error('runCliContainer(...): each mount must have { source, target }');
635 + }
636 + const parts = [`type=bind`, `src=${m.source}`, `dst=${m.target}`];
637 + if (m.readonly) parts.push('readonly');
638 + dockerArgs.push('--mount', parts.join(','));
639 + }
640 + }
641 +
642 + if (volumes) {
643 + for (const v of volumes) {
644 + if (!v?.volume || !v?.target) {
645 + throw new Error('runCliContainer(...): each volume must have { volume, target }');
646 + }
647 + const parts = [`type=volume`, `src=${v.volume}`, `dst=${v.target}`];
648 + if (v.readonly) parts.push('readonly');
649 + dockerArgs.push('--mount', parts.join(','));
650 + }
651 + }
652 +
653 + dockerArgs.push(imageTag);
654 +
655 + if (Array.isArray(cmd)) {
656 + dockerArgs.push(...cmd.map(String));
657 + } else if (typeof cmd === 'string' && cmd.length > 0) {
658 + // Use -c (not -lc) for busybox sh compatibility in scratch images
659 + dockerArgs.push('sh', '-c', cmd);
660 + } else {
661 + throw new Error('runCliContainer({ cmd, ... }): cmd must be a non-empty string or string[]');
662 + }
663 +
664 + return exec('docker', dockerArgs);
665 +}
ts/e2e-tests/_utils/src/index.ts
+2
@@ -2,10 +2,12 @@ export type {
2 E2EConfig,
3 E2ETestResult,
4 E2ETestResultWithSetup,
5 + E2ETestResultWithFiles,
6 RunFixtureOptions,
7 DefineTestsContext,
8 NodeVersionMeta as NodeVersion,
9 DenoVersionMeta as DenoVersion,
10 + CliVersionMeta as CliVersion,
11 RuntimeVersions,
12 RuntimeKind,
13 SkipInCI,
ts/e2e-tests/_utils/src/runner.ts
+268 -10
@@ -1,20 +1,37 @@
1 import { describe, beforeAll, afterAll, it } from 'bun:test';
2 -import { appendFile, writeFile } from 'node:fs/promises';
3 -import { resolve } from 'node:path';
2 +import { $ } from 'bun';
3 +import { appendFile, writeFile, mkdtemp, readFile, rm } from 'node:fs/promises';
4 +import { resolve, join } from 'node:path';
5 +import { tmpdir } from 'node:os';
6 import type {
7 DefineTestsContext,
8 E2EConfig,
9 E2ETestResult,
10 E2ETestResultWithSetup,
11 + E2ETestResultWithFiles,
12 NodeVersionMeta,
13 DenoVersionMeta,
14 + CliVersionMeta,
15 NonEmptyString,
16 RunFixtureOptions,
17 + RunCmdOptions,
18 RuntimeVersions,
19 RuntimeKind,
20 } from './types';
16 -import { getRepoRoot, resolveNodeVersionMetaList, resolveDenoVersionMetaList } from './config';
17 -import { ensureNodeImage, runNodeContainer, ensureDenoImage, runDenoContainer } from './image-lifecycle';
21 +import {
22 + getRepoRoot,
23 + resolveNodeVersionMetaList,
24 + resolveDenoVersionMetaList,
25 + resolveCliVersionMetaList,
26 +} from './config';
27 +import {
28 + ensureNodeImage,
29 + runNodeContainer,
30 + ensureDenoImage,
31 + runDenoContainer,
32 + ensureCliImage,
33 + runCliContainer,
34 +} from './image-lifecycle';
35 import { WELL_KNOWN_ENV_VARS } from './const';
36 import { createVolume, generateVolumeName, initializeVolumeOwnership, removeVolume } from './volume';
37
@@ -123,7 +140,8 @@ class DebugLogManager {
140
141 private formatVersionSection(result: VersionResult): string {
142 const dividerHeavy = '#'.repeat(80);
126 - const runtimeLabel = result.runtime === 'node' ? 'Node.js' : 'Deno';
143 + const runtimeLabel =
144 + result.runtime === 'node' ? 'Node.js' : result.runtime === 'deno' ? 'Deno' : 'CLI';
145 const lines: string[] = [
146 '',
147 dividerHeavy,
@@ -181,7 +199,8 @@ class DebugLogManager {
199 ];
200
201 for (const result of this.versionResults) {
184 - const runtimeLabel = result.runtime === 'node' ? 'Node.js' : 'Deno';
202 + const runtimeLabel =
203 + result.runtime === 'node' ? 'Node.js' : result.runtime === 'deno' ? 'Deno' : 'CLI';
204 if (result.status === 'skipped') {
205 lines.push(`${runtimeLabel} ${result.version}: SKIPPED`);
206 } else {
@@ -258,6 +277,59 @@ function validateRequiredEnvVars(
277 }
278 }
279
280 +// ============================================================================
281 +// Command execution helpers
282 +// ============================================================================
283 +
284 +type RunCmdInput = string | RunCmdOptions;
285 +
286 +function normalizeRunCmdInput(input: RunCmdInput): { command: string; files?: string[] } {
287 + if (typeof input === 'string') {
288 + return { command: input };
289 + }
290 + return { command: input.command, files: input.files };
291 +}
292 +
293 +function toContainerCwd(cwd: string): string {
294 + return cwd.startsWith('/') ? cwd : `/app/${cwd}`;
295 +}
296 +
297 +async function dockerCp(source: string, destination: string): Promise<void> {
298 + const result = await $`docker cp ${source} ${destination}`.nothrow();
299 + if (result.exitCode !== 0) {
300 + const details = (result.stderr || result.stdout).toString().trim();
301 + throw new Error(`docker cp failed: ${details || 'unknown error'}`);
302 + }
303 +}
304 +
305 +async function dockerRm(containerName: string): Promise<void> {
306 + await $`docker rm ${containerName}`.nothrow();
307 +}
308 +
309 +async function captureFilesFromContainer(options: {
310 + containerName: string;
311 + containerCwd: string;
312 + files: string[];
313 +}): Promise<Record<string, string>> {
314 + const { containerName, containerCwd, files } = options;
315 + const tempDir = await mkdtemp(join(tmpdir(), 'composio-e2e-files-'));
316 + const outputs: Record<string, string> = {};
317 +
318 + try {
319 + for (const file of files) {
320 + const containerPath = file.startsWith('/') ? file : `${containerCwd}/${file}`;
321 + const safeName = file.replace(/^\/+/, '').replace(/[\\/]/g, '__');
322 + const hostPath = join(tempDir, safeName);
323 + await dockerCp(`${containerName}:${containerPath}`, hostPath);
324 + outputs[file] = await readFile(hostPath, 'utf-8');
325 + }
326 + } finally {
327 + await rm(tempDir, { recursive: true, force: true });
328 + }
329 +
330 + return outputs;
331 +}
332 +
333 /**
334 * Internal configuration for runE2E.
335 */
@@ -299,7 +371,9 @@ function createNodeDockerExecutors(
371 imageTag,
372 };
373
302 - const runCmd = async (command: string): Promise<E2ETestResult> => {
374 + const runCmd = async (input: RunCmdInput): Promise<E2ETestResult | E2ETestResultWithFiles> => {
375 + const { command, files } = normalizeRunCmdInput(input);
376 + const shouldCapture = Array.isArray(files) && files.length > 0;
377 const containerName = `e2e-${suiteName}-node-${nodeVersion.replace(/\./g, '-')}-${Date.now()}`;
378 const startTime = Date.now();
379
@@ -309,6 +383,7 @@ function createNodeDockerExecutors(
383 cwd: effectiveCwd,
384 env: containerEnv,
385 name: containerName,
386 + remove: !shouldCapture,
387 });
388
389 const durationMs = Date.now() - startTime;
@@ -325,6 +400,20 @@ function createNodeDockerExecutors(
400 stderr: result.stderr,
401 });
402
403 + if (shouldCapture) {
404 + const containerCwd = toContainerCwd(effectiveCwd);
405 + try {
406 + const captured = await captureFilesFromContainer({
407 + containerName,
408 + containerCwd,
409 + files: files as string[],
410 + });
411 + return { ...result, files: captured };
412 + } finally {
413 + await dockerRm(containerName);
414 + }
415 + }
416 +
417 return result;
418 };
419
@@ -448,7 +537,9 @@ function createDenoDockerExecutors(
537 imageTag,
538 };
539
451 - const runCmd = async (command: string): Promise<E2ETestResult> => {
540 + const runCmd = async (input: RunCmdInput): Promise<E2ETestResult | E2ETestResultWithFiles> => {
541 + const { command, files } = normalizeRunCmdInput(input);
542 + const shouldCapture = Array.isArray(files) && files.length > 0;
543 const containerName = `e2e-${suiteName}-deno-${denoVersion.replace(/\./g, '-')}-${Date.now()}`;
544 const startTime = Date.now();
545
@@ -458,6 +549,7 @@ function createDenoDockerExecutors(
549 cwd: effectiveCwd,
550 env: containerEnv,
551 name: containerName,
552 + remove: !shouldCapture,
553 });
554
555 const durationMs = Date.now() - startTime;
@@ -474,6 +566,20 @@ function createDenoDockerExecutors(
566 stderr: result.stderr,
567 });
568
569 + if (shouldCapture) {
570 + const containerCwd = toContainerCwd(effectiveCwd);
571 + try {
572 + const captured = await captureFilesFromContainer({
573 + containerName,
574 + containerCwd,
575 + files: files as string[],
576 + });
577 + return { ...result, files: captured };
578 + } finally {
579 + await dockerRm(containerName);
580 + }
581 + }
582 +
583 return result;
584 };
585
@@ -571,6 +677,98 @@ function createDenoDockerExecutors(
677 return { runCmd, runFixture, finalizeVersionLog, context };
678 }
679
680 +// ============================================================================
681 +// CLI Docker Executors
682 +// ============================================================================
683 +
684 +/**
685 + * Creates Docker execution utilities for a specific CLI version.
686 + */
687 +function createCliDockerExecutors(
688 + config: RunE2EInternalConfig,
689 + cliVersion: string,
690 + imageTag: string,
691 + repoRoot: string,
692 + logManager: DebugLogManager
693 +) {
694 + const { cwd, suiteName, env, usesFixtures } = config;
695 + const containerEnv = buildContainerEnv(env);
696 + const effectiveCwd = usesFixtures ? `${cwd}/fixtures` : cwd;
697 +
698 + const context: VersionExecutionContext = {
699 + phases: [],
700 + versionStartTime: Date.now(),
701 + imageTag,
702 + };
703 +
704 + const runCmd = async (input: RunCmdInput): Promise<E2ETestResult | E2ETestResultWithFiles> => {
705 + const { command, files } = normalizeRunCmdInput(input);
706 + const shouldCapture = Array.isArray(files) && files.length > 0;
707 + const containerName = `e2e-${suiteName}-cli-${cliVersion.replace(/\./g, '-')}-${Date.now()}`;
708 + const startTime = Date.now();
709 +
710 + const result = await runCliContainer({
711 + imageTag,
712 + cmd: command,
713 + cwd: effectiveCwd,
714 + env: containerEnv,
715 + name: containerName,
716 + remove: !shouldCapture,
717 + });
718 +
719 + const durationMs = Date.now() - startTime;
720 +
721 + context.phases.push({
722 + phase: 'fixture',
723 + phaseIndex: context.phases.length + 1,
724 + totalPhases: context.phases.length + 1,
725 + command,
726 + containerName,
727 + durationMs,
728 + exitCode: result.exitCode,
729 + stdout: result.stdout,
730 + stderr: result.stderr,
731 + });
732 +
733 + if (shouldCapture) {
734 + const containerCwd = toContainerCwd(effectiveCwd);
735 + try {
736 + const captured = await captureFilesFromContainer({
737 + containerName,
738 + containerCwd,
739 + files: files as string[],
740 + });
741 + return { ...result, files: captured };
742 + } finally {
743 + await dockerRm(containerName);
744 + }
745 + }
746 +
747 + return result;
748 + };
749 +
750 + const runFixture = ((_: RunFixtureOptions) => {
751 + throw new Error('runFixture is not supported for CLI runtime tests');
752 + }) as DefineTestsContext['runFixture'];
753 +
754 + const finalizeVersionLog = async (): Promise<void> => {
755 + const totalDurationMs = Date.now() - context.versionStartTime;
756 + const allPassed = context.phases.every(p => p.exitCode === 0);
757 + const status: 'pass' | 'fail' = allPassed ? 'pass' : 'fail';
758 +
759 + await logManager.writeVersionSection({
760 + runtime: 'cli',
761 + version: cliVersion,
762 + status,
763 + imageTag: context.imageTag,
764 + phases: context.phases,
765 + totalDurationMs,
766 + });
767 + };
768 +
769 + return { runCmd, runFixture, finalizeVersionLog, context };
770 +}
771 +
772 // ============================================================================
773 // Version normalization and resolution
774 // ============================================================================
@@ -582,7 +780,7 @@ function normalizeVersionConfig(config: E2EConfig): RuntimeVersions {
780 if (config.versions) {
781 return config.versions;
782 }
585 - // Default: use current version for Node.js and Deno
783 + // Default: use current version for Node.js and Deno (CLI is opt-in)
784 return { node: ['current'], deno: ['current'] };
785 }
786
@@ -614,6 +812,20 @@ function renderDenoVersionMeta({ kind, value }: DenoVersionMeta): string {
812 }
813 }
814
815 +/**
816 + * Render a CLI version meta for display in test names.
817 + */
818 +function renderCliVersionMeta({ kind, value }: CliVersionMeta): string {
819 + switch (kind) {
820 + case 'current':
821 + return `CLI ${value} [current]`;
822 + case 'overridden':
823 + return `CLI ${value} [overridden]`;
824 + case 'static':
825 + return `CLI ${value}`;
826 + }
827 +}
828 +
829 // ============================================================================
830 // Main test runner
831 // ============================================================================
@@ -622,7 +834,7 @@ function renderDenoVersionMeta({ kind, value }: DenoVersionMeta): string {
834 * Runs e2e tests using bun:test.
835 * Creates a describe block per runtime version and passes test utilities to defineTests.
836 *
625 - * Supports both Node.js and Deno runtimes.
837 + * Supports Node.js, Deno, and CLI runtimes.
838 * In CI mode, versions not matching their respective env vars are skipped.
839 */
840 export function runE2E(config: RunE2EInternalConfig): void {
@@ -637,11 +849,13 @@ export function runE2E(config: RunE2EInternalConfig): void {
849 // Resolve versions for each runtime
850 const nodeVersionMetaList = versions.node ? resolveNodeVersionMetaList(versions.node) : [];
851 const denoVersionMetaList = versions.deno ? resolveDenoVersionMetaList(versions.deno) : [];
852 + const cliVersionMetaList = versions.cli ? resolveCliVersionMetaList(versions.cli) : [];
853
854 // Build runtime version labels for logging
855 const allRuntimeVersions = [
856 ...nodeVersionMetaList.map(v => `Node.js ${v.value}`),
857 ...denoVersionMetaList.map(v => `Deno ${v.value}`),
858 + ...cliVersionMetaList.map(v => `CLI ${v.value}`),
859 ];
860
861 // Create the debug log manager for structured output
@@ -748,6 +962,50 @@ export function runE2E(config: RunE2EInternalConfig): void {
962 });
963 }
964
965 + // =========== CLI Tests ===========
966 + for (const cliVersionMeta of cliVersionMetaList) {
967 + const testName = renderCliVersionMeta(cliVersionMeta);
968 +
969 + // Skip this version in CI if not selected
970 + if (cliVersionMeta.skip.value) {
971 + describe(`${testName} - skipped: ${cliVersionMeta.skip.reason}`, () => {
972 + beforeAll(async () => {
973 + await logManager.writeVersionSection({
974 + runtime: 'cli',
975 + version: cliVersionMeta.value,
976 + status: 'skipped',
977 + reason: cliVersionMeta.skip.reason || 'Skipped in CI',
978 + phases: [],
979 + });
980 + });
981 +
982 + it('this version should not run in this CI job', () => {});
983 + });
984 + continue;
985 + }
986 +
987 + describe(testName, () => {
988 + let executors: ReturnType<typeof createCliDockerExecutors>;
989 +
990 + beforeAll(async () => {
991 + const imageTag = await ensureCliImage(cliVersionMeta.value, { repoRoot });
992 + executors = createCliDockerExecutors(config, cliVersionMeta.value, imageTag, repoRoot, logManager);
993 + }, 600_000);
994 +
995 + defineTests({
996 + runtime: 'cli',
997 + runCmd: (cmd) => executors.runCmd(cmd),
998 + runFixture: ((opts: RunFixtureOptions) => executors.runFixture(opts)) as DefineTestsContext['runFixture'],
999 + });
1000 +
1001 + afterAll(async () => {
1002 + if (executors) {
1003 + await executors.finalizeVersionLog();
1004 + }
1005 + });
1006 + });
1007 + }
1008 +
1009 // Write summary after all versions complete
1010 afterAll(async () => {
1011 await logManager.writeSummary();
ts/e2e-tests/_utils/src/sanitize.ts
+3 -6
@@ -3,8 +3,9 @@
3 */
4
5 /**
6 - * Sanitize command output for stable snapshot comparisons.
7 - * Removes ANSI escape codes, normalizes line endings, and trims whitespace.
6 + * Sanitize command output for stable comparisons.
7 + * Removes ANSI escape codes, normalizes line endings, and trims leading/trailing
8 + * whitespace around the first/last meaningful output.
9 *
10 * @param output - Raw command output (stdout or stderr)
11 * @returns Sanitized output suitable for assertions and snapshots
@@ -16,10 +17,6 @@ export function sanitizeOutput(output: string): string {
17 .replace(/\x1b\[[0-9;]*m/g, '')
18 // Normalize line endings (Windows → Unix)
19 .replace(/\r\n/g, '\n')
19 - // Trim trailing whitespace from each line
20 - .split('\n')
21 - .map((line) => line.trimEnd())
22 - .join('\n')
20 // Trim leading/trailing whitespace from entire output
21 .trim()
22 );
ts/e2e-tests/_utils/src/types.ts
+38 -5
@@ -1,4 +1,4 @@
1 -import { WELL_KNOWN_NODE_VERSIONS, WELL_KNOWN_DENO_VERSIONS } from './const';
1 +import { WELL_KNOWN_NODE_VERSIONS, WELL_KNOWN_DENO_VERSIONS, WELL_KNOWN_CLI_VERSIONS } from './const';
2
3 export type NonEmptyArray<T> = [T, ...T[]];
4
@@ -15,6 +15,7 @@ export type NonEmptyString<T extends string> = T extends '' ? never : T;
15
16 export type NodeVersionFromUser = typeof WELL_KNOWN_NODE_VERSIONS[number];
17 export type DenoVersionFromUser = typeof WELL_KNOWN_DENO_VERSIONS[number];
18 +export type CliVersionFromUser = typeof WELL_KNOWN_CLI_VERSIONS[number];
19
20 /**
21 * Result of CI skip check for a specific Node version.
@@ -42,15 +43,26 @@ export type DenoVersionMeta =
43 | { kind: 'overridden'; value: string; skip: SkipInCI }
44 | { kind: 'current'; value: string; skip: SkipInCI };
45
46 +/**
47 + * Metadata for a resolved CLI version to test against.
48 + * Includes skip state for CI mode.
49 + */
50 +export type CliVersionMeta =
51 + | { kind: 'static'; value: string; skip: SkipInCI }
52 + | { kind: 'overridden'; value: string; skip: SkipInCI }
53 + | { kind: 'current'; value: string; skip: SkipInCI };
54 +
55 /**
56 * Runtime versions to test against.
47 - * Supports both Node.js and Deno runtimes.
57 + * Supports Node.js, Deno, and CLI runtimes.
58 */
59 export interface RuntimeVersions {
60 /** Node.js versions to test */
61 node?: readonly NodeVersionFromUser[];
62 /** Deno versions to test */
63 deno?: readonly DenoVersionFromUser[];
64 + /** CLI versions to test */
65 + cli?: readonly CliVersionFromUser[];
66 }
67
68 /**
@@ -65,6 +77,14 @@ export interface E2ETestResult {
77 stderr: string;
78 }
79
80 +/**
81 + * Result of executing a command with requested file captures.
82 + */
83 +export interface E2ETestResultWithFiles extends E2ETestResult {
84 + /** Captured file contents keyed by requested file path */
85 + files: Record<string, string>;
86 +}
87 +
88 /**
89 * Result of runFixture when a setup command is provided.
90 * Top-level fields (exitCode, stdout, stderr) reflect the fixture result.
@@ -87,7 +107,17 @@ export interface RunFixtureOptions {
107 /**
108 * Supported runtime environments for e2e tests.
109 */
90 -export type RuntimeKind = 'node' | 'deno';
110 +export type RuntimeKind = 'node' | 'deno' | 'cli';
111 +
112 +/**
113 + * Options for runCmd.
114 + */
115 +export interface RunCmdOptions {
116 + /** Command to execute */
117 + command: string;
118 + /** Files to copy out of the container after execution */
119 + files?: string[];
120 +}
121
122 /**
123 * Context passed to defineTests callback.
@@ -96,7 +126,10 @@ export interface DefineTestsContext {
126 /** The runtime environment for this test execution */
127 runtime: RuntimeKind;
128 /** Run an arbitrary command in the Docker container */
99 - runCmd: (command: string) => Promise<E2ETestResult>;
129 + runCmd: {
130 + <const C extends string>(command: NonEmptyString<C>): Promise<E2ETestResult>;
131 + <const C extends string>(options: { command: NonEmptyString<C>; files: NonEmptyArray<string> }): Promise<E2ETestResultWithFiles>;
132 + };
133 /**
134 * Run a fixture file with Node.js.
135 *
@@ -135,7 +168,7 @@ export interface DefineTestsContext {
168 export interface E2EConfig {
169 /**
170 * Runtime versions to test against.
138 - * Supports Node.js and Deno runtimes.
171 + * Supports Node.js, Deno, and CLI runtimes.
172 *
173 * @example
174 * ```typescript
ts/e2e-tests/cli/version/e2e.test.ts new
+61
@@ -0,0 +1,61 @@
1 +/**
2 + * CLI version command e2e test
3 + *
4 + * Verifies that the compiled composio CLI behaves correctly in a scratch container.
5 + */
6 +
7 +import { e2e, sanitizeOutput, type E2ETestResult, type E2ETestResultWithFiles } from '@e2e-tests/utils';
8 +import { TIMEOUTS } from '@e2e-tests/utils/const';
9 +import { describe, it, expect, beforeAll } from 'bun:test';
10 +import cliPkg from '../../../packages/cli/package.json' with { type: 'json' };
11 +
12 +e2e(import.meta.url, {
13 + versions: {
14 + cli: ['current'],
15 + },
16 + defineTests: ({ runCmd }) => {
17 + const expectedVersion = String(cliPkg.version ?? '').trim();
18 + let versionResult: E2ETestResult;
19 + let redirectedResult: E2ETestResultWithFiles;
20 +
21 + beforeAll(async () => {
22 + versionResult = await runCmd('composio version');
23 + redirectedResult = await runCmd({
24 + command: 'composio version > out.txt',
25 + files: ['out.txt'],
26 + });
27 + }, TIMEOUTS.FIXTURE);
28 +
29 + describe('composio version', () => {
30 + it('exits successfully', () => {
31 + expect(versionResult.exitCode).toBe(0);
32 + });
33 +
34 + it('stdout matches snapshot', () => {
35 + expect(sanitizeOutput(versionResult.stdout)).toBe(expectedVersion);
36 + });
37 +
38 + it('stderr matches snapshot', () => {
39 + expect(sanitizeOutput(versionResult.stderr)).toBe('');
40 + });
41 + });
42 +
43 + describe('stdout redirection to out.txt', () => {
44 + it('exits successfully', () => {
45 + expect(redirectedResult.exitCode).toBe(0);
46 + });
47 +
48 + it('stdout is empty', () => {
49 + expect(sanitizeOutput(redirectedResult.stdout)).toBe('');
50 + });
51 +
52 + it('stderr is empty', () => {
53 + expect(sanitizeOutput(redirectedResult.stderr)).toBe('');
54 + });
55 +
56 + it('out.txt matches snapshot', () => {
57 + expect(sanitizeOutput(redirectedResult.files['out.txt'] ?? '')).toBe(expectedVersion);
58 + });
59 + });
60 + },
61 +});
ts/e2e-tests/cli/version/package.json new
+14
@@ -0,0 +1,14 @@
1 +{
2 + "name": "@e2e-tests/cli-version",
3 + "version": "0.0.0",
4 + "private": true,
5 + "type": "module",
6 + "scripts": {
7 + "typecheck": "tsc --noEmit",
8 + "test:e2e": "bun test e2e.test.ts",
9 + "test:e2e:cli": "bun test e2e.test.ts"
10 + },
11 + "devDependencies": {
12 + "@e2e-tests/utils": "workspace:*"
13 + }
14 +}
turbo.jsonc
+5
@@ -32,6 +32,11 @@
32 "env": ["COMPOSIO_API_KEY", "COMPOSIO_E2E_DENO_VERSION", "OPENAI_API_KEY"],
33 "cache": false
34 },
35 + "test:e2e:cli": {
36 + "dependsOn": ["^build"],
37 + "env": ["COMPOSIO_API_KEY", "COMPOSIO_E2E_CLI_VERSION", "OPENAI_API_KEY"],
38 + "cache": false
39 + },
40 "test:e2e:cloudflare": {
41 "dependsOn": ["^build"],
42 "env": ["COMPOSIO_API_KEY", "OPENAI_API_KEY"],