@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 662 lines 19.3 KB
Raw
1 import { $ } from 'bun';
2 import { existsSync } from 'node:fs';
3 import { homedir } from 'node:os';
4 import { resolve } from 'node:path';
5 import { getRepoRoot } from './config';
6 import { nodeDigestFor } from './const';
7
8 /**
9 * Result of executing a command.
10 */
11 export interface ExecResult {
12 exitCode: number;
13 stdout: string;
14 stderr: string;
15 }
16
17 /**
18 * Options for exec function.
19 */
20 interface ExecOptions {
21 cwd?: string;
22 env?: Record<string, string | undefined>;
23 }
24
25 /**
26 * Docker bind mount configuration.
27 */
28 export interface DockerMount {
29 source: string;
30 target: string;
31 readonly?: boolean;
32 }
33
34 /**
35 * Docker named volume mount configuration.
36 */
37 export interface DockerVolumeMount {
38 /** Named volume name (must be created with `docker volume create` first) */
39 volume: string;
40 /** Container path to mount the volume at */
41 target: string;
42 /** Whether to mount read-only */
43 readonly?: boolean;
44 }
45
46 /**
47 * Options for running a Node container.
48 */
49 export interface RunNodeContainerOptions {
50 imageTag: string;
51 cmd: string | string[];
52 cwd?: string;
53 mounts?: DockerMount[];
54 /** Named volume mounts (volumes must be created beforehand with `docker volume create`) */
55 volumes?: DockerVolumeMount[];
56 env?: Record<string, string | undefined>;
57 labels?: Record<string, string>;
58 name?: string;
59 /** Whether to auto-remove the container after exit (default: true) */
60 remove?: boolean;
61 }
62
63 /**
64 * Options for ensureNodeImage.
65 */
66 export interface EnsureNodeImageOptions {
67 repoRoot?: string;
68 dockerfilePath?: string;
69 }
70
71 /**
72 * Options for checkDocker.
73 */
74 export interface CheckDockerOptions {
75 repoRoot?: string;
76 }
77
78 /**
79 * Escapes a shell argument for safe use in Bun shell raw strings.
80 */
81 function escapeShellArg(arg: string): string {
82 // Single-quote the arg and escape any embedded single quotes
83 return `'${arg.replace(/'/g, "'\\''")}'`;
84 }
85
86 /**
87 * Creates default Docker labels for Node.js e2e images.
88 */
89 function defaultNodeLabels(nodeVersion?: string): Record<string, string> {
90 const labels: Record<string, string> = {
91 'composio.e2e': 'true',
92 'composio.runtime': 'node',
93 };
94 if (nodeVersion) {
95 labels['composio.node_version'] = nodeVersion;
96 }
97 return labels;
98 }
99
100 /**
101 * Creates default Docker labels for CLI e2e images.
102 */
103 function defaultCliLabels(cliVersion?: string): Record<string, string> {
104 const labels: Record<string, string> = {
105 'composio.e2e': 'true',
106 'composio.runtime': 'cli',
107 };
108 if (cliVersion) {
109 labels['composio.cli_version'] = cliVersion;
110 }
111 return labels;
112 }
113
114 /**
115 * Converts labels object to Docker CLI arguments array.
116 */
117 function labelsToArgs(labels: Record<string, string> = {}): string[] {
118 const args: string[] = [];
119 for (const [k, v] of Object.entries(labels)) {
120 args.push('--label', `${k}=${v}`);
121 }
122 return args;
123 }
124
125 /**
126 * Converts environment variables to Docker CLI arguments array.
127 * Skips undefined values to avoid passing empty env vars to Docker.
128 */
129 function envToArgs(env: Record<string, string | undefined> = {}): string[] {
130 const args: string[] = [];
131 for (const [k, v] of Object.entries(env)) {
132 if (v === undefined) continue;
133 args.push('-e', `${k}=${v}`);
134 }
135 return args;
136 }
137
138 /**
139 * Adds a bind mount for host ~/.composio into container home.
140 * This allows E2E jobs to inject `user_data.json` once per job.
141 */
142 function addHostComposioMount(dockerArgs: string[], containerComposioPath: string): void {
143 const hostComposioPath = resolve(homedir(), '.composio');
144 if (!existsSync(hostComposioPath)) {
145 return;
146 }
147
148 dockerArgs.push(
149 '--mount',
150 `type=bind,src=${hostComposioPath},dst=${containerComposioPath},readonly`
151 );
152 }
153
154 /**
155 * Executes a command and captures stdout/stderr using Bun shell.
156 */
157 async function exec(cmd: string, args: string[], options: ExecOptions = {}): Promise<ExecResult> {
158 const { cwd, env } = options;
159 const escapedArgs = args.map(escapeShellArg).join(' ');
160
161 let shell = $`${{ raw: cmd }} ${{ raw: escapedArgs }}`.nothrow().quiet();
162
163 if (cwd) {
164 shell = shell.cwd(cwd);
165 }
166
167 if (env) {
168 shell = shell.env({ ...process.env, ...env });
169 }
170
171 const result = await shell;
172
173 return {
174 exitCode: result.exitCode,
175 stdout: result.stdout.toString(),
176 stderr: result.stderr.toString(),
177 };
178 }
179
180 /**
181 * Checks if Docker is available.
182 */
183 export async function checkDocker(options: CheckDockerOptions = {}): Promise<ExecResult> {
184 const repoRoot = options.repoRoot ?? getRepoRoot();
185 return exec('docker', ['info'], { cwd: repoRoot });
186 }
187
188 // bun and pnpm are installed inside the images from mise.toml/mise.lock, so only the
189 // matrixed runtime (node) needs to be resolved on the host and threaded as a build arg.
190 async function getMiseVersion(tool: 'node', repoRoot: string): Promise<string> {
191 const result = await exec('mise', ['current', tool], { cwd: repoRoot });
192 const version = result.stdout.trim();
193 if (result.exitCode !== 0 || !version) {
194 throw new Error(
195 `Failed to resolve ${tool} version from mise.toml: ${result.stderr || result.stdout}`
196 );
197 }
198 return version;
199 }
200
201 /**
202 * Creates image tag for a given Node version.
203 */
204 function imageTagForNodeVersion(nodeVersion: string): string {
205 return `composio-e2e-node:${nodeVersion}`;
206 }
207
208 /**
209 * Parses Node version from an image tag.
210 */
211 function parseNodeVersionFromImageTag(imageTag: string): string | undefined {
212 const match = /^composio-e2e-node:(.+)$/.exec(imageTag);
213 return match?.[1];
214 }
215
216 /**
217 * Ensures a Docker image exists for the given Node version, building it if necessary.
218 */
219 export async function ensureNodeImage(
220 nodeVersion: string,
221 options: EnsureNodeImageOptions = {}
222 ): Promise<string> {
223 if (!nodeVersion || typeof nodeVersion !== 'string') {
224 throw new Error(`ensureNodeImage(${nodeVersion}): nodeVersion must be a non-empty string`);
225 }
226
227 const repoRoot = options.repoRoot ?? getRepoRoot();
228 // `current` must resolve to the mise-pinned version so the tag, build arg, and
229 // digest lookup all reference a concrete Node.js release.
230 const resolvedVersion =
231 nodeVersion === 'current' ? await getMiseVersion('node', repoRoot) : nodeVersion;
232 const dockerfilePath =
233 options.dockerfilePath ?? resolve(repoRoot, 'ts/e2e-tests/_utils/Dockerfile.node');
234 const imageTag = imageTagForNodeVersion(resolvedVersion);
235
236 const inspect = await exec('docker', ['image', 'inspect', imageTag], { cwd: repoRoot });
237 if (inspect.exitCode === 0) {
238 return imageTag;
239 }
240
241 const buildArgs = [
242 'build',
243 '-f',
244 dockerfilePath,
245 '--build-arg',
246 `NODE_VERSION=${resolvedVersion}`,
247 '--build-arg',
248 `NODE_VERSION_DIGEST=${nodeDigestFor(resolvedVersion)}`,
249 ...labelsToArgs(defaultNodeLabels(resolvedVersion)),
250 '-t',
251 imageTag,
252 repoRoot,
253 ];
254
255 const built = await exec('docker', buildArgs, { cwd: repoRoot });
256 if (built.exitCode !== 0) {
257 // Handle race condition: concurrent builds may fail because another build
258 // already tagged the image. If the image now exists, treat it as success.
259 const output = built.stderr || built.stdout;
260 if (output.includes('already exists')) {
261 const recheck = await exec('docker', ['image', 'inspect', imageTag], { cwd: repoRoot });
262 if (recheck.exitCode === 0) {
263 return imageTag;
264 }
265 }
266
267 const err = new Error(`Failed to build Docker image ${imageTag}`);
268 (err as Error & { cause: Error }).cause = new Error(output);
269 throw err;
270 }
271
272 return imageTag;
273 }
274
275 /**
276 * Runs a command in a Node Docker container.
277 */
278 export async function runNodeContainer(options: RunNodeContainerOptions): Promise<ExecResult> {
279 const { imageTag, cmd, cwd, mounts, volumes, env, labels, name, remove } = options;
280
281 // Validate imageTag is provided, as it determines which Docker image to run.
282 if (!imageTag || typeof imageTag !== 'string') {
283 throw new Error('runNodeContainer({ imageTag, ... }): imageTag must be a non-empty string');
284 }
285
286 const nodeVersion = parseNodeVersionFromImageTag(imageTag);
287
288 const dockerArgs = ['run'];
289 if (remove !== false) {
290 dockerArgs.push('--rm');
291 }
292
293 dockerArgs.push(...labelsToArgs({ ...defaultNodeLabels(nodeVersion), ...(labels ?? {}) }));
294
295 // Assign a custom container name for easier identification and cleanup.
296 if (name) {
297 dockerArgs.push('--name', name);
298 }
299
300 // Set the working directory inside the container, defaulting relative paths to /app.
301 if (cwd) {
302 const containerCwd = cwd.startsWith('/') ? cwd : `/app/${cwd}`;
303 dockerArgs.push('--workdir', containerCwd);
304 }
305
306 addHostComposioMount(dockerArgs, '/root/.composio');
307
308 // Pass environment variables to the container for runtime configuration.
309 dockerArgs.push(...envToArgs(env));
310
311 // Configure bind mounts to share host directories with the container.
312 if (mounts) {
313 for (const m of mounts) {
314 // Ensure each mount has required source and target paths.
315 if (!m?.source || !m?.target) {
316 throw new Error('runNodeContainer(...): each mount must have { source, target }');
317 }
318 const parts = [`type=bind`, `src=${m.source}`, `dst=${m.target}`];
319 // Mark mount as read-only to prevent container from modifying host files.
320 if (m.readonly) parts.push('readonly');
321 dockerArgs.push('--mount', parts.join(','));
322 }
323 }
324
325 // Configure named volume mounts for sharing state between container runs.
326 if (volumes) {
327 for (const v of volumes) {
328 // Ensure each volume mount has required volume name and target path.
329 if (!v?.volume || !v?.target) {
330 throw new Error('runNodeContainer(...): each volume must have { volume, target }');
331 }
332 const parts = [`type=volume`, `src=${v.volume}`, `dst=${v.target}`];
333 // Mark mount as read-only if specified.
334 if (v.readonly) parts.push('readonly');
335 dockerArgs.push('--mount', parts.join(','));
336 }
337 }
338
339 dockerArgs.push(imageTag);
340
341 // Handle cmd as array for direct execution without shell interpretation.
342 if (Array.isArray(cmd)) {
343 dockerArgs.push(...cmd.map(String));
344 // Preserve the image's configured PATH instead of resetting it in a login shell.
345 } else if (typeof cmd === 'string' && cmd.length > 0) {
346 dockerArgs.push('sh', '-c', cmd);
347 } else {
348 throw new Error('runNodeContainer({ cmd, ... }): cmd must be a non-empty string or string[]');
349 }
350
351 return exec('docker', dockerArgs);
352 }
353
354 // ============================================================================
355 // CLI Docker utilities
356 // ============================================================================
357
358 /**
359 * Options for running a CLI container.
360 */
361 export interface RunCliContainerOptions {
362 imageTag: string;
363 cmd: string | string[];
364 cwd?: string;
365 mounts?: DockerMount[];
366 /** Named volume mounts (volumes must be created beforehand with `docker volume create`) */
367 volumes?: DockerVolumeMount[];
368 env?: Record<string, string | undefined>;
369 labels?: Record<string, string>;
370 name?: string;
371 /** Whether to auto-remove the container after exit (default: true) */
372 remove?: boolean;
373 }
374
375 /**
376 * Options for ensureCliImage.
377 */
378 export interface EnsureCliImageOptions {
379 repoRoot?: string;
380 dockerfilePath?: string;
381 }
382
383 /**
384 * Creates image tag for a given CLI version.
385 */
386 function imageTagForCliVersion(cliVersion: string): string {
387 return `composio-e2e-cli:${cliVersion}`;
388 }
389
390 /**
391 * Parses CLI version from an image tag.
392 */
393 function parseCliVersionFromImageTag(imageTag: string): string | undefined {
394 const match = /^composio-e2e-cli:(.+)$/.exec(imageTag);
395 return match?.[1];
396 }
397
398 /**
399 * Ensures a Docker image exists for the given CLI version, building it if necessary.
400 */
401 export async function ensureCliImage(
402 cliVersion: string,
403 options: EnsureCliImageOptions = {}
404 ): Promise<string> {
405 if (!cliVersion || typeof cliVersion !== 'string') {
406 throw new Error(`ensureCliImage(${cliVersion}): cliVersion must be a non-empty string`);
407 }
408
409 const repoRoot = options.repoRoot ?? getRepoRoot();
410 const dockerfilePath =
411 options.dockerfilePath ?? resolve(repoRoot, 'ts/e2e-tests/_utils/Dockerfile.cli');
412 const imageTag = imageTagForCliVersion(cliVersion);
413 const nodeVersion = await getMiseVersion('node', repoRoot);
414
415 const revision = await exec('git', ['rev-parse', 'HEAD'], { cwd: repoRoot });
416 if (revision.exitCode !== 0) throw new Error('Cannot identify the CLI checkout revision');
417 const sourceRevision = revision.stdout.trim();
418 // CI prebuilds this image from the checkout. Locally, always let Docker compare COPY
419 // inputs, including dirty and ignored build outputs, against its content cache.
420 if (process.env.GITHUB_ACTIONS === 'true') {
421 const inspect = await exec(
422 'docker',
423 [
424 'image',
425 'inspect',
426 '--format',
427 '{{index .Config.Labels "org.opencontainers.image.revision"}}',
428 imageTag,
429 ],
430 { cwd: repoRoot }
431 );
432 if (inspect.exitCode === 0 && inspect.stdout.trim() === sourceRevision) return imageTag;
433 }
434
435 const buildArgs = [
436 'build',
437 '-f',
438 dockerfilePath,
439 '--build-arg',
440 `CLI_VERSION=${cliVersion}`,
441 '--build-arg',
442 `NODE_VERSION=${nodeVersion}`,
443 '--build-arg',
444 `NODE_VERSION_DIGEST=${nodeDigestFor(nodeVersion)}`,
445 ...labelsToArgs({
446 ...defaultCliLabels(cliVersion),
447 'org.opencontainers.image.revision': sourceRevision,
448 }),
449 '-t',
450 imageTag,
451 repoRoot,
452 ];
453
454 const built = await exec('docker', buildArgs, { cwd: repoRoot });
455 if (built.exitCode !== 0) {
456 const output = built.stderr || built.stdout;
457
458 const err = new Error(`Failed to build Docker image ${imageTag}`);
459 (err as Error & { cause: Error }).cause = new Error(output);
460 throw err;
461 }
462
463 return imageTag;
464 }
465
466 /**
467 * Runs a command in a CLI Docker container.
468 */
469 export async function runCliContainer(options: RunCliContainerOptions): Promise<ExecResult> {
470 const { imageTag, cmd, cwd, mounts, volumes, env, labels, name, remove } = options;
471
472 if (!imageTag || typeof imageTag !== 'string') {
473 throw new Error('runCliContainer({ imageTag, ... }): imageTag must be a non-empty string');
474 }
475
476 const cliVersion = parseCliVersionFromImageTag(imageTag);
477
478 const dockerArgs = ['run'];
479 if (remove !== false) {
480 dockerArgs.push('--rm');
481 }
482
483 dockerArgs.push(...labelsToArgs({ ...defaultCliLabels(cliVersion), ...(labels ?? {}) }));
484
485 if (name) {
486 dockerArgs.push('--name', name);
487 }
488
489 if (cwd) {
490 const containerCwd = cwd.startsWith('/') ? cwd : `/app/${cwd}`;
491 dockerArgs.push('--workdir', containerCwd);
492 }
493
494 // Allow CLI e2e suites to reach host-side mock servers deterministically.
495 dockerArgs.push('--add-host', 'host.docker.internal:host-gateway');
496
497 addHostComposioMount(dockerArgs, '/tmp/.composio');
498
499 dockerArgs.push(...envToArgs(env));
500
501 if (mounts) {
502 for (const m of mounts) {
503 if (!m?.source || !m?.target) {
504 throw new Error('runCliContainer(...): each mount must have { source, target }');
505 }
506 const parts = [`type=bind`, `src=${m.source}`, `dst=${m.target}`];
507 if (m.readonly) parts.push('readonly');
508 dockerArgs.push('--mount', parts.join(','));
509 }
510 }
511
512 if (volumes) {
513 for (const v of volumes) {
514 if (!v?.volume || !v?.target) {
515 throw new Error('runCliContainer(...): each volume must have { volume, target }');
516 }
517 const parts = [`type=volume`, `src=${v.volume}`, `dst=${v.target}`];
518 if (v.readonly) parts.push('readonly');
519 dockerArgs.push('--mount', parts.join(','));
520 }
521 }
522
523 dockerArgs.push(imageTag);
524
525 if (Array.isArray(cmd)) {
526 dockerArgs.push(...cmd.map(String));
527 } else if (typeof cmd === 'string' && cmd.length > 0) {
528 // Use -c (not -lc) for busybox sh compatibility in scratch images
529 dockerArgs.push('sh', '-c', cmd);
530 } else {
531 throw new Error('runCliContainer({ cmd, ... }): cmd must be a non-empty string or string[]');
532 }
533
534 return exec('docker', dockerArgs);
535 }
536
537 // ============================================================================
538 // CLI installer Docker utilities
539 // ============================================================================
540
541 export type InstallShell = 'bash' | 'zsh' | 'fish';
542 export type InstallPlatform = 'linux/amd64' | 'linux/arm64';
543
544 export interface EnsureInstallImageOptions {
545 repoRoot?: string;
546 dockerfilePath?: string;
547 platform?: InstallPlatform;
548 }
549
550 export interface InstallImage {
551 imageTag: string;
552 platform: InstallPlatform;
553 }
554
555 export interface RunInstallContainerOptions {
556 imageTag: string;
557 shell: InstallShell;
558 platform: InstallPlatform;
559 cmd: string[];
560 env?: Record<string, string | undefined>;
561 labels?: Record<string, string>;
562 }
563
564 function imageTagForInstallShell(shell: InstallShell): string {
565 return `composio-e2e-install:${shell}`;
566 }
567
568 function defaultInstallLabels(shell: InstallShell): Record<string, string> {
569 return {
570 'composio.e2e': 'true',
571 'composio.runtime': 'cli-install',
572 'composio.install_shell': shell,
573 };
574 }
575
576 export async function ensureInstallImage(
577 shell: InstallShell,
578 options: EnsureInstallImageOptions = {}
579 ): Promise<InstallImage> {
580 if (shell !== 'bash' && shell !== 'zsh' && shell !== 'fish') {
581 throw new Error(`ensureInstallImage(${shell}): shell must be bash, zsh, or fish`);
582 }
583
584 const repoRoot = options.repoRoot ?? getRepoRoot();
585 const dockerfilePath =
586 options.dockerfilePath ?? resolve(repoRoot, 'ts/e2e-tests/_utils/Dockerfile.install');
587 const platform = options.platform ?? (process.arch === 'arm64' ? 'linux/arm64' : 'linux/amd64');
588 const architecture = platform === 'linux/arm64' ? 'arm64' : 'amd64';
589 const imageTag = imageTagForInstallShell(shell);
590 const inspect = await exec(
591 'docker',
592 ['image', 'inspect', '--format', '{{.Architecture}}', imageTag],
593 { cwd: repoRoot }
594 );
595 if (inspect.exitCode === 0 && inspect.stdout.trim() === architecture) {
596 return { imageTag, platform };
597 }
598
599 const built = await exec(
600 'docker',
601 [
602 'build',
603 '--platform',
604 platform,
605 '-f',
606 dockerfilePath,
607 '--build-arg',
608 `INSTALL_SHELL=${shell}`,
609 ...labelsToArgs(defaultInstallLabels(shell)),
610 '-t',
611 imageTag,
612 repoRoot,
613 ],
614 { cwd: repoRoot }
615 );
616 if (built.exitCode !== 0) {
617 // Handle race condition: concurrent builds may fail because another build
618 // already tagged the image. If the image now exists, treat it as success.
619 const output = built.stderr || built.stdout;
620 if (output.includes('already exists')) {
621 const recheck = await exec(
622 'docker',
623 ['image', 'inspect', '--format', '{{.Architecture}}', imageTag],
624 { cwd: repoRoot }
625 );
626 if (recheck.exitCode === 0 && recheck.stdout.trim() === architecture) {
627 return { imageTag, platform };
628 }
629 }
630
631 const err = new Error(`Failed to build Docker image ${imageTag}`);
632 (err as Error & { cause: Error }).cause = new Error(output);
633 throw err;
634 }
635
636 return { imageTag, platform };
637 }
638
639 export async function runInstallContainer(
640 options: RunInstallContainerOptions
641 ): Promise<ExecResult> {
642 const { imageTag, shell, platform, cmd, env, labels } = options;
643 if (!Array.isArray(cmd) || cmd.length === 0) {
644 throw new Error('runInstallContainer({ cmd, ... }): cmd must be a non-empty array');
645 }
646
647 const dockerArgs = [
648 'run',
649 '--rm',
650 '--platform',
651 platform,
652 ...labelsToArgs({ ...defaultInstallLabels(shell), ...(labels ?? {}) }),
653 '--add-host',
654 'host.docker.internal:host-gateway',
655 ];
656
657 dockerArgs.push(...envToArgs(env));
658
659 // Installer tests intentionally do not mount the host's ~/.composio directory.
660 dockerArgs.push(imageTag, ...cmd.map(String));
661 return exec('docker', dockerArgs);
662 }