@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 110 lines 2.79 KB
Raw
1 #!/usr/bin/env bun
2
3 import { spawnSync } from 'node:child_process';
4
5 const DEFAULT_LOCAL_FLOWS = ['gateway', 'mercury'] as const;
6 const EVE_BIN = process.env.EVE_BIN ?? './node_modules/.bin/eve';
7 const extraArgs = process.argv.slice(2);
8
9 type EvalRun = {
10 name: string;
11 env?: Record<string, string>;
12 url?: string;
13 };
14
15 const parseList = (value: string | undefined): string[] =>
16 value
17 ?.split(',')
18 .map(item => item.trim())
19 .filter(Boolean) ?? [];
20
21 const parseRemoteTargets = (value: string | undefined): EvalRun[] =>
22 parseList(value).map(entry => {
23 const separator = entry.indexOf('=');
24 if (separator === -1) {
25 throw new Error(
26 `Invalid DOCS_AGENT_EVAL_TARGETS entry "${entry}". Use name=https://deployment.example.`
27 );
28 }
29
30 const name = entry.slice(0, separator).trim();
31 const url = entry.slice(separator + 1).trim();
32
33 if (!name || !url) {
34 throw new Error(
35 `Invalid DOCS_AGENT_EVAL_TARGETS entry "${entry}". Both name and URL are required.`
36 );
37 }
38
39 return { name, url };
40 });
41
42 const buildRuns = (): EvalRun[] => {
43 const remoteTargets = parseRemoteTargets(process.env.DOCS_AGENT_EVAL_TARGETS);
44
45 if (remoteTargets.length > 0) {
46 return remoteTargets;
47 }
48
49 const flows = parseList(process.env.DOCS_AGENT_EVAL_FLOWS);
50 const selectedFlows = flows.length > 0 ? flows : [...DEFAULT_LOCAL_FLOWS];
51
52 return selectedFlows.map(flow => ({
53 name: flow,
54 env: { DOCS_AGENT_MODEL_FLOW: flow },
55 }));
56 };
57
58 const warnForMissingCredentials = (run: EvalRun) => {
59 const flow = run.env?.DOCS_AGENT_MODEL_FLOW;
60
61 if (flow === 'mercury' && !process.env.INCEPTION_API_KEY) {
62 console.warn(
63 '[eval-agent-flows] INCEPTION_API_KEY is not set; Mercury evals will fail or skip model calls.'
64 );
65 }
66
67 if (flow === 'gateway' && !process.env.AI_GATEWAY_API_KEY && !process.env.VERCEL_OIDC_TOKEN) {
68 console.warn(
69 '[eval-agent-flows] AI_GATEWAY_API_KEY/VERCEL_OIDC_TOKEN is not set; gateway evals will fail or skip model calls.'
70 );
71 }
72 };
73
74 const runEval = (run: EvalRun) => {
75 const args = ['eval', 'docs-agent', '--skip-report'];
76
77 if (run.url) {
78 args.push('--url', run.url);
79 }
80
81 args.push(...extraArgs);
82
83 console.log(`\n## ${run.url ? 'Remote target' : 'Local model flow'}: ${run.name}`);
84 console.log(`$ ${EVE_BIN} ${args.join(' ')}`);
85 warnForMissingCredentials(run);
86
87 return (
88 spawnSync(EVE_BIN, args, {
89 env: { ...process.env, ...run.env },
90 stdio: 'inherit',
91 }).status ?? 1
92 );
93 };
94
95 const runs = buildRuns();
96 let failed = 0;
97
98 for (const run of runs) {
99 const status = runEval(run);
100 if (status !== 0) {
101 failed += 1;
102 }
103 }
104
105 if (failed > 0) {
106 console.error(`\n${failed}/${runs.length} docs-agent eval run(s) failed.`);
107 process.exit(1);
108 }
109
110 console.log(`\nAll ${runs.length} docs-agent eval run(s) passed.`);