@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
mjs 736 lines 29.9 KB
Raw
1 #!/usr/bin/env node
2 // Examples runner: sweeps the entrypoints in examples-manifest.json against
3 // the live backend, recording per-entry results and Composio traces.
4 //
5 // node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--exclude-toolkits a,b] [--llm live|mock]
6 // node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--exclude-toolkits a,b] [--sample N] [--seed S]
7 // node harness/run.mjs selftest
8 //
9 // Exit codes: 0 ok · 1 findings (red entries / failed expectations) · 2 usage/env error.
10
11 import { spawn } from 'node:child_process';
12 import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, rmSync } from 'node:fs';
13 import { dirname, join, relative, resolve } from 'node:path';
14 import { fileURLToPath } from 'node:url';
15 import {
16 describeExclusions,
17 emptySelectionMessage,
18 entryToolkits,
19 loadManifest,
20 parseOption,
21 parseSelectionOptions,
22 requiredBrowserGrantToolkits,
23 requiresDemoToolkit,
24 selectManifestEntries,
25 validateManifestEntries,
26 } from './manifest.mjs';
27 import { resolveBackendBaseUrl, STAGING_BASE_URL } from './backend-url.mjs';
28 import { HarnessError } from './errors.mjs';
29
30 const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
31 const ARTIFACTS = join(ROOT, '.artifacts', 'examples-parity');
32 const AIMOCK_VERSION = '1.47.0';
33 const AIMOCK_PORT = Number(process.env.AIMOCK_PORT ?? 4010);
34
35 // Set by cmdSweep when --llm mock: LLM traffic goes to a local aimock server
36 // and provider keys are overridden so a mock sweep can never spend tokens.
37 let LLM_MOCK = false;
38
39 const args = process.argv.slice(2);
40 const cmd = args[0];
41 const opt = (name, fallback) => parseOption(args, name, fallback);
42
43 const fail = (msg, code = 2) => {
44 throw new HarnessError(msg, code);
45 };
46
47 const selectEntries = () => selectManifestEntries(loadManifest(), parseSelectionOptions(args));
48
49 const reportExclusions = selection => {
50 for (const line of describeExclusions(selection)) console.log(line);
51 };
52
53 const baseUrl = () => {
54 try {
55 return resolveBackendBaseUrl();
56 } catch (error) {
57 fail(error instanceof Error ? error.message : String(error));
58 }
59 };
60
61 // Deterministic PRNG so --seed reproduces a negative-control sample.
62 const mulberry32 = (a) => () => {
63 a |= 0; a = (a + 0x6d2b79f5) | 0;
64 let t = Math.imul(a ^ (a >>> 15), 1 | a);
65 t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
66 return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
67 };
68
69 const runId = () => new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14) + '-' + Math.random().toString(36).slice(2, 8);
70
71 /** Spawn one entry; resolve {exit, readinessMatched, timedOut, durationMs}. */
72 const runEntry = (entry, env, { garbage = false } = {}) =>
73 new Promise((resolveRun) => {
74 const isTs = entry.lang === 'ts';
75 let cmdline;
76 if (isTs && entry.script) {
77 cmdline = ['pnpm', '--filter', entry.pkg, 'run', entry.script];
78 } else if (isTs) {
79 // pnpm --filter runs with the example package as cwd; derive the package
80 // dir from the entry file path (ts/examples/<dir>/src/...).
81 const pkgDir = entry.file.split('/').slice(0, 3).join('/');
82 const srcRel = relative(join(ROOT, pkgDir), join(ROOT, entry.file));
83 cmdline = ['pnpm', '--filter', entry.pkg, 'exec', 'tsx', srcRel];
84 } else {
85 cmdline = ['uv', 'run', '--project', 'python'];
86 for (const dep of entry.pyWith ?? []) cmdline.push('--with', dep);
87 cmdline.push('python', join(ROOT, entry.file));
88 }
89
90 const child = spawn(cmdline[0], cmdline.slice(1), {
91 cwd: ROOT,
92 env,
93 detached: true,
94 stdio: ['ignore', 'pipe', 'pipe'],
95 });
96
97 const started = Date.now();
98 const timeoutMs = (entry.timeoutSec ?? 120) * 1000;
99 let readinessMatched = false;
100 let timedOut = false;
101 let settled = false;
102 let outputTail = '';
103 const readiness = entry.tier === '3' && entry.readiness ? new RegExp(entry.readiness) : null;
104
105 const killTree = () => {
106 try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ }
107 setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ } }, 10_000).unref();
108 };
109
110 const onChunk = (chunk) => {
111 const text = chunk.toString();
112 outputTail = (outputTail + text).slice(-4000);
113 if (readiness && !readinessMatched && readiness.test(outputTail)) {
114 readinessMatched = true;
115 // Grace period so the in-flight backend call lands in the trace, then stop the listener.
116 setTimeout(killTree, 3000).unref();
117 }
118 };
119 child.stdout.on('data', onChunk);
120 child.stderr.on('data', onChunk);
121
122 const timer = setTimeout(() => {
123 timedOut = true;
124 killTree();
125 }, timeoutMs);
126
127 child.on('exit', (code, signal) => {
128 if (settled) return;
129 settled = true;
130 clearTimeout(timer);
131 resolveRun({
132 exit: code ?? (signal ? 128 : 1),
133 readinessMatched,
134 timedOut,
135 durationMs: Date.now() - started,
136 outputTail: garbage ? undefined : outputTail.slice(-1500),
137 });
138 });
139 child.on('error', () => {
140 if (settled) return;
141 settled = true;
142 clearTimeout(timer);
143 resolveRun({ exit: 127, readinessMatched: false, timedOut: false, durationMs: Date.now() - started });
144 });
145 });
146
147 const isGreen = (entry, r) => {
148 if (entry.tier === '3') return r.readinessMatched || (!r.timedOut && r.exit === 0);
149 return !r.timedOut && r.exit === 0;
150 };
151
152 const buildEnv = (entry, runDir, { garbage = false } = {}) => {
153 const traceFile = join(runDir, 'traces', `${entry.id.replace(/[/]/g, '__')}.jsonl`);
154 mkdirSync(dirname(traceFile), { recursive: true });
155 const env = { ...process.env };
156 env.COMPOSIO_BASE_URL = baseUrl();
157 env.COMPOSIO_TRACE_FILE = traceFile;
158 env.COMPOSIO_LOG_LEVEL ??= 'error';
159 // The outbound-email denylist is fixed by the shims; never overridable per run.
160 delete env.COMPOSIO_TOOL_DENYLIST;
161 if (entry.lang === 'ts') {
162 env.NODE_OPTIONS = `${env.NODE_OPTIONS ?? ''} --import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}`.trim();
163 } else {
164 env.PYTHONPATH = `${join(ROOT, 'harness', 'trace-py')}${env.PYTHONPATH ? `:${env.PYTHONPATH}` : ''}`;
165 }
166 if (garbage) {
167 env.COMPOSIO_API_KEY = 'nc-invalid-0000000000000000';
168 env.OPENAI_API_KEY = 'nc-invalid';
169 env.ANTHROPIC_API_KEY = 'nc-invalid';
170 env.GEMINI_API_KEY = 'nc-invalid';
171 env.NOTION_API_KEY = 'nc-invalid';
172 } else if (LLM_MOCK) {
173 const mock = `http://127.0.0.1:${AIMOCK_PORT}`;
174 env.OPENAI_BASE_URL = `${mock}/v1`;
175 // langchain/litellm read the older variable name.
176 env.OPENAI_API_BASE = `${mock}/v1`;
177 env.ANTHROPIC_BASE_URL = mock;
178 env.OPENAI_API_KEY = 'aimock';
179 env.ANTHROPIC_API_KEY = 'aimock';
180 }
181 return { env, traceFile };
182 };
183
184 const LLM_KEYS = new Set(['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GEMINI_API_KEY']);
185
186 const missingEnv = (entry) => {
187 const wanted = [...(entry.env ?? []), ...(entry.ids ?? [])].filter(
188 (k) => !(LLM_MOCK && LLM_KEYS.has(k))
189 );
190 if (entry.backend !== false) wanted.unshift('COMPOSIO_API_KEY');
191 return wanted.filter((k) => !process.env[k]);
192 };
193
194 /** Run entries with bounded concurrency; serial entries run last, alone. */
195 const runPool = async (entries, runDir, { garbage = false, concurrency = 4 } = {}) => {
196 const parallel = entries.filter((e) => !e.serial);
197 const serial = entries.filter((e) => e.serial);
198 const results = [];
199 const worker = async (queue) => {
200 for (;;) {
201 const entry = queue.shift();
202 if (!entry) return;
203 if (LLM_MOCK && entry.llmMock === false) {
204 results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: 'llm-mock-unsupported' });
205 console.log(` ~ ${entry.id} skipped (llm-mock-unsupported)`);
206 continue;
207 }
208 const missing = garbage ? [] : missingEnv(entry);
209 if (missing.length > 0) {
210 results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: `missing-env:${missing.join(',')}` });
211 console.log(` ~ ${entry.id} skipped (${missing.join(',')})`);
212 continue;
213 }
214 const { env, traceFile } = buildEnv(entry, runDir, { garbage });
215 const r = await runEntry(entry, env, { garbage });
216 // An entry that tripped the outbound-email guard is red no matter how it
217 // exited: catching the refusal and continuing is not coverage.
218 const blocked =
219 existsSync(traceFile) && readFileSync(traceFile, 'utf8').includes('"s":"BLOCKED"');
220 const green = !blocked && isGreen(entry, r);
221 results.push({
222 id: entry.id,
223 tier: entry.tier,
224 status: green ? 'green' : 'red',
225 ...(blocked ? { reason: 'blocked-tool' } : {}),
226 exit: r.exit,
227 timedOut: r.timedOut,
228 readinessMatched: r.readinessMatched,
229 durationMs: r.durationMs,
230 traceFile: relative(ROOT, traceFile),
231 outputTail: green ? undefined : r.outputTail,
232 });
233 console.log(` ${green ? '✓' : '✗'} ${entry.id} (${(r.durationMs / 1000).toFixed(1)}s${r.timedOut ? ', timeout' : ''})`);
234 }
235 };
236 const queue = [...parallel];
237 await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(queue.length, 1)) }, () => worker(queue)));
238 for (const entry of serial) await worker([entry]);
239 return results;
240 };
241
242 // --- candidate swap (TS) --------------------------------------------------
243
244 const WORKSPACE_YAML = join(ROOT, 'pnpm-workspace.yaml');
245 const LOCKFILE = join(ROOT, 'pnpm-lock.yaml');
246
247 const sh = (cmdline, opts = {}) =>
248 new Promise((resolveSh, rejectSh) => {
249 const child = spawn(cmdline[0], cmdline.slice(1), { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'], ...opts });
250 let out = '';
251 child.stdout.on('data', (c) => (out += c));
252 child.stderr.on('data', (c) => (out += c));
253 child.on('exit', (code) => (code === 0 ? resolveSh(out) : rejectSh(new Error(`${cmdline.join(' ')} → exit ${code}\n${out.slice(-2000)}`))));
254 });
255
256 /**
257 * Resolve @composio/client as @composio/core sees it. The realpath identifies
258 * the installed copy: a `file:` override lands in its own store directory even
259 * when the tarball reports the same version as the catalog pin, so comparing
260 * paths (not versions) is what tells a real swap from a no-op.
261 */
262 const resolvedTsClient = async () => {
263 const out = await sh(['node', '-e', "const path=require('path');const fs=require('fs');const manifest=require.resolve('@composio/client/package.json', {paths:[path.join(process.cwd(),'ts/packages/core')]});console.log(JSON.stringify({path:fs.realpathSync(manifest),version:JSON.parse(fs.readFileSync(manifest,'utf8')).version}))"]);
264 return JSON.parse(out.trim().split('\n').pop());
265 };
266
267 const assertTsCandidateApplied = (baseline, candidate) => {
268 if (candidate.path === baseline.path) {
269 fail(
270 `override did not take effect: @composio/client still resolves to the installed baseline at ${candidate.path} (${candidate.version})`
271 );
272 }
273 };
274
275 const applyTsCandidate = async (tarball) => {
276 if (!existsSync(tarball)) fail(`COMPOSIO_CLIENT_TARBALL not found: ${tarball}`);
277 const baseline = await resolvedTsClient();
278 const yaml = readFileSync(WORKSPACE_YAML, 'utf8');
279 if (/^overrides:$/m.test(yaml)) {
280 // Merge into the repo's existing overrides block (e.g. security pins);
281 // restoreTsBaseline() writes back the exact pre-run snapshot either way.
282 writeFileSync(WORKSPACE_YAML, yaml.replace(/^overrides:$/m, `overrides:\n '@composio/client': file:${tarball}`));
283 } else {
284 writeFileSync(WORKSPACE_YAML, `${yaml}\noverrides:\n '@composio/client': file:${tarball}\n`);
285 }
286 await sh(['pnpm', 'install', '--no-frozen-lockfile']);
287 const candidate = await resolvedTsClient();
288 assertTsCandidateApplied(baseline, candidate);
289 console.log(`candidate @composio/client resolved: ${candidate.version} (${candidate.path})`);
290 await sh(['pnpm', 'run', 'build:packages']);
291 };
292
293 const snapshotFiles = (paths) => new Map(paths.map((path) => [path, readFileSync(path)]));
294
295 const restoreFiles = (snapshot) => {
296 for (const [path, contents] of snapshot) writeFileSync(path, contents);
297 };
298
299 const snapshotTsBaseline = () => snapshotFiles([WORKSPACE_YAML, LOCKFILE]);
300
301 const restoreTsBaseline = async (snapshot) => {
302 restoreFiles(snapshot);
303 await sh(['pnpm', 'install', '--frozen-lockfile']);
304 };
305
306 /**
307 * Resolve composio-client as the python project sees it. `uv run --with <wheel>`
308 * layers an ephemeral overlay on the project environment, so the realpath tells
309 * a real swap from a no-op even when the wheel matches the pinned version.
310 */
311 const resolvedPyClient = async (extraArgs = []) => {
312 const out = await sh(['uv', 'run', '--project', 'python', ...extraArgs, 'python', '-c', 'import composio_client, importlib.metadata, json, os; print(json.dumps({"path": os.path.realpath(composio_client.__file__), "version": importlib.metadata.version("composio-client")}))']);
313 return JSON.parse(out.trim().split('\n').pop());
314 };
315
316 const assertPyCandidateApplied = (baseline, candidate) => {
317 if (candidate.path === baseline.path) {
318 fail(
319 `python candidate did not take effect: composio-client still resolves to the installed baseline at ${candidate.path} (${candidate.version})`
320 );
321 }
322 };
323
324 const verifyPyCandidate = async (wheel) => {
325 if (!existsSync(wheel)) fail(`COMPOSIO_CLIENT_WHEEL not found: ${wheel}`);
326 const baseline = await resolvedPyClient();
327 const candidate = await resolvedPyClient(['--with', wheel]);
328 assertPyCandidateApplied(baseline, candidate);
329 return candidate.version;
330 };
331
332 const PY_PROJECT = join(ROOT, 'python', 'pyproject.toml');
333 const UV_LOCK = join(ROOT, 'uv.lock');
334 const PY_CLIENT_PIN = /^(\s*)"composio-client==[^"]+",$/m;
335
336 /**
337 * Drop the exact `composio-client==` pin from the python project.
338 *
339 * A candidate sweep layers the wheel onto every entry with `uv run --with`, but
340 * several entries also install the local `./python` project, whose exact pin the
341 * wheel contradicts — uv then refuses the whole overlay ("no solution found")
342 * and those entries go red for a packaging reason rather than a client one.
343 * Without the pin the wheel is the only `composio-client` in play. Baseline
344 * sweeps never call this, and restorePyBaseline() writes the pre-run files back
345 * either way.
346 */
347 const relaxPyClientPin = (toml) => {
348 if (!PY_CLIENT_PIN.test(toml)) {
349 fail(`no exact composio-client pin found in ${relative(ROOT, PY_PROJECT)}`);
350 }
351 return toml.replace(PY_CLIENT_PIN, '$1"composio-client",');
352 };
353
354 const snapshotPyBaseline = () => snapshotFiles([PY_PROJECT, UV_LOCK]);
355
356 const restorePyBaseline = async (snapshot) => {
357 restoreFiles(snapshot);
358 await sh(['uv', 'sync', '--project', 'python', '--frozen']);
359 };
360
361 // --- llm mock ---------------------------------------------------------------
362
363 /** Start aimock serving harness/llm-mock/fixtures; resolve once it answers. */
364 const startLlmMock = async (runDir) => {
365 const mockDir = join(ROOT, 'harness', 'llm-mock');
366 const logFile = join(runDir, 'aimock.log');
367 const child = spawn(
368 'npx',
369 ['-y', `@copilotkit/aimock@${AIMOCK_VERSION}`, '-c', 'aimock.json', '-p', String(AIMOCK_PORT), '--host', '127.0.0.1'],
370 { cwd: mockDir, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }
371 );
372 child.stdout.on('data', (c) => appendFileSync(logFile, c));
373 child.stderr.on('data', (c) => appendFileSync(logFile, c));
374
375 // First run may download the package; allow a generous readiness window.
376 const deadline = Date.now() + 120_000;
377 for (;;) {
378 try {
379 const res = await fetch(`http://127.0.0.1:${AIMOCK_PORT}/v1/models`);
380 if (res.ok) break;
381 } catch { /* not up yet */ }
382 if (Date.now() > deadline) {
383 try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ }
384 fail(`aimock did not become ready on :${AIMOCK_PORT} (see ${relative(ROOT, logFile)})`);
385 }
386 await new Promise((r) => setTimeout(r, 500));
387 }
388 console.log(`aimock ${AIMOCK_VERSION} ready on :${AIMOCK_PORT}`);
389 return () => {
390 try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ }
391 };
392 };
393
394 // --- commands ---------------------------------------------------------------
395
396 const cmdSweep = async () => {
397 const client = opt('client', 'baseline');
398 if (!['baseline', 'candidate'].includes(client)) fail(`--client must be baseline|candidate`);
399 const llm = opt('llm', 'live');
400 if (!['live', 'mock'].includes(llm)) fail(`--llm must be live|mock`);
401 LLM_MOCK = llm === 'mock';
402 const selection = selectEntries();
403 reportExclusions(selection);
404 const entries = selection.entries;
405 if (entries.length === 0) fail(emptySelectionMessage(selection));
406 if (!process.env.COMPOSIO_API_KEY) fail('COMPOSIO_API_KEY is required for a sweep (disposable examples-project key)', 2);
407
408 const id = `${runId()}-${client}${LLM_MOCK ? '-mock' : ''}`;
409 const runDir = join(ARTIFACTS, id);
410 mkdirSync(join(runDir, 'traces'), { recursive: true });
411
412 let tsBaselineSnapshot;
413 let pyBaselineSnapshot;
414 let stopLlmMock;
415 const pyWheel = process.env.COMPOSIO_CLIENT_WHEEL;
416 try {
417 if (LLM_MOCK) stopLlmMock = await startLlmMock(runDir);
418 if (client === 'candidate') {
419 const wantTs = entries.some((e) => e.lang === 'ts');
420 const wantPy = entries.some((e) => e.lang === 'py');
421 if (wantTs) {
422 if (!process.env.COMPOSIO_CLIENT_TARBALL) fail('candidate sweep with ts entries needs COMPOSIO_CLIENT_TARBALL');
423 tsBaselineSnapshot = snapshotTsBaseline();
424 await applyTsCandidate(process.env.COMPOSIO_CLIENT_TARBALL);
425 }
426 if (wantPy) {
427 if (!pyWheel) fail('candidate sweep with py entries needs COMPOSIO_CLIENT_WHEEL');
428 const v = await verifyPyCandidate(pyWheel);
429 console.log(`candidate composio-client (py) resolved: ${v}`);
430 pyBaselineSnapshot = snapshotPyBaseline();
431 writeFileSync(PY_PROJECT, relaxPyClientPin(readFileSync(PY_PROJECT, 'utf8')));
432 for (const e of entries) if (e.lang === 'py') e.pyWith = [...(e.pyWith ?? []), pyWheel];
433 }
434 }
435
436 console.log(`sweep ${id}: ${entries.length} entries → ${runDir}`);
437 const results = await runPool(entries, runDir, { concurrency: Number(opt('concurrency', '4')) });
438 for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`);
439 const counts = {
440 green: results.filter((r) => r.status === 'green').length,
441 red: results.filter((r) => r.status === 'red').length,
442 skipped: results.filter((r) => r.status === 'skipped').length,
443 };
444 writeFileSync(
445 join(runDir, 'summary.json'),
446 JSON.stringify(
447 {
448 runId: id,
449 client,
450 llm,
451 baseUrl: baseUrl(),
452 counts,
453 selection: {
454 excludedToolkits: selection.excludedToolkits,
455 excludedEntries: selection.excludedEntries.map(entry => entry.id),
456 },
457 startedAt: new Date().toISOString(),
458 },
459 null,
460 2
461 )
462 );
463 console.log(`sweep ${id}: ${counts.green} green / ${counts.red} red / ${counts.skipped} skipped`);
464 process.exitCode = counts.red > 0 ? 1 : 0;
465 } finally {
466 if (stopLlmMock) stopLlmMock();
467 if (tsBaselineSnapshot) await restoreTsBaseline(tsBaselineSnapshot);
468 if (pyBaselineSnapshot) await restorePyBaseline(pyBaselineSnapshot);
469 }
470 };
471
472 const cmdNeg = async () => {
473 const selection = selectEntries();
474 reportExclusions(selection);
475 let entries = selection.entries.filter((e) => e.backend !== false);
476 const sample = opt('sample');
477 if (sample) {
478 const rand = mulberry32(Number(opt('seed', '42')));
479 entries = [...entries].sort(() => rand() - 0.5).slice(0, Number(sample));
480 }
481 const id = `${runId()}-neg`;
482 const runDir = join(ARTIFACTS, id);
483 mkdirSync(join(runDir, 'traces'), { recursive: true });
484 console.log(`negative controls ${id}: ${entries.length} entries (garbage credentials; expectation: red)`);
485 const results = await runPool(entries, runDir, { garbage: true, concurrency: Number(opt('concurrency', '6')) });
486 const swallowers = results.filter((r) => r.status === 'green').map((r) => r.id);
487 for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`);
488 writeFileSync(join(runDir, 'summary.json'), JSON.stringify({ runId: id, mode: 'neg', total: results.length, swallowers }, null, 2));
489 if (swallowers.length > 0) {
490 console.log(`NEGATIVE-CONTROL FAILURES (green under garbage credentials — these swallow errors):`);
491 for (const s of swallowers) console.log(` ✗ ${s}`);
492 process.exitCode = 1;
493 } else {
494 console.log('all negative controls red, as expected');
495 }
496 };
497
498 const cmdSelftest = async () => {
499 const runDir = join(ARTIFACTS, `${runId()}-selftest`);
500 mkdirSync(join(runDir, 'traces'), { recursive: true });
501 let failures = 0;
502 const check = (name, ok) => {
503 console.log(` ${ok ? '✓' : '✗'} ${name}`);
504 if (!ok) failures += 1;
505 };
506
507 // 0. backend URL: staging is the default, any other bare https root is
508 // honoured as given, and anything structurally unusable is refused.
509 check('staging backend is the default', resolveBackendBaseUrl(undefined) === STAGING_BASE_URL);
510 check(
511 'a non-staging bare https root is honoured',
512 resolveBackendBaseUrl('https://backend.composio.dev') === 'https://backend.composio.dev'
513 );
514 const refuses = (value) => {
515 try {
516 resolveBackendBaseUrl(value);
517 return false;
518 } catch {
519 return true;
520 }
521 };
522 check('a base URL carrying a path is refused', refuses('https://backend.composio.dev/api/v3'));
523 check('a base URL carrying a query is refused', refuses('https://backend.composio.dev/?x=1'));
524 check(
525 'a base URL carrying credentials is refused',
526 refuses('https://user:pass@backend.composio.dev')
527 );
528 check('a plaintext base URL is refused', refuses('http://backend.composio.dev'));
529 check('an unparseable base URL is refused', refuses('not-a-url'));
530
531 let cleanupRan = false;
532 let cleanupError;
533 try {
534 try {
535 fail('expected selftest failure');
536 } finally {
537 cleanupRan = true;
538 }
539 } catch (error) {
540 cleanupError = error;
541 }
542 check(
543 'usage failures unwind through cleanup',
544 cleanupRan && cleanupError instanceof HarnessError
545 );
546
547 const installedBaseline = {
548 path: '/repo/node_modules/.pnpm/@composio+client@2.0.0-rc.3/node_modules/@composio/client',
549 version: '2.0.0-rc.3',
550 };
551 let noOpCandidateRejected = false;
552 try {
553 assertTsCandidateApplied(installedBaseline, { ...installedBaseline });
554 } catch {
555 noOpCandidateRejected = true;
556 }
557 check('candidate swap rejects the installed baseline copy', noOpCandidateRejected);
558
559 let sameVersionCandidateAccepted = true;
560 try {
561 assertTsCandidateApplied(installedBaseline, {
562 path: '/repo/node_modules/.pnpm/file+client.tgz/node_modules/@composio/client',
563 version: installedBaseline.version,
564 });
565 } catch {
566 sameVersionCandidateAccepted = false;
567 }
568 check('candidate swap accepts a tarball reporting the pinned version', sameVersionCandidateAccepted);
569
570 const installedPyBaseline = {
571 path: '/repo/.venv/lib/python3.12/site-packages/composio_client/__init__.py',
572 version: '2.0.0rc5',
573 };
574 let noOpPyCandidateRejected = false;
575 try {
576 assertPyCandidateApplied(installedPyBaseline, { ...installedPyBaseline });
577 } catch {
578 noOpPyCandidateRejected = true;
579 }
580 check('python candidate swap rejects the installed baseline copy', noOpPyCandidateRejected);
581
582 let sameVersionPyCandidateAccepted = true;
583 try {
584 assertPyCandidateApplied(installedPyBaseline, {
585 path: '/cache/uv/archive-v0/abc123/lib/python3.12/site-packages/composio_client/__init__.py',
586 version: installedPyBaseline.version,
587 });
588 } catch {
589 sameVersionPyCandidateAccepted = false;
590 }
591 check('python candidate swap accepts a wheel reporting the pinned version', sameVersionPyCandidateAccepted);
592
593 let invalidManifestIsUsageError = false;
594 try {
595 validateManifestEntries([{ id: 'missing-required-fields' }]);
596 } catch (error) {
597 invalidManifestIsUsageError = error instanceof HarnessError && error.exitCode === 2;
598 }
599 check('invalid manifest entries are usage errors', invalidManifestIsUsageError);
600
601 const manifest = loadManifest();
602 const withoutGoogleDrive = selectManifestEntries(manifest, {
603 excludeToolkits: 'googledrive',
604 });
605 check(
606 'Google Drive exclusion removes every dependent entry and preserves the rest',
607 withoutGoogleDrive.excludedEntries.length > 0 &&
608 withoutGoogleDrive.excludedEntries.every(entry =>
609 entryToolkits(entry).includes('googledrive')
610 ) &&
611 withoutGoogleDrive.entries.every(entry => !entryToolkits(entry).includes('googledrive')) &&
612 withoutGoogleDrive.entries.length + withoutGoogleDrive.excludedEntries.length ===
613 selectManifestEntries(manifest).entries.length
614 );
615 check(
616 'Google Drive exclusion removes it from provisioning',
617 !requiredBrowserGrantToolkits(withoutGoogleDrive.entries).some(
618 toolkit => toolkit.slug === 'googledrive'
619 )
620 );
621 const apiKeyOnly = selectManifestEntries(manifest, {
622 ids: 'ts/connected-accounts/api-key',
623 }).entries;
624 check(
625 'selected API-key example provisions only its demo toolkit',
626 requiredBrowserGrantToolkits(apiKeyOnly).length === 0 && requiresDemoToolkit(apiKeyOnly)
627 );
628 const implicitGmail = selectManifestEntries(manifest, {
629 ids: 'py/fastapi_app',
630 }).entries;
631 check(
632 'resource identifiers contribute implicit toolkit requirements',
633 requiredBrowserGrantToolkits(implicitGmail).some(toolkit => toolkit.slug === 'gmail')
634 );
635
636 const cleanupFixture = join(runDir, 'cleanup-fixture.txt');
637 writeFileSync(cleanupFixture, 'user contents');
638 const cleanupSnapshot = snapshotFiles([cleanupFixture]);
639 writeFileSync(cleanupFixture, 'candidate contents');
640 restoreFiles(cleanupSnapshot);
641 check('cleanup restores pre-run file contents', readFileSync(cleanupFixture, 'utf8') === 'user contents');
642
643 // The python candidate swap only resolves once the project's exact
644 // composio-client pin is out of the way; see relaxPyClientPin.
645 const pinnedProject = readFileSync(PY_PROJECT, 'utf8');
646 const relaxedProject = relaxPyClientPin(pinnedProject);
647 check('the python project pins composio-client exactly', /"composio-client==/.test(pinnedProject));
648 check(
649 'relaxing the pin keeps composio-client a dependency',
650 /^\s*"composio-client",$/m.test(relaxedProject) && !/"composio-client==/.test(relaxedProject)
651 );
652 check(
653 'relaxing the pin touches nothing else',
654 relaxedProject.replace(/^\s*"composio-client",$/m, '') ===
655 pinnedProject.replace(/^\s*"composio-client==[^"]+",$/m, '')
656 );
657 let unpinnedProjectRejected = false;
658 try {
659 relaxPyClientPin(relaxedProject);
660 } catch {
661 unpinnedProjectRejected = true;
662 }
663 check('relaxing an already-unpinned project is refused', unpinnedProjectRejected);
664
665 // 1. known-good: error-handling-demo (no backend, no keys).
666 const good = loadManifest().find((e) => e.id === 'ts/error-handling-demo/index');
667 const { env: goodEnv } = buildEnv(good, runDir);
668 delete goodEnv.COMPOSIO_API_KEY;
669 const goodRes = await runEntry(good, goodEnv);
670 check('known-good (error-handling-demo) exits 0', goodRes.exit === 0);
671
672 // 2. known-bad fixtures: must exit non-zero in both languages.
673 const badTsOut = await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/always-fails.ts']).then(() => 0).catch(() => 1);
674 check('known-bad ts fixture exits non-zero', badTsOut === 1);
675 const badPy = await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/always_fails.py']).then(() => 0).catch(() => 1);
676 check('known-bad py fixture exits non-zero', badPy === 1);
677
678 // 3. trace shims: an unauthenticated backend request must appear in the trace.
679 const tsTrace = join(runDir, 'traces', 'shim-ts.jsonl');
680 await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/trace-check.ts'], {
681 env: { ...process.env, COMPOSIO_TRACE_FILE: tsTrace, COMPOSIO_BASE_URL: baseUrl(), NODE_OPTIONS: `--import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}` },
682 }).catch(() => {});
683 const tsLines = existsSync(tsTrace) ? readFileSync(tsTrace, 'utf8').trim().split('\n').filter(Boolean) : [];
684 check('ts fetch shim recorded a composio call', tsLines.some((l) => JSON.parse(l).p?.includes('/api/')));
685
686 const pyTrace = join(runDir, 'traces', 'shim-py.jsonl');
687 await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/trace_check.py'], {
688 env: { ...process.env, COMPOSIO_TRACE_FILE: pyTrace, COMPOSIO_BASE_URL: baseUrl(), PYTHONPATH: join(ROOT, 'harness', 'trace-py') },
689 }).catch(() => {});
690 const pyLines = existsSync(pyTrace) ? readFileSync(pyTrace, 'utf8').trim().split('\n').filter(Boolean) : [];
691 check('py httpx shim recorded a composio call', pyLines.some((l) => JSON.parse(l).p?.includes('/api/')));
692
693 // 4. parity must compare at least one entry and reject missing candidates.
694 const parityBase = join(runDir, 'parity-base');
695 const parityCandidate = join(runDir, 'parity-candidate');
696 for (const dir of [parityBase, parityCandidate]) {
697 mkdirSync(join(dir, 'traces'), { recursive: true });
698 writeFileSync(join(dir, 'results.jsonl'), `${JSON.stringify({ id: 'fixture', status: 'green' })}\n`);
699 writeFileSync(
700 join(dir, 'traces', 'fixture.jsonl'),
701 `${JSON.stringify({ m: 'GET', p: '/api/v3/toolkits', s: '2xx' })}\n`
702 );
703 }
704 const parityMatches = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate])
705 .then(() => true)
706 .catch(() => false);
707 check('parity accepts matching runs', parityMatches);
708 writeFileSync(join(parityCandidate, 'results.jsonl'), '');
709 const missingCandidateRejected = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate])
710 .then(() => false)
711 .catch(() => true);
712 check('parity rejects missing candidate entries', missingCandidateRejected);
713
714 rmSync(runDir, { recursive: true, force: true });
715 console.log(failures === 0 ? 'selftest: PASS' : `selftest: FAIL (${failures})`);
716 process.exitCode = failures === 0 ? 0 : 1;
717 };
718
719 try {
720 switch (cmd) {
721 case 'sweep':
722 await cmdSweep();
723 break;
724 case 'neg':
725 await cmdNeg();
726 break;
727 case 'selftest':
728 await cmdSelftest();
729 break;
730 default:
731 fail('usage: run.mjs <sweep|neg|selftest> [options]');
732 }
733 } catch (error) {
734 console.error(`run.mjs: ${error instanceof Error ? error.message : String(error)}`);
735 process.exitCode = error instanceof HarnessError ? error.exitCode : 1;
736 }