Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | #!/usr/bin/env node |
| 2 | // Examples runner: sweeps the entrypoints in examples-manifest.json against |
| 3 | // the live backend, recording per-entry results and Composio traces. |
| 4 | // |
| 5 | // node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--exclude-toolkits a,b] [--llm live|mock] |
| 6 | // node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--exclude-toolkits a,b] [--sample N] [--seed S] |
| 7 | // node harness/run.mjs selftest |
| 8 | // |
| 9 | // Exit codes: 0 ok · 1 findings (red entries / failed expectations) · 2 usage/env error. |
| 10 | |
| 11 | import { spawn } from 'node:child_process'; |
| 12 | import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, rmSync } from 'node:fs'; |
| 13 | import { dirname, join, relative, resolve } from 'node:path'; |
| 14 | import { fileURLToPath } from 'node:url'; |
| 15 | import { |
| 16 | describeExclusions, |
| 17 | emptySelectionMessage, |
| 18 | entryToolkits, |
| 19 | loadManifest, |
| 20 | parseOption, |
| 21 | parseSelectionOptions, |
| 22 | requiredBrowserGrantToolkits, |
| 23 | requiresDemoToolkit, |
| 24 | selectManifestEntries, |
| 25 | validateManifestEntries, |
| 26 | } from './manifest.mjs'; |
| 27 | import { resolveBackendBaseUrl, STAGING_BASE_URL } from './backend-url.mjs'; |
| 28 | import { HarnessError } from './errors.mjs'; |
| 29 | |
| 30 | const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); |
| 31 | const ARTIFACTS = join(ROOT, '.artifacts', 'examples-parity'); |
| 32 | const AIMOCK_VERSION = '1.47.0'; |
| 33 | const AIMOCK_PORT = Number(process.env.AIMOCK_PORT ?? 4010); |
| 34 | |
| 35 | // Set by cmdSweep when --llm mock: LLM traffic goes to a local aimock server |
| 36 | // and provider keys are overridden so a mock sweep can never spend tokens. |
| 37 | let LLM_MOCK = false; |
| 38 | |
| 39 | const args = process.argv.slice(2); |
| 40 | const cmd = args[0]; |
| 41 | const opt = (name, fallback) => parseOption(args, name, fallback); |
| 42 | |
| 43 | const fail = (msg, code = 2) => { |
| 44 | throw new HarnessError(msg, code); |
| 45 | }; |
| 46 | |
| 47 | const selectEntries = () => selectManifestEntries(loadManifest(), parseSelectionOptions(args)); |
| 48 | |
| 49 | const reportExclusions = selection => { |
| 50 | for (const line of describeExclusions(selection)) console.log(line); |
| 51 | }; |
| 52 | |
| 53 | const baseUrl = () => { |
| 54 | try { |
| 55 | return resolveBackendBaseUrl(); |
| 56 | } catch (error) { |
| 57 | fail(error instanceof Error ? error.message : String(error)); |
| 58 | } |
| 59 | }; |
| 60 | |
| 61 | // Deterministic PRNG so --seed reproduces a negative-control sample. |
| 62 | const mulberry32 = (a) => () => { |
| 63 | a |= 0; a = (a + 0x6d2b79f5) | 0; |
| 64 | let t = Math.imul(a ^ (a >>> 15), 1 | a); |
| 65 | t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; |
| 66 | return ((t ^ (t >>> 14)) >>> 0) / 4294967296; |
| 67 | }; |
| 68 | |
| 69 | const runId = () => new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14) + '-' + Math.random().toString(36).slice(2, 8); |
| 70 | |
| 71 | /** Spawn one entry; resolve {exit, readinessMatched, timedOut, durationMs}. */ |
| 72 | const runEntry = (entry, env, { garbage = false } = {}) => |
| 73 | new Promise((resolveRun) => { |
| 74 | const isTs = entry.lang === 'ts'; |
| 75 | let cmdline; |
| 76 | if (isTs && entry.script) { |
| 77 | cmdline = ['pnpm', '--filter', entry.pkg, 'run', entry.script]; |
| 78 | } else if (isTs) { |
| 79 | // pnpm --filter runs with the example package as cwd; derive the package |
| 80 | // dir from the entry file path (ts/examples/<dir>/src/...). |
| 81 | const pkgDir = entry.file.split('/').slice(0, 3).join('/'); |
| 82 | const srcRel = relative(join(ROOT, pkgDir), join(ROOT, entry.file)); |
| 83 | cmdline = ['pnpm', '--filter', entry.pkg, 'exec', 'tsx', srcRel]; |
| 84 | } else { |
| 85 | cmdline = ['uv', 'run', '--project', 'python']; |
| 86 | for (const dep of entry.pyWith ?? []) cmdline.push('--with', dep); |
| 87 | cmdline.push('python', join(ROOT, entry.file)); |
| 88 | } |
| 89 | |
| 90 | const child = spawn(cmdline[0], cmdline.slice(1), { |
| 91 | cwd: ROOT, |
| 92 | env, |
| 93 | detached: true, |
| 94 | stdio: ['ignore', 'pipe', 'pipe'], |
| 95 | }); |
| 96 | |
| 97 | const started = Date.now(); |
| 98 | const timeoutMs = (entry.timeoutSec ?? 120) * 1000; |
| 99 | let readinessMatched = false; |
| 100 | let timedOut = false; |
| 101 | let settled = false; |
| 102 | let outputTail = ''; |
| 103 | const readiness = entry.tier === '3' && entry.readiness ? new RegExp(entry.readiness) : null; |
| 104 | |
| 105 | const killTree = () => { |
| 106 | try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ } |
| 107 | setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ } }, 10_000).unref(); |
| 108 | }; |
| 109 | |
| 110 | const onChunk = (chunk) => { |
| 111 | const text = chunk.toString(); |
| 112 | outputTail = (outputTail + text).slice(-4000); |
| 113 | if (readiness && !readinessMatched && readiness.test(outputTail)) { |
| 114 | readinessMatched = true; |
| 115 | // Grace period so the in-flight backend call lands in the trace, then stop the listener. |
| 116 | setTimeout(killTree, 3000).unref(); |
| 117 | } |
| 118 | }; |
| 119 | child.stdout.on('data', onChunk); |
| 120 | child.stderr.on('data', onChunk); |
| 121 | |
| 122 | const timer = setTimeout(() => { |
| 123 | timedOut = true; |
| 124 | killTree(); |
| 125 | }, timeoutMs); |
| 126 | |
| 127 | child.on('exit', (code, signal) => { |
| 128 | if (settled) return; |
| 129 | settled = true; |
| 130 | clearTimeout(timer); |
| 131 | resolveRun({ |
| 132 | exit: code ?? (signal ? 128 : 1), |
| 133 | readinessMatched, |
| 134 | timedOut, |
| 135 | durationMs: Date.now() - started, |
| 136 | outputTail: garbage ? undefined : outputTail.slice(-1500), |
| 137 | }); |
| 138 | }); |
| 139 | child.on('error', () => { |
| 140 | if (settled) return; |
| 141 | settled = true; |
| 142 | clearTimeout(timer); |
| 143 | resolveRun({ exit: 127, readinessMatched: false, timedOut: false, durationMs: Date.now() - started }); |
| 144 | }); |
| 145 | }); |
| 146 | |
| 147 | const isGreen = (entry, r) => { |
| 148 | if (entry.tier === '3') return r.readinessMatched || (!r.timedOut && r.exit === 0); |
| 149 | return !r.timedOut && r.exit === 0; |
| 150 | }; |
| 151 | |
| 152 | const buildEnv = (entry, runDir, { garbage = false } = {}) => { |
| 153 | const traceFile = join(runDir, 'traces', `${entry.id.replace(/[/]/g, '__')}.jsonl`); |
| 154 | mkdirSync(dirname(traceFile), { recursive: true }); |
| 155 | const env = { ...process.env }; |
| 156 | env.COMPOSIO_BASE_URL = baseUrl(); |
| 157 | env.COMPOSIO_TRACE_FILE = traceFile; |
| 158 | env.COMPOSIO_LOG_LEVEL ??= 'error'; |
| 159 | // The outbound-email denylist is fixed by the shims; never overridable per run. |
| 160 | delete env.COMPOSIO_TOOL_DENYLIST; |
| 161 | if (entry.lang === 'ts') { |
| 162 | env.NODE_OPTIONS = `${env.NODE_OPTIONS ?? ''} --import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}`.trim(); |
| 163 | } else { |
| 164 | env.PYTHONPATH = `${join(ROOT, 'harness', 'trace-py')}${env.PYTHONPATH ? `:${env.PYTHONPATH}` : ''}`; |
| 165 | } |
| 166 | if (garbage) { |
| 167 | env.COMPOSIO_API_KEY = 'nc-invalid-0000000000000000'; |
| 168 | env.OPENAI_API_KEY = 'nc-invalid'; |
| 169 | env.ANTHROPIC_API_KEY = 'nc-invalid'; |
| 170 | env.GEMINI_API_KEY = 'nc-invalid'; |
| 171 | env.NOTION_API_KEY = 'nc-invalid'; |
| 172 | } else if (LLM_MOCK) { |
| 173 | const mock = `http://127.0.0.1:${AIMOCK_PORT}`; |
| 174 | env.OPENAI_BASE_URL = `${mock}/v1`; |
| 175 | // langchain/litellm read the older variable name. |
| 176 | env.OPENAI_API_BASE = `${mock}/v1`; |
| 177 | env.ANTHROPIC_BASE_URL = mock; |
| 178 | env.OPENAI_API_KEY = 'aimock'; |
| 179 | env.ANTHROPIC_API_KEY = 'aimock'; |
| 180 | } |
| 181 | return { env, traceFile }; |
| 182 | }; |
| 183 | |
| 184 | const LLM_KEYS = new Set(['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GEMINI_API_KEY']); |
| 185 | |
| 186 | const missingEnv = (entry) => { |
| 187 | const wanted = [...(entry.env ?? []), ...(entry.ids ?? [])].filter( |
| 188 | (k) => !(LLM_MOCK && LLM_KEYS.has(k)) |
| 189 | ); |
| 190 | if (entry.backend !== false) wanted.unshift('COMPOSIO_API_KEY'); |
| 191 | return wanted.filter((k) => !process.env[k]); |
| 192 | }; |
| 193 | |
| 194 | /** Run entries with bounded concurrency; serial entries run last, alone. */ |
| 195 | const runPool = async (entries, runDir, { garbage = false, concurrency = 4 } = {}) => { |
| 196 | const parallel = entries.filter((e) => !e.serial); |
| 197 | const serial = entries.filter((e) => e.serial); |
| 198 | const results = []; |
| 199 | const worker = async (queue) => { |
| 200 | for (;;) { |
| 201 | const entry = queue.shift(); |
| 202 | if (!entry) return; |
| 203 | if (LLM_MOCK && entry.llmMock === false) { |
| 204 | results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: 'llm-mock-unsupported' }); |
| 205 | console.log(` ~ ${entry.id} skipped (llm-mock-unsupported)`); |
| 206 | continue; |
| 207 | } |
| 208 | const missing = garbage ? [] : missingEnv(entry); |
| 209 | if (missing.length > 0) { |
| 210 | results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: `missing-env:${missing.join(',')}` }); |
| 211 | console.log(` ~ ${entry.id} skipped (${missing.join(',')})`); |
| 212 | continue; |
| 213 | } |
| 214 | const { env, traceFile } = buildEnv(entry, runDir, { garbage }); |
| 215 | const r = await runEntry(entry, env, { garbage }); |
| 216 | // An entry that tripped the outbound-email guard is red no matter how it |
| 217 | // exited: catching the refusal and continuing is not coverage. |
| 218 | const blocked = |
| 219 | existsSync(traceFile) && readFileSync(traceFile, 'utf8').includes('"s":"BLOCKED"'); |
| 220 | const green = !blocked && isGreen(entry, r); |
| 221 | results.push({ |
| 222 | id: entry.id, |
| 223 | tier: entry.tier, |
| 224 | status: green ? 'green' : 'red', |
| 225 | ...(blocked ? { reason: 'blocked-tool' } : {}), |
| 226 | exit: r.exit, |
| 227 | timedOut: r.timedOut, |
| 228 | readinessMatched: r.readinessMatched, |
| 229 | durationMs: r.durationMs, |
| 230 | traceFile: relative(ROOT, traceFile), |
| 231 | outputTail: green ? undefined : r.outputTail, |
| 232 | }); |
| 233 | console.log(` ${green ? '✓' : '✗'} ${entry.id} (${(r.durationMs / 1000).toFixed(1)}s${r.timedOut ? ', timeout' : ''})`); |
| 234 | } |
| 235 | }; |
| 236 | const queue = [...parallel]; |
| 237 | await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(queue.length, 1)) }, () => worker(queue))); |
| 238 | for (const entry of serial) await worker([entry]); |
| 239 | return results; |
| 240 | }; |
| 241 | |
| 242 | // --- candidate swap (TS) -------------------------------------------------- |
| 243 | |
| 244 | const WORKSPACE_YAML = join(ROOT, 'pnpm-workspace.yaml'); |
| 245 | const LOCKFILE = join(ROOT, 'pnpm-lock.yaml'); |
| 246 | |
| 247 | const sh = (cmdline, opts = {}) => |
| 248 | new Promise((resolveSh, rejectSh) => { |
| 249 | const child = spawn(cmdline[0], cmdline.slice(1), { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'], ...opts }); |
| 250 | let out = ''; |
| 251 | child.stdout.on('data', (c) => (out += c)); |
| 252 | child.stderr.on('data', (c) => (out += c)); |
| 253 | child.on('exit', (code) => (code === 0 ? resolveSh(out) : rejectSh(new Error(`${cmdline.join(' ')} → exit ${code}\n${out.slice(-2000)}`)))); |
| 254 | }); |
| 255 | |
| 256 | /** |
| 257 | * Resolve @composio/client as @composio/core sees it. The realpath identifies |
| 258 | * the installed copy: a `file:` override lands in its own store directory even |
| 259 | * when the tarball reports the same version as the catalog pin, so comparing |
| 260 | * paths (not versions) is what tells a real swap from a no-op. |
| 261 | */ |
| 262 | const resolvedTsClient = async () => { |
| 263 | const out = await sh(['node', '-e', "const path=require('path');const fs=require('fs');const manifest=require.resolve('@composio/client/package.json', {paths:[path.join(process.cwd(),'ts/packages/core')]});console.log(JSON.stringify({path:fs.realpathSync(manifest),version:JSON.parse(fs.readFileSync(manifest,'utf8')).version}))"]); |
| 264 | return JSON.parse(out.trim().split('\n').pop()); |
| 265 | }; |
| 266 | |
| 267 | const assertTsCandidateApplied = (baseline, candidate) => { |
| 268 | if (candidate.path === baseline.path) { |
| 269 | fail( |
| 270 | `override did not take effect: @composio/client still resolves to the installed baseline at ${candidate.path} (${candidate.version})` |
| 271 | ); |
| 272 | } |
| 273 | }; |
| 274 | |
| 275 | const applyTsCandidate = async (tarball) => { |
| 276 | if (!existsSync(tarball)) fail(`COMPOSIO_CLIENT_TARBALL not found: ${tarball}`); |
| 277 | const baseline = await resolvedTsClient(); |
| 278 | const yaml = readFileSync(WORKSPACE_YAML, 'utf8'); |
| 279 | if (/^overrides:$/m.test(yaml)) { |
| 280 | // Merge into the repo's existing overrides block (e.g. security pins); |
| 281 | // restoreTsBaseline() writes back the exact pre-run snapshot either way. |
| 282 | writeFileSync(WORKSPACE_YAML, yaml.replace(/^overrides:$/m, `overrides:\n '@composio/client': file:${tarball}`)); |
| 283 | } else { |
| 284 | writeFileSync(WORKSPACE_YAML, `${yaml}\noverrides:\n '@composio/client': file:${tarball}\n`); |
| 285 | } |
| 286 | await sh(['pnpm', 'install', '--no-frozen-lockfile']); |
| 287 | const candidate = await resolvedTsClient(); |
| 288 | assertTsCandidateApplied(baseline, candidate); |
| 289 | console.log(`candidate @composio/client resolved: ${candidate.version} (${candidate.path})`); |
| 290 | await sh(['pnpm', 'run', 'build:packages']); |
| 291 | }; |
| 292 | |
| 293 | const snapshotFiles = (paths) => new Map(paths.map((path) => [path, readFileSync(path)])); |
| 294 | |
| 295 | const restoreFiles = (snapshot) => { |
| 296 | for (const [path, contents] of snapshot) writeFileSync(path, contents); |
| 297 | }; |
| 298 | |
| 299 | const snapshotTsBaseline = () => snapshotFiles([WORKSPACE_YAML, LOCKFILE]); |
| 300 | |
| 301 | const restoreTsBaseline = async (snapshot) => { |
| 302 | restoreFiles(snapshot); |
| 303 | await sh(['pnpm', 'install', '--frozen-lockfile']); |
| 304 | }; |
| 305 | |
| 306 | /** |
| 307 | * Resolve composio-client as the python project sees it. `uv run --with <wheel>` |
| 308 | * layers an ephemeral overlay on the project environment, so the realpath tells |
| 309 | * a real swap from a no-op even when the wheel matches the pinned version. |
| 310 | */ |
| 311 | const resolvedPyClient = async (extraArgs = []) => { |
| 312 | const out = await sh(['uv', 'run', '--project', 'python', ...extraArgs, 'python', '-c', 'import composio_client, importlib.metadata, json, os; print(json.dumps({"path": os.path.realpath(composio_client.__file__), "version": importlib.metadata.version("composio-client")}))']); |
| 313 | return JSON.parse(out.trim().split('\n').pop()); |
| 314 | }; |
| 315 | |
| 316 | const assertPyCandidateApplied = (baseline, candidate) => { |
| 317 | if (candidate.path === baseline.path) { |
| 318 | fail( |
| 319 | `python candidate did not take effect: composio-client still resolves to the installed baseline at ${candidate.path} (${candidate.version})` |
| 320 | ); |
| 321 | } |
| 322 | }; |
| 323 | |
| 324 | const verifyPyCandidate = async (wheel) => { |
| 325 | if (!existsSync(wheel)) fail(`COMPOSIO_CLIENT_WHEEL not found: ${wheel}`); |
| 326 | const baseline = await resolvedPyClient(); |
| 327 | const candidate = await resolvedPyClient(['--with', wheel]); |
| 328 | assertPyCandidateApplied(baseline, candidate); |
| 329 | return candidate.version; |
| 330 | }; |
| 331 | |
| 332 | const PY_PROJECT = join(ROOT, 'python', 'pyproject.toml'); |
| 333 | const UV_LOCK = join(ROOT, 'uv.lock'); |
| 334 | const PY_CLIENT_PIN = /^(\s*)"composio-client==[^"]+",$/m; |
| 335 | |
| 336 | /** |
| 337 | * Drop the exact `composio-client==` pin from the python project. |
| 338 | * |
| 339 | * A candidate sweep layers the wheel onto every entry with `uv run --with`, but |
| 340 | * several entries also install the local `./python` project, whose exact pin the |
| 341 | * wheel contradicts — uv then refuses the whole overlay ("no solution found") |
| 342 | * and those entries go red for a packaging reason rather than a client one. |
| 343 | * Without the pin the wheel is the only `composio-client` in play. Baseline |
| 344 | * sweeps never call this, and restorePyBaseline() writes the pre-run files back |
| 345 | * either way. |
| 346 | */ |
| 347 | const relaxPyClientPin = (toml) => { |
| 348 | if (!PY_CLIENT_PIN.test(toml)) { |
| 349 | fail(`no exact composio-client pin found in ${relative(ROOT, PY_PROJECT)}`); |
| 350 | } |
| 351 | return toml.replace(PY_CLIENT_PIN, '$1"composio-client",'); |
| 352 | }; |
| 353 | |
| 354 | const snapshotPyBaseline = () => snapshotFiles([PY_PROJECT, UV_LOCK]); |
| 355 | |
| 356 | const restorePyBaseline = async (snapshot) => { |
| 357 | restoreFiles(snapshot); |
| 358 | await sh(['uv', 'sync', '--project', 'python', '--frozen']); |
| 359 | }; |
| 360 | |
| 361 | // --- llm mock --------------------------------------------------------------- |
| 362 | |
| 363 | /** Start aimock serving harness/llm-mock/fixtures; resolve once it answers. */ |
| 364 | const startLlmMock = async (runDir) => { |
| 365 | const mockDir = join(ROOT, 'harness', 'llm-mock'); |
| 366 | const logFile = join(runDir, 'aimock.log'); |
| 367 | const child = spawn( |
| 368 | 'npx', |
| 369 | ['-y', `@copilotkit/aimock@${AIMOCK_VERSION}`, '-c', 'aimock.json', '-p', String(AIMOCK_PORT), '--host', '127.0.0.1'], |
| 370 | { cwd: mockDir, detached: true, stdio: ['ignore', 'pipe', 'pipe'] } |
| 371 | ); |
| 372 | child.stdout.on('data', (c) => appendFileSync(logFile, c)); |
| 373 | child.stderr.on('data', (c) => appendFileSync(logFile, c)); |
| 374 | |
| 375 | // First run may download the package; allow a generous readiness window. |
| 376 | const deadline = Date.now() + 120_000; |
| 377 | for (;;) { |
| 378 | try { |
| 379 | const res = await fetch(`http://127.0.0.1:${AIMOCK_PORT}/v1/models`); |
| 380 | if (res.ok) break; |
| 381 | } catch { /* not up yet */ } |
| 382 | if (Date.now() > deadline) { |
| 383 | try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ } |
| 384 | fail(`aimock did not become ready on :${AIMOCK_PORT} (see ${relative(ROOT, logFile)})`); |
| 385 | } |
| 386 | await new Promise((r) => setTimeout(r, 500)); |
| 387 | } |
| 388 | console.log(`aimock ${AIMOCK_VERSION} ready on :${AIMOCK_PORT}`); |
| 389 | return () => { |
| 390 | try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ } |
| 391 | }; |
| 392 | }; |
| 393 | |
| 394 | // --- commands --------------------------------------------------------------- |
| 395 | |
| 396 | const cmdSweep = async () => { |
| 397 | const client = opt('client', 'baseline'); |
| 398 | if (!['baseline', 'candidate'].includes(client)) fail(`--client must be baseline|candidate`); |
| 399 | const llm = opt('llm', 'live'); |
| 400 | if (!['live', 'mock'].includes(llm)) fail(`--llm must be live|mock`); |
| 401 | LLM_MOCK = llm === 'mock'; |
| 402 | const selection = selectEntries(); |
| 403 | reportExclusions(selection); |
| 404 | const entries = selection.entries; |
| 405 | if (entries.length === 0) fail(emptySelectionMessage(selection)); |
| 406 | if (!process.env.COMPOSIO_API_KEY) fail('COMPOSIO_API_KEY is required for a sweep (disposable examples-project key)', 2); |
| 407 | |
| 408 | const id = `${runId()}-${client}${LLM_MOCK ? '-mock' : ''}`; |
| 409 | const runDir = join(ARTIFACTS, id); |
| 410 | mkdirSync(join(runDir, 'traces'), { recursive: true }); |
| 411 | |
| 412 | let tsBaselineSnapshot; |
| 413 | let pyBaselineSnapshot; |
| 414 | let stopLlmMock; |
| 415 | const pyWheel = process.env.COMPOSIO_CLIENT_WHEEL; |
| 416 | try { |
| 417 | if (LLM_MOCK) stopLlmMock = await startLlmMock(runDir); |
| 418 | if (client === 'candidate') { |
| 419 | const wantTs = entries.some((e) => e.lang === 'ts'); |
| 420 | const wantPy = entries.some((e) => e.lang === 'py'); |
| 421 | if (wantTs) { |
| 422 | if (!process.env.COMPOSIO_CLIENT_TARBALL) fail('candidate sweep with ts entries needs COMPOSIO_CLIENT_TARBALL'); |
| 423 | tsBaselineSnapshot = snapshotTsBaseline(); |
| 424 | await applyTsCandidate(process.env.COMPOSIO_CLIENT_TARBALL); |
| 425 | } |
| 426 | if (wantPy) { |
| 427 | if (!pyWheel) fail('candidate sweep with py entries needs COMPOSIO_CLIENT_WHEEL'); |
| 428 | const v = await verifyPyCandidate(pyWheel); |
| 429 | console.log(`candidate composio-client (py) resolved: ${v}`); |
| 430 | pyBaselineSnapshot = snapshotPyBaseline(); |
| 431 | writeFileSync(PY_PROJECT, relaxPyClientPin(readFileSync(PY_PROJECT, 'utf8'))); |
| 432 | for (const e of entries) if (e.lang === 'py') e.pyWith = [...(e.pyWith ?? []), pyWheel]; |
| 433 | } |
| 434 | } |
| 435 | |
| 436 | console.log(`sweep ${id}: ${entries.length} entries → ${runDir}`); |
| 437 | const results = await runPool(entries, runDir, { concurrency: Number(opt('concurrency', '4')) }); |
| 438 | for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`); |
| 439 | const counts = { |
| 440 | green: results.filter((r) => r.status === 'green').length, |
| 441 | red: results.filter((r) => r.status === 'red').length, |
| 442 | skipped: results.filter((r) => r.status === 'skipped').length, |
| 443 | }; |
| 444 | writeFileSync( |
| 445 | join(runDir, 'summary.json'), |
| 446 | JSON.stringify( |
| 447 | { |
| 448 | runId: id, |
| 449 | client, |
| 450 | llm, |
| 451 | baseUrl: baseUrl(), |
| 452 | counts, |
| 453 | selection: { |
| 454 | excludedToolkits: selection.excludedToolkits, |
| 455 | excludedEntries: selection.excludedEntries.map(entry => entry.id), |
| 456 | }, |
| 457 | startedAt: new Date().toISOString(), |
| 458 | }, |
| 459 | null, |
| 460 | 2 |
| 461 | ) |
| 462 | ); |
| 463 | console.log(`sweep ${id}: ${counts.green} green / ${counts.red} red / ${counts.skipped} skipped`); |
| 464 | process.exitCode = counts.red > 0 ? 1 : 0; |
| 465 | } finally { |
| 466 | if (stopLlmMock) stopLlmMock(); |
| 467 | if (tsBaselineSnapshot) await restoreTsBaseline(tsBaselineSnapshot); |
| 468 | if (pyBaselineSnapshot) await restorePyBaseline(pyBaselineSnapshot); |
| 469 | } |
| 470 | }; |
| 471 | |
| 472 | const cmdNeg = async () => { |
| 473 | const selection = selectEntries(); |
| 474 | reportExclusions(selection); |
| 475 | let entries = selection.entries.filter((e) => e.backend !== false); |
| 476 | const sample = opt('sample'); |
| 477 | if (sample) { |
| 478 | const rand = mulberry32(Number(opt('seed', '42'))); |
| 479 | entries = [...entries].sort(() => rand() - 0.5).slice(0, Number(sample)); |
| 480 | } |
| 481 | const id = `${runId()}-neg`; |
| 482 | const runDir = join(ARTIFACTS, id); |
| 483 | mkdirSync(join(runDir, 'traces'), { recursive: true }); |
| 484 | console.log(`negative controls ${id}: ${entries.length} entries (garbage credentials; expectation: red)`); |
| 485 | const results = await runPool(entries, runDir, { garbage: true, concurrency: Number(opt('concurrency', '6')) }); |
| 486 | const swallowers = results.filter((r) => r.status === 'green').map((r) => r.id); |
| 487 | for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`); |
| 488 | writeFileSync(join(runDir, 'summary.json'), JSON.stringify({ runId: id, mode: 'neg', total: results.length, swallowers }, null, 2)); |
| 489 | if (swallowers.length > 0) { |
| 490 | console.log(`NEGATIVE-CONTROL FAILURES (green under garbage credentials — these swallow errors):`); |
| 491 | for (const s of swallowers) console.log(` ✗ ${s}`); |
| 492 | process.exitCode = 1; |
| 493 | } else { |
| 494 | console.log('all negative controls red, as expected'); |
| 495 | } |
| 496 | }; |
| 497 | |
| 498 | const cmdSelftest = async () => { |
| 499 | const runDir = join(ARTIFACTS, `${runId()}-selftest`); |
| 500 | mkdirSync(join(runDir, 'traces'), { recursive: true }); |
| 501 | let failures = 0; |
| 502 | const check = (name, ok) => { |
| 503 | console.log(` ${ok ? '✓' : '✗'} ${name}`); |
| 504 | if (!ok) failures += 1; |
| 505 | }; |
| 506 | |
| 507 | // 0. backend URL: staging is the default, any other bare https root is |
| 508 | // honoured as given, and anything structurally unusable is refused. |
| 509 | check('staging backend is the default', resolveBackendBaseUrl(undefined) === STAGING_BASE_URL); |
| 510 | check( |
| 511 | 'a non-staging bare https root is honoured', |
| 512 | resolveBackendBaseUrl('https://backend.composio.dev') === 'https://backend.composio.dev' |
| 513 | ); |
| 514 | const refuses = (value) => { |
| 515 | try { |
| 516 | resolveBackendBaseUrl(value); |
| 517 | return false; |
| 518 | } catch { |
| 519 | return true; |
| 520 | } |
| 521 | }; |
| 522 | check('a base URL carrying a path is refused', refuses('https://backend.composio.dev/api/v3')); |
| 523 | check('a base URL carrying a query is refused', refuses('https://backend.composio.dev/?x=1')); |
| 524 | check( |
| 525 | 'a base URL carrying credentials is refused', |
| 526 | refuses('https://user:pass@backend.composio.dev') |
| 527 | ); |
| 528 | check('a plaintext base URL is refused', refuses('http://backend.composio.dev')); |
| 529 | check('an unparseable base URL is refused', refuses('not-a-url')); |
| 530 | |
| 531 | let cleanupRan = false; |
| 532 | let cleanupError; |
| 533 | try { |
| 534 | try { |
| 535 | fail('expected selftest failure'); |
| 536 | } finally { |
| 537 | cleanupRan = true; |
| 538 | } |
| 539 | } catch (error) { |
| 540 | cleanupError = error; |
| 541 | } |
| 542 | check( |
| 543 | 'usage failures unwind through cleanup', |
| 544 | cleanupRan && cleanupError instanceof HarnessError |
| 545 | ); |
| 546 | |
| 547 | const installedBaseline = { |
| 548 | path: '/repo/node_modules/.pnpm/@composio+client@2.0.0-rc.3/node_modules/@composio/client', |
| 549 | version: '2.0.0-rc.3', |
| 550 | }; |
| 551 | let noOpCandidateRejected = false; |
| 552 | try { |
| 553 | assertTsCandidateApplied(installedBaseline, { ...installedBaseline }); |
| 554 | } catch { |
| 555 | noOpCandidateRejected = true; |
| 556 | } |
| 557 | check('candidate swap rejects the installed baseline copy', noOpCandidateRejected); |
| 558 | |
| 559 | let sameVersionCandidateAccepted = true; |
| 560 | try { |
| 561 | assertTsCandidateApplied(installedBaseline, { |
| 562 | path: '/repo/node_modules/.pnpm/file+client.tgz/node_modules/@composio/client', |
| 563 | version: installedBaseline.version, |
| 564 | }); |
| 565 | } catch { |
| 566 | sameVersionCandidateAccepted = false; |
| 567 | } |
| 568 | check('candidate swap accepts a tarball reporting the pinned version', sameVersionCandidateAccepted); |
| 569 | |
| 570 | const installedPyBaseline = { |
| 571 | path: '/repo/.venv/lib/python3.12/site-packages/composio_client/__init__.py', |
| 572 | version: '2.0.0rc5', |
| 573 | }; |
| 574 | let noOpPyCandidateRejected = false; |
| 575 | try { |
| 576 | assertPyCandidateApplied(installedPyBaseline, { ...installedPyBaseline }); |
| 577 | } catch { |
| 578 | noOpPyCandidateRejected = true; |
| 579 | } |
| 580 | check('python candidate swap rejects the installed baseline copy', noOpPyCandidateRejected); |
| 581 | |
| 582 | let sameVersionPyCandidateAccepted = true; |
| 583 | try { |
| 584 | assertPyCandidateApplied(installedPyBaseline, { |
| 585 | path: '/cache/uv/archive-v0/abc123/lib/python3.12/site-packages/composio_client/__init__.py', |
| 586 | version: installedPyBaseline.version, |
| 587 | }); |
| 588 | } catch { |
| 589 | sameVersionPyCandidateAccepted = false; |
| 590 | } |
| 591 | check('python candidate swap accepts a wheel reporting the pinned version', sameVersionPyCandidateAccepted); |
| 592 | |
| 593 | let invalidManifestIsUsageError = false; |
| 594 | try { |
| 595 | validateManifestEntries([{ id: 'missing-required-fields' }]); |
| 596 | } catch (error) { |
| 597 | invalidManifestIsUsageError = error instanceof HarnessError && error.exitCode === 2; |
| 598 | } |
| 599 | check('invalid manifest entries are usage errors', invalidManifestIsUsageError); |
| 600 | |
| 601 | const manifest = loadManifest(); |
| 602 | const withoutGoogleDrive = selectManifestEntries(manifest, { |
| 603 | excludeToolkits: 'googledrive', |
| 604 | }); |
| 605 | check( |
| 606 | 'Google Drive exclusion removes every dependent entry and preserves the rest', |
| 607 | withoutGoogleDrive.excludedEntries.length > 0 && |
| 608 | withoutGoogleDrive.excludedEntries.every(entry => |
| 609 | entryToolkits(entry).includes('googledrive') |
| 610 | ) && |
| 611 | withoutGoogleDrive.entries.every(entry => !entryToolkits(entry).includes('googledrive')) && |
| 612 | withoutGoogleDrive.entries.length + withoutGoogleDrive.excludedEntries.length === |
| 613 | selectManifestEntries(manifest).entries.length |
| 614 | ); |
| 615 | check( |
| 616 | 'Google Drive exclusion removes it from provisioning', |
| 617 | !requiredBrowserGrantToolkits(withoutGoogleDrive.entries).some( |
| 618 | toolkit => toolkit.slug === 'googledrive' |
| 619 | ) |
| 620 | ); |
| 621 | const apiKeyOnly = selectManifestEntries(manifest, { |
| 622 | ids: 'ts/connected-accounts/api-key', |
| 623 | }).entries; |
| 624 | check( |
| 625 | 'selected API-key example provisions only its demo toolkit', |
| 626 | requiredBrowserGrantToolkits(apiKeyOnly).length === 0 && requiresDemoToolkit(apiKeyOnly) |
| 627 | ); |
| 628 | const implicitGmail = selectManifestEntries(manifest, { |
| 629 | ids: 'py/fastapi_app', |
| 630 | }).entries; |
| 631 | check( |
| 632 | 'resource identifiers contribute implicit toolkit requirements', |
| 633 | requiredBrowserGrantToolkits(implicitGmail).some(toolkit => toolkit.slug === 'gmail') |
| 634 | ); |
| 635 | |
| 636 | const cleanupFixture = join(runDir, 'cleanup-fixture.txt'); |
| 637 | writeFileSync(cleanupFixture, 'user contents'); |
| 638 | const cleanupSnapshot = snapshotFiles([cleanupFixture]); |
| 639 | writeFileSync(cleanupFixture, 'candidate contents'); |
| 640 | restoreFiles(cleanupSnapshot); |
| 641 | check('cleanup restores pre-run file contents', readFileSync(cleanupFixture, 'utf8') === 'user contents'); |
| 642 | |
| 643 | // The python candidate swap only resolves once the project's exact |
| 644 | // composio-client pin is out of the way; see relaxPyClientPin. |
| 645 | const pinnedProject = readFileSync(PY_PROJECT, 'utf8'); |
| 646 | const relaxedProject = relaxPyClientPin(pinnedProject); |
| 647 | check('the python project pins composio-client exactly', /"composio-client==/.test(pinnedProject)); |
| 648 | check( |
| 649 | 'relaxing the pin keeps composio-client a dependency', |
| 650 | /^\s*"composio-client",$/m.test(relaxedProject) && !/"composio-client==/.test(relaxedProject) |
| 651 | ); |
| 652 | check( |
| 653 | 'relaxing the pin touches nothing else', |
| 654 | relaxedProject.replace(/^\s*"composio-client",$/m, '') === |
| 655 | pinnedProject.replace(/^\s*"composio-client==[^"]+",$/m, '') |
| 656 | ); |
| 657 | let unpinnedProjectRejected = false; |
| 658 | try { |
| 659 | relaxPyClientPin(relaxedProject); |
| 660 | } catch { |
| 661 | unpinnedProjectRejected = true; |
| 662 | } |
| 663 | check('relaxing an already-unpinned project is refused', unpinnedProjectRejected); |
| 664 | |
| 665 | // 1. known-good: error-handling-demo (no backend, no keys). |
| 666 | const good = loadManifest().find((e) => e.id === 'ts/error-handling-demo/index'); |
| 667 | const { env: goodEnv } = buildEnv(good, runDir); |
| 668 | delete goodEnv.COMPOSIO_API_KEY; |
| 669 | const goodRes = await runEntry(good, goodEnv); |
| 670 | check('known-good (error-handling-demo) exits 0', goodRes.exit === 0); |
| 671 | |
| 672 | // 2. known-bad fixtures: must exit non-zero in both languages. |
| 673 | const badTsOut = await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/always-fails.ts']).then(() => 0).catch(() => 1); |
| 674 | check('known-bad ts fixture exits non-zero', badTsOut === 1); |
| 675 | const badPy = await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/always_fails.py']).then(() => 0).catch(() => 1); |
| 676 | check('known-bad py fixture exits non-zero', badPy === 1); |
| 677 | |
| 678 | // 3. trace shims: an unauthenticated backend request must appear in the trace. |
| 679 | const tsTrace = join(runDir, 'traces', 'shim-ts.jsonl'); |
| 680 | await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/trace-check.ts'], { |
| 681 | env: { ...process.env, COMPOSIO_TRACE_FILE: tsTrace, COMPOSIO_BASE_URL: baseUrl(), NODE_OPTIONS: `--import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}` }, |
| 682 | }).catch(() => {}); |
| 683 | const tsLines = existsSync(tsTrace) ? readFileSync(tsTrace, 'utf8').trim().split('\n').filter(Boolean) : []; |
| 684 | check('ts fetch shim recorded a composio call', tsLines.some((l) => JSON.parse(l).p?.includes('/api/'))); |
| 685 | |
| 686 | const pyTrace = join(runDir, 'traces', 'shim-py.jsonl'); |
| 687 | await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/trace_check.py'], { |
| 688 | env: { ...process.env, COMPOSIO_TRACE_FILE: pyTrace, COMPOSIO_BASE_URL: baseUrl(), PYTHONPATH: join(ROOT, 'harness', 'trace-py') }, |
| 689 | }).catch(() => {}); |
| 690 | const pyLines = existsSync(pyTrace) ? readFileSync(pyTrace, 'utf8').trim().split('\n').filter(Boolean) : []; |
| 691 | check('py httpx shim recorded a composio call', pyLines.some((l) => JSON.parse(l).p?.includes('/api/'))); |
| 692 | |
| 693 | // 4. parity must compare at least one entry and reject missing candidates. |
| 694 | const parityBase = join(runDir, 'parity-base'); |
| 695 | const parityCandidate = join(runDir, 'parity-candidate'); |
| 696 | for (const dir of [parityBase, parityCandidate]) { |
| 697 | mkdirSync(join(dir, 'traces'), { recursive: true }); |
| 698 | writeFileSync(join(dir, 'results.jsonl'), `${JSON.stringify({ id: 'fixture', status: 'green' })}\n`); |
| 699 | writeFileSync( |
| 700 | join(dir, 'traces', 'fixture.jsonl'), |
| 701 | `${JSON.stringify({ m: 'GET', p: '/api/v3/toolkits', s: '2xx' })}\n` |
| 702 | ); |
| 703 | } |
| 704 | const parityMatches = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate]) |
| 705 | .then(() => true) |
| 706 | .catch(() => false); |
| 707 | check('parity accepts matching runs', parityMatches); |
| 708 | writeFileSync(join(parityCandidate, 'results.jsonl'), ''); |
| 709 | const missingCandidateRejected = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate]) |
| 710 | .then(() => false) |
| 711 | .catch(() => true); |
| 712 | check('parity rejects missing candidate entries', missingCandidateRejected); |
| 713 | |
| 714 | rmSync(runDir, { recursive: true, force: true }); |
| 715 | console.log(failures === 0 ? 'selftest: PASS' : `selftest: FAIL (${failures})`); |
| 716 | process.exitCode = failures === 0 ? 0 : 1; |
| 717 | }; |
| 718 | |
| 719 | try { |
| 720 | switch (cmd) { |
| 721 | case 'sweep': |
| 722 | await cmdSweep(); |
| 723 | break; |
| 724 | case 'neg': |
| 725 | await cmdNeg(); |
| 726 | break; |
| 727 | case 'selftest': |
| 728 | await cmdSelftest(); |
| 729 | break; |
| 730 | default: |
| 731 | fail('usage: run.mjs <sweep|neg|selftest> [options]'); |
| 732 | } |
| 733 | } catch (error) { |
| 734 | console.error(`run.mjs: ${error instanceof Error ? error.message : String(error)}`); |
| 735 | process.exitCode = error instanceof HarnessError ? error.exitCode : 1; |
| 736 | } |