1
+#!/usr/bin/env node
2
+// Examples runner: sweeps the entrypoints in examples-manifest.json against
3
+// the live backend, recording per-entry results and Composio traces.
4
+//
5
+// node harness/run.mjs sweep --client baseline|candidate [--lang ts|py] [--ids a,b] [--tiers 1,2,3] [--llm live|mock]
6
+// node harness/run.mjs neg [--lang ts|py] [--ids a,b] [--sample N] [--seed S]
7
+// node harness/run.mjs selftest
8
+//
9
+// Exit codes: 0 ok · 1 findings (red entries / failed expectations) · 2 usage/env error.
10
+
11
+import { spawn } from 'node:child_process';
12
+import { mkdirSync, readFileSync, writeFileSync, appendFileSync, existsSync, rmSync } from 'node:fs';
13
+import { dirname, join, relative, resolve } from 'node:path';
14
+import { fileURLToPath } from 'node:url';
15
+import { requireStagingBaseUrl, STAGING_BASE_URL } from './staging-backend.mjs';
16
+
17
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
18
+const ARTIFACTS = join(ROOT, '.artifacts', 'examples-parity');
19
+const STAINLESS_TS_VERSION = '0.1.0-alpha.76';
20
+const AIMOCK_VERSION = '1.38.0';
21
+const AIMOCK_PORT = Number(process.env.AIMOCK_PORT ?? 4010);
22
+
23
+// Set by cmdSweep when --llm mock: LLM traffic goes to a local aimock server
24
+// and provider keys are overridden so a mock sweep can never spend tokens.
25
+let LLM_MOCK = false;
26
+
27
+const args = process.argv.slice(2);
28
+const cmd = args[0];
29
+const opt = (name, fallback) => {
30
+ const i = args.indexOf(`--${name}`);
31
+ return i >= 0 && args[i + 1] !== undefined ? args[i + 1] : fallback;
32
+};
33
+
34
+class HarnessError extends Error {
35
+ constructor(message, exitCode = 2) {
36
+ super(message);
37
+ this.exitCode = exitCode;
38
+ }
39
+}
40
+
41
+const fail = (msg, code = 2) => {
42
+ throw new HarnessError(msg, code);
43
+};
44
+
45
+const loadManifest = () => {
46
+ const manifest = JSON.parse(readFileSync(join(ROOT, 'examples-manifest.json'), 'utf8'));
47
+ for (const e of manifest.entries) {
48
+ if (!e.id || !e.lang || !e.file || !e.tier) fail(`manifest entry missing required fields: ${JSON.stringify(e)}`);
49
+ if (e.lang === 'ts' && e.tier !== 'X' && !e.pkg) fail(`ts entry ${e.id} missing pkg`);
50
+ if (e.tier === '3' && !e.readiness) fail(`tier-3 entry ${e.id} missing readiness regex`);
51
+ }
52
+ return manifest.entries;
53
+};
54
+
55
+const selectEntries = () => {
56
+ const lang = opt('lang');
57
+ const ids = opt('ids') ? opt('ids').split(',') : null;
58
+ const tiers = (opt('tiers', '1,2,3')).split(',');
59
+ return loadManifest().filter(
60
+ (e) =>
61
+ e.tier !== 'X' &&
62
+ tiers.includes(e.tier) &&
63
+ (!lang || e.lang === lang) &&
64
+ (!ids || ids.includes(e.id))
65
+ );
66
+};
67
+
68
+const baseUrl = () => {
69
+ try {
70
+ return requireStagingBaseUrl();
71
+ } catch (error) {
72
+ fail(error instanceof Error ? error.message : String(error));
73
+ }
74
+};
75
+
76
+// Deterministic PRNG so --seed reproduces a negative-control sample.
77
+const mulberry32 = (a) => () => {
78
+ a |= 0; a = (a + 0x6d2b79f5) | 0;
79
+ let t = Math.imul(a ^ (a >>> 15), 1 | a);
80
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
81
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
82
+};
83
+
84
+const runId = () => new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14) + '-' + Math.random().toString(36).slice(2, 8);
85
+
86
+/** Spawn one entry; resolve {exit, readinessMatched, timedOut, durationMs}. */
87
+const runEntry = (entry, env, { garbage = false } = {}) =>
88
+ new Promise((resolveRun) => {
89
+ const isTs = entry.lang === 'ts';
90
+ let cmdline;
91
+ if (isTs && entry.script) {
92
+ cmdline = ['pnpm', '--filter', entry.pkg, 'run', entry.script];
93
+ } else if (isTs) {
94
+ // pnpm --filter runs with the example package as cwd; derive the package
95
+ // dir from the entry file path (ts/examples/<dir>/src/...).
96
+ const pkgDir = entry.file.split('/').slice(0, 3).join('/');
97
+ const srcRel = relative(join(ROOT, pkgDir), join(ROOT, entry.file));
98
+ cmdline = ['pnpm', '--filter', entry.pkg, 'exec', 'tsx', srcRel];
99
+ } else {
100
+ cmdline = ['uv', 'run', '--project', 'python'];
101
+ for (const dep of entry.pyWith ?? []) cmdline.push('--with', dep);
102
+ cmdline.push('python', join(ROOT, entry.file));
103
+ }
104
+
105
+ const child = spawn(cmdline[0], cmdline.slice(1), {
106
+ cwd: ROOT,
107
+ env,
108
+ detached: true,
109
+ stdio: ['ignore', 'pipe', 'pipe'],
110
+ });
111
+
112
+ const started = Date.now();
113
+ const timeoutMs = (entry.timeoutSec ?? 120) * 1000;
114
+ let readinessMatched = false;
115
+ let timedOut = false;
116
+ let settled = false;
117
+ let outputTail = '';
118
+ const readiness = entry.tier === '3' && entry.readiness ? new RegExp(entry.readiness) : null;
119
+
120
+ const killTree = () => {
121
+ try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ }
122
+ setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ } }, 10_000).unref();
123
+ };
124
+
125
+ const onChunk = (chunk) => {
126
+ const text = chunk.toString();
127
+ outputTail = (outputTail + text).slice(-4000);
128
+ if (readiness && !readinessMatched && readiness.test(outputTail)) {
129
+ readinessMatched = true;
130
+ // Grace period so the in-flight backend call lands in the trace, then stop the listener.
131
+ setTimeout(killTree, 3000).unref();
132
+ }
133
+ };
134
+ child.stdout.on('data', onChunk);
135
+ child.stderr.on('data', onChunk);
136
+
137
+ const timer = setTimeout(() => {
138
+ timedOut = true;
139
+ killTree();
140
+ }, timeoutMs);
141
+
142
+ child.on('exit', (code, signal) => {
143
+ if (settled) return;
144
+ settled = true;
145
+ clearTimeout(timer);
146
+ resolveRun({
147
+ exit: code ?? (signal ? 128 : 1),
148
+ readinessMatched,
149
+ timedOut,
150
+ durationMs: Date.now() - started,
151
+ outputTail: garbage ? undefined : outputTail.slice(-1500),
152
+ });
153
+ });
154
+ child.on('error', () => {
155
+ if (settled) return;
156
+ settled = true;
157
+ clearTimeout(timer);
158
+ resolveRun({ exit: 127, readinessMatched: false, timedOut: false, durationMs: Date.now() - started });
159
+ });
160
+ });
161
+
162
+const isGreen = (entry, r) => {
163
+ if (entry.tier === '3') return r.readinessMatched || (!r.timedOut && r.exit === 0);
164
+ return !r.timedOut && r.exit === 0;
165
+};
166
+
167
+const buildEnv = (entry, runDir, { garbage = false } = {}) => {
168
+ const traceFile = join(runDir, 'traces', `${entry.id.replace(/[/]/g, '__')}.jsonl`);
169
+ mkdirSync(dirname(traceFile), { recursive: true });
170
+ const env = { ...process.env };
171
+ env.COMPOSIO_BASE_URL = baseUrl();
172
+ env.COMPOSIO_TRACE_FILE = traceFile;
173
+ env.COMPOSIO_LOG_LEVEL ??= 'error';
174
+ // The outbound-email denylist is fixed by the shims; never overridable per run.
175
+ delete env.COMPOSIO_TOOL_DENYLIST;
176
+ if (entry.lang === 'ts') {
177
+ env.NODE_OPTIONS = `${env.NODE_OPTIONS ?? ''} --import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}`.trim();
178
+ } else {
179
+ env.PYTHONPATH = `${join(ROOT, 'harness', 'trace-py')}${env.PYTHONPATH ? `:${env.PYTHONPATH}` : ''}`;
180
+ }
181
+ if (garbage) {
182
+ env.COMPOSIO_API_KEY = 'nc-invalid-0000000000000000';
183
+ env.OPENAI_API_KEY = 'nc-invalid';
184
+ env.ANTHROPIC_API_KEY = 'nc-invalid';
185
+ env.GEMINI_API_KEY = 'nc-invalid';
186
+ env.NOTION_API_KEY = 'nc-invalid';
187
+ } else if (LLM_MOCK) {
188
+ const mock = `http://127.0.0.1:${AIMOCK_PORT}`;
189
+ env.OPENAI_BASE_URL = `${mock}/v1`;
190
+ // langchain/litellm read the older variable name.
191
+ env.OPENAI_API_BASE = `${mock}/v1`;
192
+ env.ANTHROPIC_BASE_URL = mock;
193
+ env.OPENAI_API_KEY = 'aimock';
194
+ env.ANTHROPIC_API_KEY = 'aimock';
195
+ }
196
+ return { env, traceFile };
197
+};
198
+
199
+const LLM_KEYS = new Set(['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GEMINI_API_KEY']);
200
+
201
+const missingEnv = (entry) => {
202
+ const wanted = [...(entry.env ?? []), ...(entry.ids ?? [])].filter(
203
+ (k) => !(LLM_MOCK && LLM_KEYS.has(k))
204
+ );
205
+ if (entry.backend !== false) wanted.unshift('COMPOSIO_API_KEY');
206
+ return wanted.filter((k) => !process.env[k]);
207
+};
208
+
209
+/** Run entries with bounded concurrency; serial entries run last, alone. */
210
+const runPool = async (entries, runDir, { garbage = false, concurrency = 4 } = {}) => {
211
+ const parallel = entries.filter((e) => !e.serial);
212
+ const serial = entries.filter((e) => e.serial);
213
+ const results = [];
214
+ const worker = async (queue) => {
215
+ for (;;) {
216
+ const entry = queue.shift();
217
+ if (!entry) return;
218
+ if (LLM_MOCK && entry.llmMock === false) {
219
+ results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: 'llm-mock-unsupported' });
220
+ console.log(` ~ ${entry.id} skipped (llm-mock-unsupported)`);
221
+ continue;
222
+ }
223
+ const missing = garbage ? [] : missingEnv(entry);
224
+ if (missing.length > 0) {
225
+ results.push({ id: entry.id, tier: entry.tier, status: 'skipped', reason: `missing-env:${missing.join(',')}` });
226
+ console.log(` ~ ${entry.id} skipped (${missing.join(',')})`);
227
+ continue;
228
+ }
229
+ const { env, traceFile } = buildEnv(entry, runDir, { garbage });
230
+ const r = await runEntry(entry, env, { garbage });
231
+ // An entry that tripped the outbound-email guard is red no matter how it
232
+ // exited: catching the refusal and continuing is not coverage.
233
+ const blocked =
234
+ existsSync(traceFile) && readFileSync(traceFile, 'utf8').includes('"s":"BLOCKED"');
235
+ const green = !blocked && isGreen(entry, r);
236
+ results.push({
237
+ id: entry.id,
238
+ tier: entry.tier,
239
+ status: green ? 'green' : 'red',
240
+ ...(blocked ? { reason: 'blocked-tool' } : {}),
241
+ exit: r.exit,
242
+ timedOut: r.timedOut,
243
+ readinessMatched: r.readinessMatched,
244
+ durationMs: r.durationMs,
245
+ traceFile: relative(ROOT, traceFile),
246
+ outputTail: green ? undefined : r.outputTail,
247
+ });
248
+ console.log(` ${green ? '✓' : '✗'} ${entry.id} (${(r.durationMs / 1000).toFixed(1)}s${r.timedOut ? ', timeout' : ''})`);
249
+ }
250
+ };
251
+ const queue = [...parallel];
252
+ await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(queue.length, 1)) }, () => worker(queue)));
253
+ for (const entry of serial) await worker([entry]);
254
+ return results;
255
+};
256
+
257
+// --- candidate swap (TS) --------------------------------------------------
258
+
259
+const WORKSPACE_YAML = join(ROOT, 'pnpm-workspace.yaml');
260
+const LOCKFILE = join(ROOT, 'pnpm-lock.yaml');
261
+
262
+const sh = (cmdline, opts = {}) =>
263
+ new Promise((resolveSh, rejectSh) => {
264
+ const child = spawn(cmdline[0], cmdline.slice(1), { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'], ...opts });
265
+ let out = '';
266
+ child.stdout.on('data', (c) => (out += c));
267
+ child.stderr.on('data', (c) => (out += c));
268
+ child.on('exit', (code) => (code === 0 ? resolveSh(out) : rejectSh(new Error(`${cmdline.join(' ')} → exit ${code}\n${out.slice(-2000)}`))));
269
+ });
270
+
271
+const resolvedTsClientVersion = async () => {
272
+ const out = await sh(['node', '-e', "console.log(JSON.parse(require('fs').readFileSync(require.resolve('@composio/client/package.json', {paths:[require('path').join(process.cwd(),'ts/packages/core')]}), 'utf8')).version)"]);
273
+ return out.trim().split('\n').pop();
274
+};
275
+
276
+const applyTsCandidate = async (tarball) => {
277
+ if (!existsSync(tarball)) fail(`COMPOSIO_CLIENT_TARBALL not found: ${tarball}`);
278
+ const yaml = readFileSync(WORKSPACE_YAML, 'utf8');
279
+ if (/^overrides:$/m.test(yaml)) {
280
+ // Merge into the repo's existing overrides block (e.g. security pins);
281
+ // restoreTsBaseline() writes back the exact pre-run snapshot either way.
282
+ writeFileSync(WORKSPACE_YAML, yaml.replace(/^overrides:$/m, `overrides:\n '@composio/client': file:${tarball}`));
283
+ } else {
284
+ writeFileSync(WORKSPACE_YAML, `${yaml}\noverrides:\n '@composio/client': file:${tarball}\n`);
285
+ }
286
+ await sh(['pnpm', 'install', '--no-frozen-lockfile']);
287
+ const version = await resolvedTsClientVersion();
288
+ if (version === STAINLESS_TS_VERSION) {
289
+ fail(`override did not take effect: @composio/client still resolves to ${version}`);
290
+ }
291
+ console.log(`candidate @composio/client resolved: ${version}`);
292
+ await sh(['pnpm', 'run', 'build:packages']);
293
+};
294
+
295
+const snapshotFiles = (paths) => new Map(paths.map((path) => [path, readFileSync(path)]));
296
+
297
+const restoreFiles = (snapshot) => {
298
+ for (const [path, contents] of snapshot) writeFileSync(path, contents);
299
+};
300
+
301
+const snapshotTsBaseline = () => snapshotFiles([WORKSPACE_YAML, LOCKFILE]);
302
+
303
+const restoreTsBaseline = async (snapshot) => {
304
+ restoreFiles(snapshot);
305
+ await sh(['pnpm', 'install', '--frozen-lockfile']);
306
+};
307
+
308
+const verifyPyCandidate = async (wheel) => {
309
+ if (!existsSync(wheel)) fail(`COMPOSIO_CLIENT_WHEEL not found: ${wheel}`);
310
+ const out = await sh(['uv', 'run', '--project', 'python', '--with', wheel, 'python', '-c', 'import composio_client, importlib.metadata; print(importlib.metadata.version("composio-client"))']);
311
+ const version = out.trim().split('\n').pop();
312
+ if (!version.startsWith('2.')) {
313
+ fail(`python candidate did not take effect (resolved ${version}); Stage B (SDK migration) must land first`);
314
+ }
315
+ return version;
316
+};
317
+
318
+// --- llm mock ---------------------------------------------------------------
319
+
320
+/** Start aimock serving harness/llm-mock/fixtures; resolve once it answers. */
321
+const startLlmMock = async (runDir) => {
322
+ const mockDir = join(ROOT, 'harness', 'llm-mock');
323
+ const logFile = join(runDir, 'aimock.log');
324
+ const child = spawn(
325
+ 'npx',
326
+ ['-y', `@copilotkit/aimock@${AIMOCK_VERSION}`, '-c', 'aimock.json', '-p', String(AIMOCK_PORT), '-h', '127.0.0.1'],
327
+ { cwd: mockDir, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }
328
+ );
329
+ child.stdout.on('data', (c) => appendFileSync(logFile, c));
330
+ child.stderr.on('data', (c) => appendFileSync(logFile, c));
331
+
332
+ // First run may download the package; allow a generous readiness window.
333
+ const deadline = Date.now() + 120_000;
334
+ for (;;) {
335
+ try {
336
+ const res = await fetch(`http://127.0.0.1:${AIMOCK_PORT}/v1/models`);
337
+ if (res.ok) break;
338
+ } catch { /* not up yet */ }
339
+ if (Date.now() > deadline) {
340
+ try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ }
341
+ fail(`aimock did not become ready on :${AIMOCK_PORT} (see ${relative(ROOT, logFile)})`);
342
+ }
343
+ await new Promise((r) => setTimeout(r, 500));
344
+ }
345
+ console.log(`aimock ${AIMOCK_VERSION} ready on :${AIMOCK_PORT}`);
346
+ return () => {
347
+ try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ }
348
+ };
349
+};
350
+
351
+// --- commands ---------------------------------------------------------------
352
+
353
+const cmdSweep = async () => {
354
+ const client = opt('client', 'baseline');
355
+ if (!['baseline', 'candidate'].includes(client)) fail(`--client must be baseline|candidate`);
356
+ const llm = opt('llm', 'live');
357
+ if (!['live', 'mock'].includes(llm)) fail(`--llm must be live|mock`);
358
+ LLM_MOCK = llm === 'mock';
359
+ const entries = selectEntries();
360
+ if (entries.length === 0) fail('no entries selected');
361
+ if (!process.env.COMPOSIO_API_KEY) fail('COMPOSIO_API_KEY is required for a sweep (disposable examples-project key)', 2);
362
+
363
+ const id = `${runId()}-${client}${LLM_MOCK ? '-mock' : ''}`;
364
+ const runDir = join(ARTIFACTS, id);
365
+ mkdirSync(join(runDir, 'traces'), { recursive: true });
366
+
367
+ let tsBaselineSnapshot;
368
+ let stopLlmMock;
369
+ const pyWheel = process.env.COMPOSIO_CLIENT_WHEEL;
370
+ try {
371
+ if (LLM_MOCK) stopLlmMock = await startLlmMock(runDir);
372
+ if (client === 'candidate') {
373
+ const wantTs = entries.some((e) => e.lang === 'ts');
374
+ const wantPy = entries.some((e) => e.lang === 'py');
375
+ if (wantTs) {
376
+ if (!process.env.COMPOSIO_CLIENT_TARBALL) fail('candidate sweep with ts entries needs COMPOSIO_CLIENT_TARBALL');
377
+ tsBaselineSnapshot = snapshotTsBaseline();
378
+ await applyTsCandidate(process.env.COMPOSIO_CLIENT_TARBALL);
379
+ }
380
+ if (wantPy) {
381
+ if (!pyWheel) fail('candidate sweep with py entries needs COMPOSIO_CLIENT_WHEEL');
382
+ const v = await verifyPyCandidate(pyWheel);
383
+ console.log(`candidate composio-client (py) resolved: ${v}`);
384
+ for (const e of entries) if (e.lang === 'py') e.pyWith = [...(e.pyWith ?? []), pyWheel];
385
+ }
386
+ }
387
+
388
+ console.log(`sweep ${id}: ${entries.length} entries → ${runDir}`);
389
+ const results = await runPool(entries, runDir, { concurrency: Number(opt('concurrency', '4')) });
390
+ for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`);
391
+ const counts = {
392
+ green: results.filter((r) => r.status === 'green').length,
393
+ red: results.filter((r) => r.status === 'red').length,
394
+ skipped: results.filter((r) => r.status === 'skipped').length,
395
+ };
396
+ writeFileSync(
397
+ join(runDir, 'summary.json'),
398
+ JSON.stringify({ runId: id, client, llm, baseUrl: baseUrl(), counts, startedAt: new Date().toISOString() }, null, 2)
399
+ );
400
+ console.log(`sweep ${id}: ${counts.green} green / ${counts.red} red / ${counts.skipped} skipped`);
401
+ process.exitCode = counts.red > 0 ? 1 : 0;
402
+ } finally {
403
+ if (stopLlmMock) stopLlmMock();
404
+ if (tsBaselineSnapshot) await restoreTsBaseline(tsBaselineSnapshot);
405
+ }
406
+};
407
+
408
+const cmdNeg = async () => {
409
+ let entries = selectEntries().filter((e) => e.backend !== false);
410
+ const sample = opt('sample');
411
+ if (sample) {
412
+ const rand = mulberry32(Number(opt('seed', '42')));
413
+ entries = [...entries].sort(() => rand() - 0.5).slice(0, Number(sample));
414
+ }
415
+ const id = `${runId()}-neg`;
416
+ const runDir = join(ARTIFACTS, id);
417
+ mkdirSync(join(runDir, 'traces'), { recursive: true });
418
+ console.log(`negative controls ${id}: ${entries.length} entries (garbage credentials; expectation: red)`);
419
+ const results = await runPool(entries, runDir, { garbage: true, concurrency: Number(opt('concurrency', '6')) });
420
+ const swallowers = results.filter((r) => r.status === 'green').map((r) => r.id);
421
+ for (const r of results) appendFileSync(join(runDir, 'results.jsonl'), `${JSON.stringify(r)}\n`);
422
+ writeFileSync(join(runDir, 'summary.json'), JSON.stringify({ runId: id, mode: 'neg', total: results.length, swallowers }, null, 2));
423
+ if (swallowers.length > 0) {
424
+ console.log(`NEGATIVE-CONTROL FAILURES (green under garbage credentials — these swallow errors):`);
425
+ for (const s of swallowers) console.log(` ✗ ${s}`);
426
+ process.exitCode = 1;
427
+ } else {
428
+ console.log('all negative controls red, as expected');
429
+ }
430
+};
431
+
432
+const cmdSelftest = async () => {
433
+ const runDir = join(ARTIFACTS, `${runId()}-selftest`);
434
+ mkdirSync(join(runDir, 'traces'), { recursive: true });
435
+ let failures = 0;
436
+ const check = (name, ok) => {
437
+ console.log(` ${ok ? '✓' : '✗'} ${name}`);
438
+ if (!ok) failures += 1;
439
+ };
440
+
441
+ // 0. backend safety: the harness accepts only the canonical staging root.
442
+ check('staging backend is accepted', requireStagingBaseUrl() === STAGING_BASE_URL);
443
+ let productionRejected = false;
444
+ try {
445
+ requireStagingBaseUrl('https://backend.composio.dev');
446
+ } catch {
447
+ productionRejected = true;
448
+ }
449
+ check('production backend is rejected', productionRejected);
450
+
451
+ let cleanupRan = false;
452
+ let cleanupError;
453
+ try {
454
+ try {
455
+ fail('expected selftest failure');
456
+ } finally {
457
+ cleanupRan = true;
458
+ }
459
+ } catch (error) {
460
+ cleanupError = error;
461
+ }
462
+ check(
463
+ 'usage failures unwind through cleanup',
464
+ cleanupRan && cleanupError instanceof HarnessError
465
+ );
466
+
467
+ const cleanupFixture = join(runDir, 'cleanup-fixture.txt');
468
+ writeFileSync(cleanupFixture, 'user contents');
469
+ const cleanupSnapshot = snapshotFiles([cleanupFixture]);
470
+ writeFileSync(cleanupFixture, 'candidate contents');
471
+ restoreFiles(cleanupSnapshot);
472
+ check('cleanup restores pre-run file contents', readFileSync(cleanupFixture, 'utf8') === 'user contents');
473
+
474
+ // 1. known-good: error-handling-demo (no backend, no keys).
475
+ const good = loadManifest().find((e) => e.id === 'ts/error-handling-demo/index');
476
+ const { env: goodEnv } = buildEnv(good, runDir);
477
+ delete goodEnv.COMPOSIO_API_KEY;
478
+ const goodRes = await runEntry(good, goodEnv);
479
+ check('known-good (error-handling-demo) exits 0', goodRes.exit === 0);
480
+
481
+ // 2. known-bad fixtures: must exit non-zero in both languages.
482
+ const badTsOut = await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/always-fails.ts']).then(() => 0).catch(() => 1);
483
+ check('known-bad ts fixture exits non-zero', badTsOut === 1);
484
+ const badPy = await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/always_fails.py']).then(() => 0).catch(() => 1);
485
+ check('known-bad py fixture exits non-zero', badPy === 1);
486
+
487
+ // 3. trace shims: an unauthenticated backend request must appear in the trace.
488
+ const tsTrace = join(runDir, 'traces', 'shim-ts.jsonl');
489
+ await sh(['pnpm', 'exec', 'tsx', 'harness/fixtures/trace-check.ts'], {
490
+ env: { ...process.env, COMPOSIO_TRACE_FILE: tsTrace, COMPOSIO_BASE_URL: baseUrl(), NODE_OPTIONS: `--import=file://${join(ROOT, 'harness', 'trace', 'register.mjs')}` },
491
+ }).catch(() => {});
492
+ const tsLines = existsSync(tsTrace) ? readFileSync(tsTrace, 'utf8').trim().split('\n').filter(Boolean) : [];
493
+ check('ts fetch shim recorded a composio call', tsLines.some((l) => JSON.parse(l).p?.includes('/api/')));
494
+
495
+ const pyTrace = join(runDir, 'traces', 'shim-py.jsonl');
496
+ await sh(['uv', 'run', '--project', 'python', 'python', 'harness/fixtures/trace_check.py'], {
497
+ env: { ...process.env, COMPOSIO_TRACE_FILE: pyTrace, COMPOSIO_BASE_URL: baseUrl(), PYTHONPATH: join(ROOT, 'harness', 'trace-py') },
498
+ }).catch(() => {});
499
+ const pyLines = existsSync(pyTrace) ? readFileSync(pyTrace, 'utf8').trim().split('\n').filter(Boolean) : [];
500
+ check('py httpx shim recorded a composio call', pyLines.some((l) => JSON.parse(l).p?.includes('/api/')));
501
+
502
+ // 4. parity must compare at least one entry and reject missing candidates.
503
+ const parityBase = join(runDir, 'parity-base');
504
+ const parityCandidate = join(runDir, 'parity-candidate');
505
+ for (const dir of [parityBase, parityCandidate]) {
506
+ mkdirSync(join(dir, 'traces'), { recursive: true });
507
+ writeFileSync(join(dir, 'results.jsonl'), `${JSON.stringify({ id: 'fixture', status: 'green' })}\n`);
508
+ writeFileSync(
509
+ join(dir, 'traces', 'fixture.jsonl'),
510
+ `${JSON.stringify({ m: 'GET', p: '/api/v3/toolkits', s: '2xx' })}\n`
511
+ );
512
+ }
513
+ const parityMatches = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate])
514
+ .then(() => true)
515
+ .catch(() => false);
516
+ check('parity accepts matching runs', parityMatches);
517
+ writeFileSync(join(parityCandidate, 'results.jsonl'), '');
518
+ const missingCandidateRejected = await sh(['node', 'harness/parity.mjs', parityBase, parityCandidate])
519
+ .then(() => false)
520
+ .catch(() => true);
521
+ check('parity rejects missing candidate entries', missingCandidateRejected);
522
+
523
+ rmSync(runDir, { recursive: true, force: true });
524
+ console.log(failures === 0 ? 'selftest: PASS' : `selftest: FAIL (${failures})`);
525
+ process.exitCode = failures === 0 ? 0 : 1;
526
+};
527
+
528
+try {
529
+ switch (cmd) {
530
+ case 'sweep':
531
+ await cmdSweep();
532
+ break;
533
+ case 'neg':
534
+ await cmdNeg();
535
+ break;
536
+ case 'selftest':
537
+ await cmdSelftest();
538
+ break;
539
+ default:
540
+ fail('usage: run.mjs <sweep|neg|selftest> [options]');
541
+ }
542
+} catch (error) {
543
+ console.error(`run.mjs: ${error instanceof Error ? error.message : String(error)}`);
544
+ process.exitCode = error instanceof HarnessError ? error.exitCode : 1;
545
+}