@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 119 lines 3.63 KB
Raw
1 import { describe, expect, it } from 'bun:test';
2 import {
3 fingerprint,
4 renderComparison,
5 summarize,
6 type BatchInfo,
7 type RunRecord,
8 } from './src/batch';
9
10 const run = (fields: Partial<RunRecord>): RunRecord => ({
11 journey: 'consumer-slack-message',
12 split: 'train',
13 arm: 'skill',
14 attempt: 1,
15 fingerprint: 'f',
16 outcome: 'passed',
17 checks: [],
18 detours: [],
19 stop: 'finished',
20 modelTurns: 3,
21 commands: [],
22 costUsd: 0.01,
23 durationMs: 1000,
24 ...fields,
25 });
26
27 const info = (fields: Partial<BatchInfo> = {}): BatchInfo => ({
28 id: 'batch',
29 startedAt: '2026-11-11T00:00:00.000Z',
30 agent: 'claude-code',
31 agentVersion: '2.1.310',
32 model: 'claude-haiku-5-5',
33 runs: 3,
34 arms: ['skill'],
35 journeys: ['consumer-slack-message'],
36 revision: 'abc',
37 dirty: false,
38 skillSha256: 'skill-a',
39 harnessSha256: 'harness-a',
40 cliSha256: 'cli-a',
41 maxCostUsd: 1,
42 ...fields,
43 });
44
45 describe('summarize', () => {
46 it('leaves infra failures out of every rate', () => {
47 const [cell] = summarize([
48 run({ attempt: 1 }),
49 run({ attempt: 2, outcome: 'failed', checks: [{ name: 'it went to #general', ok: false }] }),
50 run({ attempt: 3, outcome: 'infra' }),
51 ]);
52 expect(cell?.n).toBe(2);
53 expect(cell?.infra).toBe(1);
54 expect(cell?.passAt1).toBe(0.5);
55 expect(cell?.passAtK).toBe(1);
56 expect(cell?.passHatK).toBe(0);
57 expect(cell?.failedChecks).toEqual({ 'it went to #general': 1 });
58 });
59
60 it('counts a run with a detour toward the wrong-tree rate, passed or not', () => {
61 const detour = [{ name: 'no `composio dev` command ran', ok: false }];
62 const [cell] = summarize([run({ detours: detour }), run({ attempt: 2 })]);
63 expect(cell?.wrongTreeRate).toBe(0.5);
64 expect(cell?.passAt1).toBe(1);
65 });
66 });
67
68 describe('fingerprint', () => {
69 const parts = {
70 journey: 'first-run-login',
71 agent: 'claude-code' as const,
72 agentVersion: '2.1.310',
73 model: 'claude-haiku-5-5',
74 harnessSha256: 'h',
75 cliSha256: 'i',
76 };
77
78 it('changes with the skill on the skill arm only', () => {
79 expect(fingerprint({ ...parts, arm: 'skill', skillSha256: 'a' })).not.toBe(
80 fingerprint({ ...parts, arm: 'skill', skillSha256: 'b' })
81 );
82 expect(fingerprint({ ...parts, arm: 'no-skill', skillSha256: 'a' })).toBe(
83 fingerprint({ ...parts, arm: 'no-skill', skillSha256: 'b' })
84 );
85 });
86
87 it('changes with the harness, the CLI and the model', () => {
88 const base = fingerprint({ ...parts, arm: 'skill', skillSha256: 'a' });
89 expect(fingerprint({ ...parts, arm: 'skill', skillSha256: 'a', harnessSha256: 'x' })).not.toBe(
90 base
91 );
92 expect(fingerprint({ ...parts, arm: 'skill', skillSha256: 'a', cliSha256: 'x' })).not.toBe(
93 base
94 );
95 expect(
96 fingerprint({ ...parts, arm: 'skill', skillSha256: 'a', model: 'claude-opus-5-5' })
97 ).not.toBe(base);
98 });
99 });
100
101 describe('renderComparison', () => {
102 const baseline = { info: info(), runs: [run({ outcome: 'failed' }), run({ attempt: 2 })] };
103
104 it('reports the change in pass@1 per journey and split', () => {
105 const candidate = {
106 info: info({ id: 'after', skillSha256: 'skill-b' }),
107 runs: [run({}), run({ attempt: 2 })],
108 };
109 const text = renderComparison(baseline, candidate, { arm: 'skill', allowHarnessChange: false });
110 expect(text).toContain('| train | consumer-slack-message | 50% → 100% | +50 |');
111 });
112
113 it('refuses batches graded by different harnesses', () => {
114 const candidate = { info: info({ harnessSha256: 'harness-b' }), runs: [run({})] };
115 expect(() =>
116 renderComparison(baseline, candidate, { arm: 'skill', allowHarnessChange: false })
117 ).toThrow(/different harnesses/);
118 });
119 });