Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | import { describe, expect, it } from 'bun:test'; |
| 2 | import { |
| 3 | fingerprint, |
| 4 | renderComparison, |
| 5 | summarize, |
| 6 | type BatchInfo, |
| 7 | type RunRecord, |
| 8 | } from './src/batch'; |
| 9 | |
| 10 | const run = (fields: Partial<RunRecord>): RunRecord => ({ |
| 11 | journey: 'consumer-slack-message', |
| 12 | split: 'train', |
| 13 | arm: 'skill', |
| 14 | attempt: 1, |
| 15 | fingerprint: 'f', |
| 16 | outcome: 'passed', |
| 17 | checks: [], |
| 18 | detours: [], |
| 19 | stop: 'finished', |
| 20 | modelTurns: 3, |
| 21 | commands: [], |
| 22 | costUsd: 0.01, |
| 23 | durationMs: 1000, |
| 24 | ...fields, |
| 25 | }); |
| 26 | |
| 27 | const info = (fields: Partial<BatchInfo> = {}): BatchInfo => ({ |
| 28 | id: 'batch', |
| 29 | startedAt: '2026-11-11T00:00:00.000Z', |
| 30 | agent: 'claude-code', |
| 31 | agentVersion: '2.1.310', |
| 32 | model: 'claude-haiku-5-5', |
| 33 | runs: 3, |
| 34 | arms: ['skill'], |
| 35 | journeys: ['consumer-slack-message'], |
| 36 | revision: 'abc', |
| 37 | dirty: false, |
| 38 | skillSha256: 'skill-a', |
| 39 | harnessSha256: 'harness-a', |
| 40 | cliSha256: 'cli-a', |
| 41 | maxCostUsd: 1, |
| 42 | ...fields, |
| 43 | }); |
| 44 | |
| 45 | describe('summarize', () => { |
| 46 | it('leaves infra failures out of every rate', () => { |
| 47 | const [cell] = summarize([ |
| 48 | run({ attempt: 1 }), |
| 49 | run({ attempt: 2, outcome: 'failed', checks: [{ name: 'it went to #general', ok: false }] }), |
| 50 | run({ attempt: 3, outcome: 'infra' }), |
| 51 | ]); |
| 52 | expect(cell?.n).toBe(2); |
| 53 | expect(cell?.infra).toBe(1); |
| 54 | expect(cell?.passAt1).toBe(0.5); |
| 55 | expect(cell?.passAtK).toBe(1); |
| 56 | expect(cell?.passHatK).toBe(0); |
| 57 | expect(cell?.failedChecks).toEqual({ 'it went to #general': 1 }); |
| 58 | }); |
| 59 | |
| 60 | it('counts a run with a detour toward the wrong-tree rate, passed or not', () => { |
| 61 | const detour = [{ name: 'no `composio dev` command ran', ok: false }]; |
| 62 | const [cell] = summarize([run({ detours: detour }), run({ attempt: 2 })]); |
| 63 | expect(cell?.wrongTreeRate).toBe(0.5); |
| 64 | expect(cell?.passAt1).toBe(1); |
| 65 | }); |
| 66 | }); |
| 67 | |
| 68 | describe('fingerprint', () => { |
| 69 | const parts = { |
| 70 | journey: 'first-run-login', |
| 71 | agent: 'claude-code' as const, |
| 72 | agentVersion: '2.1.310', |
| 73 | model: 'claude-haiku-5-5', |
| 74 | harnessSha256: 'h', |
| 75 | cliSha256: 'i', |
| 76 | }; |
| 77 | |
| 78 | it('changes with the skill on the skill arm only', () => { |
| 79 | expect(fingerprint({ ...parts, arm: 'skill', skillSha256: 'a' })).not.toBe( |
| 80 | fingerprint({ ...parts, arm: 'skill', skillSha256: 'b' }) |
| 81 | ); |
| 82 | expect(fingerprint({ ...parts, arm: 'no-skill', skillSha256: 'a' })).toBe( |
| 83 | fingerprint({ ...parts, arm: 'no-skill', skillSha256: 'b' }) |
| 84 | ); |
| 85 | }); |
| 86 | |
| 87 | it('changes with the harness, the CLI and the model', () => { |
| 88 | const base = fingerprint({ ...parts, arm: 'skill', skillSha256: 'a' }); |
| 89 | expect(fingerprint({ ...parts, arm: 'skill', skillSha256: 'a', harnessSha256: 'x' })).not.toBe( |
| 90 | base |
| 91 | ); |
| 92 | expect(fingerprint({ ...parts, arm: 'skill', skillSha256: 'a', cliSha256: 'x' })).not.toBe( |
| 93 | base |
| 94 | ); |
| 95 | expect( |
| 96 | fingerprint({ ...parts, arm: 'skill', skillSha256: 'a', model: 'claude-opus-5-5' }) |
| 97 | ).not.toBe(base); |
| 98 | }); |
| 99 | }); |
| 100 | |
| 101 | describe('renderComparison', () => { |
| 102 | const baseline = { info: info(), runs: [run({ outcome: 'failed' }), run({ attempt: 2 })] }; |
| 103 | |
| 104 | it('reports the change in pass@1 per journey and split', () => { |
| 105 | const candidate = { |
| 106 | info: info({ id: 'after', skillSha256: 'skill-b' }), |
| 107 | runs: [run({}), run({ attempt: 2 })], |
| 108 | }; |
| 109 | const text = renderComparison(baseline, candidate, { arm: 'skill', allowHarnessChange: false }); |
| 110 | expect(text).toContain('| train | consumer-slack-message | 50% → 100% | +50 |'); |
| 111 | }); |
| 112 | |
| 113 | it('refuses batches graded by different harnesses', () => { |
| 114 | const candidate = { info: info({ harnessSha256: 'harness-b' }), runs: [run({})] }; |
| 115 | expect(() => |
| 116 | renderComparison(baseline, candidate, { arm: 'skill', allowHarnessChange: false }) |
| 117 | ).toThrow(/different harnesses/); |
| 118 | }); |
| 119 | }); |