Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { describe, expect, it } from 'bun:test'; |
| 2 | import { passAtK, passHatK, wilson } from './src/metrics'; |
| 3 | |
| 4 | describe('pass@k and pass^k', () => { |
| 5 | it('agree with pass@1 when k is 1', () => { |
| 6 | expect(passAtK(5, 3, 1)).toBeCloseTo(0.6); |
| 7 | expect(passHatK(5, 3, 1)).toBeCloseTo(0.6); |
| 8 | }); |
| 9 | |
| 10 | it('are 1 or 0 when every run passed or none did', () => { |
| 11 | expect(passAtK(3, 3, 3)).toBe(1); |
| 12 | expect(passHatK(3, 3, 3)).toBe(1); |
| 13 | expect(passAtK(3, 0, 3)).toBe(0); |
| 14 | expect(passHatK(3, 0, 3)).toBe(0); |
| 15 | }); |
| 16 | |
| 17 | it('pass@k rises and pass^k falls as k grows', () => { |
| 18 | // 2 of 3 runs passed. |
| 19 | expect(passAtK(3, 2, 2)).toBe(1); |
| 20 | expect(passHatK(3, 2, 2)).toBeCloseTo(1 / 3); |
| 21 | expect(passHatK(3, 2, 3)).toBe(0); |
| 22 | }); |
| 23 | |
| 24 | it('match the closed forms for n=10, c=7, k=3', () => { |
| 25 | // 1 - C(3,3)/C(10,3) and C(7,3)/C(10,3). |
| 26 | expect(passAtK(10, 7, 3)).toBeCloseTo(1 - 1 / 120); |
| 27 | expect(passHatK(10, 7, 3)).toBeCloseTo(35 / 120); |
| 28 | }); |
| 29 | |
| 30 | it('refuse counts that cannot happen', () => { |
| 31 | expect(() => passAtK(3, 4, 1)).toThrow(); |
| 32 | expect(() => passHatK(3, 1, 4)).toThrow(); |
| 33 | expect(() => passHatK(3, 1, 0)).toThrow(); |
| 34 | }); |
| 35 | }); |
| 36 | |
| 37 | describe('wilson', () => { |
| 38 | it('brackets the observed rate', () => { |
| 39 | const interval = wilson(2, 3); |
| 40 | expect(interval.low).toBeLessThan(2 / 3); |
| 41 | expect(interval.high).toBeGreaterThan(2 / 3); |
| 42 | }); |
| 43 | |
| 44 | it('stays inside [0, 1] at the edges', () => { |
| 45 | expect(wilson(0, 3).low).toBe(0); |
| 46 | expect(wilson(3, 3).high).toBe(1); |
| 47 | expect(wilson(0, 0)).toEqual({ low: 0, high: 1 }); |
| 48 | }); |
| 49 | }); |