@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 49 lines 1.52 KB
Raw
1 import { describe, expect, it } from 'bun:test';
2 import { passAtK, passHatK, wilson } from './src/metrics';
3
4 describe('pass@k and pass^k', () => {
5 it('agree with pass@1 when k is 1', () => {
6 expect(passAtK(5, 3, 1)).toBeCloseTo(0.6);
7 expect(passHatK(5, 3, 1)).toBeCloseTo(0.6);
8 });
9
10 it('are 1 or 0 when every run passed or none did', () => {
11 expect(passAtK(3, 3, 3)).toBe(1);
12 expect(passHatK(3, 3, 3)).toBe(1);
13 expect(passAtK(3, 0, 3)).toBe(0);
14 expect(passHatK(3, 0, 3)).toBe(0);
15 });
16
17 it('pass@k rises and pass^k falls as k grows', () => {
18 // 2 of 3 runs passed.
19 expect(passAtK(3, 2, 2)).toBe(1);
20 expect(passHatK(3, 2, 2)).toBeCloseTo(1 / 3);
21 expect(passHatK(3, 2, 3)).toBe(0);
22 });
23
24 it('match the closed forms for n=10, c=7, k=3', () => {
25 // 1 - C(3,3)/C(10,3) and C(7,3)/C(10,3).
26 expect(passAtK(10, 7, 3)).toBeCloseTo(1 - 1 / 120);
27 expect(passHatK(10, 7, 3)).toBeCloseTo(35 / 120);
28 });
29
30 it('refuse counts that cannot happen', () => {
31 expect(() => passAtK(3, 4, 1)).toThrow();
32 expect(() => passHatK(3, 1, 4)).toThrow();
33 expect(() => passHatK(3, 1, 0)).toThrow();
34 });
35 });
36
37 describe('wilson', () => {
38 it('brackets the observed rate', () => {
39 const interval = wilson(2, 3);
40 expect(interval.low).toBeLessThan(2 / 3);
41 expect(interval.high).toBeGreaterThan(2 / 3);
42 });
43
44 it('stays inside [0, 1] at the edges', () => {
45 expect(wilson(0, 3).low).toBe(0);
46 expect(wilson(3, 3).high).toBe(1);
47 expect(wilson(0, 0)).toEqual({ low: 0, high: 1 });
48 });
49 });