Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | /** |
| 2 | * Pass rates over repeated runs of one journey. |
| 3 | * |
| 4 | * With n graded runs of which c passed: |
| 5 | * |
| 6 | * - pass@k: the chance that at least one of k runs passes. It answers "can |
| 7 | * the agent do this at all", and rises with k. |
| 8 | * - pass^k: the chance that all k runs pass. It answers "can a user rely on |
| 9 | * it", and falls with k. A skill change that makes the agent dependable |
| 10 | * moves pass^k even when pass@k is already 1. |
| 11 | * |
| 12 | * Both are the unbiased estimates over the n runs (Chen et al. 2021 for |
| 13 | * pass@k; Yao et al. 2024, tau-bench, for pass^k), so k may be smaller than n. |
| 14 | */ |
| 15 | |
| 16 | const choose = (n: number, k: number): number => { |
| 17 | if (k < 0 || k > n) return 0; |
| 18 | let result = 1; |
| 19 | for (let i = 1; i <= k; i += 1) result = (result * (n - k + i)) / i; |
| 20 | return result; |
| 21 | }; |
| 22 | |
| 23 | const assertCounts = (n: number, c: number, k: number) => { |
| 24 | if (!Number.isInteger(n) || !Number.isInteger(c) || !Number.isInteger(k)) { |
| 25 | throw new Error(`counts must be integers (n=${n}, c=${c}, k=${k})`); |
| 26 | } |
| 27 | if (c < 0 || c > n || k < 1 || k > n) { |
| 28 | throw new Error(`need 0 <= c <= n and 1 <= k <= n (n=${n}, c=${c}, k=${k})`); |
| 29 | } |
| 30 | }; |
| 31 | |
| 32 | export const passAtK = (n: number, c: number, k: number): number => { |
| 33 | assertCounts(n, c, k); |
| 34 | return 1 - choose(n - c, k) / choose(n, k); |
| 35 | }; |
| 36 | |
| 37 | export const passHatK = (n: number, c: number, k: number): number => { |
| 38 | assertCounts(n, c, k); |
| 39 | return choose(c, k) / choose(n, k); |
| 40 | }; |
| 41 | |
| 42 | /** The 95% Wilson score interval for c successes in n runs. */ |
| 43 | export const wilson = (c: number, n: number): { low: number; high: number } => { |
| 44 | if (n === 0) return { low: 0, high: 1 }; |
| 45 | const z = 1.96; |
| 46 | const p = c / n; |
| 47 | const denominator = 1 + (z * z) / n; |
| 48 | const centre = (p + (z * z) / (2 * n)) / denominator; |
| 49 | const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denominator; |
| 50 | return { low: Math.max(0, centre - half), high: Math.min(1, centre + half) }; |
| 51 | }; |