@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 51 lines 1.86 KB
Raw
1 /**
2 * Pass rates over repeated runs of one journey.
3 *
4 * With n graded runs of which c passed:
5 *
6 * - pass@k: the chance that at least one of k runs passes. It answers "can
7 * the agent do this at all", and rises with k.
8 * - pass^k: the chance that all k runs pass. It answers "can a user rely on
9 * it", and falls with k. A skill change that makes the agent dependable
10 * moves pass^k even when pass@k is already 1.
11 *
12 * Both are the unbiased estimates over the n runs (Chen et al. 2021 for
13 * pass@k; Yao et al. 2024, tau-bench, for pass^k), so k may be smaller than n.
14 */
15
16 const choose = (n: number, k: number): number => {
17 if (k < 0 || k > n) return 0;
18 let result = 1;
19 for (let i = 1; i <= k; i += 1) result = (result * (n - k + i)) / i;
20 return result;
21 };
22
23 const assertCounts = (n: number, c: number, k: number) => {
24 if (!Number.isInteger(n) || !Number.isInteger(c) || !Number.isInteger(k)) {
25 throw new Error(`counts must be integers (n=${n}, c=${c}, k=${k})`);
26 }
27 if (c < 0 || c > n || k < 1 || k > n) {
28 throw new Error(`need 0 <= c <= n and 1 <= k <= n (n=${n}, c=${c}, k=${k})`);
29 }
30 };
31
32 export const passAtK = (n: number, c: number, k: number): number => {
33 assertCounts(n, c, k);
34 return 1 - choose(n - c, k) / choose(n, k);
35 };
36
37 export const passHatK = (n: number, c: number, k: number): number => {
38 assertCounts(n, c, k);
39 return choose(c, k) / choose(n, k);
40 };
41
42 /** The 95% Wilson score interval for c successes in n runs. */
43 export const wilson = (c: number, n: number): { low: number; high: number } => {
44 if (n === 0) return { low: 0, high: 1 };
45 const z = 1.96;
46 const p = c / n;
47 const denominator = 1 + (z * z) / n;
48 const centre = (p + (z * z) / (2 * n)) / denominator;
49 const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denominator;
50 return { low: Math.max(0, centre - half), high: Math.min(1, centre + half) };
51 };