Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 129 lines 4.64 KB
Raw
1 import type { KnowledgeSourceType } from './types';
2
3 export type KbSearchEvalKind = 'exact' | 'paraphrase' | 'no-answer';
4
5 export interface KbSearchEvalRanking {
6 canonicalUrl: string;
7 sourceType: KnowledgeSourceType;
8 }
9
10 export interface KbSearchEvalCase {
11 id: string;
12 kind: KbSearchEvalKind;
13 query: string;
14 expectedUrls: string[];
15 expectedSourceTypes?: KnowledgeSourceType[];
16 note?: string;
17 }
18
19 export interface KbSearchEvalCaseResult extends KbSearchEvalCase {
20 rankedUrls: string[];
21 rankedSourceTypes: Array<KnowledgeSourceType | null>;
22 firstRelevantRank: number | null;
23 hitAt5: boolean;
24 }
25
26 interface AnswerableMetrics {
27 count: number;
28 recallAt5: number;
29 mrrAt10: number;
30 }
31
32 export interface KbSearchEvalReport {
33 answerable: AnswerableMetrics;
34 byKind: Record<'exact' | 'paraphrase', AnswerableMetrics>;
35 noAnswer: {
36 count: number;
37 emptyAt5Rate: number;
38 };
39 cases: KbSearchEvalCaseResult[];
40 }
41
42 function canonicalPage(url: string): string {
43 return url.split('#', 1)[0] ?? url;
44 }
45
46 function answerableMetrics(results: KbSearchEvalCaseResult[]): AnswerableMetrics {
47 if (results.length === 0) return { count: 0, recallAt5: 0, mrrAt10: 0 };
48 const hitsAt5 = results.filter(result => result.hitAt5).length;
49 const reciprocalRanks = results.reduce((total, result) => {
50 const rank = result.firstRelevantRank;
51 return total + (rank !== null && rank <= 10 ? 1 / rank : 0);
52 }, 0);
53 return {
54 count: results.length,
55 recallAt5: hitsAt5 / results.length,
56 mrrAt10: reciprocalRanks / results.length,
57 };
58 }
59
60 export function evaluateKbSearchRankings(
61 cases: KbSearchEvalCase[],
62 rankings: ReadonlyMap<string, Array<string | KbSearchEvalRanking>>,
63 ): KbSearchEvalReport {
64 const seen = new Set<string>();
65 const results = cases.map(evalCase => {
66 if (!evalCase.id || seen.has(evalCase.id)) {
67 throw new Error(`Eval case IDs must be unique and non-empty: ${evalCase.id}`);
68 }
69 seen.add(evalCase.id);
70 if (!evalCase.query.trim()) throw new Error(`Eval case query is empty: ${evalCase.id}`);
71 if (evalCase.kind === 'no-answer' && evalCase.expectedUrls.length > 0) {
72 throw new Error(`No-answer eval case has expected URLs: ${evalCase.id}`);
73 }
74 if (evalCase.kind === 'no-answer' && (evalCase.expectedSourceTypes?.length ?? 0) > 0) {
75 throw new Error(`No-answer eval case has expected source types: ${evalCase.id}`);
76 }
77 if (evalCase.kind !== 'no-answer' && evalCase.expectedUrls.length === 0 &&
78 (evalCase.expectedSourceTypes?.length ?? 0) === 0) {
79 throw new Error(`Answerable eval case has no expected URLs or source types: ${evalCase.id}`);
80 }
81 if (evalCase.expectedSourceTypes && evalCase.expectedSourceTypes.length === 0) {
82 throw new Error(`Eval case has an empty expected source-type list: ${evalCase.id}`);
83 }
84 const ranked = rankings.get(evalCase.id);
85 if (!ranked) throw new Error(`Missing ranking for eval case: ${evalCase.id}`);
86 const rankedResults = ranked.map(result => typeof result === 'string'
87 ? { canonicalUrl: result, sourceType: null }
88 : result);
89 const rankedUrls = rankedResults.map(result => result.canonicalUrl);
90 const rankedSourceTypes = rankedResults.map(result => result.sourceType);
91 const expectedPages = evalCase.kind !== 'no-answer' && evalCase.expectedUrls.length === 0
92 ? null
93 : new Set(evalCase.expectedUrls.map(canonicalPage));
94 const expectedSourceTypes = evalCase.expectedSourceTypes
95 ? new Set(evalCase.expectedSourceTypes)
96 : null;
97 const firstRelevantIndex = rankedResults.findIndex(result =>
98 (expectedPages === null || expectedPages.has(canonicalPage(result.canonicalUrl))) &&
99 (expectedSourceTypes === null || (
100 result.sourceType !== null && expectedSourceTypes.has(result.sourceType)
101 )));
102 const firstRelevantRank = firstRelevantIndex === -1 ? null : firstRelevantIndex + 1;
103 return {
104 ...evalCase,
105 rankedUrls,
106 rankedSourceTypes,
107 firstRelevantRank,
108 hitAt5: firstRelevantRank !== null && firstRelevantRank <= 5,
109 };
110 });
111
112 const exact = results.filter(result => result.kind === 'exact');
113 const paraphrase = results.filter(result => result.kind === 'paraphrase');
114 const noAnswer = results.filter(result => result.kind === 'no-answer');
115 return {
116 answerable: answerableMetrics([...exact, ...paraphrase]),
117 byKind: {
118 exact: answerableMetrics(exact),
119 paraphrase: answerableMetrics(paraphrase),
120 },
121 noAnswer: {
122 count: noAnswer.length,
123 emptyAt5Rate: noAnswer.length === 0
124 ? 0
125 : noAnswer.filter(result => result.rankedUrls.slice(0, 5).length === 0).length / noAnswer.length,
126 },
127 cases: results,
128 };
129 }