Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import type { KnowledgeSourceType } from './types'; |
| 2 | |
| 3 | export type KbSearchEvalKind = 'exact' | 'paraphrase' | 'no-answer'; |
| 4 | |
| 5 | export interface KbSearchEvalRanking { |
| 6 | canonicalUrl: string; |
| 7 | sourceType: KnowledgeSourceType; |
| 8 | } |
| 9 | |
| 10 | export interface KbSearchEvalCase { |
| 11 | id: string; |
| 12 | kind: KbSearchEvalKind; |
| 13 | query: string; |
| 14 | expectedUrls: string[]; |
| 15 | expectedSourceTypes?: KnowledgeSourceType[]; |
| 16 | note?: string; |
| 17 | } |
| 18 | |
| 19 | export interface KbSearchEvalCaseResult extends KbSearchEvalCase { |
| 20 | rankedUrls: string[]; |
| 21 | rankedSourceTypes: Array<KnowledgeSourceType | null>; |
| 22 | firstRelevantRank: number | null; |
| 23 | hitAt5: boolean; |
| 24 | } |
| 25 | |
| 26 | interface AnswerableMetrics { |
| 27 | count: number; |
| 28 | recallAt5: number; |
| 29 | mrrAt10: number; |
| 30 | } |
| 31 | |
| 32 | export interface KbSearchEvalReport { |
| 33 | answerable: AnswerableMetrics; |
| 34 | byKind: Record<'exact' | 'paraphrase', AnswerableMetrics>; |
| 35 | noAnswer: { |
| 36 | count: number; |
| 37 | emptyAt5Rate: number; |
| 38 | }; |
| 39 | cases: KbSearchEvalCaseResult[]; |
| 40 | } |
| 41 | |
| 42 | function canonicalPage(url: string): string { |
| 43 | return url.split('#', 1)[0] ?? url; |
| 44 | } |
| 45 | |
| 46 | function answerableMetrics(results: KbSearchEvalCaseResult[]): AnswerableMetrics { |
| 47 | if (results.length === 0) return { count: 0, recallAt5: 0, mrrAt10: 0 }; |
| 48 | const hitsAt5 = results.filter(result => result.hitAt5).length; |
| 49 | const reciprocalRanks = results.reduce((total, result) => { |
| 50 | const rank = result.firstRelevantRank; |
| 51 | return total + (rank !== null && rank <= 10 ? 1 / rank : 0); |
| 52 | }, 0); |
| 53 | return { |
| 54 | count: results.length, |
| 55 | recallAt5: hitsAt5 / results.length, |
| 56 | mrrAt10: reciprocalRanks / results.length, |
| 57 | }; |
| 58 | } |
| 59 | |
| 60 | export function evaluateKbSearchRankings( |
| 61 | cases: KbSearchEvalCase[], |
| 62 | rankings: ReadonlyMap<string, Array<string | KbSearchEvalRanking>>, |
| 63 | ): KbSearchEvalReport { |
| 64 | const seen = new Set<string>(); |
| 65 | const results = cases.map(evalCase => { |
| 66 | if (!evalCase.id || seen.has(evalCase.id)) { |
| 67 | throw new Error(`Eval case IDs must be unique and non-empty: ${evalCase.id}`); |
| 68 | } |
| 69 | seen.add(evalCase.id); |
| 70 | if (!evalCase.query.trim()) throw new Error(`Eval case query is empty: ${evalCase.id}`); |
| 71 | if (evalCase.kind === 'no-answer' && evalCase.expectedUrls.length > 0) { |
| 72 | throw new Error(`No-answer eval case has expected URLs: ${evalCase.id}`); |
| 73 | } |
| 74 | if (evalCase.kind === 'no-answer' && (evalCase.expectedSourceTypes?.length ?? 0) > 0) { |
| 75 | throw new Error(`No-answer eval case has expected source types: ${evalCase.id}`); |
| 76 | } |
| 77 | if (evalCase.kind !== 'no-answer' && evalCase.expectedUrls.length === 0 && |
| 78 | (evalCase.expectedSourceTypes?.length ?? 0) === 0) { |
| 79 | throw new Error(`Answerable eval case has no expected URLs or source types: ${evalCase.id}`); |
| 80 | } |
| 81 | if (evalCase.expectedSourceTypes && evalCase.expectedSourceTypes.length === 0) { |
| 82 | throw new Error(`Eval case has an empty expected source-type list: ${evalCase.id}`); |
| 83 | } |
| 84 | const ranked = rankings.get(evalCase.id); |
| 85 | if (!ranked) throw new Error(`Missing ranking for eval case: ${evalCase.id}`); |
| 86 | const rankedResults = ranked.map(result => typeof result === 'string' |
| 87 | ? { canonicalUrl: result, sourceType: null } |
| 88 | : result); |
| 89 | const rankedUrls = rankedResults.map(result => result.canonicalUrl); |
| 90 | const rankedSourceTypes = rankedResults.map(result => result.sourceType); |
| 91 | const expectedPages = evalCase.kind !== 'no-answer' && evalCase.expectedUrls.length === 0 |
| 92 | ? null |
| 93 | : new Set(evalCase.expectedUrls.map(canonicalPage)); |
| 94 | const expectedSourceTypes = evalCase.expectedSourceTypes |
| 95 | ? new Set(evalCase.expectedSourceTypes) |
| 96 | : null; |
| 97 | const firstRelevantIndex = rankedResults.findIndex(result => |
| 98 | (expectedPages === null || expectedPages.has(canonicalPage(result.canonicalUrl))) && |
| 99 | (expectedSourceTypes === null || ( |
| 100 | result.sourceType !== null && expectedSourceTypes.has(result.sourceType) |
| 101 | ))); |
| 102 | const firstRelevantRank = firstRelevantIndex === -1 ? null : firstRelevantIndex + 1; |
| 103 | return { |
| 104 | ...evalCase, |
| 105 | rankedUrls, |
| 106 | rankedSourceTypes, |
| 107 | firstRelevantRank, |
| 108 | hitAt5: firstRelevantRank !== null && firstRelevantRank <= 5, |
| 109 | }; |
| 110 | }); |
| 111 | |
| 112 | const exact = results.filter(result => result.kind === 'exact'); |
| 113 | const paraphrase = results.filter(result => result.kind === 'paraphrase'); |
| 114 | const noAnswer = results.filter(result => result.kind === 'no-answer'); |
| 115 | return { |
| 116 | answerable: answerableMetrics([...exact, ...paraphrase]), |
| 117 | byKind: { |
| 118 | exact: answerableMetrics(exact), |
| 119 | paraphrase: answerableMetrics(paraphrase), |
| 120 | }, |
| 121 | noAnswer: { |
| 122 | count: noAnswer.length, |
| 123 | emptyAt5Rate: noAnswer.length === 0 |
| 124 | ? 0 |
| 125 | : noAnswer.filter(result => result.rankedUrls.slice(0, 5).length === 0).length / noAnswer.length, |
| 126 | }, |
| 127 | cases: results, |
| 128 | }; |
| 129 | } |