Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | #!/usr/bin/env bun |
| 2 | |
| 3 | import { readFileSync } from 'node:fs'; |
| 4 | import { join } from 'node:path'; |
| 5 | import { |
| 6 | evaluateKbSearchRankings, |
| 7 | type KbSearchEvalCase, |
| 8 | type KbSearchEvalRanking, |
| 9 | } from '@/lib/knowledge/evaluation'; |
| 10 | import { searchKnowledgeRecords, type KnowledgeSearchResponse } from '@/lib/knowledge/search'; |
| 11 | import { getAlgoliaSearchDocuments } from '@/lib/search-index'; |
| 12 | |
| 13 | function argument(name: string): string | undefined { |
| 14 | const index = process.argv.indexOf(name); |
| 15 | return index === -1 ? undefined : process.argv[index + 1]; |
| 16 | } |
| 17 | |
| 18 | const baseUrl = argument('--base-url')?.replace(/\/$/, ''); |
| 19 | const expectedMode = argument('--expect-mode'); |
| 20 | const fixturePath = argument('--fixture') ?? join(process.cwd(), 'evals', 'kb-search-v1.json'); |
| 21 | const jsonOutput = process.argv.includes('--json'); |
| 22 | const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as { |
| 23 | cases: KbSearchEvalCase[]; |
| 24 | }; |
| 25 | const cases = fixture.cases; |
| 26 | const rankings = new Map<string, KbSearchEvalRanking[]>(); |
| 27 | const observedModes = new Set<string>(); |
| 28 | |
| 29 | if (baseUrl) { |
| 30 | for (const evalCase of cases) { |
| 31 | const url = new URL('/api/knowledge-search', baseUrl); |
| 32 | url.searchParams.set('q', evalCase.query); |
| 33 | const response = await fetch(url); |
| 34 | if (!response.ok) { |
| 35 | throw new Error(`Search endpoint returned HTTP ${response.status} for eval case: ${evalCase.id}`); |
| 36 | } |
| 37 | const body = await response.json() as KnowledgeSearchResponse; |
| 38 | observedModes.add(body.mode ?? 'unspecified'); |
| 39 | if (expectedMode && body.mode !== expectedMode) { |
| 40 | throw new Error( |
| 41 | `Expected retrieval mode ${expectedMode}, got ${body.mode ?? 'unspecified'} for: ${evalCase.id}`, |
| 42 | ); |
| 43 | } |
| 44 | rankings.set(evalCase.id, body.results.map(result => ({ |
| 45 | canonicalUrl: result.canonicalUrl, |
| 46 | sourceType: result.sourceType, |
| 47 | }))); |
| 48 | } |
| 49 | } else { |
| 50 | const records = await getAlgoliaSearchDocuments(); |
| 51 | for (const evalCase of cases) { |
| 52 | const response = searchKnowledgeRecords(records, { |
| 53 | query: evalCase.query, |
| 54 | filter: 'all', |
| 55 | limit: 20, |
| 56 | }); |
| 57 | rankings.set(evalCase.id, response.results.map(result => ({ |
| 58 | canonicalUrl: result.canonicalUrl, |
| 59 | sourceType: result.sourceType, |
| 60 | }))); |
| 61 | } |
| 62 | observedModes.add('local-keyword'); |
| 63 | } |
| 64 | |
| 65 | const report = evaluateKbSearchRankings(cases, rankings); |
| 66 | if (jsonOutput) { |
| 67 | console.log(JSON.stringify({ modes: [...observedModes], ...report }, null, 2)); |
| 68 | process.exit(0); |
| 69 | } |
| 70 | |
| 71 | const percent = (value: number): string => `${(value * 100).toFixed(1)}%`; |
| 72 | console.log(`KB search eval (${[...observedModes].join(', ')})`); |
| 73 | console.log(`Answerable: ${report.answerable.count}`); |
| 74 | console.log(`Recall@5: ${percent(report.answerable.recallAt5)}`); |
| 75 | console.log(`MRR@10: ${report.answerable.mrrAt10.toFixed(3)}`); |
| 76 | console.log(`Exact Recall@5: ${percent(report.byKind.exact.recallAt5)}`); |
| 77 | console.log(`Paraphrase Recall@5: ${percent(report.byKind.paraphrase.recallAt5)}`); |
| 78 | console.log(`No-answer empty@5: ${percent(report.noAnswer.emptyAt5Rate)}`); |
| 79 | |
| 80 | const misses = report.cases.filter(result => result.kind !== 'no-answer' && !result.hitAt5); |
| 81 | if (misses.length > 0) { |
| 82 | console.log('\nAnswerable misses at 5:'); |
| 83 | for (const miss of misses) { |
| 84 | console.log(`- ${miss.id}: top=${miss.rankedUrls[0] ?? '(empty)'}`); |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | const noisy = report.cases.filter(result => result.kind === 'no-answer' && result.rankedUrls.length > 0); |
| 89 | if (noisy.length > 0) { |
| 90 | console.log('\nNo-answer queries with results:'); |
| 91 | for (const result of noisy) { |
| 92 | console.log(`- ${result.id}: top=${result.rankedUrls[0]}`); |
| 93 | } |
| 94 | } |