@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 94 lines 3.46 KB
Raw
1 #!/usr/bin/env bun
2
3 import { readFileSync } from 'node:fs';
4 import { join } from 'node:path';
5 import {
6 evaluateKbSearchRankings,
7 type KbSearchEvalCase,
8 type KbSearchEvalRanking,
9 } from '@/lib/knowledge/evaluation';
10 import { searchKnowledgeRecords, type KnowledgeSearchResponse } from '@/lib/knowledge/search';
11 import { getAlgoliaSearchDocuments } from '@/lib/search-index';
12
13 function argument(name: string): string | undefined {
14 const index = process.argv.indexOf(name);
15 return index === -1 ? undefined : process.argv[index + 1];
16 }
17
18 const baseUrl = argument('--base-url')?.replace(/\/$/, '');
19 const expectedMode = argument('--expect-mode');
20 const fixturePath = argument('--fixture') ?? join(process.cwd(), 'evals', 'kb-search-v1.json');
21 const jsonOutput = process.argv.includes('--json');
22 const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as {
23 cases: KbSearchEvalCase[];
24 };
25 const cases = fixture.cases;
26 const rankings = new Map<string, KbSearchEvalRanking[]>();
27 const observedModes = new Set<string>();
28
29 if (baseUrl) {
30 for (const evalCase of cases) {
31 const url = new URL('/api/knowledge-search', baseUrl);
32 url.searchParams.set('q', evalCase.query);
33 const response = await fetch(url);
34 if (!response.ok) {
35 throw new Error(`Search endpoint returned HTTP ${response.status} for eval case: ${evalCase.id}`);
36 }
37 const body = await response.json() as KnowledgeSearchResponse;
38 observedModes.add(body.mode ?? 'unspecified');
39 if (expectedMode && body.mode !== expectedMode) {
40 throw new Error(
41 `Expected retrieval mode ${expectedMode}, got ${body.mode ?? 'unspecified'} for: ${evalCase.id}`,
42 );
43 }
44 rankings.set(evalCase.id, body.results.map(result => ({
45 canonicalUrl: result.canonicalUrl,
46 sourceType: result.sourceType,
47 })));
48 }
49 } else {
50 const records = await getAlgoliaSearchDocuments();
51 for (const evalCase of cases) {
52 const response = searchKnowledgeRecords(records, {
53 query: evalCase.query,
54 filter: 'all',
55 limit: 20,
56 });
57 rankings.set(evalCase.id, response.results.map(result => ({
58 canonicalUrl: result.canonicalUrl,
59 sourceType: result.sourceType,
60 })));
61 }
62 observedModes.add('local-keyword');
63 }
64
65 const report = evaluateKbSearchRankings(cases, rankings);
66 if (jsonOutput) {
67 console.log(JSON.stringify({ modes: [...observedModes], ...report }, null, 2));
68 process.exit(0);
69 }
70
71 const percent = (value: number): string => `${(value * 100).toFixed(1)}%`;
72 console.log(`KB search eval (${[...observedModes].join(', ')})`);
73 console.log(`Answerable: ${report.answerable.count}`);
74 console.log(`Recall@5: ${percent(report.answerable.recallAt5)}`);
75 console.log(`MRR@10: ${report.answerable.mrrAt10.toFixed(3)}`);
76 console.log(`Exact Recall@5: ${percent(report.byKind.exact.recallAt5)}`);
77 console.log(`Paraphrase Recall@5: ${percent(report.byKind.paraphrase.recallAt5)}`);
78 console.log(`No-answer empty@5: ${percent(report.noAnswer.emptyAt5Rate)}`);
79
80 const misses = report.cases.filter(result => result.kind !== 'no-answer' && !result.hitAt5);
81 if (misses.length > 0) {
82 console.log('\nAnswerable misses at 5:');
83 for (const miss of misses) {
84 console.log(`- ${miss.id}: top=${miss.rankedUrls[0] ?? '(empty)'}`);
85 }
86 }
87
88 const noisy = report.cases.filter(result => result.kind === 'no-answer' && result.rankedUrls.length > 0);
89 if (noisy.length > 0) {
90 console.log('\nNo-answer queries with results:');
91 for (const result of noisy) {
92 console.log(`- ${result.id}: top=${result.rankedUrls[0]}`);
93 }
94 }