Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 111 lines 3.67 KB
Raw
1 import { describe, expect, test } from 'bun:test';
2 import {
3 evaluateKbSearchRankings,
4 type KbSearchEvalCase,
5 } from '@/lib/knowledge/evaluation';
6
7 const cases: KbSearchEvalCase[] = [
8 {
9 id: 'exact-hit',
10 kind: 'exact',
11 query: 'CALENDLY_POST_INVITEE',
12 expectedUrls: ['/kb/guide/calendly'],
13 },
14 {
15 id: 'paraphrase-hit',
16 kind: 'paraphrase',
17 query: 'my calendar alias fails',
18 expectedUrls: ['/kb/guide/google-calendar'],
19 },
20 {
21 id: 'paraphrase-miss',
22 kind: 'paraphrase',
23 query: 'connection stopped unexpectedly',
24 expectedUrls: ['/kb/guide/connected-accounts'],
25 },
26 {
27 id: 'out-of-scope',
28 kind: 'no-answer',
29 query: 'best pizza near the office',
30 expectedUrls: [],
31 },
32 ];
33
34 describe('KB search evaluation metrics', () => {
35 test('computes recall, reciprocal rank, and no-answer empty rate by query kind', () => {
36 const report = evaluateKbSearchRankings(cases, new Map([
37 ['exact-hit', ['/kb/guide/calendly#answer']],
38 ['paraphrase-hit', ['/kb/guide/unrelated', '/kb/guide/google-calendar#answer']],
39 ['paraphrase-miss', ['/kb/guide/unrelated']],
40 ['out-of-scope', []],
41 ]));
42
43 expect(report.answerable.count).toBe(3);
44 expect(report.answerable.recallAt5).toBeCloseTo(2 / 3);
45 expect(report.answerable.mrrAt10).toBeCloseTo((1 + 0.5) / 3);
46 expect(report.byKind.exact.recallAt5).toBe(1);
47 expect(report.byKind.paraphrase.recallAt5).toBe(0.5);
48 expect(report.noAnswer.emptyAt5Rate).toBe(1);
49 expect(report.cases.find(result => result.id === 'paraphrase-hit')?.firstRelevantRank).toBe(2);
50 expect(report.cases.find(result => result.id === 'out-of-scope')).toMatchObject({
51 firstRelevantRank: null,
52 hitAt5: false,
53 });
54 });
55
56 test('matches expected pages regardless of result anchors and rejects missing rankings', () => {
57 expect(() => evaluateKbSearchRankings(cases, new Map([
58 ['exact-hit', ['/kb/guide/calendly']],
59 ]))).toThrow('Missing ranking for eval case: paraphrase-hit');
60 });
61
62 test('requires an accepted source class for structural eval cases', () => {
63 const structuralCase: KbSearchEvalCase = {
64 id: 'connect-claude',
65 kind: 'paraphrase',
66 query: 'how to connect to claude',
67 expectedUrls: ['/docs/composio-connect'],
68 expectedSourceTypes: ['docs'],
69 };
70
71 const report = evaluateKbSearchRankings([structuralCase], new Map([
72 ['connect-claude', [
73 { canonicalUrl: '/docs/composio-connect', sourceType: 'kb' },
74 { canonicalUrl: '/docs/composio-connect#claude-code', sourceType: 'docs' },
75 ]],
76 ]));
77
78 expect(report.cases[0]?.firstRelevantRank).toBe(2);
79 expect(report.cases[0]?.rankedSourceTypes).toEqual(['kb', 'docs']);
80 });
81
82 test('supports source-only structural cases without mutable answer expectations', () => {
83 const report = evaluateKbSearchRankings([{
84 id: 'calendly-source',
85 kind: 'exact',
86 query: 'CALENDLY_POST_INVITEE',
87 expectedUrls: [],
88 expectedSourceTypes: ['toolkit', 'reference'],
89 }], new Map([
90 ['calendly-source', [
91 { canonicalUrl: '/kb/guide/toolkits-calendly', sourceType: 'kb' },
92 { canonicalUrl: '/toolkits/calendly', sourceType: 'toolkit' },
93 ]],
94 ]));
95
96 expect(report.cases[0]?.firstRelevantRank).toBe(2);
97 });
98
99 test('never marks a no-answer result as relevant', () => {
100 const report = evaluateKbSearchRankings([{
101 id: 'no-answer-with-noise',
102 kind: 'no-answer',
103 query: 'best pizza tonight',
104 expectedUrls: [],
105 }], new Map([
106 ['no-answer-with-noise', [{ canonicalUrl: '/docs', sourceType: 'docs' }]],
107 ]));
108
109 expect(report.cases[0]).toMatchObject({ firstRelevantRank: null, hitAt5: false });
110 });
111 });