Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 308 lines 10.1 KB
Raw
1 import { describe, expect, test } from 'bun:test';
2 import { readFileSync } from 'node:fs';
3 import { join } from 'node:path';
4 import {
5 buildProductCatalog,
6 extractExternalLinks,
7 extractToolSlugs,
8 extractToolkitLinks,
9 isComposioToolSlug,
10 renderVerifyReport,
11 verifyKb,
12 type CatalogToolkit,
13 } from '@/lib/kb/verify';
14 import { getKbCatalog, readKbManifest } from '@/lib/kb/repository';
15 import type { KbGuide, KbManifest } from '@/lib/kb/types';
16
17 const TOOLKITS: CatalogToolkit[] = [
18 {
19 slug: 'canvas',
20 authSchemes: ['OAUTH2', 'BEARER_TOKEN'],
21 tools: [{ slug: 'CANVAS_GET_SINGLE_ACCOUNT' }, { slug: 'CANVAS_LIST_MANAGEABLE_ACCOUNTS' }],
22 triggers: [],
23 },
24 {
25 slug: 'composio',
26 authSchemes: [],
27 tools: [{ slug: 'COMPOSIO_SEARCH_TOOLS' }],
28 triggers: [],
29 },
30 {
31 slug: 'gmail',
32 authSchemes: ['OAUTH2'],
33 tools: [{ slug: 'GMAIL_CREATE_FILTER' }],
34 triggers: [{ slug: 'GMAIL_EMAIL_SENT_TRIGGER' }],
35 },
36 ];
37
38 const catalog = buildProductCatalog(TOOLKITS);
39
40 const MANIFEST: KbManifest = {
41 schemaVersion: 2,
42 topics: [],
43 guides: [],
44 };
45
46 function guide(overrides: Partial<KbGuide> & { slug: string; body: string }): KbGuide {
47 return {
48 title: overrides.slug,
49 description: '',
50 sources: [],
51 topics: [],
52 tags: [],
53 aliases: [],
54 relatedGuides: [],
55 externalResources: [],
56 updatedAt: '2026-07-22',
57 lastVerifiedAt: '2026-07-22',
58 reviewAfter: '2027-01-01',
59 freshness: 'evergreen',
60 state: 'published',
61 featured: false,
62 sourceMetadata: [],
63 ...overrides,
64 } as unknown as KbGuide;
65 }
66
67 describe('tool slug detection', () => {
68 test('recognizes tokens prefixed by a known toolkit slug', () => {
69 expect(isComposioToolSlug('CANVAS_GET_SINGLE_ACCOUNT', catalog)).toBe(true);
70 expect(isComposioToolSlug('GMAIL_CREATE_FILTER', catalog)).toBe(true);
71 });
72
73 test('ignores provider error codes that merely share the shape', () => {
74 expect(isComposioToolSlug('NONEXISTENT_VERSION', catalog)).toBe(false);
75 expect(isComposioToolSlug('RESOURCE_EXHAUSTED', catalog)).toBe(false);
76 });
77
78 test('skips substitution placeholders in code samples', () => {
79 const text = "curl --header 'x-api-key: <COMPOSIO_API_KEY>' and $COMPOSIO_BASE_URL";
80 expect(extractToolSlugs(text, catalog)).toEqual([]);
81 });
82
83 test('still checks a token that also appears outside a placeholder', () => {
84 const text = 'Call COMPOSIO_SEARCH_TOOLS first. Example: <COMPOSIO_SEARCH_TOOLS>';
85 expect(extractToolSlugs(text, catalog)).toEqual(['COMPOSIO_SEARCH_TOOLS']);
86 });
87
88 test('detects trigger slugs alongside tool slugs', () => {
89 expect(catalog.toolSlugs.has('GMAIL_EMAIL_SENT_TRIGGER')).toBe(true);
90 });
91 });
92
93 describe('link extraction', () => {
94 test('collects markdown link targets only', () => {
95 const text =
96 'See the [pagination guide](https://example.com/docs/pagination). ' +
97 'Calls must use `https://api.example.com/v3`, not `https://example.com/v3`.';
98 expect(extractExternalLinks(text)).toEqual(['https://example.com/docs/pagination']);
99 });
100
101 test('collects toolkit links', () => {
102 expect(extractToolkitLinks('the [Canvas toolkit](/toolkits/canvas) page')).toEqual(['canvas']);
103 });
104 });
105
106 describe('verifyKb', () => {
107 const now = new Date('2026-07-24T00:00:00Z');
108
109 test('reports a cited tool slug that is absent from the catalog', () => {
110 const report = verifyKb({
111 manifest: MANIFEST,
112 catalog,
113 now,
114 guides: [guide({ slug: 'canvas-accounts', body: 'Use `CANVAS_GET_ACCOUNTS` for this.' })],
115 });
116 const finding = report.findings.find((entry) => entry.kind === 'unknown-tool-slug');
117 expect(finding?.severity).toBe('error');
118 expect(finding?.detail).toContain('CANVAS_GET_ACCOUNTS');
119 });
120
121 test('accepts a cited slug that exists', () => {
122 const report = verifyKb({
123 manifest: MANIFEST,
124 catalog,
125 now,
126 guides: [guide({ slug: 'ok', body: 'Use `CANVAS_GET_SINGLE_ACCOUNT`.' })],
127 });
128 expect(report.findings.filter((entry) => entry.kind === 'unknown-tool-slug')).toEqual([]);
129 expect(report.checkedToolSlugs).toBe(1);
130 });
131
132 test('reports a link to a toolkit that left the catalog', () => {
133 const report = verifyKb({
134 manifest: MANIFEST,
135 catalog,
136 now,
137 guides: [guide({ slug: 'gone', body: 'see [X](/toolkits/retired_toolkit)' })],
138 });
139 expect(report.findings.some((entry) => entry.kind === 'unknown-toolkit-link')).toBe(true);
140 });
141
142 test('ignores unpublished guides', () => {
143 const report = verifyKb({
144 manifest: MANIFEST,
145 catalog,
146 now,
147 guides: [
148 guide({ slug: 'held', body: 'Use `CANVAS_GET_ACCOUNTS`.', state: 'needs-review' } as never),
149 ],
150 });
151 expect(report.findings).toEqual([]);
152 expect(report.checkedGuides).toBe(0);
153 });
154
155 test('honors a manifest-declared exemption for deliberately dead slugs', () => {
156 const report = verifyKb({
157 manifest: MANIFEST,
158 catalog,
159 now,
160 guides: [
161 guide({
162 slug: 'deprecated-note',
163 body: '`CANVAS_GET_ACCOUNTS` was removed.',
164 verifyIgnoreToolSlugs: ['CANVAS_GET_ACCOUNTS'],
165 } as never),
166 ],
167 });
168 expect(report.findings).toEqual([]);
169 });
170
171 test('flags explicit internal handoff instructions', () => {
172 const report = verifyKb({
173 manifest: MANIFEST,
174 catalog,
175 now,
176 guides: [
177 guide({
178 slug: 'internal-handoff',
179 body: 'Collect the request details, then route the case to a human for review.',
180 }),
181 ],
182 });
183 const finding = report.findings.find((entry) => entry.kind === 'unedited-support-prose');
184 expect(finding?.evidence).toContain('route the case to a human');
185 });
186
187 test('reports an expired review window instead of throwing', () => {
188 const report = verifyKb({
189 manifest: MANIFEST,
190 catalog,
191 now,
192 guides: [guide({ slug: 'expired', body: 'text', reviewAfter: '2026-01-01' })],
193 });
194 const finding = report.findings.find((entry) => entry.kind === 'review-window-expired');
195 expect(finding?.severity).toBe('error');
196 });
197
198 describe('review window boundary', () => {
199 const expiryFor = (at: string) =>
200 verifyKb({
201 manifest: MANIFEST,
202 catalog,
203 now: new Date(at),
204 guides: [guide({ slug: 'due', body: 'text', reviewAfter: '2026-11-10' })],
205 }).findings.find((entry) => entry.kind === 'review-window-expired');
206
207 test.each(['2026-11-09T12:25:00Z', '2026-11-09T15:05:00Z', '2026-11-09T23:59:59.999Z'])(
208 'stays open before the deadline (%s)',
209 (at) => {
210 expect(expiryFor(at)).toBeUndefined();
211 }
212 );
213
214 test('expires at the deadline', () => {
215 expect(expiryFor('2026-11-10T00:00:00Z')?.detail).toBe('review window expired 0 day(s) ago');
216 });
217
218 test('counts whole days elapsed since the deadline', () => {
219 expect(expiryFor('2026-11-11T12:00:00Z')?.detail).toBe('review window expired 1 day(s) ago');
220 });
221 });
222
223 test('flags a cohort of guides that all expire on one date', () => {
224 const guides = Array.from({ length: 6 }, (_, index) =>
225 guide({ slug: `guide-${index}`, body: 'text', reviewAfter: '2026-10-20' })
226 );
227 const report = verifyKb({ manifest: MANIFEST, catalog, now, guides });
228 const finding = report.findings.find((entry) => entry.kind === 'review-window-cliff');
229 expect(finding?.detail).toContain('6 published guides');
230 });
231
232 test('warns when verification has aged past the threshold', () => {
233 const report = verifyKb({
234 manifest: MANIFEST,
235 catalog,
236 now,
237 guides: [guide({ slug: 'stale', body: 'text', lastVerifiedAt: '2026-01-01' })],
238 options: { staleVerificationDays: 30 },
239 });
240 expect(report.findings.some((entry) => entry.kind === 'verification-stale')).toBe(true);
241 });
242
243 test('treats provider rate-limiting as inconclusive, not as rot', () => {
244 const body = 'see [docs](https://example.com/a) and [more](https://example.com/b)';
245 const report = verifyKb({
246 manifest: MANIFEST,
247 catalog,
248 now,
249 guides: [guide({ slug: 'links', body })],
250 options: {
251 linkStatuses: new Map<string, number | 'unreachable'>([
252 ['https://example.com/a', 503],
253 ['https://example.com/b', 404],
254 ]),
255 },
256 });
257 const linkFindings = report.findings.filter((entry) => entry.kind === 'unreachable-link');
258 expect(linkFindings).toHaveLength(1);
259 expect(linkFindings[0]?.detail).toContain('https://example.com/b');
260 });
261 });
262
263 describe('report rendering', () => {
264 test('states success plainly when nothing is wrong', () => {
265 const rendered = renderVerifyReport({
266 checkedGuides: 3,
267 checkedToolSlugs: 5,
268 checkedToolkitLinks: 2,
269 externalLinks: [],
270 findings: [],
271 });
272 expect(rendered).toContain('All mechanical claims verified');
273 });
274 });
275
276 describe('live corpus', () => {
277 /**
278 * The first publish tranche predated article bodies and was edited in place
279 * inside `kb/source`, which silently made the snapshot a derivative of the
280 * commit it claims to come from. Nine of twenty-five sources had drifted.
281 * Published prose belongs in `kb/articles`; `kb/source` must stay verbatim.
282 */
283 test('no published guide renders from the source snapshot', () => {
284 const manifest = readKbManifest();
285 const rendersFromSource = manifest.guides
286 .filter((guide) => guide.state === 'published' && !guide.articlePath)
287 .map((guide) => guide.slug);
288 expect(rendersFromSource).toEqual([]);
289 });
290
291 test('contains no internal support prose or unexplained dead tool slugs', () => {
292 const toolkitData = JSON.parse(
293 readFileSync(join(process.cwd(), 'public/data/toolkits.json'), 'utf8')
294 ) as CatalogToolkit[];
295 const kbCatalog = getKbCatalog();
296 const report = verifyKb({
297 manifest: kbCatalog.manifest,
298 guides: kbCatalog.guides,
299 catalog: buildProductCatalog(toolkitData),
300 now: new Date('2026-08-17T00:00:00Z'),
301 });
302 const publicationFindings = report.findings.filter(
303 (finding) =>
304 finding.kind === 'unedited-support-prose' || finding.kind === 'unknown-tool-slug'
305 );
306 expect(publicationFindings).toEqual([]);
307 });
308 });