Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { describe, expect, test } from 'bun:test'; |
| 2 | import { readFileSync } from 'node:fs'; |
| 3 | import { join } from 'node:path'; |
| 4 | import { |
| 5 | buildProductCatalog, |
| 6 | extractExternalLinks, |
| 7 | extractToolSlugs, |
| 8 | extractToolkitLinks, |
| 9 | isComposioToolSlug, |
| 10 | renderVerifyReport, |
| 11 | verifyKb, |
| 12 | type CatalogToolkit, |
| 13 | } from '@/lib/kb/verify'; |
| 14 | import { getKbCatalog, readKbManifest } from '@/lib/kb/repository'; |
| 15 | import type { KbGuide, KbManifest } from '@/lib/kb/types'; |
| 16 | |
| 17 | const TOOLKITS: CatalogToolkit[] = [ |
| 18 | { |
| 19 | slug: 'canvas', |
| 20 | authSchemes: ['OAUTH2', 'BEARER_TOKEN'], |
| 21 | tools: [{ slug: 'CANVAS_GET_SINGLE_ACCOUNT' }, { slug: 'CANVAS_LIST_MANAGEABLE_ACCOUNTS' }], |
| 22 | triggers: [], |
| 23 | }, |
| 24 | { |
| 25 | slug: 'composio', |
| 26 | authSchemes: [], |
| 27 | tools: [{ slug: 'COMPOSIO_SEARCH_TOOLS' }], |
| 28 | triggers: [], |
| 29 | }, |
| 30 | { |
| 31 | slug: 'gmail', |
| 32 | authSchemes: ['OAUTH2'], |
| 33 | tools: [{ slug: 'GMAIL_CREATE_FILTER' }], |
| 34 | triggers: [{ slug: 'GMAIL_EMAIL_SENT_TRIGGER' }], |
| 35 | }, |
| 36 | ]; |
| 37 | |
| 38 | const catalog = buildProductCatalog(TOOLKITS); |
| 39 | |
| 40 | const MANIFEST: KbManifest = { |
| 41 | schemaVersion: 2, |
| 42 | topics: [], |
| 43 | guides: [], |
| 44 | }; |
| 45 | |
| 46 | function guide(overrides: Partial<KbGuide> & { slug: string; body: string }): KbGuide { |
| 47 | return { |
| 48 | title: overrides.slug, |
| 49 | description: '', |
| 50 | sources: [], |
| 51 | topics: [], |
| 52 | tags: [], |
| 53 | aliases: [], |
| 54 | relatedGuides: [], |
| 55 | externalResources: [], |
| 56 | updatedAt: '2026-07-22', |
| 57 | lastVerifiedAt: '2026-07-22', |
| 58 | reviewAfter: '2027-01-01', |
| 59 | freshness: 'evergreen', |
| 60 | state: 'published', |
| 61 | featured: false, |
| 62 | sourceMetadata: [], |
| 63 | ...overrides, |
| 64 | } as unknown as KbGuide; |
| 65 | } |
| 66 | |
| 67 | describe('tool slug detection', () => { |
| 68 | test('recognizes tokens prefixed by a known toolkit slug', () => { |
| 69 | expect(isComposioToolSlug('CANVAS_GET_SINGLE_ACCOUNT', catalog)).toBe(true); |
| 70 | expect(isComposioToolSlug('GMAIL_CREATE_FILTER', catalog)).toBe(true); |
| 71 | }); |
| 72 | |
| 73 | test('ignores provider error codes that merely share the shape', () => { |
| 74 | expect(isComposioToolSlug('NONEXISTENT_VERSION', catalog)).toBe(false); |
| 75 | expect(isComposioToolSlug('RESOURCE_EXHAUSTED', catalog)).toBe(false); |
| 76 | }); |
| 77 | |
| 78 | test('skips substitution placeholders in code samples', () => { |
| 79 | const text = "curl --header 'x-api-key: <COMPOSIO_API_KEY>' and $COMPOSIO_BASE_URL"; |
| 80 | expect(extractToolSlugs(text, catalog)).toEqual([]); |
| 81 | }); |
| 82 | |
| 83 | test('still checks a token that also appears outside a placeholder', () => { |
| 84 | const text = 'Call COMPOSIO_SEARCH_TOOLS first. Example: <COMPOSIO_SEARCH_TOOLS>'; |
| 85 | expect(extractToolSlugs(text, catalog)).toEqual(['COMPOSIO_SEARCH_TOOLS']); |
| 86 | }); |
| 87 | |
| 88 | test('detects trigger slugs alongside tool slugs', () => { |
| 89 | expect(catalog.toolSlugs.has('GMAIL_EMAIL_SENT_TRIGGER')).toBe(true); |
| 90 | }); |
| 91 | }); |
| 92 | |
| 93 | describe('link extraction', () => { |
| 94 | test('collects markdown link targets only', () => { |
| 95 | const text = |
| 96 | 'See the [pagination guide](https://example.com/docs/pagination). ' + |
| 97 | 'Calls must use `https://api.example.com/v3`, not `https://example.com/v3`.'; |
| 98 | expect(extractExternalLinks(text)).toEqual(['https://example.com/docs/pagination']); |
| 99 | }); |
| 100 | |
| 101 | test('collects toolkit links', () => { |
| 102 | expect(extractToolkitLinks('the [Canvas toolkit](/toolkits/canvas) page')).toEqual(['canvas']); |
| 103 | }); |
| 104 | }); |
| 105 | |
| 106 | describe('verifyKb', () => { |
| 107 | const now = new Date('2026-07-24T00:00:00Z'); |
| 108 | |
| 109 | test('reports a cited tool slug that is absent from the catalog', () => { |
| 110 | const report = verifyKb({ |
| 111 | manifest: MANIFEST, |
| 112 | catalog, |
| 113 | now, |
| 114 | guides: [guide({ slug: 'canvas-accounts', body: 'Use `CANVAS_GET_ACCOUNTS` for this.' })], |
| 115 | }); |
| 116 | const finding = report.findings.find((entry) => entry.kind === 'unknown-tool-slug'); |
| 117 | expect(finding?.severity).toBe('error'); |
| 118 | expect(finding?.detail).toContain('CANVAS_GET_ACCOUNTS'); |
| 119 | }); |
| 120 | |
| 121 | test('accepts a cited slug that exists', () => { |
| 122 | const report = verifyKb({ |
| 123 | manifest: MANIFEST, |
| 124 | catalog, |
| 125 | now, |
| 126 | guides: [guide({ slug: 'ok', body: 'Use `CANVAS_GET_SINGLE_ACCOUNT`.' })], |
| 127 | }); |
| 128 | expect(report.findings.filter((entry) => entry.kind === 'unknown-tool-slug')).toEqual([]); |
| 129 | expect(report.checkedToolSlugs).toBe(1); |
| 130 | }); |
| 131 | |
| 132 | test('reports a link to a toolkit that left the catalog', () => { |
| 133 | const report = verifyKb({ |
| 134 | manifest: MANIFEST, |
| 135 | catalog, |
| 136 | now, |
| 137 | guides: [guide({ slug: 'gone', body: 'see [X](/toolkits/retired_toolkit)' })], |
| 138 | }); |
| 139 | expect(report.findings.some((entry) => entry.kind === 'unknown-toolkit-link')).toBe(true); |
| 140 | }); |
| 141 | |
| 142 | test('ignores unpublished guides', () => { |
| 143 | const report = verifyKb({ |
| 144 | manifest: MANIFEST, |
| 145 | catalog, |
| 146 | now, |
| 147 | guides: [ |
| 148 | guide({ slug: 'held', body: 'Use `CANVAS_GET_ACCOUNTS`.', state: 'needs-review' } as never), |
| 149 | ], |
| 150 | }); |
| 151 | expect(report.findings).toEqual([]); |
| 152 | expect(report.checkedGuides).toBe(0); |
| 153 | }); |
| 154 | |
| 155 | test('honors a manifest-declared exemption for deliberately dead slugs', () => { |
| 156 | const report = verifyKb({ |
| 157 | manifest: MANIFEST, |
| 158 | catalog, |
| 159 | now, |
| 160 | guides: [ |
| 161 | guide({ |
| 162 | slug: 'deprecated-note', |
| 163 | body: '`CANVAS_GET_ACCOUNTS` was removed.', |
| 164 | verifyIgnoreToolSlugs: ['CANVAS_GET_ACCOUNTS'], |
| 165 | } as never), |
| 166 | ], |
| 167 | }); |
| 168 | expect(report.findings).toEqual([]); |
| 169 | }); |
| 170 | |
| 171 | test('flags explicit internal handoff instructions', () => { |
| 172 | const report = verifyKb({ |
| 173 | manifest: MANIFEST, |
| 174 | catalog, |
| 175 | now, |
| 176 | guides: [ |
| 177 | guide({ |
| 178 | slug: 'internal-handoff', |
| 179 | body: 'Collect the request details, then route the case to a human for review.', |
| 180 | }), |
| 181 | ], |
| 182 | }); |
| 183 | const finding = report.findings.find((entry) => entry.kind === 'unedited-support-prose'); |
| 184 | expect(finding?.evidence).toContain('route the case to a human'); |
| 185 | }); |
| 186 | |
| 187 | test('reports an expired review window instead of throwing', () => { |
| 188 | const report = verifyKb({ |
| 189 | manifest: MANIFEST, |
| 190 | catalog, |
| 191 | now, |
| 192 | guides: [guide({ slug: 'expired', body: 'text', reviewAfter: '2026-01-01' })], |
| 193 | }); |
| 194 | const finding = report.findings.find((entry) => entry.kind === 'review-window-expired'); |
| 195 | expect(finding?.severity).toBe('error'); |
| 196 | }); |
| 197 | |
| 198 | describe('review window boundary', () => { |
| 199 | const expiryFor = (at: string) => |
| 200 | verifyKb({ |
| 201 | manifest: MANIFEST, |
| 202 | catalog, |
| 203 | now: new Date(at), |
| 204 | guides: [guide({ slug: 'due', body: 'text', reviewAfter: '2026-11-10' })], |
| 205 | }).findings.find((entry) => entry.kind === 'review-window-expired'); |
| 206 | |
| 207 | test.each(['2026-11-09T12:25:00Z', '2026-11-09T15:05:00Z', '2026-11-09T23:59:59.999Z'])( |
| 208 | 'stays open before the deadline (%s)', |
| 209 | (at) => { |
| 210 | expect(expiryFor(at)).toBeUndefined(); |
| 211 | } |
| 212 | ); |
| 213 | |
| 214 | test('expires at the deadline', () => { |
| 215 | expect(expiryFor('2026-11-10T00:00:00Z')?.detail).toBe('review window expired 0 day(s) ago'); |
| 216 | }); |
| 217 | |
| 218 | test('counts whole days elapsed since the deadline', () => { |
| 219 | expect(expiryFor('2026-11-11T12:00:00Z')?.detail).toBe('review window expired 1 day(s) ago'); |
| 220 | }); |
| 221 | }); |
| 222 | |
| 223 | test('flags a cohort of guides that all expire on one date', () => { |
| 224 | const guides = Array.from({ length: 6 }, (_, index) => |
| 225 | guide({ slug: `guide-${index}`, body: 'text', reviewAfter: '2026-10-20' }) |
| 226 | ); |
| 227 | const report = verifyKb({ manifest: MANIFEST, catalog, now, guides }); |
| 228 | const finding = report.findings.find((entry) => entry.kind === 'review-window-cliff'); |
| 229 | expect(finding?.detail).toContain('6 published guides'); |
| 230 | }); |
| 231 | |
| 232 | test('warns when verification has aged past the threshold', () => { |
| 233 | const report = verifyKb({ |
| 234 | manifest: MANIFEST, |
| 235 | catalog, |
| 236 | now, |
| 237 | guides: [guide({ slug: 'stale', body: 'text', lastVerifiedAt: '2026-01-01' })], |
| 238 | options: { staleVerificationDays: 30 }, |
| 239 | }); |
| 240 | expect(report.findings.some((entry) => entry.kind === 'verification-stale')).toBe(true); |
| 241 | }); |
| 242 | |
| 243 | test('treats provider rate-limiting as inconclusive, not as rot', () => { |
| 244 | const body = 'see [docs](https://example.com/a) and [more](https://example.com/b)'; |
| 245 | const report = verifyKb({ |
| 246 | manifest: MANIFEST, |
| 247 | catalog, |
| 248 | now, |
| 249 | guides: [guide({ slug: 'links', body })], |
| 250 | options: { |
| 251 | linkStatuses: new Map<string, number | 'unreachable'>([ |
| 252 | ['https://example.com/a', 503], |
| 253 | ['https://example.com/b', 404], |
| 254 | ]), |
| 255 | }, |
| 256 | }); |
| 257 | const linkFindings = report.findings.filter((entry) => entry.kind === 'unreachable-link'); |
| 258 | expect(linkFindings).toHaveLength(1); |
| 259 | expect(linkFindings[0]?.detail).toContain('https://example.com/b'); |
| 260 | }); |
| 261 | }); |
| 262 | |
| 263 | describe('report rendering', () => { |
| 264 | test('states success plainly when nothing is wrong', () => { |
| 265 | const rendered = renderVerifyReport({ |
| 266 | checkedGuides: 3, |
| 267 | checkedToolSlugs: 5, |
| 268 | checkedToolkitLinks: 2, |
| 269 | externalLinks: [], |
| 270 | findings: [], |
| 271 | }); |
| 272 | expect(rendered).toContain('All mechanical claims verified'); |
| 273 | }); |
| 274 | }); |
| 275 | |
| 276 | describe('live corpus', () => { |
| 277 | /** |
| 278 | * The first publish tranche predated article bodies and was edited in place |
| 279 | * inside `kb/source`, which silently made the snapshot a derivative of the |
| 280 | * commit it claims to come from. Nine of twenty-five sources had drifted. |
| 281 | * Published prose belongs in `kb/articles`; `kb/source` must stay verbatim. |
| 282 | */ |
| 283 | test('no published guide renders from the source snapshot', () => { |
| 284 | const manifest = readKbManifest(); |
| 285 | const rendersFromSource = manifest.guides |
| 286 | .filter((guide) => guide.state === 'published' && !guide.articlePath) |
| 287 | .map((guide) => guide.slug); |
| 288 | expect(rendersFromSource).toEqual([]); |
| 289 | }); |
| 290 | |
| 291 | test('contains no internal support prose or unexplained dead tool slugs', () => { |
| 292 | const toolkitData = JSON.parse( |
| 293 | readFileSync(join(process.cwd(), 'public/data/toolkits.json'), 'utf8') |
| 294 | ) as CatalogToolkit[]; |
| 295 | const kbCatalog = getKbCatalog(); |
| 296 | const report = verifyKb({ |
| 297 | manifest: kbCatalog.manifest, |
| 298 | guides: kbCatalog.guides, |
| 299 | catalog: buildProductCatalog(toolkitData), |
| 300 | now: new Date('2026-08-17T00:00:00Z'), |
| 301 | }); |
| 302 | const publicationFindings = report.findings.filter( |
| 303 | (finding) => |
| 304 | finding.kind === 'unedited-support-prose' || finding.kind === 'unknown-tool-slug' |
| 305 | ); |
| 306 | expect(publicationFindings).toEqual([]); |
| 307 | }); |
| 308 | }); |