Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { beforeAll, expect, test } from 'bun:test'; |
| 2 | import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; |
| 3 | import { tmpdir } from 'node:os'; |
| 4 | import { join } from 'node:path'; |
| 5 | import type { AdvancedIndex } from 'fumadocs-core/search/server'; |
| 6 | import { buildCompleteSearchReplacement } from '../../lib/knowledge/search-replacement'; |
| 7 | import { getDocsSearchIndexes, type AlgoliaDocsRecord } from '../../lib/search-index'; |
| 8 | import { assertPageFacts, corpusPages, parityPages } from '../fixtures/semantic-parity'; |
| 9 | |
| 10 | let records: AlgoliaDocsRecord[]; |
| 11 | let localIndexes: AdvancedIndex[]; |
| 12 | beforeAll(async () => { |
| 13 | // Exercise both production search ingestion and the /api/search fallback corpus. |
| 14 | [records, localIndexes] = await Promise.all([ |
| 15 | buildCompleteSearchReplacement({ validateExternal: false }), |
| 16 | getDocsSearchIndexes(), |
| 17 | ]); |
| 18 | }); |
| 19 | |
| 20 | for (const page of parityPages) { |
| 21 | const surfaces = [ |
| 22 | { |
| 23 | name: 'Algolia ingestion', |
| 24 | text: () => records.filter(record => record.page_id === page.url) |
| 25 | .sort((a, b) => a.position - b.position) |
| 26 | .map(record => record.content).join('\n\n'), |
| 27 | }, |
| 28 | { |
| 29 | name: 'local search corpus', |
| 30 | text: () => localIndexes.find(index => index.url === page.url) |
| 31 | ?.structuredData.contents.map(item => item.content).join('\n\n') ?? '', |
| 32 | }, |
| 33 | ]; |
| 34 | for (const surface of surfaces) { |
| 35 | test(`${surface.name} semantic parity ${page.url}`, () => { |
| 36 | assertPageFacts(surface.text(), page, surface.name); |
| 37 | }); |
| 38 | for (const [id, expected] of page.facts) { |
| 39 | test(`mutation catches ${surface.name} ${page.url} lost fact ${id}`, () => { |
| 40 | const text = surface.text(); |
| 41 | assertPageFacts(text, page, surface.name); |
| 42 | const mutated = text.replace(/\s+/g, ' ') |
| 43 | .split(expected.replace(/\s+/g, ' ')).join(''); |
| 44 | expect(() => assertPageFacts(mutated, page, surface.name)).toThrow( |
| 45 | `${surface.name} ${page.url} lost fact ${id}`, |
| 46 | ); |
| 47 | }); |
| 48 | } |
| 49 | } |
| 50 | } |
| 51 | |
| 52 | test('legacy REST records remain classified as legacy in the search payload', () => { |
| 53 | const legacy = records.filter(record => record.page_id === '/reference/v3/api-reference/tools'); |
| 54 | expect(legacy.length).toBeGreaterThan(0); |
| 55 | expect(legacy.every(record => record.source_type === 'legacy')).toBe(true); |
| 56 | }); |
| 57 | |
| 58 | test('facts on an adjacent corpus page cannot mask a missing fact', () => { |
| 59 | const page = parityPages[1]; |
| 60 | const pages = corpusPages(`# Quickstart (${page.url})\nMissing install.\n\n---\n\n# Other (/docs/other)\n${page.facts.map(([, text]) => text).join('\n')}`); |
| 61 | expect(() => assertPageFacts(pages.get(page.url)!, page, 'llms-full.txt')).toThrow( |
| 62 | 'llms-full.txt /docs/quickstart lost fact typescript-install', |
| 63 | ); |
| 64 | }); |
| 65 | |
| 66 | test('external checker preserves ordered line chunks and names missing facts and pages', () => { |
| 67 | const directory = mkdtempSync(join(tmpdir(), 'docs-parity-')); |
| 68 | const path = join(directory, 'snapshot.json'); |
| 69 | const snapshot = { |
| 70 | surface: 'test external export', capturedAt: '2026-10-14T00:00:00Z', includesLegacy: true, |
| 71 | pages: parityPages.flatMap(page => records |
| 72 | .filter(record => record.page_id === page.url) |
| 73 | .sort((a, b) => a.position - b.position) |
| 74 | // Regression for #4443: split multi-line facts between adjacent chunks. |
| 75 | .flatMap(record => record.content.split('\n').map(content => ({ |
| 76 | url: `https://docs.composio.dev${page.url}.md#chunk`, content, |
| 77 | })))), |
| 78 | }; |
| 79 | const run = (input: unknown) => { |
| 80 | writeFileSync(path, JSON.stringify(input)); |
| 81 | return Bun.spawnSync([process.execPath, 'scripts/check-docs-parity.ts', path]); |
| 82 | }; |
| 83 | try { |
| 84 | const complete = run(snapshot); |
| 85 | expect(complete.exitCode, complete.stderr.toString()).toBe(0); |
| 86 | const missingFact = run({ ...snapshot, pages: snapshot.pages.map(page => ({ |
| 87 | ...page, |
| 88 | content: page.content.replaceAll('composio login', ''), |
| 89 | })) }); |
| 90 | expect(missingFact.exitCode).not.toBe(0); |
| 91 | const diagnostic = 'test external export /docs/agent-plugins lost fact login-and-setup'; |
| 92 | expect(missingFact.stderr.toString()).toContain(diagnostic); |
| 93 | console.info(`Expected mutation caught (checker exit ${missingFact.exitCode}): ${diagnostic}`); |
| 94 | const missingPage = run({ ...snapshot, pages: snapshot.pages.filter(page => !page.url.includes('/quickstart')) }); |
| 95 | expect(missingPage.exitCode).not.toBe(0); |
| 96 | expect(missingPage.stderr.toString()).toContain('test external export /docs/quickstart lost fact typescript-install'); |
| 97 | const currentOnly = run({ ...snapshot, includesLegacy: false, |
| 98 | pages: snapshot.pages.filter(page => !page.url.includes('/reference/v3/')) }); |
| 99 | expect(currentOnly.exitCode, currentOnly.stderr.toString()).toBe(0); |
| 100 | expect(run({ ...snapshot, pages: [{ url: '/docs', content: 42 }] }).exitCode).not.toBe(0); |
| 101 | expect(run({ ...snapshot, pages: [{ url: 'https://example.com/docs', content: '' }] }).exitCode).not.toBe(0); |
| 102 | } finally { |
| 103 | rmSync(directory, { recursive: true, force: true }); |
| 104 | } |
| 105 | }); |