Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { |
| 2 | buildBundledIndex, |
| 3 | buildIndex, |
| 4 | extractSections, |
| 5 | getBundleSearchIndex, |
| 6 | pageSearchKey, |
| 7 | readPageByUrl, |
| 8 | toCleanMarkdown, |
| 9 | tokenize, |
| 10 | type DocPage, |
| 11 | } from './docs'; |
| 12 | |
| 13 | /** |
| 14 | * search_docs — find the most relevant Composio docs pages for a query. |
| 15 | * |
| 16 | * This is a local, in-memory lexical retriever. It uses a BM25-style body score |
| 17 | * plus field boosts for title, description, headings, and URL. The top results |
| 18 | * include full page content (bounded per page), so the model gets rich context |
| 19 | * in the same fast tool call instead of doing a serial search -> read round trip. |
| 20 | */ |
| 21 | |
| 22 | // Collection priority: docs first, then examples, then references and toolkits. |
| 23 | // (Knowledge base guides rank with docs.) A toolkit-name query still surfaces its |
| 24 | // toolkit page because nothing else matches it. |
| 25 | const PRIORITY: Record<DocPage['collection'], number> = { |
| 26 | docs: 1.3, |
| 27 | kb: 1.3, |
| 28 | examples: 1.1, |
| 29 | reference: 0.85, |
| 30 | toolkits: 0.9, |
| 31 | }; |
| 32 | |
| 33 | export const DEFAULT_SEARCH_LIMIT = 5; |
| 34 | const DEFAULT_LIMIT = DEFAULT_SEARCH_LIMIT; |
| 35 | export const DEFAULT_CONTENT_RESULT_COUNT = 4; |
| 36 | export const DEFAULT_MAX_CONTENT_CHARS = 10_000; |
| 37 | export const DEFAULT_MAX_SECTIONS = 16; |
| 38 | const BM25_K1 = 1.2; |
| 39 | const BM25_B = 0.75; |
| 40 | const PERF_LOG_ENABLED = process.env.DOCS_AGENT_SEARCH_PERF_LOG === '1'; |
| 41 | const PERF_LOG_QUERY = process.env.DOCS_AGENT_SEARCH_LOG_QUERY === '1'; |
| 42 | |
| 43 | type CorpusEntry = { |
| 44 | page: DocPage; |
| 45 | termCounts: Map<string, number>; |
| 46 | length: number; |
| 47 | }; |
| 48 | |
| 49 | type Corpus = { |
| 50 | entries: CorpusEntry[]; |
| 51 | documentFrequency: Map<string, number>; |
| 52 | averageLength: number; |
| 53 | }; |
| 54 | |
| 55 | type CorpusSource = 'precomputed' | 'runtime'; |
| 56 | |
| 57 | type PrecomputedCorpusResult = |
| 58 | | { corpus: Corpus; fallbackReason?: never } |
| 59 | | { corpus?: never; fallbackReason: string }; |
| 60 | |
| 61 | type CorpusLoad = { |
| 62 | corpus: Corpus; |
| 63 | source: CorpusSource; |
| 64 | cached: boolean; |
| 65 | loadMs: number; |
| 66 | fallbackReason?: string; |
| 67 | }; |
| 68 | |
| 69 | let corpusCache: Corpus | undefined; |
| 70 | let corpusCacheSource: CorpusSource | undefined; |
| 71 | let corpusCacheFallbackReason: string | undefined; |
| 72 | |
| 73 | function roundMs(ms: number): number { |
| 74 | return Math.round(ms * 100) / 100; |
| 75 | } |
| 76 | |
| 77 | function logPerf(payload: Record<string, unknown>) { |
| 78 | if (!PERF_LOG_ENABLED) return; |
| 79 | console.info(`[docs-agent:search_docs] ${JSON.stringify(payload)}`); |
| 80 | } |
| 81 | |
| 82 | function termCountsFor(page: DocPage): Map<string, number> { |
| 83 | const counts = new Map<string, number>(); |
| 84 | |
| 85 | for (const token of tokenize(page.lowerText)) { |
| 86 | counts.set(token, (counts.get(token) ?? 0) + 1); |
| 87 | } |
| 88 | |
| 89 | return counts; |
| 90 | } |
| 91 | |
| 92 | function buildRuntimeCorpus(pages: DocPage[]): Corpus { |
| 93 | const entries = pages.map(page => { |
| 94 | const termCounts = termCountsFor(page); |
| 95 | return { |
| 96 | page, |
| 97 | termCounts, |
| 98 | length: [...termCounts.values()].reduce((sum, count) => sum + count, 0), |
| 99 | }; |
| 100 | }); |
| 101 | const documentFrequency = new Map<string, number>(); |
| 102 | |
| 103 | for (const entry of entries) { |
| 104 | for (const term of entry.termCounts.keys()) { |
| 105 | documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1); |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | return { |
| 110 | entries, |
| 111 | documentFrequency, |
| 112 | averageLength: |
| 113 | entries.reduce((sum, entry) => sum + entry.length, 0) / Math.max(entries.length, 1), |
| 114 | }; |
| 115 | } |
| 116 | |
| 117 | function buildPrecomputedCorpus(pages: DocPage[]): PrecomputedCorpusResult { |
| 118 | const search = getBundleSearchIndex(); |
| 119 | if (!search) return { fallbackReason: 'missing-precomputed-index' }; |
| 120 | if (search.entries.length !== pages.length) { |
| 121 | return { fallbackReason: `entry-count-mismatch:${search.entries.length}:${pages.length}` }; |
| 122 | } |
| 123 | |
| 124 | const entries: CorpusEntry[] = []; |
| 125 | |
| 126 | for (let index = 0; index < pages.length; index++) { |
| 127 | const page = pages[index]; |
| 128 | const entry = search.entries[index]; |
| 129 | |
| 130 | if (!entry) return { fallbackReason: `missing-entry:${index}` }; |
| 131 | if (entry.key !== pageSearchKey(page)) { |
| 132 | return { fallbackReason: `entry-key-mismatch:${index}:${page.url}` }; |
| 133 | } |
| 134 | |
| 135 | entries.push({ |
| 136 | page, |
| 137 | termCounts: new Map(entry.terms), |
| 138 | length: entry.length, |
| 139 | }); |
| 140 | } |
| 141 | |
| 142 | return { |
| 143 | corpus: { |
| 144 | entries, |
| 145 | documentFrequency: new Map(search.documentFrequency), |
| 146 | averageLength: search.averageLength, |
| 147 | }, |
| 148 | }; |
| 149 | } |
| 150 | |
| 151 | function getCorpus(): CorpusLoad { |
| 152 | if (corpusCache && corpusCacheSource) { |
| 153 | return { |
| 154 | corpus: corpusCache, |
| 155 | source: corpusCacheSource, |
| 156 | cached: true, |
| 157 | loadMs: 0, |
| 158 | fallbackReason: corpusCacheFallbackReason, |
| 159 | }; |
| 160 | } |
| 161 | |
| 162 | const started = performance.now(); |
| 163 | const bundledPages = buildBundledIndex(); |
| 164 | const bundledPrecomputed = buildPrecomputedCorpus(bundledPages); |
| 165 | |
| 166 | if (bundledPrecomputed.corpus) { |
| 167 | corpusCache = bundledPrecomputed.corpus; |
| 168 | corpusCacheSource = 'precomputed'; |
| 169 | } else { |
| 170 | const pages = buildIndex(); |
| 171 | const livePrecomputed = buildPrecomputedCorpus(pages); |
| 172 | |
| 173 | if (livePrecomputed.corpus) { |
| 174 | corpusCache = livePrecomputed.corpus; |
| 175 | corpusCacheSource = 'precomputed'; |
| 176 | } else { |
| 177 | corpusCache = buildRuntimeCorpus(pages); |
| 178 | corpusCacheSource = 'runtime'; |
| 179 | corpusCacheFallbackReason = `bundle:${bundledPrecomputed.fallbackReason};live:${livePrecomputed.fallbackReason}`; |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | return { |
| 184 | corpus: corpusCache, |
| 185 | source: corpusCacheSource, |
| 186 | cached: false, |
| 187 | loadMs: performance.now() - started, |
| 188 | fallbackReason: corpusCacheFallbackReason, |
| 189 | }; |
| 190 | } |
| 191 | |
| 192 | function idf(term: string, corpus: Corpus): number { |
| 193 | const n = corpus.entries.length; |
| 194 | const df = corpus.documentFrequency.get(term) ?? 0; |
| 195 | return Math.log(1 + (n - df + 0.5) / (df + 0.5)); |
| 196 | } |
| 197 | |
| 198 | function bm25(entry: CorpusEntry, terms: string[], corpus: Corpus): number { |
| 199 | let total = 0; |
| 200 | |
| 201 | for (const term of terms) { |
| 202 | const tf = entry.termCounts.get(term) ?? 0; |
| 203 | if (tf === 0) continue; |
| 204 | |
| 205 | const denominator = |
| 206 | tf + BM25_K1 * (1 - BM25_B + BM25_B * (entry.length / corpus.averageLength)); |
| 207 | total += idf(term, corpus) * ((tf * (BM25_K1 + 1)) / denominator); |
| 208 | } |
| 209 | |
| 210 | return total; |
| 211 | } |
| 212 | |
| 213 | function fieldBoost(page: DocPage, terms: string[]): number { |
| 214 | const title = page.title.toLowerCase(); |
| 215 | const description = page.description.toLowerCase(); |
| 216 | const url = page.url.toLowerCase(); |
| 217 | let total = 0; |
| 218 | |
| 219 | for (const term of terms) { |
| 220 | if (title.includes(term)) total += 12; |
| 221 | if (description.includes(term)) total += 5; |
| 222 | for (const heading of page.headings) if (heading.includes(term)) total += 4; |
| 223 | if (url.includes(term)) total += 6; |
| 224 | } |
| 225 | |
| 226 | return total; |
| 227 | } |
| 228 | |
| 229 | function score(entry: CorpusEntry, terms: string[], corpus: Corpus): number { |
| 230 | let total = bm25(entry, terms, corpus) * 8 + fieldBoost(entry.page, terms); |
| 231 | const isMigrationIntent = terms.some(term => |
| 232 | ['migration', 'migrate', 'direct', 'legacy', 'v1', 'v2'].includes(term) |
| 233 | ); |
| 234 | |
| 235 | // Heavily downrank legacy (direct-execution) pages so they only surface when |
| 236 | // nothing in the session-based docs matches. |
| 237 | if (entry.page.legacy) total *= 0.12; |
| 238 | // Migration pages mention both old and current APIs a lot; keep them for |
| 239 | // migration/direct-execution questions, but don't let them beat canonical |
| 240 | // session docs for ordinary usage questions. |
| 241 | if (!isMigrationIntent && entry.page.url.includes('/migration-guide')) total *= 0.35; |
| 242 | return total * (PRIORITY[entry.page.collection] ?? 1); |
| 243 | } |
| 244 | |
| 245 | function firstTermMatch(text: string, terms: string[]): number { |
| 246 | const lower = text.toLowerCase(); |
| 247 | return ( |
| 248 | terms |
| 249 | .map(term => lower.indexOf(term)) |
| 250 | .filter(index => index >= 0) |
| 251 | .sort((a, b) => a - b)[0] ?? 0 |
| 252 | ); |
| 253 | } |
| 254 | |
| 255 | function excerpt( |
| 256 | text: string, |
| 257 | terms: string[], |
| 258 | maxChars: number |
| 259 | ): { value: string; truncated: boolean } { |
| 260 | const at = firstTermMatch(text, terms); |
| 261 | const start = Math.max(0, at - 180); |
| 262 | const end = Math.min(text.length, start + maxChars); |
| 263 | const slice = text.slice(start, end).trim(); |
| 264 | const prefix = start > 0 ? '…' : ''; |
| 265 | const suffix = end < text.length ? '…' : ''; |
| 266 | |
| 267 | return { value: `${prefix}${slice}${suffix}`, truncated: start > 0 || end < text.length }; |
| 268 | } |
| 269 | |
| 270 | function snippet(page: DocPage, terms: string[]): string { |
| 271 | return excerpt(page.text, terms, 360).value; |
| 272 | } |
| 273 | |
| 274 | function dedupeByUrl(ranked: { page: DocPage; s: number }[]): { page: DocPage; s: number }[] { |
| 275 | const seen = new Set<string>(); |
| 276 | const deduped: { page: DocPage; s: number }[] = []; |
| 277 | |
| 278 | for (const item of ranked) { |
| 279 | if (seen.has(item.page.url)) continue; |
| 280 | seen.add(item.page.url); |
| 281 | deduped.push(item); |
| 282 | } |
| 283 | |
| 284 | return deduped; |
| 285 | } |
| 286 | |
| 287 | function contentFor( |
| 288 | page: DocPage, |
| 289 | terms: string[], |
| 290 | maxContentChars: number, |
| 291 | maxSections: number |
| 292 | ) { |
| 293 | const found = readPageByUrl(page.url); |
| 294 | |
| 295 | if (found) { |
| 296 | const markdown = toCleanMarkdown(found.raw); |
| 297 | const evidence = excerpt(markdown, terms, maxContentChars); |
| 298 | |
| 299 | return { |
| 300 | sections: extractSections(markdown).slice(0, maxSections), |
| 301 | content: evidence.value, |
| 302 | contentTruncated: evidence.truncated, |
| 303 | }; |
| 304 | } |
| 305 | |
| 306 | const evidence = excerpt(page.text, terms, maxContentChars); |
| 307 | return { |
| 308 | content: evidence.value, |
| 309 | contentTruncated: evidence.truncated, |
| 310 | }; |
| 311 | } |
| 312 | |
| 313 | |
| 314 | export type SearchDocsInvocation = 'tool' | 'eager_context' | 'eager_preview' | (string & {}); |
| 315 | |
| 316 | export type SearchDocsOptions = { |
| 317 | limit?: number; |
| 318 | invocation?: SearchDocsInvocation; |
| 319 | contentResultCount?: number; |
| 320 | maxContentChars?: number; |
| 321 | maxSections?: number; |
| 322 | /** When false, skip page hydration and return metadata/snippets only. */ |
| 323 | hydrateContent?: boolean; |
| 324 | }; |
| 325 | |
| 326 | export const EAGER_SEARCH_ENABLED = process.env.DOCS_AGENT_EAGER_SEARCH !== '0'; |
| 327 | |
| 328 | export function shouldRunEagerDocsSearch(text: string): boolean { |
| 329 | if (!EAGER_SEARCH_ENABLED) return false; |
| 330 | if (text.trim().length < 3) return false; |
| 331 | |
| 332 | const normalized = text.toLowerCase(); |
| 333 | const accountTerms = |
| 334 | /\b(account|billing|invoice|payment|refund|subscription|ticket|dashboard|workspace|organization|org|api key)\b/; |
| 335 | const personalTerms = |
| 336 | /\b(my|our|me|us|latest|current|status|paid|check|look up|lookup|change|cancel|delete|update)\b/; |
| 337 | |
| 338 | return !(accountTerms.test(normalized) && personalTerms.test(normalized)); |
| 339 | } |
| 340 | |
| 341 | export type SearchDocsResult = { |
| 342 | retrieval: 'bm25-lexical-local'; |
| 343 | results: Array<{ |
| 344 | title: string; |
| 345 | url: string; |
| 346 | description: string; |
| 347 | snippet: string; |
| 348 | sections?: { title: string; anchor: string }[]; |
| 349 | content?: string; |
| 350 | contentTruncated?: boolean; |
| 351 | }>; |
| 352 | }; |
| 353 | |
| 354 | export function searchDocs(query: string, options: SearchDocsOptions = {}): SearchDocsResult { |
| 355 | const limit = options.limit ?? DEFAULT_LIMIT; |
| 356 | const contentResultCount = options.contentResultCount ?? DEFAULT_CONTENT_RESULT_COUNT; |
| 357 | const maxContentChars = options.maxContentChars ?? DEFAULT_MAX_CONTENT_CHARS; |
| 358 | const maxSections = options.maxSections ?? DEFAULT_MAX_SECTIONS; |
| 359 | const hydrateContent = options.hydrateContent ?? true; |
| 360 | const totalStarted = performance.now(); |
| 361 | const tokenizeStarted = performance.now(); |
| 362 | const terms = tokenize(query); |
| 363 | // Fall back to raw terms if the query was all stopwords. |
| 364 | const effective = |
| 365 | terms.length > 0 |
| 366 | ? terms |
| 367 | : query |
| 368 | .toLowerCase() |
| 369 | .split(/\s+/) |
| 370 | .filter(t => t.length > 1); |
| 371 | const tokenizeMs = performance.now() - tokenizeStarted; |
| 372 | |
| 373 | const corpusLoad = getCorpus(); |
| 374 | const corpus = corpusLoad.corpus; |
| 375 | |
| 376 | const rankStarted = performance.now(); |
| 377 | const ranked = dedupeByUrl( |
| 378 | corpus.entries |
| 379 | .map(entry => ({ page: entry.page, s: score(entry, effective, corpus) })) |
| 380 | .filter(({ s }) => s > 0) |
| 381 | .sort((a, b) => b.s - a.s) |
| 382 | ).slice(0, limit); |
| 383 | const rankMs = performance.now() - rankStarted; |
| 384 | |
| 385 | const hydrateStarted = performance.now(); |
| 386 | const results = ranked.map(({ page }, index) => ({ |
| 387 | title: page.title, |
| 388 | url: page.url, |
| 389 | description: page.description, |
| 390 | snippet: snippet(page, effective), |
| 391 | ...(hydrateContent && index < contentResultCount |
| 392 | ? contentFor(page, effective, maxContentChars, maxSections) |
| 393 | : {}), |
| 394 | })); |
| 395 | const hydrateMs = performance.now() - hydrateStarted; |
| 396 | const totalMs = performance.now() - totalStarted; |
| 397 | |
| 398 | logPerf({ |
| 399 | event: 'search_docs', |
| 400 | invocation: options.invocation ?? 'tool', |
| 401 | retrieval: 'bm25-lexical-local', |
| 402 | totalMs: roundMs(totalMs), |
| 403 | tokenizeMs: roundMs(tokenizeMs), |
| 404 | corpusLoadMs: roundMs(corpusLoad.loadMs), |
| 405 | rankMs: roundMs(rankMs), |
| 406 | hydrateMs: roundMs(hydrateMs), |
| 407 | corpusSource: corpusLoad.source, |
| 408 | corpusCached: corpusLoad.cached, |
| 409 | corpusFallbackReason: corpusLoad.fallbackReason, |
| 410 | queryChars: query.length, |
| 411 | termCount: effective.length, |
| 412 | limit, |
| 413 | resultCount: results.length, |
| 414 | contentResultCount: results.filter(result => 'content' in result).length, |
| 415 | corpusPages: corpus.entries.length, |
| 416 | corpusTerms: corpus.documentFrequency.size, |
| 417 | topUrls: results.slice(0, 5).map(result => result.url), |
| 418 | ...(PERF_LOG_QUERY ? { query, terms: effective } : {}), |
| 419 | }); |
| 420 | |
| 421 | return { |
| 422 | retrieval: 'bm25-lexical-local', |
| 423 | results, |
| 424 | }; |
| 425 | } |