@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 425 lines 12.5 KB
Raw
1 import {
2 buildBundledIndex,
3 buildIndex,
4 extractSections,
5 getBundleSearchIndex,
6 pageSearchKey,
7 readPageByUrl,
8 toCleanMarkdown,
9 tokenize,
10 type DocPage,
11 } from './docs';
12
13 /**
14 * search_docs — find the most relevant Composio docs pages for a query.
15 *
16 * This is a local, in-memory lexical retriever. It uses a BM25-style body score
17 * plus field boosts for title, description, headings, and URL. The top results
18 * include full page content (bounded per page), so the model gets rich context
19 * in the same fast tool call instead of doing a serial search -> read round trip.
20 */
21
22 // Collection priority: docs first, then examples, then references and toolkits.
23 // (Knowledge base guides rank with docs.) A toolkit-name query still surfaces its
24 // toolkit page because nothing else matches it.
25 const PRIORITY: Record<DocPage['collection'], number> = {
26 docs: 1.3,
27 kb: 1.3,
28 examples: 1.1,
29 reference: 0.85,
30 toolkits: 0.9,
31 };
32
33 export const DEFAULT_SEARCH_LIMIT = 5;
34 const DEFAULT_LIMIT = DEFAULT_SEARCH_LIMIT;
35 export const DEFAULT_CONTENT_RESULT_COUNT = 4;
36 export const DEFAULT_MAX_CONTENT_CHARS = 10_000;
37 export const DEFAULT_MAX_SECTIONS = 16;
38 const BM25_K1 = 1.2;
39 const BM25_B = 0.75;
40 const PERF_LOG_ENABLED = process.env.DOCS_AGENT_SEARCH_PERF_LOG === '1';
41 const PERF_LOG_QUERY = process.env.DOCS_AGENT_SEARCH_LOG_QUERY === '1';
42
43 type CorpusEntry = {
44 page: DocPage;
45 termCounts: Map<string, number>;
46 length: number;
47 };
48
49 type Corpus = {
50 entries: CorpusEntry[];
51 documentFrequency: Map<string, number>;
52 averageLength: number;
53 };
54
55 type CorpusSource = 'precomputed' | 'runtime';
56
57 type PrecomputedCorpusResult =
58 | { corpus: Corpus; fallbackReason?: never }
59 | { corpus?: never; fallbackReason: string };
60
61 type CorpusLoad = {
62 corpus: Corpus;
63 source: CorpusSource;
64 cached: boolean;
65 loadMs: number;
66 fallbackReason?: string;
67 };
68
69 let corpusCache: Corpus | undefined;
70 let corpusCacheSource: CorpusSource | undefined;
71 let corpusCacheFallbackReason: string | undefined;
72
73 function roundMs(ms: number): number {
74 return Math.round(ms * 100) / 100;
75 }
76
77 function logPerf(payload: Record<string, unknown>) {
78 if (!PERF_LOG_ENABLED) return;
79 console.info(`[docs-agent:search_docs] ${JSON.stringify(payload)}`);
80 }
81
82 function termCountsFor(page: DocPage): Map<string, number> {
83 const counts = new Map<string, number>();
84
85 for (const token of tokenize(page.lowerText)) {
86 counts.set(token, (counts.get(token) ?? 0) + 1);
87 }
88
89 return counts;
90 }
91
92 function buildRuntimeCorpus(pages: DocPage[]): Corpus {
93 const entries = pages.map(page => {
94 const termCounts = termCountsFor(page);
95 return {
96 page,
97 termCounts,
98 length: [...termCounts.values()].reduce((sum, count) => sum + count, 0),
99 };
100 });
101 const documentFrequency = new Map<string, number>();
102
103 for (const entry of entries) {
104 for (const term of entry.termCounts.keys()) {
105 documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
106 }
107 }
108
109 return {
110 entries,
111 documentFrequency,
112 averageLength:
113 entries.reduce((sum, entry) => sum + entry.length, 0) / Math.max(entries.length, 1),
114 };
115 }
116
117 function buildPrecomputedCorpus(pages: DocPage[]): PrecomputedCorpusResult {
118 const search = getBundleSearchIndex();
119 if (!search) return { fallbackReason: 'missing-precomputed-index' };
120 if (search.entries.length !== pages.length) {
121 return { fallbackReason: `entry-count-mismatch:${search.entries.length}:${pages.length}` };
122 }
123
124 const entries: CorpusEntry[] = [];
125
126 for (let index = 0; index < pages.length; index++) {
127 const page = pages[index];
128 const entry = search.entries[index];
129
130 if (!entry) return { fallbackReason: `missing-entry:${index}` };
131 if (entry.key !== pageSearchKey(page)) {
132 return { fallbackReason: `entry-key-mismatch:${index}:${page.url}` };
133 }
134
135 entries.push({
136 page,
137 termCounts: new Map(entry.terms),
138 length: entry.length,
139 });
140 }
141
142 return {
143 corpus: {
144 entries,
145 documentFrequency: new Map(search.documentFrequency),
146 averageLength: search.averageLength,
147 },
148 };
149 }
150
151 function getCorpus(): CorpusLoad {
152 if (corpusCache && corpusCacheSource) {
153 return {
154 corpus: corpusCache,
155 source: corpusCacheSource,
156 cached: true,
157 loadMs: 0,
158 fallbackReason: corpusCacheFallbackReason,
159 };
160 }
161
162 const started = performance.now();
163 const bundledPages = buildBundledIndex();
164 const bundledPrecomputed = buildPrecomputedCorpus(bundledPages);
165
166 if (bundledPrecomputed.corpus) {
167 corpusCache = bundledPrecomputed.corpus;
168 corpusCacheSource = 'precomputed';
169 } else {
170 const pages = buildIndex();
171 const livePrecomputed = buildPrecomputedCorpus(pages);
172
173 if (livePrecomputed.corpus) {
174 corpusCache = livePrecomputed.corpus;
175 corpusCacheSource = 'precomputed';
176 } else {
177 corpusCache = buildRuntimeCorpus(pages);
178 corpusCacheSource = 'runtime';
179 corpusCacheFallbackReason = `bundle:${bundledPrecomputed.fallbackReason};live:${livePrecomputed.fallbackReason}`;
180 }
181 }
182
183 return {
184 corpus: corpusCache,
185 source: corpusCacheSource,
186 cached: false,
187 loadMs: performance.now() - started,
188 fallbackReason: corpusCacheFallbackReason,
189 };
190 }
191
192 function idf(term: string, corpus: Corpus): number {
193 const n = corpus.entries.length;
194 const df = corpus.documentFrequency.get(term) ?? 0;
195 return Math.log(1 + (n - df + 0.5) / (df + 0.5));
196 }
197
198 function bm25(entry: CorpusEntry, terms: string[], corpus: Corpus): number {
199 let total = 0;
200
201 for (const term of terms) {
202 const tf = entry.termCounts.get(term) ?? 0;
203 if (tf === 0) continue;
204
205 const denominator =
206 tf + BM25_K1 * (1 - BM25_B + BM25_B * (entry.length / corpus.averageLength));
207 total += idf(term, corpus) * ((tf * (BM25_K1 + 1)) / denominator);
208 }
209
210 return total;
211 }
212
213 function fieldBoost(page: DocPage, terms: string[]): number {
214 const title = page.title.toLowerCase();
215 const description = page.description.toLowerCase();
216 const url = page.url.toLowerCase();
217 let total = 0;
218
219 for (const term of terms) {
220 if (title.includes(term)) total += 12;
221 if (description.includes(term)) total += 5;
222 for (const heading of page.headings) if (heading.includes(term)) total += 4;
223 if (url.includes(term)) total += 6;
224 }
225
226 return total;
227 }
228
229 function score(entry: CorpusEntry, terms: string[], corpus: Corpus): number {
230 let total = bm25(entry, terms, corpus) * 8 + fieldBoost(entry.page, terms);
231 const isMigrationIntent = terms.some(term =>
232 ['migration', 'migrate', 'direct', 'legacy', 'v1', 'v2'].includes(term)
233 );
234
235 // Heavily downrank legacy (direct-execution) pages so they only surface when
236 // nothing in the session-based docs matches.
237 if (entry.page.legacy) total *= 0.12;
238 // Migration pages mention both old and current APIs a lot; keep them for
239 // migration/direct-execution questions, but don't let them beat canonical
240 // session docs for ordinary usage questions.
241 if (!isMigrationIntent && entry.page.url.includes('/migration-guide')) total *= 0.35;
242 return total * (PRIORITY[entry.page.collection] ?? 1);
243 }
244
245 function firstTermMatch(text: string, terms: string[]): number {
246 const lower = text.toLowerCase();
247 return (
248 terms
249 .map(term => lower.indexOf(term))
250 .filter(index => index >= 0)
251 .sort((a, b) => a - b)[0] ?? 0
252 );
253 }
254
255 function excerpt(
256 text: string,
257 terms: string[],
258 maxChars: number
259 ): { value: string; truncated: boolean } {
260 const at = firstTermMatch(text, terms);
261 const start = Math.max(0, at - 180);
262 const end = Math.min(text.length, start + maxChars);
263 const slice = text.slice(start, end).trim();
264 const prefix = start > 0 ? '…' : '';
265 const suffix = end < text.length ? '…' : '';
266
267 return { value: `${prefix}${slice}${suffix}`, truncated: start > 0 || end < text.length };
268 }
269
270 function snippet(page: DocPage, terms: string[]): string {
271 return excerpt(page.text, terms, 360).value;
272 }
273
274 function dedupeByUrl(ranked: { page: DocPage; s: number }[]): { page: DocPage; s: number }[] {
275 const seen = new Set<string>();
276 const deduped: { page: DocPage; s: number }[] = [];
277
278 for (const item of ranked) {
279 if (seen.has(item.page.url)) continue;
280 seen.add(item.page.url);
281 deduped.push(item);
282 }
283
284 return deduped;
285 }
286
287 function contentFor(
288 page: DocPage,
289 terms: string[],
290 maxContentChars: number,
291 maxSections: number
292 ) {
293 const found = readPageByUrl(page.url);
294
295 if (found) {
296 const markdown = toCleanMarkdown(found.raw);
297 const evidence = excerpt(markdown, terms, maxContentChars);
298
299 return {
300 sections: extractSections(markdown).slice(0, maxSections),
301 content: evidence.value,
302 contentTruncated: evidence.truncated,
303 };
304 }
305
306 const evidence = excerpt(page.text, terms, maxContentChars);
307 return {
308 content: evidence.value,
309 contentTruncated: evidence.truncated,
310 };
311 }
312
313
314 export type SearchDocsInvocation = 'tool' | 'eager_context' | 'eager_preview' | (string & {});
315
316 export type SearchDocsOptions = {
317 limit?: number;
318 invocation?: SearchDocsInvocation;
319 contentResultCount?: number;
320 maxContentChars?: number;
321 maxSections?: number;
322 /** When false, skip page hydration and return metadata/snippets only. */
323 hydrateContent?: boolean;
324 };
325
326 export const EAGER_SEARCH_ENABLED = process.env.DOCS_AGENT_EAGER_SEARCH !== '0';
327
328 export function shouldRunEagerDocsSearch(text: string): boolean {
329 if (!EAGER_SEARCH_ENABLED) return false;
330 if (text.trim().length < 3) return false;
331
332 const normalized = text.toLowerCase();
333 const accountTerms =
334 /\b(account|billing|invoice|payment|refund|subscription|ticket|dashboard|workspace|organization|org|api key)\b/;
335 const personalTerms =
336 /\b(my|our|me|us|latest|current|status|paid|check|look up|lookup|change|cancel|delete|update)\b/;
337
338 return !(accountTerms.test(normalized) && personalTerms.test(normalized));
339 }
340
341 export type SearchDocsResult = {
342 retrieval: 'bm25-lexical-local';
343 results: Array<{
344 title: string;
345 url: string;
346 description: string;
347 snippet: string;
348 sections?: { title: string; anchor: string }[];
349 content?: string;
350 contentTruncated?: boolean;
351 }>;
352 };
353
354 export function searchDocs(query: string, options: SearchDocsOptions = {}): SearchDocsResult {
355 const limit = options.limit ?? DEFAULT_LIMIT;
356 const contentResultCount = options.contentResultCount ?? DEFAULT_CONTENT_RESULT_COUNT;
357 const maxContentChars = options.maxContentChars ?? DEFAULT_MAX_CONTENT_CHARS;
358 const maxSections = options.maxSections ?? DEFAULT_MAX_SECTIONS;
359 const hydrateContent = options.hydrateContent ?? true;
360 const totalStarted = performance.now();
361 const tokenizeStarted = performance.now();
362 const terms = tokenize(query);
363 // Fall back to raw terms if the query was all stopwords.
364 const effective =
365 terms.length > 0
366 ? terms
367 : query
368 .toLowerCase()
369 .split(/\s+/)
370 .filter(t => t.length > 1);
371 const tokenizeMs = performance.now() - tokenizeStarted;
372
373 const corpusLoad = getCorpus();
374 const corpus = corpusLoad.corpus;
375
376 const rankStarted = performance.now();
377 const ranked = dedupeByUrl(
378 corpus.entries
379 .map(entry => ({ page: entry.page, s: score(entry, effective, corpus) }))
380 .filter(({ s }) => s > 0)
381 .sort((a, b) => b.s - a.s)
382 ).slice(0, limit);
383 const rankMs = performance.now() - rankStarted;
384
385 const hydrateStarted = performance.now();
386 const results = ranked.map(({ page }, index) => ({
387 title: page.title,
388 url: page.url,
389 description: page.description,
390 snippet: snippet(page, effective),
391 ...(hydrateContent && index < contentResultCount
392 ? contentFor(page, effective, maxContentChars, maxSections)
393 : {}),
394 }));
395 const hydrateMs = performance.now() - hydrateStarted;
396 const totalMs = performance.now() - totalStarted;
397
398 logPerf({
399 event: 'search_docs',
400 invocation: options.invocation ?? 'tool',
401 retrieval: 'bm25-lexical-local',
402 totalMs: roundMs(totalMs),
403 tokenizeMs: roundMs(tokenizeMs),
404 corpusLoadMs: roundMs(corpusLoad.loadMs),
405 rankMs: roundMs(rankMs),
406 hydrateMs: roundMs(hydrateMs),
407 corpusSource: corpusLoad.source,
408 corpusCached: corpusLoad.cached,
409 corpusFallbackReason: corpusLoad.fallbackReason,
410 queryChars: query.length,
411 termCount: effective.length,
412 limit,
413 resultCount: results.length,
414 contentResultCount: results.filter(result => 'content' in result).length,
415 corpusPages: corpus.entries.length,
416 corpusTerms: corpus.documentFrequency.size,
417 topUrls: results.slice(0, 5).map(result => result.url),
418 ...(PERF_LOG_QUERY ? { query, terms: effective } : {}),
419 });
420
421 return {
422 retrieval: 'bm25-lexical-local',
423 results,
424 };
425 }