@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 495 lines 14.3 KB
Raw
1 /**
2 * Shared docs access for the agent's retrieval tools.
3 *
4 * Builds a lazy in-memory index over `content/` (docs, reference, examples,
5 * toolkits and the public knowledge base guides under `content/kb`), maps page
6 * URLs to files, and flags legacy pages so the search tool can heavily downrank
7 * them.
8 */
9
10 import { resolveToolkitCount } from '../../lib/toolkit-count';
11 import { existsSync, readdirSync, readFileSync } from 'node:fs';
12 import { dirname, join, relative } from 'node:path';
13 import { fileURLToPath } from 'node:url';
14 // Build-time snapshot of content/, imported so eve bundles it into the deployed
15 // service. The deployed runtime has the agent/ dir but NOT content/, so we fall
16 // back to this bundle whenever the content tree isn't on disk. Regenerate with
17 // `bun scripts/build-agent-index.ts` (runs automatically on dev/build).
18 import bundledIndex from './docs-index';
19
20 /**
21 * Resolve the docs app root (the directory containing `content/docs`) without
22 * assuming the runtime's `process.cwd()`. In a deployed eve function the cwd may
23 * not be `docs/`, so we probe the cwd, `cwd/docs`, and the ancestors of this
24 * module until we find the content tree.
25 */
26 function resolveAppRoot(): string {
27 const candidates: string[] = [process.cwd(), join(process.cwd(), 'docs')];
28 try {
29 let dir = dirname(fileURLToPath(import.meta.url));
30 for (let i = 0; i < 10; i++) {
31 candidates.push(dir);
32 const parent = dirname(dir);
33 if (parent === dir) break;
34 dir = parent;
35 }
36 } catch {
37 // import.meta.url unavailable (e.g. CJS) — rely on the cwd candidates.
38 }
39 for (const candidate of candidates) {
40 if (existsSync(join(candidate, 'content', 'docs'))) return candidate;
41 }
42 return process.cwd();
43 }
44
45 const APP_ROOT = resolveAppRoot();
46 const CONTENT_ROOT = join(APP_ROOT, 'content');
47 const COLLECTIONS = ['docs', 'reference', 'examples', 'toolkits', 'kb'] as const;
48
49 /** URL prefixes for the legacy, pre-session / direct-execution docs. */
50 const LEGACY_URL_PATTERNS = [
51 '/docs/tools-direct',
52 '/docs/auth-configuration',
53 '/docs/sessions-vs-direct-execution',
54 ];
55
56 const STOPWORDS = new Set([
57 'the',
58 'a',
59 'an',
60 'and',
61 'or',
62 'but',
63 'is',
64 'are',
65 'was',
66 'were',
67 'be',
68 'been',
69 'to',
70 'of',
71 'in',
72 'on',
73 'for',
74 'with',
75 'as',
76 'at',
77 'by',
78 'from',
79 'how',
80 'do',
81 'does',
82 'did',
83 'what',
84 'why',
85 'when',
86 'which',
87 'who',
88 'can',
89 'i',
90 'you',
91 'it',
92 'this',
93 'that',
94 'these',
95 'those',
96 'my',
97 'your',
98 'we',
99 'they',
100 'use',
101 'using',
102 'work',
103 'works',
104 'about',
105 'into',
106 'so',
107 'if',
108 'me',
109 'get',
110 'set',
111 'up',
112 ]);
113
114 export type Collection = (typeof COLLECTIONS)[number];
115
116 export interface DocPage {
117 collection: Collection;
118 title: string;
119 description: string;
120 url: string;
121 legacy: boolean;
122 /** Headings, lowercased, for ranking. */
123 headings: string[];
124 /** Original-case plain text for snippets. */
125 text: string;
126 /** Lowercased title + description + text for scoring. */
127 lowerText: string;
128 }
129
130 interface BundlePage {
131 collection: Collection;
132 url: string;
133 title: string;
134 description: string;
135 legacy: boolean;
136 /** Cleaned Markdown body (no frontmatter), enough for search + read_doc. */
137 markdown: string;
138 }
139 interface BundleToolkit {
140 slug?: string;
141 name?: string;
142 description?: string;
143 category?: string;
144 authSchemes?: string[];
145 }
146 export interface BundleSearchIndex {
147 averageLength: number;
148 documentFrequency: [string, number][];
149 entries: {
150 key: string;
151 length: number;
152 terms: [string, number][];
153 }[];
154 }
155 const BUNDLE = bundledIndex as {
156 pages: BundlePage[];
157 toolkits: BundleToolkit[];
158 search?: BundleSearchIndex;
159 };
160
161 /**
162 * Whether the live content tree is on disk. True in dev / local (read fresh
163 * files); false in the deployed eve service (use the bundled snapshot). Set
164 * `EVE_FORCE_BUNDLE=1` to exercise the bundle path locally.
165 */
166 const CONTENT_AVAILABLE =
167 process.env.EVE_FORCE_BUNDLE !== '1' && existsSync(join(CONTENT_ROOT, 'docs'));
168
169 /** Mirror of lib/search-index.ts `urlFromContentPath` so links match real routes. */
170 export function urlFromContentPath(absPath: string): string | undefined {
171 const rel = relative(CONTENT_ROOT, absPath).replace(/\\/g, '/');
172 const withoutExt = rel.replace(/\.mdx?$/, '');
173 const parts = withoutExt.split('/');
174 const collection = parts.shift();
175 if (!collection) return undefined;
176 if (collection === 'docs') return `/docs/${parts.join('/')}`.replace(/\/index$/, '');
177 if (collection === 'examples') return `/examples/${parts.join('/')}`.replace(/\/index$/, '');
178 if (collection === 'reference') return `/reference/${parts.join('/')}`.replace(/\/index$/, '');
179 if (collection === 'kb') return `/kb/${parts.join('/')}`.replace(/\/index$/, '');
180 if (collection === 'toolkits') {
181 if (parts[0] === 'faq') return undefined;
182 return `/toolkits/${parts.join('/')}`.replace(/\/index$/, '');
183 }
184 return undefined;
185 }
186
187 export function isLegacyUrl(url: string): boolean {
188 return LEGACY_URL_PATTERNS.some(p => url === p || url.startsWith(`${p}/`));
189 }
190
191 export function tokenize(query: string): string[] {
192 return query
193 .toLowerCase()
194 .split(/[^a-z0-9_]+/)
195 .filter(t => t.length > 1 && !STOPWORDS.has(t));
196 }
197
198 export function parseFrontmatter(raw: string): {
199 title: string;
200 description: string;
201 legacy: boolean;
202 body: string;
203 } {
204 const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
205 if (!match) return { title: '', description: '', legacy: false, body: raw };
206 const [, fm, body] = match;
207 const get = (key: string) => {
208 const m = fm.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
209 return m ? m[1].trim().replace(/^["']|["']$/g, '') : '';
210 };
211 return {
212 title: get('title'),
213 description: get('description'),
214 legacy: get('legacy') === 'true',
215 body,
216 };
217 }
218
219 /** Strip MDX/JSX noise to readable text (keeps prose and inline code words). */
220 function toPlainText(body: string): string {
221 return body
222 .replace(/```[\s\S]*?```/g, ' ')
223 .replace(/<\/?[A-Za-z][^>]*>/g, ' ')
224 .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ')
225 .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
226 .replace(/[#>*_`|]/g, ' ')
227 .replace(/\s+/g, ' ')
228 .trim();
229 }
230
231 /** Clean MDX into Markdown the model can read: drop frontmatter and bare JSX tags. */
232 export function toCleanMarkdown(raw: string): string {
233 const { body } = parseFrontmatter(raw);
234 return body
235 .replace(/<\/?[A-Za-z][A-Za-z0-9.]*(\s[^>]*)?\/?>/g, '')
236 .replace(/\n{3,}/g, '\n\n')
237 .trim();
238 }
239
240 /**
241 * Extract a page's section anchors from its Markdown headings, so the assistant
242 * can link a specific section (e.g. `/docs/how-composio-works#users`) instead
243 * of just the page. Anchor slugs match the docs' heading-id convention.
244 */
245 export function extractSections(markdown: string): { title: string; anchor: string }[] {
246 return [...markdown.matchAll(/^#{2,4}\s+(.+?)\s*$/gm)].map(m => {
247 const title = m[1].replace(/[`*_]/g, '').trim();
248 const slug = title
249 .toLowerCase()
250 .replace(/[^a-z0-9]+/g, '-')
251 .replace(/(^-|-$)/g, '');
252 return { title, anchor: `#${slug}` };
253 });
254 }
255
256 export function pageSearchKey(page: Pick<DocPage, 'collection' | 'title' | 'url'>): string {
257 return `${page.collection}\u0000${page.url}\u0000${page.title}`;
258 }
259
260 export function getBundleSearchIndex(): BundleSearchIndex | undefined {
261 return BUNDLE.search;
262 }
263
264 function makePage(args: {
265 collection: Collection;
266 title: string;
267 description: string;
268 url: string;
269 legacy: boolean;
270 body: string;
271 }): DocPage {
272 const headings = (args.body.match(/^#{1,4}\s+(.+)$/gm) ?? []).map(h =>
273 h
274 .replace(/^#{1,4}\s+/, '')
275 .toLowerCase()
276 .trim()
277 );
278 const text = toPlainText(args.body);
279 return {
280 collection: args.collection,
281 title: args.title || args.url,
282 description: args.description,
283 url: args.url,
284 legacy: args.legacy,
285 headings,
286 text,
287 lowerText: `${args.title} ${args.description} ${text}`.toLowerCase(),
288 };
289 }
290
291 /**
292 * Index the toolkit catalog from `public/data/toolkits.json` so the assistant
293 * can answer "do you have <toolkit>?" — individual toolkit pages are generated
294 * from this data, not from MDX, so they aren't covered by the content scan.
295 */
296 interface Toolkit {
297 slug?: string;
298 name?: string;
299 description?: string;
300 category?: string;
301 authSchemes?: string[];
302 }
303
304 let toolkitMap: Map<string, Toolkit> | undefined;
305
306 /** Parse `public/data/toolkits.json` once into a slug -> toolkit map. */
307 function getToolkitMap(): Map<string, Toolkit> {
308 if (toolkitMap) return toolkitMap;
309 const map = new Map<string, Toolkit>();
310 if (CONTENT_AVAILABLE) {
311 try {
312 const parsed = JSON.parse(
313 readFileSync(join(APP_ROOT, 'public', 'data', 'toolkits.json'), 'utf8')
314 );
315 const list: Toolkit[] = Array.isArray(parsed)
316 ? parsed
317 : (parsed.toolkits ?? parsed.items ?? []);
318 for (const tk of list) if (tk?.slug) map.set(tk.slug, tk);
319 } catch {
320 // no catalog available
321 }
322 }
323 // Deployed runtime (or partial traced catalog): merge in the bundled toolkit
324 // snapshot so search/read_doc still cover every generated toolkit page.
325 for (const tk of BUNDLE.toolkits) if (tk?.slug && !map.has(tk.slug)) map.set(tk.slug, tk);
326 toolkitMap = map;
327 return map;
328 }
329
330 function loadBundledToolkits(): DocPage[] {
331 const pages: DocPage[] = [];
332 for (const tk of BUNDLE.toolkits) {
333 if (!tk?.slug) continue;
334 const name = tk.name ?? tk.slug;
335 const body = `${tk.description ?? ''} Category: ${tk.category ?? ''}. Toolkit slug: ${tk.slug}.`;
336 pages.push(
337 makePage({
338 collection: 'toolkits',
339 title: `${name} toolkit`,
340 description: tk.description ?? '',
341 url: `/toolkits/${tk.slug}`,
342 legacy: false,
343 body,
344 })
345 );
346 }
347 return pages;
348 }
349
350 function loadToolkits(): DocPage[] {
351 const pages: DocPage[] = [];
352 for (const tk of getToolkitMap().values()) {
353 const name = tk.name ?? tk.slug!;
354 const body = `${tk.description ?? ''} Category: ${tk.category ?? ''}. Toolkit slug: ${tk.slug}.`;
355 pages.push(
356 makePage({
357 collection: 'toolkits',
358 title: `${name} toolkit`,
359 description: tk.description ?? '',
360 url: `/toolkits/${tk.slug}`,
361 legacy: false,
362 body,
363 })
364 );
365 }
366 return pages;
367 }
368
369 let indexCache: DocPage[] | undefined;
370
371 export function buildBundledIndex(): DocPage[] {
372 const pages: DocPage[] = [];
373 for (const p of BUNDLE.pages) {
374 pages.push(
375 makePage({
376 collection: p.collection,
377 title: p.title,
378 description: p.description,
379 url: p.url,
380 legacy: p.legacy || isLegacyUrl(p.url),
381 body: p.markdown,
382 })
383 );
384 }
385 pages.push(...loadBundledToolkits());
386 return pages;
387 }
388
389 export function buildIndex(): DocPage[] {
390 if (indexCache) return indexCache;
391 const pages: DocPage[] = [];
392 if (CONTENT_AVAILABLE) {
393 for (const collection of COLLECTIONS) {
394 const dir = join(CONTENT_ROOT, collection);
395 let entries: string[];
396 try {
397 entries = readdirSync(dir, { recursive: true }) as string[];
398 } catch {
399 continue;
400 }
401 for (const entry of entries) {
402 if (!/\.mdx?$/.test(entry)) continue;
403 const absPath = join(dir, entry);
404 const url = urlFromContentPath(absPath);
405 if (!url) continue;
406 let raw: string;
407 try {
408 raw = resolveToolkitCount(readFileSync(absPath, 'utf8'));
409 } catch {
410 continue;
411 }
412 const { title, description, legacy, body } = parseFrontmatter(raw);
413 pages.push(
414 makePage({
415 collection: collection as Collection,
416 title,
417 description,
418 url,
419 legacy: legacy || isLegacyUrl(url),
420 body,
421 })
422 );
423 }
424 }
425 } else {
426 // Deployed runtime: content/ isn't on disk, so index the bundled snapshot.
427 pages.push(...buildBundledIndex());
428 indexCache = pages;
429 return pages;
430 }
431 pages.push(...loadToolkits());
432 indexCache = pages;
433 return pages;
434 }
435
436 /** Resolve a page URL back to its source file and return the raw MDX. */
437 function authSchemesMarkdown(schemes: string[] = []): string {
438 if (!schemes.length) return 'n/a';
439 return schemes.map(scheme => `\`${scheme}\``).join(', ');
440 }
441
442 export function readPageByUrl(url: string): { title: string; raw: string } | undefined {
443 const clean = url.split('#')[0].split('?')[0].replace(/\/$/, '');
444 const parts = clean.split('/').filter(Boolean);
445 const collection = parts.shift();
446 if (!collection || !COLLECTIONS.includes(collection as (typeof COLLECTIONS)[number]))
447 return undefined;
448
449 // Toolkit pages are generated from the catalog, not MDX. Synthesize content
450 // so the assistant can confirm support and describe the toolkit.
451 if (collection === 'toolkits') {
452 const tk = getToolkitMap().get(parts.join('/'));
453 if (tk) {
454 const name = tk.name ?? tk.slug!;
455 const raw = [
456 `# ${name}`,
457 '',
458 tk.description ?? '',
459 '',
460 `- **Supported:** yes, \`${tk.slug}\` is a Composio toolkit.`,
461 `- **Category:** ${tk.category ?? 'n/a'}`,
462 `- **Auth:** ${authSchemesMarkdown(tk.authSchemes)}`,
463 '',
464 `Connect it for a user with \`session.authorize("${tk.slug}")\` (or in-chat auth), then use its tools through the session. See [Configuring sessions](/docs/configuring-sessions), [Authentication](/docs/authentication), and [auth schemes](/reference/api-reference/auth-configs#auth-schemes) for how its auth mode works.`,
465 ].join('\n');
466 return { title: `${name} toolkit`, raw };
467 }
468 }
469
470 if (CONTENT_AVAILABLE) {
471 const base = join(CONTENT_ROOT, collection, ...parts);
472 const candidates = [
473 `${base}.mdx`,
474 `${base}.md`,
475 join(base, 'index.mdx'),
476 join(base, 'index.md'),
477 ];
478 for (const candidate of candidates) {
479 try {
480 const raw = resolveToolkitCount(readFileSync(candidate, 'utf8'));
481 const { title } = parseFrontmatter(raw);
482 return { title: title || url, raw };
483 } catch {
484 // try next candidate
485 }
486 }
487 // Fall through to the bundled snapshot. Vercel may trace only part of the
488 // content tree, while the generated agent snapshot still has the full docs.
489 }
490
491 // Deployed runtime: serve the page from the bundled snapshot.
492 const page = BUNDLE.pages.find(p => p.url === clean);
493 if (page) return { title: page.title || url, raw: page.markdown };
494 return undefined;
495 }