Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 341 lines 9.36 KB
Raw
1 /**
2 * Build the docs snapshot the Ask AI agent imports (`agent/lib/docs-index.ts`).
3 *
4 * The deployed eve service bundles the `agent/` directory but NOT `content/` or
5 * `public/data/`, so the agent can't read the docs from disk at runtime. This
6 * script snapshots every content page (cleaned Markdown + metadata) and the
7 * toolkit catalog into a generated module the agent imports, so eve bundles it
8 * into the deployed service. It also precomputes BM25-style lexical term stats
9 * for the docs assistant search tool, avoiding cold-start corpus construction in
10 * the deployed function.
11 *
12 * In dev the agent still reads `content/` live when the generated search snapshot
13 * does not match; the bundle is always used when the content tree isn't on disk.
14 *
15 * Self-contained on purpose: it does NOT import `agent/lib/docs.ts` (which
16 * imports the generated file), so it can run from a clean checkout where
17 * `docs-index.ts` doesn't exist yet. The generated file is gitignored and
18 * regenerated before `dev`, `build`, and `types:check`.
19 *
20 * Run manually with `bun scripts/build-agent-index.ts`.
21 */
22 import { readdirSync, readFileSync, writeFileSync } from 'node:fs';
23 import { join, relative } from 'node:path';
24 import { resolveToolkitCount } from '../lib/toolkit-count';
25
26 const ROOT = process.cwd();
27 const CONTENT = join(ROOT, 'content');
28 const COLLECTIONS = ['docs', 'reference', 'examples', 'toolkits', 'kb'] as const;
29 const LEGACY_URL_PATTERNS = [
30 '/docs/tools-direct',
31 '/docs/auth-configuration',
32 '/docs/sessions-vs-direct-execution',
33 ];
34 const STOPWORDS = new Set([
35 'the',
36 'a',
37 'an',
38 'and',
39 'or',
40 'but',
41 'is',
42 'are',
43 'was',
44 'were',
45 'be',
46 'been',
47 'to',
48 'of',
49 'in',
50 'on',
51 'for',
52 'with',
53 'as',
54 'at',
55 'by',
56 'from',
57 'how',
58 'do',
59 'does',
60 'did',
61 'what',
62 'why',
63 'when',
64 'which',
65 'who',
66 'can',
67 'i',
68 'you',
69 'it',
70 'this',
71 'that',
72 'these',
73 'those',
74 'my',
75 'your',
76 'we',
77 'they',
78 'use',
79 'using',
80 'work',
81 'works',
82 'about',
83 'into',
84 'so',
85 'if',
86 'me',
87 'get',
88 'set',
89 'up',
90 ]);
91
92 // The helpers below mirror agent/lib/docs.ts so generated URLs, cleaned
93 // Markdown, page keys, and BM25 token stats match what the runtime path uses.
94 function isLegacyUrl(url: string): boolean {
95 return LEGACY_URL_PATTERNS.some(p => url === p || url.startsWith(`${p}/`));
96 }
97
98 function urlFromContentPath(absPath: string): string | undefined {
99 const rel = relative(CONTENT, absPath).replace(/\\/g, '/');
100 const withoutExt = rel.replace(/\.mdx?$/, '');
101 const parts = withoutExt.split('/');
102 const collection = parts.shift();
103 if (!collection) return undefined;
104 if (collection === 'docs') return `/docs/${parts.join('/')}`.replace(/\/index$/, '');
105 if (collection === 'examples') return `/examples/${parts.join('/')}`.replace(/\/index$/, '');
106 if (collection === 'reference') return `/reference/${parts.join('/')}`.replace(/\/index$/, '');
107 if (collection === 'kb') return `/kb/${parts.join('/')}`.replace(/\/index$/, '');
108 if (collection === 'toolkits') {
109 if (parts[0] === 'faq') return undefined;
110 return `/toolkits/${parts.join('/')}`.replace(/\/index$/, '');
111 }
112 return undefined;
113 }
114
115 function tokenize(query: string): string[] {
116 return query
117 .toLowerCase()
118 .split(/[^a-z0-9_]+/)
119 .filter(t => t.length > 1 && !STOPWORDS.has(t));
120 }
121
122 function parseFrontmatter(raw: string): {
123 title: string;
124 description: string;
125 legacy: boolean;
126 body: string;
127 } {
128 const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
129 if (!match) return { title: '', description: '', legacy: false, body: raw };
130 const [, fm, body] = match;
131 const get = (key: string) => {
132 const m = fm.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
133 return m ? m[1].trim().replace(/^["']|["']$/g, '') : '';
134 };
135 return {
136 title: get('title'),
137 description: get('description'),
138 legacy: get('legacy') === 'true',
139 body,
140 };
141 }
142
143 function toPlainText(body: string): string {
144 return body
145 .replace(/```[\s\S]*?```/g, ' ')
146 .replace(/<\/?[A-Za-z][^>]*>/g, ' ')
147 .replace(/!\[[^\]]*\]\([^)]*\)/g, ' ')
148 .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
149 .replace(/[#>*_`|]/g, ' ')
150 .replace(/\s+/g, ' ')
151 .trim();
152 }
153
154 function toCleanMarkdown(raw: string): string {
155 const { body } = parseFrontmatter(raw);
156 return body
157 .replace(/<\/?[A-Za-z][A-Za-z0-9.]*(\s[^>]*)?\/?>/g, '')
158 .replace(/\n{3,}/g, '\n\n')
159 .trim();
160 }
161
162 interface BundlePage {
163 collection: string;
164 url: string;
165 title: string;
166 description: string;
167 legacy: boolean;
168 markdown: string;
169 }
170
171 interface BundleToolkit {
172 slug: string;
173 name?: string;
174 description?: string;
175 category?: string;
176 authSchemes?: string[];
177 }
178
179 interface SearchPage {
180 collection: string;
181 title: string;
182 description: string;
183 url: string;
184 headings: string[];
185 lowerText: string;
186 }
187
188 function pageSearchKey(page: Pick<SearchPage, 'collection' | 'title' | 'url'>): string {
189 return `${page.collection}\u0000${page.url}\u0000${page.title}`;
190 }
191
192 function makeSearchPage(args: {
193 collection: string;
194 title: string;
195 description: string;
196 url: string;
197 body: string;
198 }): SearchPage {
199 const headings = (args.body.match(/^#{1,4}\s+(.+)$/gm) ?? []).map(h =>
200 h
201 .replace(/^#{1,4}\s+/, '')
202 .toLowerCase()
203 .trim()
204 );
205 const text = toPlainText(args.body);
206
207 return {
208 collection: args.collection,
209 title: args.title || args.url,
210 description: args.description,
211 url: args.url,
212 headings,
213 lowerText: `${args.title} ${args.description} ${text}`.toLowerCase(),
214 };
215 }
216
217 function searchPageForToolkit(tk: BundleToolkit): SearchPage {
218 const name = tk.name ?? tk.slug;
219 const body = `${tk.description ?? ''} Category: ${tk.category ?? ''}. Toolkit slug: ${tk.slug}.`;
220
221 return makeSearchPage({
222 collection: 'toolkits',
223 title: `${name} toolkit`,
224 description: tk.description ?? '',
225 url: `/toolkits/${tk.slug}`,
226 body,
227 });
228 }
229
230 function termCountsFor(page: SearchPage): Map<string, number> {
231 const counts = new Map<string, number>();
232
233 for (const token of tokenize(page.lowerText)) {
234 counts.set(token, (counts.get(token) ?? 0) + 1);
235 }
236
237 return counts;
238 }
239
240 function buildSearchIndex(searchPages: SearchPage[]) {
241 const entries = searchPages.map(page => {
242 const counts = termCountsFor(page);
243 const terms = [...counts.entries()];
244
245 return {
246 key: pageSearchKey(page),
247 terms,
248 length: terms.reduce((sum, [, count]) => sum + count, 0),
249 };
250 });
251 const documentFrequency = new Map<string, number>();
252
253 for (const entry of entries) {
254 for (const [term] of entry.terms) {
255 documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
256 }
257 }
258
259 return {
260 averageLength:
261 entries.reduce((sum, entry) => sum + entry.length, 0) / Math.max(entries.length, 1),
262 documentFrequency: [...documentFrequency.entries()],
263 entries,
264 };
265 }
266
267 const pages: BundlePage[] = [];
268 for (const collection of COLLECTIONS) {
269 const dir = join(CONTENT, collection);
270 let entries: string[];
271 try {
272 entries = readdirSync(dir, { recursive: true }) as string[];
273 } catch {
274 continue;
275 }
276 for (const entry of entries) {
277 if (!/\.mdx?$/.test(entry)) continue;
278 const absPath = join(dir, entry);
279 const url = urlFromContentPath(absPath);
280 if (!url) continue;
281 const raw = resolveToolkitCount(readFileSync(absPath, 'utf8'));
282 const { title, description, legacy } = parseFrontmatter(raw);
283 pages.push({
284 collection,
285 url,
286 title: title || url,
287 description,
288 legacy: legacy || isLegacyUrl(url),
289 markdown: toCleanMarkdown(raw),
290 });
291 }
292 }
293
294 let toolkits: BundleToolkit[] = [];
295 try {
296 const parsed = JSON.parse(readFileSync(join(ROOT, 'public', 'data', 'toolkits.json'), 'utf8'));
297 const list = Array.isArray(parsed) ? parsed : (parsed.toolkits ?? parsed.items ?? []);
298 toolkits = list
299 .filter((tk: { slug?: string }) => Boolean(tk?.slug))
300 .map((tk: BundleToolkit) => ({
301 slug: tk.slug,
302 name: tk.name,
303 description: tk.description,
304 category: tk.category,
305 authSchemes: tk.authSchemes,
306 }));
307 } catch {
308 // no catalog available
309 }
310
311 const contentSearchPages = pages.map(page =>
312 makeSearchPage({
313 collection: page.collection,
314 title: page.title,
315 description: page.description,
316 url: page.url,
317 body: page.markdown,
318 })
319 );
320 const toolkitSearchPages = toolkits.map(searchPageForToolkit);
321 const search = buildSearchIndex([
322 ...contentSearchPages,
323 ...toolkitSearchPages,
324 ]);
325
326 // Emit a .ts module (not .json): eve's discovery scans agent/lib/ and only
327 // accepts authored modules there, so a raw .json fails `eve build`. We embed the
328 // snapshot as a JSON string parsed at runtime — a plain string literal keeps the
329 // source fast for tsc and free of escaping pitfalls, and lets eve bundle it.
330 const json = JSON.stringify({ pages, toolkits, search });
331 const ts =
332 `/* eslint-disable */\n` +
333 `// Auto-generated by scripts/build-agent-index.ts — do not edit by hand.\n` +
334 `// Gitignored snapshot of content/ + the toolkit catalog, imported by\n` +
335 `// lib/docs.ts so eve bundles it into the deployed service (agent/ but not content/).\n` +
336 `export default JSON.parse(\n ${JSON.stringify(json)},\n) as unknown;\n`;
337 const out = join(ROOT, 'agent', 'lib', 'docs-index.ts');
338 writeFileSync(out, ts);
339 console.log(
340 `[build-agent-index] wrote ${pages.length} pages + ${toolkits.length} toolkits + ${search.entries.length} BM25 rows (${Math.round(json.length / 1024)} KB) -> ${out}`
341 );