Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 805 lines 26.1 KB
Raw
1 // Use direct imports from collections to avoid top-level await in lib/source.ts.
2 import { resolveToolkitCount } from './toolkit-count';
3 import { createHash } from 'node:crypto';
4 import { existsSync, readdirSync, readFileSync } from 'node:fs';
5 import { join, relative } from 'node:path';
6 import {
7 docs,
8 reference,
9 examples,
10 toolkits,
11 knowledgeBase,
12 } from 'fumadocs-mdx:collections/server';
13 import { loader, multiple } from 'fumadocs-core/source';
14 import { lucideIconsPlugin } from 'fumadocs-core/source/lucide-icons';
15 import { openapiSource, openapiPlugin } from 'fumadocs-openapi/server';
16 import type { AdvancedIndex } from 'fumadocs-core/search/server';
17 import type { BaseIndex } from 'fumadocs-core/search/algolia';
18 import { openapi } from '@/lib/openapi';
19 import { mdxToCleanMarkdown } from '@/lib/source';
20 import { isHiddenApiTagUrl } from '@/lib/filter-api-version';
21 import { getAllToolkitsSync } from '@/lib/toolkit-data';
22 import {
23 classifyKnowledgeRecord,
24 normalizeKnowledgeKeywords,
25 } from '@/lib/knowledge/metadata';
26 import { getAuthGuideSearchRecords } from '@/lib/knowledge/auth-guides';
27 import type {
28 KnowledgeIntent,
29 KnowledgeMetadata,
30 KnowledgeSourceType,
31 ProductAreaSlug,
32 } from '@/lib/knowledge/types';
33
34 export const ALGOLIA_DEFAULT_APP_ID = '62HI9PQZ1L';
35 export const ALGOLIA_DEFAULT_INDEX_NAME = 'docs_composio';
36
37 const MAX_CHUNK_CHARS = 3_800;
38 const MAX_CHUNK_BYTES = 9_000;
39 const MAX_TOOL_ALIAS_ITEMS = 80;
40 const MAX_TOOL_ALIAS_BYTES = 2_500;
41
42 // Create loaders directly here to avoid the problematic lib/source.ts import in the
43 // fallback route. This route is intentionally still Fumadocs/Orama-backed for local
44 // development when Algolia env vars aren't configured.
45 const docsSource = loader({
46 baseUrl: '/docs',
47 source: docs.toFumadocsSource(),
48 plugins: [lucideIconsPlugin()],
49 });
50
51 const examplesSource = loader({
52 baseUrl: '/examples',
53 source: examples.toFumadocsSource(),
54 plugins: [lucideIconsPlugin()],
55 });
56
57 const toolkitsSource = loader({
58 baseUrl: '/toolkits',
59 source: toolkits.toFumadocsSource(),
60 plugins: [lucideIconsPlugin()],
61 });
62
63 const knowledgeBaseSource = loader({
64 baseUrl: '/kb',
65 source: knowledgeBase.toFumadocsSource(),
66 plugins: [lucideIconsPlugin()],
67 });
68
69 type SearchIndex = AdvancedIndex & {
70 keywords?: string[];
71 };
72
73 export type AlgoliaDocsRecord = BaseIndex & KnowledgeMetadata & {
74 objectID: string;
75 description?: string;
76 keywords?: string[];
77 slug?: string;
78 headings?: string[];
79 tool_names?: string[];
80 tool_slugs?: string[];
81 type: string;
82 lang: string;
83 page_rank: number;
84 toolkit_popularity: number;
85 section_rank: number;
86 position: number;
87 depth: number;
88 tags?: string[];
89 };
90
91 function getFrontmatter(source: string): string {
92 return source.match(/^---\n([\s\S]*?)\n---/)?.[1] ?? '';
93 }
94
95 function getFrontmatterValue(frontmatter: string, key: string): string | undefined {
96 // The double-quoted alternative must accept YAML escapes: a title like
97 // `"Resolve an \"App is blocked\" error"` otherwise fails the quoted branch,
98 // falls through to the catch-all, and surfaces raw quotes-and-backslashes in
99 // search results and browse pages.
100 const match = frontmatter.match(
101 new RegExp(`^${key}:\\s*(?:"((?:[^"\\\\]|\\\\.)*)"|'([^']*)'|(.+))$`, 'm')
102 );
103 if (match?.[1] !== undefined) {
104 return match[1].replace(/\\(["\\])/g, '$1').trim();
105 }
106 const value = match?.[2] ?? match?.[3];
107 return value?.trim();
108 }
109
110 function getFrontmatterList(frontmatter: string, key: string): string[] {
111 const raw = getFrontmatterValue(frontmatter, key);
112 if (!raw) return [];
113
114 if (raw.startsWith('[') && raw.endsWith(']')) {
115 return raw
116 .slice(1, -1)
117 .split(',')
118 .map((value) => value.trim().replace(/^['"]|['"]$/g, ''))
119 .filter(Boolean);
120 }
121
122 return [raw];
123 }
124
125 function listContentFiles(dir: string): string[] {
126 if (!existsSync(dir)) return [];
127
128 return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
129 const path = join(dir, entry.name);
130 if (entry.isDirectory()) return listContentFiles(path);
131 if (entry.isFile() && /\.mdx?$/.test(entry.name)) return [path];
132 return [];
133 });
134 }
135
136 function slugify(value: string): string {
137 return value
138 .toLowerCase()
139 .trim()
140 .replace(/[<>]/g, '')
141 .replace(/[`*_~\[\]()]/g, '')
142 .replace(/&[a-z]+;/g, '')
143 .replace(/[^\w\s-]/g, '')
144 .replace(/\s+/g, '-')
145 .replace(/-+/g, '-')
146 .replace(/^-|-$/g, '');
147 }
148
149 function uniqueSlug(value: string, seen: Map<string, number>): string {
150 const base = slugify(value) || 'section';
151 const count = seen.get(base) ?? 0;
152 seen.set(base, count + 1);
153 return count === 0 ? base : `${base}-${count + 1}`;
154 }
155
156 function splitIntoChunks(text: string): string[] {
157 const paragraphs = text
158 .split(/\n{2,}/)
159 .map((paragraph) => paragraph.trim())
160 .filter(Boolean);
161
162 const chunks: string[] = [];
163 let current = '';
164
165 const pushCurrent = () => {
166 const trimmed = current.trim();
167 if (trimmed) chunks.push(trimmed);
168 current = '';
169 };
170
171 for (const paragraph of paragraphs) {
172 const next = current ? `${current}\n\n${paragraph}` : paragraph;
173 if (next.length <= MAX_CHUNK_CHARS && Buffer.byteLength(next, 'utf8') <= MAX_CHUNK_BYTES) {
174 current = next;
175 continue;
176 }
177
178 pushCurrent();
179
180 if (paragraph.length <= MAX_CHUNK_CHARS && Buffer.byteLength(paragraph, 'utf8') <= MAX_CHUNK_BYTES) {
181 current = paragraph;
182 continue;
183 }
184
185 for (let i = 0; i < paragraph.length; i += MAX_CHUNK_CHARS) {
186 chunks.push(paragraph.slice(i, i + MAX_CHUNK_CHARS).trim());
187 }
188 }
189
190 pushCurrent();
191 return chunks;
192 }
193
194 function contentHash(value: string): string {
195 return createHash('sha1').update(value).digest('hex').slice(0, 10);
196 }
197
198 // Sections intentionally kept out of search entirely (not just deprioritized).
199 // Direct tool execution is the legacy pre-sessions flow — we don't want it
200 // surfacing in results at all. Matches the prefix and anything beneath it.
201 const SEARCH_EXCLUDED_PREFIXES = ['/docs/tools-direct'];
202
203 function isExcludedFromSearch(url: string): boolean {
204 const path = url.replace(/\/$/, '');
205 return SEARCH_EXCLUDED_PREFIXES.some((prefix) => path === prefix || path.startsWith(`${prefix}/`));
206 }
207
208 function urlFromContentPath(path: string): { url: string; type: string } | undefined {
209 const rel = relative(join(process.cwd(), 'content'), path).replace(/\\/g, '/');
210 const withoutExt = rel.replace(/\.mdx?$/, '');
211 const parts = withoutExt.split('/');
212 const collection = parts.shift();
213 if (!collection) return undefined;
214
215 if (collection === 'docs') {
216 return { url: `/docs/${parts.join('/')}`.replace(/\/index$/, ''), type: 'docs' };
217 }
218 if (collection === 'examples') {
219 return { url: `/examples/${parts.join('/')}`.replace(/\/index$/, ''), type: 'examples' };
220 }
221 if (collection === 'reference') {
222 const url = `/reference/${parts.join('/')}`.replace(/\/index$/, '');
223 return { url, type: url.startsWith('/reference/v3') ? 'v3-reference' : 'reference' };
224 }
225 if (collection === 'toolkits') {
226 if (parts[0] === 'faq') return undefined;
227 return { url: `/toolkits/${parts.join('/')}`.replace(/\/index$/, ''), type: 'toolkits' };
228 }
229 if (collection === 'kb') {
230 return { url: `/kb/${parts.join('/')}`.replace(/\/index$/, ''), type: 'kb' };
231 }
232 if (collection === 'changelog') return undefined;
233
234 return undefined;
235 }
236
237 function slugTokens(url: string): string {
238 return url
239 .split(/[/-]/)
240 .map((part) => part.replace(/[_#?=&.]/g, ' '))
241 .join(' ')
242 .replace(/\s+/g, ' ')
243 .trim();
244 }
245
246 const typeLabels: Record<string, string> = {
247 docs: 'Docs',
248 examples: 'Example',
249 reference: 'Reference',
250 'v3-reference': 'Legacy v3 Reference',
251 toolkits: 'Toolkit',
252 kb: 'Knowledge Base',
253 changelog: 'Changelog',
254 'api-reference': 'API Reference',
255 };
256
257 function titleizeSlug(value: string): string {
258 return value
259 .split('-')
260 .filter(Boolean)
261 .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
262 .join(' ')
263 .replace(/\bApi\b/g, 'API')
264 .replace(/\bMcp\b/g, 'MCP')
265 .replace(/\bSdk\b/g, 'SDK')
266 .replace(/\bOauth\b/g, 'OAuth');
267 }
268
269 function breadcrumbsForUrl(url: string, type: string): string[] {
270 const label = typeLabels[type] ?? titleizeSlug(type);
271
272 if (type === 'toolkits' || type === 'examples' || type === 'changelog') {
273 return [label];
274 }
275
276 const parts = url.split('/').filter(Boolean);
277 const parentParts = parts.slice(1, -1);
278 return [label, ...parentParts.map(titleizeSlug)].filter(Boolean);
279 }
280
281 const TOOLKIT_POPULARITY_OVERRIDES: Record<string, number> = {
282 gmail: 1_000,
283 github: 980,
284 slack: 960,
285 googledrive: 940,
286 googlecalendar: 930,
287 notion: 920,
288 linear: 900,
289 jira: 890,
290 hubspot: 860,
291 salesforce: 850,
292 resend: 650,
293 zoho_mail: 520,
294 mailchimp: 500,
295 sendgrid: 480,
296 mailsoftly: 260,
297 };
298
299 function getToolkitSlugFromUrl(url: string): string | null {
300 return url.match(/^\/toolkits\/([^/#?]+)/)?.[1] ?? null;
301 }
302
303 function toolkitPopularity(url: string, type: string): number {
304 if (type !== 'toolkits') return 0;
305
306 const slug = getToolkitSlugFromUrl(url);
307 if (!slug) return 0;
308
309 const toolkit = toolkitBySlug.get(slug);
310 const override = TOOLKIT_POPULARITY_OVERRIDES[slug] ?? 0;
311 const managedAuthBoost = toolkit?.composioManagedAuthSchemes?.length ? 120 : 0;
312 const authBoost = toolkit?.authSchemes?.includes('OAUTH2') ? 40 : 0;
313 const triggerBoost = Math.min((toolkit?.triggerCount ?? 0) * 10, 80);
314 const toolCountBoost = Math.min((toolkit?.toolCount ?? 0), 100);
315
316 return override + managedAuthBoost + authBoost + triggerBoost + toolCountBoost;
317 }
318
319 function pageRank(url: string, sourceType: KnowledgeSourceType): number {
320 // Prefer conceptual docs over generated/reference material when textual
321 // relevance is otherwise close. Toolkit aliases can still win earlier via
322 // searchableAttributes when the query matches a tool name/slug exactly.
323 //
324 // Hints use precise path matches against the current (nested) docs structure.
325 // When pages move, update these — a stale `.includes()` hint silently boosts
326 // nothing. See content/docs/ for the canonical layout.
327 if (sourceType === 'docs') {
328 const path = url.replace(/\/$/, '');
329 if (path === '/docs') return 2_400;
330 if (path === '/docs/quickstart') return 2_300;
331 if (path === '/docs/how-composio-works') return 2_250;
332 if (path === '/docs/authentication') return 2_220;
333 if (path.startsWith('/docs/tools-direct/')) return 2_180;
334 if (path === '/docs/configuring-sessions') return 2_120;
335 if (path.startsWith('/docs/auth-configuration/')) return 2_080;
336 if (path === '/docs/triggers' || path.startsWith('/docs/setting-up-triggers/')) return 2_060;
337 return 2_000;
338 }
339
340 if (sourceType === 'kb') return 1_900;
341 if (sourceType === 'oauth-guide') return 1_700;
342 if (sourceType === 'toolkit') return 1_500;
343 if (sourceType === 'example') return 1_300;
344 if (sourceType === 'reference') return 700;
345 if (sourceType === 'changelog') return 350;
346 if (sourceType === 'legacy') return 25;
347 return 400;
348 }
349
350 function knowledgeSourceType(type: string, legacy: boolean): KnowledgeSourceType {
351 if (legacy || type === 'v3-reference') return 'legacy';
352 if (type === 'docs') return 'docs';
353 if (type === 'kb') return 'kb';
354 if (type === 'toolkits') return 'toolkit';
355 if (type === 'examples') return 'example';
356 if (type === 'reference' || type === 'api-reference') return 'reference';
357 if (type === 'changelog') return 'changelog';
358 throw new Error(`Unsupported search source type: ${type}`);
359 }
360
361 export function recordsFromMarkdownPage(input: {
362 url: string;
363 type: string;
364 title: string;
365 description?: string;
366 keywords?: string[];
367 markdown: string;
368 breadcrumbs?: string[];
369 tags?: string[];
370 toolNames?: string[];
371 toolSlugs?: string[];
372 legacy?: boolean;
373 topics?: string[];
374 productAreas?: ProductAreaSlug[];
375 toolkitSlugs?: string[];
376 intents?: KnowledgeIntent[];
377 lastVerifiedAt?: string | null;
378 }): AlgoliaDocsRecord[] {
379 const isLegacy = input.legacy === true;
380 const sourceType = knowledgeSourceType(input.type, isLegacy);
381 const metadata = classifyKnowledgeRecord({
382 sourceType,
383 canonicalUrl: input.url,
384 productAreas: input.productAreas,
385 topics: input.topics,
386 toolkitSlugs: input.toolkitSlugs,
387 intents: input.intents,
388 lastVerifiedAt: input.lastVerifiedAt,
389 });
390 const resolvedPageRank = pageRank(input.url, sourceType);
391 const resolvedTags = isLegacy ? [...(input.tags ?? []), 'legacy'] : input.tags;
392 const resolvedKeywords = normalizeKnowledgeKeywords(input.keywords ?? []);
393 const clean = mdxToCleanMarkdown(input.markdown, input.url);
394 const lines = clean.split('\n');
395 const headingSlugs = new Map<string, number>();
396 const headings: string[] = [];
397 const sections: Array<{ heading?: string; section_id?: string; depth: number; text: string; position: number }> = [];
398 let currentHeading: string | undefined;
399 let currentSectionId: string | undefined;
400 let currentDepth = 0;
401 let currentLines: string[] = [];
402 let sectionPosition = 0;
403 let fenceMarker: string | null = null;
404 let fenceLength = 0;
405
406 const flush = () => {
407 const text = currentLines.join('\n').trim();
408 if (!text) return;
409 sections.push({
410 heading: currentHeading,
411 section_id: currentSectionId,
412 depth: currentDepth,
413 text,
414 position: sectionPosition++,
415 });
416 currentLines = [];
417 };
418
419 for (const line of lines) {
420 const fenceMatch = line.match(/^\s{0,3}(`{3,}|~{3,})(.*)$/);
421 if (fenceMatch) {
422 const marker = fenceMatch[1];
423 const markerCharacter = marker[0];
424 const suffix = fenceMatch[2];
425
426 if (!fenceMarker) {
427 fenceMarker = markerCharacter;
428 fenceLength = marker.length;
429 } else if (
430 markerCharacter === fenceMarker
431 && marker.length >= fenceLength
432 && suffix.trim() === ''
433 ) {
434 fenceMarker = null;
435 fenceLength = 0;
436 }
437
438 currentLines.push(line);
439 continue;
440 }
441
442 const headingMatch = fenceMarker ? null : line.match(/^(#{1,6})\s+(.+)$/);
443 if (headingMatch) {
444 flush();
445 const depth = headingMatch[1].length;
446 const heading = headingMatch[2].trim();
447 currentHeading = heading;
448 currentSectionId = uniqueSlug(heading, headingSlugs);
449 currentDepth = depth;
450 headings.push(heading);
451 currentLines.push(line);
452 continue;
453 }
454
455 currentLines.push(line);
456 }
457 flush();
458
459 const fallbackText = clean.trim() || [input.title, input.description, ...resolvedKeywords].filter(Boolean).join('\n');
460 if (sections.length === 0 && fallbackText) {
461 sections.push({ text: fallbackText, position: 0, depth: 0 });
462 }
463
464 let recordPosition = 0;
465 return sections.flatMap((section) => {
466 const chunks = splitIntoChunks(section.text);
467 return chunks.map((chunk, chunkIndex) => {
468 const sectionPart = section.section_id ?? 'overview';
469 const objectID = `${input.url}__${sectionPart}__${chunkIndex}__${contentHash(chunk)}`;
470 const sectionRank = Math.max(10, 120 - section.depth * 12 - chunkIndex * 2);
471
472 const position = recordPosition++;
473 const includeToolkitAliases = input.type === 'toolkits' && position === 0;
474
475 return {
476 objectID,
477 title: input.title,
478 description: input.description,
479 breadcrumbs: input.breadcrumbs,
480 url: input.url,
481 page_id: input.url,
482 section: section.heading,
483 section_id: section.section_id,
484 content: chunk,
485 keywords: resolvedKeywords,
486 slug: slugTokens(input.url),
487 headings,
488 tool_names: includeToolkitAliases ? input.toolNames : undefined,
489 tool_slugs: includeToolkitAliases ? input.toolSlugs : undefined,
490 type: input.type,
491 lang: 'en',
492 tags: resolvedTags,
493 page_rank: resolvedPageRank,
494 toolkit_popularity: toolkitPopularity(
495 input.url,
496 sourceType === 'toolkit' ? 'toolkits' : input.type,
497 ),
498 section_rank: sectionRank,
499 position,
500 depth: section.depth,
501 ...metadata,
502 } satisfies AlgoliaDocsRecord;
503 });
504 });
505 }
506
507 const toolkitBySlug = new Map(getAllToolkitsSync().map((toolkit) => [toolkit.slug, toolkit]));
508
509 function limitToolkitAliases(values: string[]): string[] {
510 const aliases: string[] = [];
511 let bytes = 0;
512
513 for (const value of values) {
514 const trimmed = value.trim();
515 if (!trimmed) continue;
516
517 const nextBytes = Buffer.byteLength(JSON.stringify(trimmed), 'utf8');
518 if (aliases.length >= MAX_TOOL_ALIAS_ITEMS || bytes + nextBytes > MAX_TOOL_ALIAS_BYTES) break;
519
520 aliases.push(trimmed);
521 bytes += nextBytes;
522 }
523
524 return aliases;
525 }
526
527 function getToolkitSearchFields(slug: string): Pick<Parameters<typeof recordsFromMarkdownPage>[0], 'toolNames' | 'toolSlugs' | 'tags'> {
528 const toolkit = toolkitBySlug.get(slug);
529 if (!toolkit) return {};
530
531 return {
532 toolNames: limitToolkitAliases(toolkit.tools.map((tool) => tool.name).filter(Boolean)),
533 toolSlugs: limitToolkitAliases(toolkit.tools.map((tool) => tool.slug).filter(Boolean)),
534 tags: [toolkit.category].filter(Boolean) as string[],
535 };
536 }
537
538 function getFilesystemRecords(): AlgoliaDocsRecord[] {
539 const contentDir = join(process.cwd(), 'content');
540 return listContentFiles(contentDir).flatMap((file) => {
541 const route = urlFromContentPath(file);
542 if (!route) return [];
543 if (isExcludedFromSearch(route.url)) return [];
544
545 const source = resolveToolkitCount(readFileSync(file, 'utf8'));
546 const frontmatter = getFrontmatter(source);
547 const title = getFrontmatterValue(frontmatter, 'title');
548 if (!title) return [];
549
550 const keywords = getFrontmatterList(frontmatter, 'keywords');
551 const topics = getFrontmatterList(frontmatter, 'topics');
552 const explicitToolkitSlugs = getFrontmatterList(frontmatter, 'toolkitSlugs');
553 const taggedToolkitSlugs = route.type === 'kb' && topics.includes('toolkits')
554 ? keywords.filter((keyword) => toolkitBySlug.has(keyword))
555 : [];
556 const routeToolkitSlug = route.type === 'toolkits'
557 ? route.url.replace(/^\/toolkits\//, '').split('/')[0]
558 : null;
559
560 const toolkitFields = route.type === 'toolkits'
561 ? getToolkitSearchFields(route.url.replace(/^\/toolkits\//, ''))
562 : {};
563
564 const legacy =
565 getFrontmatterValue(frontmatter, 'legacy') === 'true' ||
566 getFrontmatterValue(frontmatter, 'deprecated') === 'true';
567
568 return recordsFromMarkdownPage({
569 url: route.url,
570 type: route.type,
571 title,
572 description: getFrontmatterValue(frontmatter, 'description'),
573 keywords,
574 markdown: source,
575 breadcrumbs: breadcrumbsForUrl(route.url, route.type),
576 legacy,
577 topics,
578 productAreas: getFrontmatterList(frontmatter, 'productAreas').filter(
579 (area): area is ProductAreaSlug => [
580 'authentication-and-connected-accounts',
581 'tools-actions-and-execution',
582 'triggers-and-workflows',
583 'sdk-api-and-mcp',
584 'account-billing-and-security',
585 ].includes(area),
586 ),
587 toolkitSlugs: [
588 ...explicitToolkitSlugs,
589 ...taggedToolkitSlugs,
590 ...(routeToolkitSlug ? [routeToolkitSlug] : []),
591 ],
592 intents: getFrontmatterList(frontmatter, 'intents') as KnowledgeIntent[],
593 lastVerifiedAt: getFrontmatterValue(frontmatter, 'lastVerifiedAt'),
594 ...toolkitFields,
595 });
596 });
597 }
598
599 // Dynamic toolkit entries from toolkits.json. We include a compact sample of tool
600 // names/slugs so queries like "gmail send email" can still find the Gmail toolkit
601 // page without indexing the full tool catalog as noisy giant records.
602 const mdxToolkitSlugs = new Set(
603 toolkitsSource.getPages().map((page) => page.slugs.join('/')),
604 );
605
606 const dynamicToolkitIndexes = getAllToolkitsSync()
607 .filter((toolkit) => !mdxToolkitSlugs.has(toolkit.slug))
608 .map((toolkit) => ({
609 id: `/toolkits/${toolkit.slug}`,
610 title: toolkit.name,
611 description: toolkit.description,
612 url: `/toolkits/${toolkit.slug}`,
613 structuredData: { headings: [], contents: [] },
614 keywords: [toolkit.slug, toolkit.category].filter(Boolean) as string[],
615 } satisfies SearchIndex));
616
617 function getDynamicToolkitRecords(): AlgoliaDocsRecord[] {
618 return getAllToolkitsSync()
619 .filter((toolkit) => !mdxToolkitSlugs.has(toolkit.slug))
620 .flatMap((toolkit) => {
621 const toolsText = (toolkit.tools ?? [])
622 .slice(0, 60)
623 .map((tool) => `${tool.slug ?? ''} ${tool.name ?? ''}`)
624 .join('\n');
625
626 return recordsFromMarkdownPage({
627 url: `/toolkits/${toolkit.slug}`,
628 type: 'toolkits',
629 title: toolkit.name,
630 description: toolkit.description,
631 keywords: [toolkit.slug, toolkit.category].filter(Boolean) as string[],
632 markdown: `# ${toolkit.name}\n\n${toolkit.description ?? ''}\n\n## Available tools\n\n${toolsText}`,
633 breadcrumbs: breadcrumbsForUrl(`/toolkits/${toolkit.slug}`, 'toolkits'),
634 toolkitSlugs: [toolkit.slug],
635 ...getToolkitSearchFields(toolkit.slug),
636 });
637 });
638 }
639
640 function getChangelogIndexes(): SearchIndex[] {
641 const changelogDir = join(process.cwd(), 'content/changelog');
642 if (!existsSync(changelogDir)) return [];
643
644 return readdirSync(changelogDir)
645 .filter((file) => file.endsWith('.mdx'))
646 .flatMap((file) => {
647 const source = readFileSync(join(changelogDir, file), 'utf8');
648 const frontmatter = getFrontmatter(source);
649 const date = getFrontmatterValue(frontmatter, 'date');
650 const title = getFrontmatterValue(frontmatter, 'title');
651 if (!date || !title) return [];
652
653 const url = `/docs/changelog/${date.replace(/-/g, '/')}`;
654
655 return [{
656 id: `${url}#${title}`,
657 title,
658 description: getFrontmatterValue(frontmatter, 'description') ?? '',
659 url,
660 structuredData: { headings: [], contents: [] },
661 keywords: ['changelog'],
662 } satisfies SearchIndex];
663 });
664 }
665
666 function getChangelogRecords(): AlgoliaDocsRecord[] {
667 const changelogDir = join(process.cwd(), 'content/changelog');
668 if (!existsSync(changelogDir)) return [];
669
670 return readdirSync(changelogDir)
671 .filter((file) => file.endsWith('.mdx'))
672 .flatMap((file) => {
673 const source = readFileSync(join(changelogDir, file), 'utf8');
674 const frontmatter = getFrontmatter(source);
675 const date = getFrontmatterValue(frontmatter, 'date');
676 const title = getFrontmatterValue(frontmatter, 'title');
677 if (!date || !title) return [];
678
679 return recordsFromMarkdownPage({
680 url: `/docs/changelog/${date.replace(/-/g, '/')}`,
681 type: 'changelog',
682 title,
683 description: getFrontmatterValue(frontmatter, 'description'),
684 keywords: ['changelog', date],
685 markdown: source,
686 breadcrumbs: breadcrumbsForUrl(`/docs/changelog/${date.replace(/-/g, '/')}`, 'changelog'),
687 });
688 });
689 }
690
691 export async function getDocsSearchIndexes(): Promise<SearchIndex[]> {
692 // Load OpenAPI pages and build full reference source.
693 const openapiPages = await openapiSource(openapi, {
694 groupBy: 'tag',
695 baseDir: 'api-reference',
696 });
697
698 const fullReferenceSource = loader({
699 baseUrl: '/reference',
700 source: multiple({
701 mdx: reference.toFumadocsSource(),
702 openapi: openapiPages,
703 }),
704 plugins: [lucideIconsPlugin(), openapiPlugin()],
705 });
706
707 const mdxIndexes = await Promise.all([
708 ...docsSource.getPages(),
709 ...examplesSource.getPages(),
710 ...toolkitsSource.getPages(),
711 ...knowledgeBaseSource.getPages(),
712 ...fullReferenceSource.getPages(),
713 ].filter((page) =>
714 !isExcludedFromSearch(page.url) && !isHiddenApiTagUrl(page.url)
715 ).map(async (page) => {
716 // Keep section-linked prose and add code and custom-component instructions
717 // omitted by structured MDX search data.
718 const content = 'getText' in page.data
719 ? mdxToCleanMarkdown(await page.data.getText('raw'), page.url)
720 : undefined;
721 return {
722 id: page.url,
723 title: page.data.title ?? 'Untitled',
724 description: page.data.description,
725 url: page.url,
726 structuredData: content === undefined
727 ? page.data.structuredData
728 : {
729 ...page.data.structuredData,
730 contents: [
731 ...(page.data.structuredData?.contents ?? []),
732 { heading: undefined, content },
733 ],
734 },
735 keywords: 'keywords' in page.data ? (page.data.keywords as string[]) : undefined,
736 } satisfies SearchIndex;
737 }));
738 const oauthIndexes = getAuthGuideSearchRecords().map((record) => ({
739 id: record.page_id,
740 title: record.title,
741 description: record.description,
742 url: record.canonical_url,
743 structuredData: {
744 headings: [],
745 contents: [{ heading: undefined, content: record.content }],
746 },
747 keywords: record.keywords,
748 } satisfies SearchIndex));
749
750 // Migrated auth guides share the toolkit page, so keep one local search hit.
751 const oauthUrls = new Set(oauthIndexes.map(index => index.url));
752 return [
753 ...mdxIndexes,
754 ...dynamicToolkitIndexes.filter(index => !oauthUrls.has(index.url)),
755 ...getChangelogIndexes(),
756 ...oauthIndexes,
757 ];
758 }
759
760 async function getOpenApiRecords(): Promise<AlgoliaDocsRecord[]> {
761 const openapiPages = await openapiSource(openapi, {
762 groupBy: 'tag',
763 baseDir: 'api-reference',
764 });
765
766 const openapiOnlySource = loader({
767 baseUrl: '/reference',
768 source: openapiPages,
769 plugins: [lucideIconsPlugin(), openapiPlugin()],
770 });
771
772 return openapiOnlySource.getPages().filter((page) =>
773 !isHiddenApiTagUrl(page.url)
774 ).flatMap((page) => {
775 const contents = [
776 page.data.description,
777 ...(page.data.structuredData?.headings ?? []).map((heading) => heading.content),
778 ...(page.data.structuredData?.contents ?? []).map((content) => content.content),
779 ].filter(Boolean).join('\n\n');
780
781 return recordsFromMarkdownPage({
782 url: page.url,
783 type: 'api-reference',
784 title: page.data.title ?? 'Untitled',
785 description: page.data.description,
786 markdown: `# ${page.data.title ?? 'Untitled'}\n\n${contents}`,
787 breadcrumbs: breadcrumbsForUrl(page.url, 'api-reference'),
788 });
789 });
790 }
791
792 export async function getAlgoliaSearchDocuments(): Promise<AlgoliaDocsRecord[]> {
793 const records = [
794 ...getFilesystemRecords(),
795 ...getDynamicToolkitRecords(),
796 ...getChangelogRecords(),
797 ...getAuthGuideSearchRecords(),
798 ...await getOpenApiRecords(),
799 ];
800
801 const deduped = new Map<string, AlgoliaDocsRecord>();
802 for (const record of records) deduped.set(record.objectID, record);
803
804 return Array.from(deduped.values());
805 }