Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 749 lines 26.8 KB
Raw
1 import {
2 docs,
3 reference,
4 examples,
5 toolkits,
6 knowledgeBase,
7 changelog,
8 } from 'fumadocs-mdx:collections/server';
9 import type { DocCollectionEntry } from 'fumadocs-mdx/runtime/server';
10 import type { Folder, Node } from 'fumadocs-core/page-tree';
11 import {
12 type ContentStorage,
13 type InferPageType,
14 loader,
15 multiple,
16 } from 'fumadocs-core/source';
17 import { lucideIconsPlugin } from 'fumadocs-core/source/lucide-icons';
18 import { openapi, openapiV3 } from './openapi';
19 import { openapiSource, openapiPlugin } from 'fumadocs-openapi/server';
20 import { getGuardrails } from './llm-guardrails';
21 import { isHiddenApiTagUrl } from './filter-api-version';
22 import { FILE_BUILDS } from './file-builds';
23 import { replaceRepoBrowserMarkdown } from './repo-browser-markdown';
24 import { transformDeprecatedApiSidebarNode } from './deprecated-api-sidebar';
25 import { API_BASE_URLS, detectApiVersion, type ApiVersion } from './api-version';
26 import { apiVersionPointer } from './api-version-guidance';
27 import { apiEndpointsSchema } from './api-endpoints-table-schema';
28 import { replaceHomeNavigationMarkdown } from './home-navigation';
29 import { PACKAGE_MANAGERS } from './package-install';
30 import { z } from 'zod';
31 import { promptFor, SETUP_PROMPT } from './agent-prompts';
32 import { AGENTS } from './agent-setup-clients';
33 import { HOME_OG_DESCRIPTION, resolveToolkitCount } from './toolkit-count';
34 import { componentsToMarkdown } from './example-document';
35 import { providerListMarkdown } from './providers';
36
37 /**
38 * True if a reference URL belongs to an intentionally-hidden API tag
39 * (consumer, invite-codes) in either v3.1 or v3.0. These tags exist in the
40 * upstream OpenAPI spec but are hidden on our side. The page tree is filtered
41 * via `prepareTree` (lib/filter-api-version.ts); this mirror keeps the flat
42 * `getPages()` list (consumed by validate-links, llms.mdx, sitemap) in sync.
43 */
44 export const source = loader({
45 baseUrl: '/docs',
46 source: docs.toFumadocsSource(),
47 plugins: [lucideIconsPlugin()],
48 });
49
50 function loadOpenapiPages() {
51 return Promise.all([
52 openapiSource(openapi, { groupBy: 'tag', baseDir: 'api-reference' }),
53 openapiSource(openapiV3, { groupBy: 'tag', baseDir: 'v3/api-reference' }),
54 ]);
55 }
56
57 type OpenapiPages = Awaited<ReturnType<typeof loadOpenapiPages>>;
58
59 const API_METHOD_ORDER: Partial<Record<string, number>> = {
60 get: 0, // read
61 post: 1, // create
62 patch: 2, // update
63 put: 2, // update
64 delete: 3,
65 };
66
67 const openApiPageDataSchema = z.object({
68 _openapi: z.object({ method: z.string() }),
69 });
70
71 /** Orders generated OpenAPI siblings while leaving authored sidebar items in place. */
72 function orderApiOperations(folder: Folder, storage: ContentStorage): Folder {
73 const ranks = new Map<Node, number>();
74 for (const node of folder.children) {
75 if (node.type !== 'page' || !node.$ref) continue;
76 const file = storage.read(node.$ref);
77 if (!file || file.format !== 'page') continue;
78
79 const parsed = openApiPageDataSchema.safeParse(file.data);
80 if (parsed.success) ranks.set(node, API_METHOD_ORDER[parsed.data._openapi.method] ?? 4);
81 }
82
83 const ordered = [...ranks].sort((a, b) => a[1] - b[1]).map(([node]) => node);
84 let index = 0;
85
86 return {
87 ...folder,
88 children: folder.children.map(node => (ranks.has(node) ? ordered[index++] : node)),
89 };
90 }
91
92 // One combined reference source with both v3.1 and v3.0 OpenAPI pages.
93 // v3.1 at api-reference/, v3.0 at api-reference/v3/
94 let _openapiPagesPromise: ReturnType<typeof loadOpenapiPages> | null = null;
95
96 async function getOpenapiPages() {
97 if (!_openapiPagesPromise) {
98 _openapiPagesPromise = loadOpenapiPages().catch(e => {
99 // Don't permanently cache a failed load (e.g. a transient OpenAPI spec
100 // resolution error in a serverless instance). Clearing the memo lets the
101 // next request retry instead of re-throwing the same cached rejection.
102 _openapiPagesPromise = null;
103 throw e;
104 });
105 }
106 return _openapiPagesPromise;
107 }
108
109 function createReferenceSource(openapiLatest: OpenapiPages[0], openapiV3Pages: OpenapiPages[1]) {
110 const loaded = loader({
111 baseUrl: '/reference',
112 source: multiple({
113 mdx: reference.toFumadocsSource(),
114 openapi: openapiLatest,
115 'openapi-v3': openapiV3Pages,
116 }),
117 plugins: [lucideIconsPlugin(), openapiPlugin()],
118 pageTree: {
119 transformers: [
120 {
121 folder(node, folderPath) {
122 node = orderApiOperations(node, this.storage);
123 if (
124 folderPath === 'api-reference' ||
125 folderPath === 'sdk-reference' ||
126 folderPath === 'v3/api-reference'
127 ) {
128 return { ...node, defaultOpen: true };
129 }
130 return node;
131 },
132 },
133 {
134 file(node, filePath) {
135 return transformDeprecatedApiSidebarNode(node, filePath, this.storage);
136 },
137 },
138 ],
139 },
140 });
141
142 // Exclude intentionally-hidden API tags (consumer, invite-codes) from the
143 // flat page list so validate-links, llms.mdx, llms.txt, and sitemap skip
144 // their fumadocs-openapi operation pages. The sidebar tree is filtered
145 // separately via prepareTree (lib/filter-api-version.ts).
146 const originalGetPages = loaded.getPages.bind(loaded);
147 loaded.getPages = (...args: Parameters<typeof originalGetPages>) =>
148 originalGetPages(...args).filter((page: { url: string }) => !isHiddenApiTagUrl(page.url));
149
150 return loaded;
151 }
152
153 type ReferenceSource = ReturnType<typeof createReferenceSource>;
154 let _referenceSource: ReferenceSource | null = null;
155
156 export async function getReferenceSource() {
157 if (!_referenceSource) {
158 const [openapiLatest, openapiV3Pages] = await getOpenapiPages();
159 _referenceSource = createReferenceSource(openapiLatest, openapiV3Pages);
160 }
161 return _referenceSource;
162 }
163
164 // Synchronous reference source for cases where OpenAPI isn't needed
165 export const referenceSource = loader({
166 baseUrl: '/reference',
167 source: reference.toFumadocsSource(),
168 plugins: [lucideIconsPlugin()],
169 });
170
171 export type ReferenceMdxPageData = InferPageType<typeof referenceSource>['data'];
172
173 export const examplesSource = loader({
174 baseUrl: '/examples',
175 source: examples.toFumadocsSource(),
176 plugins: [lucideIconsPlugin()],
177 });
178
179 export const toolkitsSource = loader({
180 baseUrl: '/toolkits',
181 source: toolkits.toFumadocsSource(),
182 plugins: [lucideIconsPlugin()],
183 });
184
185 export const knowledgeBaseSource = loader({
186 baseUrl: '/kb',
187 source: knowledgeBase.toFumadocsSource(),
188 plugins: [lucideIconsPlugin()],
189 });
190
191 export type ChangelogEntry = DocCollectionEntry<
192 'changelog',
193 {
194 date: string;
195 title: string;
196 description?: string;
197 icon?: string;
198 full?: boolean;
199 }
200 >;
201
202 // The generated Fumadocs virtual module is untyped in Next's production
203 // checker. Preserve the collection's public shape for all route consumers.
204 export const changelogEntries = changelog as ChangelogEntry[];
205
206 export interface OgImageExtras {
207 /** Toolkit logo URL. Only https://logos.composio.dev and https://assets.composio.dev are rendered. */
208 logo?: string | null;
209 /** Changelog date, already formatted for display. */
210 date?: string | null;
211 /** API reference version label, e.g. "v3.1". */
212 version?: string | null;
213 }
214
215 const OG_ROUTE = 'https://docs.composio.dev/api/og';
216
217 export function getOgImageUrl(
218 section: string,
219 slugs: string[],
220 title?: string,
221 description?: string,
222 extras: OgImageExtras = {}
223 ): string {
224 const params = new URLSearchParams();
225 if (section === 'docs' && slugs.length === 0) {
226 // The home card has fixed copy; the page's own description is ignored so
227 // the root layout and the /docs index page produce the same image URL.
228 params.set('section', 'home');
229 params.set('description', HOME_OG_DESCRIPTION);
230 return `${OG_ROUTE}?${params.toString()}`;
231 }
232 const isChangelog = section === 'docs' && slugs[0] === 'changelog';
233 params.set('section', isChangelog ? 'changelog' : section);
234 params.set('title', title ?? 'Composio Docs');
235 if (description) params.set('description', description);
236 for (const key of ['logo', 'date', 'version'] as const) {
237 const value = extras[key];
238 if (value) params.set(key, value);
239 }
240 return `${OG_ROUTE}?${params.toString()}`;
241 }
242
243 /**
244 * `<ApiEndpointsTable />` reaches this converter in two different shapes.
245 *
246 * `getLLMText` reads fumadocs' *processed* markdown, which re-serializes the
247 * JSX expression attribute as a quoted string with the inner quotes escaped:
248 *
249 * <ApiEndpointsTable endpoints="[{&#x22;method&#x22;:&#x22;GET&#x22;, ...}]" />
250 *
251 * while `lib/search-index.ts` passes the raw file content, which keeps the
252 * authored form:
253 *
254 * <ApiEndpointsTable endpoints={[{"method":"GET", ...}]} />
255 *
256 * Matching only the authored form is what left the Endpoints section empty on
257 * every live tag page, so both are matched here. Braces are not escaped in the
258 * processed form, and every inner `"` is — so a non-greedy match to the next
259 * unescaped quote is exact.
260 */
261 const API_ENDPOINTS_TABLE_REGEX =
262 /<ApiEndpointsTable\s+endpoints=(?:\{([\s\S]*?)\}\s*\/>|"([\s\S]*?)"\s*\/>)/g;
263
264 /** Reverses the entity escaping fumadocs applies to JSX attribute values. */
265 function decodeHtmlEntities(value: string): string {
266 return value
267 .replace(/&#x([0-9a-fA-F]+);/g, (_, hex: string) =>
268 String.fromCodePoint(Number.parseInt(hex, 16))
269 )
270 .replace(/&#(\d+);/g, (_, dec: string) => String.fromCodePoint(Number.parseInt(dec, 10)))
271 .replace(/&quot;/g, '"')
272 .replace(/&apos;/g, "'")
273 .replace(/&lt;/g, '<')
274 .replace(/&gt;/g, '>')
275 .replace(/&amp;/g, '&');
276 }
277
278 const packageInstallSchema = z.object({
279 packages: z.string().min(1),
280 ecosystem: z.enum(['node', 'python']).default('node'),
281 comment: z.array(z.string()).default([]),
282 });
283
284 /** Only literal attributes are supported; MDX expressions are never evaluated. */
285 function packageInstallToMarkdown(attributes: string): string {
286 const quoted = (name: string) => {
287 const match = attributes.match(new RegExp(`\\b${name}=(?:"([^"]*)"|'([^']*)')`));
288 return match ? decodeHtmlEntities(match[1] ?? match[2]) : undefined;
289 };
290 const comment = quoted('comment') ?? attributes.match(/\bcomment=\{(\[[\s\S]*?\])\}/)?.[1];
291 const comments = comment?.trim().startsWith('[')
292 ? Array.from(comment.matchAll(/(['"])((?:\\.|(?!\1)[^\\])*)\1/g), match =>
293 match[2].replace(/\\(['"\\])/g, '$1'),
294 )
295 : comment ? [comment] : [];
296 const props = packageInstallSchema.parse({
297 packages: quoted('packages'),
298 ecosystem: quoted('ecosystem'),
299 comment: comments,
300 });
301 return PACKAGE_MANAGERS[props.ecosystem].map(manager =>
302 `\n**${manager.id}:**\n\n\`\`\`bash\n${manager.install} ${props.packages}${props.comment.map(line => `\n# ${line}`).join('')}\n\`\`\`\n`,
303 ).join('\n');
304 }
305
306 /**
307 * Renders an `<ApiEndpointsTable />` payload as a markdown table.
308 *
309 * Degrades rather than throws: a malformed payload emits nothing for that one
310 * table and warns, because taking the whole `.md` response down over one bad
311 * page is worse. The failure signal for checked-in content lives in the static
312 * suite instead — `tests/static/api-reference-routes.test.ts` runs every
313 * committed payload through the same schema, and
314 * `scripts/generate-api-index.ts` refuses to write an invalid one.
315 */
316 function endpointsTableToMarkdown(payload: string, version: ApiVersion, url?: string): string {
317 const where = url ?? '(no page url)';
318
319 let json: unknown;
320 try {
321 json = JSON.parse(payload);
322 } catch {
323 console.warn(`[mdxToCleanMarkdown] unparseable ApiEndpointsTable payload on ${where}`);
324 return '';
325 }
326
327 const parsed = apiEndpointsSchema.safeParse(json);
328 if (!parsed.success) {
329 console.warn(`[mdxToCleanMarkdown] invalid ApiEndpointsTable payload on ${where}`);
330 return '';
331 }
332
333 const rows = parsed.data.map(endpoint => {
334 const path = version === '3.0' ? endpoint.pathV3 : endpoint.pathV31;
335 const summary = endpoint.summary
336 .replace(/\\/g, '\\\\')
337 .replace(/\|/g, '\\|')
338 .replace(/\n/g, ' ');
339 const label = endpoint.legacy ? `${summary} (Legacy)` : summary;
340 return `| \`${endpoint.method}\` | \`${path}\` | [${label}](${endpoint.href}) |`;
341 });
342
343 return ['| Method | Path | Endpoint |', '| --- | --- | --- |', ...rows].join('\n');
344 }
345
346 /**
347 * Converts MDX content to clean markdown for AI agents.
348 * Strips JSX components and converts them to plain text equivalents.
349 *
350 * `url` is the page URL. `ApiBaseUrl` and `ApiEndpointsTable` are client
351 * components that pick a version from `usePathname()`, which the `.md` channel
352 * has no access to — so the URL is passed in and resolved with the same
353 * `detectApiVersion`. Optional because the changelog call sites have no page
354 * URL; with none, the page is treated as current (v3.1), which is correct
355 * there since the changelog tree is not versioned.
356 */
357 export function mdxToCleanMarkdown(content: string, url?: string): string {
358 let result = resolveToolkitCount(content);
359 const version = url ? detectApiVersion(url) : '3.1';
360
361 // Remove frontmatter
362 result = result.replace(/^---[\s\S]*?---\n*/m, '');
363
364 // Version-dependent API components. These must run before the generic JSX
365 // strippers at the bottom of this function, which would otherwise drop both
366 // tags — publishing an empty `Base URL` bullet and an empty `Endpoints`
367 // section to every agent while the superseded v3.0 operation pages published
368 // a complete working request. That asymmetry is why agents reached for v3.
369 result = result.replace(/<ApiBaseUrl\s*\/>/g, `\`${API_BASE_URLS[version]}\``);
370 result = result.replace(
371 API_ENDPOINTS_TABLE_REGEX,
372 (_, bracedPayload?: string, quotedPayload?: string) =>
373 endpointsTableToMarkdown(
374 bracedPayload ?? decodeHtmlEntities(quotedPayload ?? ''),
375 version,
376 url
377 )
378 );
379 result = componentsToMarkdown(result);
380
381 result = replaceHomeNavigationMarkdown(result);
382
383 // Keep installation commands in both raw search input and processed page Markdown.
384 result = result.replace(/<PackageInstall\b([\s\S]*?)\/>/g, (_, attributes: string) =>
385 packageInstallToMarkdown(attributes),
386 );
387 result = result.replace(/<AgentSetupActions\b[^>]*\/>/g,
388 `\n[Agent setup](/docs/agent-setup)\n\n${SETUP_PROMPT}\n`,
389 );
390 result = result.replace(/<AgentSetupGrid\s*\/>/g,
391 AGENTS.map(agent => `- [${agent.name}](${agent.href}): ${agent.description}`).join('\n'),
392 );
393 result = result.replace(/<AgentFirstPrompt\s+agent="([^"]+)"\s*\/>/g, (_, agent: string) => {
394 const parsed = z.enum(['claude-code', 'cline', 'codex', 'cursor', 'gemini-cli',
395 'github-copilot', 'grok', 'openclaw', 'opencode']).parse(agent);
396 return `\n\`\`\`text\n${promptFor(parsed)}\n\`\`\`\n`;
397 });
398 result = result.replace(/<Video\b[^>]*src="([^"]+)"[^>]*caption="([^"]+)"[^>]*\/>/g,
399 '[Video: $2]($1)',
400 );
401
402 // Convert YouTube to link
403 result = result.replace(
404 /<YouTube\s+id="([^"]+)"\s+title="([^"]+)"\s*\/>/g,
405 '[Video: $2](https://youtube.com/watch?v=$1)'
406 );
407
408 // Convert Callout to blockquote - trim content to avoid empty lines
409 result = result.replace(
410 /<Callout[^>]*title="([^"]*)"[^>]*>([\s\S]*?)<\/Callout>/g,
411 (_, title, content) => `> **${title}**: ${content.trim()}`
412 );
413 result = result.replace(
414 /<Callout[^>]*>([\s\S]*?)<\/Callout>/g,
415 (_, content) => `> ${content.trim()}`
416 );
417
418 // Cards were converted with the parsed document above; drop their wrapper.
419 result = result.replace(/<\/?Cards\b[^>]*>/g, '');
420
421 result = result.replace(
422 /<ProviderList\s+group="(ai-sdks|agent-frameworks|custom)"\s*\/>/g,
423 (_, group: 'ai-sdks' | 'agent-frameworks' | 'custom') => providerListMarkdown(group),
424 );
425
426 result = result.replace(/^[ \t]+(- \[)/gm, '$1');
427
428
429 result = result.replace(/<StepTitle>([\s\S]*?)<\/StepTitle>/g, (_, title) => {
430 const cleanTitle = title
431 .replace(/^[\s#]*#\s*/, '')
432 .replace(/\s+$/, '')
433 .trim();
434 return cleanTitle ? `#### ${cleanTitle}` : '';
435 });
436 result = result.replace(/<Step>\s*###\s*(.+)/g, '#### $1');
437 result = result.replace(/<\/?Steps>/g, '');
438 result = result.replace(/<\/?Step>/g, '');
439 result = result.replace(/^(\s*#{1,6})\s+#\s+(.+)$/gm, '$1 $2');
440 result = result.replace(/^\s*#\s*$/gm, '');
441
442 result = result.replace(/<FrameworkOption[\s\S]*?name="([^"]*)"[\s\S]*?>/g, '\n## $1\n');
443 result = result.replace(/<\/FrameworkOption>/g, '');
444
445 const tabLabelMap: Record<string, string> = { native: 'Native Tools', mcp: 'MCP' };
446 result = result.replace(
447 /<IntegrationTabs[\s\S]*?tabs=\{\[([\s\S]*?)\]\}[\s\S]*?>/g,
448 (_, tabsContent: string) => {
449 const labelRegex = /value:\s*"([^"]+)"[\s\S]*?label:\s*"([^"]+)"/g;
450 let match;
451 while ((match = labelRegex.exec(tabsContent)) !== null) {
452 tabLabelMap[match[1]] = match[2];
453 }
454 return '\n> Choose your integration type · [Use this guide to decide](/docs/native-tools-vs-mcp)\n';
455 }
456 );
457 result = result.replace(
458 /<IntegrationTabs(?![^>]*tabs=)[\s\S]*?>/g,
459 '\n> Choose your integration type · [Use this guide to decide](/docs/native-tools-vs-mcp)\n'
460 );
461
462 result = result.replace(
463 /<IntegrationContent[\s\S]*?value="([^"]*)"[\s\S]*?>/g,
464 (_, value: string) => `\n### ${tabLabelMap[value] || value}\n`
465 );
466 result = result.replace(/<\/IntegrationContent>/g, '');
467
468 result = result.replace(
469 /<Accordion[\s\S]*?title="([^"]*)"[\s\S]*?>([\s\S]*?)<\/Accordion>/g,
470 '\n**$1**\n$2'
471 );
472
473 result = result.replace(
474 /<Figure[\s\S]*?src="([^"]*)"[\s\S]*?alt="([^"]*)"[\s\S]*?caption="([^"]*)"[\s\S]*?\/>/g,
475 '![$2]($1)\n*$3*'
476 );
477 result = result.replace(
478 /<Figure[\s\S]*?src="([^"]*)"[\s\S]*?alt="([^"]*)"[\s\S]*?\/>/g,
479 '![$2]($1)'
480 );
481
482 result = result.replace(/<ToolTypeOption[\s\S]*?name="([^"]*)"[\s\S]*?>/g, '\n### $1\n');
483 result = result.replace(/<\/ToolTypeOption>/g, '');
484
485 result = result.replace(
486 /<TemplateCard[\s\S]*?title="([^"]*)"[\s\S]*?description="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?\/>/g,
487 '- [$1]($3): $2'
488 );
489 result = result.replace(
490 /<TemplateCard[\s\S]*?href="([^"]*)"[\s\S]*?title="([^"]*)"[\s\S]*?description="([^"]*)"[\s\S]*?\/>/g,
491 '- [$2]($1): $3'
492 );
493
494 result = result.replace(
495 /<GlossaryTerm[\s\S]*?name="([^"]*)"[\s\S]*?>([\s\S]*?)<\/GlossaryTerm>/g,
496 (_, name, content) => `### ${name}\n\n${content.trim()}`
497 );
498
499 result = result.replace(
500 /<AIToolsBanner\s*\/>/g,
501 '### For AI tools\n\n' +
502 '**Skills:**\n' +
503 '```bash\nnpx skills add ComposioHQ/composio --skill composio -y\n```\n' +
504 '[GitHub](https://github.com/ComposioHQ/composio/tree/next/skills/composio)\n\n' +
505 '**CLI:**\n' +
506 '```bash\ncurl -fsSL https://composio.dev/install | sh\n```\n' +
507 '[CLI Reference](/docs/cli)\n\n' +
508 '**Context:**\n' +
509 '- [llms.txt](/llms.txt) — Documentation index with links\n' +
510 '- [llms-full.txt](/llms-full.txt) — Complete documentation in one file'
511 );
512
513 result = result.replace(
514 /<ConnectClientOption[^>]*\bname="([^"]*)"[^>]*>/g,
515 (_, name) => `## ${name}\n`
516 );
517
518 // FileBuildup renders an example's file growing step by step. The JSX can't
519 // serialize to markdown, so the .md an agent reads would otherwise lose every
520 // line of real code. Emit the actual source from the FILE_BUILDS registry:
521 // `<FileBuildup name="bot" step={2} />` -> the full file at that step;
522 // without `step` -> the final complete file.
523 result = result.replace(
524 /<FileBuildup\s+name="([^"]+)"(?:\s+step=\{(\d+)\})?\s*\/>/g,
525 (_, name: string, step?: string) => {
526 const build = FILE_BUILDS[name];
527 if (!build || !build.stages?.length) return '';
528 const lang = /\.tsx?$/.test(build.file)
529 ? 'typescript'
530 : /\.py$/.test(build.file)
531 ? 'python'
532 : '';
533 const idx = step ? Number(step) - 1 : build.stages.length - 1;
534 const stage = build.stages[idx];
535 if (!stage) return '';
536 const label = step ? ` — step ${step}: ${stage.title}` : ' — complete file';
537 return `\n**\`${build.file}\`${label}**\n\n\`\`\`${lang}\n${stage.code.trim()}\n\`\`\`\n`;
538 }
539 );
540
541 result = replaceRepoBrowserMarkdown(result);
542
543 result = result.replace(
544 /<\/?(Tabs|Frame|div|QuickstartFlow|IntegrationTabs|Accordions|ToolTypeFlow|ToolkitsLanding|TemplateGrid|Glossary|ConnectFlow|ConnectClientOption)[^>]*>/g,
545 ''
546 );
547
548 result = result.replace(/<[A-Z][a-zA-Z]*[\s\S]*?\/>/g, '');
549 result = result.replace(/<\/?[A-Z][a-zA-Z]*[^>]*>/g, '');
550
551 const lines = result.split('\n');
552 const normalizedLines: string[] = [];
553 let inCodeBlock = false;
554 let codeBlockLines: string[] = [];
555
556 const flushCodeBlock = () => {
557 if (codeBlockLines.length > 0) {
558 const nonEmptyLines = codeBlockLines.filter(l => l.trim().length > 0);
559 const minIndent =
560 nonEmptyLines.length > 0
561 ? Math.min(...nonEmptyLines.map(l => l.match(/^(\s*)/)?.[1]?.length || 0))
562 : 0;
563 for (const codeLine of codeBlockLines) {
564 normalizedLines.push(codeLine.slice(minIndent));
565 }
566 codeBlockLines = [];
567 }
568 };
569
570 for (const line of lines) {
571 if (line.trim().startsWith('```')) {
572 if (inCodeBlock) {
573 flushCodeBlock();
574 inCodeBlock = false;
575 normalizedLines.push(line.trim());
576 } else {
577 inCodeBlock = true;
578 normalizedLines.push(line.trim());
579 }
580 } else if (inCodeBlock) {
581 codeBlockLines.push(line);
582 } else {
583 const trimmedLine = line.trimStart();
584 if (/^\s*[{}]\s*$/.test(line)) {
585 continue;
586 }
587 if (trimmedLine.match(/^[-*+]\s/) || trimmedLine.match(/^\d+\.\s/)) {
588 const leadingSpaces = line.length - trimmedLine.length;
589 const indentLevel = Math.floor(leadingSpaces / 2);
590 const normalizedIndent = ' '.repeat(Math.min(indentLevel, 4));
591 normalizedLines.push(normalizedIndent + trimmedLine);
592 } else {
593 normalizedLines.push(trimmedLine);
594 }
595 }
596 }
597
598 if (inCodeBlock) {
599 flushCodeBlock();
600 }
601
602 result = normalizedLines.join('\n');
603 result = result.replace(/\n{3,}/g, '\n\n');
604 result = stripTwoslashFromCodeBlocks(result);
605
606 return result.trim();
607 }
608
609 function stripTwoslashFromCodeBlocks(content: string): string {
610 return content.replace(/(```[\w]*\n)([\s\S]*?)(```)/g, (match, open, code, close) => {
611 let cleanCode = code;
612 cleanCode = cleanCode.replace(/^\/\/\s*---cut---.*\n?/gm, '');
613 cleanCode = cleanCode.replace(/^\/\/\s*@errors?:.*\n?/gm, '');
614 cleanCode = cleanCode.replace(/^\/\/\s*@noErrors.*\n?/gm, '');
615 cleanCode = cleanCode.replace(/^\/\/\s*@filename:.*\n?/gm, '');
616 cleanCode = cleanCode.replace(/^\/\/\s*@highlight.*\n?/gm, '');
617 cleanCode = cleanCode.replace(/^\/\/\s*\^[\?\!].*\n?/gm, '');
618 cleanCode = cleanCode.replace(/^\n+/, '');
619 return open + cleanCode + close;
620 });
621 }
622
623 export interface LLMPage {
624 url: string;
625 data: {
626 title: string;
627 description?: string;
628 getText?: (mode: 'processed' | 'raw') => Promise<string>;
629 legacy?: boolean;
630 written?: string;
631 llmGuardrails?: Parameters<typeof getGuardrails>[0];
632 };
633 }
634
635 export async function getLLMText(
636 page: LLMPage,
637 options?: { includeFooter?: boolean; includeGuardrails?: boolean }
638 ) {
639 const includeFooter = options?.includeFooter ?? true;
640 const includeGuardrails = options?.includeGuardrails ?? true;
641 if (typeof page.data.getText !== 'function') {
642 return `# ${page.data.title} (${page.url})
643
644 ${page.data.description || ''}`;
645 }
646
647 let content: string | null = null;
648
649 try {
650 content = await page.data.getText('processed');
651 } catch (e) {
652 console.error('getText(processed) failed:', e);
653 try {
654 content = await page.data.getText('raw');
655 } catch (e2) {
656 console.error('getText(raw) also failed:', e2);
657 }
658 }
659
660 if (!content) {
661 return `# ${page.data.title} (${page.url})
662
663 ${page.data.description || ''}`;
664 }
665
666 const mermaidRegex = /<Mermaid\s+chart="([\s\S]*?)"\s*\/>/g;
667 const segments: string[] = [];
668 const mermaidCharts: string[] = [];
669 let lastIndex = 0;
670 let match;
671
672 while ((match = mermaidRegex.exec(content)) !== null) {
673 segments.push(content.slice(lastIndex, match.index));
674 mermaidCharts.push(match[1]);
675 lastIndex = match.index + match[0].length;
676 }
677 segments.push(content.slice(lastIndex));
678
679 const cleanSegments = segments.map(s => mdxToCleanMarkdown(s, page.url));
680 let cleanContent = cleanSegments[0];
681 for (let i = 0; i < mermaidCharts.length; i++) {
682 const chart = mermaidCharts[i]
683 .replace(/&#x22;/g, '"')
684 .replace(/&#x27;/g, "'")
685 .replace(/&amp;/g, '&');
686 cleanContent += `\n\n\`\`\`mermaid\n${chart}\n\`\`\`\n\n${cleanSegments[i + 1]}`;
687 }
688
689 const footer = includeFooter
690 ? `\n\n---\n\n📚 **More documentation:** [View all docs](https://docs.composio.dev/llms.txt) | [Changelog](https://docs.composio.dev/docs/changelog.md) | [Glossary](https://docs.composio.dev/llms.mdx/reference/glossary) | [Examples](https://docs.composio.dev/llms.mdx/examples) | [API Reference](https://docs.composio.dev/llms.mdx/reference)`
691 : '';
692
693 // Legacy pages (frontmatter `legacy: true`) document point-in-time migrations
694 // and may show outdated APIs. Mark the .md so an agent reading it knows, and
695 // skip the "enforce the CURRENT patterns" guardrail block — appending it to a
696 // legacy guide contradicts the guide's own (older) content.
697 const isLegacy = page.data.legacy === true;
698 const written = page.data.written;
699 const frontmatterNote = isLegacy
700 ? `\n> **Legacy${written ? ` · written ${written}` : ''}.** This is a point-in-time migration/legacy guide and may describe outdated APIs. For current guidance, see https://docs.composio.dev.\n`
701 : written
702 ? `\n> _Written ${written}._\n`
703 : '';
704
705 // Which REST version this page documents. Scoped to the reference tree —
706 // /docs/** has no REST version — and carries no guidance paragraph, because
707 // the guardrail block further down this same response already does.
708 const topNote = `${frontmatterNote}${apiVersionPointer(page.url)}`;
709
710 const guardrails = includeGuardrails && !isLegacy ? getGuardrails(page.data.llmGuardrails) : '';
711
712 return `# ${page.data.title} (${page.url})
713 ${topNote}
714 ${cleanContent}${footer}${guardrails}`;
715 }
716
717 export function formatDate(dateStr: string): string {
718 return new Date(`${dateStr}T12:00:00`).toLocaleDateString('en-US', {
719 year: 'numeric',
720 month: 'short',
721 day: 'numeric',
722 });
723 }
724
725 const DATE_REGEX = /^\d{4}-\d{2}-\d{2}$/;
726
727 function validateDateFormat(dateStr: string): void {
728 if (!DATE_REGEX.test(dateStr)) {
729 throw new Error(`Invalid date format: "${dateStr}". Expected YYYY-MM-DD (e.g., "2025-12-29")`);
730 }
731 }
732
733 export function dateToChangelogUrl(dateStr: string): string {
734 validateDateFormat(dateStr);
735 const [year, month, day] = dateStr.split('-');
736 return `/docs/changelog/${year}/${month}/${day}`;
737 }
738
739 export function dateToSlug(dateStr: string): string[] {
740 validateDateFormat(dateStr);
741 const [year, month, day] = dateStr.split('-');
742 return [year, month, day];
743 }
744
745 export function slugToDate(slug: string[]): string | null {
746 if (slug.length !== 3) return null;
747 const [year, month, day] = slug;
748 return `${year}-${month}-${day}`;
749 }