@devries-cameron20 / composio-1 / commits / c66fa32ba
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(docs): clean MDX content for AI agents

Convert MDX components to plain markdown for better AI consumption: - YouTube embeds → markdown links - Cards → list items with links - Callouts → blockquotes - Steps → headers - Strip remaining JSX components Also update llms-full.txt to use the same clean output. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Sushmithamallesh committed Jan 19, 2026 at 02:27 UTC c66fa32ba83831b9b896f1fa3d2cf76ac1311cbd
2 files changed +72 -15
docs/app/llms-full.txt/route.ts
+3 -13
@@ -1,23 +1,13 @@
1 -import { source } from '@/lib/source';
1 +import { getLLMText, source } from '@/lib/source';
2
3 export const revalidate = false;
4
5 export async function GET() {
6 try {
7 const pages = source.getPages();
8 - const results: string[] = [];
8 + const results = await Promise.all(pages.map(getLLMText));
9
10 - for (const page of pages) {
11 - try {
12 - const processed = await page.data.getText('processed');
13 - results.push(`# ${page.data.title}\n\n${processed}`);
14 - } catch {
15 - // Fallback to basic info if getText fails
16 - results.push(`# ${page.data.title}\n\n${page.data.description || ''}`);
17 - }
18 - }
19 -
20 - return new Response(results.join('\n\n'), {
10 + return new Response(results.join('\n\n---\n\n'), {
11 headers: {
12 'Content-Type': 'text/plain; charset=utf-8',
13 },
docs/lib/source.ts
+69 -2
@@ -55,12 +55,79 @@ export function getPageImage(page: InferPageType<typeof source>) {
55 };
56 }
57
58 +/**
59 + * Converts MDX content to clean markdown for AI agents.
60 + * Strips JSX components and converts them to plain text equivalents.
61 + */
62 +function mdxToCleanMarkdown(content: string): string {
63 + let result = content;
64 +
65 + // Remove frontmatter
66 + result = result.replace(/^---[\s\S]*?---\n*/m, '');
67 +
68 + // Convert YouTube to link
69 + result = result.replace(
70 + /<YouTube\s+id="([^"]+)"\s+title="([^"]+)"\s*\/>/g,
71 + '[Video: $2](https://youtube.com/watch?v=$1)'
72 + );
73 +
74 + // Convert Callout to blockquote
75 + result = result.replace(
76 + /<Callout[^>]*title="([^"]*)"[^>]*>([\s\S]*?)<\/Callout>/g,
77 + '> **$1**: $2'
78 + );
79 + result = result.replace(
80 + /<Callout[^>]*>([\s\S]*?)<\/Callout>/g,
81 + '> $1'
82 + );
83 +
84 + // Convert Card - handle multiline and various attribute orders
85 + result = result.replace(
86 + /<Card[\s\S]*?title="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?description="([^"]*)"[\s\S]*?\/>/g,
87 + '- [$1]($2): $3'
88 + );
89 +
90 + // Convert ProviderCard to markdown link
91 + result = result.replace(
92 + /<ProviderCard[\s\S]*?name="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?languages=\{\[([^\]]*)\]\}[\s\S]*?\/>/g,
93 + (_, name, href, langs) => `- [${name}](${href}) (${langs.replace(/"/g, '')})`
94 + );
95 +
96 + // Convert Tabs/Tab content - keep inner content
97 + result = result.replace(/<TabsList>[\s\S]*?<\/TabsList>/g, '');
98 + result = result.replace(/<TabsTrigger[^>]*>[^<]*<\/TabsTrigger>/g, '');
99 + result = result.replace(/<TabsContent[\s\S]*?value="([^"]*)"[\s\S]*?>([\s\S]*?)<\/TabsContent>/g, '\n**$1:**\n$2');
100 + result = result.replace(/<Tab[\s\S]*?value="([^"]*)"[\s\S]*?>([\s\S]*?)<\/Tab>/g, '\n**$1:**\n$2');
101 +
102 + // Convert Steps/Step
103 + result = result.replace(/<Step>\s*###\s*(.+)/g, '### Step: $1');
104 + result = result.replace(/<\/?Steps?>/g, '');
105 +
106 + // Remove wrapper components (Cards, ProviderGrid, Tabs, Frame, div, etc.)
107 + result = result.replace(/<\/?(Cards|ProviderGrid|Tabs|Frame|div)[^>]*>/g, '');
108 +
109 + // Remove remaining self-closing JSX tags (including those with JSX expressions)
110 + result = result.replace(/<[A-Z][a-zA-Z]*[\s\S]*?\/>/g, '');
111 +
112 + // Remove remaining JSX opening/closing tags but keep content
113 + result = result.replace(/<\/?[A-Z][a-zA-Z]*[^>]*>/g, '');
114 +
115 + // Clean up leftover JSX artifacts like lone } or {
116 + result = result.replace(/^\s*[{}]\s*$/gm, '');
117 +
118 + // Clean up excessive newlines
119 + result = result.replace(/\n{3,}/g, '\n\n');
120 +
121 + return result.trim();
122 +}
123 +
124 export async function getLLMText(page: InferPageType<typeof source>) {
59 - const processed = await page.data.getText('processed');
125 + const content = await page.data.getText('raw');
126 + const cleanContent = mdxToCleanMarkdown(content);
127
128 return `# ${page.data.title} (${page.url})
129
63 -${processed}`;
130 +${cleanContent}`;
131 }
132
133 export function formatDate(dateStr: string): string {