Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
fix(docs): clean MDX content for AI agents
Convert MDX components to plain markdown for better AI consumption: - YouTube embeds → markdown links - Cards → list items with links - Callouts → blockquotes - Steps → headers - Strip remaining JSX components Also update llms-full.txt to use the same clean output. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
Sushmithamallesh committed
Jan 19, 2026 at 02:27 UTC
c66fa32ba83831b9b896f1fa3d2cf76ac1311cbd
2 files changed
+72
-15
docs/app/llms-full.txt/route.ts
+3
-13
@@ -1,23 +1,13 @@
1
-import { source } from '@/lib/source';
1
+import { getLLMText, source } from '@/lib/source';
2
3
export const revalidate = false;
4
5
export async function GET() {
6
try {
7
const pages = source.getPages();
8
- const results: string[] = [];
8
+ const results = await Promise.all(pages.map(getLLMText));
9
10
- for (const page of pages) {
11
- try {
12
- const processed = await page.data.getText('processed');
13
- results.push(`# ${page.data.title}\n\n${processed}`);
14
- } catch {
15
- // Fallback to basic info if getText fails
16
- results.push(`# ${page.data.title}\n\n${page.data.description || ''}`);
17
- }
18
- }
19
-
20
- return new Response(results.join('\n\n'), {
10
+ return new Response(results.join('\n\n---\n\n'), {
11
headers: {
12
'Content-Type': 'text/plain; charset=utf-8',
13
},
docs/lib/source.ts
+69
-2
@@ -55,12 +55,79 @@ export function getPageImage(page: InferPageType<typeof source>) {
55
};
56
}
57
58
+/**
59
+ * Converts MDX content to clean markdown for AI agents.
60
+ * Strips JSX components and converts them to plain text equivalents.
61
+ */
62
+function mdxToCleanMarkdown(content: string): string {
63
+ let result = content;
64
+
65
+ // Remove frontmatter
66
+ result = result.replace(/^---[\s\S]*?---\n*/m, '');
67
+
68
+ // Convert YouTube to link
69
+ result = result.replace(
70
+ /<YouTube\s+id="([^"]+)"\s+title="([^"]+)"\s*\/>/g,
71
+ '[Video: $2](https://youtube.com/watch?v=$1)'
72
+ );
73
+
74
+ // Convert Callout to blockquote
75
+ result = result.replace(
76
+ /<Callout[^>]*title="([^"]*)"[^>]*>([\s\S]*?)<\/Callout>/g,
77
+ '> **$1**: $2'
78
+ );
79
+ result = result.replace(
80
+ /<Callout[^>]*>([\s\S]*?)<\/Callout>/g,
81
+ '> $1'
82
+ );
83
+
84
+ // Convert Card - handle multiline and various attribute orders
85
+ result = result.replace(
86
+ /<Card[\s\S]*?title="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?description="([^"]*)"[\s\S]*?\/>/g,
87
+ '- [$1]($2): $3'
88
+ );
89
+
90
+ // Convert ProviderCard to markdown link
91
+ result = result.replace(
92
+ /<ProviderCard[\s\S]*?name="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?languages=\{\[([^\]]*)\]\}[\s\S]*?\/>/g,
93
+ (_, name, href, langs) => `- [${name}](${href}) (${langs.replace(/"/g, '')})`
94
+ );
95
+
96
+ // Convert Tabs/Tab content - keep inner content
97
+ result = result.replace(/<TabsList>[\s\S]*?<\/TabsList>/g, '');
98
+ result = result.replace(/<TabsTrigger[^>]*>[^<]*<\/TabsTrigger>/g, '');
99
+ result = result.replace(/<TabsContent[\s\S]*?value="([^"]*)"[\s\S]*?>([\s\S]*?)<\/TabsContent>/g, '\n**$1:**\n$2');
100
+ result = result.replace(/<Tab[\s\S]*?value="([^"]*)"[\s\S]*?>([\s\S]*?)<\/Tab>/g, '\n**$1:**\n$2');
101
+
102
+ // Convert Steps/Step
103
+ result = result.replace(/<Step>\s*###\s*(.+)/g, '### Step: $1');
104
+ result = result.replace(/<\/?Steps?>/g, '');
105
+
106
+ // Remove wrapper components (Cards, ProviderGrid, Tabs, Frame, div, etc.)
107
+ result = result.replace(/<\/?(Cards|ProviderGrid|Tabs|Frame|div)[^>]*>/g, '');
108
+
109
+ // Remove remaining self-closing JSX tags (including those with JSX expressions)
110
+ result = result.replace(/<[A-Z][a-zA-Z]*[\s\S]*?\/>/g, '');
111
+
112
+ // Remove remaining JSX opening/closing tags but keep content
113
+ result = result.replace(/<\/?[A-Z][a-zA-Z]*[^>]*>/g, '');
114
+
115
+ // Clean up leftover JSX artifacts like lone } or {
116
+ result = result.replace(/^\s*[{}]\s*$/gm, '');
117
+
118
+ // Clean up excessive newlines
119
+ result = result.replace(/\n{3,}/g, '\n\n');
120
+
121
+ return result.trim();
122
+}
123
+
124
export async function getLLMText(page: InferPageType<typeof source>) {
59
- const processed = await page.data.getText('processed');
125
+ const content = await page.data.getText('raw');
126
+ const cleanContent = mdxToCleanMarkdown(content);
127
128
return `# ${page.data.title} (${page.url})
129
63
-${processed}`;
130
+${cleanContent}`;
131
}
132
133
export function formatDate(dateStr: string): string {