Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 22, 18:15 UTC
some changes to build llm
Sushmithamallesh committed
Nov 5, 2025 at 21:42 UTC
f93f0ac506c7e9395162982b3d9252aa28b10dde
1 file changed
+92
-121
fern/scripts/build-llms-txt.ts
+92
-121
@@ -10,16 +10,16 @@ import {
10
ensureOutputDir,
11
type DocsConfig,
12
type ContentItem,
13
+ type LayoutItem,
14
} from './lib/docs-utils';
15
16
interface LlmsEntry {
17
title: string;
18
url: string;
19
description?: string;
19
- parentSection?: string; // Track parent section for nested items
20
+ parentSection?: string; // for prefixing nested items
21
}
22
22
-// Extract title and description from MDX file frontmatter
23
function extractFrontmatter(mdxPath: string): { title?: string; description?: string } {
24
try {
25
const fullPath = path.join(FERN_DIR, mdxPath);
@@ -28,78 +28,67 @@ function extractFrontmatter(mdxPath: string): { title?: string; description?: st
28
}
29
30
const content = fs.readFileSync(fullPath, 'utf8');
31
-
32
- // Try to extract from frontmatter
31
const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
34
- if (frontmatterMatch) {
35
- const frontmatter = frontmatterMatch[1];
36
-
37
- // Extract title
38
- const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
39
- const title = titleMatch ? titleMatch[1].trim() : undefined;
40
-
41
- // Extract subtitle only
42
- const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
43
- const description = descMatch ? descMatch[1].trim() : undefined;
44
-
45
- return { title, description };
32
+
33
+ if (!frontmatterMatch) {
34
+ return {};
35
}
36
48
- return {};
37
+ const frontmatter = frontmatterMatch[1];
38
+
39
+ // Extract title from frontmatter
40
+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
41
+ const title = titleMatch ? titleMatch[1].trim() : undefined;
42
+
43
+ // Extract subtitle (used as description in llms.txt)
44
+ const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
45
+ const description = descMatch ? descMatch[1].trim() : undefined;
46
+
47
+ return { title, description };
48
} catch (error) {
49
return {};
50
}
51
}
52
54
-// Process a single content item
53
+function buildUrl(item: ContentItem, tabSlug: string, parentSlug?: string): string {
54
+ const BASE_URL = 'https://docs.composio.dev';
55
+
56
+ if (item.slug) {
57
+ const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
58
+ return `${BASE_URL}/${urlParts.join('/')}`;
59
+ }
60
+
61
+ if (item.path) {
62
+ const urlPath = item.path
63
+ .replace(/^pages\/(dist|src)\//, '')
64
+ .replace(/^pages\//, '')
65
+ .replace(/\.(mdx?|md)$/, '');
66
+ return `${BASE_URL}/${urlPath}`;
67
+ }
68
+
69
+ console.warn(`No slug or path for: ${item.page}`);
70
+ return BASE_URL;
71
+}
72
+
73
function processItem(
74
item: ContentItem,
75
tabSlug: string,
76
parentSlug?: string,
77
parentSection?: string
78
): LlmsEntry | null {
61
- // Skip items without a page name and path
62
- if (!item.page || !item.path) {
63
- return null;
64
- }
65
-
66
- // Simple URL generation
67
- let url: string;
68
-
69
- if (item.slug) {
70
- // Use explicit slug from docs.yml
71
- const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
72
- url = `https://docs.composio.dev/${urlParts.join('/')}`;
73
- } else if (item.path) {
74
- // Derive URL from file path
75
- let urlPath = item.path
76
- .replace('pages/dist/', '')
77
- .replace('pages/src/', '')
78
- .replace('pages/', '')
79
- .replace('.mdx', '')
80
- .replace('.md', '');
81
-
82
- url = `https://docs.composio.dev/${urlPath}`;
83
- } else {
84
- // This shouldn't happen - all items should have either slug or path
85
- console.warn(`Warning: No slug or path for item: ${item.page}`);
86
- return null;
87
- }
79
+ if (!item.page || !item.path) return null;
80
89
- // Try to extract title and description from MDX file
81
+ const url = buildUrl(item, tabSlug, parentSlug);
82
const frontmatter = extractFrontmatter(item.path);
91
- const title = frontmatter.title || item.page;
92
- const description = frontmatter.description;
93
-
83
+
84
return {
95
- title,
85
+ title: frontmatter.title || item.page,
86
url,
97
- description,
87
+ description: frontmatter.description,
88
parentSection,
89
};
90
}
91
102
-// Process content items recursively (flattened)
92
function processContentItems(
93
items: ContentItem[],
94
tabSlug: string,
@@ -109,20 +98,23 @@ function processContentItems(
98
const entries: LlmsEntry[] = [];
99
100
for (const item of items) {
112
- // Process the item itself
101
const entry = processItem(item, tabSlug, parentSlug, parentSection);
102
if (entry) {
103
entries.push(entry);
104
}
105
118
- // Process nested items
106
if (item.section && item.contents) {
120
- // Combine parent slug with item slug for nested sections
107
const nestedSlug = item.slug
108
? (parentSlug ? `${parentSlug}/${item.slug}` : item.slug)
109
: parentSlug;
124
- // Pass the section name down to nested items
125
- const nestedEntries = processContentItems(item.contents, tabSlug, nestedSlug, item.section);
110
+
111
+ // pass section name to children for prefixing
112
+ const nestedEntries = processContentItems(
113
+ item.contents,
114
+ tabSlug,
115
+ nestedSlug,
116
+ item.section
117
+ );
118
entries.push(...nestedEntries);
119
}
120
}
@@ -130,9 +122,15 @@ function processContentItems(
122
return entries;
123
}
124
133
-// Process all navigation and organize by sections
125
+function getSectionName(tab: string, layoutItem: LayoutItem, tabDisplayName: string): string {
126
+ if (tab === 'docs' && layoutItem.section) {
127
+ return layoutItem.section;
128
+ }
129
+ return tabDisplayName;
130
+}
131
+
132
function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
135
- const sections: Array<[string, LlmsEntry[]]> = [];
133
+ const sections = new Map<string, LlmsEntry[]>();
134
135
for (const navItem of config.navigation) {
136
if (!navItem.layout) continue;
@@ -143,122 +141,95 @@ function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
141
const tabSlug = skipSlug ? '' : tab;
142
const tabDisplayName = tabConfig?.['display-name'] || tab;
143
146
- // Process each layout item as a section
144
for (const layoutItem of navItem.layout) {
148
- // Determine section name
149
- let sectionName: string;
150
- if (tab === 'docs' && layoutItem.section) {
151
- sectionName = layoutItem.section;
152
- } else if (layoutItem.section) {
153
- sectionName = tabDisplayName; // Use tab name for non-docs sections
154
- } else {
155
- sectionName = tabDisplayName;
156
- }
157
-
145
+ const sectionName = getSectionName(tab, layoutItem, tabDisplayName);
146
const sectionEntries: LlmsEntry[] = [];
147
160
- // Process direct pages
148
+ // direct page
149
if (layoutItem.page && layoutItem.path && layoutItem.slug) {
162
- const entry = processItem(layoutItem, tabSlug);
150
+ const entry = processItem(layoutItem as ContentItem, tabSlug);
151
if (entry) {
152
sectionEntries.push(entry);
153
}
154
}
155
168
- // Process section contents
156
+ // nested contents
157
if (layoutItem.contents) {
158
const entries = processContentItems(
159
layoutItem.contents,
160
tabSlug,
161
layoutItem.slug,
174
- undefined // Don't pass parentSection for top-level section items
162
+ undefined // no parent section for top-level
163
);
164
sectionEntries.push(...entries);
165
}
166
167
if (sectionEntries.length > 0) {
180
- // Find existing section or add new one
181
- const existingIndex = sections.findIndex(([name]) => name === sectionName);
182
- if (existingIndex >= 0) {
183
- sections[existingIndex][1].push(...sectionEntries);
184
- } else {
185
- sections.push([sectionName, sectionEntries]);
186
- }
168
+ const existing = sections.get(sectionName) || [];
169
+ sections.set(sectionName, [...existing, ...sectionEntries]);
170
}
171
}
172
}
173
191
- return sections;
174
+ return Array.from(sections.entries());
175
+}
176
+
177
+function formatEntry(entry: LlmsEntry): string {
178
+ const displayTitle = entry.parentSection
179
+ ? `${entry.parentSection}: ${entry.title}`
180
+ : entry.title;
181
+
182
+ const link = `[${displayTitle}](${entry.url})`;
183
+ return entry.description
184
+ ? `- ${link}: ${entry.description}`
185
+ : `- ${link}`;
186
}
187
194
-// Generate llms.txt content following best practices
188
function generateLlmsTxt(sections: Array<[string, LlmsEntry[]]>): string {
189
let content = '# Composio\n\n';
190
content += '> Composio is a platform that connects AI agents to 500+ external tools and services through managed authentication, tool execution, and triggers. It simplifies integrating AI agents with SaaS applications, developer tools, and custom APIs.\n\n';
198
- content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n\n';
191
+ content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n';
192
200
- // Process all sections
193
for (const [sectionName, entries] of sections) {
202
- content += `## ${sectionName}\n`;
194
+ content += `\n## ${sectionName}\n`;
195
204
- // Special handling for Toolkits section only
196
if (sectionName === 'Toolkits') {
206
- content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n`;
197
+ content += '- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n';
198
} else {
199
for (const entry of entries) {
209
- // Add parent section prefix if it exists
210
- const displayTitle = entry.parentSection
211
- ? `${entry.parentSection}: ${entry.title}`
212
- : entry.title;
213
-
214
- if (entry.description) {
215
- content += `- [${displayTitle}](${entry.url}): ${entry.description}\n`;
216
- } else {
217
- content += `- [${displayTitle}](${entry.url})\n`;
218
- }
200
+ content += formatEntry(entry) + '\n';
201
}
202
}
221
- content += '\n';
203
}
204
205
return content;
206
}
207
227
-// Main function
208
async function main(): Promise<void> {
229
- console.log('🔨 Building llms.txt from docs.yml...');
209
+ console.log('Building llms.txt...');
210
211
try {
232
- // Load docs configuration
212
const config = loadDocsConfig();
234
-
235
- // Process navigation to extract all entries
213
const sections = processNavigation(config);
214
238
- // Count total entries
239
- let totalEntries = 0;
240
- for (const [, entries] of sections) {
241
- totalEntries += entries.length;
242
- }
243
-
244
- console.log(`📄 Found ${totalEntries} documentation pages`);
215
+ const totalEntries = sections.reduce((total, [_, entries]) => total + entries.length, 0);
216
+ console.log(`Found ${totalEntries} pages across ${sections.length} sections`);
217
246
- // Generate llms.txt content
218
const llmsTxtContent = generateLlmsTxt(sections);
248
-
249
- // Write llms.txt
219
ensureOutputDir(LLMS_TXT_PATH);
220
fs.writeFileSync(LLMS_TXT_PATH, llmsTxtContent);
221
253
- console.log(`✅ Generated llms.txt at ${LLMS_TXT_PATH}`);
222
+ console.log(`Generated ${LLMS_TXT_PATH}`);
223
} catch (error) {
255
- console.error('❌ Error building llms.txt:', error);
224
+ console.error('Error:', error);
225
process.exit(1);
226
}
227
}
228
260
-// Run the script
261
-main().catch(error => {
262
- console.error('❌ Fatal error:', error);
263
- process.exit(1);
264
-});
\ No newline at end of file
229
+// @ts-ignore - bun-specific
230
+if (import.meta.main) {
231
+ main().catch(error => {
232
+ console.error('Fatal error:', error);
233
+ process.exit(1);
234
+ });
235
+}
\ No newline at end of file