@devries-cameron20 / composio-1 / commits / f93f0ac50
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

some changes to build llm

Sushmithamallesh committed Nov 5, 2025 at 21:42 UTC f93f0ac506c7e9395162982b3d9252aa28b10dde
1 file changed +92 -121
fern/scripts/build-llms-txt.ts
+92 -121
@@ -10,16 +10,16 @@ import {
10 ensureOutputDir,
11 type DocsConfig,
12 type ContentItem,
13 + type LayoutItem,
14 } from './lib/docs-utils';
15
16 interface LlmsEntry {
17 title: string;
18 url: string;
19 description?: string;
19 - parentSection?: string; // Track parent section for nested items
20 + parentSection?: string; // for prefixing nested items
21 }
22
22 -// Extract title and description from MDX file frontmatter
23 function extractFrontmatter(mdxPath: string): { title?: string; description?: string } {
24 try {
25 const fullPath = path.join(FERN_DIR, mdxPath);
@@ -28,78 +28,67 @@ function extractFrontmatter(mdxPath: string): { title?: string; description?: st
28 }
29
30 const content = fs.readFileSync(fullPath, 'utf8');
31 -
32 - // Try to extract from frontmatter
31 const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
34 - if (frontmatterMatch) {
35 - const frontmatter = frontmatterMatch[1];
36 -
37 - // Extract title
38 - const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
39 - const title = titleMatch ? titleMatch[1].trim() : undefined;
40 -
41 - // Extract subtitle only
42 - const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
43 - const description = descMatch ? descMatch[1].trim() : undefined;
44 -
45 - return { title, description };
32 +
33 + if (!frontmatterMatch) {
34 + return {};
35 }
36
48 - return {};
37 + const frontmatter = frontmatterMatch[1];
38 +
39 + // Extract title from frontmatter
40 + const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
41 + const title = titleMatch ? titleMatch[1].trim() : undefined;
42 +
43 + // Extract subtitle (used as description in llms.txt)
44 + const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
45 + const description = descMatch ? descMatch[1].trim() : undefined;
46 +
47 + return { title, description };
48 } catch (error) {
49 return {};
50 }
51 }
52
54 -// Process a single content item
53 +function buildUrl(item: ContentItem, tabSlug: string, parentSlug?: string): string {
54 + const BASE_URL = 'https://docs.composio.dev';
55 +
56 + if (item.slug) {
57 + const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
58 + return `${BASE_URL}/${urlParts.join('/')}`;
59 + }
60 +
61 + if (item.path) {
62 + const urlPath = item.path
63 + .replace(/^pages\/(dist|src)\//, '')
64 + .replace(/^pages\//, '')
65 + .replace(/\.(mdx?|md)$/, '');
66 + return `${BASE_URL}/${urlPath}`;
67 + }
68 +
69 + console.warn(`No slug or path for: ${item.page}`);
70 + return BASE_URL;
71 +}
72 +
73 function processItem(
74 item: ContentItem,
75 tabSlug: string,
76 parentSlug?: string,
77 parentSection?: string
78 ): LlmsEntry | null {
61 - // Skip items without a page name and path
62 - if (!item.page || !item.path) {
63 - return null;
64 - }
65 -
66 - // Simple URL generation
67 - let url: string;
68 -
69 - if (item.slug) {
70 - // Use explicit slug from docs.yml
71 - const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
72 - url = `https://docs.composio.dev/${urlParts.join('/')}`;
73 - } else if (item.path) {
74 - // Derive URL from file path
75 - let urlPath = item.path
76 - .replace('pages/dist/', '')
77 - .replace('pages/src/', '')
78 - .replace('pages/', '')
79 - .replace('.mdx', '')
80 - .replace('.md', '');
81 -
82 - url = `https://docs.composio.dev/${urlPath}`;
83 - } else {
84 - // This shouldn't happen - all items should have either slug or path
85 - console.warn(`Warning: No slug or path for item: ${item.page}`);
86 - return null;
87 - }
79 + if (!item.page || !item.path) return null;
80
89 - // Try to extract title and description from MDX file
81 + const url = buildUrl(item, tabSlug, parentSlug);
82 const frontmatter = extractFrontmatter(item.path);
91 - const title = frontmatter.title || item.page;
92 - const description = frontmatter.description;
93 -
83 +
84 return {
95 - title,
85 + title: frontmatter.title || item.page,
86 url,
97 - description,
87 + description: frontmatter.description,
88 parentSection,
89 };
90 }
91
102 -// Process content items recursively (flattened)
92 function processContentItems(
93 items: ContentItem[],
94 tabSlug: string,
@@ -109,20 +98,23 @@ function processContentItems(
98 const entries: LlmsEntry[] = [];
99
100 for (const item of items) {
112 - // Process the item itself
101 const entry = processItem(item, tabSlug, parentSlug, parentSection);
102 if (entry) {
103 entries.push(entry);
104 }
105
118 - // Process nested items
106 if (item.section && item.contents) {
120 - // Combine parent slug with item slug for nested sections
107 const nestedSlug = item.slug
108 ? (parentSlug ? `${parentSlug}/${item.slug}` : item.slug)
109 : parentSlug;
124 - // Pass the section name down to nested items
125 - const nestedEntries = processContentItems(item.contents, tabSlug, nestedSlug, item.section);
110 +
111 + // pass section name to children for prefixing
112 + const nestedEntries = processContentItems(
113 + item.contents,
114 + tabSlug,
115 + nestedSlug,
116 + item.section
117 + );
118 entries.push(...nestedEntries);
119 }
120 }
@@ -130,9 +122,15 @@ function processContentItems(
122 return entries;
123 }
124
133 -// Process all navigation and organize by sections
125 +function getSectionName(tab: string, layoutItem: LayoutItem, tabDisplayName: string): string {
126 + if (tab === 'docs' && layoutItem.section) {
127 + return layoutItem.section;
128 + }
129 + return tabDisplayName;
130 +}
131 +
132 function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
135 - const sections: Array<[string, LlmsEntry[]]> = [];
133 + const sections = new Map<string, LlmsEntry[]>();
134
135 for (const navItem of config.navigation) {
136 if (!navItem.layout) continue;
@@ -143,122 +141,95 @@ function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
141 const tabSlug = skipSlug ? '' : tab;
142 const tabDisplayName = tabConfig?.['display-name'] || tab;
143
146 - // Process each layout item as a section
144 for (const layoutItem of navItem.layout) {
148 - // Determine section name
149 - let sectionName: string;
150 - if (tab === 'docs' && layoutItem.section) {
151 - sectionName = layoutItem.section;
152 - } else if (layoutItem.section) {
153 - sectionName = tabDisplayName; // Use tab name for non-docs sections
154 - } else {
155 - sectionName = tabDisplayName;
156 - }
157 -
145 + const sectionName = getSectionName(tab, layoutItem, tabDisplayName);
146 const sectionEntries: LlmsEntry[] = [];
147
160 - // Process direct pages
148 + // direct page
149 if (layoutItem.page && layoutItem.path && layoutItem.slug) {
162 - const entry = processItem(layoutItem, tabSlug);
150 + const entry = processItem(layoutItem as ContentItem, tabSlug);
151 if (entry) {
152 sectionEntries.push(entry);
153 }
154 }
155
168 - // Process section contents
156 + // nested contents
157 if (layoutItem.contents) {
158 const entries = processContentItems(
159 layoutItem.contents,
160 tabSlug,
161 layoutItem.slug,
174 - undefined // Don't pass parentSection for top-level section items
162 + undefined // no parent section for top-level
163 );
164 sectionEntries.push(...entries);
165 }
166
167 if (sectionEntries.length > 0) {
180 - // Find existing section or add new one
181 - const existingIndex = sections.findIndex(([name]) => name === sectionName);
182 - if (existingIndex >= 0) {
183 - sections[existingIndex][1].push(...sectionEntries);
184 - } else {
185 - sections.push([sectionName, sectionEntries]);
186 - }
168 + const existing = sections.get(sectionName) || [];
169 + sections.set(sectionName, [...existing, ...sectionEntries]);
170 }
171 }
172 }
173
191 - return sections;
174 + return Array.from(sections.entries());
175 +}
176 +
177 +function formatEntry(entry: LlmsEntry): string {
178 + const displayTitle = entry.parentSection
179 + ? `${entry.parentSection}: ${entry.title}`
180 + : entry.title;
181 +
182 + const link = `[${displayTitle}](${entry.url})`;
183 + return entry.description
184 + ? `- ${link}: ${entry.description}`
185 + : `- ${link}`;
186 }
187
194 -// Generate llms.txt content following best practices
188 function generateLlmsTxt(sections: Array<[string, LlmsEntry[]]>): string {
189 let content = '# Composio\n\n';
190 content += '> Composio is a platform that connects AI agents to 500+ external tools and services through managed authentication, tool execution, and triggers. It simplifies integrating AI agents with SaaS applications, developer tools, and custom APIs.\n\n';
198 - content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n\n';
191 + content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n';
192
200 - // Process all sections
193 for (const [sectionName, entries] of sections) {
202 - content += `## ${sectionName}\n`;
194 + content += `\n## ${sectionName}\n`;
195
204 - // Special handling for Toolkits section only
196 if (sectionName === 'Toolkits') {
206 - content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n`;
197 + content += '- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n';
198 } else {
199 for (const entry of entries) {
209 - // Add parent section prefix if it exists
210 - const displayTitle = entry.parentSection
211 - ? `${entry.parentSection}: ${entry.title}`
212 - : entry.title;
213 -
214 - if (entry.description) {
215 - content += `- [${displayTitle}](${entry.url}): ${entry.description}\n`;
216 - } else {
217 - content += `- [${displayTitle}](${entry.url})\n`;
218 - }
200 + content += formatEntry(entry) + '\n';
201 }
202 }
221 - content += '\n';
203 }
204
205 return content;
206 }
207
227 -// Main function
208 async function main(): Promise<void> {
229 - console.log('🔨 Building llms.txt from docs.yml...');
209 + console.log('Building llms.txt...');
210
211 try {
232 - // Load docs configuration
212 const config = loadDocsConfig();
234 -
235 - // Process navigation to extract all entries
213 const sections = processNavigation(config);
214
238 - // Count total entries
239 - let totalEntries = 0;
240 - for (const [, entries] of sections) {
241 - totalEntries += entries.length;
242 - }
243 -
244 - console.log(`📄 Found ${totalEntries} documentation pages`);
215 + const totalEntries = sections.reduce((total, [_, entries]) => total + entries.length, 0);
216 + console.log(`Found ${totalEntries} pages across ${sections.length} sections`);
217
246 - // Generate llms.txt content
218 const llmsTxtContent = generateLlmsTxt(sections);
248 -
249 - // Write llms.txt
219 ensureOutputDir(LLMS_TXT_PATH);
220 fs.writeFileSync(LLMS_TXT_PATH, llmsTxtContent);
221
253 - console.log(`✅ Generated llms.txt at ${LLMS_TXT_PATH}`);
222 + console.log(`Generated ${LLMS_TXT_PATH}`);
223 } catch (error) {
255 - console.error('❌ Error building llms.txt:', error);
224 + console.error('Error:', error);
225 process.exit(1);
226 }
227 }
228
260 -// Run the script
261 -main().catch(error => {
262 - console.error('❌ Fatal error:', error);
263 - process.exit(1);
264 -});
\ No newline at end of file
229 +// @ts-ignore - bun-specific
230 +if (import.meta.main) {
231 + main().catch(error => {
232 + console.error('Fatal error:', error);
233 + process.exit(1);
234 + });
235 +}
\ No newline at end of file