Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 22, 18:15 UTC
update build scripts
Sushmithamallesh committed
Nov 5, 2025 at 21:25 UTC
f639ad3ac7ca9740dd19cd6983243bcf75c0e534
5 files changed
+482
-362
fern/docs.yml
+14
@@ -351,40 +351,54 @@ navigation:
351
- tab: sdk-reference
352
layout:
353
- section: Python
354
+ slug: python
355
icon: fa-brands fa-python
356
contents:
357
- page: Python SDK Reference
358
+ slug: python-sdk-reference
359
path: pages/dist/sdk/python/sdk-reference.md
360
- section: TypeScript
361
+ slug: type-script
362
icon: fa-brands fa-js
363
contents:
364
- section: Core Classes
365
slug: core-classes
366
contents:
367
- page: Composio
368
+ slug: composio
369
path: pages/dist/sdk/typescript/composio/classes/Composio.mdx
370
- page: ComposioProvider
371
+ slug: composio-provider
372
path: pages/dist/sdk/typescript/provider/ComposioProvider/classes/ComposioProvider.mdx
373
- page: OpenAIProvider
374
+ slug: open-ai-provider
375
path: pages/dist/sdk/typescript/provider/OpenAIProvider/classes/OpenAIProvider.mdx
376
- page: BaseAgenticProvider
377
+ slug: base-agentic-provider
378
path: pages/dist/sdk/typescript/provider/BaseProvider/classes/BaseAgenticProvider.mdx
379
- page: BaseNonAgenticProvider
380
+ slug: base-non-agentic-provider
381
path: pages/dist/sdk/typescript/provider/BaseProvider/classes/BaseNonAgenticProvider.mdx
382
- section: Models
383
slug: models
384
contents:
385
- page: Tools
386
+ slug: tools
387
path: pages/dist/sdk/typescript/models/Tools/classes/Tools.mdx
388
- page: Toolkits
389
+ slug: toolkits
390
path: pages/dist/sdk/typescript/models/Toolkits/classes/Toolkits.mdx
391
- page: Triggers
392
+ slug: triggers
393
path: pages/dist/sdk/typescript/models/Triggers/classes/Triggers.mdx
394
- page: AuthConfigs
395
+ slug: auth-configs
396
path: pages/dist/sdk/typescript/AuthConfigs/classes/AuthConfigs.mdx
397
- page: ConnectedAccounts
398
+ slug: connected-accounts
399
path: pages/dist/sdk/typescript/ConnectedAccounts/classes/ConnectedAccounts.mdx
400
- page: CustomTools
401
+ slug: custom-tools
402
path: pages/dist/sdk/typescript/CustomTools/classes/CustomTools.mdx
403
- tab: api-reference
404
layout:
fern/pages/src/get-started/providers.mdx
+1
@@ -1,5 +1,6 @@
1
---
2
title: Providers
3
+subtitle: Use Composio with any AI framework
4
image: "https://og.composio.dev/api/og?title=Providers" # image for socials
5
keywords: ''
6
hide-nav-links: false
fern/scripts/build-llms-txt.ts
+175
-362
@@ -3,303 +3,136 @@
3
4
import fs from 'fs';
5
import path from 'path';
6
-import yaml from 'yaml';
7
-import { watch } from 'fs';
8
-
9
-// Define directories
10
-const FERN_DIR = path.join(import.meta.dir, '..');
11
-const DOCS_YML_PATH = path.join(FERN_DIR, 'docs.yml');
12
-const LLMS_TXT_PATH = path.join(FERN_DIR, 'llms-txt-worker', 'public', 'llms.txt');
13
-const TOOLS_JSON_PATH = path.join(
6
+import {
7
+ LLMS_TXT_PATH,
8
FERN_DIR,
15
- 'llms-txt-worker',
16
- 'public',
17
- 'robots-only',
18
- 'tools.json'
19
-);
20
-const PAGES_DIR = path.join(FERN_DIR, 'pages');
21
-const TOOLKITS_DIR = path.join(FERN_DIR, 'toolkits');
22
-
23
-interface DocsConfig {
24
- title: string;
25
- navigation: Array<{
26
- tab: string;
27
- layout?: LayoutItem[];
28
- }>;
29
- tabs?: {
30
- [key: string]: {
31
- 'display-name': string;
32
- 'skip-slug'?: boolean;
33
- };
34
- };
35
-}
36
-
37
-interface LayoutItem {
38
- section?: string;
39
- contents?: ContentItem[];
40
- page?: string;
41
- path?: string;
42
- slug?: string;
43
-}
44
-
45
-interface ContentItem {
46
- page?: string;
47
- path?: string;
48
- slug?: string;
49
- section?: string;
50
- contents?: ContentItem[];
51
-}
9
+ loadDocsConfig,
10
+ ensureOutputDir,
11
+ type DocsConfig,
12
+ type ContentItem,
13
+} from './lib/docs-utils';
14
15
interface LlmsEntry {
16
title: string;
17
url: string;
18
description?: string;
19
+ parentSection?: string; // Track parent section for nested items
20
}
21
59
-interface ToolkitInfo {
60
- slug: string;
61
- description: string;
62
- tools: string[];
63
-}
64
-
65
-interface ToolsData {
66
- toolkitNames: string[];
67
- [key: string]: string[] | ToolkitInfo;
68
-}
69
-
70
-// Parse tool MDX files to extract toolkit information
71
-function parseToolMdx(toolMdxPath: string): ToolkitInfo | null {
72
- try {
73
- const fullPath = path.join(TOOLKITS_DIR, toolMdxPath);
74
- if (!fs.existsSync(fullPath)) {
75
- console.warn(`⚠️ Tool file not found: ${fullPath}`);
76
- return null;
77
- }
78
-
79
- const content = fs.readFileSync(fullPath, 'utf8');
80
-
81
- // Extract frontmatter
82
- const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
83
- if (!frontmatterMatch) {
84
- console.warn(`⚠️ No frontmatter found in: ${toolMdxPath}`);
85
- return null;
86
- }
87
-
88
- const frontmatter = frontmatterMatch[1];
89
- const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
90
- if (!titleMatch) {
91
- console.warn(`⚠️ No title found in: ${toolMdxPath}`);
92
- return null;
93
- }
94
-
95
- // Extract SLUG from content
96
- const slugMatch = content.match(/\*\*SLUG\*\*:\s*`([^`]+)`/);
97
- if (!slugMatch) {
98
- console.warn(`⚠️ No SLUG found in: ${toolMdxPath}`);
99
- return null;
100
- }
101
-
102
- // Extract description
103
- const descriptionMatch = content.match(/### Description\s*\n(.+?)(?:\n|$)/);
104
- const description = descriptionMatch ? descriptionMatch[1].trim() : '';
105
-
106
- // Extract tool names from Accordion titles
107
- const toolNames: string[] = [];
108
- const accordionMatches = content.matchAll(/<Accordion title="([^"]+)">/g);
109
-
110
- for (const match of accordionMatches) {
111
- const toolName = match[1];
112
- // Only include tools that start with the toolkit prefix
113
- if (toolName.startsWith(slugMatch[1] + '_')) {
114
- toolNames.push(toolName);
115
- }
116
- }
117
-
118
- return {
119
- slug: slugMatch[1],
120
- description,
121
- tools: toolNames,
122
- };
123
- } catch (error) {
124
- console.warn(`⚠️ Error parsing tool file ${toolMdxPath}:`, error);
125
- return null;
126
- }
127
-}
128
-
129
-// Extract description from MDX content
130
-function extractDescription(mdxPath: string): string | undefined {
22
+// Extract title and description from MDX file frontmatter
23
+function extractFrontmatter(mdxPath: string): { title?: string; description?: string } {
24
try {
25
const fullPath = path.join(FERN_DIR, mdxPath);
26
if (!fs.existsSync(fullPath)) {
134
- console.warn(`⚠️ File not found: ${fullPath}`);
135
- return undefined;
27
+ return {};
28
}
29
30
const content = fs.readFileSync(fullPath, 'utf8');
31
140
- // Try to extract from frontmatter description
32
+ // Try to extract from frontmatter
33
const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
34
if (frontmatterMatch) {
35
const frontmatter = frontmatterMatch[1];
144
- const descMatch = frontmatter.match(/description:\s*["']?(.+?)["']?\s*$/m);
145
- if (descMatch) {
146
- return descMatch[1].trim();
147
- }
36
+
37
+ // Extract title
38
+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
39
+ const title = titleMatch ? titleMatch[1].trim() : undefined;
40
+
41
+ // Extract subtitle only
42
+ const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
43
+ const description = descMatch ? descMatch[1].trim() : undefined;
44
+
45
+ return { title, description };
46
}
47
150
- // Extract first meaningful paragraph after the title
151
- const lines = content.split('\n');
152
- let inFrontmatter = false;
153
- let foundTitle = false;
154
- let description = '';
155
-
156
- for (const line of lines) {
157
- const trimmedLine = line.trim();
158
-
159
- // Handle frontmatter
160
- if (trimmedLine === '---') {
161
- inFrontmatter = !inFrontmatter;
162
- continue;
163
- }
164
-
165
- if (inFrontmatter) continue;
166
-
167
- // Skip titles
168
- if (trimmedLine.startsWith('#')) {
169
- foundTitle = true;
170
- continue;
171
- }
172
-
173
- // Skip empty lines, code blocks, and JSX
174
- if (
175
- !trimmedLine ||
176
- trimmedLine.startsWith('```') ||
177
- trimmedLine.startsWith('<') ||
178
- trimmedLine.startsWith('import ') ||
179
- trimmedLine.startsWith('export ')
180
- ) {
181
- continue;
182
- }
183
-
184
- // Found a paragraph
185
- if (foundTitle && trimmedLine.length > 20) {
186
- description = trimmedLine;
187
- break;
188
- }
189
- }
190
-
191
- // Clean up the description
192
- if (description) {
193
- // Remove MDX components and markdown syntax
194
- description = description
195
- .replace(/<[^>]+>/g, '') // Remove HTML/JSX tags
196
- .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert [text](url) to text
197
- .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks
198
- .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold
199
- .replace(/\*([^*]+)\*/g, '$1') // Remove italic
200
- .trim();
201
-
202
- // Truncate if too long
203
- if (description.length > 150) {
204
- description = description.substring(0, 147) + '...';
205
- }
206
-
207
- return description;
208
- }
209
-
210
- return undefined;
48
+ return {};
49
} catch (error) {
212
- console.warn(`⚠️ Error reading ${mdxPath}:`, error);
213
- return undefined;
50
+ return {};
51
}
52
}
53
217
-// Process content items recursively
218
-function processContentItems(
219
- items: ContentItem[],
54
+// Process a single content item
55
+function processItem(
56
+ item: ContentItem,
57
tabSlug: string,
221
- entries: LlmsEntry[],
222
- toolsData: ToolsData,
223
- parentSlug?: string
224
-): void {
225
- for (const item of items) {
226
- if (item.page && item.path) {
227
- // Check if this is a tool page (in toolkits/ directory) with a SLUG
228
- if (item.path.includes('toolkits/') && item.path.endsWith('.mdx')) {
229
- // Parse tool MDX file and check if it has a SLUG (actual tool page)
230
- const toolMdxFile = item.path.match(/toolkits\/(.+\.mdx)$/)?.[1];
231
- if (toolMdxFile) {
232
- const toolkitInfo = parseToolMdx(toolMdxFile);
233
- if (toolkitInfo) {
234
- // This is an actual tool page with SLUG - add to toolsData and skip main entries
235
- toolsData.toolkitNames.push(toolkitInfo.slug);
236
- toolsData[toolkitInfo.slug] = toolkitInfo;
237
- continue;
238
- }
239
- // If no SLUG found, it's a documentation page in toolkits/ - include in main entries
240
- }
241
- }
58
+ parentSlug?: string,
59
+ parentSection?: string
60
+): LlmsEntry | null {
61
+ // Skip items without a page name and path
62
+ if (!item.page || !item.path) {
63
+ return null;
64
+ }
65
243
- let url: string;
66
+ // Simple URL generation
67
+ let url: string;
68
+
69
+ if (item.slug) {
70
+ // Use explicit slug from docs.yml
71
+ const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
72
+ url = `https://docs.composio.dev/${urlParts.join('/')}`;
73
+ } else if (item.path) {
74
+ // Derive URL from file path
75
+ let urlPath = item.path
76
+ .replace('pages/dist/', '')
77
+ .replace('pages/src/', '')
78
+ .replace('pages/', '')
79
+ .replace('.mdx', '')
80
+ .replace('.md', '');
81
+
82
+ url = `https://docs.composio.dev/${urlPath}`;
83
+ } else {
84
+ // This shouldn't happen - all items should have either slug or path
85
+ console.warn(`Warning: No slug or path for item: ${item.page}`);
86
+ return null;
87
+ }
88
245
- if (item.slug) {
246
- // Use explicit slug if provided
247
- const slugParts = [tabSlug, item.slug].filter(Boolean);
248
- url = `https://docs.composio.dev/${slugParts.join('/')}`;
249
- // Add .mdx extension if not already present
250
- if (!url.endsWith('.mdx') && !url.endsWith('.md')) {
251
- url = url + '.mdx';
252
- }
253
- } else if (item.path) {
254
- // For SDK reference pages without explicit slugs, construct from path
255
- const pathMatch = item.path.match(/pages\/dist\/(.+)$/);
256
- if (pathMatch) {
257
- let constructedPath = pathMatch[1];
258
- // Handle sdk-reference special case
259
- if (tabSlug === 'sdk-reference' && constructedPath.startsWith('sdk/')) {
260
- constructedPath = constructedPath.replace('sdk/', 'sdk-reference/');
261
- // Convert typescript to type-script
262
- constructedPath = constructedPath.replace('typescript/', 'type-script/');
263
- }
264
- url = `https://docs.composio.dev/${constructedPath}`;
265
- } else {
266
- // Fallback
267
- const slugParts = [tabSlug, parentSlug].filter(Boolean);
268
- url = `https://docs.composio.dev/${slugParts.join('/')}/${item.page.toLowerCase().replace(/\s+/g, '-')}.mdx`;
269
- }
270
- } else {
271
- // Fallback
272
- const slugParts = [tabSlug, parentSlug].filter(Boolean);
273
- url = `https://docs.composio.dev/${slugParts.join('/')}/${item.page.toLowerCase().replace(/\s+/g, '-')}.mdx`;
274
- }
89
+ // Try to extract title and description from MDX file
90
+ const frontmatter = extractFrontmatter(item.path);
91
+ const title = frontmatter.title || item.page;
92
+ const description = frontmatter.description;
93
276
- const description = extractDescription(item.path);
94
+ return {
95
+ title,
96
+ url,
97
+ description,
98
+ parentSection,
99
+ };
100
+}
101
278
- // Override specific page titles
279
- let title = item.page;
280
- if (title === 'Overview' && url.includes('mcp-overview')) {
281
- title = 'Composio MCP Servers';
282
- }
102
+// Process content items recursively (flattened)
103
+function processContentItems(
104
+ items: ContentItem[],
105
+ tabSlug: string,
106
+ parentSlug?: string,
107
+ parentSection?: string
108
+): LlmsEntry[] {
109
+ const entries: LlmsEntry[] = [];
110
284
- entries.push({
285
- title: title,
286
- url: url,
287
- description,
288
- });
111
+ for (const item of items) {
112
+ // Process the item itself
113
+ const entry = processItem(item, tabSlug, parentSlug, parentSection);
114
+ if (entry) {
115
+ entries.push(entry);
116
}
117
291
- // Process nested sections
118
+ // Process nested items
119
if (item.section && item.contents) {
293
- const newParentSlug = item.slug || parentSlug;
294
- processContentItems(item.contents, tabSlug, entries, toolsData, newParentSlug);
120
+ // Combine parent slug with item slug for nested sections
121
+ const nestedSlug = item.slug
122
+ ? (parentSlug ? `${parentSlug}/${item.slug}` : item.slug)
123
+ : parentSlug;
124
+ // Pass the section name down to nested items
125
+ const nestedEntries = processContentItems(item.contents, tabSlug, nestedSlug, item.section);
126
+ entries.push(...nestedEntries);
127
}
128
}
129
+
130
+ return entries;
131
}
132
299
-// Process navigation items
300
-function processNavigation(config: DocsConfig): { entries: LlmsEntry[]; toolsData: ToolsData } {
301
- const entries: LlmsEntry[] = [];
302
- const toolsData: ToolsData = { toolkitNames: [] };
133
+// Process all navigation and organize by sections
134
+function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
135
+ const sections: Array<[string, LlmsEntry[]]> = [];
136
137
for (const navItem of config.navigation) {
138
if (!navItem.layout) continue;
@@ -308,144 +141,124 @@ function processNavigation(config: DocsConfig): { entries: LlmsEntry[]; toolsDat
141
const tabConfig = config.tabs?.[tab];
142
const skipSlug = tabConfig?.['skip-slug'] || false;
143
const tabSlug = skipSlug ? '' : tab;
144
+ const tabDisplayName = tabConfig?.['display-name'] || tab;
145
146
+ // Process each layout item as a section
147
for (const layoutItem of navItem.layout) {
148
+ // Determine section name
149
+ let sectionName: string;
150
+ if (tab === 'docs' && layoutItem.section) {
151
+ sectionName = layoutItem.section;
152
+ } else if (layoutItem.section) {
153
+ sectionName = tabDisplayName; // Use tab name for non-docs sections
154
+ } else {
155
+ sectionName = tabDisplayName;
156
+ }
157
+
158
+ const sectionEntries: LlmsEntry[] = [];
159
+
160
// Process direct pages
161
if (layoutItem.page && layoutItem.path && layoutItem.slug) {
315
- const slugParts = [tabSlug, layoutItem.slug].filter(Boolean);
316
- const url = `https://docs.composio.dev/${slugParts.join('/')}`;
317
- const description = extractDescription(layoutItem.path);
318
-
319
- entries.push({
320
- title: layoutItem.page,
321
- url: url.endsWith('.mdx') || url.endsWith('.md') ? url : url + '.mdx',
322
- description,
323
- });
162
+ const entry = processItem(layoutItem, tabSlug);
163
+ if (entry) {
164
+ sectionEntries.push(entry);
165
+ }
166
}
167
168
// Process section contents
327
- if (layoutItem.section && layoutItem.contents) {
328
- processContentItems(layoutItem.contents, tabSlug, entries, toolsData, layoutItem.slug);
169
+ if (layoutItem.contents) {
170
+ const entries = processContentItems(
171
+ layoutItem.contents,
172
+ tabSlug,
173
+ layoutItem.slug,
174
+ undefined // Don't pass parentSection for top-level section items
175
+ );
176
+ sectionEntries.push(...entries);
177
+ }
178
+
179
+ if (sectionEntries.length > 0) {
180
+ // Find existing section or add new one
181
+ const existingIndex = sections.findIndex(([name]) => name === sectionName);
182
+ if (existingIndex >= 0) {
183
+ sections[existingIndex][1].push(...sectionEntries);
184
+ } else {
185
+ sections.push([sectionName, sectionEntries]);
186
+ }
187
}
188
}
189
}
190
333
- return { entries, toolsData };
191
+ return sections;
192
}
193
336
-// Generate llms.txt content
337
-function generateLlmsTxt(entries: LlmsEntry[], toolkitCount: number): string {
338
- let content = '# Composio Docs\n\n## Docs\n\n';
339
-
340
- for (const entry of entries) {
341
- if (entry.description) {
342
- content += `- [${entry.title}](${entry.url}): ${entry.description}\n`;
194
+// Generate llms.txt content following best practices
195
+function generateLlmsTxt(sections: Array<[string, LlmsEntry[]]>): string {
196
+ let content = '# Composio\n\n';
197
+ content += '> Composio is a platform that connects AI agents to 500+ external tools and services through managed authentication, tool execution, and triggers. It simplifies integrating AI agents with SaaS applications, developer tools, and custom APIs.\n\n';
198
+ content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n\n';
199
+
200
+ // Process all sections
201
+ for (const [sectionName, entries] of sections) {
202
+ content += `## ${sectionName}\n`;
203
+
204
+ // Special handling for Toolkits section only
205
+ if (sectionName === 'Toolkits') {
206
+ content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n`;
207
} else {
344
- content += `- [${entry.title}](${entry.url})\n`;
208
+ for (const entry of entries) {
209
+ // Add parent section prefix if it exists
210
+ const displayTitle = entry.parentSection
211
+ ? `${entry.parentSection}: ${entry.title}`
212
+ : entry.title;
213
+
214
+ if (entry.description) {
215
+ content += `- [${displayTitle}](${entry.url}): ${entry.description}\n`;
216
+ } else {
217
+ content += `- [${displayTitle}](${entry.url})\n`;
218
+ }
219
+ }
220
}
221
+ content += '\n';
222
}
223
348
- // Add reference to tools JSON
349
- content += `\n## Complete list of composio tools\n\n`;
350
- content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Complete JSON list of all ${toolkitCount} toolkits with their tools and descriptions\n`;
351
-
224
return content;
225
}
226
355
-// Main build function
356
-async function buildLlmsTxt(): Promise<void> {
227
+// Main function
228
+async function main(): Promise<void> {
229
console.log('🔨 Building llms.txt from docs.yml...');
230
231
try {
360
- // Read and parse docs.yml
361
- const docsYmlContent = fs.readFileSync(DOCS_YML_PATH, 'utf8');
362
- const config = yaml.parse(docsYmlContent) as DocsConfig;
363
-
232
+ // Load docs configuration
233
+ const config = loadDocsConfig();
234
+
235
// Process navigation to extract all entries
365
- const { entries, toolsData } = processNavigation(config);
366
- console.log(`📄 Found ${entries.length} documentation pages`);
236
+ const sections = processNavigation(config);
237
+
238
+ // Count total entries
239
+ let totalEntries = 0;
240
+ for (const [, entries] of sections) {
241
+ totalEntries += entries.length;
242
+ }
243
+
244
+ console.log(`📄 Found ${totalEntries} documentation pages`);
245
246
// Generate llms.txt content
369
- const llmsTxtContent = generateLlmsTxt(entries, toolsData.toolkitNames.length);
370
-
371
- // Ensure output directory exists
372
- const outputDir = path.dirname(LLMS_TXT_PATH);
373
- if (!fs.existsSync(outputDir)) {
374
- fs.mkdirSync(outputDir, { recursive: true });
375
- }
247
+ const llmsTxtContent = generateLlmsTxt(sections);
248
249
// Write llms.txt
250
+ ensureOutputDir(LLMS_TXT_PATH);
251
fs.writeFileSync(LLMS_TXT_PATH, llmsTxtContent);
379
- console.log(`✅ Generated llms.txt at ${path.relative(FERN_DIR, LLMS_TXT_PATH)}`);
380
-
381
- // Write tools.json
382
- const toolsDir = path.dirname(TOOLS_JSON_PATH);
383
- if (!fs.existsSync(toolsDir)) {
384
- fs.mkdirSync(toolsDir, { recursive: true });
385
- }
386
- const toolsJsonContent = JSON.stringify(toolsData, null, 2);
387
- fs.writeFileSync(TOOLS_JSON_PATH, toolsJsonContent);
388
- console.log(
389
- `✅ Generated tools.json at ${path.relative(FERN_DIR, TOOLS_JSON_PATH)} with ${toolsData.toolkitNames.length} toolkits`
390
- );
252
+
253
+ console.log(`✅ Generated llms.txt at ${LLMS_TXT_PATH}`);
254
} catch (error) {
255
console.error('❌ Error building llms.txt:', error);
256
process.exit(1);
257
}
258
}
259
397
-// Watch mode
398
-function startWatchMode(): void {
399
- console.log('👀 Watching for changes...');
400
-
401
- // Watch docs.yml
402
- const docsWatcher = watch(DOCS_YML_PATH, async eventType => {
403
- if (eventType === 'change') {
404
- console.log('🔄 docs.yml changed, rebuilding...');
405
- await buildLlmsTxt();
406
- }
407
- });
408
-
409
- // Watch MDX files in pages directory
410
- const pagesWatcher = watch(PAGES_DIR, { recursive: true }, async (eventType, filename) => {
411
- if (filename && (filename.endsWith('.mdx') || filename.endsWith('.md'))) {
412
- console.log(`🔄 Page changed: ${filename}, rebuilding...`);
413
- await buildLlmsTxt();
414
- }
415
- });
416
-
417
- // Watch MDX files in toolkits directory
418
- const toolkitsWatcher = watch(TOOLKITS_DIR, { recursive: true }, async (eventType, filename) => {
419
- if (filename && (filename.endsWith('.mdx') || filename.endsWith('.md'))) {
420
- console.log(`🔄 Toolkit doc changed: ${filename}, rebuilding...`);
421
- await buildLlmsTxt();
422
- }
423
- });
424
-
425
- // Cleanup on exit
426
- process.on('SIGINT', () => {
427
- console.log('\n👋 Stopping watchers...');
428
- docsWatcher.close();
429
- pagesWatcher.close();
430
- toolkitsWatcher.close();
431
- process.exit(0);
432
- });
433
-}
434
-
435
-// Main function
436
-async function main(): Promise<void> {
437
- await buildLlmsTxt();
438
-
439
- // Check if we should run in watch mode
440
- const isWatchMode = process.argv.includes('--watch') || process.argv.includes('-w');
441
-
442
- if (isWatchMode) {
443
- startWatchMode();
444
- }
445
-}
446
-
260
// Run the script
261
main().catch(error => {
262
console.error('❌ Fatal error:', error);
263
process.exit(1);
451
-});
264
+});
\ No newline at end of file
fern/scripts/build-tools-json.ts
new
+142
@@ -0,0 +1,142 @@
1
+#!/usr/bin/env bun
2
+/* eslint-disable no-console */
3
+
4
+import fs from 'fs';
5
+import path from 'path';
6
+import { TOOLKITS_DIR, TOOLS_JSON_PATH, ensureOutputDir } from './lib/docs-utils';
7
+
8
+interface ToolkitInfo {
9
+ slug: string;
10
+ description: string;
11
+ tools: string[];
12
+}
13
+
14
+interface ToolsData {
15
+ toolkitNames: string[];
16
+ [key: string]: string[] | ToolkitInfo;
17
+}
18
+
19
+// Parse tool MDX files to extract toolkit information
20
+function parseToolkitMdx(toolMdxPath: string): ToolkitInfo | null {
21
+ try {
22
+ const fullPath = path.join(TOOLKITS_DIR, toolMdxPath);
23
+ if (!fs.existsSync(fullPath)) {
24
+ console.warn(`⚠️ Tool file not found: ${fullPath}`);
25
+ return null;
26
+ }
27
+
28
+ const content = fs.readFileSync(fullPath, 'utf8');
29
+
30
+ // Extract frontmatter
31
+ const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
32
+ if (!frontmatterMatch) {
33
+ console.warn(`⚠️ No frontmatter found in: ${toolMdxPath}`);
34
+ return null;
35
+ }
36
+
37
+ const frontmatter = frontmatterMatch[1];
38
+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
39
+ if (!titleMatch) {
40
+ console.warn(`⚠️ No title found in: ${toolMdxPath}`);
41
+ return null;
42
+ }
43
+
44
+ // Extract SLUG from content
45
+ const slugMatch = content.match(/\*\*SLUG\*\*:\s*`([^`]+)`/);
46
+ if (!slugMatch) {
47
+ // Not a toolkit page, skip
48
+ return null;
49
+ }
50
+
51
+ // Extract description
52
+ const descriptionMatch = content.match(/### Description\s*\n(.+?)(?:\n|$)/);
53
+ const description = descriptionMatch ? descriptionMatch[1].trim() : '';
54
+
55
+ // Extract tool names from Accordion titles
56
+ const toolNames: string[] = [];
57
+ const accordionMatches = content.matchAll(/<Accordion title="([^"]+)">/g);
58
+
59
+ for (const match of accordionMatches) {
60
+ const toolName = match[1];
61
+ // Only include tools that start with the toolkit prefix
62
+ if (toolName.startsWith(slugMatch[1] + '_')) {
63
+ toolNames.push(toolName);
64
+ }
65
+ }
66
+
67
+ return {
68
+ slug: slugMatch[1],
69
+ description,
70
+ tools: toolNames,
71
+ };
72
+ } catch (error) {
73
+ console.warn(`⚠️ Error parsing tool file ${toolMdxPath}:`, error);
74
+ return null;
75
+ }
76
+}
77
+
78
+// Scan toolkits directory and build tools data
79
+function buildToolsData(): ToolsData {
80
+ const toolsData: ToolsData = { toolkitNames: [] };
81
+
82
+ if (!fs.existsSync(TOOLKITS_DIR)) {
83
+ console.warn(`⚠️ Toolkits directory not found: ${TOOLKITS_DIR}`);
84
+ return toolsData;
85
+ }
86
+
87
+ // Read all MDX files in toolkits directory
88
+ const files = fs.readdirSync(TOOLKITS_DIR);
89
+ const mdxFiles = files.filter(file => file.endsWith('.mdx'));
90
+
91
+ console.log(`📂 Found ${mdxFiles.length} MDX files in toolkits directory`);
92
+
93
+ for (const file of mdxFiles) {
94
+ const toolkitInfo = parseToolkitMdx(file);
95
+ if (toolkitInfo) {
96
+ toolsData.toolkitNames.push(toolkitInfo.slug);
97
+ toolsData[toolkitInfo.slug] = toolkitInfo;
98
+ console.log(` ✓ ${toolkitInfo.slug}: ${toolkitInfo.tools.length} tools`);
99
+ }
100
+ }
101
+
102
+ // Sort toolkit names alphabetically
103
+ toolsData.toolkitNames.sort();
104
+
105
+ return toolsData;
106
+}
107
+
108
+// Main function
109
+async function main(): Promise<void> {
110
+ console.log('🔨 Building tools.json...');
111
+
112
+ try {
113
+ // Build tools data
114
+ const toolsData = buildToolsData();
115
+
116
+ console.log(`\n📊 Summary:`);
117
+ console.log(` - Total toolkits: ${toolsData.toolkitNames.length}`);
118
+
119
+ let totalTools = 0;
120
+ for (const name of toolsData.toolkitNames) {
121
+ const toolkit = toolsData[name] as ToolkitInfo;
122
+ totalTools += toolkit.tools.length;
123
+ }
124
+ console.log(` - Total tools: ${totalTools}`);
125
+
126
+ // Write tools.json
127
+ ensureOutputDir(TOOLS_JSON_PATH);
128
+ const toolsJsonContent = JSON.stringify(toolsData, null, 2);
129
+ fs.writeFileSync(TOOLS_JSON_PATH, toolsJsonContent);
130
+
131
+ console.log(`\n✅ Generated tools.json at ${TOOLS_JSON_PATH}`);
132
+ } catch (error) {
133
+ console.error('❌ Error building tools.json:', error);
134
+ process.exit(1);
135
+ }
136
+}
137
+
138
+// Run the script
139
+main().catch(error => {
140
+ console.error('❌ Fatal error:', error);
141
+ process.exit(1);
142
+});
\ No newline at end of file
fern/scripts/lib/docs-utils.ts
new
+150
@@ -0,0 +1,150 @@
1
+#!/usr/bin/env bun
2
+
3
+import fs from 'fs';
4
+import path from 'path';
5
+import yaml from 'yaml';
6
+
7
+// Define directories
8
+export const FERN_DIR = path.join(import.meta.dir, '../..');
9
+export const DOCS_YML_PATH = path.join(FERN_DIR, 'docs.yml');
10
+export const LLMS_TXT_PATH = path.join(FERN_DIR, 'llms-txt-worker', 'public', 'llms.txt');
11
+export const TOOLS_JSON_PATH = path.join(
12
+ FERN_DIR,
13
+ 'llms-txt-worker',
14
+ 'public',
15
+ 'robots-only',
16
+ 'tools.json'
17
+);
18
+export const PAGES_DIR = path.join(FERN_DIR, 'pages');
19
+export const TOOLKITS_DIR = path.join(FERN_DIR, 'toolkits');
20
+
21
+export interface DocsConfig {
22
+ title: string;
23
+ navigation: Array<{
24
+ tab: string;
25
+ layout?: LayoutItem[];
26
+ }>;
27
+ tabs?: {
28
+ [key: string]: {
29
+ 'display-name': string;
30
+ 'skip-slug'?: boolean;
31
+ };
32
+ };
33
+}
34
+
35
+export interface LayoutItem {
36
+ section?: string;
37
+ contents?: ContentItem[];
38
+ page?: string;
39
+ path?: string;
40
+ slug?: string;
41
+}
42
+
43
+export interface ContentItem {
44
+ page?: string;
45
+ path?: string;
46
+ slug?: string;
47
+ section?: string;
48
+ contents?: ContentItem[];
49
+}
50
+
51
+export function loadDocsConfig(): DocsConfig {
52
+ const docsYmlContent = fs.readFileSync(DOCS_YML_PATH, 'utf8');
53
+ return yaml.parse(docsYmlContent) as DocsConfig;
54
+}
55
+
56
+// Extract description from MDX content
57
+export function extractDescription(mdxPath: string): string | undefined {
58
+ try {
59
+ const fullPath = path.join(FERN_DIR, mdxPath);
60
+ if (!fs.existsSync(fullPath)) {
61
+ console.warn(`⚠️ File not found: ${fullPath}`);
62
+ return undefined;
63
+ }
64
+
65
+ const content = fs.readFileSync(fullPath, 'utf8');
66
+
67
+ // Try to extract from frontmatter description
68
+ const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
69
+ if (frontmatterMatch) {
70
+ const frontmatter = frontmatterMatch[1];
71
+ const descMatch = frontmatter.match(/description:\s*["']?(.+?)["']?\s*$/m);
72
+ if (descMatch) {
73
+ return descMatch[1].trim();
74
+ }
75
+ }
76
+
77
+ // Extract first meaningful paragraph after the title
78
+ const lines = content.split('\n');
79
+ let inFrontmatter = false;
80
+ let foundTitle = false;
81
+ let description = '';
82
+
83
+ for (const line of lines) {
84
+ const trimmedLine = line.trim();
85
+
86
+ // Handle frontmatter
87
+ if (trimmedLine === '---') {
88
+ inFrontmatter = !inFrontmatter;
89
+ continue;
90
+ }
91
+
92
+ if (inFrontmatter) continue;
93
+
94
+ // Skip titles
95
+ if (trimmedLine.startsWith('#')) {
96
+ foundTitle = true;
97
+ continue;
98
+ }
99
+
100
+ // Skip empty lines, code blocks, and JSX
101
+ if (
102
+ !trimmedLine ||
103
+ trimmedLine.startsWith('```') ||
104
+ trimmedLine.startsWith('<') ||
105
+ trimmedLine.startsWith('import ') ||
106
+ trimmedLine.startsWith('export ')
107
+ ) {
108
+ continue;
109
+ }
110
+
111
+ // Found a paragraph
112
+ if (foundTitle && trimmedLine.length > 20) {
113
+ description = trimmedLine;
114
+ break;
115
+ }
116
+ }
117
+
118
+ // Clean up the description
119
+ if (description) {
120
+ // Remove MDX components and markdown syntax
121
+ description = description
122
+ .replace(/<[^>]+>/g, '') // Remove HTML/JSX tags
123
+ .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert [text](url) to text
124
+ .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks
125
+ .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold
126
+ .replace(/\*([^*]+)\*/g, '$1') // Remove italic
127
+ .trim();
128
+
129
+ // Truncate if too long
130
+ if (description.length > 150) {
131
+ description = description.substring(0, 147) + '...';
132
+ }
133
+
134
+ return description;
135
+ }
136
+
137
+ return undefined;
138
+ } catch (error) {
139
+ console.warn(`⚠️ Error reading ${mdxPath}:`, error);
140
+ return undefined;
141
+ }
142
+}
143
+
144
+// Ensure output directory exists
145
+export function ensureOutputDir(filePath: string): void {
146
+ const dir = path.dirname(filePath);
147
+ if (!fs.existsSync(dir)) {
148
+ fs.mkdirSync(dir, { recursive: true });
149
+ }
150
+}
\ No newline at end of file