@devries-cameron20 / composio-1 / commits / f639ad3ac
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

update build scripts

Sushmithamallesh committed Nov 5, 2025 at 21:25 UTC f639ad3ac7ca9740dd19cd6983243bcf75c0e534
5 files changed +482 -362
fern/docs.yml
+14
@@ -351,40 +351,54 @@ navigation:
351 - tab: sdk-reference
352 layout:
353 - section: Python
354 + slug: python
355 icon: fa-brands fa-python
356 contents:
357 - page: Python SDK Reference
358 + slug: python-sdk-reference
359 path: pages/dist/sdk/python/sdk-reference.md
360 - section: TypeScript
361 + slug: type-script
362 icon: fa-brands fa-js
363 contents:
364 - section: Core Classes
365 slug: core-classes
366 contents:
367 - page: Composio
368 + slug: composio
369 path: pages/dist/sdk/typescript/composio/classes/Composio.mdx
370 - page: ComposioProvider
371 + slug: composio-provider
372 path: pages/dist/sdk/typescript/provider/ComposioProvider/classes/ComposioProvider.mdx
373 - page: OpenAIProvider
374 + slug: open-ai-provider
375 path: pages/dist/sdk/typescript/provider/OpenAIProvider/classes/OpenAIProvider.mdx
376 - page: BaseAgenticProvider
377 + slug: base-agentic-provider
378 path: pages/dist/sdk/typescript/provider/BaseProvider/classes/BaseAgenticProvider.mdx
379 - page: BaseNonAgenticProvider
380 + slug: base-non-agentic-provider
381 path: pages/dist/sdk/typescript/provider/BaseProvider/classes/BaseNonAgenticProvider.mdx
382 - section: Models
383 slug: models
384 contents:
385 - page: Tools
386 + slug: tools
387 path: pages/dist/sdk/typescript/models/Tools/classes/Tools.mdx
388 - page: Toolkits
389 + slug: toolkits
390 path: pages/dist/sdk/typescript/models/Toolkits/classes/Toolkits.mdx
391 - page: Triggers
392 + slug: triggers
393 path: pages/dist/sdk/typescript/models/Triggers/classes/Triggers.mdx
394 - page: AuthConfigs
395 + slug: auth-configs
396 path: pages/dist/sdk/typescript/AuthConfigs/classes/AuthConfigs.mdx
397 - page: ConnectedAccounts
398 + slug: connected-accounts
399 path: pages/dist/sdk/typescript/ConnectedAccounts/classes/ConnectedAccounts.mdx
400 - page: CustomTools
401 + slug: custom-tools
402 path: pages/dist/sdk/typescript/CustomTools/classes/CustomTools.mdx
403 - tab: api-reference
404 layout:
fern/pages/src/get-started/providers.mdx
+1
@@ -1,5 +1,6 @@
1 ---
2 title: Providers
3 +subtitle: Use Composio with any AI framework
4 image: "https://og.composio.dev/api/og?title=Providers" # image for socials
5 keywords: ''
6 hide-nav-links: false
fern/scripts/build-llms-txt.ts
+175 -362
@@ -3,303 +3,136 @@
3
4 import fs from 'fs';
5 import path from 'path';
6 -import yaml from 'yaml';
7 -import { watch } from 'fs';
8 -
9 -// Define directories
10 -const FERN_DIR = path.join(import.meta.dir, '..');
11 -const DOCS_YML_PATH = path.join(FERN_DIR, 'docs.yml');
12 -const LLMS_TXT_PATH = path.join(FERN_DIR, 'llms-txt-worker', 'public', 'llms.txt');
13 -const TOOLS_JSON_PATH = path.join(
6 +import {
7 + LLMS_TXT_PATH,
8 FERN_DIR,
15 - 'llms-txt-worker',
16 - 'public',
17 - 'robots-only',
18 - 'tools.json'
19 -);
20 -const PAGES_DIR = path.join(FERN_DIR, 'pages');
21 -const TOOLKITS_DIR = path.join(FERN_DIR, 'toolkits');
22 -
23 -interface DocsConfig {
24 - title: string;
25 - navigation: Array<{
26 - tab: string;
27 - layout?: LayoutItem[];
28 - }>;
29 - tabs?: {
30 - [key: string]: {
31 - 'display-name': string;
32 - 'skip-slug'?: boolean;
33 - };
34 - };
35 -}
36 -
37 -interface LayoutItem {
38 - section?: string;
39 - contents?: ContentItem[];
40 - page?: string;
41 - path?: string;
42 - slug?: string;
43 -}
44 -
45 -interface ContentItem {
46 - page?: string;
47 - path?: string;
48 - slug?: string;
49 - section?: string;
50 - contents?: ContentItem[];
51 -}
9 + loadDocsConfig,
10 + ensureOutputDir,
11 + type DocsConfig,
12 + type ContentItem,
13 +} from './lib/docs-utils';
14
15 interface LlmsEntry {
16 title: string;
17 url: string;
18 description?: string;
19 + parentSection?: string; // Track parent section for nested items
20 }
21
59 -interface ToolkitInfo {
60 - slug: string;
61 - description: string;
62 - tools: string[];
63 -}
64 -
65 -interface ToolsData {
66 - toolkitNames: string[];
67 - [key: string]: string[] | ToolkitInfo;
68 -}
69 -
70 -// Parse tool MDX files to extract toolkit information
71 -function parseToolMdx(toolMdxPath: string): ToolkitInfo | null {
72 - try {
73 - const fullPath = path.join(TOOLKITS_DIR, toolMdxPath);
74 - if (!fs.existsSync(fullPath)) {
75 - console.warn(`⚠️ Tool file not found: ${fullPath}`);
76 - return null;
77 - }
78 -
79 - const content = fs.readFileSync(fullPath, 'utf8');
80 -
81 - // Extract frontmatter
82 - const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
83 - if (!frontmatterMatch) {
84 - console.warn(`⚠️ No frontmatter found in: ${toolMdxPath}`);
85 - return null;
86 - }
87 -
88 - const frontmatter = frontmatterMatch[1];
89 - const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
90 - if (!titleMatch) {
91 - console.warn(`⚠️ No title found in: ${toolMdxPath}`);
92 - return null;
93 - }
94 -
95 - // Extract SLUG from content
96 - const slugMatch = content.match(/\*\*SLUG\*\*:\s*`([^`]+)`/);
97 - if (!slugMatch) {
98 - console.warn(`⚠️ No SLUG found in: ${toolMdxPath}`);
99 - return null;
100 - }
101 -
102 - // Extract description
103 - const descriptionMatch = content.match(/### Description\s*\n(.+?)(?:\n|$)/);
104 - const description = descriptionMatch ? descriptionMatch[1].trim() : '';
105 -
106 - // Extract tool names from Accordion titles
107 - const toolNames: string[] = [];
108 - const accordionMatches = content.matchAll(/<Accordion title="([^"]+)">/g);
109 -
110 - for (const match of accordionMatches) {
111 - const toolName = match[1];
112 - // Only include tools that start with the toolkit prefix
113 - if (toolName.startsWith(slugMatch[1] + '_')) {
114 - toolNames.push(toolName);
115 - }
116 - }
117 -
118 - return {
119 - slug: slugMatch[1],
120 - description,
121 - tools: toolNames,
122 - };
123 - } catch (error) {
124 - console.warn(`⚠️ Error parsing tool file ${toolMdxPath}:`, error);
125 - return null;
126 - }
127 -}
128 -
129 -// Extract description from MDX content
130 -function extractDescription(mdxPath: string): string | undefined {
22 +// Extract title and description from MDX file frontmatter
23 +function extractFrontmatter(mdxPath: string): { title?: string; description?: string } {
24 try {
25 const fullPath = path.join(FERN_DIR, mdxPath);
26 if (!fs.existsSync(fullPath)) {
134 - console.warn(`⚠️ File not found: ${fullPath}`);
135 - return undefined;
27 + return {};
28 }
29
30 const content = fs.readFileSync(fullPath, 'utf8');
31
140 - // Try to extract from frontmatter description
32 + // Try to extract from frontmatter
33 const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
34 if (frontmatterMatch) {
35 const frontmatter = frontmatterMatch[1];
144 - const descMatch = frontmatter.match(/description:\s*["']?(.+?)["']?\s*$/m);
145 - if (descMatch) {
146 - return descMatch[1].trim();
147 - }
36 +
37 + // Extract title
38 + const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
39 + const title = titleMatch ? titleMatch[1].trim() : undefined;
40 +
41 + // Extract subtitle only
42 + const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
43 + const description = descMatch ? descMatch[1].trim() : undefined;
44 +
45 + return { title, description };
46 }
47
150 - // Extract first meaningful paragraph after the title
151 - const lines = content.split('\n');
152 - let inFrontmatter = false;
153 - let foundTitle = false;
154 - let description = '';
155 -
156 - for (const line of lines) {
157 - const trimmedLine = line.trim();
158 -
159 - // Handle frontmatter
160 - if (trimmedLine === '---') {
161 - inFrontmatter = !inFrontmatter;
162 - continue;
163 - }
164 -
165 - if (inFrontmatter) continue;
166 -
167 - // Skip titles
168 - if (trimmedLine.startsWith('#')) {
169 - foundTitle = true;
170 - continue;
171 - }
172 -
173 - // Skip empty lines, code blocks, and JSX
174 - if (
175 - !trimmedLine ||
176 - trimmedLine.startsWith('```') ||
177 - trimmedLine.startsWith('<') ||
178 - trimmedLine.startsWith('import ') ||
179 - trimmedLine.startsWith('export ')
180 - ) {
181 - continue;
182 - }
183 -
184 - // Found a paragraph
185 - if (foundTitle && trimmedLine.length > 20) {
186 - description = trimmedLine;
187 - break;
188 - }
189 - }
190 -
191 - // Clean up the description
192 - if (description) {
193 - // Remove MDX components and markdown syntax
194 - description = description
195 - .replace(/<[^>]+>/g, '') // Remove HTML/JSX tags
196 - .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert [text](url) to text
197 - .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks
198 - .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold
199 - .replace(/\*([^*]+)\*/g, '$1') // Remove italic
200 - .trim();
201 -
202 - // Truncate if too long
203 - if (description.length > 150) {
204 - description = description.substring(0, 147) + '...';
205 - }
206 -
207 - return description;
208 - }
209 -
210 - return undefined;
48 + return {};
49 } catch (error) {
212 - console.warn(`⚠️ Error reading ${mdxPath}:`, error);
213 - return undefined;
50 + return {};
51 }
52 }
53
217 -// Process content items recursively
218 -function processContentItems(
219 - items: ContentItem[],
54 +// Process a single content item
55 +function processItem(
56 + item: ContentItem,
57 tabSlug: string,
221 - entries: LlmsEntry[],
222 - toolsData: ToolsData,
223 - parentSlug?: string
224 -): void {
225 - for (const item of items) {
226 - if (item.page && item.path) {
227 - // Check if this is a tool page (in toolkits/ directory) with a SLUG
228 - if (item.path.includes('toolkits/') && item.path.endsWith('.mdx')) {
229 - // Parse tool MDX file and check if it has a SLUG (actual tool page)
230 - const toolMdxFile = item.path.match(/toolkits\/(.+\.mdx)$/)?.[1];
231 - if (toolMdxFile) {
232 - const toolkitInfo = parseToolMdx(toolMdxFile);
233 - if (toolkitInfo) {
234 - // This is an actual tool page with SLUG - add to toolsData and skip main entries
235 - toolsData.toolkitNames.push(toolkitInfo.slug);
236 - toolsData[toolkitInfo.slug] = toolkitInfo;
237 - continue;
238 - }
239 - // If no SLUG found, it's a documentation page in toolkits/ - include in main entries
240 - }
241 - }
58 + parentSlug?: string,
59 + parentSection?: string
60 +): LlmsEntry | null {
61 + // Skip items without a page name and path
62 + if (!item.page || !item.path) {
63 + return null;
64 + }
65
243 - let url: string;
66 + // Simple URL generation
67 + let url: string;
68 +
69 + if (item.slug) {
70 + // Use explicit slug from docs.yml
71 + const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
72 + url = `https://docs.composio.dev/${urlParts.join('/')}`;
73 + } else if (item.path) {
74 + // Derive URL from file path
75 + let urlPath = item.path
76 + .replace('pages/dist/', '')
77 + .replace('pages/src/', '')
78 + .replace('pages/', '')
79 + .replace('.mdx', '')
80 + .replace('.md', '');
81 +
82 + url = `https://docs.composio.dev/${urlPath}`;
83 + } else {
84 + // This shouldn't happen - all items should have either slug or path
85 + console.warn(`Warning: No slug or path for item: ${item.page}`);
86 + return null;
87 + }
88
245 - if (item.slug) {
246 - // Use explicit slug if provided
247 - const slugParts = [tabSlug, item.slug].filter(Boolean);
248 - url = `https://docs.composio.dev/${slugParts.join('/')}`;
249 - // Add .mdx extension if not already present
250 - if (!url.endsWith('.mdx') && !url.endsWith('.md')) {
251 - url = url + '.mdx';
252 - }
253 - } else if (item.path) {
254 - // For SDK reference pages without explicit slugs, construct from path
255 - const pathMatch = item.path.match(/pages\/dist\/(.+)$/);
256 - if (pathMatch) {
257 - let constructedPath = pathMatch[1];
258 - // Handle sdk-reference special case
259 - if (tabSlug === 'sdk-reference' && constructedPath.startsWith('sdk/')) {
260 - constructedPath = constructedPath.replace('sdk/', 'sdk-reference/');
261 - // Convert typescript to type-script
262 - constructedPath = constructedPath.replace('typescript/', 'type-script/');
263 - }
264 - url = `https://docs.composio.dev/${constructedPath}`;
265 - } else {
266 - // Fallback
267 - const slugParts = [tabSlug, parentSlug].filter(Boolean);
268 - url = `https://docs.composio.dev/${slugParts.join('/')}/${item.page.toLowerCase().replace(/\s+/g, '-')}.mdx`;
269 - }
270 - } else {
271 - // Fallback
272 - const slugParts = [tabSlug, parentSlug].filter(Boolean);
273 - url = `https://docs.composio.dev/${slugParts.join('/')}/${item.page.toLowerCase().replace(/\s+/g, '-')}.mdx`;
274 - }
89 + // Try to extract title and description from MDX file
90 + const frontmatter = extractFrontmatter(item.path);
91 + const title = frontmatter.title || item.page;
92 + const description = frontmatter.description;
93
276 - const description = extractDescription(item.path);
94 + return {
95 + title,
96 + url,
97 + description,
98 + parentSection,
99 + };
100 +}
101
278 - // Override specific page titles
279 - let title = item.page;
280 - if (title === 'Overview' && url.includes('mcp-overview')) {
281 - title = 'Composio MCP Servers';
282 - }
102 +// Process content items recursively (flattened)
103 +function processContentItems(
104 + items: ContentItem[],
105 + tabSlug: string,
106 + parentSlug?: string,
107 + parentSection?: string
108 +): LlmsEntry[] {
109 + const entries: LlmsEntry[] = [];
110
284 - entries.push({
285 - title: title,
286 - url: url,
287 - description,
288 - });
111 + for (const item of items) {
112 + // Process the item itself
113 + const entry = processItem(item, tabSlug, parentSlug, parentSection);
114 + if (entry) {
115 + entries.push(entry);
116 }
117
291 - // Process nested sections
118 + // Process nested items
119 if (item.section && item.contents) {
293 - const newParentSlug = item.slug || parentSlug;
294 - processContentItems(item.contents, tabSlug, entries, toolsData, newParentSlug);
120 + // Combine parent slug with item slug for nested sections
121 + const nestedSlug = item.slug
122 + ? (parentSlug ? `${parentSlug}/${item.slug}` : item.slug)
123 + : parentSlug;
124 + // Pass the section name down to nested items
125 + const nestedEntries = processContentItems(item.contents, tabSlug, nestedSlug, item.section);
126 + entries.push(...nestedEntries);
127 }
128 }
129 +
130 + return entries;
131 }
132
299 -// Process navigation items
300 -function processNavigation(config: DocsConfig): { entries: LlmsEntry[]; toolsData: ToolsData } {
301 - const entries: LlmsEntry[] = [];
302 - const toolsData: ToolsData = { toolkitNames: [] };
133 +// Process all navigation and organize by sections
134 +function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
135 + const sections: Array<[string, LlmsEntry[]]> = [];
136
137 for (const navItem of config.navigation) {
138 if (!navItem.layout) continue;
@@ -308,144 +141,124 @@ function processNavigation(config: DocsConfig): { entries: LlmsEntry[]; toolsDat
141 const tabConfig = config.tabs?.[tab];
142 const skipSlug = tabConfig?.['skip-slug'] || false;
143 const tabSlug = skipSlug ? '' : tab;
144 + const tabDisplayName = tabConfig?.['display-name'] || tab;
145
146 + // Process each layout item as a section
147 for (const layoutItem of navItem.layout) {
148 + // Determine section name
149 + let sectionName: string;
150 + if (tab === 'docs' && layoutItem.section) {
151 + sectionName = layoutItem.section;
152 + } else if (layoutItem.section) {
153 + sectionName = tabDisplayName; // Use tab name for non-docs sections
154 + } else {
155 + sectionName = tabDisplayName;
156 + }
157 +
158 + const sectionEntries: LlmsEntry[] = [];
159 +
160 // Process direct pages
161 if (layoutItem.page && layoutItem.path && layoutItem.slug) {
315 - const slugParts = [tabSlug, layoutItem.slug].filter(Boolean);
316 - const url = `https://docs.composio.dev/${slugParts.join('/')}`;
317 - const description = extractDescription(layoutItem.path);
318 -
319 - entries.push({
320 - title: layoutItem.page,
321 - url: url.endsWith('.mdx') || url.endsWith('.md') ? url : url + '.mdx',
322 - description,
323 - });
162 + const entry = processItem(layoutItem, tabSlug);
163 + if (entry) {
164 + sectionEntries.push(entry);
165 + }
166 }
167
168 // Process section contents
327 - if (layoutItem.section && layoutItem.contents) {
328 - processContentItems(layoutItem.contents, tabSlug, entries, toolsData, layoutItem.slug);
169 + if (layoutItem.contents) {
170 + const entries = processContentItems(
171 + layoutItem.contents,
172 + tabSlug,
173 + layoutItem.slug,
174 + undefined // Don't pass parentSection for top-level section items
175 + );
176 + sectionEntries.push(...entries);
177 + }
178 +
179 + if (sectionEntries.length > 0) {
180 + // Find existing section or add new one
181 + const existingIndex = sections.findIndex(([name]) => name === sectionName);
182 + if (existingIndex >= 0) {
183 + sections[existingIndex][1].push(...sectionEntries);
184 + } else {
185 + sections.push([sectionName, sectionEntries]);
186 + }
187 }
188 }
189 }
190
333 - return { entries, toolsData };
191 + return sections;
192 }
193
336 -// Generate llms.txt content
337 -function generateLlmsTxt(entries: LlmsEntry[], toolkitCount: number): string {
338 - let content = '# Composio Docs\n\n## Docs\n\n';
339 -
340 - for (const entry of entries) {
341 - if (entry.description) {
342 - content += `- [${entry.title}](${entry.url}): ${entry.description}\n`;
194 +// Generate llms.txt content following best practices
195 +function generateLlmsTxt(sections: Array<[string, LlmsEntry[]]>): string {
196 + let content = '# Composio\n\n';
197 + content += '> Composio is a platform that connects AI agents to 500+ external tools and services through managed authentication, tool execution, and triggers. It simplifies integrating AI agents with SaaS applications, developer tools, and custom APIs.\n\n';
198 + content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n\n';
199 +
200 + // Process all sections
201 + for (const [sectionName, entries] of sections) {
202 + content += `## ${sectionName}\n`;
203 +
204 + // Special handling for Toolkits section only
205 + if (sectionName === 'Toolkits') {
206 + content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n`;
207 } else {
344 - content += `- [${entry.title}](${entry.url})\n`;
208 + for (const entry of entries) {
209 + // Add parent section prefix if it exists
210 + const displayTitle = entry.parentSection
211 + ? `${entry.parentSection}: ${entry.title}`
212 + : entry.title;
213 +
214 + if (entry.description) {
215 + content += `- [${displayTitle}](${entry.url}): ${entry.description}\n`;
216 + } else {
217 + content += `- [${displayTitle}](${entry.url})\n`;
218 + }
219 + }
220 }
221 + content += '\n';
222 }
223
348 - // Add reference to tools JSON
349 - content += `\n## Complete list of composio tools\n\n`;
350 - content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Complete JSON list of all ${toolkitCount} toolkits with their tools and descriptions\n`;
351 -
224 return content;
225 }
226
355 -// Main build function
356 -async function buildLlmsTxt(): Promise<void> {
227 +// Main function
228 +async function main(): Promise<void> {
229 console.log('🔨 Building llms.txt from docs.yml...');
230
231 try {
360 - // Read and parse docs.yml
361 - const docsYmlContent = fs.readFileSync(DOCS_YML_PATH, 'utf8');
362 - const config = yaml.parse(docsYmlContent) as DocsConfig;
363 -
232 + // Load docs configuration
233 + const config = loadDocsConfig();
234 +
235 // Process navigation to extract all entries
365 - const { entries, toolsData } = processNavigation(config);
366 - console.log(`📄 Found ${entries.length} documentation pages`);
236 + const sections = processNavigation(config);
237 +
238 + // Count total entries
239 + let totalEntries = 0;
240 + for (const [, entries] of sections) {
241 + totalEntries += entries.length;
242 + }
243 +
244 + console.log(`📄 Found ${totalEntries} documentation pages`);
245
246 // Generate llms.txt content
369 - const llmsTxtContent = generateLlmsTxt(entries, toolsData.toolkitNames.length);
370 -
371 - // Ensure output directory exists
372 - const outputDir = path.dirname(LLMS_TXT_PATH);
373 - if (!fs.existsSync(outputDir)) {
374 - fs.mkdirSync(outputDir, { recursive: true });
375 - }
247 + const llmsTxtContent = generateLlmsTxt(sections);
248
249 // Write llms.txt
250 + ensureOutputDir(LLMS_TXT_PATH);
251 fs.writeFileSync(LLMS_TXT_PATH, llmsTxtContent);
379 - console.log(`✅ Generated llms.txt at ${path.relative(FERN_DIR, LLMS_TXT_PATH)}`);
380 -
381 - // Write tools.json
382 - const toolsDir = path.dirname(TOOLS_JSON_PATH);
383 - if (!fs.existsSync(toolsDir)) {
384 - fs.mkdirSync(toolsDir, { recursive: true });
385 - }
386 - const toolsJsonContent = JSON.stringify(toolsData, null, 2);
387 - fs.writeFileSync(TOOLS_JSON_PATH, toolsJsonContent);
388 - console.log(
389 - `✅ Generated tools.json at ${path.relative(FERN_DIR, TOOLS_JSON_PATH)} with ${toolsData.toolkitNames.length} toolkits`
390 - );
252 +
253 + console.log(`✅ Generated llms.txt at ${LLMS_TXT_PATH}`);
254 } catch (error) {
255 console.error('❌ Error building llms.txt:', error);
256 process.exit(1);
257 }
258 }
259
397 -// Watch mode
398 -function startWatchMode(): void {
399 - console.log('👀 Watching for changes...');
400 -
401 - // Watch docs.yml
402 - const docsWatcher = watch(DOCS_YML_PATH, async eventType => {
403 - if (eventType === 'change') {
404 - console.log('🔄 docs.yml changed, rebuilding...');
405 - await buildLlmsTxt();
406 - }
407 - });
408 -
409 - // Watch MDX files in pages directory
410 - const pagesWatcher = watch(PAGES_DIR, { recursive: true }, async (eventType, filename) => {
411 - if (filename && (filename.endsWith('.mdx') || filename.endsWith('.md'))) {
412 - console.log(`🔄 Page changed: ${filename}, rebuilding...`);
413 - await buildLlmsTxt();
414 - }
415 - });
416 -
417 - // Watch MDX files in toolkits directory
418 - const toolkitsWatcher = watch(TOOLKITS_DIR, { recursive: true }, async (eventType, filename) => {
419 - if (filename && (filename.endsWith('.mdx') || filename.endsWith('.md'))) {
420 - console.log(`🔄 Toolkit doc changed: ${filename}, rebuilding...`);
421 - await buildLlmsTxt();
422 - }
423 - });
424 -
425 - // Cleanup on exit
426 - process.on('SIGINT', () => {
427 - console.log('\n👋 Stopping watchers...');
428 - docsWatcher.close();
429 - pagesWatcher.close();
430 - toolkitsWatcher.close();
431 - process.exit(0);
432 - });
433 -}
434 -
435 -// Main function
436 -async function main(): Promise<void> {
437 - await buildLlmsTxt();
438 -
439 - // Check if we should run in watch mode
440 - const isWatchMode = process.argv.includes('--watch') || process.argv.includes('-w');
441 -
442 - if (isWatchMode) {
443 - startWatchMode();
444 - }
445 -}
446 -
260 // Run the script
261 main().catch(error => {
262 console.error('❌ Fatal error:', error);
263 process.exit(1);
451 -});
264 +});
\ No newline at end of file
fern/scripts/build-tools-json.ts new
+142
@@ -0,0 +1,142 @@
1 +#!/usr/bin/env bun
2 +/* eslint-disable no-console */
3 +
4 +import fs from 'fs';
5 +import path from 'path';
6 +import { TOOLKITS_DIR, TOOLS_JSON_PATH, ensureOutputDir } from './lib/docs-utils';
7 +
8 +interface ToolkitInfo {
9 + slug: string;
10 + description: string;
11 + tools: string[];
12 +}
13 +
14 +interface ToolsData {
15 + toolkitNames: string[];
16 + [key: string]: string[] | ToolkitInfo;
17 +}
18 +
19 +// Parse tool MDX files to extract toolkit information
20 +function parseToolkitMdx(toolMdxPath: string): ToolkitInfo | null {
21 + try {
22 + const fullPath = path.join(TOOLKITS_DIR, toolMdxPath);
23 + if (!fs.existsSync(fullPath)) {
24 + console.warn(`⚠️ Tool file not found: ${fullPath}`);
25 + return null;
26 + }
27 +
28 + const content = fs.readFileSync(fullPath, 'utf8');
29 +
30 + // Extract frontmatter
31 + const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
32 + if (!frontmatterMatch) {
33 + console.warn(`⚠️ No frontmatter found in: ${toolMdxPath}`);
34 + return null;
35 + }
36 +
37 + const frontmatter = frontmatterMatch[1];
38 + const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
39 + if (!titleMatch) {
40 + console.warn(`⚠️ No title found in: ${toolMdxPath}`);
41 + return null;
42 + }
43 +
44 + // Extract SLUG from content
45 + const slugMatch = content.match(/\*\*SLUG\*\*:\s*`([^`]+)`/);
46 + if (!slugMatch) {
47 + // Not a toolkit page, skip
48 + return null;
49 + }
50 +
51 + // Extract description
52 + const descriptionMatch = content.match(/### Description\s*\n(.+?)(?:\n|$)/);
53 + const description = descriptionMatch ? descriptionMatch[1].trim() : '';
54 +
55 + // Extract tool names from Accordion titles
56 + const toolNames: string[] = [];
57 + const accordionMatches = content.matchAll(/<Accordion title="([^"]+)">/g);
58 +
59 + for (const match of accordionMatches) {
60 + const toolName = match[1];
61 + // Only include tools that start with the toolkit prefix
62 + if (toolName.startsWith(slugMatch[1] + '_')) {
63 + toolNames.push(toolName);
64 + }
65 + }
66 +
67 + return {
68 + slug: slugMatch[1],
69 + description,
70 + tools: toolNames,
71 + };
72 + } catch (error) {
73 + console.warn(`⚠️ Error parsing tool file ${toolMdxPath}:`, error);
74 + return null;
75 + }
76 +}
77 +
78 +// Scan toolkits directory and build tools data
79 +function buildToolsData(): ToolsData {
80 + const toolsData: ToolsData = { toolkitNames: [] };
81 +
82 + if (!fs.existsSync(TOOLKITS_DIR)) {
83 + console.warn(`⚠️ Toolkits directory not found: ${TOOLKITS_DIR}`);
84 + return toolsData;
85 + }
86 +
87 + // Read all MDX files in toolkits directory
88 + const files = fs.readdirSync(TOOLKITS_DIR);
89 + const mdxFiles = files.filter(file => file.endsWith('.mdx'));
90 +
91 + console.log(`📂 Found ${mdxFiles.length} MDX files in toolkits directory`);
92 +
93 + for (const file of mdxFiles) {
94 + const toolkitInfo = parseToolkitMdx(file);
95 + if (toolkitInfo) {
96 + toolsData.toolkitNames.push(toolkitInfo.slug);
97 + toolsData[toolkitInfo.slug] = toolkitInfo;
98 + console.log(` ✓ ${toolkitInfo.slug}: ${toolkitInfo.tools.length} tools`);
99 + }
100 + }
101 +
102 + // Sort toolkit names alphabetically
103 + toolsData.toolkitNames.sort();
104 +
105 + return toolsData;
106 +}
107 +
108 +// Main function
109 +async function main(): Promise<void> {
110 + console.log('🔨 Building tools.json...');
111 +
112 + try {
113 + // Build tools data
114 + const toolsData = buildToolsData();
115 +
116 + console.log(`\n📊 Summary:`);
117 + console.log(` - Total toolkits: ${toolsData.toolkitNames.length}`);
118 +
119 + let totalTools = 0;
120 + for (const name of toolsData.toolkitNames) {
121 + const toolkit = toolsData[name] as ToolkitInfo;
122 + totalTools += toolkit.tools.length;
123 + }
124 + console.log(` - Total tools: ${totalTools}`);
125 +
126 + // Write tools.json
127 + ensureOutputDir(TOOLS_JSON_PATH);
128 + const toolsJsonContent = JSON.stringify(toolsData, null, 2);
129 + fs.writeFileSync(TOOLS_JSON_PATH, toolsJsonContent);
130 +
131 + console.log(`\n✅ Generated tools.json at ${TOOLS_JSON_PATH}`);
132 + } catch (error) {
133 + console.error('❌ Error building tools.json:', error);
134 + process.exit(1);
135 + }
136 +}
137 +
138 +// Run the script
139 +main().catch(error => {
140 + console.error('❌ Fatal error:', error);
141 + process.exit(1);
142 +});
\ No newline at end of file
fern/scripts/lib/docs-utils.ts new
+150
@@ -0,0 +1,150 @@
1 +#!/usr/bin/env bun
2 +
3 +import fs from 'fs';
4 +import path from 'path';
5 +import yaml from 'yaml';
6 +
7 +// Define directories
8 +export const FERN_DIR = path.join(import.meta.dir, '../..');
9 +export const DOCS_YML_PATH = path.join(FERN_DIR, 'docs.yml');
10 +export const LLMS_TXT_PATH = path.join(FERN_DIR, 'llms-txt-worker', 'public', 'llms.txt');
11 +export const TOOLS_JSON_PATH = path.join(
12 + FERN_DIR,
13 + 'llms-txt-worker',
14 + 'public',
15 + 'robots-only',
16 + 'tools.json'
17 +);
18 +export const PAGES_DIR = path.join(FERN_DIR, 'pages');
19 +export const TOOLKITS_DIR = path.join(FERN_DIR, 'toolkits');
20 +
21 +export interface DocsConfig {
22 + title: string;
23 + navigation: Array<{
24 + tab: string;
25 + layout?: LayoutItem[];
26 + }>;
27 + tabs?: {
28 + [key: string]: {
29 + 'display-name': string;
30 + 'skip-slug'?: boolean;
31 + };
32 + };
33 +}
34 +
35 +export interface LayoutItem {
36 + section?: string;
37 + contents?: ContentItem[];
38 + page?: string;
39 + path?: string;
40 + slug?: string;
41 +}
42 +
43 +export interface ContentItem {
44 + page?: string;
45 + path?: string;
46 + slug?: string;
47 + section?: string;
48 + contents?: ContentItem[];
49 +}
50 +
51 +export function loadDocsConfig(): DocsConfig {
52 + const docsYmlContent = fs.readFileSync(DOCS_YML_PATH, 'utf8');
53 + return yaml.parse(docsYmlContent) as DocsConfig;
54 +}
55 +
56 +// Extract description from MDX content
57 +export function extractDescription(mdxPath: string): string | undefined {
58 + try {
59 + const fullPath = path.join(FERN_DIR, mdxPath);
60 + if (!fs.existsSync(fullPath)) {
61 + console.warn(`⚠️ File not found: ${fullPath}`);
62 + return undefined;
63 + }
64 +
65 + const content = fs.readFileSync(fullPath, 'utf8');
66 +
67 + // Try to extract from frontmatter description
68 + const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
69 + if (frontmatterMatch) {
70 + const frontmatter = frontmatterMatch[1];
71 + const descMatch = frontmatter.match(/description:\s*["']?(.+?)["']?\s*$/m);
72 + if (descMatch) {
73 + return descMatch[1].trim();
74 + }
75 + }
76 +
77 + // Extract first meaningful paragraph after the title
78 + const lines = content.split('\n');
79 + let inFrontmatter = false;
80 + let foundTitle = false;
81 + let description = '';
82 +
83 + for (const line of lines) {
84 + const trimmedLine = line.trim();
85 +
86 + // Handle frontmatter
87 + if (trimmedLine === '---') {
88 + inFrontmatter = !inFrontmatter;
89 + continue;
90 + }
91 +
92 + if (inFrontmatter) continue;
93 +
94 + // Skip titles
95 + if (trimmedLine.startsWith('#')) {
96 + foundTitle = true;
97 + continue;
98 + }
99 +
100 + // Skip empty lines, code blocks, and JSX
101 + if (
102 + !trimmedLine ||
103 + trimmedLine.startsWith('```') ||
104 + trimmedLine.startsWith('<') ||
105 + trimmedLine.startsWith('import ') ||
106 + trimmedLine.startsWith('export ')
107 + ) {
108 + continue;
109 + }
110 +
111 + // Found a paragraph
112 + if (foundTitle && trimmedLine.length > 20) {
113 + description = trimmedLine;
114 + break;
115 + }
116 + }
117 +
118 + // Clean up the description
119 + if (description) {
120 + // Remove MDX components and markdown syntax
121 + description = description
122 + .replace(/<[^>]+>/g, '') // Remove HTML/JSX tags
123 + .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert [text](url) to text
124 + .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks
125 + .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold
126 + .replace(/\*([^*]+)\*/g, '$1') // Remove italic
127 + .trim();
128 +
129 + // Truncate if too long
130 + if (description.length > 150) {
131 + description = description.substring(0, 147) + '...';
132 + }
133 +
134 + return description;
135 + }
136 +
137 + return undefined;
138 + } catch (error) {
139 + console.warn(`⚠️ Error reading ${mdxPath}:`, error);
140 + return undefined;
141 + }
142 +}
143 +
144 +// Ensure output directory exists
145 +export function ensureOutputDir(filePath: string): void {
146 + const dir = path.dirname(filePath);
147 + if (!fs.existsSync(dir)) {
148 + fs.mkdirSync(dir, { recursive: true });
149 + }
150 +}
\ No newline at end of file