@devries-cameron20 / composio-1 / commits / 9c569470c
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(docs): auto-generate llms.txt from page tree instead of hardcoded lists

Both llms.txt and llms-full.txt maintained a manual copy of the sidebar structure, causing new pages to silently go missing. Now they walk the fumadocs page tree (derived from meta.json) so any page added to the sidebar automatically appears. Also removes the repeated footer from llms-full.txt pages via an includeFooter option on getLLMText. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Sushmithamallesh committed Feb 11, 2026 at 11:58 UTC 9c569470c8b98922aa7270e2dc6fc1c8fdfcc846
3 files changed +141 -163
docs/app/llms-full.txt/route.ts
+69 -69
@@ -1,7 +1,29 @@
1 import { getLLMText, source, examplesSource, referenceSource, toolkitsSource } from '@/lib/source';
2 +import type { ReactNode } from 'react';
3
4 export const revalidate = false;
5
6 +// Fumadocs page tree node types
7 +interface PageNode {
8 + type: 'page';
9 + name: ReactNode;
10 + url: string;
11 +}
12 +
13 +interface SeparatorNode {
14 + type: 'separator';
15 + name?: ReactNode;
16 +}
17 +
18 +interface FolderNode {
19 + type: 'folder';
20 + name: ReactNode;
21 + index?: PageNode;
22 + children: TreeNode[];
23 +}
24 +
25 +type TreeNode = PageNode | SeparatorNode | FolderNode;
26 +
27 // Generic page type that works for all sources
28 interface PageLike {
29 url: string;
@@ -12,13 +34,55 @@ interface PageLike {
34 };
35 }
36
37 +/**
38 + * Collect page URLs from the page tree in sidebar order.
39 + * This ensures pages appear in the same order as the docs sidebar.
40 + */
41 +function collectPageUrls(nodes: TreeNode[]): string[] {
42 + const urls: string[] = [];
43 +
44 + for (const node of nodes) {
45 + switch (node.type) {
46 + case 'page':
47 + urls.push(node.url);
48 + break;
49 +
50 + case 'folder':
51 + if (node.index) {
52 + urls.push(node.index.url);
53 + }
54 + urls.push(...collectPageUrls(node.children));
55 + break;
56 +
57 + // separators don't have URLs
58 + }
59 + }
60 +
61 + return urls;
62 +}
63 +
64 +/**
65 + * Order pages according to the page tree structure from meta.json.
66 + * Pages not in the tree are appended at the end.
67 + */
68 +function orderDocPages(pages: PageLike[], treeNodes: TreeNode[]): PageLike[] {
69 + const orderedUrls = collectPageUrls(treeNodes);
70 + const urlOrder = new Map(orderedUrls.map((url, i) => [url, i]));
71 +
72 + return [...pages].sort((a, b) => {
73 + const orderA = urlOrder.get(a.url) ?? 999;
74 + const orderB = urlOrder.get(b.url) ?? 999;
75 + return orderA - orderB;
76 + });
77 +}
78 +
79 // eslint-disable-next-line @typescript-eslint/no-explicit-any
80 async function getTextForPages(pages: PageLike[]) {
81 return Promise.all(
82 pages.map(async (page) => {
83 try {
84 // eslint-disable-next-line @typescript-eslint/no-explicit-any
21 - return await getLLMText(page as any);
85 + return await getLLMText(page as any, { includeFooter: false });
86 } catch {
87 // Graceful fallback if getText fails
88 return `# ${page.data.title} (${page.url})\n\n${page.data.description || ''}`;
@@ -27,76 +91,12 @@ async function getTextForPages(pages: PageLike[]) {
91 );
92 }
93
30 -// Order pages according to sidebar structure from meta.json
31 -function orderDocPages(pages: PageLike[]) {
32 - // Define the order based on meta.json sidebar structure
33 - const sidebarOrder = [
34 - // Get Started
35 - 'index',
36 - 'quickstart',
37 - // Providers folder
38 - 'providers/openai-agents',
39 - 'providers/anthropic',
40 - 'providers/vercel',
41 - 'providers/langchain',
42 - 'providers/mastra',
43 - 'providers/openai',
44 - 'providers/google',
45 - 'providers/llamaindex',
46 - 'providers/crewai',
47 - 'providers/custom-providers',
48 - // Core Concepts
49 - 'users-and-sessions',
50 - 'authentication',
51 - 'tools-and-toolkits',
52 - // Getting Started
53 - 'configuring-sessions',
54 - 'authenticating-users/in-chat-authentication',
55 - 'authenticating-users/manually-authenticating',
56 - // Toolkits folder (from docs/toolkits)
57 - 'toolkits',
58 - // Guides
59 - 'white-labeling-authentication',
60 - 'managing-multiple-connected-accounts',
61 - 'using-custom-auth-configuration',
62 - // Features
63 - 'triggers',
64 - 'cli',
65 - 'single-toolkit-mcp',
66 - // Direct Tool Execution - Tools
67 - 'tools-direct/fetching-tools',
68 - 'tools-direct/authenticating-tools',
69 - 'tools-direct/executing-tools',
70 - 'tools-direct/modify-tool-behavior',
71 - 'tools-direct/custom-tools',
72 - 'tools-direct/toolkit-versioning',
73 - // Direct Tool Execution - Auth Configuration
74 - 'auth-configuration',
75 - // Resources
76 - 'debugging-info',
77 - 'migration-guide',
78 - 'troubleshooting',
79 - ];
80 -
81 - // Create a map for ordering
82 - const orderMap = new Map<string, number>();
83 - sidebarOrder.forEach((slug, index) => {
84 - orderMap.set(slug, index);
85 - });
86 -
87 - // Sort pages based on sidebar order, unmatched pages go to end
88 - return [...pages].sort((a, b) => {
89 - const slugA = a.slugs.join('/');
90 - const slugB = b.slugs.join('/');
91 - const orderA = orderMap.get(slugA) ?? 999;
92 - const orderB = orderMap.get(slugB) ?? 999;
93 - return orderA - orderB;
94 - });
95 -}
96 -
94 export async function GET() {
95 try {
99 - const orderedDocsPages = orderDocPages(source.getPages() as PageLike[]);
96 + const orderedDocsPages = orderDocPages(
97 + source.getPages() as PageLike[],
98 + source.pageTree.children as TreeNode[]
99 + );
100
101 const [docsResults, examplesResults, referenceResults, toolkitsResults] = await Promise.all([
102 getTextForPages(orderedDocsPages),
docs/app/llms.txt/route.ts
+65 -88
@@ -1,105 +1,82 @@
1 import { source, examplesSource, referenceSource, toolkitsSource } from '@/lib/source';
2 +import type { ReactNode } from 'react';
3
4 export const revalidate = false;
5
5 -export async function GET() {
6 - try {
7 - const docsPages = source.getPages();
8 - const examplesPages = examplesSource.getPages();
9 - const referencePages = referenceSource.getPages();
10 - const toolkitsPages = toolkitsSource.getPages();
11 -
12 - // Create a map for quick lookup by slug path
13 - // eslint-disable-next-line @typescript-eslint/no-explicit-any
14 - const pageMap = new Map<string, any>();
15 - for (const page of docsPages) {
16 - pageMap.set(page.slugs.join('/'), page);
17 - }
18 -
19 - // Format page as simple URL
20 - // eslint-disable-next-line @typescript-eslint/no-explicit-any
21 - const formatPage = (page: any) =>
22 - `- https://docs.composio.dev${page.url}.md`;
23 -
24 - // Get page by slug path, return empty string if not found
25 - const getPage = (slugPath: string) => {
26 - const page = pageMap.get(slugPath);
27 - return page ? formatPage(page) : '';
28 - };
29 -
30 - // Get all pages in a folder
31 - const getFolderPages = (folder: string) => {
32 - return docsPages
33 - .filter(p => p.slugs[0] === folder)
34 - .map(formatPage)
35 - .join('\n');
36 - };
37 -
38 - // Build the index following the exact sidebar structure from meta.json
39 - const index = `# Composio Documentation
40 -
41 -> Composio powers 800+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
42 -
43 -## Get Started
44 -
45 -${getPage('index')}
46 -${getPage('quickstart')}
47 -
48 -### Providers
49 -
50 -${getFolderPages('providers')}
51 -
52 -## Core Concepts
53 -
54 -${getPage('users-and-sessions')}
55 -${getPage('authentication')}
56 -${getPage('tools-and-toolkits')}
57 -
58 -## Getting Started
59 -
60 -${getPage('configuring-sessions')}
61 -
62 -### Authenticating Users
63 -
64 -${getFolderPages('authenticating-users')}
65 -
66 -### Toolkits
67 -
68 -${getFolderPages('toolkits')}
69 -
70 -## Guides
71 -
72 -${getPage('white-labeling-authentication')}
73 -${getPage('managing-multiple-connected-accounts')}
74 -${getPage('using-custom-auth-configuration')}
75 -
76 -## Features
77 -
78 -${getPage('triggers')}
79 -${getPage('cli')}
80 -${getPage('single-toolkit-mcp')}
81 -
82 -## Direct Tool Execution Guides
6 +// Fumadocs page tree node types
7 +interface PageNode {
8 + type: 'page';
9 + name: ReactNode;
10 + url: string;
11 +}
12
84 -### Tools
13 +interface SeparatorNode {
14 + type: 'separator';
15 + name?: ReactNode;
16 +}
17
86 -${getFolderPages('tools-direct')}
18 +interface FolderNode {
19 + type: 'folder';
20 + name: ReactNode;
21 + index?: PageNode;
22 + children: TreeNode[];
23 +}
24
88 -### Auth Configuration
25 +type TreeNode = PageNode | SeparatorNode | FolderNode;
26 +
27 +/**
28 + * Walk the fumadocs page tree and generate a markdown index.
29 + * Separators become ## headings, pages become URL entries, folders recurse.
30 + */
31 +function walkPageTree(nodes: TreeNode[], depth = 2): string {
32 + const lines: string[] = [];
33 +
34 + for (const node of nodes) {
35 + switch (node.type) {
36 + case 'separator':
37 + if (node.name) {
38 + lines.push(`${'#'.repeat(depth)} ${String(node.name)}`, '');
39 + }
40 + break;
41 +
42 + case 'page':
43 + lines.push(`- https://docs.composio.dev${node.url}.md`);
44 + break;
45 +
46 + case 'folder':
47 + // If folder has an index page, include it
48 + if (node.index) {
49 + lines.push(`- https://docs.composio.dev${node.index.url}.md`);
50 + }
51 + // Recurse into children
52 + if (node.children.length > 0) {
53 + lines.push(walkPageTree(node.children, depth + 1));
54 + }
55 + break;
56 + }
57 + }
58
90 -${getFolderPages('auth-configuration')}
59 + return lines.filter(Boolean).join('\n');
60 +}
61
92 -## Resources
62 +// eslint-disable-next-line @typescript-eslint/no-explicit-any
63 +function formatPage(page: any) {
64 + return `- https://docs.composio.dev${page.url}.md`;
65 +}
66
94 -${getPage('debugging-info')}
67 +export async function GET() {
68 + try {
69 + const docsTree = walkPageTree(source.pageTree.children as TreeNode[]);
70
96 -### Migration Guide
71 + const examplesPages = examplesSource.getPages();
72 + const referencePages = referenceSource.getPages();
73 + const toolkitsPages = toolkitsSource.getPages();
74
98 -${getFolderPages('migration-guide')}
75 + const index = `# Composio Documentation
76
100 -### Troubleshooting
77 +> Composio powers 800+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
78
102 -${getFolderPages('troubleshooting')}
79 +${docsTree}
80
81 ## Examples
82
docs/lib/source.ts
+7 -6
@@ -306,7 +306,8 @@ function stripTwoslashFromCodeBlocks(content: string): string {
306 });
307 }
308
309 -export async function getLLMText(page: InferPageType<typeof source>) {
309 +export async function getLLMText(page: InferPageType<typeof source>, options?: { includeFooter?: boolean }) {
310 + const includeFooter = options?.includeFooter ?? true;
311 // Fall back to description if getText is not available
312 if (typeof page.data.getText !== 'function') {
313 return `# ${page.data.title} (${page.url})
@@ -337,13 +338,13 @@ ${page.data.description || ''}`;
338
339 const cleanContent = mdxToCleanMarkdown(content);
340
340 - return `# ${page.data.title} (${page.url})
341 -
342 -${cleanContent}
341 + const footer = includeFooter
342 + ? `\n\n---\n\n📚 **More documentation:** [View all docs](https://docs.composio.dev/llms.txt) | [Examples](https://docs.composio.dev/llms.mdx/examples) | [API Reference](https://docs.composio.dev/llms.mdx/reference)`
343 + : '';
344
344 ----
345 + return `# ${page.data.title} (${page.url})
346
346 -📚 **More documentation:** [View all docs](https://docs.composio.dev/llms.txt) | [Examples](https://docs.composio.dev/llms.mdx/examples) | [API Reference](https://docs.composio.dev/llms.mdx/reference)`;
347 +${cleanContent}${footer}`;
348 }
349
350 export function formatDate(dateStr: string): string {