Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { |
| 2 | getLLMText, |
| 3 | source, |
| 4 | examplesSource, |
| 5 | referenceSource, |
| 6 | toolkitsSource, |
| 7 | knowledgeBaseSource, |
| 8 | type LLMPage, |
| 9 | } from '@/lib/source'; |
| 10 | import { SESSION_GUARDRAILS } from '@/lib/llm-guardrails'; |
| 11 | import { detectReferenceApiVersion } from '@/lib/api-version'; |
| 12 | import { TOOLKIT_COUNT_LABEL } from '@/lib/toolkit-count'; |
| 13 | import { |
| 14 | formatKnowledgeDiscoveryLinks, |
| 15 | getLocalKnowledgeDiscoveryPaths, |
| 16 | } from '@/lib/knowledge/discovery'; |
| 17 | import type { ReactNode } from 'react'; |
| 18 | |
| 19 | export const revalidate = false; |
| 20 | |
| 21 | // Fumadocs page tree node types |
| 22 | interface PageNode { |
| 23 | type: 'page'; |
| 24 | name: ReactNode; |
| 25 | url: string; |
| 26 | } |
| 27 | |
| 28 | interface SeparatorNode { |
| 29 | type: 'separator'; |
| 30 | name?: ReactNode; |
| 31 | } |
| 32 | |
| 33 | interface FolderNode { |
| 34 | type: 'folder'; |
| 35 | name: ReactNode; |
| 36 | index?: PageNode; |
| 37 | children: TreeNode[]; |
| 38 | } |
| 39 | |
| 40 | type TreeNode = PageNode | SeparatorNode | FolderNode; |
| 41 | |
| 42 | // Generic page type that works for all sources |
| 43 | type PageLike = LLMPage & { slugs: string[] }; |
| 44 | |
| 45 | /** |
| 46 | * Collect page URLs from the page tree in sidebar order. |
| 47 | * This ensures pages appear in the same order as the docs sidebar. |
| 48 | */ |
| 49 | function collectPageUrls(nodes: TreeNode[]): string[] { |
| 50 | const urls: string[] = []; |
| 51 | |
| 52 | for (const node of nodes) { |
| 53 | switch (node.type) { |
| 54 | case 'page': |
| 55 | urls.push(node.url); |
| 56 | break; |
| 57 | |
| 58 | case 'folder': |
| 59 | if (node.index) { |
| 60 | urls.push(node.index.url); |
| 61 | } |
| 62 | urls.push(...collectPageUrls(node.children)); |
| 63 | break; |
| 64 | |
| 65 | // separators don't have URLs |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | return urls; |
| 70 | } |
| 71 | |
| 72 | /** All page URLs under a folder, including its index. */ |
| 73 | function collectFolderUrls(folder: FolderNode): string[] { |
| 74 | const urls: string[] = []; |
| 75 | if (folder.index) urls.push(folder.index.url); |
| 76 | for (const child of folder.children) { |
| 77 | if (child.type === 'page') urls.push(child.url); |
| 78 | else if (child.type === 'folder') urls.push(...collectFolderUrls(child)); |
| 79 | } |
| 80 | return urls; |
| 81 | } |
| 82 | |
| 83 | /** |
| 84 | * URLs that live under a legacy/deprecated separator section (e.g. "Direct |
| 85 | * Tool Execution Guides (Legacy)"). We drop their full text from the default |
| 86 | * LLM context so generators don't learn deprecated patterns. |
| 87 | */ |
| 88 | function collectLegacyUrls(nodes: TreeNode[]): Set<string> { |
| 89 | const legacy = new Set<string>(); |
| 90 | let inLegacySection = false; |
| 91 | |
| 92 | for (const node of nodes) { |
| 93 | if (node.type === 'separator') { |
| 94 | const text = typeof node.name === 'string' ? node.name : ''; |
| 95 | inLegacySection = /legacy|deprecated/i.test(text); |
| 96 | continue; |
| 97 | } |
| 98 | if (!inLegacySection) continue; |
| 99 | if (node.type === 'page') legacy.add(node.url); |
| 100 | else if (node.type === 'folder') { |
| 101 | for (const url of collectFolderUrls(node)) legacy.add(url); |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | return legacy; |
| 106 | } |
| 107 | |
| 108 | /** |
| 109 | * Order pages according to the page tree structure from meta.json. |
| 110 | * Pages not in the tree are appended at the end. |
| 111 | */ |
| 112 | function orderDocPages(pages: PageLike[], treeNodes: TreeNode[]): PageLike[] { |
| 113 | const orderedUrls = collectPageUrls(treeNodes); |
| 114 | const urlOrder = new Map(orderedUrls.map((url, i) => [url, i])); |
| 115 | |
| 116 | return [...pages].sort((a, b) => { |
| 117 | const orderA = urlOrder.get(a.url) ?? 999; |
| 118 | const orderB = urlOrder.get(b.url) ?? 999; |
| 119 | return orderA - orderB; |
| 120 | }); |
| 121 | } |
| 122 | |
| 123 | async function getTextForPages(pages: PageLike[]) { |
| 124 | return Promise.all( |
| 125 | pages.map(async page => { |
| 126 | try { |
| 127 | return await getLLMText(page, { |
| 128 | includeFooter: false, |
| 129 | includeGuardrails: false, |
| 130 | }); |
| 131 | } catch { |
| 132 | return `# ${page.data.title} (${page.url})\n\n${page.data.description || ''}`; |
| 133 | } |
| 134 | }) |
| 135 | ); |
| 136 | } |
| 137 | |
| 138 | export async function GET() { |
| 139 | try { |
| 140 | const treeChildren = source.pageTree.children as TreeNode[]; |
| 141 | const legacyUrls = collectLegacyUrls(treeChildren); |
| 142 | const orderedDocsPages = orderDocPages( |
| 143 | source.getPages().filter(page => !legacyUrls.has(page.url)), |
| 144 | treeChildren |
| 145 | ); |
| 146 | const knowledgeDiscoveryLinks = formatKnowledgeDiscoveryLinks( |
| 147 | (await getLocalKnowledgeDiscoveryPaths()).filter( |
| 148 | (path) => !path.startsWith('/kb/guide/'), |
| 149 | ), |
| 150 | ); |
| 151 | |
| 152 | const [ |
| 153 | docsResults, |
| 154 | knowledgeBaseResults, |
| 155 | examplesResults, |
| 156 | referenceResults, |
| 157 | toolkitsResults, |
| 158 | ] = await Promise.all([ |
| 159 | getTextForPages(orderedDocsPages), |
| 160 | getTextForPages(knowledgeBaseSource.getPages()), |
| 161 | getTextForPages(examplesSource.getPages()), |
| 162 | getTextForPages( |
| 163 | referenceSource.getPages().filter(page => detectReferenceApiVersion(page.url) !== '3.0') |
| 164 | ), |
| 165 | getTextForPages(toolkitsSource.getPages()), |
| 166 | ]); |
| 167 | |
| 168 | const results = [ |
| 169 | `# Composio Documentation\n\n> Composio powers ${TOOLKIT_COUNT_LABEL} apps, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.${SESSION_GUARDRAILS}\n\n[Changelog](https://docs.composio.dev/docs/changelog.md): Browse dated release notes and follow each date for its full Markdown.\n# Documentation\n`, |
| 170 | ...docsResults, |
| 171 | '\n# Knowledge Hub navigation\n', |
| 172 | knowledgeDiscoveryLinks, |
| 173 | '\n# Knowledge Base\n', |
| 174 | ...knowledgeBaseResults, |
| 175 | '\n# Examples\n', |
| 176 | ...examplesResults, |
| 177 | '\n# API Reference\n', |
| 178 | ...referenceResults, |
| 179 | '\n# Toolkits\n', |
| 180 | ...toolkitsResults, |
| 181 | ]; |
| 182 | |
| 183 | return new Response(results.join('\n\n---\n\n'), { |
| 184 | headers: { |
| 185 | 'Content-Type': 'text/plain; charset=utf-8', |
| 186 | }, |
| 187 | }); |
| 188 | } catch (error) { |
| 189 | console.error('[llms-full.txt] Error generating content:', error); |
| 190 | return new Response('Error generating LLM content', { |
| 191 | status: 500, |
| 192 | headers: { |
| 193 | 'Content-Type': 'text/plain; charset=utf-8', |
| 194 | }, |
| 195 | }); |
| 196 | } |
| 197 | } |