@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 197 lines 5.53 KB
Raw
1 import {
2 getLLMText,
3 source,
4 examplesSource,
5 referenceSource,
6 toolkitsSource,
7 knowledgeBaseSource,
8 type LLMPage,
9 } from '@/lib/source';
10 import { SESSION_GUARDRAILS } from '@/lib/llm-guardrails';
11 import { detectReferenceApiVersion } from '@/lib/api-version';
12 import { TOOLKIT_COUNT_LABEL } from '@/lib/toolkit-count';
13 import {
14 formatKnowledgeDiscoveryLinks,
15 getLocalKnowledgeDiscoveryPaths,
16 } from '@/lib/knowledge/discovery';
17 import type { ReactNode } from 'react';
18
19 export const revalidate = false;
20
21 // Fumadocs page tree node types
22 interface PageNode {
23 type: 'page';
24 name: ReactNode;
25 url: string;
26 }
27
28 interface SeparatorNode {
29 type: 'separator';
30 name?: ReactNode;
31 }
32
33 interface FolderNode {
34 type: 'folder';
35 name: ReactNode;
36 index?: PageNode;
37 children: TreeNode[];
38 }
39
40 type TreeNode = PageNode | SeparatorNode | FolderNode;
41
42 // Generic page type that works for all sources
43 type PageLike = LLMPage & { slugs: string[] };
44
45 /**
46 * Collect page URLs from the page tree in sidebar order.
47 * This ensures pages appear in the same order as the docs sidebar.
48 */
49 function collectPageUrls(nodes: TreeNode[]): string[] {
50 const urls: string[] = [];
51
52 for (const node of nodes) {
53 switch (node.type) {
54 case 'page':
55 urls.push(node.url);
56 break;
57
58 case 'folder':
59 if (node.index) {
60 urls.push(node.index.url);
61 }
62 urls.push(...collectPageUrls(node.children));
63 break;
64
65 // separators don't have URLs
66 }
67 }
68
69 return urls;
70 }
71
72 /** All page URLs under a folder, including its index. */
73 function collectFolderUrls(folder: FolderNode): string[] {
74 const urls: string[] = [];
75 if (folder.index) urls.push(folder.index.url);
76 for (const child of folder.children) {
77 if (child.type === 'page') urls.push(child.url);
78 else if (child.type === 'folder') urls.push(...collectFolderUrls(child));
79 }
80 return urls;
81 }
82
83 /**
84 * URLs that live under a legacy/deprecated separator section (e.g. "Direct
85 * Tool Execution Guides (Legacy)"). We drop their full text from the default
86 * LLM context so generators don't learn deprecated patterns.
87 */
88 function collectLegacyUrls(nodes: TreeNode[]): Set<string> {
89 const legacy = new Set<string>();
90 let inLegacySection = false;
91
92 for (const node of nodes) {
93 if (node.type === 'separator') {
94 const text = typeof node.name === 'string' ? node.name : '';
95 inLegacySection = /legacy|deprecated/i.test(text);
96 continue;
97 }
98 if (!inLegacySection) continue;
99 if (node.type === 'page') legacy.add(node.url);
100 else if (node.type === 'folder') {
101 for (const url of collectFolderUrls(node)) legacy.add(url);
102 }
103 }
104
105 return legacy;
106 }
107
108 /**
109 * Order pages according to the page tree structure from meta.json.
110 * Pages not in the tree are appended at the end.
111 */
112 function orderDocPages(pages: PageLike[], treeNodes: TreeNode[]): PageLike[] {
113 const orderedUrls = collectPageUrls(treeNodes);
114 const urlOrder = new Map(orderedUrls.map((url, i) => [url, i]));
115
116 return [...pages].sort((a, b) => {
117 const orderA = urlOrder.get(a.url) ?? 999;
118 const orderB = urlOrder.get(b.url) ?? 999;
119 return orderA - orderB;
120 });
121 }
122
123 async function getTextForPages(pages: PageLike[]) {
124 return Promise.all(
125 pages.map(async page => {
126 try {
127 return await getLLMText(page, {
128 includeFooter: false,
129 includeGuardrails: false,
130 });
131 } catch {
132 return `# ${page.data.title} (${page.url})\n\n${page.data.description || ''}`;
133 }
134 })
135 );
136 }
137
138 export async function GET() {
139 try {
140 const treeChildren = source.pageTree.children as TreeNode[];
141 const legacyUrls = collectLegacyUrls(treeChildren);
142 const orderedDocsPages = orderDocPages(
143 source.getPages().filter(page => !legacyUrls.has(page.url)),
144 treeChildren
145 );
146 const knowledgeDiscoveryLinks = formatKnowledgeDiscoveryLinks(
147 (await getLocalKnowledgeDiscoveryPaths()).filter(
148 (path) => !path.startsWith('/kb/guide/'),
149 ),
150 );
151
152 const [
153 docsResults,
154 knowledgeBaseResults,
155 examplesResults,
156 referenceResults,
157 toolkitsResults,
158 ] = await Promise.all([
159 getTextForPages(orderedDocsPages),
160 getTextForPages(knowledgeBaseSource.getPages()),
161 getTextForPages(examplesSource.getPages()),
162 getTextForPages(
163 referenceSource.getPages().filter(page => detectReferenceApiVersion(page.url) !== '3.0')
164 ),
165 getTextForPages(toolkitsSource.getPages()),
166 ]);
167
168 const results = [
169 `# Composio Documentation\n\n> Composio powers ${TOOLKIT_COUNT_LABEL} apps, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.${SESSION_GUARDRAILS}\n\n[Changelog](https://docs.composio.dev/docs/changelog.md): Browse dated release notes and follow each date for its full Markdown.\n# Documentation\n`,
170 ...docsResults,
171 '\n# Knowledge Hub navigation\n',
172 knowledgeDiscoveryLinks,
173 '\n# Knowledge Base\n',
174 ...knowledgeBaseResults,
175 '\n# Examples\n',
176 ...examplesResults,
177 '\n# API Reference\n',
178 ...referenceResults,
179 '\n# Toolkits\n',
180 ...toolkitsResults,
181 ];
182
183 return new Response(results.join('\n\n---\n\n'), {
184 headers: {
185 'Content-Type': 'text/plain; charset=utf-8',
186 },
187 });
188 } catch (error) {
189 console.error('[llms-full.txt] Error generating content:', error);
190 return new Response('Error generating LLM content', {
191 status: 500,
192 headers: {
193 'Content-Type': 'text/plain; charset=utf-8',
194 },
195 });
196 }
197 }