@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 303 lines 9.98 KB
Raw
1 import { readFile } from 'node:fs/promises';
2 import { glob } from 'node:fs/promises';
3 import { resolve } from 'node:path';
4 import { type FileObject, printErrors, scanURLs, validateFiles } from 'next-validate-link';
5 import GithubSlugger from 'github-slugger';
6 import { z } from 'zod';
7 import {
8 source,
9 getReferenceSource,
10 examplesSource,
11 toolkitsSource,
12 knowledgeBaseSource,
13 } from '../lib/source';
14 import { getKnowledgeToolkitSummaries } from '../lib/knowledge/catalog';
15 import { getLocalKnowledgeDiscoveryPaths } from '../lib/knowledge/discovery';
16
17 /**
18 * `--external` additionally HEAD/GET-checks every external URL. Slow and
19 * dependent on third-party uptime, so it runs on a schedule (nightly CI),
20 * never on the PR path.
21 */
22 const checkExternalLinks = process.argv.includes('--external');
23
24 type AnySource =
25 | typeof source
26 | Awaited<ReturnType<typeof getReferenceSource>>
27 | typeof examplesSource
28 | typeof toolkitsSource
29 | typeof knowledgeBaseSource;
30
31 type PageOf = ReturnType<AnySource['getPages']>[number];
32
33 export function withoutFrontmatter(content: string): string {
34 return content.replace(/^(?:\uFEFF)?---\r?\n[\s\S]*?\r?\n---\r?\n?/, '');
35 }
36
37 /**
38 * Extract heading anchors from raw MDX/markdown content.
39 * Falls back to this when data.toc is unavailable (outside Next.js runtime).
40 * Uses github-slugger to match rehype-slug's algorithm (handles Unicode, duplicate suffixes).
41 */
42 function extractHeadingsFromContent(content: string): string[] {
43 const slugger = new GithubSlugger();
44 const headings: string[] = [];
45 let inCodeBlock = false;
46 for (const line of content.split('\n')) {
47 if (line.startsWith('```')) {
48 inCodeBlock = !inCodeBlock;
49 continue;
50 }
51 if (inCodeBlock) continue;
52 const match = line.match(/^#{1,6}\s+(.+)$/);
53 if (match) {
54 headings.push(slugger.slug(match[1]));
55 }
56 }
57 return headings;
58 }
59
60 // Page data differs per source (MDX pages carry toc/getText, OpenAPI pages do
61 // not), so the pieces this script walks are parsed instead of guarded by hand.
62 const tocEntrySchema = z.object({ url: z.string() });
63 const tocPageDataSchema = z.object({ toc: z.array(z.unknown()) });
64 const textPageDataSchema = z.object({
65 getText: z.custom<(mode: 'processed' | 'raw') => Promise<string>>(
66 value => typeof value === 'function'
67 ),
68 });
69
70 /**
71 * Get headings for a page, trying data.toc first then falling back to raw content parsing.
72 */
73 async function getHeadingsForPage(page: PageOf): Promise<string[]> {
74 const tocData = tocPageDataSchema.safeParse(page.data);
75 if (tocData.success && tocData.data.toc.length > 0) {
76 return tocData.data.toc.flatMap(item => {
77 const entry = tocEntrySchema.safeParse(item);
78 return entry.success ? [entry.data.url.slice(1)] : [];
79 });
80 }
81 const textData = textPageDataSchema.safeParse(page.data);
82 if (textData.success) {
83 try {
84 const content = await textData.data.getText('raw');
85 return extractHeadingsFromContent(content);
86 } catch {
87 // fall through
88 }
89 }
90 if (page.absolutePath) {
91 try {
92 const content = await readFile(page.absolutePath, 'utf-8');
93 return extractHeadingsFromContent(content);
94 } catch {
95 // fall through
96 }
97 }
98 return [];
99 }
100
101 /**
102 * Build populate entries for a source, resolving headings asynchronously.
103 */
104 async function buildPopulateEntries(src: AnySource) {
105 return Promise.all(
106 src.getPages().map(async (page: PageOf) => ({
107 value: { slug: page.slugs },
108 hashes: await getHeadingsForPage(page),
109 }))
110 );
111 }
112
113 const toolkitSlugsSchema = z.array(z.object({ slug: z.string() }));
114
115 export async function getDynamicToolkitEntries() {
116 const raw = await readFile('public/data/toolkits.json', 'utf-8');
117 const toolkits = toolkitSlugsSchema.parse(JSON.parse(raw));
118 return toolkits.flatMap(t =>
119 ['', '.md', '.mdx'].map(suffix => ({
120 value: { slug: [`${t.slug}${suffix}`] },
121 hashes: [] as string[],
122 })),
123 );
124 }
125
126 export async function getKnowledgeToolkitRouteEntries() {
127 return (await getKnowledgeToolkitSummaries()).map(toolkit => ({
128 value: toolkit.slug,
129 hashes: [] as string[],
130 }));
131 }
132
133 const EXTERNAL_FETCH_HEADERS = {
134 // Some hosts reject requests without a browser-like UA (bot filters).
135 'user-agent':
136 'Mozilla/5.0 (compatible; composio-docs-link-check; +https://docs.composio.dev)',
137 accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8',
138 };
139
140 async function fetchStatus(url: URL, method: 'HEAD' | 'GET'): Promise<number> {
141 const response = await fetch(url, {
142 method,
143 headers: EXTERNAL_FETCH_HEADERS,
144 redirect: 'follow',
145 signal: AbortSignal.timeout(15_000),
146 });
147 await response.body?.cancel();
148 return response.status;
149 }
150
151 const externalResultCache = new Map<
152 string,
153 Promise<{ success: true } | { success: false; message?: string }>
154 >();
155
156 /**
157 * Replaces next-validate-link's default external validator (bare HEAD, no
158 * timeout, no UA, no retry). This one only fails on evidence the link is dead
159 * — 404/410, or a network error after a retry. Auth walls, rate limits, and
160 * transient 5xx count as reachable: the nightly job hunts dead links, not
161 * third-party uptime.
162 */
163 function validateExternalUrl(url: URL) {
164 if (url.hostname === 'localhost' || url.hostname === '127.0.0.1') {
165 return Promise.resolve({ success: true as const });
166 }
167 const cached = externalResultCache.get(url.href);
168 if (cached) return cached;
169
170 const result = (async () => {
171 for (let attempt = 0; attempt < 2; attempt++) {
172 try {
173 let status = await fetchStatus(url, 'HEAD');
174 // Many servers reject HEAD (405, bot filters); confirm with GET.
175 if (status >= 400) status = await fetchStatus(url, 'GET');
176 if (status === 404 || status === 410) {
177 return { success: false as const, message: `responded status ${status}` };
178 }
179 return { success: true as const };
180 } catch (error) {
181 if (attempt === 0) continue;
182 return {
183 success: false as const,
184 message: error instanceof Error ? error.message : 'fetch failed',
185 };
186 }
187 }
188 return { success: false as const };
189 })();
190 externalResultCache.set(url.href, result);
191 return result;
192 }
193
194 async function checkLinks() {
195 const referenceSource = await getReferenceSource();
196 const knowledgeDiscoveryPaths = await getLocalKnowledgeDiscoveryPaths();
197 const knowledgeTopicEntries = knowledgeDiscoveryPaths
198 .filter((path) => path.startsWith('/kb/topic/'))
199 .map((path) => ({ value: path.slice('/kb/topic/'.length), hashes: [] as string[] }));
200 const [
201 docsEntries,
202 refEntries,
203 exampleEntries,
204 toolkitEntries,
205 knowledgeBaseEntries,
206 dynamicToolkitEntries,
207 knowledgeToolkitEntries,
208 ] = await Promise.all([
209 buildPopulateEntries(source),
210 buildPopulateEntries(referenceSource),
211 buildPopulateEntries(examplesSource),
212 buildPopulateEntries(toolkitsSource),
213 buildPopulateEntries(knowledgeBaseSource),
214 getDynamicToolkitEntries(),
215 getKnowledgeToolkitRouteEntries(),
216 ]);
217 const knowledgeGuideEntries = knowledgeBaseEntries.flatMap((entry) => {
218 const value = entry.value;
219 if (!value || Array.isArray(value) || typeof value === 'string' || !('slug' in value)) return [];
220 const slug = value.slug;
221 const segments = Array.isArray(slug) ? slug : [slug];
222 return segments.length === 2 && segments[0] === 'guide'
223 ? [{ ...entry, value: segments[1] }]
224 : [];
225 });
226
227 const scanned = await scanURLs({
228 preset: 'next',
229 populate: {
230 // Keys must include (home) route group to match app directory structure
231 '(home)/docs/[[...slug]]': docsEntries,
232 '(home)/reference/[[...slug]]': refEntries,
233 '(home)/examples/[[...slug]]': exampleEntries,
234 '(home)/toolkits/[[...slug]]': [...toolkitEntries, ...dynamicToolkitEntries],
235 '(home)/kb/guide/[slug]': knowledgeGuideEntries,
236 '(home)/kb/topic/[slug]': knowledgeTopicEntries,
237 '(home)/kb/toolkit/[slug]': knowledgeToolkitEntries,
238 },
239 });
240
241 const errors = await validateFiles(await getFiles(), {
242 scanned,
243 markdown: {
244 components: {
245 Card: { attributes: ['href'] },
246 },
247 },
248 checkRelativePaths: 'as-url',
249 checkExternal: checkExternalLinks ? { validate: validateExternalUrl } : false,
250 });
251
252 // Filter out API route URLs (these are valid but not detected as pages)
253 const ignoredUrls = ['/llms.txt', '/llms-full.txt'];
254 const filteredErrors = errors
255 .map(fileError => ({
256 ...fileError,
257 errors: fileError.errors.filter(e => !ignoredUrls.includes(e.url)),
258 detected: fileError.detected.filter(d => !ignoredUrls.includes(d[0] as string)),
259 }))
260 .filter(fileError => fileError.errors.length > 0);
261
262 printErrors(filteredErrors, true);
263 if (filteredErrors.length > 0) {
264 process.exit(1);
265 }
266 }
267
268 async function getFiles(): Promise<FileObject[]> {
269 const referenceSource = await getReferenceSource();
270 const sources = [source, referenceSource, examplesSource, toolkitsSource, knowledgeBaseSource];
271 const allFiles: FileObject[] = [];
272
273 for (const src of sources) {
274 const pages = src.getPages();
275 for (const page of pages) {
276 if (!page.absolutePath) continue;
277 if (!page.absolutePath.endsWith('.mdx') && !page.absolutePath.endsWith('.md')) continue;
278 // Skip OpenAPI-generated pages (they don't have getText)
279 const textData = textPageDataSchema.safeParse(page.data);
280 if (!textData.success) continue;
281
282 allFiles.push({
283 path: page.absolutePath,
284 content: withoutFrontmatter(await textData.data.getText('raw')),
285 url: page.url,
286 data: page.data,
287 });
288 }
289 }
290
291 // Scan any .md files under content/ not already covered by Fumadocs sources
292 const coveredPaths = new Set(allFiles.map(f => resolve(f.path)));
293 const extraMdFiles = await Array.fromAsync(glob('content/**/*.md'));
294 for (const filePath of extraMdFiles) {
295 if (coveredPaths.has(resolve(filePath))) continue;
296 const content = withoutFrontmatter(await readFile(filePath, 'utf-8'));
297 allFiles.push({ path: filePath, content });
298 }
299
300 return allFiles;
301 }
302
303 if (import.meta.main) void checkLinks();