Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { readFile } from 'node:fs/promises'; |
| 2 | import { glob } from 'node:fs/promises'; |
| 3 | import { resolve } from 'node:path'; |
| 4 | import { type FileObject, printErrors, scanURLs, validateFiles } from 'next-validate-link'; |
| 5 | import GithubSlugger from 'github-slugger'; |
| 6 | import { z } from 'zod'; |
| 7 | import { |
| 8 | source, |
| 9 | getReferenceSource, |
| 10 | examplesSource, |
| 11 | toolkitsSource, |
| 12 | knowledgeBaseSource, |
| 13 | } from '../lib/source'; |
| 14 | import { getKnowledgeToolkitSummaries } from '../lib/knowledge/catalog'; |
| 15 | import { getLocalKnowledgeDiscoveryPaths } from '../lib/knowledge/discovery'; |
| 16 | |
| 17 | /** |
| 18 | * `--external` additionally HEAD/GET-checks every external URL. Slow and |
| 19 | * dependent on third-party uptime, so it runs on a schedule (nightly CI), |
| 20 | * never on the PR path. |
| 21 | */ |
| 22 | const checkExternalLinks = process.argv.includes('--external'); |
| 23 | |
| 24 | type AnySource = |
| 25 | | typeof source |
| 26 | | Awaited<ReturnType<typeof getReferenceSource>> |
| 27 | | typeof examplesSource |
| 28 | | typeof toolkitsSource |
| 29 | | typeof knowledgeBaseSource; |
| 30 | |
| 31 | type PageOf = ReturnType<AnySource['getPages']>[number]; |
| 32 | |
| 33 | export function withoutFrontmatter(content: string): string { |
| 34 | return content.replace(/^(?:\uFEFF)?---\r?\n[\s\S]*?\r?\n---\r?\n?/, ''); |
| 35 | } |
| 36 | |
| 37 | /** |
| 38 | * Extract heading anchors from raw MDX/markdown content. |
| 39 | * Falls back to this when data.toc is unavailable (outside Next.js runtime). |
| 40 | * Uses github-slugger to match rehype-slug's algorithm (handles Unicode, duplicate suffixes). |
| 41 | */ |
| 42 | function extractHeadingsFromContent(content: string): string[] { |
| 43 | const slugger = new GithubSlugger(); |
| 44 | const headings: string[] = []; |
| 45 | let inCodeBlock = false; |
| 46 | for (const line of content.split('\n')) { |
| 47 | if (line.startsWith('```')) { |
| 48 | inCodeBlock = !inCodeBlock; |
| 49 | continue; |
| 50 | } |
| 51 | if (inCodeBlock) continue; |
| 52 | const match = line.match(/^#{1,6}\s+(.+)$/); |
| 53 | if (match) { |
| 54 | headings.push(slugger.slug(match[1])); |
| 55 | } |
| 56 | } |
| 57 | return headings; |
| 58 | } |
| 59 | |
| 60 | // Page data differs per source (MDX pages carry toc/getText, OpenAPI pages do |
| 61 | // not), so the pieces this script walks are parsed instead of guarded by hand. |
| 62 | const tocEntrySchema = z.object({ url: z.string() }); |
| 63 | const tocPageDataSchema = z.object({ toc: z.array(z.unknown()) }); |
| 64 | const textPageDataSchema = z.object({ |
| 65 | getText: z.custom<(mode: 'processed' | 'raw') => Promise<string>>( |
| 66 | value => typeof value === 'function' |
| 67 | ), |
| 68 | }); |
| 69 | |
| 70 | /** |
| 71 | * Get headings for a page, trying data.toc first then falling back to raw content parsing. |
| 72 | */ |
| 73 | async function getHeadingsForPage(page: PageOf): Promise<string[]> { |
| 74 | const tocData = tocPageDataSchema.safeParse(page.data); |
| 75 | if (tocData.success && tocData.data.toc.length > 0) { |
| 76 | return tocData.data.toc.flatMap(item => { |
| 77 | const entry = tocEntrySchema.safeParse(item); |
| 78 | return entry.success ? [entry.data.url.slice(1)] : []; |
| 79 | }); |
| 80 | } |
| 81 | const textData = textPageDataSchema.safeParse(page.data); |
| 82 | if (textData.success) { |
| 83 | try { |
| 84 | const content = await textData.data.getText('raw'); |
| 85 | return extractHeadingsFromContent(content); |
| 86 | } catch { |
| 87 | // fall through |
| 88 | } |
| 89 | } |
| 90 | if (page.absolutePath) { |
| 91 | try { |
| 92 | const content = await readFile(page.absolutePath, 'utf-8'); |
| 93 | return extractHeadingsFromContent(content); |
| 94 | } catch { |
| 95 | // fall through |
| 96 | } |
| 97 | } |
| 98 | return []; |
| 99 | } |
| 100 | |
| 101 | /** |
| 102 | * Build populate entries for a source, resolving headings asynchronously. |
| 103 | */ |
| 104 | async function buildPopulateEntries(src: AnySource) { |
| 105 | return Promise.all( |
| 106 | src.getPages().map(async (page: PageOf) => ({ |
| 107 | value: { slug: page.slugs }, |
| 108 | hashes: await getHeadingsForPage(page), |
| 109 | })) |
| 110 | ); |
| 111 | } |
| 112 | |
| 113 | const toolkitSlugsSchema = z.array(z.object({ slug: z.string() })); |
| 114 | |
| 115 | export async function getDynamicToolkitEntries() { |
| 116 | const raw = await readFile('public/data/toolkits.json', 'utf-8'); |
| 117 | const toolkits = toolkitSlugsSchema.parse(JSON.parse(raw)); |
| 118 | return toolkits.flatMap(t => |
| 119 | ['', '.md', '.mdx'].map(suffix => ({ |
| 120 | value: { slug: [`${t.slug}${suffix}`] }, |
| 121 | hashes: [] as string[], |
| 122 | })), |
| 123 | ); |
| 124 | } |
| 125 | |
| 126 | export async function getKnowledgeToolkitRouteEntries() { |
| 127 | return (await getKnowledgeToolkitSummaries()).map(toolkit => ({ |
| 128 | value: toolkit.slug, |
| 129 | hashes: [] as string[], |
| 130 | })); |
| 131 | } |
| 132 | |
| 133 | const EXTERNAL_FETCH_HEADERS = { |
| 134 | // Some hosts reject requests without a browser-like UA (bot filters). |
| 135 | 'user-agent': |
| 136 | 'Mozilla/5.0 (compatible; composio-docs-link-check; +https://docs.composio.dev)', |
| 137 | accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8', |
| 138 | }; |
| 139 | |
| 140 | async function fetchStatus(url: URL, method: 'HEAD' | 'GET'): Promise<number> { |
| 141 | const response = await fetch(url, { |
| 142 | method, |
| 143 | headers: EXTERNAL_FETCH_HEADERS, |
| 144 | redirect: 'follow', |
| 145 | signal: AbortSignal.timeout(15_000), |
| 146 | }); |
| 147 | await response.body?.cancel(); |
| 148 | return response.status; |
| 149 | } |
| 150 | |
| 151 | const externalResultCache = new Map< |
| 152 | string, |
| 153 | Promise<{ success: true } | { success: false; message?: string }> |
| 154 | >(); |
| 155 | |
| 156 | /** |
| 157 | * Replaces next-validate-link's default external validator (bare HEAD, no |
| 158 | * timeout, no UA, no retry). This one only fails on evidence the link is dead |
| 159 | * — 404/410, or a network error after a retry. Auth walls, rate limits, and |
| 160 | * transient 5xx count as reachable: the nightly job hunts dead links, not |
| 161 | * third-party uptime. |
| 162 | */ |
| 163 | function validateExternalUrl(url: URL) { |
| 164 | if (url.hostname === 'localhost' || url.hostname === '127.0.0.1') { |
| 165 | return Promise.resolve({ success: true as const }); |
| 166 | } |
| 167 | const cached = externalResultCache.get(url.href); |
| 168 | if (cached) return cached; |
| 169 | |
| 170 | const result = (async () => { |
| 171 | for (let attempt = 0; attempt < 2; attempt++) { |
| 172 | try { |
| 173 | let status = await fetchStatus(url, 'HEAD'); |
| 174 | // Many servers reject HEAD (405, bot filters); confirm with GET. |
| 175 | if (status >= 400) status = await fetchStatus(url, 'GET'); |
| 176 | if (status === 404 || status === 410) { |
| 177 | return { success: false as const, message: `responded status ${status}` }; |
| 178 | } |
| 179 | return { success: true as const }; |
| 180 | } catch (error) { |
| 181 | if (attempt === 0) continue; |
| 182 | return { |
| 183 | success: false as const, |
| 184 | message: error instanceof Error ? error.message : 'fetch failed', |
| 185 | }; |
| 186 | } |
| 187 | } |
| 188 | return { success: false as const }; |
| 189 | })(); |
| 190 | externalResultCache.set(url.href, result); |
| 191 | return result; |
| 192 | } |
| 193 | |
| 194 | async function checkLinks() { |
| 195 | const referenceSource = await getReferenceSource(); |
| 196 | const knowledgeDiscoveryPaths = await getLocalKnowledgeDiscoveryPaths(); |
| 197 | const knowledgeTopicEntries = knowledgeDiscoveryPaths |
| 198 | .filter((path) => path.startsWith('/kb/topic/')) |
| 199 | .map((path) => ({ value: path.slice('/kb/topic/'.length), hashes: [] as string[] })); |
| 200 | const [ |
| 201 | docsEntries, |
| 202 | refEntries, |
| 203 | exampleEntries, |
| 204 | toolkitEntries, |
| 205 | knowledgeBaseEntries, |
| 206 | dynamicToolkitEntries, |
| 207 | knowledgeToolkitEntries, |
| 208 | ] = await Promise.all([ |
| 209 | buildPopulateEntries(source), |
| 210 | buildPopulateEntries(referenceSource), |
| 211 | buildPopulateEntries(examplesSource), |
| 212 | buildPopulateEntries(toolkitsSource), |
| 213 | buildPopulateEntries(knowledgeBaseSource), |
| 214 | getDynamicToolkitEntries(), |
| 215 | getKnowledgeToolkitRouteEntries(), |
| 216 | ]); |
| 217 | const knowledgeGuideEntries = knowledgeBaseEntries.flatMap((entry) => { |
| 218 | const value = entry.value; |
| 219 | if (!value || Array.isArray(value) || typeof value === 'string' || !('slug' in value)) return []; |
| 220 | const slug = value.slug; |
| 221 | const segments = Array.isArray(slug) ? slug : [slug]; |
| 222 | return segments.length === 2 && segments[0] === 'guide' |
| 223 | ? [{ ...entry, value: segments[1] }] |
| 224 | : []; |
| 225 | }); |
| 226 | |
| 227 | const scanned = await scanURLs({ |
| 228 | preset: 'next', |
| 229 | populate: { |
| 230 | // Keys must include (home) route group to match app directory structure |
| 231 | '(home)/docs/[[...slug]]': docsEntries, |
| 232 | '(home)/reference/[[...slug]]': refEntries, |
| 233 | '(home)/examples/[[...slug]]': exampleEntries, |
| 234 | '(home)/toolkits/[[...slug]]': [...toolkitEntries, ...dynamicToolkitEntries], |
| 235 | '(home)/kb/guide/[slug]': knowledgeGuideEntries, |
| 236 | '(home)/kb/topic/[slug]': knowledgeTopicEntries, |
| 237 | '(home)/kb/toolkit/[slug]': knowledgeToolkitEntries, |
| 238 | }, |
| 239 | }); |
| 240 | |
| 241 | const errors = await validateFiles(await getFiles(), { |
| 242 | scanned, |
| 243 | markdown: { |
| 244 | components: { |
| 245 | Card: { attributes: ['href'] }, |
| 246 | }, |
| 247 | }, |
| 248 | checkRelativePaths: 'as-url', |
| 249 | checkExternal: checkExternalLinks ? { validate: validateExternalUrl } : false, |
| 250 | }); |
| 251 | |
| 252 | // Filter out API route URLs (these are valid but not detected as pages) |
| 253 | const ignoredUrls = ['/llms.txt', '/llms-full.txt']; |
| 254 | const filteredErrors = errors |
| 255 | .map(fileError => ({ |
| 256 | ...fileError, |
| 257 | errors: fileError.errors.filter(e => !ignoredUrls.includes(e.url)), |
| 258 | detected: fileError.detected.filter(d => !ignoredUrls.includes(d[0] as string)), |
| 259 | })) |
| 260 | .filter(fileError => fileError.errors.length > 0); |
| 261 | |
| 262 | printErrors(filteredErrors, true); |
| 263 | if (filteredErrors.length > 0) { |
| 264 | process.exit(1); |
| 265 | } |
| 266 | } |
| 267 | |
| 268 | async function getFiles(): Promise<FileObject[]> { |
| 269 | const referenceSource = await getReferenceSource(); |
| 270 | const sources = [source, referenceSource, examplesSource, toolkitsSource, knowledgeBaseSource]; |
| 271 | const allFiles: FileObject[] = []; |
| 272 | |
| 273 | for (const src of sources) { |
| 274 | const pages = src.getPages(); |
| 275 | for (const page of pages) { |
| 276 | if (!page.absolutePath) continue; |
| 277 | if (!page.absolutePath.endsWith('.mdx') && !page.absolutePath.endsWith('.md')) continue; |
| 278 | // Skip OpenAPI-generated pages (they don't have getText) |
| 279 | const textData = textPageDataSchema.safeParse(page.data); |
| 280 | if (!textData.success) continue; |
| 281 | |
| 282 | allFiles.push({ |
| 283 | path: page.absolutePath, |
| 284 | content: withoutFrontmatter(await textData.data.getText('raw')), |
| 285 | url: page.url, |
| 286 | data: page.data, |
| 287 | }); |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | // Scan any .md files under content/ not already covered by Fumadocs sources |
| 292 | const coveredPaths = new Set(allFiles.map(f => resolve(f.path))); |
| 293 | const extraMdFiles = await Array.fromAsync(glob('content/**/*.md')); |
| 294 | for (const filePath of extraMdFiles) { |
| 295 | if (coveredPaths.has(resolve(filePath))) continue; |
| 296 | const content = withoutFrontmatter(await readFile(filePath, 'utf-8')); |
| 297 | allFiles.push({ path: filePath, content }); |
| 298 | } |
| 299 | |
| 300 | return allFiles; |
| 301 | } |
| 302 | |
| 303 | if (import.meta.main) void checkLinks(); |