Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; |
| 2 | import { basename, dirname, join, parse, posix, relative, resolve } from 'node:path'; |
| 3 | import GithubSlugger from 'github-slugger'; |
| 4 | import { getKbCatalog, getKbGuideUrl, getPublishedKbGuides } from './repository'; |
| 5 | import type { KbCatalog, KbGuide } from './types'; |
| 6 | import toolkitsData from '@/public/data/toolkits-list.json'; |
| 7 | import type { ToolkitSummary } from '@/types/toolkit'; |
| 8 | import { markdownLanguageGroups, parseExampleDocument } from '../example-document'; |
| 9 | |
| 10 | export interface KbGenerationSummary { |
| 11 | published: number; |
| 12 | held: number; |
| 13 | files: number; |
| 14 | } |
| 15 | |
| 16 | export interface GenerateKbContentOptions { |
| 17 | outputDir?: string; |
| 18 | check?: boolean; |
| 19 | catalog?: KbCatalog; |
| 20 | } |
| 21 | |
| 22 | const EXTERNAL_RESOURCES: Record<string, { title: string; href: string; description: string }> = { |
| 23 | 'docs-sessions': { |
| 24 | title: 'Configure Composio sessions', |
| 25 | href: '/docs/configuring-sessions', |
| 26 | description: 'Create and configure Tool Router sessions.', |
| 27 | }, |
| 28 | 'docs-api-reference': { |
| 29 | title: 'Composio API reference', |
| 30 | href: '/reference', |
| 31 | description: 'Inspect current endpoint schemas and parameters.', |
| 32 | }, |
| 33 | }; |
| 34 | |
| 35 | const toolkitSlugByLooseKey = new Map( |
| 36 | (toolkitsData as ToolkitSummary[]).map(toolkit => [ |
| 37 | toolkit.slug.toLowerCase().replace(/[-_\s]+/g, ''), |
| 38 | toolkit.slug, |
| 39 | ]), |
| 40 | ); |
| 41 | |
| 42 | function toolkitSlugsForGuide(guide: KbGuide): string[] { |
| 43 | return [...new Set(guide.sources.flatMap(source => { |
| 44 | const sourceSlug = source.sourcePath.match(/^toolkits\/([^/]+)\/public\.md$/)?.[1]; |
| 45 | if (!sourceSlug) return []; |
| 46 | const looseKey = sourceSlug.toLowerCase().replace(/[-_\s]+/g, ''); |
| 47 | return [toolkitSlugByLooseKey.get(looseKey) ?? sourceSlug]; |
| 48 | }))].sort(); |
| 49 | } |
| 50 | |
| 51 | function yamlString(value: string): string { |
| 52 | return JSON.stringify(value); |
| 53 | } |
| 54 | |
| 55 | function yamlArray(values: string[]): string { |
| 56 | return JSON.stringify(values); |
| 57 | } |
| 58 | |
| 59 | function relatedResources(guide: KbGuide, guides: KbGuide[]) { |
| 60 | const bySlug = new Map(guides.map(candidate => [candidate.slug, candidate])); |
| 61 | const internal = guide.relatedGuides.flatMap(slug => { |
| 62 | const related = bySlug.get(slug); |
| 63 | if (!related) return []; |
| 64 | return [ |
| 65 | { |
| 66 | title: related.title, |
| 67 | href: getKbGuideUrl(related), |
| 68 | description: related.description, |
| 69 | }, |
| 70 | ]; |
| 71 | }); |
| 72 | const external = guide.externalResources.flatMap(id => { |
| 73 | const resource = EXTERNAL_RESOURCES[id]; |
| 74 | return resource ? [resource] : []; |
| 75 | }); |
| 76 | return [...internal, ...external]; |
| 77 | } |
| 78 | |
| 79 | function relatedFrontmatter( |
| 80 | resources: Array<{ title: string; href: string; description: string }> |
| 81 | ): string[] { |
| 82 | if (resources.length === 0) return []; |
| 83 | return [ |
| 84 | 'related:', |
| 85 | ...resources.flatMap(resource => [ |
| 86 | ` - title: ${yamlString(resource.title)}`, |
| 87 | ` href: ${yamlString(resource.href)}`, |
| 88 | ` description: ${yamlString(resource.description)}`, |
| 89 | ]), |
| 90 | ]; |
| 91 | } |
| 92 | |
| 93 | /** |
| 94 | * Identifier URLs cite machine identifiers — OAuth scope URIs and API |
| 95 | * surface roots such as `https://api.example.com/v3` — not documents. They |
| 96 | * respond 404 by design, so they must never publish as links the nightly |
| 97 | * external-link sweep would have to keep alive: they render as code spans. |
| 98 | */ |
| 99 | export function isIdentifierUrl(raw: string): boolean { |
| 100 | let url: URL; |
| 101 | try { |
| 102 | url = new URL(raw); |
| 103 | } catch { |
| 104 | return false; |
| 105 | } |
| 106 | if ((url.protocol !== 'https:' && url.protocol !== 'http:') || url.search || url.hash) { |
| 107 | return false; |
| 108 | } |
| 109 | // Google OAuth scope URIs (https://www.googleapis.com/auth/…) identify a |
| 110 | // permission; the namespace serves no pages. |
| 111 | if (url.hostname === 'www.googleapis.com' && url.pathname.startsWith('/auth/')) { |
| 112 | return true; |
| 113 | } |
| 114 | // An API surface root: a version-only path such as `/v3` or `/v1beta/`. |
| 115 | // Deep paths (`/v3/tools/X`) stay links because they name real resources. |
| 116 | return /^\/v\d[\w.-]*\/?$/.test(url.pathname); |
| 117 | } |
| 118 | |
| 119 | /** |
| 120 | * Wraps bare identifier URLs in code spans. URLs already presented as links — |
| 121 | * markdown link labels `[url](…)`, link targets `](url)`, and anything |
| 122 | * adjacent to a code span — keep their form; only bare citations and |
| 123 | * `<url>` autolinks (normalized earlier) become code. |
| 124 | */ |
| 125 | function identifierUrlsToCodeSpans(segment: string): string { |
| 126 | return segment.replace(/(?<![`(\]\[])https?:\/\/[^\s`<>\[\]()]+/g, match => { |
| 127 | // GFM autolinks drop trailing punctuation; keep it outside the code span. |
| 128 | const url = match.replace(/[.,;:!?'"]+$/, ''); |
| 129 | if (!isIdentifierUrl(url)) return match; |
| 130 | return `\`${url}\`${match.slice(url.length)}`; |
| 131 | }); |
| 132 | } |
| 133 | |
| 134 | function escapeMdxProse(line: string): string { |
| 135 | const escapeSegment = (segment: string): string => identifierUrlsToCodeSpans( |
| 136 | segment |
| 137 | .replace(/<(https?:\/\/[^>\s]+)>/g, (match, url: string) => |
| 138 | isIdentifierUrl(url) ? `\`${url}\`` : `[${url}](${url})`) |
| 139 | .replace(/<([^>\n]+)>/g, '<$1>') |
| 140 | .replace(/\{/g, '{') |
| 141 | .replace(/\}/g, '}'), |
| 142 | ); |
| 143 | |
| 144 | let result = ''; |
| 145 | let cursor = 0; |
| 146 | while (cursor < line.length) { |
| 147 | const opening = line.indexOf('`', cursor); |
| 148 | if (opening < 0) return result + escapeSegment(line.slice(cursor)); |
| 149 | result += escapeSegment(line.slice(cursor, opening)); |
| 150 | |
| 151 | let markerEnd = opening; |
| 152 | while (line[markerEnd] === '`') markerEnd += 1; |
| 153 | const marker = line.slice(opening, markerEnd); |
| 154 | const closing = line.indexOf(marker, markerEnd); |
| 155 | if (closing < 0) return result + escapeSegment(line.slice(opening)); |
| 156 | result += line.slice(opening, closing + marker.length); |
| 157 | cursor = closing + marker.length; |
| 158 | } |
| 159 | return result; |
| 160 | } |
| 161 | |
| 162 | /** Escape prose without changing code fence language or code content. */ |
| 163 | function escapeMarkdownForMdx(markdown: string): string { |
| 164 | let fence: { marker: string; length: number } | null = null; |
| 165 | return markdown.split('\n').map(line => { |
| 166 | const match = line.match(/^(\s*)(`{3,}|~{3,})(.*)$/); |
| 167 | if (match) { |
| 168 | const marker = match[2] ?? ''; |
| 169 | const info = match[3] ?? ''; |
| 170 | if (!fence) { |
| 171 | fence = { marker: marker[0] ?? '`', length: marker.length }; |
| 172 | return line; |
| 173 | } |
| 174 | if (marker[0] === fence.marker && marker.length >= fence.length && info.trim() === '') { |
| 175 | fence = null; |
| 176 | } |
| 177 | return line; |
| 178 | } |
| 179 | return fence ? line : escapeMdxProse(line); |
| 180 | }).join('\n'); |
| 181 | } |
| 182 | |
| 183 | function safeMarkdownForMdx(markdown: string): string { |
| 184 | const changes: { start: number; end: number; text: string }[] = []; |
| 185 | for (const node of parseExampleDocument(markdown, 'kb.md').children) { |
| 186 | const start = node.position?.start.offset; |
| 187 | const end = node.position?.end.offset; |
| 188 | if (start === undefined || end === undefined) continue; |
| 189 | if (node.type === 'html' && /^<!--\s*code-example:/.test(node.value)) { |
| 190 | // Validation metadata belongs to the canonical authored Markdown, never |
| 191 | // visible prose in the generated article. |
| 192 | changes.push({ start, end, text: '' }); |
| 193 | } else if (node.type === 'blockquote') { |
| 194 | const first = node.children[0]; |
| 195 | const text = first?.type === 'paragraph' ? first.children[0] : undefined; |
| 196 | if (text?.type !== 'text' || !/^\[!NOTE\](?:\s|$)/.test(text.value)) continue; |
| 197 | const body = markdown.slice(start, end).split('\n') |
| 198 | .map((line) => line.replace(/^ {0,3}> ?/, '')).join('\n') |
| 199 | .replace(/^\[!NOTE\]\s*/, ''); |
| 200 | changes.push({ start, end, text: `<Callout type="info">\n\n${markdownForMdx(body)}\n\n</Callout>` }); |
| 201 | } |
| 202 | } |
| 203 | let result = ''; |
| 204 | let cursor = 0; |
| 205 | for (const change of changes) { |
| 206 | result += escapeMarkdownForMdx(markdown.slice(cursor, change.start)) + change.text; |
| 207 | cursor = change.end; |
| 208 | } |
| 209 | return result + escapeMarkdownForMdx(markdown.slice(cursor)); |
| 210 | } |
| 211 | |
| 212 | /** |
| 213 | * Preserve authoritative CommonMark as the agent-facing source, while turning |
| 214 | * its language headings and NOTE blocks into real Fumadocs Tabs and Callouts. |
| 215 | * TypeScript fences keep their language and remain type-checkable. |
| 216 | */ |
| 217 | export function markdownForMdx(markdown: string): string { |
| 218 | let result = ''; |
| 219 | let cursor = 0; |
| 220 | for (const group of markdownLanguageGroups(markdown)) { |
| 221 | result += safeMarkdownForMdx(markdown.slice(cursor, group.start)); |
| 222 | result += `<Tabs groupId="language" items={${JSON.stringify(group.tabs.map((tab) => tab.value))}} persist>\n`; |
| 223 | for (const tab of group.tabs) { |
| 224 | result += `\n<Tab value="${tab.value}">\n${markdownForMdx(tab.source).trim()}\n</Tab>\n`; |
| 225 | } |
| 226 | result += '\n</Tabs>\n\n'; |
| 227 | cursor = group.end; |
| 228 | } |
| 229 | return result + safeMarkdownForMdx(markdown.slice(cursor)); |
| 230 | } |
| 231 | |
| 232 | function rewriteSourceRepositoryLinks(markdown: string, guide: KbGuide, guides: KbGuide[]): string { |
| 233 | const guideBySourcePath = new Map( |
| 234 | guides.flatMap(candidate => candidate.sources.map(source => [source.sourcePath, candidate] as const)), |
| 235 | ); |
| 236 | const headingFragmentsByGuide = new Map(guides.map(candidate => { |
| 237 | const slugger = new GithubSlugger(); |
| 238 | const fragments = candidate.body.split('\n').flatMap(line => { |
| 239 | const heading = line.match(/^#{1,6}\s+(.+)$/)?.[1]; |
| 240 | return heading ? [slugger.slug(heading)] : []; |
| 241 | }); |
| 242 | return [candidate.slug, new Set(fragments)] as const; |
| 243 | })); |
| 244 | const sourcePaths = [...new Set(guide.sources.map(source => source.sourcePath))]; |
| 245 | |
| 246 | return markdown.replace( |
| 247 | /\]\((\.\.?\/[^)\s]*?public\.md)(#[^)\s]+)?\)/g, |
| 248 | (match, relativeTarget: string, fragment = '') => { |
| 249 | const targets = new Set( |
| 250 | sourcePaths.flatMap(sourcePath => { |
| 251 | const resolved = posix.normalize(posix.join(posix.dirname(sourcePath), relativeTarget)); |
| 252 | const target = guideBySourcePath.get(resolved); |
| 253 | return target ? [target] : []; |
| 254 | }), |
| 255 | ); |
| 256 | if (targets.size !== 1) return match; |
| 257 | const [target] = targets; |
| 258 | const validFragment = fragment |
| 259 | && headingFragmentsByGuide.get(target!.slug)?.has(fragment.slice(1)) |
| 260 | ? fragment |
| 261 | : ''; |
| 262 | return `](${getKbGuideUrl(target!)}${validFragment})`; |
| 263 | }, |
| 264 | ); |
| 265 | } |
| 266 | |
| 267 | function guideMdx(guide: KbGuide, guides: KbGuide[]): string { |
| 268 | const related = relatedResources(guide, guides); |
| 269 | const toolkitSlugs = toolkitSlugsForGuide(guide); |
| 270 | const frontmatter = [ |
| 271 | '---', |
| 272 | `title: ${yamlString(guide.title)}`, |
| 273 | `description: ${yamlString(guide.description)}`, |
| 274 | `keywords: ${yamlArray([...guide.tags, ...guide.topics, ...guide.aliases])}`, |
| 275 | `sources: ${JSON.stringify(guide.sources)}`, |
| 276 | `lastVerifiedAt: ${yamlString(guide.lastVerifiedAt ?? '')}`, |
| 277 | `reviewAfter: ${yamlString(guide.reviewAfter ?? '')}`, |
| 278 | `freshness: ${yamlString(guide.freshness)}`, |
| 279 | `topics: ${yamlArray(guide.topics)}`, |
| 280 | ...(toolkitSlugs.length > 0 ? [`toolkitSlugs: ${yamlArray(toolkitSlugs)}`] : []), |
| 281 | `aliases: ${yamlArray(guide.aliases)}`, |
| 282 | ...relatedFrontmatter(related), |
| 283 | '---', |
| 284 | '', |
| 285 | ]; |
| 286 | const body = rewriteSourceRepositoryLinks(guide.body.trim(), guide, guides); |
| 287 | return `${frontmatter.join('\n')}${markdownForMdx(body)}\n`; |
| 288 | } |
| 289 | |
| 290 | function rootIndex(): string { |
| 291 | return `--- |
| 292 | title: "Knowledge Base" |
| 293 | description: "Verified troubleshooting guides and answers for building with Composio." |
| 294 | --- |
| 295 | |
| 296 | Verified troubleshooting guides, operational answers, and known-good patterns from Composio support. |
| 297 | `; |
| 298 | } |
| 299 | |
| 300 | function buildExpectedFiles(catalog: KbCatalog): Map<string, string> { |
| 301 | const guides = getPublishedKbGuides(catalog); |
| 302 | const files = new Map<string, string>(); |
| 303 | files.set('index.mdx', rootIndex()); |
| 304 | files.set( |
| 305 | 'meta.json', |
| 306 | `${JSON.stringify({ title: 'Knowledge Base', root: true, pages: ['index', 'guide'] }, null, 2)}\n` |
| 307 | ); |
| 308 | files.set( |
| 309 | 'guide/meta.json', |
| 310 | `${JSON.stringify({ title: 'Guides', pages: guides.map(guide => guide.slug) }, null, 2)}\n` |
| 311 | ); |
| 312 | for (const guide of guides) { |
| 313 | files.set(`guide/${guide.slug}.mdx`, guideMdx(guide, guides)); |
| 314 | } |
| 315 | return files; |
| 316 | } |
| 317 | |
| 318 | function listRelativeFiles(directory: string): string[] { |
| 319 | if (!existsSync(directory)) return []; |
| 320 | return readdirSync(directory, { recursive: true, withFileTypes: true }) |
| 321 | .filter(entry => entry.isFile()) |
| 322 | .map(entry => relative(directory, join(entry.parentPath, entry.name))) |
| 323 | .sort(); |
| 324 | } |
| 325 | |
| 326 | function planFileChanges(outputDir: string, expected: Map<string, string>) { |
| 327 | const actualFiles = listRelativeFiles(outputDir); |
| 328 | const actualFileSet = new Set(actualFiles); |
| 329 | const writes = [...expected].filter( |
| 330 | ([path, content]) => |
| 331 | !actualFileSet.has(path) || readFileSync(join(outputDir, path), 'utf8') !== content, |
| 332 | ); |
| 333 | const removals = actualFiles.filter(path => !expected.has(path)); |
| 334 | return { writes, removals }; |
| 335 | } |
| 336 | |
| 337 | function assertSafeOutputDirectory(outputDir: string): void { |
| 338 | const resolved = resolve(outputDir); |
| 339 | if (resolved === parse(resolved).root || basename(resolved) === '') { |
| 340 | throw new Error(`Refusing to generate KB content into unsafe path: ${resolved}`); |
| 341 | } |
| 342 | } |
| 343 | |
| 344 | export function generateKbContent(options: GenerateKbContentOptions = {}): KbGenerationSummary { |
| 345 | const outputDir = resolve(options.outputDir ?? join(process.cwd(), 'content', 'kb')); |
| 346 | assertSafeOutputDirectory(outputDir); |
| 347 | const catalog = options.catalog ?? getKbCatalog(); |
| 348 | const expected = buildExpectedFiles(catalog); |
| 349 | const published = getPublishedKbGuides(catalog).length; |
| 350 | const held = catalog.guides.filter(guide => guide.state === 'needs-review').length; |
| 351 | const changes = planFileChanges(outputDir, expected); |
| 352 | |
| 353 | if (options.check) { |
| 354 | if (changes.writes.length > 0 || changes.removals.length > 0) { |
| 355 | throw new Error('Generated KB content is out of date; run bun run generate:kb'); |
| 356 | } |
| 357 | return { published, held, files: expected.size }; |
| 358 | } |
| 359 | |
| 360 | for (const [path, content] of changes.writes) { |
| 361 | const target = join(outputDir, path); |
| 362 | mkdirSync(dirname(target), { recursive: true }); |
| 363 | writeFileSync(target, content, 'utf8'); |
| 364 | } |
| 365 | for (const path of changes.removals) { |
| 366 | rmSync(join(outputDir, path), { force: true }); |
| 367 | } |
| 368 | return { published, held, files: expected.size }; |
| 369 | } |