Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 369 lines 13.6 KB
Raw
1 import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
2 import { basename, dirname, join, parse, posix, relative, resolve } from 'node:path';
3 import GithubSlugger from 'github-slugger';
4 import { getKbCatalog, getKbGuideUrl, getPublishedKbGuides } from './repository';
5 import type { KbCatalog, KbGuide } from './types';
6 import toolkitsData from '@/public/data/toolkits-list.json';
7 import type { ToolkitSummary } from '@/types/toolkit';
8 import { markdownLanguageGroups, parseExampleDocument } from '../example-document';
9
10 export interface KbGenerationSummary {
11 published: number;
12 held: number;
13 files: number;
14 }
15
16 export interface GenerateKbContentOptions {
17 outputDir?: string;
18 check?: boolean;
19 catalog?: KbCatalog;
20 }
21
22 const EXTERNAL_RESOURCES: Record<string, { title: string; href: string; description: string }> = {
23 'docs-sessions': {
24 title: 'Configure Composio sessions',
25 href: '/docs/configuring-sessions',
26 description: 'Create and configure Tool Router sessions.',
27 },
28 'docs-api-reference': {
29 title: 'Composio API reference',
30 href: '/reference',
31 description: 'Inspect current endpoint schemas and parameters.',
32 },
33 };
34
35 const toolkitSlugByLooseKey = new Map(
36 (toolkitsData as ToolkitSummary[]).map(toolkit => [
37 toolkit.slug.toLowerCase().replace(/[-_\s]+/g, ''),
38 toolkit.slug,
39 ]),
40 );
41
42 function toolkitSlugsForGuide(guide: KbGuide): string[] {
43 return [...new Set(guide.sources.flatMap(source => {
44 const sourceSlug = source.sourcePath.match(/^toolkits\/([^/]+)\/public\.md$/)?.[1];
45 if (!sourceSlug) return [];
46 const looseKey = sourceSlug.toLowerCase().replace(/[-_\s]+/g, '');
47 return [toolkitSlugByLooseKey.get(looseKey) ?? sourceSlug];
48 }))].sort();
49 }
50
51 function yamlString(value: string): string {
52 return JSON.stringify(value);
53 }
54
55 function yamlArray(values: string[]): string {
56 return JSON.stringify(values);
57 }
58
59 function relatedResources(guide: KbGuide, guides: KbGuide[]) {
60 const bySlug = new Map(guides.map(candidate => [candidate.slug, candidate]));
61 const internal = guide.relatedGuides.flatMap(slug => {
62 const related = bySlug.get(slug);
63 if (!related) return [];
64 return [
65 {
66 title: related.title,
67 href: getKbGuideUrl(related),
68 description: related.description,
69 },
70 ];
71 });
72 const external = guide.externalResources.flatMap(id => {
73 const resource = EXTERNAL_RESOURCES[id];
74 return resource ? [resource] : [];
75 });
76 return [...internal, ...external];
77 }
78
79 function relatedFrontmatter(
80 resources: Array<{ title: string; href: string; description: string }>
81 ): string[] {
82 if (resources.length === 0) return [];
83 return [
84 'related:',
85 ...resources.flatMap(resource => [
86 ` - title: ${yamlString(resource.title)}`,
87 ` href: ${yamlString(resource.href)}`,
88 ` description: ${yamlString(resource.description)}`,
89 ]),
90 ];
91 }
92
93 /**
94 * Identifier URLs cite machine identifiers — OAuth scope URIs and API
95 * surface roots such as `https://api.example.com/v3` — not documents. They
96 * respond 404 by design, so they must never publish as links the nightly
97 * external-link sweep would have to keep alive: they render as code spans.
98 */
99 export function isIdentifierUrl(raw: string): boolean {
100 let url: URL;
101 try {
102 url = new URL(raw);
103 } catch {
104 return false;
105 }
106 if ((url.protocol !== 'https:' && url.protocol !== 'http:') || url.search || url.hash) {
107 return false;
108 }
109 // Google OAuth scope URIs (https://www.googleapis.com/auth/…) identify a
110 // permission; the namespace serves no pages.
111 if (url.hostname === 'www.googleapis.com' && url.pathname.startsWith('/auth/')) {
112 return true;
113 }
114 // An API surface root: a version-only path such as `/v3` or `/v1beta/`.
115 // Deep paths (`/v3/tools/X`) stay links because they name real resources.
116 return /^\/v\d[\w.-]*\/?$/.test(url.pathname);
117 }
118
119 /**
120 * Wraps bare identifier URLs in code spans. URLs already presented as links —
121 * markdown link labels `[url](…)`, link targets `](url)`, and anything
122 * adjacent to a code span — keep their form; only bare citations and
123 * `<url>` autolinks (normalized earlier) become code.
124 */
125 function identifierUrlsToCodeSpans(segment: string): string {
126 return segment.replace(/(?<![`(\]\[])https?:\/\/[^\s`<>\[\]()]+/g, match => {
127 // GFM autolinks drop trailing punctuation; keep it outside the code span.
128 const url = match.replace(/[.,;:!?'"]+$/, '');
129 if (!isIdentifierUrl(url)) return match;
130 return `\`${url}\`${match.slice(url.length)}`;
131 });
132 }
133
134 function escapeMdxProse(line: string): string {
135 const escapeSegment = (segment: string): string => identifierUrlsToCodeSpans(
136 segment
137 .replace(/<(https?:\/\/[^>\s]+)>/g, (match, url: string) =>
138 isIdentifierUrl(url) ? `\`${url}\`` : `[${url}](${url})`)
139 .replace(/<([^>\n]+)>/g, '&lt;$1&gt;')
140 .replace(/\{/g, '&#123;')
141 .replace(/\}/g, '&#125;'),
142 );
143
144 let result = '';
145 let cursor = 0;
146 while (cursor < line.length) {
147 const opening = line.indexOf('`', cursor);
148 if (opening < 0) return result + escapeSegment(line.slice(cursor));
149 result += escapeSegment(line.slice(cursor, opening));
150
151 let markerEnd = opening;
152 while (line[markerEnd] === '`') markerEnd += 1;
153 const marker = line.slice(opening, markerEnd);
154 const closing = line.indexOf(marker, markerEnd);
155 if (closing < 0) return result + escapeSegment(line.slice(opening));
156 result += line.slice(opening, closing + marker.length);
157 cursor = closing + marker.length;
158 }
159 return result;
160 }
161
162 /** Escape prose without changing code fence language or code content. */
163 function escapeMarkdownForMdx(markdown: string): string {
164 let fence: { marker: string; length: number } | null = null;
165 return markdown.split('\n').map(line => {
166 const match = line.match(/^(\s*)(`{3,}|~{3,})(.*)$/);
167 if (match) {
168 const marker = match[2] ?? '';
169 const info = match[3] ?? '';
170 if (!fence) {
171 fence = { marker: marker[0] ?? '`', length: marker.length };
172 return line;
173 }
174 if (marker[0] === fence.marker && marker.length >= fence.length && info.trim() === '') {
175 fence = null;
176 }
177 return line;
178 }
179 return fence ? line : escapeMdxProse(line);
180 }).join('\n');
181 }
182
183 function safeMarkdownForMdx(markdown: string): string {
184 const changes: { start: number; end: number; text: string }[] = [];
185 for (const node of parseExampleDocument(markdown, 'kb.md').children) {
186 const start = node.position?.start.offset;
187 const end = node.position?.end.offset;
188 if (start === undefined || end === undefined) continue;
189 if (node.type === 'html' && /^<!--\s*code-example:/.test(node.value)) {
190 // Validation metadata belongs to the canonical authored Markdown, never
191 // visible prose in the generated article.
192 changes.push({ start, end, text: '' });
193 } else if (node.type === 'blockquote') {
194 const first = node.children[0];
195 const text = first?.type === 'paragraph' ? first.children[0] : undefined;
196 if (text?.type !== 'text' || !/^\[!NOTE\](?:\s|$)/.test(text.value)) continue;
197 const body = markdown.slice(start, end).split('\n')
198 .map((line) => line.replace(/^ {0,3}> ?/, '')).join('\n')
199 .replace(/^\[!NOTE\]\s*/, '');
200 changes.push({ start, end, text: `<Callout type="info">\n\n${markdownForMdx(body)}\n\n</Callout>` });
201 }
202 }
203 let result = '';
204 let cursor = 0;
205 for (const change of changes) {
206 result += escapeMarkdownForMdx(markdown.slice(cursor, change.start)) + change.text;
207 cursor = change.end;
208 }
209 return result + escapeMarkdownForMdx(markdown.slice(cursor));
210 }
211
212 /**
213 * Preserve authoritative CommonMark as the agent-facing source, while turning
214 * its language headings and NOTE blocks into real Fumadocs Tabs and Callouts.
215 * TypeScript fences keep their language and remain type-checkable.
216 */
217 export function markdownForMdx(markdown: string): string {
218 let result = '';
219 let cursor = 0;
220 for (const group of markdownLanguageGroups(markdown)) {
221 result += safeMarkdownForMdx(markdown.slice(cursor, group.start));
222 result += `<Tabs groupId="language" items={${JSON.stringify(group.tabs.map((tab) => tab.value))}} persist>\n`;
223 for (const tab of group.tabs) {
224 result += `\n<Tab value="${tab.value}">\n${markdownForMdx(tab.source).trim()}\n</Tab>\n`;
225 }
226 result += '\n</Tabs>\n\n';
227 cursor = group.end;
228 }
229 return result + safeMarkdownForMdx(markdown.slice(cursor));
230 }
231
232 function rewriteSourceRepositoryLinks(markdown: string, guide: KbGuide, guides: KbGuide[]): string {
233 const guideBySourcePath = new Map(
234 guides.flatMap(candidate => candidate.sources.map(source => [source.sourcePath, candidate] as const)),
235 );
236 const headingFragmentsByGuide = new Map(guides.map(candidate => {
237 const slugger = new GithubSlugger();
238 const fragments = candidate.body.split('\n').flatMap(line => {
239 const heading = line.match(/^#{1,6}\s+(.+)$/)?.[1];
240 return heading ? [slugger.slug(heading)] : [];
241 });
242 return [candidate.slug, new Set(fragments)] as const;
243 }));
244 const sourcePaths = [...new Set(guide.sources.map(source => source.sourcePath))];
245
246 return markdown.replace(
247 /\]\((\.\.?\/[^)\s]*?public\.md)(#[^)\s]+)?\)/g,
248 (match, relativeTarget: string, fragment = '') => {
249 const targets = new Set(
250 sourcePaths.flatMap(sourcePath => {
251 const resolved = posix.normalize(posix.join(posix.dirname(sourcePath), relativeTarget));
252 const target = guideBySourcePath.get(resolved);
253 return target ? [target] : [];
254 }),
255 );
256 if (targets.size !== 1) return match;
257 const [target] = targets;
258 const validFragment = fragment
259 && headingFragmentsByGuide.get(target!.slug)?.has(fragment.slice(1))
260 ? fragment
261 : '';
262 return `](${getKbGuideUrl(target!)}${validFragment})`;
263 },
264 );
265 }
266
267 function guideMdx(guide: KbGuide, guides: KbGuide[]): string {
268 const related = relatedResources(guide, guides);
269 const toolkitSlugs = toolkitSlugsForGuide(guide);
270 const frontmatter = [
271 '---',
272 `title: ${yamlString(guide.title)}`,
273 `description: ${yamlString(guide.description)}`,
274 `keywords: ${yamlArray([...guide.tags, ...guide.topics, ...guide.aliases])}`,
275 `sources: ${JSON.stringify(guide.sources)}`,
276 `lastVerifiedAt: ${yamlString(guide.lastVerifiedAt ?? '')}`,
277 `reviewAfter: ${yamlString(guide.reviewAfter ?? '')}`,
278 `freshness: ${yamlString(guide.freshness)}`,
279 `topics: ${yamlArray(guide.topics)}`,
280 ...(toolkitSlugs.length > 0 ? [`toolkitSlugs: ${yamlArray(toolkitSlugs)}`] : []),
281 `aliases: ${yamlArray(guide.aliases)}`,
282 ...relatedFrontmatter(related),
283 '---',
284 '',
285 ];
286 const body = rewriteSourceRepositoryLinks(guide.body.trim(), guide, guides);
287 return `${frontmatter.join('\n')}${markdownForMdx(body)}\n`;
288 }
289
290 function rootIndex(): string {
291 return `---
292 title: "Knowledge Base"
293 description: "Verified troubleshooting guides and answers for building with Composio."
294 ---
295
296 Verified troubleshooting guides, operational answers, and known-good patterns from Composio support.
297 `;
298 }
299
300 function buildExpectedFiles(catalog: KbCatalog): Map<string, string> {
301 const guides = getPublishedKbGuides(catalog);
302 const files = new Map<string, string>();
303 files.set('index.mdx', rootIndex());
304 files.set(
305 'meta.json',
306 `${JSON.stringify({ title: 'Knowledge Base', root: true, pages: ['index', 'guide'] }, null, 2)}\n`
307 );
308 files.set(
309 'guide/meta.json',
310 `${JSON.stringify({ title: 'Guides', pages: guides.map(guide => guide.slug) }, null, 2)}\n`
311 );
312 for (const guide of guides) {
313 files.set(`guide/${guide.slug}.mdx`, guideMdx(guide, guides));
314 }
315 return files;
316 }
317
318 function listRelativeFiles(directory: string): string[] {
319 if (!existsSync(directory)) return [];
320 return readdirSync(directory, { recursive: true, withFileTypes: true })
321 .filter(entry => entry.isFile())
322 .map(entry => relative(directory, join(entry.parentPath, entry.name)))
323 .sort();
324 }
325
326 function planFileChanges(outputDir: string, expected: Map<string, string>) {
327 const actualFiles = listRelativeFiles(outputDir);
328 const actualFileSet = new Set(actualFiles);
329 const writes = [...expected].filter(
330 ([path, content]) =>
331 !actualFileSet.has(path) || readFileSync(join(outputDir, path), 'utf8') !== content,
332 );
333 const removals = actualFiles.filter(path => !expected.has(path));
334 return { writes, removals };
335 }
336
337 function assertSafeOutputDirectory(outputDir: string): void {
338 const resolved = resolve(outputDir);
339 if (resolved === parse(resolved).root || basename(resolved) === '') {
340 throw new Error(`Refusing to generate KB content into unsafe path: ${resolved}`);
341 }
342 }
343
344 export function generateKbContent(options: GenerateKbContentOptions = {}): KbGenerationSummary {
345 const outputDir = resolve(options.outputDir ?? join(process.cwd(), 'content', 'kb'));
346 assertSafeOutputDirectory(outputDir);
347 const catalog = options.catalog ?? getKbCatalog();
348 const expected = buildExpectedFiles(catalog);
349 const published = getPublishedKbGuides(catalog).length;
350 const held = catalog.guides.filter(guide => guide.state === 'needs-review').length;
351 const changes = planFileChanges(outputDir, expected);
352
353 if (options.check) {
354 if (changes.writes.length > 0 || changes.removals.length > 0) {
355 throw new Error('Generated KB content is out of date; run bun run generate:kb');
356 }
357 return { published, held, files: expected.size };
358 }
359
360 for (const [path, content] of changes.writes) {
361 const target = join(outputDir, path);
362 mkdirSync(dirname(target), { recursive: true });
363 writeFileSync(target, content, 'utf8');
364 }
365 for (const path of changes.removals) {
366 rmSync(join(outputDir, path), { force: true });
367 }
368 return { published, held, files: expected.size };
369 }