Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 459 lines 15.9 KB
Raw
1 /**
2 * Generates markdown index pages for each OpenAPI tag.
3 * Reads both v3.1 and v3.0 specs and generates a table that
4 * uses the ApiEndpointsTable component to switch versions dynamically.
5 *
6 * Run: bun scripts/generate-api-index.ts
7 */
8
9 import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync, readdirSync } from 'fs';
10 import { join } from 'path';
11 import { z } from 'zod';
12 import { HIDDEN_API_TAGS } from '../lib/filter-api-version';
13 import { apiEndpointsSchema } from '../lib/api-endpoints-table-schema';
14
15 /**
16 * Serializes an endpoints array for the `<ApiEndpointsTable />` prop, refusing
17 * to write a payload the readers cannot parse.
18 *
19 * At runtime `mdxToCleanMarkdown` degrades a malformed payload to an empty
20 * table (one bad page must not 500 the whole `.md` response), so an invalid
21 * payload written here would render as a silently empty Endpoints section —
22 * exactly the defect this pipeline exists to avoid. A generator that cannot
23 * produce a valid payload should fail the generation run instead.
24 */
25 function serializeEndpoints(endpoints: unknown, label: string): string {
26 const parsed = apiEndpointsSchema.safeParse(endpoints);
27 if (!parsed.success) {
28 throw new Error(
29 `Invalid ApiEndpointsTable payload for ${label}:\n${z.prettifyError(parsed.error)}`
30 );
31 }
32 return JSON.stringify(endpoints);
33 }
34
35 /**
36 * API-reference tags hidden on our side even though the upstream OpenAPI spec
37 * (from hermes) includes them. Matched by slug. We neither generate their
38 * `index.mdx` overview pages nor leave stale ones behind. Shared with the
39 * reference page-tree filter so both stay in sync.
40 */
41 const HIDDEN_TAGS: ReadonlySet<string> = HIDDEN_API_TAGS;
42
43 /**
44 * Display-title overrides for API-reference tags whose upstream OpenAPI tag
45 * name is stale or off-brand. Keyed by tag slug.
46 */
47 const TITLE_OVERRIDES: Record<string, string> = {
48 'tool-router': 'Sessions (prev Tool Router)',
49 };
50
51 interface OpenAPIOperation {
52 summary?: string;
53 tags?: string[];
54 description?: string;
55 operationId?: string;
56 deprecated?: boolean;
57 'x-api-version'?: string;
58 }
59
60 interface OpenAPISpec {
61 tags: Array<{ name: string; description?: string }>;
62 paths: Record<string, Record<string, OpenAPIOperation>>;
63 }
64
65 interface OperationEntry {
66 summary: string;
67 method: string;
68 path: string;
69 operationId: string;
70 deprecated: boolean;
71 }
72
73 function slugify(text: string): string {
74 return text
75 .toLowerCase()
76 .replace(/[^a-z0-9]+/g, '-')
77 .replace(/^-|-$/g, '');
78 }
79
80 /**
81 * Optional hand-written overview for a tag, merged above the generated
82 * endpoints table. Lives in `api-overviews/<tagSlug>.mdx` (outside `content/`,
83 * so Fumadocs never renders it as its own page). Use this to fold a conceptual
84 * guide into the API reference page instead of keeping a separate docs page.
85 * Frontmatter, if present, is stripped — the generator owns the frontmatter.
86 */
87 function readOverview(tagSlug: string): string | null {
88 const overviewPath = join(process.cwd(), 'api-overviews', `${tagSlug}.mdx`);
89 if (!existsSync(overviewPath)) return null;
90 const raw = readFileSync(overviewPath, 'utf-8');
91 const stripped = raw.replace(/^---\n[\s\S]*?\n---\n/, '').trim();
92 return stripped.length > 0 ? stripped : null;
93 }
94
95 function getOperationsByTag(spec: OpenAPISpec): Record<string, OperationEntry[]> {
96 const tagOps: Record<string, OperationEntry[]> = {};
97
98 for (const tag of spec.tags) {
99 tagOps[tag.name] = [];
100 }
101
102 for (const [path, methods] of Object.entries(spec.paths)) {
103 for (const [method, operation] of Object.entries(methods)) {
104 if (operation.tags) {
105 for (const tag of operation.tags) {
106 if (!tagOps[tag]) tagOps[tag] = [];
107 tagOps[tag].push({
108 summary: operation.summary || `${method.toUpperCase()} ${path}`,
109 method: method.toUpperCase(),
110 path,
111 operationId: operation.operationId || slugify(operation.summary || path),
112 deprecated: operation.deprecated === true,
113 });
114 }
115 }
116 }
117 }
118
119 return tagOps;
120 }
121
122 function activeTagSlugs(opsByTag: Record<string, OperationEntry[]>): Set<string> {
123 const active = new Set<string>();
124 for (const [tagName, ops] of Object.entries(opsByTag)) {
125 const tagSlug = slugify(tagName);
126 if (ops.length > 0 && !HIDDEN_TAGS.has(tagSlug)) {
127 active.add(tagSlug);
128 }
129 }
130 return active;
131 }
132
133 interface WebhookSpec {
134 tags?: { name: string; description?: string }[];
135 webhooks?: Record<string, Record<string, OpenAPIOperation>>;
136 }
137
138 function loadWebhookSpec(): WebhookSpec | null {
139 const specPath = join(process.cwd(), 'public/openapi-webhooks.json');
140 if (!existsSync(specPath)) return null;
141 return JSON.parse(readFileSync(specPath, 'utf-8'));
142 }
143
144 /**
145 * Tag slugs contributed by the separate webhook-events spec (PLEN-2793). Its
146 * operations live under the OpenAPI 3.1 `webhooks` block (not `paths`) and its
147 * tags (e.g. "Webhook Events") are absent from openapi.json — so without folding
148 * these into the active set, `removeStaleTagIndexes` would recursively delete
149 * their generated overview folders on the next docs data run.
150 */
151 function webhookTagSlugs(spec: WebhookSpec | null): Set<string> {
152 const slugs = new Set<string>();
153 if (!spec) return slugs;
154
155 for (const item of Object.values(spec.webhooks ?? {})) {
156 for (const operation of Object.values(item)) {
157 for (const tag of operation.tags ?? []) {
158 slugs.add(slugify(tag));
159 }
160 }
161 }
162 return slugs;
163 }
164
165 type WebhookEntry = [eventKey: string, operation: OpenAPIOperation];
166
167 /**
168 * Generates one overview page per webhook tag. The webhooks spec can group its
169 * events under several tags (e.g. Webhook Events and Session Webhooks), and
170 * Fumadocs files each event page under its tag, so each tag gets its own index.
171 *
172 * The event list MUST be derived, not hand-maintained: adding an event to the
173 * Apollo registry would otherwise leave it off this page, and removing one would
174 * leave a link to a page Fumadocs no longer builds (a 404). Prose lives in
175 * `api-overviews/<tagSlug>.mdx`; everything below it is generated.
176 *
177 * Individual event pages are rendered by Fumadocs straight from the spec, so
178 * these pages only need the index tables.
179 */
180 function generateWebhookIndexes(spec: WebhookSpec | null, outputDir: string) {
181 if (!spec) return;
182
183 const entriesByTag = new Map<string, WebhookEntry[]>();
184 for (const [key, item] of Object.entries(spec.webhooks ?? {})) {
185 const operation = item.post;
186 if (!operation?.operationId) continue;
187
188 const tags = operation.tags ?? [];
189 if (tags.length === 0) {
190 throw new Error(`Webhook operation "${key}" has no tag`);
191 }
192 for (const tagName of tags) {
193 entriesByTag.set(tagName, [...(entriesByTag.get(tagName) ?? []), [key, operation]]);
194 }
195 }
196
197 for (const [tagName, entries] of entriesByTag) {
198 const tag = spec.tags?.find(candidate => candidate.name === tagName);
199 if (!tag) {
200 throw new Error(`Webhook operation tag "${tagName}" is not declared in spec.tags`);
201 }
202 generateWebhookTagIndex(tag, entries, outputDir);
203 }
204 }
205
206 function generateWebhookTagIndex(
207 tag: { name: string; description?: string },
208 entries: WebhookEntry[],
209 outputDir: string,
210 ) {
211 const tagSlug = slugify(tag.name);
212 const current: string[] = [];
213 const legacy: string[] = [];
214
215 for (const [key, operation] of entries) {
216 const href = `/reference/api-reference/${tagSlug}/${operation.operationId}`;
217 const label = operation.summary ?? key;
218
219 if (operation.deprecated !== true) {
220 current.push(`| \`${key}\` | [${label}](${href}) |`);
221 continue;
222 }
223
224 // Legacy payloads are keyed `<event>.<version>` so each format gets its own
225 // page — but `composio.trigger.message.v2` is NOT an event type anyone ever
226 // receives. Split the synthetic key back apart so the table shows the real
227 // event plus the payload version it applies to.
228 const versioned = /^(.*)\.(v\d+)$/.exec(key);
229 const event = versioned ? versioned[1] : key;
230 const version = versioned ? versioned[2].toUpperCase() : '—';
231 legacy.push(`| \`${event}\` | ${version} | [${label}](${href}) |`);
232 }
233
234 const overview = readOverview(tagSlug);
235 const body = overview ?? tag.description ?? '';
236
237 const legacySection =
238 legacy.length > 0
239 ? `
240
241 ## Legacy payloads (deprecated)
242
243 Older subscriptions may still receive these payload formats. The event type is unchanged — only the payload shape differs, selected by the subscription's version. You can upgrade an existing subscription at any time by updating its \`version\` — see [Update a webhook subscription](/reference/api-reference/webhook-subscriptions/patchWebhookSubscriptionsById). New integrations should use the current events above.
244
245 | Event | Version | Description |
246 |-------|---------|-------------|
247 ${legacy.join('\n')}`
248 : '';
249
250 const content = `---
251 title: ${tag.name}
252 description: "${tag.description ?? ''}"
253 ---
254
255 {/* Auto-generated from openapi-webhooks.json. Edit the overview at api-overviews/${tagSlug}.mdx, not this file. */}
256
257 ${body}
258
259 ## Events
260
261 | Event | Description |
262 |-------|-------------|
263 ${current.join('\n')}${legacySection}
264 `;
265
266 const folderPath = join(outputDir, tagSlug);
267 mkdirSync(folderPath, { recursive: true });
268 writeFileSync(join(folderPath, 'index.mdx'), content);
269 console.log(`Generated: ${tagSlug}/index.mdx (${current.length} events, ${legacy.length} legacy)`);
270 }
271
272 function removeStaleTagIndexes(baseDir: string, activeSlugs: Set<string>) {
273 if (!existsSync(baseDir)) return;
274
275 for (const entry of readdirSync(baseDir, { withFileTypes: true })) {
276 if (!entry.isDirectory()) continue;
277
278 const tagSlug = entry.name;
279 if (activeSlugs.has(tagSlug)) continue;
280
281 const tagDir = join(baseDir, tagSlug);
282 const indexPath = join(tagDir, 'index.mdx');
283 if (existsSync(indexPath)) {
284 rmSync(tagDir, { recursive: true, force: true });
285 console.log(`Removed stale tag: ${tagDir}`);
286 }
287 }
288 }
289
290 function generateIndexPages() {
291 const specV31Path = join(process.cwd(), 'public/openapi.json');
292 const specV3Path = join(process.cwd(), 'public/openapi-v3.json');
293
294 const specV31: OpenAPISpec = JSON.parse(readFileSync(specV31Path, 'utf-8'));
295 const v31Ops = getOperationsByTag(specV31);
296
297 let v3Ops: Record<string, OperationEntry[]> = {};
298 if (existsSync(specV3Path)) {
299 const loadedSpecV3: OpenAPISpec = JSON.parse(readFileSync(specV3Path, 'utf-8'));
300 v3Ops = getOperationsByTag(loadedSpecV3);
301 }
302
303 const v31TagDescriptions: Record<string, string> = {};
304 for (const tag of specV31.tags) {
305 v31TagDescriptions[tag.name] = tag.description || '';
306 }
307 const outputDir = join(process.cwd(), 'content/reference/api-reference');
308 const webhookSpec = loadWebhookSpec();
309
310 // Webhook tags (e.g. `Webhook Events`) are used only in openapi-webhooks.json,
311 // so they never appear in the openapi.json-derived active set. Without folding
312 // them in, their sections are recursively deleted on every run (and immediately
313 // regenerated below) — and if that ordering ever changed, the docs would
314 // silently lose those sections.
315 removeStaleTagIndexes(
316 outputDir,
317 new Set([...activeTagSlugs(v31Ops), ...webhookTagSlugs(webhookSpec)]),
318 );
319 removeStaleTagIndexes(
320 join(process.cwd(), 'content/reference/v3/api-reference'),
321 activeTagSlugs(v3Ops),
322 );
323
324 // Webhook events come from a separate 3.1 spec, so they're generated here
325 // rather than in the tag loop below (which iterates openapi.json operations).
326 generateWebhookIndexes(webhookSpec, outputDir);
327
328 // Get all unique tag names
329 const allTags = new Set([...Object.keys(v31Ops), ...Object.keys(v3Ops)]);
330
331 for (const tagName of allTags) {
332 const ops31 = v31Ops[tagName] || [];
333 const ops3 = v3Ops[tagName] || [];
334 const tagSlug = slugify(tagName);
335
336 // Intentionally-hidden tag — skip generation and delete any existing index.mdx
337 // (v3.1 and v3.0) so neither overview page lingers in the sidebar.
338 if (HIDDEN_TAGS.has(tagSlug)) {
339 for (const baseDir of [
340 join(process.cwd(), 'content/reference/api-reference'),
341 join(process.cwd(), 'content/reference/v3/api-reference'),
342 ]) {
343 const hidden = join(baseDir, tagSlug, 'index.mdx');
344 if (existsSync(hidden)) {
345 rmSync(hidden);
346 console.log(`Removed hidden tag: ${hidden}`);
347 }
348 }
349 continue;
350 }
351
352 // Tag declared in spec.tags but no operations reference it — clean up any stale index.mdx from a prior run.
353 if (ops31.length === 0 && ops3.length === 0) {
354 for (const baseDir of [
355 join(process.cwd(), 'content/reference/api-reference'),
356 join(process.cwd(), 'content/reference/v3/api-reference'),
357 ]) {
358 const stale = join(baseDir, tagSlug, 'index.mdx');
359 if (existsSync(stale)) {
360 rmSync(stale);
361 console.log(`Removed stale: ${stale}`);
362 }
363 }
364 continue;
365 }
366
367 const tagDescription = v31TagDescriptions[tagName] || `${tagName} API endpoints`;
368 // Display-title overrides for tags whose OpenAPI name is stale (e.g. the
369 // tool router is now Sessions). Keyed by slug.
370 const displayTitle = TITLE_OVERRIDES[tagSlug] ?? tagName;
371 const overview = readOverview(tagSlug);
372 // Body above the endpoints table: hand-written overview when present,
373 // otherwise the thin OpenAPI tag description.
374 const body = overview ?? tagDescription;
375 const genComment = overview
376 ? `{/* Auto-generated from OpenAPI spec. Edit the overview at api-overviews/${tagSlug}.mdx, not this file. */}`
377 : '{/* Auto-generated from OpenAPI spec. Do not edit directly. */}';
378
379 // Only generate v3.1 index page if the tag has v3.1 operations
380 if (ops31.length > 0) {
381 const v3ByOpId: Record<string, OperationEntry> = {};
382 for (const op of ops3) {
383 v3ByOpId[op.operationId] = op;
384 }
385
386 const endpoints = ops31.map(op => {
387 const v3Op = v3ByOpId[op.operationId];
388 return {
389 method: op.method,
390 pathV31: op.path,
391 pathV3: v3Op ? v3Op.path : op.path.replace('/v3.1/', '/v3/'),
392 summary: op.summary,
393 href: `/reference/api-reference/${tagSlug}/${op.operationId}`,
394 // OpenAPI marks these `deprecated`; we surface them with the existing
395 // "Legacy" tag rather than a separate one. Only emitted when true so
396 // non-legacy pages stay byte-identical.
397 ...(op.deprecated ? { legacy: true } : {}),
398 };
399 });
400
401 const content = `---
402 title: ${displayTitle}
403 description: "${tagDescription}"
404 ---
405
406 ${genComment}
407
408 ${body}
409
410 ## Endpoints
411
412 <ApiEndpointsTable endpoints={${serializeEndpoints(endpoints, `${tagSlug} (v3.1)`)}} />
413 `;
414
415 const folderPath = join(outputDir, tagSlug);
416 mkdirSync(folderPath, { recursive: true });
417 writeFileSync(join(folderPath, 'index.mdx'), content);
418 console.log(`Generated: ${tagSlug}/index.mdx`);
419 }
420
421 // Also generate v3 index page with v3-specific hrefs
422 if (ops3.length > 0) {
423 const v3Endpoints = ops3.map(op => ({
424 method: op.method,
425 pathV31: op.path.replace('/v3/', '/v3.1/'),
426 pathV3: op.path,
427 summary: op.summary,
428 href: `/reference/v3/api-reference/${tagSlug}/${op.operationId}`,
429 // OpenAPI marks these `deprecated`; we surface them with the existing
430 // "Legacy" tag rather than a separate one. Only emitted when true so
431 // non-legacy pages stay byte-identical.
432 ...(op.deprecated ? { legacy: true } : {}),
433 }));
434
435 const v3Content = `---
436 title: ${displayTitle}
437 description: "${tagDescription}"
438 ---
439
440 ${genComment}
441
442 ${body}
443
444 ## Endpoints
445
446 <ApiEndpointsTable endpoints={${serializeEndpoints(v3Endpoints, `${tagSlug} (v3.0)`)}} />
447 `;
448
449 const v3FolderPath = join(process.cwd(), 'content/reference/v3/api-reference', tagSlug);
450 mkdirSync(v3FolderPath, { recursive: true });
451 writeFileSync(join(v3FolderPath, 'index.mdx'), v3Content);
452 console.log(`Generated: v3/api-reference/${tagSlug}/index.mdx`);
453 }
454 }
455
456 console.log('Done generating API index pages');
457 }
458
459 generateIndexPages();