Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | /** |
| 2 | * Generates markdown index pages for each OpenAPI tag. |
| 3 | * Reads both v3.1 and v3.0 specs and generates a table that |
| 4 | * uses the ApiEndpointsTable component to switch versions dynamically. |
| 5 | * |
| 6 | * Run: bun scripts/generate-api-index.ts |
| 7 | */ |
| 8 | |
| 9 | import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync, readdirSync } from 'fs'; |
| 10 | import { join } from 'path'; |
| 11 | import { z } from 'zod'; |
| 12 | import { HIDDEN_API_TAGS } from '../lib/filter-api-version'; |
| 13 | import { apiEndpointsSchema } from '../lib/api-endpoints-table-schema'; |
| 14 | |
| 15 | /** |
| 16 | * Serializes an endpoints array for the `<ApiEndpointsTable />` prop, refusing |
| 17 | * to write a payload the readers cannot parse. |
| 18 | * |
| 19 | * At runtime `mdxToCleanMarkdown` degrades a malformed payload to an empty |
| 20 | * table (one bad page must not 500 the whole `.md` response), so an invalid |
| 21 | * payload written here would render as a silently empty Endpoints section — |
| 22 | * exactly the defect this pipeline exists to avoid. A generator that cannot |
| 23 | * produce a valid payload should fail the generation run instead. |
| 24 | */ |
| 25 | function serializeEndpoints(endpoints: unknown, label: string): string { |
| 26 | const parsed = apiEndpointsSchema.safeParse(endpoints); |
| 27 | if (!parsed.success) { |
| 28 | throw new Error( |
| 29 | `Invalid ApiEndpointsTable payload for ${label}:\n${z.prettifyError(parsed.error)}` |
| 30 | ); |
| 31 | } |
| 32 | return JSON.stringify(endpoints); |
| 33 | } |
| 34 | |
| 35 | /** |
| 36 | * API-reference tags hidden on our side even though the upstream OpenAPI spec |
| 37 | * (from hermes) includes them. Matched by slug. We neither generate their |
| 38 | * `index.mdx` overview pages nor leave stale ones behind. Shared with the |
| 39 | * reference page-tree filter so both stay in sync. |
| 40 | */ |
| 41 | const HIDDEN_TAGS: ReadonlySet<string> = HIDDEN_API_TAGS; |
| 42 | |
| 43 | /** |
| 44 | * Display-title overrides for API-reference tags whose upstream OpenAPI tag |
| 45 | * name is stale or off-brand. Keyed by tag slug. |
| 46 | */ |
| 47 | const TITLE_OVERRIDES: Record<string, string> = { |
| 48 | 'tool-router': 'Sessions (prev Tool Router)', |
| 49 | }; |
| 50 | |
| 51 | interface OpenAPIOperation { |
| 52 | summary?: string; |
| 53 | tags?: string[]; |
| 54 | description?: string; |
| 55 | operationId?: string; |
| 56 | deprecated?: boolean; |
| 57 | 'x-api-version'?: string; |
| 58 | } |
| 59 | |
| 60 | interface OpenAPISpec { |
| 61 | tags: Array<{ name: string; description?: string }>; |
| 62 | paths: Record<string, Record<string, OpenAPIOperation>>; |
| 63 | } |
| 64 | |
| 65 | interface OperationEntry { |
| 66 | summary: string; |
| 67 | method: string; |
| 68 | path: string; |
| 69 | operationId: string; |
| 70 | deprecated: boolean; |
| 71 | } |
| 72 | |
| 73 | function slugify(text: string): string { |
| 74 | return text |
| 75 | .toLowerCase() |
| 76 | .replace(/[^a-z0-9]+/g, '-') |
| 77 | .replace(/^-|-$/g, ''); |
| 78 | } |
| 79 | |
| 80 | /** |
| 81 | * Optional hand-written overview for a tag, merged above the generated |
| 82 | * endpoints table. Lives in `api-overviews/<tagSlug>.mdx` (outside `content/`, |
| 83 | * so Fumadocs never renders it as its own page). Use this to fold a conceptual |
| 84 | * guide into the API reference page instead of keeping a separate docs page. |
| 85 | * Frontmatter, if present, is stripped — the generator owns the frontmatter. |
| 86 | */ |
| 87 | function readOverview(tagSlug: string): string | null { |
| 88 | const overviewPath = join(process.cwd(), 'api-overviews', `${tagSlug}.mdx`); |
| 89 | if (!existsSync(overviewPath)) return null; |
| 90 | const raw = readFileSync(overviewPath, 'utf-8'); |
| 91 | const stripped = raw.replace(/^---\n[\s\S]*?\n---\n/, '').trim(); |
| 92 | return stripped.length > 0 ? stripped : null; |
| 93 | } |
| 94 | |
| 95 | function getOperationsByTag(spec: OpenAPISpec): Record<string, OperationEntry[]> { |
| 96 | const tagOps: Record<string, OperationEntry[]> = {}; |
| 97 | |
| 98 | for (const tag of spec.tags) { |
| 99 | tagOps[tag.name] = []; |
| 100 | } |
| 101 | |
| 102 | for (const [path, methods] of Object.entries(spec.paths)) { |
| 103 | for (const [method, operation] of Object.entries(methods)) { |
| 104 | if (operation.tags) { |
| 105 | for (const tag of operation.tags) { |
| 106 | if (!tagOps[tag]) tagOps[tag] = []; |
| 107 | tagOps[tag].push({ |
| 108 | summary: operation.summary || `${method.toUpperCase()} ${path}`, |
| 109 | method: method.toUpperCase(), |
| 110 | path, |
| 111 | operationId: operation.operationId || slugify(operation.summary || path), |
| 112 | deprecated: operation.deprecated === true, |
| 113 | }); |
| 114 | } |
| 115 | } |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | return tagOps; |
| 120 | } |
| 121 | |
| 122 | function activeTagSlugs(opsByTag: Record<string, OperationEntry[]>): Set<string> { |
| 123 | const active = new Set<string>(); |
| 124 | for (const [tagName, ops] of Object.entries(opsByTag)) { |
| 125 | const tagSlug = slugify(tagName); |
| 126 | if (ops.length > 0 && !HIDDEN_TAGS.has(tagSlug)) { |
| 127 | active.add(tagSlug); |
| 128 | } |
| 129 | } |
| 130 | return active; |
| 131 | } |
| 132 | |
| 133 | interface WebhookSpec { |
| 134 | tags?: { name: string; description?: string }[]; |
| 135 | webhooks?: Record<string, Record<string, OpenAPIOperation>>; |
| 136 | } |
| 137 | |
| 138 | function loadWebhookSpec(): WebhookSpec | null { |
| 139 | const specPath = join(process.cwd(), 'public/openapi-webhooks.json'); |
| 140 | if (!existsSync(specPath)) return null; |
| 141 | return JSON.parse(readFileSync(specPath, 'utf-8')); |
| 142 | } |
| 143 | |
| 144 | /** |
| 145 | * Tag slugs contributed by the separate webhook-events spec (PLEN-2793). Its |
| 146 | * operations live under the OpenAPI 3.1 `webhooks` block (not `paths`) and its |
| 147 | * tags (e.g. "Webhook Events") are absent from openapi.json — so without folding |
| 148 | * these into the active set, `removeStaleTagIndexes` would recursively delete |
| 149 | * their generated overview folders on the next docs data run. |
| 150 | */ |
| 151 | function webhookTagSlugs(spec: WebhookSpec | null): Set<string> { |
| 152 | const slugs = new Set<string>(); |
| 153 | if (!spec) return slugs; |
| 154 | |
| 155 | for (const item of Object.values(spec.webhooks ?? {})) { |
| 156 | for (const operation of Object.values(item)) { |
| 157 | for (const tag of operation.tags ?? []) { |
| 158 | slugs.add(slugify(tag)); |
| 159 | } |
| 160 | } |
| 161 | } |
| 162 | return slugs; |
| 163 | } |
| 164 | |
| 165 | type WebhookEntry = [eventKey: string, operation: OpenAPIOperation]; |
| 166 | |
| 167 | /** |
| 168 | * Generates one overview page per webhook tag. The webhooks spec can group its |
| 169 | * events under several tags (e.g. Webhook Events and Session Webhooks), and |
| 170 | * Fumadocs files each event page under its tag, so each tag gets its own index. |
| 171 | * |
| 172 | * The event list MUST be derived, not hand-maintained: adding an event to the |
| 173 | * Apollo registry would otherwise leave it off this page, and removing one would |
| 174 | * leave a link to a page Fumadocs no longer builds (a 404). Prose lives in |
| 175 | * `api-overviews/<tagSlug>.mdx`; everything below it is generated. |
| 176 | * |
| 177 | * Individual event pages are rendered by Fumadocs straight from the spec, so |
| 178 | * these pages only need the index tables. |
| 179 | */ |
| 180 | function generateWebhookIndexes(spec: WebhookSpec | null, outputDir: string) { |
| 181 | if (!spec) return; |
| 182 | |
| 183 | const entriesByTag = new Map<string, WebhookEntry[]>(); |
| 184 | for (const [key, item] of Object.entries(spec.webhooks ?? {})) { |
| 185 | const operation = item.post; |
| 186 | if (!operation?.operationId) continue; |
| 187 | |
| 188 | const tags = operation.tags ?? []; |
| 189 | if (tags.length === 0) { |
| 190 | throw new Error(`Webhook operation "${key}" has no tag`); |
| 191 | } |
| 192 | for (const tagName of tags) { |
| 193 | entriesByTag.set(tagName, [...(entriesByTag.get(tagName) ?? []), [key, operation]]); |
| 194 | } |
| 195 | } |
| 196 | |
| 197 | for (const [tagName, entries] of entriesByTag) { |
| 198 | const tag = spec.tags?.find(candidate => candidate.name === tagName); |
| 199 | if (!tag) { |
| 200 | throw new Error(`Webhook operation tag "${tagName}" is not declared in spec.tags`); |
| 201 | } |
| 202 | generateWebhookTagIndex(tag, entries, outputDir); |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | function generateWebhookTagIndex( |
| 207 | tag: { name: string; description?: string }, |
| 208 | entries: WebhookEntry[], |
| 209 | outputDir: string, |
| 210 | ) { |
| 211 | const tagSlug = slugify(tag.name); |
| 212 | const current: string[] = []; |
| 213 | const legacy: string[] = []; |
| 214 | |
| 215 | for (const [key, operation] of entries) { |
| 216 | const href = `/reference/api-reference/${tagSlug}/${operation.operationId}`; |
| 217 | const label = operation.summary ?? key; |
| 218 | |
| 219 | if (operation.deprecated !== true) { |
| 220 | current.push(`| \`${key}\` | [${label}](${href}) |`); |
| 221 | continue; |
| 222 | } |
| 223 | |
| 224 | // Legacy payloads are keyed `<event>.<version>` so each format gets its own |
| 225 | // page — but `composio.trigger.message.v2` is NOT an event type anyone ever |
| 226 | // receives. Split the synthetic key back apart so the table shows the real |
| 227 | // event plus the payload version it applies to. |
| 228 | const versioned = /^(.*)\.(v\d+)$/.exec(key); |
| 229 | const event = versioned ? versioned[1] : key; |
| 230 | const version = versioned ? versioned[2].toUpperCase() : '—'; |
| 231 | legacy.push(`| \`${event}\` | ${version} | [${label}](${href}) |`); |
| 232 | } |
| 233 | |
| 234 | const overview = readOverview(tagSlug); |
| 235 | const body = overview ?? tag.description ?? ''; |
| 236 | |
| 237 | const legacySection = |
| 238 | legacy.length > 0 |
| 239 | ? ` |
| 240 | |
| 241 | ## Legacy payloads (deprecated) |
| 242 | |
| 243 | Older subscriptions may still receive these payload formats. The event type is unchanged — only the payload shape differs, selected by the subscription's version. You can upgrade an existing subscription at any time by updating its \`version\` — see [Update a webhook subscription](/reference/api-reference/webhook-subscriptions/patchWebhookSubscriptionsById). New integrations should use the current events above. |
| 244 | |
| 245 | | Event | Version | Description | |
| 246 | |-------|---------|-------------| |
| 247 | ${legacy.join('\n')}` |
| 248 | : ''; |
| 249 | |
| 250 | const content = `--- |
| 251 | title: ${tag.name} |
| 252 | description: "${tag.description ?? ''}" |
| 253 | --- |
| 254 | |
| 255 | {/* Auto-generated from openapi-webhooks.json. Edit the overview at api-overviews/${tagSlug}.mdx, not this file. */} |
| 256 | |
| 257 | ${body} |
| 258 | |
| 259 | ## Events |
| 260 | |
| 261 | | Event | Description | |
| 262 | |-------|-------------| |
| 263 | ${current.join('\n')}${legacySection} |
| 264 | `; |
| 265 | |
| 266 | const folderPath = join(outputDir, tagSlug); |
| 267 | mkdirSync(folderPath, { recursive: true }); |
| 268 | writeFileSync(join(folderPath, 'index.mdx'), content); |
| 269 | console.log(`Generated: ${tagSlug}/index.mdx (${current.length} events, ${legacy.length} legacy)`); |
| 270 | } |
| 271 | |
| 272 | function removeStaleTagIndexes(baseDir: string, activeSlugs: Set<string>) { |
| 273 | if (!existsSync(baseDir)) return; |
| 274 | |
| 275 | for (const entry of readdirSync(baseDir, { withFileTypes: true })) { |
| 276 | if (!entry.isDirectory()) continue; |
| 277 | |
| 278 | const tagSlug = entry.name; |
| 279 | if (activeSlugs.has(tagSlug)) continue; |
| 280 | |
| 281 | const tagDir = join(baseDir, tagSlug); |
| 282 | const indexPath = join(tagDir, 'index.mdx'); |
| 283 | if (existsSync(indexPath)) { |
| 284 | rmSync(tagDir, { recursive: true, force: true }); |
| 285 | console.log(`Removed stale tag: ${tagDir}`); |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | function generateIndexPages() { |
| 291 | const specV31Path = join(process.cwd(), 'public/openapi.json'); |
| 292 | const specV3Path = join(process.cwd(), 'public/openapi-v3.json'); |
| 293 | |
| 294 | const specV31: OpenAPISpec = JSON.parse(readFileSync(specV31Path, 'utf-8')); |
| 295 | const v31Ops = getOperationsByTag(specV31); |
| 296 | |
| 297 | let v3Ops: Record<string, OperationEntry[]> = {}; |
| 298 | if (existsSync(specV3Path)) { |
| 299 | const loadedSpecV3: OpenAPISpec = JSON.parse(readFileSync(specV3Path, 'utf-8')); |
| 300 | v3Ops = getOperationsByTag(loadedSpecV3); |
| 301 | } |
| 302 | |
| 303 | const v31TagDescriptions: Record<string, string> = {}; |
| 304 | for (const tag of specV31.tags) { |
| 305 | v31TagDescriptions[tag.name] = tag.description || ''; |
| 306 | } |
| 307 | const outputDir = join(process.cwd(), 'content/reference/api-reference'); |
| 308 | const webhookSpec = loadWebhookSpec(); |
| 309 | |
| 310 | // Webhook tags (e.g. `Webhook Events`) are used only in openapi-webhooks.json, |
| 311 | // so they never appear in the openapi.json-derived active set. Without folding |
| 312 | // them in, their sections are recursively deleted on every run (and immediately |
| 313 | // regenerated below) — and if that ordering ever changed, the docs would |
| 314 | // silently lose those sections. |
| 315 | removeStaleTagIndexes( |
| 316 | outputDir, |
| 317 | new Set([...activeTagSlugs(v31Ops), ...webhookTagSlugs(webhookSpec)]), |
| 318 | ); |
| 319 | removeStaleTagIndexes( |
| 320 | join(process.cwd(), 'content/reference/v3/api-reference'), |
| 321 | activeTagSlugs(v3Ops), |
| 322 | ); |
| 323 | |
| 324 | // Webhook events come from a separate 3.1 spec, so they're generated here |
| 325 | // rather than in the tag loop below (which iterates openapi.json operations). |
| 326 | generateWebhookIndexes(webhookSpec, outputDir); |
| 327 | |
| 328 | // Get all unique tag names |
| 329 | const allTags = new Set([...Object.keys(v31Ops), ...Object.keys(v3Ops)]); |
| 330 | |
| 331 | for (const tagName of allTags) { |
| 332 | const ops31 = v31Ops[tagName] || []; |
| 333 | const ops3 = v3Ops[tagName] || []; |
| 334 | const tagSlug = slugify(tagName); |
| 335 | |
| 336 | // Intentionally-hidden tag — skip generation and delete any existing index.mdx |
| 337 | // (v3.1 and v3.0) so neither overview page lingers in the sidebar. |
| 338 | if (HIDDEN_TAGS.has(tagSlug)) { |
| 339 | for (const baseDir of [ |
| 340 | join(process.cwd(), 'content/reference/api-reference'), |
| 341 | join(process.cwd(), 'content/reference/v3/api-reference'), |
| 342 | ]) { |
| 343 | const hidden = join(baseDir, tagSlug, 'index.mdx'); |
| 344 | if (existsSync(hidden)) { |
| 345 | rmSync(hidden); |
| 346 | console.log(`Removed hidden tag: ${hidden}`); |
| 347 | } |
| 348 | } |
| 349 | continue; |
| 350 | } |
| 351 | |
| 352 | // Tag declared in spec.tags but no operations reference it — clean up any stale index.mdx from a prior run. |
| 353 | if (ops31.length === 0 && ops3.length === 0) { |
| 354 | for (const baseDir of [ |
| 355 | join(process.cwd(), 'content/reference/api-reference'), |
| 356 | join(process.cwd(), 'content/reference/v3/api-reference'), |
| 357 | ]) { |
| 358 | const stale = join(baseDir, tagSlug, 'index.mdx'); |
| 359 | if (existsSync(stale)) { |
| 360 | rmSync(stale); |
| 361 | console.log(`Removed stale: ${stale}`); |
| 362 | } |
| 363 | } |
| 364 | continue; |
| 365 | } |
| 366 | |
| 367 | const tagDescription = v31TagDescriptions[tagName] || `${tagName} API endpoints`; |
| 368 | // Display-title overrides for tags whose OpenAPI name is stale (e.g. the |
| 369 | // tool router is now Sessions). Keyed by slug. |
| 370 | const displayTitle = TITLE_OVERRIDES[tagSlug] ?? tagName; |
| 371 | const overview = readOverview(tagSlug); |
| 372 | // Body above the endpoints table: hand-written overview when present, |
| 373 | // otherwise the thin OpenAPI tag description. |
| 374 | const body = overview ?? tagDescription; |
| 375 | const genComment = overview |
| 376 | ? `{/* Auto-generated from OpenAPI spec. Edit the overview at api-overviews/${tagSlug}.mdx, not this file. */}` |
| 377 | : '{/* Auto-generated from OpenAPI spec. Do not edit directly. */}'; |
| 378 | |
| 379 | // Only generate v3.1 index page if the tag has v3.1 operations |
| 380 | if (ops31.length > 0) { |
| 381 | const v3ByOpId: Record<string, OperationEntry> = {}; |
| 382 | for (const op of ops3) { |
| 383 | v3ByOpId[op.operationId] = op; |
| 384 | } |
| 385 | |
| 386 | const endpoints = ops31.map(op => { |
| 387 | const v3Op = v3ByOpId[op.operationId]; |
| 388 | return { |
| 389 | method: op.method, |
| 390 | pathV31: op.path, |
| 391 | pathV3: v3Op ? v3Op.path : op.path.replace('/v3.1/', '/v3/'), |
| 392 | summary: op.summary, |
| 393 | href: `/reference/api-reference/${tagSlug}/${op.operationId}`, |
| 394 | // OpenAPI marks these `deprecated`; we surface them with the existing |
| 395 | // "Legacy" tag rather than a separate one. Only emitted when true so |
| 396 | // non-legacy pages stay byte-identical. |
| 397 | ...(op.deprecated ? { legacy: true } : {}), |
| 398 | }; |
| 399 | }); |
| 400 | |
| 401 | const content = `--- |
| 402 | title: ${displayTitle} |
| 403 | description: "${tagDescription}" |
| 404 | --- |
| 405 | |
| 406 | ${genComment} |
| 407 | |
| 408 | ${body} |
| 409 | |
| 410 | ## Endpoints |
| 411 | |
| 412 | <ApiEndpointsTable endpoints={${serializeEndpoints(endpoints, `${tagSlug} (v3.1)`)}} /> |
| 413 | `; |
| 414 | |
| 415 | const folderPath = join(outputDir, tagSlug); |
| 416 | mkdirSync(folderPath, { recursive: true }); |
| 417 | writeFileSync(join(folderPath, 'index.mdx'), content); |
| 418 | console.log(`Generated: ${tagSlug}/index.mdx`); |
| 419 | } |
| 420 | |
| 421 | // Also generate v3 index page with v3-specific hrefs |
| 422 | if (ops3.length > 0) { |
| 423 | const v3Endpoints = ops3.map(op => ({ |
| 424 | method: op.method, |
| 425 | pathV31: op.path.replace('/v3/', '/v3.1/'), |
| 426 | pathV3: op.path, |
| 427 | summary: op.summary, |
| 428 | href: `/reference/v3/api-reference/${tagSlug}/${op.operationId}`, |
| 429 | // OpenAPI marks these `deprecated`; we surface them with the existing |
| 430 | // "Legacy" tag rather than a separate one. Only emitted when true so |
| 431 | // non-legacy pages stay byte-identical. |
| 432 | ...(op.deprecated ? { legacy: true } : {}), |
| 433 | })); |
| 434 | |
| 435 | const v3Content = `--- |
| 436 | title: ${displayTitle} |
| 437 | description: "${tagDescription}" |
| 438 | --- |
| 439 | |
| 440 | ${genComment} |
| 441 | |
| 442 | ${body} |
| 443 | |
| 444 | ## Endpoints |
| 445 | |
| 446 | <ApiEndpointsTable endpoints={${serializeEndpoints(v3Endpoints, `${tagSlug} (v3.0)`)}} /> |
| 447 | `; |
| 448 | |
| 449 | const v3FolderPath = join(process.cwd(), 'content/reference/v3/api-reference', tagSlug); |
| 450 | mkdirSync(v3FolderPath, { recursive: true }); |
| 451 | writeFileSync(join(v3FolderPath, 'index.mdx'), v3Content); |
| 452 | console.log(`Generated: v3/api-reference/${tagSlug}/index.mdx`); |
| 453 | } |
| 454 | } |
| 455 | |
| 456 | console.log('Done generating API index pages'); |
| 457 | } |
| 458 | |
| 459 | generateIndexPages(); |