Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | /** |
| 2 | * Fetches and filters the OpenAPI specs for fumadocs. |
| 3 | * |
| 4 | * Heads-up: "3.1" is overloaded below. The first two are OpenAPI 3.0.0 |
| 5 | * DOCUMENTS that happen to describe different COMPOSIO API versions; the third |
| 6 | * is an OpenAPI 3.1.0 document (a format version, not an API version). |
| 7 | * |
| 8 | * Outputs three spec files: |
| 9 | * - public/openapi.json Composio API v3.1, as an OpenAPI 3.0.0 doc |
| 10 | * (latest; operationIds cleaned) |
| 11 | * - public/openapi-v3.json Composio API v3.0, as an OpenAPI 3.0.0 doc |
| 12 | * - public/openapi-webhooks.json webhook event payloads, as an OpenAPI 3.1.0 |
| 13 | * doc. The format bump is the whole reason |
| 14 | * this is a separate file: the top-level |
| 15 | * `webhooks` object does not exist in 3.0, so |
| 16 | * these cannot live in openapi.json. |
| 17 | * |
| 18 | * Run: bun run scripts/fetch-openapi.mjs |
| 19 | */ |
| 20 | |
| 21 | import { writeFileSync } from 'node:fs'; |
| 22 | import { dirname, join } from 'node:path'; |
| 23 | import { fileURLToPath } from 'node:url'; |
| 24 | import { z } from 'zod'; |
| 25 | import { fetchWithRetry } from './fetch-with-retry'; |
| 26 | import { declareOperationTags } from '../lib/openapi-tags'; |
| 27 | import { |
| 28 | PRODUCTION_BASE_URL, |
| 29 | PRODUCTION_API_V3_URL, |
| 30 | PRODUCTION_API_V31_URL, |
| 31 | } from './production-api.mjs'; |
| 32 | |
| 33 | const OPENAPI_V3_URL = process.env.OPENAPI_SPEC_URL || `${PRODUCTION_API_V3_URL}/openapi.json`; |
| 34 | const OPENAPI_V31_URL = |
| 35 | process.env.OPENAPI_V31_SPEC_URL || `${PRODUCTION_API_V31_URL}/openapi.json`; |
| 36 | const OPENAPI_WEBHOOKS_URL = |
| 37 | process.env.OPENAPI_WEBHOOKS_SPEC_URL || `${PRODUCTION_API_V31_URL}/openapi-webhooks.json`; |
| 38 | const SCRIPT_DIRECTORY = dirname(fileURLToPath(import.meta.url)); |
| 39 | const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace']; |
| 40 | |
| 41 | // Tags to ignore (internal/admin) |
| 42 | const IGNORED_TAGS = new Set(['CLI', 'Admin', 'Profiling', 'User', 'x-internal']); |
| 43 | |
| 44 | const NonEmptyStringSchema = z.string().trim().min(1); |
| 45 | const JsonObjectSchema = z.object({}).passthrough(); |
| 46 | const TagSchema = z |
| 47 | .object({ |
| 48 | name: NonEmptyStringSchema, |
| 49 | description: z.string().optional(), |
| 50 | }) |
| 51 | .passthrough(); |
| 52 | const OperationSchema = z |
| 53 | .object({ |
| 54 | tags: z.array(NonEmptyStringSchema).optional(), |
| 55 | operationId: NonEmptyStringSchema.optional(), |
| 56 | security: z.array(z.record(z.string(), z.unknown())).optional(), |
| 57 | 'x-internal': z.boolean().optional(), |
| 58 | }) |
| 59 | .passthrough(); |
| 60 | const PathItemSchema = z |
| 61 | .object({ |
| 62 | get: OperationSchema.optional(), |
| 63 | put: OperationSchema.optional(), |
| 64 | post: OperationSchema.optional(), |
| 65 | delete: OperationSchema.optional(), |
| 66 | options: OperationSchema.optional(), |
| 67 | head: OperationSchema.optional(), |
| 68 | patch: OperationSchema.optional(), |
| 69 | trace: OperationSchema.optional(), |
| 70 | }) |
| 71 | .passthrough(); |
| 72 | const OpenApiDocumentSchema = z |
| 73 | .object({ |
| 74 | openapi: NonEmptyStringSchema, |
| 75 | paths: z.record(z.string(), PathItemSchema), |
| 76 | tags: z.array(TagSchema).optional(), |
| 77 | components: z |
| 78 | .object({ |
| 79 | securitySchemes: z.record(z.string(), z.unknown()).optional(), |
| 80 | }) |
| 81 | .passthrough() |
| 82 | .optional(), |
| 83 | }) |
| 84 | .passthrough(); |
| 85 | |
| 86 | const JsonRequestBodySchema = z |
| 87 | .object({ |
| 88 | content: z |
| 89 | .object({ |
| 90 | 'application/json': z |
| 91 | .object({ |
| 92 | schema: JsonObjectSchema, |
| 93 | }) |
| 94 | .passthrough(), |
| 95 | }) |
| 96 | .passthrough(), |
| 97 | }) |
| 98 | .passthrough(); |
| 99 | const WebhookOperationSchema = OperationSchema.extend({ |
| 100 | operationId: NonEmptyStringSchema, |
| 101 | tags: z.array(NonEmptyStringSchema).min(1), |
| 102 | requestBody: JsonRequestBodySchema, |
| 103 | }); |
| 104 | const WebhookItemSchema = z |
| 105 | .object({ |
| 106 | post: WebhookOperationSchema, |
| 107 | }) |
| 108 | .passthrough(); |
| 109 | const WebhookDocumentSchema = z |
| 110 | .object({ |
| 111 | openapi: z.string().refine(version => version.startsWith('3.1'), { |
| 112 | error: 'Expected an OpenAPI 3.1 document', |
| 113 | }), |
| 114 | tags: z.array(TagSchema).min(1), |
| 115 | webhooks: z |
| 116 | .record(NonEmptyStringSchema, WebhookItemSchema) |
| 117 | .refine(webhooks => Object.keys(webhooks).length > 0, { |
| 118 | error: 'Expected at least one webhook event', |
| 119 | }), |
| 120 | }) |
| 121 | .passthrough() |
| 122 | .superRefine((document, context) => { |
| 123 | const declaredTags = new Set(document.tags.map(tag => tag.name)); |
| 124 | const operationIds = new Set(); |
| 125 | |
| 126 | for (const [eventName, item] of Object.entries(document.webhooks)) { |
| 127 | for (const tag of item.post.tags) { |
| 128 | if (!declaredTags.has(tag)) { |
| 129 | context.addIssue({ |
| 130 | code: 'custom', |
| 131 | path: ['webhooks', eventName, 'post', 'tags'], |
| 132 | message: `Tag "${tag}" is not declared in document.tags`, |
| 133 | }); |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | if (operationIds.has(item.post.operationId)) { |
| 138 | context.addIssue({ |
| 139 | code: 'custom', |
| 140 | path: ['webhooks', eventName, 'post', 'operationId'], |
| 141 | message: `Duplicate operationId "${item.post.operationId}"`, |
| 142 | }); |
| 143 | } |
| 144 | operationIds.add(item.post.operationId); |
| 145 | } |
| 146 | }); |
| 147 | |
| 148 | async function fetchJson(url) { |
| 149 | console.log(`Fetching OpenAPI spec from ${url}...`); |
| 150 | const response = await fetchWithRetry(url); |
| 151 | if (!response.ok) { |
| 152 | throw new Error(`Failed to fetch ${url}: ${response.status}`); |
| 153 | } |
| 154 | return response.json(); |
| 155 | } |
| 156 | |
| 157 | function formatZodError(error) { |
| 158 | return error.issues |
| 159 | .map(issue => `${issue.path.join('.') || 'document'}: ${issue.message}`) |
| 160 | .join('; '); |
| 161 | } |
| 162 | |
| 163 | function parseDocument(schema, payload, label) { |
| 164 | const parsed = schema.safeParse(payload); |
| 165 | if (!parsed.success) { |
| 166 | throw new Error(`Invalid ${label}: ${formatZodError(parsed.error)}`); |
| 167 | } |
| 168 | |
| 169 | // Zod confirms the boundary; the clone preserves the upstream document's key |
| 170 | // order instead of serializing Zod's schema-key order into generated files. |
| 171 | return structuredClone(payload); |
| 172 | } |
| 173 | |
| 174 | function forEachOperation(paths, callback) { |
| 175 | for (const pathItem of Object.values(paths)) { |
| 176 | for (const method of HTTP_METHODS) { |
| 177 | const operation = pathItem[method]; |
| 178 | if (operation) callback(operation); |
| 179 | } |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | /** |
| 184 | * Filter paths: remove ignored/internal tags, keep the first public tag only. |
| 185 | */ |
| 186 | function filterPaths(paths) { |
| 187 | const filteredPaths = {}; |
| 188 | let removedCount = 0; |
| 189 | |
| 190 | for (const [path, pathItem] of Object.entries(paths)) { |
| 191 | const filteredPathItem = { ...pathItem }; |
| 192 | |
| 193 | for (const method of HTTP_METHODS) { |
| 194 | const operation = pathItem[method]; |
| 195 | if (!operation) continue; |
| 196 | |
| 197 | const tags = operation.tags ?? []; |
| 198 | const isInternal = operation['x-internal'] === true || tags.includes('x-internal'); |
| 199 | const publicTags = tags.filter(tag => !IGNORED_TAGS.has(tag)); |
| 200 | const hasOnlyIgnoredTags = tags.length > 0 && publicTags.length === 0; |
| 201 | |
| 202 | if (isInternal || hasOnlyIgnoredTags) { |
| 203 | delete filteredPathItem[method]; |
| 204 | removedCount++; |
| 205 | } else if (publicTags.length > 0) { |
| 206 | filteredPathItem[method] = { ...operation, tags: [publicTags[0]] }; |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | const hasOperation = HTTP_METHODS.some(method => filteredPathItem[method]); |
| 211 | if (hasOperation || filteredPathItem.$ref) { |
| 212 | filteredPaths[path] = filteredPathItem; |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | return { filteredPaths, removedCount }; |
| 217 | } |
| 218 | |
| 219 | /** |
| 220 | * Strip version prefixes from operationIds (e.g. getV3_1Tools → getTools). |
| 221 | */ |
| 222 | function cleanOperationIds(paths) { |
| 223 | forEachOperation(paths, operation => { |
| 224 | if (operation.operationId) { |
| 225 | operation.operationId = operation.operationId.replace(/V\d+_\d+/g, ''); |
| 226 | } |
| 227 | }); |
| 228 | } |
| 229 | |
| 230 | function mergePropertySchemas(existing, incoming) { |
| 231 | if (!existing) return structuredClone(incoming); |
| 232 | |
| 233 | const merged = structuredClone(existing); |
| 234 | if (existing.enum && incoming.enum) { |
| 235 | merged.enum = [...new Set([...existing.enum, ...incoming.enum])]; |
| 236 | } |
| 237 | if (existing.properties && incoming.properties) { |
| 238 | merged.properties = { ...existing.properties }; |
| 239 | for (const [key, value] of Object.entries(incoming.properties)) { |
| 240 | merged.properties[key] = mergePropertySchemas(merged.properties[key], value); |
| 241 | } |
| 242 | } |
| 243 | return merged; |
| 244 | } |
| 245 | |
| 246 | function visitObjects(value, visitor, parentKey = '') { |
| 247 | if (Array.isArray(value)) { |
| 248 | for (const item of value) visitObjects(item, visitor, parentKey); |
| 249 | return; |
| 250 | } |
| 251 | if (!value || typeof value !== 'object') return; |
| 252 | |
| 253 | visitor(value, parentKey); |
| 254 | for (const [key, child] of Object.entries(value)) { |
| 255 | visitObjects(child, visitor, key); |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | function normalizeLargeObjectUnions(spec) { |
| 260 | visitObjects(spec, schema => { |
| 261 | for (const unionKey of ['anyOf', 'oneOf']) { |
| 262 | const variants = schema[unionKey]; |
| 263 | if (!Array.isArray(variants) || variants.length <= 5) continue; |
| 264 | |
| 265 | const objectSchemas = variants.filter( |
| 266 | variant => variant?.type === 'object' && variant.properties |
| 267 | ); |
| 268 | const mostlyObjects = |
| 269 | objectSchemas.length > 5 && objectSchemas.length >= variants.length * 0.8; |
| 270 | if (!mostlyObjects) continue; |
| 271 | |
| 272 | const mergedProperties = {}; |
| 273 | for (const objectSchema of objectSchemas) { |
| 274 | for (const [name, property] of Object.entries(objectSchema.properties)) { |
| 275 | mergedProperties[name] = mergePropertySchemas(mergedProperties[name], property); |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | const universallyRequired = [ |
| 280 | ...new Set(objectSchemas.flatMap(objectSchema => objectSchema.required ?? [])), |
| 281 | ].filter(name => objectSchemas.every(objectSchema => objectSchema.required?.includes(name))); |
| 282 | |
| 283 | delete schema[unionKey]; |
| 284 | schema.type = 'object'; |
| 285 | schema.properties = mergedProperties; |
| 286 | schema.additionalProperties = true; |
| 287 | if (universallyRequired.length > 0) { |
| 288 | schema.required = universallyRequired; |
| 289 | } |
| 290 | } |
| 291 | }); |
| 292 | } |
| 293 | |
| 294 | function fixNullableWithoutType(spec) { |
| 295 | visitObjects(spec, (schema, parentKey) => { |
| 296 | const needsType = |
| 297 | schema.nullable === true && |
| 298 | !schema.type && |
| 299 | !schema.$ref && |
| 300 | !schema.oneOf && |
| 301 | !schema.anyOf && |
| 302 | !schema.allOf; |
| 303 | if (needsType) { |
| 304 | if (parentKey === 'additionalProperties') { |
| 305 | delete schema.nullable; |
| 306 | } else if (Array.isArray(schema.example)) { |
| 307 | schema.type = 'array'; |
| 308 | } else { |
| 309 | schema.type = 'object'; |
| 310 | } |
| 311 | } |
| 312 | }); |
| 313 | } |
| 314 | |
| 315 | function removeCookieAuthentication(spec) { |
| 316 | if (spec.components?.securitySchemes?.CookieAuth) { |
| 317 | delete spec.components.securitySchemes.CookieAuth; |
| 318 | } |
| 319 | |
| 320 | forEachOperation(spec.paths, operation => { |
| 321 | if (!operation.security) return; |
| 322 | |
| 323 | operation.security = operation.security.filter(requirement => !('CookieAuth' in requirement)); |
| 324 | if (operation.security.length === 0) delete operation.security; |
| 325 | }); |
| 326 | } |
| 327 | |
| 328 | /** |
| 329 | * Post-process a spec: pin production, hide internal API, and normalize schemas. |
| 330 | */ |
| 331 | function postProcessSpec(spec) { |
| 332 | // Pin the server to production. The published docs must always show the |
| 333 | // production base URL in their curl examples, regardless of which environment |
| 334 | // the source spec was fetched from (a staging fetch would otherwise bake a |
| 335 | // staging server URL into the committed reference). |
| 336 | spec.servers = [ |
| 337 | { |
| 338 | url: PRODUCTION_BASE_URL, |
| 339 | description: 'PRODUCTION API', |
| 340 | }, |
| 341 | ]; |
| 342 | if (spec.tags) { |
| 343 | spec.tags = spec.tags.filter(tag => !IGNORED_TAGS.has(tag.name)); |
| 344 | } |
| 345 | // fumadocs-openapi only generates pages for operations whose tags are |
| 346 | // declared top-level; the backend generator omits some (e.g. Projects). |
| 347 | const declaredUpstream = new Set((spec.tags ?? []).map(tag => tag.name)); |
| 348 | declareOperationTags(spec); |
| 349 | const addedTags = (spec.tags ?? []).filter(tag => !declaredUpstream.has(tag.name)); |
| 350 | if (addedTags.length > 0) { |
| 351 | console.warn( |
| 352 | `WARN: upstream spec uses tags missing from its top-level tags array: ${addedTags |
| 353 | .map(tag => tag.name) |
| 354 | .join(', ')}. Declared them automatically; the backend generator should emit them.` |
| 355 | ); |
| 356 | } |
| 357 | |
| 358 | removeCookieAuthentication(spec); |
| 359 | normalizeLargeObjectUnions(spec); |
| 360 | fixNullableWithoutType(spec); |
| 361 | } |
| 362 | |
| 363 | export function prepareApiSpec(payload, apiVersion) { |
| 364 | const spec = parseDocument( |
| 365 | OpenApiDocumentSchema, |
| 366 | payload, |
| 367 | `Composio API v${apiVersion} OpenAPI document` |
| 368 | ); |
| 369 | const { filteredPaths, removedCount } = filterPaths(spec.paths); |
| 370 | spec.paths = filteredPaths; |
| 371 | |
| 372 | cleanOperationIds(spec.paths); |
| 373 | forEachOperation(spec.paths, operation => { |
| 374 | operation['x-api-version'] = apiVersion; |
| 375 | }); |
| 376 | postProcessSpec(spec); |
| 377 | |
| 378 | return { spec, removedCount }; |
| 379 | } |
| 380 | |
| 381 | function writeJson(filename, value) { |
| 382 | const outputPath = join(SCRIPT_DIRECTORY, '../public', filename); |
| 383 | writeFileSync(outputPath, JSON.stringify(value, null, 2)); |
| 384 | console.log(`Written ${filename} to ${outputPath}`); |
| 385 | } |
| 386 | |
| 387 | async function fetchAndFilterSpecs() { |
| 388 | const [v3Payload, v31Payload] = await Promise.all([ |
| 389 | fetchJson(OPENAPI_V3_URL), |
| 390 | fetchJson(OPENAPI_V31_URL), |
| 391 | ]); |
| 392 | const v31 = prepareApiSpec(v31Payload, '3.1'); |
| 393 | const v3 = prepareApiSpec(v3Payload, '3.0'); |
| 394 | |
| 395 | console.log( |
| 396 | `v3.1: ${Object.keys(v31.spec.paths).length} paths (${v31.removedCount} operations removed)` |
| 397 | ); |
| 398 | console.log( |
| 399 | `v3.0: ${Object.keys(v3.spec.paths).length} paths (${v3.removedCount} operations removed)` |
| 400 | ); |
| 401 | |
| 402 | writeJson('openapi.json', v31.spec); |
| 403 | writeJson('openapi-v3.json', v3.spec); |
| 404 | } |
| 405 | |
| 406 | /** |
| 407 | * Fetch the standalone webhook-events spec and write it verbatim. |
| 408 | * |
| 409 | * It's a separate OpenAPI 3.1 document keyed on `webhooks` (not `paths`), so it |
| 410 | * skips the path filtering, server pinning, and union normalization above — none |
| 411 | * of which apply. Fetched live from production like openapi.json. |
| 412 | * |
| 413 | * A fetch or schema failure leaves the committed snapshot untouched. This keeps |
| 414 | * a transient or malformed production response from deleting generated pages. |
| 415 | */ |
| 416 | export function writeWebhookSnapshot(payload, outputPath, sourceUrl = OPENAPI_WEBHOOKS_URL) { |
| 417 | const parsed = WebhookDocumentSchema.safeParse(payload); |
| 418 | if (!parsed.success) { |
| 419 | console.warn( |
| 420 | `WARN: refusing to write webhooks spec from ${sourceUrl} — ${formatZodError(parsed.error)}. Keeping existing ${outputPath}.` |
| 421 | ); |
| 422 | return false; |
| 423 | } |
| 424 | |
| 425 | const eventCount = Object.keys(parsed.data.webhooks).length; |
| 426 | writeFileSync(outputPath, JSON.stringify(payload, null, 2)); |
| 427 | console.log(`Written webhooks spec to ${outputPath} (${eventCount} events)`); |
| 428 | return true; |
| 429 | } |
| 430 | |
| 431 | async function fetchAndWriteWebhookSpec() { |
| 432 | const outputPath = join(SCRIPT_DIRECTORY, '../public/openapi-webhooks.json'); |
| 433 | try { |
| 434 | const payload = await fetchJson(OPENAPI_WEBHOOKS_URL); |
| 435 | writeWebhookSnapshot(payload, outputPath); |
| 436 | } catch (err) { |
| 437 | const message = err instanceof Error ? err.message : String(err); |
| 438 | console.warn( |
| 439 | `WARN: could not fetch webhooks spec from ${OPENAPI_WEBHOOKS_URL}: ${message}. Keeping existing ${outputPath}.` |
| 440 | ); |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | if (import.meta.main) { |
| 445 | // Independent fetches with independent error handling; run them concurrently. |
| 446 | await Promise.all([fetchAndFilterSpecs().catch(console.error), fetchAndWriteWebhookSpec()]); |
| 447 | } |