Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
mjs 447 lines 14.5 KB
Raw
1 /**
2 * Fetches and filters the OpenAPI specs for fumadocs.
3 *
4 * Heads-up: "3.1" is overloaded below. The first two are OpenAPI 3.0.0
5 * DOCUMENTS that happen to describe different COMPOSIO API versions; the third
6 * is an OpenAPI 3.1.0 document (a format version, not an API version).
7 *
8 * Outputs three spec files:
9 * - public/openapi.json Composio API v3.1, as an OpenAPI 3.0.0 doc
10 * (latest; operationIds cleaned)
11 * - public/openapi-v3.json Composio API v3.0, as an OpenAPI 3.0.0 doc
12 * - public/openapi-webhooks.json webhook event payloads, as an OpenAPI 3.1.0
13 * doc. The format bump is the whole reason
14 * this is a separate file: the top-level
15 * `webhooks` object does not exist in 3.0, so
16 * these cannot live in openapi.json.
17 *
18 * Run: bun run scripts/fetch-openapi.mjs
19 */
20
21 import { writeFileSync } from 'node:fs';
22 import { dirname, join } from 'node:path';
23 import { fileURLToPath } from 'node:url';
24 import { z } from 'zod';
25 import { fetchWithRetry } from './fetch-with-retry';
26 import { declareOperationTags } from '../lib/openapi-tags';
27 import {
28 PRODUCTION_BASE_URL,
29 PRODUCTION_API_V3_URL,
30 PRODUCTION_API_V31_URL,
31 } from './production-api.mjs';
32
33 const OPENAPI_V3_URL = process.env.OPENAPI_SPEC_URL || `${PRODUCTION_API_V3_URL}/openapi.json`;
34 const OPENAPI_V31_URL =
35 process.env.OPENAPI_V31_SPEC_URL || `${PRODUCTION_API_V31_URL}/openapi.json`;
36 const OPENAPI_WEBHOOKS_URL =
37 process.env.OPENAPI_WEBHOOKS_SPEC_URL || `${PRODUCTION_API_V31_URL}/openapi-webhooks.json`;
38 const SCRIPT_DIRECTORY = dirname(fileURLToPath(import.meta.url));
39 const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
40
41 // Tags to ignore (internal/admin)
42 const IGNORED_TAGS = new Set(['CLI', 'Admin', 'Profiling', 'User', 'x-internal']);
43
44 const NonEmptyStringSchema = z.string().trim().min(1);
45 const JsonObjectSchema = z.object({}).passthrough();
46 const TagSchema = z
47 .object({
48 name: NonEmptyStringSchema,
49 description: z.string().optional(),
50 })
51 .passthrough();
52 const OperationSchema = z
53 .object({
54 tags: z.array(NonEmptyStringSchema).optional(),
55 operationId: NonEmptyStringSchema.optional(),
56 security: z.array(z.record(z.string(), z.unknown())).optional(),
57 'x-internal': z.boolean().optional(),
58 })
59 .passthrough();
60 const PathItemSchema = z
61 .object({
62 get: OperationSchema.optional(),
63 put: OperationSchema.optional(),
64 post: OperationSchema.optional(),
65 delete: OperationSchema.optional(),
66 options: OperationSchema.optional(),
67 head: OperationSchema.optional(),
68 patch: OperationSchema.optional(),
69 trace: OperationSchema.optional(),
70 })
71 .passthrough();
72 const OpenApiDocumentSchema = z
73 .object({
74 openapi: NonEmptyStringSchema,
75 paths: z.record(z.string(), PathItemSchema),
76 tags: z.array(TagSchema).optional(),
77 components: z
78 .object({
79 securitySchemes: z.record(z.string(), z.unknown()).optional(),
80 })
81 .passthrough()
82 .optional(),
83 })
84 .passthrough();
85
86 const JsonRequestBodySchema = z
87 .object({
88 content: z
89 .object({
90 'application/json': z
91 .object({
92 schema: JsonObjectSchema,
93 })
94 .passthrough(),
95 })
96 .passthrough(),
97 })
98 .passthrough();
99 const WebhookOperationSchema = OperationSchema.extend({
100 operationId: NonEmptyStringSchema,
101 tags: z.array(NonEmptyStringSchema).min(1),
102 requestBody: JsonRequestBodySchema,
103 });
104 const WebhookItemSchema = z
105 .object({
106 post: WebhookOperationSchema,
107 })
108 .passthrough();
109 const WebhookDocumentSchema = z
110 .object({
111 openapi: z.string().refine(version => version.startsWith('3.1'), {
112 error: 'Expected an OpenAPI 3.1 document',
113 }),
114 tags: z.array(TagSchema).min(1),
115 webhooks: z
116 .record(NonEmptyStringSchema, WebhookItemSchema)
117 .refine(webhooks => Object.keys(webhooks).length > 0, {
118 error: 'Expected at least one webhook event',
119 }),
120 })
121 .passthrough()
122 .superRefine((document, context) => {
123 const declaredTags = new Set(document.tags.map(tag => tag.name));
124 const operationIds = new Set();
125
126 for (const [eventName, item] of Object.entries(document.webhooks)) {
127 for (const tag of item.post.tags) {
128 if (!declaredTags.has(tag)) {
129 context.addIssue({
130 code: 'custom',
131 path: ['webhooks', eventName, 'post', 'tags'],
132 message: `Tag "${tag}" is not declared in document.tags`,
133 });
134 }
135 }
136
137 if (operationIds.has(item.post.operationId)) {
138 context.addIssue({
139 code: 'custom',
140 path: ['webhooks', eventName, 'post', 'operationId'],
141 message: `Duplicate operationId "${item.post.operationId}"`,
142 });
143 }
144 operationIds.add(item.post.operationId);
145 }
146 });
147
148 async function fetchJson(url) {
149 console.log(`Fetching OpenAPI spec from ${url}...`);
150 const response = await fetchWithRetry(url);
151 if (!response.ok) {
152 throw new Error(`Failed to fetch ${url}: ${response.status}`);
153 }
154 return response.json();
155 }
156
157 function formatZodError(error) {
158 return error.issues
159 .map(issue => `${issue.path.join('.') || 'document'}: ${issue.message}`)
160 .join('; ');
161 }
162
163 function parseDocument(schema, payload, label) {
164 const parsed = schema.safeParse(payload);
165 if (!parsed.success) {
166 throw new Error(`Invalid ${label}: ${formatZodError(parsed.error)}`);
167 }
168
169 // Zod confirms the boundary; the clone preserves the upstream document's key
170 // order instead of serializing Zod's schema-key order into generated files.
171 return structuredClone(payload);
172 }
173
174 function forEachOperation(paths, callback) {
175 for (const pathItem of Object.values(paths)) {
176 for (const method of HTTP_METHODS) {
177 const operation = pathItem[method];
178 if (operation) callback(operation);
179 }
180 }
181 }
182
183 /**
184 * Filter paths: remove ignored/internal tags, keep the first public tag only.
185 */
186 function filterPaths(paths) {
187 const filteredPaths = {};
188 let removedCount = 0;
189
190 for (const [path, pathItem] of Object.entries(paths)) {
191 const filteredPathItem = { ...pathItem };
192
193 for (const method of HTTP_METHODS) {
194 const operation = pathItem[method];
195 if (!operation) continue;
196
197 const tags = operation.tags ?? [];
198 const isInternal = operation['x-internal'] === true || tags.includes('x-internal');
199 const publicTags = tags.filter(tag => !IGNORED_TAGS.has(tag));
200 const hasOnlyIgnoredTags = tags.length > 0 && publicTags.length === 0;
201
202 if (isInternal || hasOnlyIgnoredTags) {
203 delete filteredPathItem[method];
204 removedCount++;
205 } else if (publicTags.length > 0) {
206 filteredPathItem[method] = { ...operation, tags: [publicTags[0]] };
207 }
208 }
209
210 const hasOperation = HTTP_METHODS.some(method => filteredPathItem[method]);
211 if (hasOperation || filteredPathItem.$ref) {
212 filteredPaths[path] = filteredPathItem;
213 }
214 }
215
216 return { filteredPaths, removedCount };
217 }
218
219 /**
220 * Strip version prefixes from operationIds (e.g. getV3_1Tools → getTools).
221 */
222 function cleanOperationIds(paths) {
223 forEachOperation(paths, operation => {
224 if (operation.operationId) {
225 operation.operationId = operation.operationId.replace(/V\d+_\d+/g, '');
226 }
227 });
228 }
229
230 function mergePropertySchemas(existing, incoming) {
231 if (!existing) return structuredClone(incoming);
232
233 const merged = structuredClone(existing);
234 if (existing.enum && incoming.enum) {
235 merged.enum = [...new Set([...existing.enum, ...incoming.enum])];
236 }
237 if (existing.properties && incoming.properties) {
238 merged.properties = { ...existing.properties };
239 for (const [key, value] of Object.entries(incoming.properties)) {
240 merged.properties[key] = mergePropertySchemas(merged.properties[key], value);
241 }
242 }
243 return merged;
244 }
245
246 function visitObjects(value, visitor, parentKey = '') {
247 if (Array.isArray(value)) {
248 for (const item of value) visitObjects(item, visitor, parentKey);
249 return;
250 }
251 if (!value || typeof value !== 'object') return;
252
253 visitor(value, parentKey);
254 for (const [key, child] of Object.entries(value)) {
255 visitObjects(child, visitor, key);
256 }
257 }
258
259 function normalizeLargeObjectUnions(spec) {
260 visitObjects(spec, schema => {
261 for (const unionKey of ['anyOf', 'oneOf']) {
262 const variants = schema[unionKey];
263 if (!Array.isArray(variants) || variants.length <= 5) continue;
264
265 const objectSchemas = variants.filter(
266 variant => variant?.type === 'object' && variant.properties
267 );
268 const mostlyObjects =
269 objectSchemas.length > 5 && objectSchemas.length >= variants.length * 0.8;
270 if (!mostlyObjects) continue;
271
272 const mergedProperties = {};
273 for (const objectSchema of objectSchemas) {
274 for (const [name, property] of Object.entries(objectSchema.properties)) {
275 mergedProperties[name] = mergePropertySchemas(mergedProperties[name], property);
276 }
277 }
278
279 const universallyRequired = [
280 ...new Set(objectSchemas.flatMap(objectSchema => objectSchema.required ?? [])),
281 ].filter(name => objectSchemas.every(objectSchema => objectSchema.required?.includes(name)));
282
283 delete schema[unionKey];
284 schema.type = 'object';
285 schema.properties = mergedProperties;
286 schema.additionalProperties = true;
287 if (universallyRequired.length > 0) {
288 schema.required = universallyRequired;
289 }
290 }
291 });
292 }
293
294 function fixNullableWithoutType(spec) {
295 visitObjects(spec, (schema, parentKey) => {
296 const needsType =
297 schema.nullable === true &&
298 !schema.type &&
299 !schema.$ref &&
300 !schema.oneOf &&
301 !schema.anyOf &&
302 !schema.allOf;
303 if (needsType) {
304 if (parentKey === 'additionalProperties') {
305 delete schema.nullable;
306 } else if (Array.isArray(schema.example)) {
307 schema.type = 'array';
308 } else {
309 schema.type = 'object';
310 }
311 }
312 });
313 }
314
315 function removeCookieAuthentication(spec) {
316 if (spec.components?.securitySchemes?.CookieAuth) {
317 delete spec.components.securitySchemes.CookieAuth;
318 }
319
320 forEachOperation(spec.paths, operation => {
321 if (!operation.security) return;
322
323 operation.security = operation.security.filter(requirement => !('CookieAuth' in requirement));
324 if (operation.security.length === 0) delete operation.security;
325 });
326 }
327
328 /**
329 * Post-process a spec: pin production, hide internal API, and normalize schemas.
330 */
331 function postProcessSpec(spec) {
332 // Pin the server to production. The published docs must always show the
333 // production base URL in their curl examples, regardless of which environment
334 // the source spec was fetched from (a staging fetch would otherwise bake a
335 // staging server URL into the committed reference).
336 spec.servers = [
337 {
338 url: PRODUCTION_BASE_URL,
339 description: 'PRODUCTION API',
340 },
341 ];
342 if (spec.tags) {
343 spec.tags = spec.tags.filter(tag => !IGNORED_TAGS.has(tag.name));
344 }
345 // fumadocs-openapi only generates pages for operations whose tags are
346 // declared top-level; the backend generator omits some (e.g. Projects).
347 const declaredUpstream = new Set((spec.tags ?? []).map(tag => tag.name));
348 declareOperationTags(spec);
349 const addedTags = (spec.tags ?? []).filter(tag => !declaredUpstream.has(tag.name));
350 if (addedTags.length > 0) {
351 console.warn(
352 `WARN: upstream spec uses tags missing from its top-level tags array: ${addedTags
353 .map(tag => tag.name)
354 .join(', ')}. Declared them automatically; the backend generator should emit them.`
355 );
356 }
357
358 removeCookieAuthentication(spec);
359 normalizeLargeObjectUnions(spec);
360 fixNullableWithoutType(spec);
361 }
362
363 export function prepareApiSpec(payload, apiVersion) {
364 const spec = parseDocument(
365 OpenApiDocumentSchema,
366 payload,
367 `Composio API v${apiVersion} OpenAPI document`
368 );
369 const { filteredPaths, removedCount } = filterPaths(spec.paths);
370 spec.paths = filteredPaths;
371
372 cleanOperationIds(spec.paths);
373 forEachOperation(spec.paths, operation => {
374 operation['x-api-version'] = apiVersion;
375 });
376 postProcessSpec(spec);
377
378 return { spec, removedCount };
379 }
380
381 function writeJson(filename, value) {
382 const outputPath = join(SCRIPT_DIRECTORY, '../public', filename);
383 writeFileSync(outputPath, JSON.stringify(value, null, 2));
384 console.log(`Written ${filename} to ${outputPath}`);
385 }
386
387 async function fetchAndFilterSpecs() {
388 const [v3Payload, v31Payload] = await Promise.all([
389 fetchJson(OPENAPI_V3_URL),
390 fetchJson(OPENAPI_V31_URL),
391 ]);
392 const v31 = prepareApiSpec(v31Payload, '3.1');
393 const v3 = prepareApiSpec(v3Payload, '3.0');
394
395 console.log(
396 `v3.1: ${Object.keys(v31.spec.paths).length} paths (${v31.removedCount} operations removed)`
397 );
398 console.log(
399 `v3.0: ${Object.keys(v3.spec.paths).length} paths (${v3.removedCount} operations removed)`
400 );
401
402 writeJson('openapi.json', v31.spec);
403 writeJson('openapi-v3.json', v3.spec);
404 }
405
406 /**
407 * Fetch the standalone webhook-events spec and write it verbatim.
408 *
409 * It's a separate OpenAPI 3.1 document keyed on `webhooks` (not `paths`), so it
410 * skips the path filtering, server pinning, and union normalization above — none
411 * of which apply. Fetched live from production like openapi.json.
412 *
413 * A fetch or schema failure leaves the committed snapshot untouched. This keeps
414 * a transient or malformed production response from deleting generated pages.
415 */
416 export function writeWebhookSnapshot(payload, outputPath, sourceUrl = OPENAPI_WEBHOOKS_URL) {
417 const parsed = WebhookDocumentSchema.safeParse(payload);
418 if (!parsed.success) {
419 console.warn(
420 `WARN: refusing to write webhooks spec from ${sourceUrl} — ${formatZodError(parsed.error)}. Keeping existing ${outputPath}.`
421 );
422 return false;
423 }
424
425 const eventCount = Object.keys(parsed.data.webhooks).length;
426 writeFileSync(outputPath, JSON.stringify(payload, null, 2));
427 console.log(`Written webhooks spec to ${outputPath} (${eventCount} events)`);
428 return true;
429 }
430
431 async function fetchAndWriteWebhookSpec() {
432 const outputPath = join(SCRIPT_DIRECTORY, '../public/openapi-webhooks.json');
433 try {
434 const payload = await fetchJson(OPENAPI_WEBHOOKS_URL);
435 writeWebhookSnapshot(payload, outputPath);
436 } catch (err) {
437 const message = err instanceof Error ? err.message : String(err);
438 console.warn(
439 `WARN: could not fetch webhooks spec from ${OPENAPI_WEBHOOKS_URL}: ${message}. Keeping existing ${outputPath}.`
440 );
441 }
442 }
443
444 if (import.meta.main) {
445 // Independent fetches with independent error handling; run them concurrently.
446 await Promise.all([fetchAndFilterSpecs().catch(console.error), fetchAndWriteWebhookSpec()]);
447 }