Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 428 lines 13.9 KB
Raw
1 /**
2 * Toolkit Generator Script
3 *
4 * Fetches all toolkits from Composio API and generates:
5 * - /public/data/toolkits.json (full data with tools & triggers - for detail pages)
6 * - /public/data/toolkits-list.json (light version without tools/triggers - for landing page)
7 *
8 * Run: bun run generate:toolkits
9 */
10
11 import { mkdir, writeFile } from 'fs/promises';
12 import { join } from 'path';
13 import { fetchWithRetry } from './fetch-with-retry';
14 import type { Composio } from '@composio/client';
15 import {
16 API_PAGE_LIMIT,
17 createComposioClient,
18 listToolkits,
19 listToolkitTools,
20 listToolkitTriggers,
21 } from '../lib/composio-api';
22 import { requireProductionApiV3Url, stripStagingHosts } from './production-api.mjs';
23 import { applyToolkitVersions, fetchProductionToolkitVersions } from './toolkit-versions';
24 import { isPublicToolkitSlug } from '../lib/public-toolkit-policy';
25 import { z } from 'zod';
26 import { instantSchema, type Instant } from '../lib/instant';
27
28 // Rejects a non-production override before any request; the client always targets production.
29 requireProductionApiV3Url(process.env.COMPOSIO_API_BASE);
30 const API_KEY = process.env.COMPOSIO_API_KEY;
31
32 const OUTPUT_DIR = join(process.cwd(), 'public/data');
33 interface Tool {
34 instant?: Instant;
35 slug: string;
36 name: string;
37 description: string;
38 }
39
40 interface Trigger {
41 slug: string;
42 name: string;
43 description: string;
44 }
45
46 interface AuthConfigField {
47 name: string;
48 displayName: string;
49 type: string;
50 description: string;
51 required: boolean;
52 default?: string | null;
53 }
54
55 interface AuthConfigDetail {
56 mode: string;
57 name: string;
58 fields: {
59 auth_config_creation: {
60 required: AuthConfigField[];
61 optional: AuthConfigField[];
62 };
63 connected_account_initiation: {
64 required: AuthConfigField[];
65 optional: AuthConfigField[];
66 };
67 };
68 }
69
70 interface Toolkit {
71 instant?: Instant;
72 slug: string;
73 name: string;
74 logo: string | null;
75 description: string;
76 category: string | null;
77 authSchemes: string[];
78 composioManagedAuthSchemes?: string[];
79 toolCount: number;
80 triggerCount: number;
81 version: string | null;
82 tools: Tool[];
83 triggers: Trigger[];
84 authConfigDetails?: AuthConfigDetail[];
85 }
86
87 // Zod schemas for the raw API payloads this script consumes. Parsing is
88 // lenient on purpose: malformed fields degrade to the same fallbacks the
89 // hand-written mapping used to apply, so a partially bad catalog entry never
90 // aborts a full generation run.
91
92 /** Optional lenient string array: undefined when absent/non-array, junk members dropped. */
93 const optionalStringArraySchema = z
94 .array(z.unknown())
95 .optional()
96 .catch(undefined)
97 .transform(items =>
98 items?.flatMap(item => {
99 const parsed = z.string().safeParse(item);
100 return parsed.success ? [parsed.data] : [];
101 })
102 );
103
104 const rawToolkitSchema = z.object({
105 instant: instantSchema,
106 slug: z.string().catch(''),
107 name: z.string().optional().catch(undefined),
108 logo: z.string().optional().catch(undefined),
109 description: z.string().catch(''),
110 meta: z
111 .object({
112 logo: z.string().optional().catch(undefined),
113 description: z.string().optional().catch(undefined),
114 categories: z.array(z.unknown()).catch([]),
115 })
116 .catch({ logo: undefined, description: undefined, categories: [] }),
117 auth_schemes: optionalStringArraySchema,
118 authSchemes: optionalStringArraySchema,
119 composio_managed_auth_schemes: optionalStringArraySchema,
120 composioManagedAuthSchemes: optionalStringArraySchema,
121 tool_count: z.number().optional().catch(undefined),
122 toolCount: z.number().optional().catch(undefined),
123 trigger_count: z.number().optional().catch(undefined),
124 triggerCount: z.number().optional().catch(undefined),
125 });
126
127 const categoryEntrySchema = z.union([
128 z.string(),
129 z.object({ name: z.string() }).transform(entry => entry.name),
130 ]);
131
132 const slugItemSchema = z.object({ slug: z.string() });
133
134 // Tools/triggers list entries share the slug/name/description shape.
135 const rawNamedItemSchema = z.object({
136 instant: instantSchema,
137 slug: z.string().catch(''),
138 name: z.string().optional().catch(undefined),
139 display_name: z.string().optional().catch(undefined),
140 description: z.string().catch(''),
141 });
142
143 const namedItemListSchema = z
144 .union([
145 z.object({ items: z.array(z.unknown()) }).transform(data => data.items),
146 z.array(z.unknown()),
147 ])
148 .catch([])
149 .transform(items =>
150 items.flatMap(item => {
151 const parsed = rawNamedItemSchema.safeParse(item);
152 if (!parsed.success) return [];
153 const { slug, name, display_name, description, instant } = parsed.data;
154 return [{ slug, name: name || display_name || slug, description, ...(instant ? { instant } : {}) }];
155 })
156 );
157
158 const requirementListsSchema = z
159 .object({
160 required: z.array(z.unknown()).catch([]),
161 optional: z.array(z.unknown()).catch([]),
162 })
163 .catch({ required: [], optional: [] });
164
165 const authConfigFieldsSchema = z
166 .object({
167 auth_config_creation: requirementListsSchema,
168 connected_account_initiation: requirementListsSchema,
169 })
170 .catch({
171 auth_config_creation: { required: [], optional: [] },
172 connected_account_initiation: { required: [], optional: [] },
173 });
174
175 const rawAuthConfigFieldSchema = z.object({
176 name: z.string().catch(''),
177 displayName: z.string().optional().catch(undefined),
178 type: z.string().catch('string'),
179 description: z.string().catch(''),
180 required: z.boolean().optional().catch(undefined),
181 default: z
182 .union([z.string(), z.number(), z.boolean()])
183 .transform(value => String(value))
184 .nullable()
185 .optional()
186 .catch(undefined),
187 });
188
189 const rawAuthConfigDetailSchema = z.object({
190 mode: z.string().catch(''),
191 name: z.string().optional().catch(undefined),
192 });
193
194 const authConfigDetailsResponseSchema = z
195 .object({ auth_config_details: z.array(z.unknown()).catch([]) })
196 .catch({ auth_config_details: [] });
197
198 export function parseNamedItems(value: unknown) {
199 return namedItemListSchema.parse(value);
200 }
201
202 // The backend caps `limit` per page (1000 for toolkits and tools, 50 for
203 // triggers) and defaults to usage ordering, so a single request returns only
204 // the first page. Every list request follows `next_cursor` until exhausted.
205 let client: Composio | undefined;
206
207 /** The production client; fetchWithRetry owns retries, so the client makes none. */
208 function api(): Composio {
209 client ??= createComposioClient(API_KEY!, {
210 maxRetries: 0,
211 timeout: 10 * 60_000,
212 fetch: (input, init) => fetchWithRetry(String(input), init),
213 });
214 return client;
215 }
216
217 async function fetchToolkits(): Promise<unknown[]> {
218 console.log('Fetching toolkits from API...');
219
220 // Pages can overlap when the catalog shifts between cursor fetches; keep the
221 // first occurrence so API-provided ordering stays stable.
222 const seen = new Set<string>();
223 return (await listToolkits(api(), { pageSize: API_PAGE_LIMIT })).filter(item => {
224 const parsedSlug = slugItemSchema.safeParse(item);
225 if (!parsedSlug.success) return true;
226 const slug = parsedSlug.data.slug.toLowerCase();
227 if (seen.has(slug)) return false;
228 seen.add(slug);
229 return true;
230 });
231 }
232
233 // A failed traversal fails the run: publishing an empty list would replace the
234 // toolkit's tools in the committed snapshot. Malformed items are still dropped.
235 async function fetchToolsForToolkit(slug: string): Promise<Tool[]> {
236 return parseNamedItems(await listToolkitTools(api(), slug, { pageSize: API_PAGE_LIMIT }));
237 }
238
239 async function fetchTriggersForToolkit(slug: string): Promise<Trigger[]> {
240 return parseNamedItems(await listToolkitTriggers(api(), slug));
241 }
242
243 async function fetchAuthConfigDetails(slug: string): Promise<AuthConfigDetail[]> {
244 let response: unknown;
245 try {
246 response = await api().toolkits.retrieve(slug);
247 } catch {
248 return [];
249 }
250
251 const data = authConfigDetailsResponseSchema.parse(response);
252
253 return data.auth_config_details.flatMap(item => {
254 const detail = transformAuthConfigDetail(item);
255 return detail ? [detail] : [];
256 });
257 }
258
259 export function transformAuthConfigField(value: unknown, required: boolean): AuthConfigField {
260 const parsed = rawAuthConfigFieldSchema.safeParse(value);
261 const field = parsed.success ? parsed.data : rawAuthConfigFieldSchema.parse({});
262
263 return {
264 name: field.name,
265 displayName: field.displayName || field.name,
266 type: field.type,
267 description: field.description,
268 required: field.required ?? required,
269 default: field.default ?? null,
270 };
271 }
272
273 export function authConfigFields(
274 raw: Record<string, unknown>,
275 phase: 'auth_config_creation' | 'connected_account_initiation',
276 requirement: 'required' | 'optional'
277 ): AuthConfigField[] {
278 const fields = authConfigFieldsSchema.parse(raw.fields);
279 return fields[phase][requirement].map(field =>
280 transformAuthConfigField(field, requirement === 'required')
281 );
282 }
283
284 export function transformAuthConfigDetail(value: unknown): AuthConfigDetail | null {
285 const raw = z.record(z.string(), z.unknown()).safeParse(value);
286 if (!raw.success) return null;
287
288 const { mode, name } = rawAuthConfigDetailSchema.parse(raw.data);
289 return {
290 mode,
291 name: name || mode,
292 fields: {
293 auth_config_creation: {
294 required: authConfigFields(raw.data, 'auth_config_creation', 'required'),
295 optional: authConfigFields(raw.data, 'auth_config_creation', 'optional'),
296 },
297 connected_account_initiation: {
298 required: authConfigFields(raw.data, 'connected_account_initiation', 'required'),
299 optional: authConfigFields(raw.data, 'connected_account_initiation', 'optional'),
300 },
301 },
302 };
303 }
304
305 export function transformToolkit(raw: unknown): Toolkit {
306 const parsed = rawToolkitSchema.safeParse(raw);
307 const toolkit = parsed.success ? parsed.data : rawToolkitSchema.parse({});
308 const slug = toolkit.slug.toLowerCase();
309 const firstCategory = categoryEntrySchema.safeParse(toolkit.meta.categories[0]);
310 const category = firstCategory.success ? firstCategory.data : null;
311 const authSchemes = toolkit.auth_schemes ?? toolkit.authSchemes ?? [];
312 const composioManaged =
313 toolkit.composio_managed_auth_schemes ?? toolkit.composioManagedAuthSchemes ?? [];
314
315 return {
316 slug,
317 name: toolkit.name || toolkit.slug,
318 ...(toolkit.instant ? { instant: toolkit.instant } : {}),
319 logo: toolkit.meta.logo || toolkit.logo || null,
320 description: toolkit.meta.description || toolkit.description,
321 category: category || null,
322 authSchemes,
323 ...(composioManaged.length > 0 ? { composioManagedAuthSchemes: composioManaged } : {}),
324 toolCount: toolkit.tool_count || toolkit.toolCount || 0,
325 triggerCount: toolkit.trigger_count || toolkit.triggerCount || 0,
326 version: null,
327 tools: [],
328 triggers: [],
329 };
330 }
331
332 export function shouldPublishToolkit(raw: unknown): boolean {
333 return isPublicToolkitSlug(transformToolkit(raw).slug);
334 }
335
336 async function main() {
337 console.log('Starting toolkit generation...\n');
338
339 // Create output directory
340 await mkdir(OUTPUT_DIR, { recursive: true });
341
342 // Fetch all toolkits and changelog in parallel
343 const [rawToolkits, versionMap] = await Promise.all([
344 fetchToolkits(),
345 fetchProductionToolkitVersions(API_KEY!),
346 ]);
347 console.log(`Found ${rawToolkits.length} toolkits\n`);
348
349 // Transform toolkits
350 const toolkits: Toolkit[] = rawToolkits.filter(shouldPublishToolkit).map(transformToolkit);
351
352 // Add versions from changelog
353 applyToolkitVersions(toolkits, versionMap);
354
355 // Fetch tools, triggers, and auth config details for each toolkit in batches.
356 // Each toolkit fires 3 requests in parallel, so batchSize N = ~3N concurrent
357 // requests. Kept low to soften burst pressure on the backend rate limit;
358 // fetchWithRetry handles the remaining 429s with backoff.
359 console.log('Fetching tools, triggers, and auth config details...');
360 const batchSize = 5;
361 let completed = 0;
362
363 for (let i = 0; i < toolkits.length; i += batchSize) {
364 const batch = toolkits.slice(i, i + batchSize);
365
366 await Promise.all(
367 batch.map(async toolkit => {
368 const [tools, triggers, authConfigDetails] = await Promise.all([
369 fetchToolsForToolkit(toolkit.slug.toUpperCase()),
370 fetchTriggersForToolkit(toolkit.slug.toUpperCase()),
371 fetchAuthConfigDetails(toolkit.slug),
372 ]);
373
374 toolkit.tools = tools;
375 toolkit.triggers = triggers;
376 toolkit.toolCount = tools.length;
377 toolkit.triggerCount = triggers.length;
378 toolkit.authConfigDetails = authConfigDetails.length > 0 ? authConfigDetails : undefined;
379
380 completed++;
381 process.stdout.write(`\r Progress: ${completed}/${toolkits.length}`);
382 })
383 );
384 }
385
386 console.log('\n');
387
388 // Write full file (for detail pages - read from filesystem).
389 // Defense in depth: rewrite any staging host embedded in production data so
390 // auth-config `default` URLs cannot publish staging endpoints. The light file
391 // below carries no URLs.
392 await writeFile(
393 join(OUTPUT_DIR, 'toolkits.json'),
394 stripStagingHosts(JSON.stringify(toolkits, null, 2))
395 );
396
397 // Write light file (for landing page - imported in client component)
398 // Excludes tools and triggers arrays to keep bundle size small
399 const toolkitsLight = toolkits.map(({ slug, name, logo, category, toolCount, triggerCount, instant }) => ({
400 slug,
401 name,
402 logo,
403 category,
404 toolCount,
405 triggerCount,
406 ...(instant ? { instant } : {}),
407 }));
408 await writeFile(join(OUTPUT_DIR, 'toolkits-list.json'), JSON.stringify(toolkitsLight, null, 2));
409
410 const fullSizeKB = Math.round(JSON.stringify(toolkits).length / 1024);
411 const lightSizeKB = Math.round(JSON.stringify(toolkitsLight).length / 1024);
412 console.log('Generation complete!');
413 console.log(` Full: public/data/toolkits.json (~${fullSizeKB}KB)`);
414 console.log(` Light: public/data/toolkits-list.json (~${lightSizeKB}KB)`);
415 console.log(` Toolkits: ${toolkits.length}`);
416 }
417
418 if (import.meta.main) {
419 if (!API_KEY) {
420 console.error('Error: COMPOSIO_API_KEY environment variable is required');
421 process.exit(1);
422 }
423
424 main().catch(error => {
425 console.error('Fatal error:', error);
426 process.exit(1);
427 });
428 }