Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | /** |
| 2 | * Toolkit Generator Script |
| 3 | * |
| 4 | * Fetches all toolkits from Composio API and generates: |
| 5 | * - /public/data/toolkits.json (full data with tools & triggers - for detail pages) |
| 6 | * - /public/data/toolkits-list.json (light version without tools/triggers - for landing page) |
| 7 | * |
| 8 | * Run: bun run generate:toolkits |
| 9 | */ |
| 10 | |
| 11 | import { mkdir, writeFile } from 'fs/promises'; |
| 12 | import { join } from 'path'; |
| 13 | import { fetchWithRetry } from './fetch-with-retry'; |
| 14 | import type { Composio } from '@composio/client'; |
| 15 | import { |
| 16 | API_PAGE_LIMIT, |
| 17 | createComposioClient, |
| 18 | listToolkits, |
| 19 | listToolkitTools, |
| 20 | listToolkitTriggers, |
| 21 | } from '../lib/composio-api'; |
| 22 | import { requireProductionApiV3Url, stripStagingHosts } from './production-api.mjs'; |
| 23 | import { applyToolkitVersions, fetchProductionToolkitVersions } from './toolkit-versions'; |
| 24 | import { isPublicToolkitSlug } from '../lib/public-toolkit-policy'; |
| 25 | import { z } from 'zod'; |
| 26 | import { instantSchema, type Instant } from '../lib/instant'; |
| 27 | |
| 28 | // Rejects a non-production override before any request; the client always targets production. |
| 29 | requireProductionApiV3Url(process.env.COMPOSIO_API_BASE); |
| 30 | const API_KEY = process.env.COMPOSIO_API_KEY; |
| 31 | |
| 32 | const OUTPUT_DIR = join(process.cwd(), 'public/data'); |
| 33 | interface Tool { |
| 34 | instant?: Instant; |
| 35 | slug: string; |
| 36 | name: string; |
| 37 | description: string; |
| 38 | } |
| 39 | |
| 40 | interface Trigger { |
| 41 | slug: string; |
| 42 | name: string; |
| 43 | description: string; |
| 44 | } |
| 45 | |
| 46 | interface AuthConfigField { |
| 47 | name: string; |
| 48 | displayName: string; |
| 49 | type: string; |
| 50 | description: string; |
| 51 | required: boolean; |
| 52 | default?: string | null; |
| 53 | } |
| 54 | |
| 55 | interface AuthConfigDetail { |
| 56 | mode: string; |
| 57 | name: string; |
| 58 | fields: { |
| 59 | auth_config_creation: { |
| 60 | required: AuthConfigField[]; |
| 61 | optional: AuthConfigField[]; |
| 62 | }; |
| 63 | connected_account_initiation: { |
| 64 | required: AuthConfigField[]; |
| 65 | optional: AuthConfigField[]; |
| 66 | }; |
| 67 | }; |
| 68 | } |
| 69 | |
| 70 | interface Toolkit { |
| 71 | instant?: Instant; |
| 72 | slug: string; |
| 73 | name: string; |
| 74 | logo: string | null; |
| 75 | description: string; |
| 76 | category: string | null; |
| 77 | authSchemes: string[]; |
| 78 | composioManagedAuthSchemes?: string[]; |
| 79 | toolCount: number; |
| 80 | triggerCount: number; |
| 81 | version: string | null; |
| 82 | tools: Tool[]; |
| 83 | triggers: Trigger[]; |
| 84 | authConfigDetails?: AuthConfigDetail[]; |
| 85 | } |
| 86 | |
| 87 | // Zod schemas for the raw API payloads this script consumes. Parsing is |
| 88 | // lenient on purpose: malformed fields degrade to the same fallbacks the |
| 89 | // hand-written mapping used to apply, so a partially bad catalog entry never |
| 90 | // aborts a full generation run. |
| 91 | |
| 92 | /** Optional lenient string array: undefined when absent/non-array, junk members dropped. */ |
| 93 | const optionalStringArraySchema = z |
| 94 | .array(z.unknown()) |
| 95 | .optional() |
| 96 | .catch(undefined) |
| 97 | .transform(items => |
| 98 | items?.flatMap(item => { |
| 99 | const parsed = z.string().safeParse(item); |
| 100 | return parsed.success ? [parsed.data] : []; |
| 101 | }) |
| 102 | ); |
| 103 | |
| 104 | const rawToolkitSchema = z.object({ |
| 105 | instant: instantSchema, |
| 106 | slug: z.string().catch(''), |
| 107 | name: z.string().optional().catch(undefined), |
| 108 | logo: z.string().optional().catch(undefined), |
| 109 | description: z.string().catch(''), |
| 110 | meta: z |
| 111 | .object({ |
| 112 | logo: z.string().optional().catch(undefined), |
| 113 | description: z.string().optional().catch(undefined), |
| 114 | categories: z.array(z.unknown()).catch([]), |
| 115 | }) |
| 116 | .catch({ logo: undefined, description: undefined, categories: [] }), |
| 117 | auth_schemes: optionalStringArraySchema, |
| 118 | authSchemes: optionalStringArraySchema, |
| 119 | composio_managed_auth_schemes: optionalStringArraySchema, |
| 120 | composioManagedAuthSchemes: optionalStringArraySchema, |
| 121 | tool_count: z.number().optional().catch(undefined), |
| 122 | toolCount: z.number().optional().catch(undefined), |
| 123 | trigger_count: z.number().optional().catch(undefined), |
| 124 | triggerCount: z.number().optional().catch(undefined), |
| 125 | }); |
| 126 | |
| 127 | const categoryEntrySchema = z.union([ |
| 128 | z.string(), |
| 129 | z.object({ name: z.string() }).transform(entry => entry.name), |
| 130 | ]); |
| 131 | |
| 132 | const slugItemSchema = z.object({ slug: z.string() }); |
| 133 | |
| 134 | // Tools/triggers list entries share the slug/name/description shape. |
| 135 | const rawNamedItemSchema = z.object({ |
| 136 | instant: instantSchema, |
| 137 | slug: z.string().catch(''), |
| 138 | name: z.string().optional().catch(undefined), |
| 139 | display_name: z.string().optional().catch(undefined), |
| 140 | description: z.string().catch(''), |
| 141 | }); |
| 142 | |
| 143 | const namedItemListSchema = z |
| 144 | .union([ |
| 145 | z.object({ items: z.array(z.unknown()) }).transform(data => data.items), |
| 146 | z.array(z.unknown()), |
| 147 | ]) |
| 148 | .catch([]) |
| 149 | .transform(items => |
| 150 | items.flatMap(item => { |
| 151 | const parsed = rawNamedItemSchema.safeParse(item); |
| 152 | if (!parsed.success) return []; |
| 153 | const { slug, name, display_name, description, instant } = parsed.data; |
| 154 | return [{ slug, name: name || display_name || slug, description, ...(instant ? { instant } : {}) }]; |
| 155 | }) |
| 156 | ); |
| 157 | |
| 158 | const requirementListsSchema = z |
| 159 | .object({ |
| 160 | required: z.array(z.unknown()).catch([]), |
| 161 | optional: z.array(z.unknown()).catch([]), |
| 162 | }) |
| 163 | .catch({ required: [], optional: [] }); |
| 164 | |
| 165 | const authConfigFieldsSchema = z |
| 166 | .object({ |
| 167 | auth_config_creation: requirementListsSchema, |
| 168 | connected_account_initiation: requirementListsSchema, |
| 169 | }) |
| 170 | .catch({ |
| 171 | auth_config_creation: { required: [], optional: [] }, |
| 172 | connected_account_initiation: { required: [], optional: [] }, |
| 173 | }); |
| 174 | |
| 175 | const rawAuthConfigFieldSchema = z.object({ |
| 176 | name: z.string().catch(''), |
| 177 | displayName: z.string().optional().catch(undefined), |
| 178 | type: z.string().catch('string'), |
| 179 | description: z.string().catch(''), |
| 180 | required: z.boolean().optional().catch(undefined), |
| 181 | default: z |
| 182 | .union([z.string(), z.number(), z.boolean()]) |
| 183 | .transform(value => String(value)) |
| 184 | .nullable() |
| 185 | .optional() |
| 186 | .catch(undefined), |
| 187 | }); |
| 188 | |
| 189 | const rawAuthConfigDetailSchema = z.object({ |
| 190 | mode: z.string().catch(''), |
| 191 | name: z.string().optional().catch(undefined), |
| 192 | }); |
| 193 | |
| 194 | const authConfigDetailsResponseSchema = z |
| 195 | .object({ auth_config_details: z.array(z.unknown()).catch([]) }) |
| 196 | .catch({ auth_config_details: [] }); |
| 197 | |
| 198 | export function parseNamedItems(value: unknown) { |
| 199 | return namedItemListSchema.parse(value); |
| 200 | } |
| 201 | |
| 202 | // The backend caps `limit` per page (1000 for toolkits and tools, 50 for |
| 203 | // triggers) and defaults to usage ordering, so a single request returns only |
| 204 | // the first page. Every list request follows `next_cursor` until exhausted. |
| 205 | let client: Composio | undefined; |
| 206 | |
| 207 | /** The production client; fetchWithRetry owns retries, so the client makes none. */ |
| 208 | function api(): Composio { |
| 209 | client ??= createComposioClient(API_KEY!, { |
| 210 | maxRetries: 0, |
| 211 | timeout: 10 * 60_000, |
| 212 | fetch: (input, init) => fetchWithRetry(String(input), init), |
| 213 | }); |
| 214 | return client; |
| 215 | } |
| 216 | |
| 217 | async function fetchToolkits(): Promise<unknown[]> { |
| 218 | console.log('Fetching toolkits from API...'); |
| 219 | |
| 220 | // Pages can overlap when the catalog shifts between cursor fetches; keep the |
| 221 | // first occurrence so API-provided ordering stays stable. |
| 222 | const seen = new Set<string>(); |
| 223 | return (await listToolkits(api(), { pageSize: API_PAGE_LIMIT })).filter(item => { |
| 224 | const parsedSlug = slugItemSchema.safeParse(item); |
| 225 | if (!parsedSlug.success) return true; |
| 226 | const slug = parsedSlug.data.slug.toLowerCase(); |
| 227 | if (seen.has(slug)) return false; |
| 228 | seen.add(slug); |
| 229 | return true; |
| 230 | }); |
| 231 | } |
| 232 | |
| 233 | // A failed traversal fails the run: publishing an empty list would replace the |
| 234 | // toolkit's tools in the committed snapshot. Malformed items are still dropped. |
| 235 | async function fetchToolsForToolkit(slug: string): Promise<Tool[]> { |
| 236 | return parseNamedItems(await listToolkitTools(api(), slug, { pageSize: API_PAGE_LIMIT })); |
| 237 | } |
| 238 | |
| 239 | async function fetchTriggersForToolkit(slug: string): Promise<Trigger[]> { |
| 240 | return parseNamedItems(await listToolkitTriggers(api(), slug)); |
| 241 | } |
| 242 | |
| 243 | async function fetchAuthConfigDetails(slug: string): Promise<AuthConfigDetail[]> { |
| 244 | let response: unknown; |
| 245 | try { |
| 246 | response = await api().toolkits.retrieve(slug); |
| 247 | } catch { |
| 248 | return []; |
| 249 | } |
| 250 | |
| 251 | const data = authConfigDetailsResponseSchema.parse(response); |
| 252 | |
| 253 | return data.auth_config_details.flatMap(item => { |
| 254 | const detail = transformAuthConfigDetail(item); |
| 255 | return detail ? [detail] : []; |
| 256 | }); |
| 257 | } |
| 258 | |
| 259 | export function transformAuthConfigField(value: unknown, required: boolean): AuthConfigField { |
| 260 | const parsed = rawAuthConfigFieldSchema.safeParse(value); |
| 261 | const field = parsed.success ? parsed.data : rawAuthConfigFieldSchema.parse({}); |
| 262 | |
| 263 | return { |
| 264 | name: field.name, |
| 265 | displayName: field.displayName || field.name, |
| 266 | type: field.type, |
| 267 | description: field.description, |
| 268 | required: field.required ?? required, |
| 269 | default: field.default ?? null, |
| 270 | }; |
| 271 | } |
| 272 | |
| 273 | export function authConfigFields( |
| 274 | raw: Record<string, unknown>, |
| 275 | phase: 'auth_config_creation' | 'connected_account_initiation', |
| 276 | requirement: 'required' | 'optional' |
| 277 | ): AuthConfigField[] { |
| 278 | const fields = authConfigFieldsSchema.parse(raw.fields); |
| 279 | return fields[phase][requirement].map(field => |
| 280 | transformAuthConfigField(field, requirement === 'required') |
| 281 | ); |
| 282 | } |
| 283 | |
| 284 | export function transformAuthConfigDetail(value: unknown): AuthConfigDetail | null { |
| 285 | const raw = z.record(z.string(), z.unknown()).safeParse(value); |
| 286 | if (!raw.success) return null; |
| 287 | |
| 288 | const { mode, name } = rawAuthConfigDetailSchema.parse(raw.data); |
| 289 | return { |
| 290 | mode, |
| 291 | name: name || mode, |
| 292 | fields: { |
| 293 | auth_config_creation: { |
| 294 | required: authConfigFields(raw.data, 'auth_config_creation', 'required'), |
| 295 | optional: authConfigFields(raw.data, 'auth_config_creation', 'optional'), |
| 296 | }, |
| 297 | connected_account_initiation: { |
| 298 | required: authConfigFields(raw.data, 'connected_account_initiation', 'required'), |
| 299 | optional: authConfigFields(raw.data, 'connected_account_initiation', 'optional'), |
| 300 | }, |
| 301 | }, |
| 302 | }; |
| 303 | } |
| 304 | |
| 305 | export function transformToolkit(raw: unknown): Toolkit { |
| 306 | const parsed = rawToolkitSchema.safeParse(raw); |
| 307 | const toolkit = parsed.success ? parsed.data : rawToolkitSchema.parse({}); |
| 308 | const slug = toolkit.slug.toLowerCase(); |
| 309 | const firstCategory = categoryEntrySchema.safeParse(toolkit.meta.categories[0]); |
| 310 | const category = firstCategory.success ? firstCategory.data : null; |
| 311 | const authSchemes = toolkit.auth_schemes ?? toolkit.authSchemes ?? []; |
| 312 | const composioManaged = |
| 313 | toolkit.composio_managed_auth_schemes ?? toolkit.composioManagedAuthSchemes ?? []; |
| 314 | |
| 315 | return { |
| 316 | slug, |
| 317 | name: toolkit.name || toolkit.slug, |
| 318 | ...(toolkit.instant ? { instant: toolkit.instant } : {}), |
| 319 | logo: toolkit.meta.logo || toolkit.logo || null, |
| 320 | description: toolkit.meta.description || toolkit.description, |
| 321 | category: category || null, |
| 322 | authSchemes, |
| 323 | ...(composioManaged.length > 0 ? { composioManagedAuthSchemes: composioManaged } : {}), |
| 324 | toolCount: toolkit.tool_count || toolkit.toolCount || 0, |
| 325 | triggerCount: toolkit.trigger_count || toolkit.triggerCount || 0, |
| 326 | version: null, |
| 327 | tools: [], |
| 328 | triggers: [], |
| 329 | }; |
| 330 | } |
| 331 | |
| 332 | export function shouldPublishToolkit(raw: unknown): boolean { |
| 333 | return isPublicToolkitSlug(transformToolkit(raw).slug); |
| 334 | } |
| 335 | |
| 336 | async function main() { |
| 337 | console.log('Starting toolkit generation...\n'); |
| 338 | |
| 339 | // Create output directory |
| 340 | await mkdir(OUTPUT_DIR, { recursive: true }); |
| 341 | |
| 342 | // Fetch all toolkits and changelog in parallel |
| 343 | const [rawToolkits, versionMap] = await Promise.all([ |
| 344 | fetchToolkits(), |
| 345 | fetchProductionToolkitVersions(API_KEY!), |
| 346 | ]); |
| 347 | console.log(`Found ${rawToolkits.length} toolkits\n`); |
| 348 | |
| 349 | // Transform toolkits |
| 350 | const toolkits: Toolkit[] = rawToolkits.filter(shouldPublishToolkit).map(transformToolkit); |
| 351 | |
| 352 | // Add versions from changelog |
| 353 | applyToolkitVersions(toolkits, versionMap); |
| 354 | |
| 355 | // Fetch tools, triggers, and auth config details for each toolkit in batches. |
| 356 | // Each toolkit fires 3 requests in parallel, so batchSize N = ~3N concurrent |
| 357 | // requests. Kept low to soften burst pressure on the backend rate limit; |
| 358 | // fetchWithRetry handles the remaining 429s with backoff. |
| 359 | console.log('Fetching tools, triggers, and auth config details...'); |
| 360 | const batchSize = 5; |
| 361 | let completed = 0; |
| 362 | |
| 363 | for (let i = 0; i < toolkits.length; i += batchSize) { |
| 364 | const batch = toolkits.slice(i, i + batchSize); |
| 365 | |
| 366 | await Promise.all( |
| 367 | batch.map(async toolkit => { |
| 368 | const [tools, triggers, authConfigDetails] = await Promise.all([ |
| 369 | fetchToolsForToolkit(toolkit.slug.toUpperCase()), |
| 370 | fetchTriggersForToolkit(toolkit.slug.toUpperCase()), |
| 371 | fetchAuthConfigDetails(toolkit.slug), |
| 372 | ]); |
| 373 | |
| 374 | toolkit.tools = tools; |
| 375 | toolkit.triggers = triggers; |
| 376 | toolkit.toolCount = tools.length; |
| 377 | toolkit.triggerCount = triggers.length; |
| 378 | toolkit.authConfigDetails = authConfigDetails.length > 0 ? authConfigDetails : undefined; |
| 379 | |
| 380 | completed++; |
| 381 | process.stdout.write(`\r Progress: ${completed}/${toolkits.length}`); |
| 382 | }) |
| 383 | ); |
| 384 | } |
| 385 | |
| 386 | console.log('\n'); |
| 387 | |
| 388 | // Write full file (for detail pages - read from filesystem). |
| 389 | // Defense in depth: rewrite any staging host embedded in production data so |
| 390 | // auth-config `default` URLs cannot publish staging endpoints. The light file |
| 391 | // below carries no URLs. |
| 392 | await writeFile( |
| 393 | join(OUTPUT_DIR, 'toolkits.json'), |
| 394 | stripStagingHosts(JSON.stringify(toolkits, null, 2)) |
| 395 | ); |
| 396 | |
| 397 | // Write light file (for landing page - imported in client component) |
| 398 | // Excludes tools and triggers arrays to keep bundle size small |
| 399 | const toolkitsLight = toolkits.map(({ slug, name, logo, category, toolCount, triggerCount, instant }) => ({ |
| 400 | slug, |
| 401 | name, |
| 402 | logo, |
| 403 | category, |
| 404 | toolCount, |
| 405 | triggerCount, |
| 406 | ...(instant ? { instant } : {}), |
| 407 | })); |
| 408 | await writeFile(join(OUTPUT_DIR, 'toolkits-list.json'), JSON.stringify(toolkitsLight, null, 2)); |
| 409 | |
| 410 | const fullSizeKB = Math.round(JSON.stringify(toolkits).length / 1024); |
| 411 | const lightSizeKB = Math.round(JSON.stringify(toolkitsLight).length / 1024); |
| 412 | console.log('Generation complete!'); |
| 413 | console.log(` Full: public/data/toolkits.json (~${fullSizeKB}KB)`); |
| 414 | console.log(` Light: public/data/toolkits-list.json (~${lightSizeKB}KB)`); |
| 415 | console.log(` Toolkits: ${toolkits.length}`); |
| 416 | } |
| 417 | |
| 418 | if (import.meta.main) { |
| 419 | if (!API_KEY) { |
| 420 | console.error('Error: COMPOSIO_API_KEY environment variable is required'); |
| 421 | process.exit(1); |
| 422 | } |
| 423 | |
| 424 | main().catch(error => { |
| 425 | console.error('Fatal error:', error); |
| 426 | process.exit(1); |
| 427 | }); |
| 428 | } |