Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import process from 'node:process'; |
| 2 | import path from 'node:path'; |
| 3 | import { $ } from 'bun'; |
| 4 | import { |
| 5 | Config, |
| 6 | ConfigProvider, |
| 7 | Console, |
| 8 | Data, |
| 9 | DateTime, |
| 10 | Effect, |
| 11 | Logger, |
| 12 | Option, |
| 13 | Schema, |
| 14 | } from 'effect'; |
| 15 | import * as FileSystem from 'effect/FileSystem'; |
| 16 | import * as BunFileSystem from '@effect/platform-bun/BunFileSystem'; |
| 17 | import * as BunRuntime from '@effect/platform-bun/BunRuntime'; |
| 18 | import { TOOLKIT_SLUG_PATTERN } from 'src/models/toolkits'; |
| 19 | import { teardown } from './_teardown'; |
| 20 | |
| 21 | /** |
| 22 | * Usage: `pnpm --filter @composio/cli generate:toolkit-slugs`. |
| 23 | * |
| 24 | * Refreshes `src/generated/toolkit-slugs.ts` — the toolkit slugs the CLI knows |
| 25 | * about without asking the API. Resolving a tool slug to its toolkit needs the |
| 26 | * catalog (`GOOGLE_ANALYTICS_RUN_REPORT` belongs to `google_analytics`, not |
| 27 | * `google`), and downloading ~800 KB of toolkit metadata to learn ~11 KB of |
| 28 | * slugs is most of the wall time of a `composio execute`. |
| 29 | * |
| 30 | * Authentication, in order of preference: |
| 31 | * - `COMPOSIO_USER_API_KEY` (+ `COMPOSIO_ORG_ID`) — a `uak_` key, what a |
| 32 | * logged-in developer has locally. |
| 33 | * - `COMPOSIO_API_KEY` — a project-scoped key, what CI has. |
| 34 | * |
| 35 | * `COMPOSIO_BASE_URL` overrides the API host; it defaults to production |
| 36 | * because the baked list ships to users of the production catalog. |
| 37 | */ |
| 38 | |
| 39 | const PRODUCTION_BASE_URL = 'https://backend.composio.dev'; |
| 40 | const PAGE_SIZE = 1000; |
| 41 | |
| 42 | /** Slugs the catalog cannot plausibly be missing. */ |
| 43 | const SENTINEL_SLUGS = ['github', 'gmail', 'google_analytics'] as const; |
| 44 | |
| 45 | /** |
| 46 | * A catalog this small means we are looking at a partial or wrong response — |
| 47 | * an empty page, a filtered org view, an error body that happened to decode. |
| 48 | * Today's production catalog holds ~1070 toolkits and the backend never |
| 49 | * removes one, so anything below this is a bad fetch, not a shrunken catalog. |
| 50 | */ |
| 51 | const MIN_EXPECTED_SLUGS = 1000; |
| 52 | |
| 53 | const ToolkitsPage = Schema.Struct({ |
| 54 | items: Schema.Array( |
| 55 | Schema.Struct({ |
| 56 | slug: Schema.String, |
| 57 | // `native` toolkits are the shared catalog. Anything else (an org-scoped |
| 58 | // custom toolkit, say) is not knowledge we can bake into a released |
| 59 | // binary for everyone. |
| 60 | type: Schema.optional(Schema.String), |
| 61 | }) |
| 62 | ), |
| 63 | next_cursor: Schema.NullishOr(Schema.String), |
| 64 | }); |
| 65 | |
| 66 | class ToolkitFetchError extends Data.TaggedError('ToolkitFetchError')<{ |
| 67 | readonly message: string; |
| 68 | }> {} |
| 69 | |
| 70 | class PrettierFormatError extends Data.TaggedError('PrettierFormatError')<{ |
| 71 | readonly message: string; |
| 72 | readonly cause: unknown; |
| 73 | }> {} |
| 74 | |
| 75 | const authHeaders: Effect.Effect< |
| 76 | Record<string, string>, |
| 77 | Config.ConfigError | ToolkitFetchError |
| 78 | > = Effect.gen(function* () { |
| 79 | const userApiKey = yield* Config.option(Config.String('COMPOSIO_USER_API_KEY')); |
| 80 | const orgId = yield* Config.option(Config.String('COMPOSIO_ORG_ID')); |
| 81 | const apiKey = yield* Config.option(Config.String('COMPOSIO_API_KEY')); |
| 82 | |
| 83 | if (Option.isSome(userApiKey)) { |
| 84 | // Built imperatively: a conditional spread of `{ 'x-org-id': string }` |
| 85 | // widens the property to `string | undefined`, which is not assignable |
| 86 | // to a `Record<string, string>` index signature under |
| 87 | // exactOptionalPropertyTypes. |
| 88 | const headers: Record<string, string> = { 'x-user-api-key': userApiKey.value }; |
| 89 | if (Option.isSome(orgId)) { |
| 90 | headers['x-org-id'] = orgId.value; |
| 91 | } |
| 92 | return headers; |
| 93 | } |
| 94 | |
| 95 | if (Option.isSome(apiKey)) { |
| 96 | return { 'x-api-key': apiKey.value }; |
| 97 | } |
| 98 | |
| 99 | return yield* new ToolkitFetchError({ |
| 100 | message: 'Set COMPOSIO_API_KEY, or COMPOSIO_USER_API_KEY (+ COMPOSIO_ORG_ID), and retry.', |
| 101 | }); |
| 102 | }); |
| 103 | |
| 104 | type ToolkitsPageDecoded = Schema.Schema.Type<typeof ToolkitsPage>; |
| 105 | |
| 106 | // The explicit return type is load-bearing: leaving it inferred makes the |
| 107 | // `const result = yield* fetchPage(...)` binding in `fetchAllSlugs` a |
| 108 | // self-referential inference cycle (TS7022). |
| 109 | const fetchPage = (params: { |
| 110 | baseUrl: string; |
| 111 | headers: Record<string, string>; |
| 112 | cursor?: string; |
| 113 | }): Effect.Effect<ToolkitsPageDecoded, ToolkitFetchError> => |
| 114 | Effect.gen(function* () { |
| 115 | const url = new URL('/api/v3/toolkits', params.baseUrl); |
| 116 | url.searchParams.set('limit', String(PAGE_SIZE)); |
| 117 | if (params.cursor) { |
| 118 | url.searchParams.set('cursor', params.cursor); |
| 119 | } |
| 120 | |
| 121 | const response = yield* Effect.tryPromise({ |
| 122 | try: () => fetch(url, { headers: params.headers }), |
| 123 | catch: cause => new ToolkitFetchError({ message: `GET ${url.pathname} failed: ${cause}` }), |
| 124 | }); |
| 125 | |
| 126 | if (!response.ok) { |
| 127 | // Reading the error body can itself fail (connection cut mid-response), |
| 128 | // so it must not be wrapped with Effect.promise, which would turn that |
| 129 | // failure into an uninterruptible-by-type defect. |
| 130 | const body = yield* Effect.tryPromise({ |
| 131 | try: () => response.text(), |
| 132 | catch: cause => |
| 133 | new ToolkitFetchError({ |
| 134 | message: `GET ${url.pathname} returned ${response.status}, and reading the error body failed: ${cause}`, |
| 135 | }), |
| 136 | }); |
| 137 | return yield* new ToolkitFetchError({ |
| 138 | message: `GET ${url.pathname} returned ${response.status}: ${body.slice(0, 400)}`, |
| 139 | }); |
| 140 | } |
| 141 | |
| 142 | const payload = yield* Effect.tryPromise({ |
| 143 | try: () => response.json(), |
| 144 | catch: cause => new ToolkitFetchError({ message: `Response was not JSON: ${cause}` }), |
| 145 | }); |
| 146 | |
| 147 | return yield* Schema.decodeUnknownEffect(ToolkitsPage)(payload).pipe( |
| 148 | Effect.mapError(cause => new ToolkitFetchError({ message: `Unexpected response: ${cause}` })) |
| 149 | ); |
| 150 | }); |
| 151 | |
| 152 | /** Walks the cursor to the end of the catalog. */ |
| 153 | const fetchAllSlugs = (params: { baseUrl: string; headers: Record<string, string> }) => |
| 154 | Effect.gen(function* () { |
| 155 | const slugs: string[] = []; |
| 156 | let cursor: string | undefined = undefined; |
| 157 | let page = 0; |
| 158 | |
| 159 | do { |
| 160 | // Explicit annotation: the inferred binding trips TS7022 through the |
| 161 | // cursor feedback loop below when `fetchPage`'s type has to be |
| 162 | // resolved from this very generator. |
| 163 | const result: ToolkitsPageDecoded = yield* fetchPage({ ...params, cursor }); |
| 164 | page += 1; |
| 165 | |
| 166 | for (const toolkit of result.items) { |
| 167 | if (toolkit.type === undefined || toolkit.type === 'native') { |
| 168 | slugs.push(toolkit.slug.toLowerCase()); |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | yield* Effect.logInfo(`Page ${page}: ${result.items.length} toolkits (${slugs.length} kept)`); |
| 173 | cursor = result.next_cursor ?? undefined; |
| 174 | } while (cursor); |
| 175 | |
| 176 | return [...new Set(slugs)].sort(); |
| 177 | }); |
| 178 | |
| 179 | /** |
| 180 | * Refuses to overwrite a good list with a bad fetch. Every failure mode we can |
| 181 | * cheaply detect — truncated pagination, an org-scoped view, a decoded error |
| 182 | * body — shows up as too few slugs, malformed slugs, or missing staples. |
| 183 | */ |
| 184 | const checkSanity = (slugs: ReadonlyArray<string>) => |
| 185 | Effect.gen(function* () { |
| 186 | if (slugs.length < MIN_EXPECTED_SLUGS) { |
| 187 | return yield* new ToolkitFetchError({ |
| 188 | message: `Refusing to write ${slugs.length} slugs; expected at least ${MIN_EXPECTED_SLUGS}.`, |
| 189 | }); |
| 190 | } |
| 191 | |
| 192 | const malformed = slugs.filter(slug => !TOOLKIT_SLUG_PATTERN.test(slug)); |
| 193 | if (malformed.length > 0) { |
| 194 | return yield* new ToolkitFetchError({ |
| 195 | message: `Refusing to write malformed slugs: ${malformed.slice(0, 10).join(', ')}`, |
| 196 | }); |
| 197 | } |
| 198 | |
| 199 | const missing = SENTINEL_SLUGS.filter(sentinel => !slugs.includes(sentinel)); |
| 200 | if (missing.length > 0) { |
| 201 | return yield* new ToolkitFetchError({ |
| 202 | message: `Refusing to write a catalog without ${missing.join(', ')}.`, |
| 203 | }); |
| 204 | } |
| 205 | }); |
| 206 | |
| 207 | const renderModule = (params: { slugs: ReadonlyArray<string>; refreshedAt: string }) => |
| 208 | `// Generated by scripts/generate-toolkit-slugs.ts — do not edit by hand. |
| 209 | // Refresh: pnpm --filter @composio/cli generate:toolkit-slugs |
| 210 | |
| 211 | /** |
| 212 | * Toolkit slugs known at build time, sorted. Resolving a tool slug to its |
| 213 | * toolkit is a longest-prefix match against the known catalog, and this list |
| 214 | * answers that without a network call. Toolkits released after this snapshot |
| 215 | * are picked up at runtime; the backend never removes a toolkit, so an entry |
| 216 | * here never goes stale in the other direction. |
| 217 | */ |
| 218 | export const BAKED_TOOLKIT_SLUGS: ReadonlyArray<string> = [ |
| 219 | ${params.slugs.map(slug => ` '${slug}',`).join('\n')} |
| 220 | ]; |
| 221 | |
| 222 | /** When the list above was generated. */ |
| 223 | export const BAKED_TOOLKIT_SLUGS_REFRESHED_AT = '${params.refreshedAt}'; |
| 224 | `; |
| 225 | |
| 226 | export function generateToolkitSlugs() { |
| 227 | return Effect.gen(function* () { |
| 228 | const fs = yield* FileSystem.FileSystem; |
| 229 | const baseUrl = yield* Config.String('COMPOSIO_BASE_URL').pipe( |
| 230 | Config.withDefault(PRODUCTION_BASE_URL) |
| 231 | ); |
| 232 | const headers = yield* authHeaders; |
| 233 | |
| 234 | yield* Effect.logInfo(`Fetching the toolkit catalog from ${baseUrl}`); |
| 235 | const slugs = yield* fetchAllSlugs({ baseUrl, headers }); |
| 236 | yield* checkSanity(slugs); |
| 237 | |
| 238 | const refreshedAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)); |
| 239 | const outputPath = path.join(process.cwd(), 'src', 'generated', 'toolkit-slugs.ts'); |
| 240 | |
| 241 | yield* fs.makeDirectory(path.dirname(outputPath), { recursive: true }); |
| 242 | yield* fs.writeFileString(outputPath, renderModule({ slugs, refreshedAt })); |
| 243 | // Bun shell promises reject on a non-zero exit, so this is a fallible |
| 244 | // Effect, not an Effect.promise defect. |
| 245 | yield* Effect.tryPromise({ |
| 246 | try: () => $`pnpm exec prettier --write ${outputPath}`.quiet(), |
| 247 | catch: cause => |
| 248 | new PrettierFormatError({ |
| 249 | message: `prettier --write ${outputPath} failed`, |
| 250 | cause, |
| 251 | }), |
| 252 | }); |
| 253 | |
| 254 | yield* Console.log(`Wrote ${slugs.length} toolkit slugs to ${outputPath}`); |
| 255 | }); |
| 256 | } |
| 257 | |
| 258 | if (require.main === module) { |
| 259 | generateToolkitSlugs().pipe( |
| 260 | Effect.provide(Logger.layer([Logger.consolePretty()])), |
| 261 | Effect.provide(BunFileSystem.layer), |
| 262 | Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv())), |
| 263 | Effect.scoped, |
| 264 | BunRuntime.runMain({ teardown }) |
| 265 | ); |
| 266 | } |