@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 266 lines 9.65 KB
Raw
1 import process from 'node:process';
2 import path from 'node:path';
3 import { $ } from 'bun';
4 import {
5 Config,
6 ConfigProvider,
7 Console,
8 Data,
9 DateTime,
10 Effect,
11 Logger,
12 Option,
13 Schema,
14 } from 'effect';
15 import * as FileSystem from 'effect/FileSystem';
16 import * as BunFileSystem from '@effect/platform-bun/BunFileSystem';
17 import * as BunRuntime from '@effect/platform-bun/BunRuntime';
18 import { TOOLKIT_SLUG_PATTERN } from 'src/models/toolkits';
19 import { teardown } from './_teardown';
20
21 /**
22 * Usage: `pnpm --filter @composio/cli generate:toolkit-slugs`.
23 *
24 * Refreshes `src/generated/toolkit-slugs.ts` — the toolkit slugs the CLI knows
25 * about without asking the API. Resolving a tool slug to its toolkit needs the
26 * catalog (`GOOGLE_ANALYTICS_RUN_REPORT` belongs to `google_analytics`, not
27 * `google`), and downloading ~800 KB of toolkit metadata to learn ~11 KB of
28 * slugs is most of the wall time of a `composio execute`.
29 *
30 * Authentication, in order of preference:
31 * - `COMPOSIO_USER_API_KEY` (+ `COMPOSIO_ORG_ID`) — a `uak_` key, what a
32 * logged-in developer has locally.
33 * - `COMPOSIO_API_KEY` — a project-scoped key, what CI has.
34 *
35 * `COMPOSIO_BASE_URL` overrides the API host; it defaults to production
36 * because the baked list ships to users of the production catalog.
37 */
38
39 const PRODUCTION_BASE_URL = 'https://backend.composio.dev';
40 const PAGE_SIZE = 1000;
41
42 /** Slugs the catalog cannot plausibly be missing. */
43 const SENTINEL_SLUGS = ['github', 'gmail', 'google_analytics'] as const;
44
45 /**
46 * A catalog this small means we are looking at a partial or wrong response —
47 * an empty page, a filtered org view, an error body that happened to decode.
48 * Today's production catalog holds ~1070 toolkits and the backend never
49 * removes one, so anything below this is a bad fetch, not a shrunken catalog.
50 */
51 const MIN_EXPECTED_SLUGS = 1000;
52
53 const ToolkitsPage = Schema.Struct({
54 items: Schema.Array(
55 Schema.Struct({
56 slug: Schema.String,
57 // `native` toolkits are the shared catalog. Anything else (an org-scoped
58 // custom toolkit, say) is not knowledge we can bake into a released
59 // binary for everyone.
60 type: Schema.optional(Schema.String),
61 })
62 ),
63 next_cursor: Schema.NullishOr(Schema.String),
64 });
65
66 class ToolkitFetchError extends Data.TaggedError('ToolkitFetchError')<{
67 readonly message: string;
68 }> {}
69
70 class PrettierFormatError extends Data.TaggedError('PrettierFormatError')<{
71 readonly message: string;
72 readonly cause: unknown;
73 }> {}
74
75 const authHeaders: Effect.Effect<
76 Record<string, string>,
77 Config.ConfigError | ToolkitFetchError
78 > = Effect.gen(function* () {
79 const userApiKey = yield* Config.option(Config.String('COMPOSIO_USER_API_KEY'));
80 const orgId = yield* Config.option(Config.String('COMPOSIO_ORG_ID'));
81 const apiKey = yield* Config.option(Config.String('COMPOSIO_API_KEY'));
82
83 if (Option.isSome(userApiKey)) {
84 // Built imperatively: a conditional spread of `{ 'x-org-id': string }`
85 // widens the property to `string | undefined`, which is not assignable
86 // to a `Record<string, string>` index signature under
87 // exactOptionalPropertyTypes.
88 const headers: Record<string, string> = { 'x-user-api-key': userApiKey.value };
89 if (Option.isSome(orgId)) {
90 headers['x-org-id'] = orgId.value;
91 }
92 return headers;
93 }
94
95 if (Option.isSome(apiKey)) {
96 return { 'x-api-key': apiKey.value };
97 }
98
99 return yield* new ToolkitFetchError({
100 message: 'Set COMPOSIO_API_KEY, or COMPOSIO_USER_API_KEY (+ COMPOSIO_ORG_ID), and retry.',
101 });
102 });
103
104 type ToolkitsPageDecoded = Schema.Schema.Type<typeof ToolkitsPage>;
105
106 // The explicit return type is load-bearing: leaving it inferred makes the
107 // `const result = yield* fetchPage(...)` binding in `fetchAllSlugs` a
108 // self-referential inference cycle (TS7022).
109 const fetchPage = (params: {
110 baseUrl: string;
111 headers: Record<string, string>;
112 cursor?: string;
113 }): Effect.Effect<ToolkitsPageDecoded, ToolkitFetchError> =>
114 Effect.gen(function* () {
115 const url = new URL('/api/v3/toolkits', params.baseUrl);
116 url.searchParams.set('limit', String(PAGE_SIZE));
117 if (params.cursor) {
118 url.searchParams.set('cursor', params.cursor);
119 }
120
121 const response = yield* Effect.tryPromise({
122 try: () => fetch(url, { headers: params.headers }),
123 catch: cause => new ToolkitFetchError({ message: `GET ${url.pathname} failed: ${cause}` }),
124 });
125
126 if (!response.ok) {
127 // Reading the error body can itself fail (connection cut mid-response),
128 // so it must not be wrapped with Effect.promise, which would turn that
129 // failure into an uninterruptible-by-type defect.
130 const body = yield* Effect.tryPromise({
131 try: () => response.text(),
132 catch: cause =>
133 new ToolkitFetchError({
134 message: `GET ${url.pathname} returned ${response.status}, and reading the error body failed: ${cause}`,
135 }),
136 });
137 return yield* new ToolkitFetchError({
138 message: `GET ${url.pathname} returned ${response.status}: ${body.slice(0, 400)}`,
139 });
140 }
141
142 const payload = yield* Effect.tryPromise({
143 try: () => response.json(),
144 catch: cause => new ToolkitFetchError({ message: `Response was not JSON: ${cause}` }),
145 });
146
147 return yield* Schema.decodeUnknownEffect(ToolkitsPage)(payload).pipe(
148 Effect.mapError(cause => new ToolkitFetchError({ message: `Unexpected response: ${cause}` }))
149 );
150 });
151
152 /** Walks the cursor to the end of the catalog. */
153 const fetchAllSlugs = (params: { baseUrl: string; headers: Record<string, string> }) =>
154 Effect.gen(function* () {
155 const slugs: string[] = [];
156 let cursor: string | undefined = undefined;
157 let page = 0;
158
159 do {
160 // Explicit annotation: the inferred binding trips TS7022 through the
161 // cursor feedback loop below when `fetchPage`'s type has to be
162 // resolved from this very generator.
163 const result: ToolkitsPageDecoded = yield* fetchPage({ ...params, cursor });
164 page += 1;
165
166 for (const toolkit of result.items) {
167 if (toolkit.type === undefined || toolkit.type === 'native') {
168 slugs.push(toolkit.slug.toLowerCase());
169 }
170 }
171
172 yield* Effect.logInfo(`Page ${page}: ${result.items.length} toolkits (${slugs.length} kept)`);
173 cursor = result.next_cursor ?? undefined;
174 } while (cursor);
175
176 return [...new Set(slugs)].sort();
177 });
178
179 /**
180 * Refuses to overwrite a good list with a bad fetch. Every failure mode we can
181 * cheaply detect — truncated pagination, an org-scoped view, a decoded error
182 * body — shows up as too few slugs, malformed slugs, or missing staples.
183 */
184 const checkSanity = (slugs: ReadonlyArray<string>) =>
185 Effect.gen(function* () {
186 if (slugs.length < MIN_EXPECTED_SLUGS) {
187 return yield* new ToolkitFetchError({
188 message: `Refusing to write ${slugs.length} slugs; expected at least ${MIN_EXPECTED_SLUGS}.`,
189 });
190 }
191
192 const malformed = slugs.filter(slug => !TOOLKIT_SLUG_PATTERN.test(slug));
193 if (malformed.length > 0) {
194 return yield* new ToolkitFetchError({
195 message: `Refusing to write malformed slugs: ${malformed.slice(0, 10).join(', ')}`,
196 });
197 }
198
199 const missing = SENTINEL_SLUGS.filter(sentinel => !slugs.includes(sentinel));
200 if (missing.length > 0) {
201 return yield* new ToolkitFetchError({
202 message: `Refusing to write a catalog without ${missing.join(', ')}.`,
203 });
204 }
205 });
206
207 const renderModule = (params: { slugs: ReadonlyArray<string>; refreshedAt: string }) =>
208 `// Generated by scripts/generate-toolkit-slugs.ts — do not edit by hand.
209 // Refresh: pnpm --filter @composio/cli generate:toolkit-slugs
210
211 /**
212 * Toolkit slugs known at build time, sorted. Resolving a tool slug to its
213 * toolkit is a longest-prefix match against the known catalog, and this list
214 * answers that without a network call. Toolkits released after this snapshot
215 * are picked up at runtime; the backend never removes a toolkit, so an entry
216 * here never goes stale in the other direction.
217 */
218 export const BAKED_TOOLKIT_SLUGS: ReadonlyArray<string> = [
219 ${params.slugs.map(slug => ` '${slug}',`).join('\n')}
220 ];
221
222 /** When the list above was generated. */
223 export const BAKED_TOOLKIT_SLUGS_REFRESHED_AT = '${params.refreshedAt}';
224 `;
225
226 export function generateToolkitSlugs() {
227 return Effect.gen(function* () {
228 const fs = yield* FileSystem.FileSystem;
229 const baseUrl = yield* Config.String('COMPOSIO_BASE_URL').pipe(
230 Config.withDefault(PRODUCTION_BASE_URL)
231 );
232 const headers = yield* authHeaders;
233
234 yield* Effect.logInfo(`Fetching the toolkit catalog from ${baseUrl}`);
235 const slugs = yield* fetchAllSlugs({ baseUrl, headers });
236 yield* checkSanity(slugs);
237
238 const refreshedAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
239 const outputPath = path.join(process.cwd(), 'src', 'generated', 'toolkit-slugs.ts');
240
241 yield* fs.makeDirectory(path.dirname(outputPath), { recursive: true });
242 yield* fs.writeFileString(outputPath, renderModule({ slugs, refreshedAt }));
243 // Bun shell promises reject on a non-zero exit, so this is a fallible
244 // Effect, not an Effect.promise defect.
245 yield* Effect.tryPromise({
246 try: () => $`pnpm exec prettier --write ${outputPath}`.quiet(),
247 catch: cause =>
248 new PrettierFormatError({
249 message: `prettier --write ${outputPath} failed`,
250 cause,
251 }),
252 });
253
254 yield* Console.log(`Wrote ${slugs.length} toolkit slugs to ${outputPath}`);
255 });
256 }
257
258 if (require.main === module) {
259 generateToolkitSlugs().pipe(
260 Effect.provide(Logger.layer([Logger.consolePretty()])),
261 Effect.provide(BunFileSystem.layer),
262 Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv())),
263 Effect.scoped,
264 BunRuntime.runMain({ teardown })
265 );
266 }