Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 511 lines 16.1 KB
Raw
1 import * as FileSystem from 'effect/FileSystem';
2 import * as Path from 'effect/Path';
3 import { Data, Effect, Option, Predicate, Schema, SchemaIssue } from 'effect';
4 import {
5 jsonSchemaToEffectSchema,
6 type JsonSchemaValidationIssue,
7 } from '@composio/json-schema-to-effect-schema';
8 import { JsonRecordSchema } from 'src/effects/json';
9 import { setupCacheDir } from 'src/effects/setup-cache-dir';
10 import { ComposioToolkitsRepository } from 'src/services/composio-clients';
11 import { getLatestToolVersion } from 'src/services/composio-account';
12 import { memoizeInProcess } from 'src/utils/memoize-in-process';
13 import { logToolDebug } from 'src/services/runtime-debug-logger';
14 import { normalizeFileUploadSchema } from 'src/services/tool-file-uploads';
15 import { ComposioUserContext } from 'src/services/user-context';
16
17 const TOOL_DEFINITIONS_DIR = 'tool_definitions';
18
19 type CachedToolInputDefinition = {
20 readonly version: string | null;
21 readonly inputSchema: Record<string, unknown>;
22 };
23
24 const CachedToolInputDefinitionEnvelope = Schema.Struct({
25 version: Schema.optional(Schema.Unknown),
26 inputSchema: JsonRecordSchema,
27 });
28 const ObjectSchemaWithProperties = Schema.Struct({ properties: JsonRecordSchema });
29
30 const decodeJsonObject = Schema.decodeUnknownEffect(Schema.fromJsonString(JsonRecordSchema));
31 const decodeCachedToolInputDefinitionEnvelope = Schema.decodeUnknownOption(
32 CachedToolInputDefinitionEnvelope
33 );
34 const decodeObjectSchemaWithProperties = Schema.decodeUnknownOption(ObjectSchemaWithProperties);
35
36 const sanitizeToolSlug = (slug: string) => slug.replace(/[^A-Za-z0-9_.-]/g, '_');
37
38 const toolDefinitionPath = (path: Path.Path, cacheDir: string, slug: string) =>
39 path.join(cacheDir, TOOL_DEFINITIONS_DIR, `${sanitizeToolSlug(slug)}.json`);
40
41 const ensureToolDefinitionsDir = (fs: FileSystem.FileSystem, path: Path.Path, cacheDir: string) =>
42 fs.makeDirectory(path.join(cacheDir, TOOL_DEFINITIONS_DIR), { recursive: true });
43
44 const parseSchemaFile = (raw: string, schemaPath: string) =>
45 decodeJsonObject(raw).pipe(
46 Effect.mapError(
47 cause => new Error(`Cached tool schema at ${schemaPath} is not valid JSON.`, { cause })
48 )
49 );
50
51 const PLACEHOLDER_TOOL_VERSION = '00000000_00';
52
53 const selectLatestVersion = (versions: ReadonlyArray<string> | undefined): string | null => {
54 if (!versions || versions.length === 0) {
55 return null;
56 }
57
58 for (const version of versions) {
59 if (
60 typeof version === 'string' &&
61 version.trim().length > 0 &&
62 version !== PLACEHOLDER_TOOL_VERSION
63 ) {
64 return version;
65 }
66 }
67
68 return versions.find(version => typeof version === 'string' && version.trim().length > 0) ?? null;
69 };
70
71 const resolveLatestAvailableVersion = (params: {
72 readonly toolLatestVersion: string | null;
73 readonly toolkitLatestVersion: string | null;
74 }): string | null => {
75 if (
76 params.toolLatestVersion &&
77 params.toolLatestVersion.trim().length > 0 &&
78 params.toolLatestVersion !== PLACEHOLDER_TOOL_VERSION
79 ) {
80 return params.toolLatestVersion;
81 }
82
83 return params.toolkitLatestVersion;
84 };
85
86 const parseCachedToolDefinition = (parsed: Record<string, unknown>): CachedToolInputDefinition => {
87 const envelope = decodeCachedToolInputDefinitionEnvelope(parsed);
88 if (Option.isSome(envelope)) {
89 return {
90 version: typeof envelope.value.version === 'string' ? envelope.value.version : null,
91 inputSchema: envelope.value.inputSchema,
92 };
93 }
94
95 // Backward compatibility for previously cached bare-schema files.
96 return {
97 version: null,
98 inputSchema: parsed,
99 };
100 };
101
102 const serializeCachedToolDefinition = (definition: CachedToolInputDefinition): string =>
103 JSON.stringify(
104 {
105 version: definition.version,
106 inputSchema: definition.inputSchema,
107 },
108 null,
109 2
110 );
111
112 export const getCachedToolInputDefinition = (slug: string) =>
113 Effect.gen(function* () {
114 const fs = yield* FileSystem.FileSystem;
115 const path = yield* Path.Path;
116 const cacheDir = yield* setupCacheDir;
117 const schemaPath = toolDefinitionPath(path, cacheDir, slug);
118
119 const raw = yield* fs
120 .readFileString(schemaPath, 'utf8')
121 .pipe(
122 Effect.catchTag('PlatformError', error =>
123 Predicate.isTagged(error.reason, 'NotFound') ? Effect.succeed(null) : Effect.fail(error)
124 )
125 );
126 if (raw === null) {
127 return null;
128 }
129
130 const parsed = yield* parseSchemaFile(raw, schemaPath);
131 const cached = parseCachedToolDefinition(parsed);
132 return {
133 schemaPath,
134 schema: cached.inputSchema,
135 version: cached.version,
136 };
137 });
138
139 export const cacheToolInputDefinition = (params: {
140 readonly slug: string;
141 readonly schema: Record<string, unknown>;
142 readonly version?: string | null;
143 }) =>
144 Effect.gen(function* () {
145 const fs = yield* FileSystem.FileSystem;
146 const path = yield* Path.Path;
147 const cacheDir = yield* setupCacheDir;
148 const schemaPath = toolDefinitionPath(path, cacheDir, params.slug);
149
150 yield* ensureToolDefinitionsDir(fs, path, cacheDir);
151 yield* fs.writeFileString(
152 schemaPath,
153 serializeCachedToolDefinition({
154 version: params.version ?? null,
155 inputSchema: params.schema,
156 })
157 );
158
159 return {
160 schemaPath,
161 schema: params.schema,
162 version: params.version ?? null,
163 };
164 });
165
166 export const invalidateToolInputDefinition = (slug: string) =>
167 Effect.gen(function* () {
168 const fs = yield* FileSystem.FileSystem;
169 const path = yield* Path.Path;
170 const cacheDir = yield* setupCacheDir;
171 const schemaPath = toolDefinitionPath(path, cacheDir, slug);
172 yield* fs
173 .remove(schemaPath)
174 .pipe(
175 Effect.catchTag('PlatformError', error =>
176 Predicate.isTagged(error.reason, 'NotFound') ? Effect.void : Effect.fail(error)
177 )
178 );
179 });
180
181 // One `get_latest_version` round trip per tool per process. `composio execute`
182 // asks twice on every call: the validation that gates the tool call in the
183 // command, and `getOrFetchToolInputDefinition` on the executor's file-upload
184 // path. Both want the same answer within the same second.
185 const fetchLatestToolVersionOnce = memoizeInProcess({
186 keyOf: (input: {
187 readonly slug: string;
188 readonly apiKey: string;
189 readonly params?: { readonly orgId?: string; readonly projectId?: string };
190 }) => `${input.slug}\u0000${input.params?.orgId ?? ''}\u0000${input.params?.projectId ?? ''}`,
191 make: ({ slug, apiKey, params }) =>
192 Effect.gen(function* () {
193 const latest = yield* getLatestToolVersion({
194 apiKey,
195 toolSlug: slug,
196 orgId: params?.orgId,
197 projectId: params?.projectId,
198 });
199 yield* logToolDebug('latest_tool_version', {
200 slug,
201 orgId: params?.orgId,
202 projectId: params?.projectId,
203 response: latest,
204 });
205 return latest.version;
206 }),
207 });
208
209 // The API-key check stays outside the memo: the user context is live state
210 // that `login` can fill in later in the same process, and caching the `null`
211 // answered before that would silence the version check for the rest of it.
212 const fetchResolvedLatestToolVersion = (input: {
213 readonly slug: string;
214 readonly params?: { readonly orgId?: string; readonly projectId?: string };
215 }) =>
216 Effect.gen(function* () {
217 const userContext = yield* ComposioUserContext;
218 const apiKey = Option.getOrUndefined(userContext.data.apiKey);
219 if (!apiKey) {
220 return null;
221 }
222 return yield* fetchLatestToolVersionOnce({ ...input, apiKey });
223 });
224
225 const fetchAndCacheToolInputDefinition = (
226 slug: string,
227 params?: { readonly orgId?: string; readonly projectId?: string }
228 ) =>
229 Effect.gen(function* () {
230 const fs = yield* FileSystem.FileSystem;
231 const path = yield* Path.Path;
232 const repo = yield* ComposioToolkitsRepository;
233 const cacheDir = yield* setupCacheDir;
234 const schemaPath = toolDefinitionPath(path, cacheDir, slug);
235 yield* ensureToolDefinitionsDir(fs, path, cacheDir);
236
237 const [tool, latestVersion] = yield* Effect.all(
238 [
239 repo.getToolDetailed(slug),
240 fetchResolvedLatestToolVersion({ slug, params }).pipe(
241 Effect.catch(() => Effect.succeed(null))
242 ),
243 ],
244 { concurrency: 2 }
245 );
246 yield* logToolDebug('tool_detail', {
247 slug,
248 tool,
249 });
250 const schema = tool.input_parameters;
251 const version =
252 latestVersion ??
253 resolveLatestAvailableVersion({
254 toolLatestVersion: selectLatestVersion(tool.available_versions),
255 toolkitLatestVersion: null,
256 });
257 yield* logToolDebug('resolved_tool_version', {
258 slug,
259 resolvedVersion: version,
260 cachePath: schemaPath,
261 });
262 yield* fs.writeFileString(
263 schemaPath,
264 serializeCachedToolDefinition({
265 version,
266 inputSchema: schema,
267 })
268 );
269
270 return { schemaPath, schema, version };
271 });
272
273 export const getOrFetchToolInputDefinition = (
274 slug: string,
275 params?: { readonly orgId?: string; readonly projectId?: string }
276 ) =>
277 Effect.gen(function* () {
278 const cached = yield* getCachedToolInputDefinition(slug);
279 if (!cached) {
280 return yield* fetchAndCacheToolInputDefinition(slug, params);
281 }
282
283 const freshness = yield* refreshAndFetchToolInputDefinitionIfVersionChanged(
284 slug,
285 cached.version,
286 params
287 ).pipe(
288 Effect.catch(() =>
289 Effect.succeed({
290 isStale: false,
291 latestVersion: cached.version,
292 definition: null,
293 skipped: false as const,
294 })
295 )
296 );
297
298 if (!freshness.isStale) {
299 return cached;
300 }
301
302 return freshness.definition ?? cached;
303 });
304
305 // Returns the freshly fetched definition on the stale path so callers that
306 // need it (getOrFetchToolInputDefinition) don't re-read the cache file that
307 // fetchAndCacheToolInputDefinition just wrote.
308 const refreshAndFetchToolInputDefinitionIfVersionChanged = (
309 slug: string,
310 cachedVersion: string | null,
311 params?: { readonly orgId?: string; readonly projectId?: string }
312 ) =>
313 Effect.gen(function* () {
314 const latestVersion = yield* fetchResolvedLatestToolVersion({ slug, params });
315 yield* logToolDebug('resolved_tool_version', {
316 slug,
317 mode: 'refresh',
318 cachedVersion,
319 resolvedVersion: latestVersion,
320 orgId: params?.orgId,
321 projectId: params?.projectId,
322 });
323 const isStale = latestVersion !== cachedVersion;
324 const definition = isStale ? yield* fetchAndCacheToolInputDefinition(slug, params) : null;
325
326 return { isStale, latestVersion, definition, skipped: false as const };
327 });
328
329 export class ToolInputValidationError extends Data.TaggedError('ToolInputValidationError')<{
330 readonly toolSlug: string;
331 readonly schemaPath: string;
332 readonly issues: ReadonlyArray<string>;
333 readonly cause?: unknown;
334 }> {
335 override get message(): string {
336 return [
337 `Input validation failed for ${this.toolSlug}.`,
338 `Schema: ${this.schemaPath}`,
339 ...this.issues.map(issue => `- ${issue}`),
340 ].join('\n');
341 }
342 }
343
344 // The schema itself is unusable, which says nothing about the arguments. Kept
345 // apart from `ToolInputValidationError` so it never blocks a tool call.
346 export class ToolInputSchemaCompileError extends Data.TaggedError('ToolInputSchemaCompileError')<{
347 readonly toolSlug: string;
348 readonly schemaPath: string;
349 readonly cause?: unknown;
350 }> {
351 override get message(): string {
352 return [
353 `Could not compile the cached JSON schema for ${this.toolSlug} into a validator.`,
354 `Schema: ${this.schemaPath}`,
355 ].join('\n');
356 }
357 }
358
359 const getObjectSchemaProperties = (schema: Record<string, unknown>): ReadonlyArray<string> => {
360 const objectSchema = decodeObjectSchemaWithProperties(schema);
361 return Option.match(objectSchema, {
362 onNone: () => [],
363 onSome: ({ properties }) => Object.keys(properties),
364 });
365 };
366
367 const normalizeKey = (value: string) => value.toLowerCase().replace(/[^a-z0-9]/g, '');
368
369 /** Simple Levenshtein distance (small N, no perf worries). */
370 const levenshteinDistance = (a: string, b: string): number => {
371 const m = a.length;
372 const n = b.length;
373 const dp: Array<Array<number>> = Array.from({ length: m + 1 }, () =>
374 Array.from({ length: n + 1 }, () => 0)
375 );
376 for (let i = 0; i <= m; i++) dp[i][0] = i;
377 for (let j = 0; j <= n; j++) dp[0][j] = j;
378 for (let i = 1; i <= m; i++) {
379 for (let j = 1; j <= n; j++) {
380 const cost = a[i - 1] === b[j - 1] ? 0 : 1;
381 dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + cost);
382 }
383 }
384 return dp[m][n];
385 };
386
387 type SchemaKeyCandidate = {
388 readonly key: string;
389 readonly score: number;
390 };
391
392 const findClosestSchemaKey = (
393 unknownKey: string,
394 allowedKeys: ReadonlyArray<string>
395 ): string | undefined => {
396 const normalizedUnknownKey = normalizeKey(unknownKey);
397 if (!normalizedUnknownKey) {
398 return undefined;
399 }
400
401 const closest = allowedKeys.reduce<SchemaKeyCandidate | undefined>((best, key) => {
402 const normalizedKey = normalizeKey(key);
403 const distance = levenshteinDistance(normalizedUnknownKey, normalizedKey);
404 const containsBonus =
405 normalizedKey.includes(normalizedUnknownKey) || normalizedUnknownKey.includes(normalizedKey)
406 ? -2
407 : 0;
408 const candidate = {
409 key,
410 score: distance + containsBonus,
411 };
412
413 if (!best || candidate.score < best.score) {
414 return candidate;
415 }
416
417 return best;
418 }, undefined);
419
420 const threshold = Math.max(3, Math.ceil(normalizedUnknownKey.length * 0.6));
421 if (!closest || closest.score > threshold) {
422 return undefined;
423 }
424
425 return closest.key;
426 };
427
428 const formatUnknownKeyIssue = (
429 unknownKeys: ReadonlyArray<string>,
430 allowedKeys: ReadonlyArray<string>
431 ): ReadonlyArray<string> => {
432 const allowedList = allowedKeys.join(', ');
433 return unknownKeys.map(key => {
434 const suggestion = findClosestSchemaKey(key, allowedKeys);
435 const lines = [`<root>: Unknown key "${key}".`];
436 if (suggestion) {
437 lines.push(`Use "${suggestion}" instead.`);
438 }
439 if (allowedList) {
440 lines.push(`Allowed top-level keys: ${allowedList}`);
441 }
442 return lines.join(' ');
443 });
444 };
445
446 const formatSchemaIssues = (
447 schemaIssues: ReadonlyArray<JsonSchemaValidationIssue>,
448 allowedKeys: ReadonlyArray<string>
449 ): ReadonlyArray<string> =>
450 schemaIssues.flatMap(issue => {
451 if (issue.code === 'unrecognized_keys' && issue.keys) {
452 return formatUnknownKeyIssue(issue.keys, allowedKeys);
453 }
454
455 const location = issue.path.length > 0 ? issue.path.join('.') : '<root>';
456 return [`${location}: ${issue.message}`];
457 });
458
459 const compileToolInputSchema = (
460 jsonSchema: Record<string, unknown>,
461 allowedKeys: ReadonlyArray<string>
462 ) =>
463 jsonSchemaToEffectSchema(jsonSchema, {
464 formatIssues: issues => formatSchemaIssues(issues, allowedKeys),
465 });
466
467 export const validateToolInputArgumentsWithDefinition = (
468 slug: string,
469 args: Record<string, unknown>,
470 definition: {
471 readonly schemaPath: string;
472 readonly schema: Record<string, unknown>;
473 }
474 ) =>
475 Effect.gen(function* () {
476 const { schemaPath, schema } = definition;
477 const allowedKeys = getObjectSchemaProperties(schema);
478 const normalizedSchema = normalizeFileUploadSchema(schema);
479
480 const inputSchema = yield* Effect.try({
481 try: () => compileToolInputSchema(normalizedSchema, allowedKeys),
482 catch: error => new ToolInputSchemaCompileError({ toolSlug: slug, schemaPath, cause: error }),
483 });
484
485 yield* Schema.decodeUnknownEffect(inputSchema, { errors: 'all' })(args).pipe(
486 Effect.mapError(error => {
487 const issues = SchemaIssue.makeFormatterStandardSchemaV1()(error.issue).issues.map(
488 issue => issue.message
489 );
490 return new ToolInputValidationError({ toolSlug: slug, schemaPath, issues, cause: error });
491 })
492 );
493
494 return { schemaPath, schema };
495 });
496
497 export const validateToolInputArguments = (
498 slug: string,
499 args: Record<string, unknown>,
500 params?: { readonly orgId?: string; readonly projectId?: string }
501 ) =>
502 Effect.gen(function* () {
503 const definition = yield* getOrFetchToolInputDefinition(slug, params);
504 return yield* validateToolInputArgumentsWithDefinition(slug, args, definition);
505 });
506
507 export const warmToolInputDefinitions = (slugs: ReadonlyArray<string>) =>
508 Effect.forEach([...new Set(slugs)].filter(Boolean), slug => getOrFetchToolInputDefinition(slug), {
509 concurrency: 'unbounded',
510 discard: true,
511 });