@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 584 lines 20.3 KB
Raw
1 import * as FileSystem from 'effect/FileSystem';
2 import * as Path from 'effect/Path';
3 import { Effect, Option, Record as EffectRecord, Schema } from 'effect';
4 import { APP_CONFIG } from 'src/config';
5 import { JsonRecordSchema } from 'src/effects/json';
6 import { setupCacheDir } from 'src/effects/setup-cache-dir';
7 import { NodeProcess } from 'src/services/node-process';
8 import { ComposioToolkitsRepository, ComposioClientSingleton } from 'src/services/composio-clients';
9 import {
10 getConsumerConnectedToolkits,
11 resolveConsumerProject,
12 } from 'src/services/composio-account';
13 import { resolveCommandProject } from 'src/services/command-project';
14 import { collectDecodedEntries } from 'src/utils/collect-decoded-entries';
15 import { djb2Hash } from 'src/utils/djb2';
16 import { resolveToolRouterSessionConnections } from 'src/services/tool-router-session-connections';
17 import { ComposioUserContext } from 'src/services/user-context';
18 import {
19 CachedConnectedAccountSummarySchema,
20 type CachedConnectedAccountSummary,
21 } from 'src/services/connected-account-selection';
22
23 const CACHE_FILE = 'consumer-short-term-cache.json';
24 const CACHE_TTL_MS = 15 * 60 * 1000;
25 const SEARCH_SESSION_EXTENSION_MS = 5 * 60 * 1000;
26
27 const StringMappingsSchema = Schema.Record(Schema.String, Schema.String);
28 const AvailableConnectedAccountsSchema = Schema.Record(
29 Schema.String,
30 Schema.Array(CachedConnectedAccountSummarySchema)
31 );
32 const ConsumerToolRouterAuthConfigMappingsSchema = Schema.Struct({
33 authConfigs: Schema.optional(StringMappingsSchema),
34 });
35 const ConsumerToolRouterConnectedAccountMappingsSchema = Schema.Struct({
36 connectedAccounts: Schema.optional(StringMappingsSchema),
37 availableConnectedAccounts: Schema.optional(AvailableConnectedAccountsSchema),
38 });
39 const CacheEntrySchema = Schema.Struct({
40 toolkits: Schema.Array(Schema.String),
41 expiresAt: Schema.String,
42 toolRouterAuthConfigs: Schema.optional(ConsumerToolRouterAuthConfigMappingsSchema),
43 toolRouterConnectedAccounts: Schema.optional(ConsumerToolRouterConnectedAccountMappingsSchema),
44 probablyMyCliSessionsByCwdHash: Schema.optional(
45 Schema.Record(Schema.String, Schema.Struct({ id: Schema.String, expiresAt: Schema.String }))
46 ),
47 });
48
49 export type ConsumerToolRouterAuthConfigMappings =
50 typeof ConsumerToolRouterAuthConfigMappingsSchema.Type;
51 export type ConsumerToolRouterConnectedAccountMappings =
52 typeof ConsumerToolRouterConnectedAccountMappingsSchema.Type;
53 type CacheEntry = typeof CacheEntrySchema.Type;
54 type CacheState = { readonly [key: string]: CacheEntry };
55 const decodeCacheShell = Schema.decodeUnknownOption(Schema.fromJsonString(JsonRecordSchema));
56 const decodeCacheEntry = Schema.decodeUnknownOption(CacheEntrySchema);
57
58 // Per-entry decode: one stale or version-skewed entry (e.g. written by a
59 // newer CLI) drops only itself; discarding the whole cache would also be
60 // persisted back on the next write, permanently destroying the good entries.
61 export const decodeCacheStateTolerant = (raw: string): CacheState =>
62 Option.match(decodeCacheShell(raw), {
63 onNone: (): CacheState => ({}),
64 onSome: shell => collectDecodedEntries(shell, decodeCacheEntry),
65 });
66
67 const cacheKey = (orgId: string, consumerUserId: string) => `${orgId}:${consumerUserId}`;
68
69 const cachePath = Effect.gen(function* () {
70 const path = yield* Path.Path;
71 const cacheDir = yield* setupCacheDir;
72 return path.join(cacheDir, CACHE_FILE);
73 });
74
75 const cwdHash = (cwd: string): string => djb2Hash(cwd).toString(36);
76
77 const createProbablyMyCliSessionId = (cwd: string): string => {
78 const random = crypto.randomUUID().replace(/-/g, '').slice(0, 10);
79 return `cli_s_${cwdHash(cwd)}_${random}`;
80 };
81
82 const resolveSearchSessionMetadata = (params: {
83 readonly currentEntry?: CacheEntry;
84 readonly cwd: string;
85 }) => {
86 const now = Date.now();
87 const currentCwdHash = cwdHash(params.cwd);
88 const previousMap = {
89 ...(params.currentEntry?.probablyMyCliSessionsByCwdHash ?? {}),
90 };
91
92 const probablyMyCliSessionsByCwdHash = EffectRecord.fromEntries(
93 EffectRecord.toEntries(previousMap).filter(([, session]) => {
94 const expiresAtMs = Date.parse(session.expiresAt);
95 return Number.isFinite(expiresAtMs) && expiresAtMs > now;
96 })
97 );
98
99 const currentSession = probablyMyCliSessionsByCwdHash[currentCwdHash];
100 if (currentSession) {
101 probablyMyCliSessionsByCwdHash[currentCwdHash] = {
102 id: currentSession.id,
103 expiresAt: new Date(
104 Math.max(now, Date.parse(currentSession.expiresAt)) + SEARCH_SESSION_EXTENSION_MS
105 ).toISOString(),
106 };
107 } else {
108 probablyMyCliSessionsByCwdHash[currentCwdHash] = {
109 id: createProbablyMyCliSessionId(params.cwd),
110 expiresAt: new Date(now + CACHE_TTL_MS).toISOString(),
111 };
112 }
113
114 return {
115 probablyMyCliSessionsByCwdHash,
116 };
117 };
118
119 const readCache = () =>
120 Effect.gen(function* () {
121 const fs = yield* FileSystem.FileSystem;
122 const filePath = yield* cachePath;
123 if (!(yield* fs.exists(filePath))) {
124 return {} satisfies CacheState;
125 }
126 const raw = yield* fs.readFileString(filePath, 'utf8');
127 return decodeCacheStateTolerant(raw);
128 });
129
130 const writeCache = (state: CacheState) =>
131 Effect.gen(function* () {
132 const fs = yield* FileSystem.FileSystem;
133 const cacheDir = yield* setupCacheDir;
134 const filePath = yield* cachePath;
135 yield* fs.makeDirectory(cacheDir, { recursive: true }).pipe(Effect.catch(() => Effect.void));
136 yield* fs
137 .writeFileString(filePath, JSON.stringify(state, null, 2))
138 .pipe(Effect.catch(() => Effect.void));
139 });
140
141 const getAlwaysConnectedNoAuthToolkits = () =>
142 Effect.gen(function* () {
143 const toolkitsRepository = yield* ComposioToolkitsRepository;
144 const toolkits = yield* toolkitsRepository.getToolkits();
145
146 return toolkits.filter(toolkit => toolkit.no_auth).map(toolkit => toolkit.slug.toLowerCase());
147 });
148
149 const normalizeCachedToolkits = (
150 toolkits: ReadonlyArray<string>,
151 noAuthToolkits: ReadonlyArray<string>
152 ) => [...new Set([...toolkits, ...noAuthToolkits].map(toolkit => toolkit.toLowerCase()))];
153
154 const normalizeAuthConfigMappings = (
155 mappings?: ConsumerToolRouterAuthConfigMappings
156 ): ConsumerToolRouterAuthConfigMappings | undefined => {
157 if (!mappings) return undefined;
158
159 const authConfigs = Object.fromEntries(
160 Object.entries(mappings.authConfigs ?? {})
161 .map(([toolkit, authConfigId]) => [toolkit.toLowerCase(), authConfigId])
162 .filter(([, authConfigId]) => typeof authConfigId === 'string' && authConfigId.length > 0)
163 );
164 if (EffectRecord.isEmptyRecord(authConfigs)) {
165 return undefined;
166 }
167
168 return {
169 authConfigs,
170 };
171 };
172
173 const mergeAuthConfigMappings = (params: {
174 readonly current?: ConsumerToolRouterAuthConfigMappings;
175 readonly next?: ConsumerToolRouterAuthConfigMappings;
176 }) => {
177 const current = normalizeAuthConfigMappings(params.current);
178 const next = normalizeAuthConfigMappings(params.next);
179 if (!current) return next;
180 if (!next) return current;
181
182 return normalizeAuthConfigMappings({
183 authConfigs: {
184 ...(current.authConfigs ?? {}),
185 ...(next.authConfigs ?? {}),
186 },
187 });
188 };
189
190 const normalizeConnectedAccountMappings = (
191 mappings?: ConsumerToolRouterConnectedAccountMappings
192 ): ConsumerToolRouterConnectedAccountMappings | undefined => {
193 if (!mappings) return undefined;
194
195 const connectedAccounts = Object.fromEntries(
196 Object.entries(mappings.connectedAccounts ?? {})
197 .map(([toolkit, connectedAccountId]) => [toolkit.toLowerCase(), connectedAccountId])
198 .filter(
199 ([, connectedAccountId]) =>
200 typeof connectedAccountId === 'string' && connectedAccountId.length > 0
201 )
202 );
203
204 const availableConnectedAccounts = Object.fromEntries(
205 Object.entries(mappings.availableConnectedAccounts ?? {})
206 .map(([toolkit, accounts]) => [
207 toolkit.toLowerCase(),
208 accounts.filter(
209 account =>
210 typeof account.id === 'string' &&
211 account.id.length > 0 &&
212 typeof account.updatedAt === 'string' &&
213 typeof account.createdAt === 'string'
214 ),
215 ])
216 .filter(([, accounts]) => accounts.length > 0)
217 );
218
219 if (
220 EffectRecord.isEmptyRecord(connectedAccounts) &&
221 EffectRecord.isEmptyRecord(availableConnectedAccounts)
222 ) {
223 return undefined;
224 }
225
226 return {
227 connectedAccounts: EffectRecord.isEmptyRecord(connectedAccounts)
228 ? undefined
229 : connectedAccounts,
230 availableConnectedAccounts: EffectRecord.isEmptyRecord(availableConnectedAccounts)
231 ? undefined
232 : availableConnectedAccounts,
233 };
234 };
235
236 const mergeConnectedAccountMappings = (params: {
237 readonly current?: ConsumerToolRouterConnectedAccountMappings;
238 readonly next?: ConsumerToolRouterConnectedAccountMappings;
239 }) => {
240 const current = normalizeConnectedAccountMappings(params.current);
241 const next = normalizeConnectedAccountMappings(params.next);
242 if (!current) return next;
243 if (!next) return current;
244
245 return normalizeConnectedAccountMappings({
246 connectedAccounts: {
247 ...(current.connectedAccounts ?? {}),
248 ...(next.connectedAccounts ?? {}),
249 },
250 availableConnectedAccounts: {
251 ...(current.availableConnectedAccounts ?? {}),
252 ...(next.availableConnectedAccounts ?? {}),
253 },
254 });
255 };
256
257 // Shared prefix of every getFresh*FromCache accessor: bail to Option.none()
258 // when the cache is disabled, the entry is missing, or its TTL has passed.
259 const getFreshCacheEntry = (params: { orgId: string; consumerUserId: string }) =>
260 Effect.gen(function* () {
261 const disabled = yield* APP_CONFIG.DISABLE_CONNECTED_ACCOUNT_CACHE;
262 if (disabled) {
263 return Option.none<CacheEntry>();
264 }
265 const state = yield* readCache();
266 const entry = state[cacheKey(params.orgId, params.consumerUserId)];
267 if (!entry) {
268 return Option.none<CacheEntry>();
269 }
270 const expiresAtMs = Date.parse(entry.expiresAt);
271 if (!Number.isFinite(expiresAtMs) || expiresAtMs <= Date.now()) {
272 return Option.none<CacheEntry>();
273 }
274 return Option.some(entry);
275 });
276
277 export const getFreshConsumerConnectedToolkitsFromCache = (params: {
278 orgId: string;
279 consumerUserId: string;
280 }) => getFreshCacheEntry(params).pipe(Effect.map(Option.map(entry => entry.toolkits)));
281
282 export const getFreshConsumerToolRouterAuthConfigsFromCache = (params: {
283 orgId: string;
284 consumerUserId: string;
285 toolkits?: ReadonlyArray<string>;
286 }) =>
287 Effect.gen(function* () {
288 const entry = yield* getFreshCacheEntry(params);
289 if (Option.isNone(entry)) {
290 return Option.none<ConsumerToolRouterAuthConfigMappings>();
291 }
292
293 const mappings = normalizeAuthConfigMappings(entry.value.toolRouterAuthConfigs);
294 if (!mappings) {
295 return Option.none<ConsumerToolRouterAuthConfigMappings>();
296 }
297
298 if (!params.toolkits || params.toolkits.length === 0) {
299 return Option.some(mappings);
300 }
301
302 const requestedToolkits = params.toolkits.map(toolkit => toolkit.toLowerCase());
303 const requestedAuthConfigs: Record<string, string> = {};
304 for (const toolkit of requestedToolkits) {
305 const authConfigId = mappings.authConfigs?.[toolkit];
306 if (typeof authConfigId !== 'string') {
307 return Option.none<ConsumerToolRouterAuthConfigMappings>();
308 }
309 requestedAuthConfigs[toolkit] = authConfigId;
310 }
311
312 const filtered = normalizeAuthConfigMappings({ authConfigs: requestedAuthConfigs });
313
314 return filtered ? Option.some(filtered) : Option.none<ConsumerToolRouterAuthConfigMappings>();
315 });
316
317 export const getFreshConsumerToolRouterConnectedAccountsFromCache = (params: {
318 orgId: string;
319 consumerUserId: string;
320 toolkits?: ReadonlyArray<string>;
321 }) =>
322 Effect.gen(function* () {
323 const entry = yield* getFreshCacheEntry(params);
324 if (Option.isNone(entry)) {
325 return Option.none<ConsumerToolRouterConnectedAccountMappings>();
326 }
327
328 const mappings = normalizeConnectedAccountMappings(entry.value.toolRouterConnectedAccounts);
329 if (!mappings) {
330 return Option.none<ConsumerToolRouterConnectedAccountMappings>();
331 }
332
333 if (!params.toolkits || params.toolkits.length === 0) {
334 return Option.some(mappings);
335 }
336
337 const requestedToolkits = params.toolkits.map(toolkit => toolkit.toLowerCase());
338 const filteredConnectedAccounts = EffectRecord.fromEntries(
339 requestedToolkits.flatMap(toolkit => {
340 const connectedAccountId = mappings.connectedAccounts?.[toolkit];
341 return typeof connectedAccountId === 'string'
342 ? [[toolkit, connectedAccountId] as const]
343 : [];
344 })
345 );
346 const filteredAvailableConnectedAccounts: Record<
347 string,
348 ReadonlyArray<CachedConnectedAccountSummary>
349 > = EffectRecord.fromEntries(
350 requestedToolkits.flatMap(toolkit => {
351 const accounts = mappings.availableConnectedAccounts?.[toolkit];
352 return Array.isArray(accounts) && accounts.length > 0 ? [[toolkit, accounts] as const] : [];
353 })
354 );
355 const filtered = normalizeConnectedAccountMappings({
356 connectedAccounts: filteredConnectedAccounts,
357 availableConnectedAccounts: filteredAvailableConnectedAccounts,
358 });
359
360 return filtered
361 ? Option.some(filtered)
362 : Option.none<ConsumerToolRouterConnectedAccountMappings>();
363 });
364
365 export const invalidateConsumerConnectedToolkitsCache = () =>
366 Effect.gen(function* () {
367 const fs = yield* FileSystem.FileSystem;
368 const filePath = yield* cachePath;
369 if (yield* fs.exists(filePath)) {
370 yield* fs.remove(filePath);
371 }
372 });
373
374 const resolveConsumerScope = (params?: {
375 readonly orgId?: string;
376 readonly consumerUserId?: string;
377 }) =>
378 Effect.gen(function* () {
379 if (params?.orgId && params.consumerUserId) {
380 return {
381 orgId: params.orgId,
382 consumerUserId: params.consumerUserId,
383 };
384 }
385
386 const project = yield* resolveCommandProject({ mode: 'consumer' }).pipe(Effect.option);
387 if (Option.isSome(project) && project.value.projectType === 'CONSUMER') {
388 return {
389 orgId: project.value.orgId,
390 consumerUserId: project.value.consumerUserId ?? '',
391 };
392 }
393
394 const userContext = yield* ComposioUserContext;
395 const apiKey = Option.getOrUndefined(userContext.data.apiKey);
396 const orgId = Option.getOrUndefined(userContext.data.orgId);
397 if (!apiKey || !orgId) {
398 return null;
399 }
400 const consumerProject = yield* resolveConsumerProject({
401 apiKey,
402 orgId,
403 }).pipe(Effect.option);
404 if (Option.isNone(consumerProject)) {
405 return null;
406 }
407 return {
408 orgId,
409 consumerUserId: consumerProject.value.consumer_user_id,
410 };
411 });
412
413 export const refreshConsumerConnectedToolkitsCache = (params?: {
414 readonly orgId?: string;
415 readonly consumerUserId?: string;
416 }) =>
417 Effect.gen(function* () {
418 const disabled = yield* APP_CONFIG.DISABLE_CONNECTED_ACCOUNT_CACHE;
419 if (disabled) return;
420 const scope = yield* resolveConsumerScope(params);
421 if (!scope?.consumerUserId) {
422 return;
423 }
424
425 const userContext = yield* ComposioUserContext;
426 const apiKey = Option.getOrUndefined(userContext.data.apiKey);
427 if (!apiKey) {
428 return;
429 }
430
431 const clientSingleton = yield* ComposioClientSingleton;
432 const directToolkits = yield* Effect.gen(function* () {
433 const consumerProject = yield* resolveConsumerProject({
434 apiKey,
435 orgId: scope.orgId,
436 });
437 // The nano id, as `resolveCommandProject` hands it to `composio execute`:
438 // the memoized connected-account list is keyed on the client's org and
439 // project, so a different project id here would mean a second
440 // `GET /connected_accounts` per execute.
441 const client = yield* clientSingleton.getFor({
442 orgId: scope.orgId,
443 projectId: consumerProject.project_nano_id,
444 });
445 const connectionContext = yield* resolveToolRouterSessionConnections(
446 client,
447 scope.consumerUserId
448 );
449 return connectionContext;
450 }).pipe(Effect.option);
451
452 const connectedToolkits =
453 Option.isSome(directToolkits) && directToolkits.value.connectedToolkits.length > 0
454 ? directToolkits.value.connectedToolkits
455 : (yield* getConsumerConnectedToolkits({
456 apiKey,
457 orgId: scope.orgId,
458 consumerUserId: scope.consumerUserId,
459 })).toolkits;
460
461 const noAuthToolkits = yield* getAlwaysConnectedNoAuthToolkits();
462 const state = yield* readCache();
463 const key = cacheKey(scope.orgId, scope.consumerUserId);
464 const currentEntry = state[key];
465 const proc = yield* NodeProcess;
466 const searchSessionFields = resolveSearchSessionMetadata({
467 currentEntry,
468 cwd: proc.cwd,
469 });
470 yield* writeCache({
471 ...state,
472 [key]: {
473 toolkits: normalizeCachedToolkits(connectedToolkits, noAuthToolkits),
474 expiresAt: new Date(Date.now() + CACHE_TTL_MS).toISOString(),
475 toolRouterAuthConfigs: mergeAuthConfigMappings({
476 current: currentEntry?.toolRouterAuthConfigs,
477 next: Option.getOrUndefined(
478 Option.map(directToolkits, ({ authConfigs }) => ({ authConfigs }))
479 ),
480 }),
481 toolRouterConnectedAccounts: mergeConnectedAccountMappings({
482 current: currentEntry?.toolRouterConnectedAccounts,
483 next: Option.getOrUndefined(
484 Option.map(directToolkits, ({ connectedAccounts, availableConnectedAccounts }) => ({
485 connectedAccounts,
486 availableConnectedAccounts,
487 }))
488 ),
489 }),
490 ...searchSessionFields,
491 },
492 });
493 });
494
495 export const writeConsumerConnectedToolkitsCache = (params: {
496 readonly orgId: string;
497 readonly consumerUserId: string;
498 readonly toolkits: ReadonlyArray<string>;
499 readonly toolRouterAuthConfigs?: ConsumerToolRouterAuthConfigMappings;
500 readonly toolRouterConnectedAccounts?: ConsumerToolRouterConnectedAccountMappings;
501 }) =>
502 Effect.gen(function* () {
503 const disabled = yield* APP_CONFIG.DISABLE_CONNECTED_ACCOUNT_CACHE;
504 if (disabled) return;
505 const noAuthToolkits = yield* getAlwaysConnectedNoAuthToolkits();
506 const state = yield* readCache();
507 const key = cacheKey(params.orgId, params.consumerUserId);
508 const currentEntry = state[key];
509 const proc = yield* NodeProcess;
510 const searchSessionFields = resolveSearchSessionMetadata({
511 currentEntry,
512 cwd: proc.cwd,
513 });
514
515 yield* writeCache({
516 ...state,
517 [key]: {
518 toolkits: normalizeCachedToolkits(
519 [...(currentEntry?.toolkits ?? []), ...params.toolkits],
520 noAuthToolkits
521 ),
522 expiresAt: new Date(Date.now() + CACHE_TTL_MS).toISOString(),
523 toolRouterAuthConfigs: mergeAuthConfigMappings({
524 current: currentEntry?.toolRouterAuthConfigs,
525 next: params.toolRouterAuthConfigs,
526 }),
527 toolRouterConnectedAccounts: mergeConnectedAccountMappings({
528 current: currentEntry?.toolRouterConnectedAccounts,
529 next: params.toolRouterConnectedAccounts,
530 }),
531 ...searchSessionFields,
532 },
533 });
534 });
535
536 export const primeConsumerConnectedToolkitsCacheInBackground = (params?: {
537 readonly orgId?: string;
538 readonly consumerUserId?: string;
539 }) =>
540 refreshConsumerConnectedToolkitsCache(params).pipe(
541 Effect.catch(() => Effect.void),
542 Effect.forkDetach,
543 Effect.asVoid
544 );
545
546 export const getOrCreateProbablyMyCliSessionIdForCurrentCwd = (params?: {
547 readonly orgId?: string;
548 readonly consumerUserId?: string;
549 }) =>
550 Effect.gen(function* () {
551 const disabled = yield* APP_CONFIG.DISABLE_CONNECTED_ACCOUNT_CACHE;
552 if (disabled) return Option.none<string>();
553 const scope = yield* resolveConsumerScope(params);
554 if (!scope?.consumerUserId) {
555 return Option.none<string>();
556 }
557
558 const proc = yield* NodeProcess;
559 const state = yield* readCache();
560 const key = cacheKey(scope.orgId, scope.consumerUserId);
561 const currentEntry = state[key];
562 const searchSessionFields = resolveSearchSessionMetadata({
563 currentEntry,
564 cwd: proc.cwd,
565 });
566 const currentCwdHash = cwdHash(proc.cwd);
567 const session = searchSessionFields.probablyMyCliSessionsByCwdHash[currentCwdHash];
568 if (!session) {
569 return Option.none<string>();
570 }
571
572 yield* writeCache({
573 ...state,
574 [key]: {
575 toolkits: currentEntry?.toolkits ?? [],
576 expiresAt: currentEntry?.expiresAt ?? new Date(Date.now() + CACHE_TTL_MS).toISOString(),
577 toolRouterAuthConfigs: currentEntry?.toolRouterAuthConfigs,
578 toolRouterConnectedAccounts: currentEntry?.toolRouterConnectedAccounts,
579 ...searchSessionFields,
580 },
581 });
582
583 return Option.some(session.id);
584 });