@devries-cameron20 / composio Mirror · read-only / commits / 2b3fa2d5a
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 22, 19:30 UTC

fix(cli): actually use cursor pagination in 'list()' methods

jkomyno committed Oct 28, 2025 at 00:20 UTC 2b3fa2d5a78c757ca4973939d694cbccea168872
1 file changed +116 -12
ts/packages/cli/src/services/composio-clients.ts
+116 -12
@@ -131,6 +131,98 @@ const callClient = <T, S extends Schema.Schema<any, any>>(
131 );
132 });
133
134 +// Schema constraint for paginated responses
135 +type PaginatedSchema = Schema.Schema<
136 + {
137 + // eslint-disable-next-line @typescript-eslint/no-explicit-any
138 + items: ReadonlyArray<any>;
139 + next_cursor: string | null;
140 + total_pages: number;
141 + },
142 + // eslint-disable-next-line @typescript-eslint/no-explicit-any
143 + any
144 +>;
145 +
146 +// Utility function for calling paginated Composio API endpoints.
147 +// Automatically fetches all pages until the specified limit is reached.
148 +const callClientWithPagination = <T, S extends PaginatedSchema>(
149 + clientSingleton: ComposioClientSingleton,
150 + apiCall: (client: _RawComposioClient, cursor?: string) => Promise<T>,
151 + responseSchema: S,
152 + limit: number
153 +): Effect.Effect<Schema.Schema.Type<S>, HttpError | NoSuchElementException> =>
154 + Effect.gen(function* () {
155 + const client = yield* clientSingleton.get();
156 +
157 + const fetchPage = (cursor?: string): Effect.Effect<T, HttpServerError> =>
158 + Effect.tryPromise({
159 + try: () => apiCall(client, cursor),
160 + catch: e => {
161 + const decodedOpt = Schema.decodeUnknownOption(HttpErrorResponse)(e);
162 +
163 + if (Option.isNone(decodedOpt)) {
164 + return new HttpServerError({
165 + cause: e,
166 + });
167 + }
168 +
169 + const {
170 + status,
171 + error: { error },
172 + } = decodedOpt.value;
173 + const pretty = renderPrettyError([
174 + ['code', error.code],
175 + ['message', error.message],
176 + ['suggested fix', error.suggested_fix],
177 + ]);
178 +
179 + return new HttpServerError({
180 + cause: `HTTP ${status}\n${pretty}`,
181 + });
182 + },
183 + });
184 +
185 + type DecodedPage = Schema.Schema.Type<S>;
186 +
187 + const decodeResponse = (json: unknown): Effect.Effect<DecodedPage, HttpDecodingError> =>
188 + pipe(
189 + Schema.decodeUnknown(responseSchema)(json),
190 + Effect.catchTag('ParseError', e => {
191 + const message = ParseResult.TreeFormatter.formatErrorSync(e);
192 +
193 + return new HttpDecodingError({
194 + cause: `ParseError\n ${message}`,
195 + });
196 + })
197 + ) as Effect.Effect<DecodedPage, HttpDecodingError>;
198 +
199 + let allItems: ReadonlyArray<unknown> = [];
200 + let currentCursor: string | null = null;
201 + let totalPages = 0;
202 +
203 + while (allItems.length < limit) {
204 + const json: T = yield* fetchPage(currentCursor ?? undefined);
205 + const decoded: DecodedPage = yield* decodeResponse(json);
206 +
207 + allItems = allItems.concat(decoded.items);
208 + totalPages = decoded.total_pages;
209 + currentCursor = decoded.next_cursor;
210 +
211 + yield* Effect.logDebug(` Total items fetched: ${allItems.length}/${limit}`);
212 +
213 + // Stop if no more pages
214 + if (currentCursor === null) {
215 + break;
216 + }
217 + }
218 +
219 + return {
220 + items: allItems.slice(0, limit),
221 + total_pages: totalPages,
222 + next_cursor: currentCursor,
223 + } as DecodedPage;
224 + });
225 +
226 /**
227 * Services
228 */
@@ -180,9 +272,17 @@ export class ComposioClientLive extends Effect.Service<ComposioClientLive>()(
272 toolkits: {
273 /**
274 * Retrieves a comprehensive list of toolkits that are available to the authenticated project.
275 + * If limit is provided, automatically handles pagination to fetch up to that many items.
276 */
184 - list: () =>
185 - callClient(clientSingleton, client => client.toolkits.list(), ToolkitsResponse),
277 + list: (params?: { limit?: number }) =>
278 + params?.limit
279 + ? callClientWithPagination(
280 + clientSingleton,
281 + (client, cursor) => client.toolkits.list({ cursor }),
282 + ToolkitsResponse,
283 + params.limit
284 + )
285 + : callClient(clientSingleton, client => client.toolkits.list(), ToolkitsResponse),
286 },
287 tools: {
288 /**
@@ -195,11 +295,15 @@ export class ComposioClientLive extends Effect.Service<ComposioClientLive>()(
295 ToolsAsEnumsResponse
296 ),
297 /**
198 - * Retrieve a list of $limit trigger types, containing their payload.
199 - * Usually, you would call this with a limit matching the length of the list returned by `retrieveEnum`.
298 + * Retrieve a list of tools, automatically handling pagination to fetch up to $limit items.
299 */
300 list: (params: { limit: number }) =>
202 - callClient(clientSingleton, client => client.tools.list(params), ToolsResponse),
301 + callClientWithPagination(
302 + clientSingleton,
303 + (client, cursor) => client.tools.list({ cursor, limit: params.limit }),
304 + ToolsResponse,
305 + params.limit
306 + ),
307 },
308 triggersTypes: {
309 /**
@@ -212,14 +316,14 @@ export class ComposioClientLive extends Effect.Service<ComposioClientLive>()(
316 TriggerTypesAsEnumsResponse
317 ),
318 /**
215 - * Retrieve a list of $limit trigger types, containing their payload.
216 - * Usually, you would call this with a limit matching the length of the list returned by `retrieveEnum`.
319 + * Retrieve a list of trigger types, automatically handling pagination to fetch up to $limit items.
320 */
321 list: (params: { limit: number }) =>
219 - callClient(
322 + callClientWithPagination(
323 clientSingleton,
221 - client => client.triggersTypes.list(params),
222 - TriggerTypesResponse
324 + (client, cursor) => client.triggersTypes.list({ cursor, limit: params.limit }),
325 + TriggerTypesResponse,
326 + params.limit
327 ),
328 },
329 cli: {
@@ -256,8 +360,8 @@ export class ComposioToolkitsRepository extends Effect.Service<ComposioToolkitsR
360 const client = yield* ComposioClientLive;
361
362 return {
259 - getToolkits: () =>
260 - client.toolkits.list().pipe(
363 + getToolkits: (limit?: number) =>
364 + client.toolkits.list({ limit }).pipe(
365 Effect.map(response => response.items),
366 Effect.flatMap(
367 Effect.fn(function* (toolkits) {