@devries-cameron20 / composio-1 / commits / 6e7d6953c
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix: codact failures were not logging properly (#3056)

Rahul Tarak committed Mar 27, 2026 at 07:43 UTC 6e7d6953c1a5fe24c257eecd21f71b8960cfaf72
4 files changed +331 -19
ts/packages/cli/src/analytics/dispatch.ts
+201 -15
@@ -8,12 +8,32 @@ import type { AnalyticsEnvelope, TrackEvent } from './types';
8 import * as constants from 'src/constants';
9
10 const INTERNAL_ANALYTICS_WORKER_FLAG = '__analytics-worker';
11 +const INTERNAL_CODACT_FAILURE_WORKER_FLAG = '__codact-failure-worker';
12 const COMPOSIO_DIR = '.composio';
13 const ANALYTICS_STATE_FILE_NAME = 'analytics.json';
14 const CONSUMER_SHORT_TERM_CACHE_FILE_NAME = 'consumer-short-term-cache.json';
15 const CLI_ANALYTICS_PATH = '/api/v3/cli/analytics';
16 +const CLI_CODACT_FAILURES_PATH = '/api/v3/cli/codact_failures';
17 const TELEMETRY_DEBUG_ENV_VAR = 'COMPOSIO_CLI_TELEMETRY_DEBUG';
18
19 +export type CliCodactFailureType = 'wrong_tool_slug' | 'wrong_tool_input_param';
20 +
21 +export type CliCodactFailureToolInfo = {
22 + readonly toolkit?: string;
23 + readonly tool?: {
24 + readonly slug: string;
25 + readonly version: string;
26 + };
27 +};
28 +
29 +export type CliCodactFailure = {
30 + readonly failureType: CliCodactFailureType;
31 + readonly toolInfo?: CliCodactFailureToolInfo;
32 + readonly ctx: Record<string, unknown>;
33 + readonly session?: Record<string, unknown>;
34 + readonly requestId?: string;
35 +};
36 +
37 const truthy = (value: string | undefined): boolean =>
38 value === '1' || value === 'true' || value === 'yes' || value === 'on';
39
@@ -165,7 +185,7 @@ const getUserApiKey = (): string | null => {
185
186 const cwdHash = (cwd: string): string => djb2Hash(cwd).toString(36);
187
168 -const getCurrentCwdSessionId = (): string | undefined => {
188 +export const getCurrentCwdSessionId = (): string | undefined => {
189 try {
190 const raw = fs.readFileSync(consumerShortTermCachePath(), 'utf8');
191 const parsed = JSON.parse(raw) as ConsumerShortTermCacheState;
@@ -202,7 +222,7 @@ const withCliSessionId = (event: TrackEvent): TrackEvent => {
222 };
223 };
224
205 -const readApiBaseUrl = (): string | null => {
225 +export const readApiBaseUrl = (): string | null => {
226 const envBaseUrl = process.env.COMPOSIO_BASE_URL?.trim();
227 if (envBaseUrl) {
228 return envBaseUrl.replace(/\/+$/u, '');
@@ -226,7 +246,12 @@ const getAnalyticsEndpoint = (): string | null => {
246 return baseUrl ? `${baseUrl}${CLI_ANALYTICS_PATH}` : null;
247 };
248
229 -const shouldDisableAnalytics = (): boolean =>
249 +const getCliCodactFailuresEndpoint = (): string | null => {
250 + const baseUrl = readApiBaseUrl();
251 + return baseUrl ? `${baseUrl}${CLI_CODACT_FAILURES_PATH}` : null;
252 +};
253 +
254 +export const shouldDisableAnalytics = (): boolean =>
255 truthy(process.env.COMPOSIO_CLI_TELEMETRY_DISABLED) ||
256 truthy(process.env.TELEMETRY_DISABLED) ||
257 truthy(process.env.COMPOSIO_DISABLE_TELEMETRY) ||
@@ -285,6 +310,70 @@ const captureToComposioAnalytics = async (envelope: AnalyticsEnvelope): Promise<
310 });
311 };
312
313 +export const createCliCodactFailureBody = (
314 + failure: CliCodactFailure
315 +): {
316 + failure_type: CliCodactFailureType;
317 + tool_info?: CliCodactFailureToolInfo;
318 + ctx: Record<string, unknown>;
319 + session: Record<string, unknown>;
320 + request_id?: string;
321 +} => ({
322 + failure_type: failure.failureType,
323 + ...(failure.toolInfo ? { tool_info: failure.toolInfo } : {}),
324 + ctx: failure.ctx,
325 + session: {
326 + source: 'cli',
327 + id: getCurrentCwdSessionId(),
328 + cli_version: constants.APP_VERSION,
329 + invocation_origin: process.env.COMPOSIO_CLI_INVOCATION_ORIGIN ?? 'cli',
330 + parent_run_id: process.env.COMPOSIO_CLI_PARENT_RUN_ID,
331 + ...(failure.session ?? {}),
332 + },
333 + ...(failure.requestId ? { request_id: failure.requestId } : {}),
334 +});
335 +
336 +const captureToComposioCodactFailures = async (failure: CliCodactFailure): Promise<void> => {
337 + const endpoint = getCliCodactFailuresEndpoint();
338 + if (!endpoint || shouldDisableAnalytics()) {
339 + telemetryDebugLog('codact_delivery_skipped', {
340 + reason: shouldDisableAnalytics() ? 'disabled' : 'missing_endpoint',
341 + endpoint,
342 + failureType: failure.failureType,
343 + });
344 + return;
345 + }
346 +
347 + const userApiKey = getUserApiKey();
348 + if (!userApiKey) {
349 + telemetryDebugLog('codact_delivery_skipped', {
350 + reason: 'missing_user_api_key',
351 + endpoint,
352 + failureType: failure.failureType,
353 + });
354 + return;
355 + }
356 +
357 + const response = await fetch(endpoint, {
358 + method: 'POST',
359 + headers: {
360 + 'Content-Type': 'application/json',
361 + 'x-user-api-key': userApiKey,
362 + },
363 + body: JSON.stringify(createCliCodactFailureBody(failure)),
364 + });
365 + const responseBody =
366 + !response.ok && isTelemetryDebugEnabled() ? await response.text() : undefined;
367 +
368 + telemetryDebugLog(response.ok ? 'codact_delivery_succeeded' : 'codact_delivery_failed', {
369 + endpoint,
370 + failureType: failure.failureType,
371 + status: response.status,
372 + ok: response.ok,
373 + responseBody: responseBody?.slice(0, 1000),
374 + });
375 +};
376 +
377 export const trackCliEvent = (event: TrackEvent): void => {
378 const endpoint = getAnalyticsEndpoint();
379 if (!event) {
@@ -337,33 +426,130 @@ export const trackCliEvent = (event: TrackEvent): void => {
426
427 export const trackCliEventEffect = (event: TrackEvent) => Effect.sync(() => trackCliEvent(event));
428
340 -const getAnalyticsWorkerFlagIndex = (argv: ReadonlyArray<string>): number =>
341 - argv.findIndex(token => token === INTERNAL_ANALYTICS_WORKER_FLAG);
429 +export const trackCliCodactFailure = (failure: CliCodactFailure): void => {
430 + const endpoint = getCliCodactFailuresEndpoint();
431 + const userApiKey = getUserApiKey();
432 + if (shouldDisableAnalytics() || !endpoint || !userApiKey) {
433 + telemetryDebugLog('codact_skip', {
434 + reason: shouldDisableAnalytics()
435 + ? 'disabled'
436 + : !endpoint
437 + ? 'missing_endpoint'
438 + : 'missing_user_api_key',
439 + failureType: failure.failureType,
440 + endpoint,
441 + });
442 + return;
443 + }
444 +
445 + try {
446 + const encodedPayload = encodeBase64Url(JSON.stringify(createCliCodactFailureBody(failure)));
447 + const { command, args } = getWorkerSpawnArgsForFlag(
448 + INTERNAL_CODACT_FAILURE_WORKER_FLAG,
449 + encodedPayload
450 + );
451 + const child = spawn(command, args, {
452 + detached: true,
453 + stdio: isTelemetryDebugEnabled() ? ['ignore', 'ignore', 'inherit'] : 'ignore',
454 + env: {
455 + ...process.env,
456 + COMPOSIO_CLI_ANALYTICS_WORKER: '1',
457 + },
458 + });
459 + child.unref();
460 + } catch {
461 + // Best-effort — codact failure tracking must never break CLI execution.
462 + }
463 +};
464 +
465 +export const trackCliCodactFailureEffect = (failure: CliCodactFailure) =>
466 + Effect.sync(() => trackCliCodactFailure(failure));
467
343 -export const isAnalyticsWorkerInvocation = (argv: ReadonlyArray<string>): boolean =>
344 - getAnalyticsWorkerFlagIndex(argv) >= 0;
468 +const getWorkerSpawnArgsForFlag = (
469 + workerFlag: string,
470 + encodedPayload: string
471 +): { command: string; args: string[] } => {
472 + const maybeScriptPath = process.argv[1];
473 + const scriptPathLooksReal =
474 + typeof maybeScriptPath === 'string' &&
475 + maybeScriptPath.length > 0 &&
476 + fs.existsSync(maybeScriptPath) &&
477 + /\.(?:[cm]?[jt]s|mjs|mts|cts)$/u.test(maybeScriptPath);
478
346 -export const runAnalyticsWorkerFromArgv = async (argv: ReadonlyArray<string>): Promise<void> => {
347 - const flagIndex = getAnalyticsWorkerFlagIndex(argv);
348 - const encodedPayload = flagIndex >= 0 ? argv[flagIndex + 1] : undefined;
479 + return scriptPathLooksReal
480 + ? {
481 + command: process.execPath,
482 + args: [maybeScriptPath, workerFlag, encodedPayload],
483 + }
484 + : {
485 + command: process.execPath,
486 + args: [workerFlag, encodedPayload],
487 + };
488 +};
489 +
490 +const getWorkerFlagIndex = (argv: ReadonlyArray<string>, flag: string): number =>
491 + argv.findIndex(token => token === flag);
492 +
493 +export const isBackgroundWorkerInvocation = (argv: ReadonlyArray<string>): boolean =>
494 + getWorkerFlagIndex(argv, INTERNAL_ANALYTICS_WORKER_FLAG) >= 0 ||
495 + getWorkerFlagIndex(argv, INTERNAL_CODACT_FAILURE_WORKER_FLAG) >= 0;
496 +
497 +export const runBackgroundWorkerFromArgv = async (argv: ReadonlyArray<string>): Promise<void> => {
498 + const analyticsFlagIndex = getWorkerFlagIndex(argv, INTERNAL_ANALYTICS_WORKER_FLAG);
499 + if (analyticsFlagIndex >= 0) {
500 + const encodedPayload = argv[analyticsFlagIndex + 1];
501 + if (!encodedPayload) {
502 + return;
503 + }
504 +
505 + try {
506 + const decoded = decodeBase64Url(encodedPayload);
507 + const envelope = JSON.parse(decoded) as AnalyticsEnvelope;
508 + if (typeof envelope?.event !== 'string' || envelope.event.length === 0) {
509 + return;
510 + }
511 + await captureToComposioAnalytics(envelope);
512 + } catch (error) {
513 + telemetryDebugLog('delivery_error', {
514 + error:
515 + error instanceof Error
516 + ? { name: error.name, message: error.message }
517 + : { message: String(error) },
518 + });
519 + }
520 + return;
521 + }
522 +
523 + const codactFlagIndex = getWorkerFlagIndex(argv, INTERNAL_CODACT_FAILURE_WORKER_FLAG);
524 + if (codactFlagIndex < 0) {
525 + return;
526 + }
527 +
528 + const encodedPayload = argv[codactFlagIndex + 1];
529 if (!encodedPayload) {
530 return;
531 }
532
533 try {
534 const decoded = decodeBase64Url(encodedPayload);
355 - const envelope = JSON.parse(decoded) as AnalyticsEnvelope;
356 - if (typeof envelope?.event !== 'string' || envelope.event.length === 0) {
535 + const body = JSON.parse(decoded) as ReturnType<typeof createCliCodactFailureBody>;
536 + if (body?.failure_type !== 'wrong_tool_slug' && body?.failure_type !== 'wrong_tool_input_param') {
537 return;
538 }
359 - await captureToComposioAnalytics(envelope);
539 +
540 + await captureToComposioCodactFailures({
541 + failureType: body.failure_type,
542 + toolInfo: body.tool_info,
543 + ctx: body.ctx,
544 + session: body.session,
545 + requestId: body.request_id,
546 + });
547 } catch (error) {
361 - telemetryDebugLog('delivery_error', {
548 + telemetryDebugLog('codact_delivery_error', {
549 error:
550 error instanceof Error
551 ? { name: error.name, message: error.message }
552 : { message: String(error) },
553 });
367 - // Analytics must never break CLI execution.
554 }
555 };
ts/packages/cli/src/bin.ts
+3 -3
@@ -1,5 +1,5 @@
1 import process from 'node:process';
2 -import { isAnalyticsWorkerInvocation, runAnalyticsWorkerFromArgv } from 'src/analytics/dispatch';
2 +import { isBackgroundWorkerInvocation, runBackgroundWorkerFromArgv } from 'src/analytics/dispatch';
3
4 const TELEMETRY_DEBUG_FLAG = '--telemetry-debug';
5 const CLI_TELEMETRY_DEBUG_ENV_VAR = 'COMPOSIO_CLI_TELEMETRY_DEBUG';
@@ -16,8 +16,8 @@ const stripTelemetryDebugFlag = (argv: ReadonlyArray<string>): string[] => {
16 return normalizedArgv;
17 };
18
19 -if (isAnalyticsWorkerInvocation(process.argv)) {
20 - void runAnalyticsWorkerFromArgv(process.argv).finally(() => {
19 +if (isBackgroundWorkerInvocation(process.argv)) {
20 + void runBackgroundWorkerFromArgv(process.argv).finally(() => {
21 process.exit(0);
22 });
23 } else {
ts/packages/cli/src/commands/tools/commands/tools.execute.cmd.ts
+95 -1
@@ -22,7 +22,7 @@ import type { ToolExecuteParams, ToolExecuteResponse } from 'src/services/tools-
22 import { ComposioToolkitsRepository } from 'src/services/composio-clients';
23 import { ComposioUserContext } from 'src/services/user-context';
24 import { ProjectContext } from 'src/services/project-context';
25 -import { trackCliEventEffect } from 'src/analytics/dispatch';
25 +import { trackCliCodactFailureEffect, trackCliEventEffect } from 'src/analytics/dispatch';
26 import {
27 getToolExecuteFailedEvent,
28 getToolExecuteToolNotFoundEvent,
@@ -53,6 +53,7 @@ import {
53 mapComposioError,
54 normalizeCliError,
55 } from 'src/services/composio-error-overrides';
56 +import * as constants from 'src/constants';
57
58 const slug = Args.text({ name: 'slug' }).pipe(
59 Args.withDescription('Tool slug (e.g. "GITHUB_CREATE_ISSUE")')
@@ -285,6 +286,27 @@ const emitExecuteFailureTelemetry = (params: {
286 logId: params.logId,
287 })
288 );
289 + yield* trackCliCodactFailureEffect({
290 + failureType: 'wrong_tool_input_param',
291 + toolInfo: {
292 + ...(toolkitFromToolSlug(params.toolSlug)
293 + ? { toolkit: toolkitFromToolSlug(params.toolSlug) }
294 + : {}),
295 + },
296 + ctx: {
297 + tool_slug: params.toolSlug,
298 + issues: [...normalized.issues].slice(0, 50),
299 + schema_path: normalized.schemaPath,
300 + stage: params.stage === 'dry_run' ? 'dry_run' : 'validation',
301 + },
302 + session: {
303 + source: 'cli',
304 + command_path: params.surface === 'root' ? 'execute' : 'dev playground-execute',
305 + project_mode: params.projectMode,
306 + surface: params.surface,
307 + cli_version: constants.APP_VERSION,
308 + },
309 + });
310 return;
311 }
312
@@ -358,6 +380,78 @@ const emitExecuteFailureTelemetry = (params: {
380 });
381
382 yield* trackCliEventEffect(event);
383 +
384 + if (isNoConnectionError) {
385 + return;
386 + }
387 +
388 + if (
389 + isMaybeToolValidationError({
390 + message,
391 + errorSlug,
392 + apiCode,
393 + })
394 + ) {
395 + yield* trackCliCodactFailureEffect({
396 + failureType: 'wrong_tool_input_param',
397 + toolInfo: {
398 + ...(toolkitFromToolSlug(params.toolSlug)
399 + ? { toolkit: toolkitFromToolSlug(params.toolSlug) }
400 + : {}),
401 + },
402 + ctx: {
403 + tool_slug: params.toolSlug,
404 + error_slug: errorSlug,
405 + api_error_code: apiCode,
406 + http_status: status,
407 + error_message: message,
408 + stage: params.stage === 'dry_run' ? 'dry_run' : 'execution',
409 + },
410 + session: {
411 + source: 'cli',
412 + command_path: params.surface === 'root' ? 'execute' : 'dev playground-execute',
413 + project_mode: params.projectMode,
414 + surface: params.surface,
415 + cli_version: constants.APP_VERSION,
416 + },
417 + requestId: apiDetails?.request_id,
418 + });
419 + return;
420 + }
421 +
422 + if (
423 + isMaybeToolNotFoundError({
424 + message,
425 + errorSlug,
426 + status,
427 + apiCode,
428 + })
429 + ) {
430 + yield* trackCliCodactFailureEffect({
431 + failureType: 'wrong_tool_slug',
432 + toolInfo: {
433 + ...(toolkitFromToolSlug(params.toolSlug)
434 + ? { toolkit: toolkitFromToolSlug(params.toolSlug) }
435 + : {}),
436 + },
437 + ctx: {
438 + invalid_tool_slug: params.toolSlug,
439 + error_slug: errorSlug,
440 + api_error_code: apiCode,
441 + http_status: status,
442 + error_message: message,
443 + stage: params.stage === 'validation' ? 'execution' : params.stage,
444 + },
445 + session: {
446 + source: 'cli',
447 + command_path: params.surface === 'root' ? 'execute' : 'dev playground-execute',
448 + project_mode: params.projectMode,
449 + surface: params.surface,
450 + cli_version: constants.APP_VERSION,
451 + },
452 + requestId: apiDetails?.request_id,
453 + });
454 + }
455 });
456
457 export const showToolsExecuteInputHelp = (toolSlug: string) =>
ts/packages/cli/test/src/analytics.events.test.ts
+32
@@ -1,4 +1,5 @@
1 import { describe, expect, it } from 'vitest';
2 +import { createCliCodactFailureBody } from 'src/analytics/dispatch';
3 import {
4 getToolExecuteFailedEvent,
5 getToolExecuteToolNotFoundEvent,
@@ -93,4 +94,35 @@ describe('CLI analytics execute failure events', () => {
94 })
95 ).toBe(true);
96 });
97 +
98 + it('builds the CLI codact failure body using backend field names', () => {
99 + const body = createCliCodactFailureBody({
100 + failureType: 'wrong_tool_slug',
101 + toolInfo: {
102 + toolkit: 'github',
103 + },
104 + ctx: {
105 + invalid_tool_slug: 'GITHUB_MAKE_ISSUE',
106 + },
107 + session: {
108 + command_path: 'execute',
109 + },
110 + requestId: 'req_123',
111 + });
112 +
113 + expect(body).toMatchObject({
114 + failure_type: 'wrong_tool_slug',
115 + tool_info: {
116 + toolkit: 'github',
117 + },
118 + ctx: {
119 + invalid_tool_slug: 'GITHUB_MAKE_ISSUE',
120 + },
121 + session: {
122 + source: 'cli',
123 + command_path: 'execute',
124 + },
125 + request_id: 'req_123',
126 + });
127 + });
128 });