@devries-cameron20 / composio Mirror · read-only / commits / f8ad9538f
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 18:30 UTC

feat(cli): track plugin setup lifecycle and runtime setup telemetry

Re-lands 4e68e652 (reverted by 333d2296) and extends it: Part 1 (re-land): - Dedicated CLI_INSTALL_* / CLI_SETUP_* lifecycle families; 'setup' added to KNOWN_COMMAND_TOKENS so it is no longer misattributed to command_path='composio' - Per-host CLI_PLUGIN_SETUP_SUCCEEDED / CLI_PLUGIN_UNINSTALL_SUCCEEDED fired from installSetupTargets/uninstallSetupTargets - install.sh auto-setup runs with COMPOSIO_CLI_INVOCATION_ORIGIN=installer Part 2 (extensions, fired from inside setup.cmd.ts / setup.ts): - CLI_SETUP_HOST_DETECTED per host after detection, with agent_host, available, supported, host_version and normalized unsupported_reason_code (codex_too_old | no_json_inspection | host_command_failed | unknown) threaded through SetupTargetDetection - CLI_PLUGIN_SETUP_FAILED per host (phase install|uninstall) tapped before the existing mapError rethrow - CLI_SETUP_CANCELLED (user declined confirm) and CLI_SETUP_SKIPPED (--if-present with no supported host) - stdout_is_tty added to the setup command properties Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Kshitij Jhunjhunwala committed Jul 21, 2026 at 11:55 UTC f8ad9538f44b2bfddf633ac93f12522436f25faa
8 files changed +891 -67
install.sh
+1 -1
@@ -407,7 +407,7 @@ rm -f "$install_err"
407 if [[ $install_plugins = true ]]; then
408 echo
409 info "Checking for supported agent hosts..."
410 - if ! "$exe" setup --target auto --yes --if-present; then
410 + if ! COMPOSIO_CLI_INVOCATION_ORIGIN=installer "$exe" setup --target auto --yes --if-present; then
411 error 'Composio CLI was installed, but agent plugin setup failed. Retry with `composio setup --target auto --yes`.'
412 fi
413 fi
test/install-sh-release-resolution.test.sh
+4
@@ -154,6 +154,10 @@ install)
154 exit 0
155 ;;
156 setup)
157 + if [[ "${COMPOSIO_CLI_INVOCATION_ORIGIN:-}" != 'installer' ]]; then
158 + echo 'Expected automatic plugin setup telemetry origin to be installer.' >&2
159 + exit 97
160 + fi
161 exit "${TEST_SETUP_EXIT:-0}"
162 ;;
163 --version|version)
ts/packages/cli/src/analytics/events.ts
+157
@@ -27,6 +27,18 @@ export const CLI_ANALYTICS_EVENTS = {
27 CLI_RUN_INVOKED: 'CLI_RUN_INVOKED',
28 CLI_RUN_SUCCEEDED: 'CLI_RUN_SUCCEEDED',
29 CLI_RUN_FAILED: 'CLI_RUN_FAILED',
30 + CLI_INSTALL_INVOKED: 'CLI_INSTALL_INVOKED',
31 + CLI_INSTALL_SUCCEEDED: 'CLI_INSTALL_SUCCEEDED',
32 + CLI_INSTALL_FAILED: 'CLI_INSTALL_FAILED',
33 + CLI_SETUP_INVOKED: 'CLI_SETUP_INVOKED',
34 + CLI_SETUP_SUCCEEDED: 'CLI_SETUP_SUCCEEDED',
35 + CLI_SETUP_FAILED: 'CLI_SETUP_FAILED',
36 + CLI_SETUP_HOST_DETECTED: 'CLI_SETUP_HOST_DETECTED',
37 + CLI_SETUP_CANCELLED: 'CLI_SETUP_CANCELLED',
38 + CLI_SETUP_SKIPPED: 'CLI_SETUP_SKIPPED',
39 + CLI_PLUGIN_SETUP_SUCCEEDED: 'CLI_PLUGIN_SETUP_SUCCEEDED',
40 + CLI_PLUGIN_SETUP_FAILED: 'CLI_PLUGIN_SETUP_FAILED',
41 + CLI_PLUGIN_UNINSTALL_SUCCEEDED: 'CLI_PLUGIN_UNINSTALL_SUCCEEDED',
42 CLI_TOOL_INVOCATION_VALIDATION_FAILED: 'CLI_TOOL_INVOCATION_VALIDATION_FAILED',
43 CLI_TOOL_INVOCATION_TOOL_NOT_FOUND: 'CLI_TOOL_INVOCATION_TOOL_NOT_FOUND',
44 CLI_TOOL_INVOCATION_FAILED: 'CLI_TOOL_INVOCATION_FAILED',
@@ -40,6 +52,7 @@ const KNOWN_COMMAND_TOKENS = new Set([
52 'logout',
53 'run',
54 'install',
55 + 'setup',
56 'dev',
57 'generate',
58 'tools',
@@ -286,6 +299,29 @@ const getRunCommandProperties = (context: CliCommandTelemetryContext) => ({
299 arg_count: Math.max(0, context.argv.length - 3),
300 });
301
302 +const getInstallCommandProperties = (context: CliCommandTelemetryContext) => ({
303 + source: 'cli',
304 + invocation_origin: getInvocationOrigin(),
305 + cli_version: context.cliVersion,
306 + command_path: context.commandPath,
307 + duration_ms: Date.now() - context.startedAt,
308 + completions: isFlagPresent(context.argv, '--completions'),
309 + no_completions: isFlagPresent(context.argv, '--no-completions'),
310 +});
311 +
312 +const getSetupCommandProperties = (context: CliCommandTelemetryContext) => ({
313 + source: 'cli',
314 + invocation_origin: getInvocationOrigin(),
315 + cli_version: context.cliVersion,
316 + command_path: context.commandPath,
317 + duration_ms: Date.now() - context.startedAt,
318 + operation: isFlagPresent(context.argv, '--uninstall') ? 'uninstall' : 'setup',
319 + target: getFlagValue(context.argv, '--target') ?? 'auto',
320 + yes: isFlagPresent(context.argv, '--yes', '-y'),
321 + if_present: isFlagPresent(context.argv, '--if-present'),
322 + stdout_is_tty: context.stdoutIsTTY,
323 +});
324 +
325 const getLoginCommandProperties = (context: CliCommandTelemetryContext) => ({
326 source: 'cli',
327 invocation_origin: getInvocationOrigin(),
@@ -338,6 +374,10 @@ const isProxyCommand = (commandPath: string): boolean => commandPath === 'proxy'
374
375 const isRunCommand = (commandPath: string): boolean => commandPath === 'run';
376
377 +const isInstallCommand = (commandPath: string): boolean => commandPath === 'install';
378 +
379 +const isSetupCommand = (commandPath: string): boolean => commandPath === 'setup';
380 +
381 const isGenericOnlyCommand = (commandPath: string): boolean =>
382 commandPath === 'composio' || commandPath.startsWith('dev');
383
@@ -461,6 +501,20 @@ const SPECIAL_LIFECYCLE_FAMILIES: ReadonlyArray<SpecialLifecycleFamily> = [
501 failedEventName: CLI_ANALYTICS_EVENTS.CLI_RUN_FAILED,
502 getProperties: getRunCommandProperties,
503 },
504 + {
505 + match: isInstallCommand,
506 + invokedEventName: CLI_ANALYTICS_EVENTS.CLI_INSTALL_INVOKED,
507 + succeededEventName: CLI_ANALYTICS_EVENTS.CLI_INSTALL_SUCCEEDED,
508 + failedEventName: CLI_ANALYTICS_EVENTS.CLI_INSTALL_FAILED,
509 + getProperties: getInstallCommandProperties,
510 + },
511 + {
512 + match: isSetupCommand,
513 + invokedEventName: CLI_ANALYTICS_EVENTS.CLI_SETUP_INVOKED,
514 + succeededEventName: CLI_ANALYTICS_EVENTS.CLI_SETUP_SUCCEEDED,
515 + failedEventName: CLI_ANALYTICS_EVENTS.CLI_SETUP_FAILED,
516 + getProperties: getSetupCommandProperties,
517 + },
518 ];
519
520 const getSpecialLifecycleFamily = (commandPath: string): SpecialLifecycleFamily | undefined =>
@@ -510,6 +564,109 @@ export const getPrimaryLifecycleFailedEvent = (
564 };
565 };
566
567 +export const getPluginLifecycleSucceededEvent = (params: {
568 + readonly operation: 'setup' | 'uninstall';
569 + readonly target: 'claude' | 'codex';
570 + readonly action: 'installed' | 'enabled' | 'configured' | 'uninstalled';
571 + readonly cliVersion: string;
572 +}): TrackEvent => ({
573 + name:
574 + params.operation === 'uninstall'
575 + ? CLI_ANALYTICS_EVENTS.CLI_PLUGIN_UNINSTALL_SUCCEEDED
576 + : CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_SUCCEEDED,
577 + properties: {
578 + source: 'cli',
579 + invocation_origin: getInvocationOrigin(),
580 + cli_version: params.cliVersion,
581 + command_path: 'setup',
582 + operation: params.operation,
583 + agent_host: params.target,
584 + action: params.action,
585 + },
586 +});
587 +
588 +export const getPluginLifecycleFailedEvent = (params: {
589 + readonly operation: 'setup' | 'uninstall';
590 + readonly target: 'claude' | 'codex';
591 + readonly phase: 'install' | 'uninstall';
592 + readonly error: unknown;
593 + readonly cliVersion: string;
594 +}): TrackEvent => ({
595 + name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_FAILED,
596 + properties: {
597 + source: 'cli',
598 + invocation_origin: getInvocationOrigin(),
599 + cli_version: params.cliVersion,
600 + command_path: 'setup',
601 + operation: params.operation,
602 + agent_host: params.target,
603 + phase: params.phase,
604 + error_name: errorNameOf(params.error),
605 + error_message: errorMessageOf(params.error),
606 + },
607 +});
608 +
609 +export const getSetupHostDetectedEvent = (params: {
610 + readonly operation: 'setup' | 'uninstall';
611 + readonly requestedTarget: 'auto' | 'claude' | 'codex' | 'all';
612 + readonly target: 'claude' | 'codex';
613 + readonly available: boolean;
614 + readonly supported: boolean;
615 + readonly hostVersion?: string;
616 + readonly unsupportedReasonCode?:
617 + 'codex_too_old' | 'no_json_inspection' | 'host_command_failed' | 'unknown';
618 + readonly cliVersion: string;
619 +}): TrackEvent => ({
620 + name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
621 + properties: {
622 + source: 'cli',
623 + invocation_origin: getInvocationOrigin(),
624 + cli_version: params.cliVersion,
625 + command_path: 'setup',
626 + operation: params.operation,
627 + requested_target: params.requestedTarget,
628 + agent_host: params.target,
629 + available: params.available,
630 + supported: params.supported,
631 + host_version: params.hostVersion,
632 + unsupported_reason_code: params.unsupportedReasonCode,
633 + },
634 +});
635 +
636 +export const getSetupCancelledEvent = (params: {
637 + readonly operation: 'setup' | 'uninstall';
638 + readonly requestedTarget: 'auto' | 'claude' | 'codex' | 'all';
639 + readonly cliVersion: string;
640 +}): TrackEvent => ({
641 + name: CLI_ANALYTICS_EVENTS.CLI_SETUP_CANCELLED,
642 + properties: {
643 + source: 'cli',
644 + invocation_origin: getInvocationOrigin(),
645 + cli_version: params.cliVersion,
646 + command_path: 'setup',
647 + operation: params.operation,
648 + requested_target: params.requestedTarget,
649 + reason: 'user_declined',
650 + },
651 +});
652 +
653 +export const getSetupSkippedEvent = (params: {
654 + readonly operation: 'setup' | 'uninstall';
655 + readonly requestedTarget: 'auto' | 'claude' | 'codex' | 'all';
656 + readonly cliVersion: string;
657 +}): TrackEvent => ({
658 + name: CLI_ANALYTICS_EVENTS.CLI_SETUP_SKIPPED,
659 + properties: {
660 + source: 'cli',
661 + invocation_origin: getInvocationOrigin(),
662 + cli_version: params.cliVersion,
663 + command_path: 'setup',
664 + operation: params.operation,
665 + requested_target: params.requestedTarget,
666 + reason: 'no_host_detected',
667 + },
668 +});
669 +
670 export const getToolExecuteValidationFailedEvent = (params: {
671 readonly toolSlug: string;
672 readonly args: Record<string, unknown>;
ts/packages/cli/src/commands/setup.cmd.ts
+44
@@ -1,5 +1,12 @@
1 import { Command, Options } from '@effect/cli';
2 import { Effect, Predicate } from 'effect';
3 +import { trackCliEventEffect } from 'src/analytics/dispatch';
4 +import {
5 + getSetupCancelledEvent,
6 + getSetupHostDetectedEvent,
7 + getSetupSkippedEvent,
8 +} from 'src/analytics/events';
9 +import { APP_VERSION } from 'src/constants';
10 import {
11 detectSetupTargets,
12 inspectSetupTargets,
@@ -64,6 +71,23 @@ const setupBaseCmd = Command.make(
71 yield* ui.intro(uninstall ? 'composio setup --uninstall' : 'composio setup');
72
73 const detections = yield* detectSetupTargets(target);
74 + yield* Effect.forEach(detections, detection =>
75 + trackCliEventEffect(
76 + getSetupHostDetectedEvent({
77 + operation,
78 + requestedTarget: target,
79 + target: detection.target,
80 + available: detection.available,
81 + supported: detection.supported,
82 + hostVersion: detection.version,
83 + unsupportedReasonCode:
84 + detection.available && !detection.supported
85 + ? (detection.unsupportedReasonCode ?? 'unknown')
86 + : undefined,
87 + cliVersion: APP_VERSION,
88 + })
89 + )
90 + );
91 const detected = detections.filter(result => result.available).map(result => result.target);
92 const supported = detections.filter(result => result.available && result.supported);
93 const unsupported = detections.filter(result => result.available && !result.supported);
@@ -96,6 +120,9 @@ const setupBaseCmd = Command.make(
120 );
121 if (supported.length === 0) {
122 if (ifPresent) {
123 + yield* trackCliEventEffect(
124 + getSetupSkippedEvent({ operation, requestedTarget: target, cliVersion: APP_VERSION })
125 + );
126 yield* ui.outro(
127 `No supported agent host detected; plugin ${uninstall ? 'uninstall' : 'setup'} skipped.`
128 );
@@ -117,6 +144,9 @@ const setupBaseCmd = Command.make(
144 operation
145 );
146 }
147 + yield* trackCliEventEffect(
148 + getSetupSkippedEvent({ operation, requestedTarget: target, cliVersion: APP_VERSION })
149 + );
150 yield* ui.outro(
151 `No supported agent host detected; plugin ${uninstall ? 'uninstall' : 'setup'} skipped.`
152 );
@@ -161,6 +191,13 @@ const setupBaseCmd = Command.make(
191 { defaultValue: false }
192 );
193 if (!confirmed) {
194 + yield* trackCliEventEffect(
195 + getSetupCancelledEvent({
196 + operation,
197 + requestedTarget: target,
198 + cliVersion: APP_VERSION,
199 + })
200 + );
201 yield* ui.outro('Uninstall cancelled.');
202 return;
203 }
@@ -204,6 +241,13 @@ const setupBaseCmd = Command.make(
241 : `Finish Composio setup for ${formatTargets(pending.map(status => status.target))}?`;
242 const confirmed = yield* ui.confirm(prompt, { defaultValue: true });
243 if (!confirmed) {
244 + yield* trackCliEventEffect(
245 + getSetupCancelledEvent({
246 + operation,
247 + requestedTarget: target,
248 + cliVersion: APP_VERSION,
249 + })
250 + );
251 yield* ui.outro('Setup cancelled.');
252 return;
253 }
ts/packages/cli/src/services/setup.ts
+73 -7
@@ -1,6 +1,12 @@
1 import { Command, Error as PlatformError } from '@effect/platform';
2 import { Data, Effect, Either, Option, Predicate, Schema } from 'effect';
3 import semver from 'semver';
4 +import { trackCliEventEffect } from 'src/analytics/dispatch';
5 +import {
6 + getPluginLifecycleFailedEvent,
7 + getPluginLifecycleSucceededEvent,
8 +} from 'src/analytics/events';
9 +import { APP_VERSION } from 'src/constants';
10 import { CommandRunner, type CommandResult } from './command-runner';
11 import { SetupSkillInstaller } from './setup-skill-installer';
12
@@ -330,7 +336,11 @@ const supportsInspection = (adapter: SetupTargetAdapter, versionOutput?: string)
336 if (adapter.target === 'codex') {
337 const version = versionOutput?.match(/\b\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?\b/)?.[0];
338 if (!version || !semver.valid(version) || semver.lt(version, MINIMUM_CODEX_SETUP_VERSION)) {
333 - return { supported: false, reason: unsupportedInspectionMessage(adapter) };
339 + return {
340 + supported: false,
341 + reason: unsupportedInspectionMessage(adapter),
342 + reasonCode: 'codex_too_old',
343 + } as const;
344 }
345 }
346
@@ -350,14 +360,19 @@ const supportsInspection = (adapter: SetupTargetAdapter, versionOutput?: string)
360 return {
361 supported: false,
362 reason: capabilityCheckFailureMessage(adapter, command, commandResultDetail(result)),
353 - };
363 + reasonCode: 'host_command_failed',
364 + } as const;
365 }
366 const help = `${result.stdout}\n${result.stderr}`;
367 if (!/^\s*--json\b/m.test(help)) {
357 - return { supported: false, reason: unsupportedInspectionMessage(adapter) };
368 + return {
369 + supported: false,
370 + reason: unsupportedInspectionMessage(adapter),
371 + reasonCode: 'no_json_inspection',
372 + } as const;
373 }
374 }
360 - return { supported: true };
375 + return { supported: true } as const;
376 });
377
378 const detectAdapter = (adapter: SetupTargetAdapter) =>
@@ -375,6 +390,7 @@ const detectAdapter = (adapter: SetupTargetAdapter) =>
390 versionCommand,
391 errorMessage(cause)
392 ),
393 + unsupportedReasonCode: 'host_command_failed' as const,
394 }),
395 onSuccess: result => {
396 if (!result || result.exitCode === 127) {
@@ -389,13 +405,18 @@ const detectAdapter = (adapter: SetupTargetAdapter) =>
405 versionCommand,
406 commandResultDetail(result)
407 ),
408 + unsupportedReasonCode: 'host_command_failed' as const,
409 });
410 }
411
412 const version = result.stdout.trim() || result.stderr.trim();
413 return supportsInspection(adapter, version).pipe(
414 Effect.match({
398 - onFailure: cause => ({ supported: false, reason: errorMessage(cause) }),
415 + onFailure: cause => ({
416 + supported: false as const,
417 + reason: errorMessage(cause),
418 + reasonCode: 'host_command_failed' as const,
419 + }),
420 onSuccess: support => support,
421 }),
422 Effect.map(support =>
@@ -404,12 +425,22 @@ const detectAdapter = (adapter: SetupTargetAdapter) =>
425 available: true,
426 supported: support.supported,
427 version,
407 - ...(support.supported ? {} : { unsupportedReason: support.reason }),
428 + ...(support.supported
429 + ? {}
430 + : {
431 + unsupportedReason: support.reason,
432 + unsupportedReasonCode: support.reasonCode,
433 + }),
434 }
435 : {
436 available: true,
437 supported: support.supported,
412 - ...(support.supported ? {} : { unsupportedReason: support.reason }),
438 + ...(support.supported
439 + ? {}
440 + : {
441 + unsupportedReason: support.reason,
442 + unsupportedReasonCode: support.reasonCode,
443 + }),
444 }
445 )
446 );
@@ -736,12 +767,16 @@ const FIXED_TARGETS: Readonly<Partial<Record<SetupTarget, ReadonlyArray<AgentHos
767 all: ['claude', 'codex'],
768 };
769
770 +export type SetupUnsupportedReasonCode =
771 + 'codex_too_old' | 'no_json_inspection' | 'host_command_failed' | 'unknown';
772 +
773 export interface SetupTargetDetection {
774 readonly target: AgentHost;
775 readonly available: boolean;
776 readonly supported: boolean;
777 readonly version?: string;
778 readonly unsupportedReason?: string;
779 + readonly unsupportedReasonCode?: SetupUnsupportedReasonCode;
780 }
781
782 export const detectSetupTargets = (target: SetupTarget) =>
@@ -786,9 +821,22 @@ const runSetupTargets = <E, R>(
821 verb: 'Setup' | 'Uninstall'
822 ) =>
823 Effect.gen(function* () {
824 + const operation = verb === 'Uninstall' ? 'uninstall' : 'setup';
825 + const phase = verb === 'Uninstall' ? 'uninstall' : 'install';
826 const completed: SetupTargetResult[] = [];
827 for (const status of inspected) {
828 const result = yield* runAdapter(ADAPTERS[status.target], status).pipe(
829 + Effect.tapError(error =>
830 + trackCliEventEffect(
831 + getPluginLifecycleFailedEvent({
832 + operation,
833 + target: status.target,
834 + phase,
835 + error,
836 + cliVersion: APP_VERSION,
837 + })
838 + )
839 + ),
840 Effect.mapError(error => {
841 if (completed.length === 0) return error;
842 const targets = completed.map(item => item.target).join(', ');
@@ -800,6 +848,24 @@ const runSetupTargets = <E, R>(
848 });
849 })
850 );
851 + if (result.plugin_changed) {
852 + const action =
853 + operation === 'uninstall'
854 + ? 'uninstalled'
855 + : !status.plugin_installed
856 + ? 'installed'
857 + : !status.plugin_enabled
858 + ? 'enabled'
859 + : 'configured';
860 + yield* trackCliEventEffect(
861 + getPluginLifecycleSucceededEvent({
862 + operation,
863 + target: result.target,
864 + action,
865 + cliVersion: APP_VERSION,
866 + })
867 + );
868 + }
869 completed.push(result);
870 }
871 return completed;
ts/packages/cli/test/__utils__/services/terminal-ui-test.ts
+58 -59
@@ -7,69 +7,68 @@ import { TerminalUI } from 'src/services/terminal-ui';
7 * all terminal UI output is captured deterministically — no animations,
8 * no ANSI codes, no spinners.
9 */
10 -export const TerminalUITest = Layer.succeed(
11 - TerminalUI,
12 - TerminalUI.of({
13 - capabilities: Effect.succeed({
14 - stdinIsTTY: false,
15 - stdoutIsTTY: false,
16 - stderrIsTTY: false,
17 - isInteractive: false,
18 - canDecorate: false,
19 - }),
20 - output: data => Console.log(data),
21 - error: data => Console.error(data),
10 +export const terminalUITestImpl = TerminalUI.of({
11 + capabilities: Effect.succeed({
12 + stdinIsTTY: false,
13 + stdoutIsTTY: false,
14 + stderrIsTTY: false,
15 + isInteractive: false,
16 + canDecorate: false,
17 + }),
18 + output: data => Console.log(data),
19 + error: data => Console.error(data),
20 +
21 + intro: title => Console.log(`-- ${title} --`),
22 + outro: message => Console.log(`-- ${message} --`),
23
23 - intro: title => Console.log(`-- ${title} --`),
24 - outro: message => Console.log(`-- ${message} --`),
24 + log: {
25 + info: message => Console.log(message),
26 + success: message => Console.log(message),
27 + warn: message => Console.warn(message),
28 + error: message => Console.error(message),
29 + step: message => Console.log(message),
30 + message: message => Console.log(message),
31 + },
32
26 - log: {
27 - info: message => Console.log(message),
28 - success: message => Console.log(message),
29 - warn: message => Console.warn(message),
30 - error: message => Console.error(message),
31 - step: message => Console.log(message),
32 - message: message => Console.log(message),
33 - },
33 + note: (message, title) => Console.log(title ? `[${title}] ${message}` : message),
34
35 - note: (message, title) => Console.log(title ? `[${title}] ${message}` : message),
35 + select: (_message, options) => Effect.succeed(options[0].value),
36
37 - select: (_message, options) => Effect.succeed(options[0].value),
37 + confirm: (_message, options) => Effect.succeed(options?.defaultValue ?? true),
38
39 - confirm: (_message, options) => Effect.succeed(options?.defaultValue ?? true),
39 + withSpinner: (message, effect, options) =>
40 + Effect.gen(function* () {
41 + const result = yield* effect;
42 + const successMsg =
43 + typeof options?.successMessage === 'function'
44 + ? options.successMessage(result)
45 + : (options?.successMessage ?? message);
46 + yield* Console.log(successMsg);
47 + return result;
48 + }),
49
41 - withSpinner: (message, effect, options) =>
42 - Effect.gen(function* () {
43 - const result = yield* effect;
44 - const successMsg =
45 - typeof options?.successMessage === 'function'
46 - ? options.successMessage(result)
47 - : (options?.successMessage ?? message);
48 - yield* Console.log(successMsg);
49 - return result;
50 - }),
50 + useMakeSpinner: (message, use) =>
51 + Effect.gen(function* () {
52 + let stopped = false;
53 + const handle = {
54 + message: (_msg: string) => Effect.void,
55 + stop: (msg?: string) =>
56 + Effect.gen(function* () {
57 + stopped = true;
58 + if (msg) yield* Console.log(msg);
59 + }),
60 + error: (msg?: string) =>
61 + Effect.gen(function* () {
62 + stopped = true;
63 + if (msg) yield* Console.error(msg);
64 + }),
65 + };
66 + const exit = yield* Effect.exit(use(handle));
67 + if (Exit.isFailure(exit) && !stopped) {
68 + yield* Console.error(message);
69 + }
70 + return yield* exit;
71 + }),
72 +});
73
52 - useMakeSpinner: (message, use) =>
53 - Effect.gen(function* () {
54 - let stopped = false;
55 - const handle = {
56 - message: (_msg: string) => Effect.void,
57 - stop: (msg?: string) =>
58 - Effect.gen(function* () {
59 - stopped = true;
60 - if (msg) yield* Console.log(msg);
61 - }),
62 - error: (msg?: string) =>
63 - Effect.gen(function* () {
64 - stopped = true;
65 - if (msg) yield* Console.error(msg);
66 - }),
67 - };
68 - const exit = yield* Effect.exit(use(handle));
69 - if (Exit.isFailure(exit) && !stopped) {
70 - yield* Console.error(message);
71 - }
72 - return yield* exit;
73 - }),
74 - })
75 -);
74 +export const TerminalUITest = Layer.succeed(TerminalUI, terminalUITestImpl);
ts/packages/cli/test/src/analytics.events.test.ts
+181
@@ -1,14 +1,23 @@
1 import { describe, expect, it } from 'vitest';
2 import { createCliCodactFailureBody } from 'src/analytics/dispatch';
3 import {
4 + CLI_ANALYTICS_EVENTS,
5 createCliCommandTelemetryContext,
6 + getPluginLifecycleFailedEvent,
7 + getPluginLifecycleSucceededEvent,
8 + getPrimaryLifecycleFailedEvent,
9 getPrimaryLifecycleInvokedEvent,
10 + getPrimaryLifecycleSucceededEvent,
11 + getSetupCancelledEvent,
12 + getSetupHostDetectedEvent,
13 + getSetupSkippedEvent,
14 getToolExecuteFailedEvent,
15 getToolExecuteToolNotFoundEvent,
16 getToolExecuteValidationFailedEvent,
17 isMaybeToolNotFoundError,
18 isMaybeToolValidationError,
19 } from 'src/analytics/events';
20 +import { APP_VERSION } from 'src/constants';
21 import { ToolInputValidationError } from 'src/services/tool-input-validation';
22
23 describe('CLI analytics execute failure events', () => {
@@ -145,3 +154,175 @@ describe('CLI analytics execute failure events', () => {
154 });
155 });
156 });
157 +
158 +describe('CLI analytics setup and install lifecycle events', () => {
159 + it('tracks setup as a dedicated lifecycle with safe option properties', () => {
160 + const context = createCliCommandTelemetryContext(
161 + ['bun', 'composio', 'setup', '--target', 'codex', '--uninstall', '--yes'],
162 + APP_VERSION,
163 + { stdoutIsTTY: false, stderrIsTTY: false }
164 + );
165 +
166 + const invoked = getPrimaryLifecycleInvokedEvent(context);
167 + const succeeded = getPrimaryLifecycleSucceededEvent(context);
168 + const failed = getPrimaryLifecycleFailedEvent(context, new Error('native failure'));
169 +
170 + expect(invoked?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_INVOKED);
171 + expect(succeeded?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_SUCCEEDED);
172 + expect(failed?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_FAILED);
173 + expect(invoked?.properties).toMatchObject({
174 + command_path: 'setup',
175 + operation: 'uninstall',
176 + target: 'codex',
177 + yes: true,
178 + if_present: false,
179 + stdout_is_tty: expect.any(Boolean),
180 + });
181 + });
182 +
183 + it('tracks shell installation separately from plugin setup', () => {
184 + const context = createCliCommandTelemetryContext(
185 + ['bun', 'composio', 'install', '--completions'],
186 + APP_VERSION,
187 + { stdoutIsTTY: false, stderrIsTTY: false }
188 + );
189 +
190 + expect(getPrimaryLifecycleInvokedEvent(context)).toMatchObject({
191 + name: CLI_ANALYTICS_EVENTS.CLI_INSTALL_INVOKED,
192 + properties: {
193 + command_path: 'install',
194 + completions: true,
195 + no_completions: false,
196 + },
197 + });
198 + });
199 +
200 + it('tracks a verified per-host plugin change', () => {
201 + expect(
202 + getPluginLifecycleSucceededEvent({
203 + operation: 'setup',
204 + target: 'claude',
205 + action: 'installed',
206 + cliVersion: APP_VERSION,
207 + })
208 + ).toMatchObject({
209 + name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_SUCCEEDED,
210 + properties: {
211 + command_path: 'setup',
212 + operation: 'setup',
213 + agent_host: 'claude',
214 + action: 'installed',
215 + },
216 + });
217 + });
218 +});
219 +
220 +describe('CLI analytics setup runtime-context events', () => {
221 + it('tracks a detected and supported host with its version', () => {
222 + expect(
223 + getSetupHostDetectedEvent({
224 + operation: 'setup',
225 + requestedTarget: 'auto',
226 + target: 'claude',
227 + available: true,
228 + supported: true,
229 + hostVersion: '2.1.0',
230 + cliVersion: APP_VERSION,
231 + })
232 + ).toMatchObject({
233 + name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
234 + properties: {
235 + source: 'cli',
236 + command_path: 'setup',
237 + operation: 'setup',
238 + requested_target: 'auto',
239 + agent_host: 'claude',
240 + available: true,
241 + supported: true,
242 + host_version: '2.1.0',
243 + unsupported_reason_code: undefined,
244 + },
245 + });
246 + });
247 +
248 + it('tracks an unsupported host with a normalized reason code', () => {
249 + expect(
250 + getSetupHostDetectedEvent({
251 + operation: 'setup',
252 + requestedTarget: 'auto',
253 + target: 'codex',
254 + available: true,
255 + supported: false,
256 + hostVersion: 'codex-cli 0.137.0',
257 + unsupportedReasonCode: 'codex_too_old',
258 + cliVersion: APP_VERSION,
259 + })
260 + ).toMatchObject({
261 + name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
262 + properties: {
263 + agent_host: 'codex',
264 + available: true,
265 + supported: false,
266 + unsupported_reason_code: 'codex_too_old',
267 + },
268 + });
269 + });
270 +
271 + it('tracks a per-host failure with truncated error details', () => {
272 + const error = new Error(`Adding the claude marketplace failed${'x'.repeat(600)}`);
273 +
274 + const event = getPluginLifecycleFailedEvent({
275 + operation: 'setup',
276 + target: 'claude',
277 + phase: 'install',
278 + error,
279 + cliVersion: APP_VERSION,
280 + });
281 +
282 + expect(event).toMatchObject({
283 + name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_FAILED,
284 + properties: {
285 + command_path: 'setup',
286 + operation: 'setup',
287 + agent_host: 'claude',
288 + phase: 'install',
289 + error_name: 'Error',
290 + },
291 + });
292 + expect(String(event?.properties?.error_message).length).toBeLessThanOrEqual(500);
293 + });
294 +
295 + it('tracks user cancellation and installer skips with normalized reasons', () => {
296 + expect(
297 + getSetupCancelledEvent({
298 + operation: 'setup',
299 + requestedTarget: 'claude',
300 + cliVersion: APP_VERSION,
301 + })
302 + ).toMatchObject({
303 + name: CLI_ANALYTICS_EVENTS.CLI_SETUP_CANCELLED,
304 + properties: {
305 + command_path: 'setup',
306 + operation: 'setup',
307 + requested_target: 'claude',
308 + reason: 'user_declined',
309 + },
310 + });
311 +
312 + expect(
313 + getSetupSkippedEvent({
314 + operation: 'uninstall',
315 + requestedTarget: 'auto',
316 + cliVersion: APP_VERSION,
317 + })
318 + ).toMatchObject({
319 + name: CLI_ANALYTICS_EVENTS.CLI_SETUP_SKIPPED,
320 + properties: {
321 + command_path: 'setup',
322 + operation: 'uninstall',
323 + requested_target: 'auto',
324 + reason: 'no_host_detected',
325 + },
326 + });
327 + });
328 +});
ts/packages/cli/test/src/commands/setup.telemetry.cmd.test.ts new
+373
@@ -0,0 +1,373 @@
1 +import { Command, CommandExecutor } from '@effect/platform';
2 +import { describe, expect, layer } from '@effect/vitest';
3 +import { Effect, Exit } from 'effect';
4 +import { afterEach, beforeEach, vi } from 'vitest';
5 +import { CommandRunner } from 'src/services/command-runner';
6 +import { SetupSkillInstaller } from 'src/services/setup-skill-installer';
7 +import { TerminalUI } from 'src/services/terminal-ui';
8 +import { cli, TestLive } from 'test/__utils__';
9 +import { terminalUITestImpl } from 'test/__utils__/services/terminal-ui-test';
10 +
11 +const tracked = vi.hoisted(() => ({
12 + events: [] as Array<{ readonly name: string; readonly properties?: Record<string, unknown> }>,
13 +}));
14 +
15 +vi.mock('src/analytics/dispatch', async importOriginal => {
16 + const actual = await importOriginal<typeof import('src/analytics/dispatch')>();
17 + const { Effect } = await import('effect');
18 + return {
19 + ...actual,
20 + trackCliEventEffect: (
21 + event: { readonly name: string; readonly properties?: Record<string, unknown> } | null
22 + ) =>
23 + Effect.sync(() => {
24 + if (event) tracked.events.push(event);
25 + }),
26 + };
27 +});
28 +
29 +const eventsNamed = (name: string) => tracked.events.filter(event => event.name === name);
30 +
31 +type AgentHost = 'claude' | 'codex';
32 +
33 +interface FakeHostState {
34 + available: boolean;
35 + marketplace: boolean;
36 + plugin: boolean;
37 +}
38 +
39 +const makeFakeHosts = (
40 + initial: Partial<Record<AgentHost, Partial<FakeHostState>>>,
41 + options: {
42 + readonly failOn?: string;
43 + readonly codexVersion?: string;
44 + } = {}
45 +) => {
46 + const state: Record<AgentHost, FakeHostState> = {
47 + claude: { available: false, marketplace: false, plugin: false, ...initial.claude },
48 + codex: { available: false, marketplace: false, plugin: false, ...initial.codex },
49 + };
50 +
51 + const marketplaceOutput = (host: AgentHost): string => {
52 + if (!state[host].marketplace) return host === 'claude' ? '[]' : '{"marketplaces":[]}';
53 + if (host === 'claude') {
54 + return JSON.stringify([
55 + {
56 + name: 'composio',
57 + source: {
58 + source: 'github',
59 + repo: 'https://github.com/ComposioHQ/composio-plugin-cc.git',
60 + },
61 + },
62 + ]);
63 + }
64 + return JSON.stringify({
65 + marketplaces: [
66 + {
67 + name: 'composio',
68 + marketplaceSource: { sourceType: 'git', source: 'ComposioHQ/composio-plugin-openai' },
69 + },
70 + ],
71 + });
72 + };
73 +
74 + const pluginOutput = (host: AgentHost): string => {
75 + if (host === 'claude') {
76 + if (!state.claude.plugin) return '[]';
77 + return JSON.stringify([{ id: 'composio@composio', scope: 'user', enabled: true }]);
78 + }
79 + if (!state.codex.plugin) return '{"installed":[],"available":[]}';
80 + return JSON.stringify({
81 + installed: [{ pluginId: 'composio@composio', installed: true, enabled: true }],
82 + available: [],
83 + });
84 + };
85 +
86 + const runner = new CommandRunner({
87 + run: () => Effect.succeed(CommandExecutor.ExitCode(0)),
88 + capture: rawCommand => {
89 + const flattened = Command.flatten(rawCommand)[0];
90 + const parts = [flattened.command, ...flattened.args];
91 + const host = parts[0] as AgentHost;
92 + const command = parts.join(' ');
93 + const respond = (result: { exitCode?: number; stdout?: string; stderr?: string }) =>
94 + Effect.succeed({
95 + exitCode: result.exitCode ?? 0,
96 + stdout: result.stdout ?? '',
97 + stderr: result.stderr ?? '',
98 + });
99 +
100 + if (parts[1] === '--version') {
101 + if (!state[host].available) return respond({ exitCode: 127, stderr: 'not found' });
102 + return respond({
103 + stdout: host === 'claude' ? '2.1.0' : (options.codexVersion ?? 'codex-cli 0.144.1'),
104 + });
105 + }
106 + if (command.endsWith('--help')) {
107 + return respond({ stdout: `Usage: ${host} plugin\n\n --json` });
108 + }
109 + if (options.failOn && command.includes(options.failOn)) {
110 + return respond({ exitCode: 2, stderr: 'native operation failed' });
111 + }
112 + if (command === `${host} plugin marketplace list --json`) {
113 + return respond({ stdout: marketplaceOutput(host) });
114 + }
115 + if (command === `${host} plugin list --json`) {
116 + return respond({ stdout: pluginOutput(host) });
117 + }
118 + if (command.includes('plugin marketplace add')) {
119 + state[host].marketplace = true;
120 + return respond({});
121 + }
122 + if (
123 + command === 'claude plugin uninstall composio@composio --scope user --yes' ||
124 + command === 'codex plugin remove composio@composio --json'
125 + ) {
126 + state[host].plugin = false;
127 + return respond({});
128 + }
129 + if (
130 + command.includes('plugin install') ||
131 + command.includes('plugin enable') ||
132 + command === 'codex plugin add composio@composio --json'
133 + ) {
134 + state[host].plugin = true;
135 + return respond({});
136 + }
137 + return respond({});
138 + },
139 + });
140 +
141 + return { runner, state };
142 +};
143 +
144 +const makeSkillInstaller = (initiallyReady = false) => {
145 + let ready = initiallyReady;
146 + return new SetupSkillInstaller({
147 + isClaudeSkillReady: Effect.sync(() => ready),
148 + hasManagedClaudeSkill: Effect.sync(() => ready),
149 + ensureClaudeSkill: Effect.sync(() => {
150 + const changed = !ready;
151 + ready = true;
152 + return changed;
153 + }),
154 + removeClaudeSkill: Effect.sync(() => {
155 + const changed = ready;
156 + ready = false;
157 + return changed;
158 + }),
159 + });
160 +};
161 +
162 +// The cancellation path requires an interactive terminal to reach the confirm prompt.
163 +const decliningUI = TerminalUI.of({
164 + ...terminalUITestImpl,
165 + capabilities: Effect.succeed({
166 + stdinIsTTY: false,
167 + stdoutIsTTY: false,
168 + stderrIsTTY: false,
169 + isInteractive: true,
170 + canDecorate: false,
171 + }),
172 + confirm: () => Effect.succeed(false),
173 +});
174 +
175 +describe('CLI: composio setup telemetry', () => {
176 + beforeEach(() => {
177 + tracked.events.length = 0;
178 + });
179 +
180 + afterEach(() => {
181 + process.exitCode = undefined;
182 + });
183 +
184 + const freshClaude = makeFakeHosts({ claude: { available: true } });
185 + layer(
186 + TestLive({
187 + commandRunner: freshClaude.runner,
188 + setupSkillInstaller: makeSkillInstaller(),
189 + })
190 + )('fresh Claude install', it => {
191 + it.scoped('tracks per-host detection and a verified plugin install', () =>
192 + Effect.gen(function* () {
193 + yield* cli(['setup', '--target', 'auto', '--yes']);
194 +
195 + const detectedEvents = eventsNamed('CLI_SETUP_HOST_DETECTED');
196 + expect(detectedEvents).toHaveLength(2);
197 + expect(detectedEvents.map(event => event.properties)).toEqual(
198 + expect.arrayContaining([
199 + expect.objectContaining({
200 + agent_host: 'claude',
201 + available: true,
202 + supported: true,
203 + host_version: '2.1.0',
204 + unsupported_reason_code: undefined,
205 + operation: 'setup',
206 + requested_target: 'auto',
207 + source: 'cli',
208 + }),
209 + expect.objectContaining({
210 + agent_host: 'codex',
211 + available: false,
212 + supported: false,
213 + unsupported_reason_code: undefined,
214 + }),
215 + ])
216 + );
217 +
218 + expect(eventsNamed('CLI_PLUGIN_SETUP_SUCCEEDED')).toEqual([
219 + expect.objectContaining({
220 + properties: expect.objectContaining({
221 + agent_host: 'claude',
222 + operation: 'setup',
223 + action: 'installed',
224 + }),
225 + }),
226 + ]);
227 + expect(eventsNamed('CLI_PLUGIN_SETUP_FAILED')).toHaveLength(0);
228 + })
229 + );
230 + });
231 +
232 + const legacyCodex = makeFakeHosts(
233 + { codex: { available: true } },
234 + { codexVersion: 'codex-cli 0.137.0' }
235 + );
236 + layer(TestLive({ commandRunner: legacyCodex.runner }))('unsupported Codex only', it => {
237 + it.scoped('tracks the normalized reason code and the installer skip', () =>
238 + Effect.gen(function* () {
239 + const exit = yield* Effect.exit(
240 + cli(['setup', '--target', 'auto', '--yes', '--if-present'])
241 + );
242 + expect(Exit.isSuccess(exit)).toBe(true);
243 +
244 + expect(eventsNamed('CLI_SETUP_HOST_DETECTED')).toEqual(
245 + expect.arrayContaining([
246 + expect.objectContaining({
247 + properties: expect.objectContaining({
248 + agent_host: 'codex',
249 + available: true,
250 + supported: false,
251 + unsupported_reason_code: 'codex_too_old',
252 + }),
253 + }),
254 + ])
255 + );
256 + expect(eventsNamed('CLI_SETUP_SKIPPED')).toEqual([
257 + expect.objectContaining({
258 + properties: expect.objectContaining({
259 + operation: 'setup',
260 + requested_target: 'auto',
261 + reason: 'no_host_detected',
262 + }),
263 + }),
264 + ]);
265 + })
266 + );
267 + });
268 +
269 + const noHosts = makeFakeHosts({});
270 + layer(TestLive({ commandRunner: noHosts.runner }))('no detected host', it => {
271 + it.scoped('tracks the installer skip when nothing is detected', () =>
272 + Effect.gen(function* () {
273 + const exit = yield* Effect.exit(
274 + cli(['setup', '--target', 'auto', '--yes', '--if-present'])
275 + );
276 + expect(Exit.isSuccess(exit)).toBe(true);
277 +
278 + expect(eventsNamed('CLI_SETUP_SKIPPED')).toEqual([
279 + expect.objectContaining({
280 + properties: expect.objectContaining({ reason: 'no_host_detected' }),
281 + }),
282 + ]);
283 + expect(eventsNamed('CLI_SETUP_CANCELLED')).toHaveLength(0);
284 + })
285 + );
286 + });
287 +
288 + const declinedSetup = makeFakeHosts({ claude: { available: true } });
289 + layer(
290 + TestLive({
291 + commandRunner: declinedSetup.runner,
292 + setupSkillInstaller: makeSkillInstaller(),
293 + terminalUI: decliningUI,
294 + })
295 + )('declined interactive setup', it => {
296 + it.scoped('tracks user cancellation without mutating the host', () =>
297 + Effect.gen(function* () {
298 + yield* cli(['setup', '--target', 'claude']);
299 +
300 + expect(eventsNamed('CLI_SETUP_CANCELLED')).toEqual([
301 + expect.objectContaining({
302 + properties: expect.objectContaining({
303 + operation: 'setup',
304 + requested_target: 'claude',
305 + reason: 'user_declined',
306 + }),
307 + }),
308 + ]);
309 + expect(declinedSetup.state.claude.plugin).toBe(false);
310 + expect(eventsNamed('CLI_PLUGIN_SETUP_SUCCEEDED')).toHaveLength(0);
311 + })
312 + );
313 + });
314 +
315 + const declinedUninstall = makeFakeHosts({
316 + claude: { available: true, marketplace: true, plugin: true },
317 + });
318 + layer(
319 + TestLive({
320 + commandRunner: declinedUninstall.runner,
321 + setupSkillInstaller: makeSkillInstaller(true),
322 + terminalUI: decliningUI,
323 + })
324 + )('declined interactive uninstall', it => {
325 + it.scoped('tracks user cancellation for the uninstall operation', () =>
326 + Effect.gen(function* () {
327 + yield* cli(['setup', '--uninstall', '--target', 'claude']);
328 +
329 + expect(eventsNamed('CLI_SETUP_CANCELLED')).toEqual([
330 + expect.objectContaining({
331 + properties: expect.objectContaining({
332 + operation: 'uninstall',
333 + requested_target: 'claude',
334 + reason: 'user_declined',
335 + }),
336 + }),
337 + ]);
338 + expect(declinedUninstall.state.claude.plugin).toBe(true);
339 + })
340 + );
341 + });
342 +
343 + const failedInstall = makeFakeHosts(
344 + { claude: { available: true } },
345 + { failOn: 'plugin install' }
346 + );
347 + layer(
348 + TestLive({
349 + commandRunner: failedInstall.runner,
350 + setupSkillInstaller: makeSkillInstaller(),
351 + })
352 + )('native install failure', it => {
353 + it.scoped('tracks the per-host failure with its phase', () =>
354 + Effect.gen(function* () {
355 + const exit = yield* Effect.exit(cli(['setup', '--target', 'claude', '--yes']));
356 + expect(Exit.isFailure(exit)).toBe(true);
357 +
358 + expect(eventsNamed('CLI_PLUGIN_SETUP_FAILED')).toEqual([
359 + expect.objectContaining({
360 + properties: expect.objectContaining({
361 + agent_host: 'claude',
362 + operation: 'setup',
363 + phase: 'install',
364 + error_name: 'services/SetupProcessError',
365 + error_message: expect.stringContaining('native operation failed'),
366 + }),
367 + }),
368 + ]);
369 + expect(eventsNamed('CLI_PLUGIN_SETUP_SUCCEEDED')).toHaveLength(0);
370 + })
371 + );
372 + });
373 +});