Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 831 lines 28.2 KB
Raw
1 import { VERSION as clientLibraryVersion } from '@composio/client/effect';
2 import { readFileSync, writeFileSync } from 'node:fs';
3 import path from 'node:path';
4 import * as tempy from 'tempy';
5 import { it } from '@effect/vitest';
6 import { afterEach, describe, expect, vi } from 'vitest';
7 import * as BunFileSystem from '@effect/platform-bun/BunFileSystem';
8 import * as BunPath from '@effect/platform-bun/BunPath';
9 import { Effect, Layer, Option } from 'effect';
10 import { CliError } from 'effect/cli';
11 import { createCliCodactFailureBody } from 'src/analytics/dispatch';
12 import {
13 CLI_ANALYTICS_EVENTS,
14 CLI_EVENT_JOURNEY_STAGES,
15 CLI_JOURNEY_STAGES,
16 configureCliAnalyticsReleaseVersion,
17 getPluginHintShownEvent,
18 getPluginLifecycleFailedEvent,
19 getPluginLifecycleSucceededEvent,
20 getPrimaryLifecycleFailedEvent,
21 getPrimaryLifecycleInvokedEvent,
22 getPrimaryLifecycleSucceededEvent,
23 getSetupCancelledEvent,
24 getSetupHostDetectedEvent,
25 getSetupSkippedEvent,
26 getToolExecuteFailedEvent,
27 getToolExecuteToolNotFoundEvent,
28 getToolExecuteValidationFailedEvent,
29 isMaybeToolNotFoundError,
30 isMaybeToolValidationError,
31 } from 'src/analytics/events';
32 import { MissingActingUserError } from 'src/commands/surface';
33 import { APP_VERSION } from 'src/constants';
34 import { inferSkillReleaseChannel } from 'src/effects/install-skill';
35 import { CLI_RELEASE_CHANNELS } from 'src/experimental-features';
36 import {
37 DEFAULT_CLI_INVOCATION_ORIGIN,
38 type CliInvocationContext,
39 } from 'src/services/runtime-cli-context';
40 import { ChildProcessExited } from 'src/effects/child-process-exited';
41 import { SetupCommandError } from 'src/services/setup-command-error';
42 import { ToolInputValidationError } from 'src/services/tool-input-validation';
43 import { resolveInstalledCliVersion } from 'src/services/run-companion-modules';
44 import type { TelemetryFamily } from 'src/analytics/types';
45 import { commandTelemetryContext } from 'test/__utils__/telemetry-context';
46
47 const CLI_INVOCATION = {
48 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
49 parentRunId: undefined,
50 } as const;
51
52 /** The context of `args` with this file's defaults: the package version, `cli` origin. */
53 const commandContext = (
54 args: ReadonlyArray<string>,
55 options: Parameters<typeof commandTelemetryContext>[1]
56 ) => commandTelemetryContext(args, { cliVersion: APP_VERSION, ...options });
57
58 describe('CLI analytics execute failure events', () => {
59 it('records terminal capabilities supplied by the terminal service', () => {
60 const context = commandContext([], {
61 path: [],
62 cliVersion: '0.3.0',
63 terminal: { stdoutIsTTY: true, stderrIsTTY: false },
64 });
65
66 const event = getPrimaryLifecycleInvokedEvent(context);
67
68 expect(event?.properties).toMatchObject({
69 stdout_is_tty: true,
70 stderr_is_tty: false,
71 });
72 });
73
74 it('keeps CLI product and installed library separate on tool-router command events', () => {
75 const context = commandContext(['execute', 'GITHUB_GET_ME'], {
76 path: ['execute'],
77 family: 'execute',
78 input: { slug: 'GITHUB_GET_ME' },
79 cliVersion: '0.4.2',
80 });
81 expect(getPrimaryLifecycleSucceededEvent(context)?.properties).toMatchObject({
82 client_name: '@composio/cli',
83 client_version: '0.4.2',
84 client_library: '@composio/client',
85 client_library_version: clientLibraryVersion,
86 execution_channel: 'tool_router',
87 duration_ms: expect.any(Number),
88 });
89 });
90
91 it('marks cached-schema validation failures as fast_fail', () => {
92 const error = new ToolInputValidationError({
93 toolSlug: 'GMAIL_SEND_EMAIL',
94 schemaPath: '/tmp/schema.json',
95 issues: ['Unknown key "recipient"'],
96 });
97
98 const event = getToolExecuteValidationFailedEvent({
99 toolSlug: 'GMAIL_SEND_EMAIL',
100 args: { recipient: 'a@example.com' },
101 error,
102 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
103 surface: 'root',
104 projectMode: 'consumer',
105 stage: 'validation',
106 failureOrigin: 'fast_fail',
107 });
108
109 expect(event).not.toBeNull();
110 expect(event!.properties?.failure_origin).toBe('fast_fail');
111 expect(event!.properties?.tool_log_id).toBeUndefined();
112 });
113
114 it('marks endpoint tool-not-found failures as main_endpoint and keeps log id', () => {
115 const event = getToolExecuteToolNotFoundEvent({
116 toolSlug: 'FAKE_TOOL',
117 args: {},
118 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
119 surface: 'root',
120 projectMode: 'consumer',
121 stage: 'execution',
122 failureOrigin: 'main_endpoint',
123 logId: 'log_123',
124 message: 'Tool not found',
125 status: 404,
126 });
127
128 expect(event).not.toBeNull();
129 expect(event!.properties?.failure_origin).toBe('main_endpoint');
130 expect(event!.properties?.tool_log_id).toBe('log_123');
131 });
132
133 it('marks endpoint execution failures as main_endpoint and keeps log id', () => {
134 const event = getToolExecuteFailedEvent({
135 toolSlug: 'GMAIL_SEND_EMAIL',
136 args: { to: 'a@example.com' },
137 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
138 surface: 'root',
139 projectMode: 'consumer',
140 stage: 'execution',
141 failureOrigin: 'main_endpoint',
142 logId: 'log_456',
143 message: 'Invalid tool arguments',
144 });
145
146 expect(event).not.toBeNull();
147 expect(event!.properties?.failure_origin).toBe('main_endpoint');
148 expect(event!.properties?.tool_log_id).toBe('log_456');
149 });
150
151 it('classifies known Hermes tool-not-found codes', () => {
152 expect(
153 isMaybeToolNotFoundError({
154 apiCode: 2306,
155 message: 'random',
156 })
157 ).toBe(true);
158 expect(
159 isMaybeToolNotFoundError({
160 apiCode: 3703,
161 message: 'random',
162 })
163 ).toBe(true);
164 });
165
166 it('classifies known Hermes validation codes', () => {
167 expect(
168 isMaybeToolValidationError({
169 apiCode: 3702,
170 message: 'random',
171 })
172 ).toBe(true);
173 expect(
174 isMaybeToolValidationError({
175 apiCode: 1149,
176 message: 'random',
177 })
178 ).toBe(true);
179 });
180
181 it('builds the CLI codact failure body using backend field names', () => {
182 const body = createCliCodactFailureBody({
183 failureType: 'wrong_tool_slug',
184 toolInfo: {
185 toolkit: 'github',
186 },
187 ctx: {
188 invalid_tool_slug: 'GITHUB_MAKE_ISSUE',
189 },
190 session: {
191 command_path: 'execute',
192 },
193 requestId: 'req_123',
194 });
195
196 expect(body).toMatchObject({
197 failure_type: 'wrong_tool_slug',
198 tool_info: {
199 toolkit: 'github',
200 },
201 ctx: {
202 invalid_tool_slug: 'GITHUB_MAKE_ISSUE',
203 },
204 session: {
205 source: 'cli',
206 command_path: 'execute',
207 cli_version: expect.any(String),
208 },
209 request_id: 'req_123',
210 });
211 });
212 });
213
214 describe('parsed telemetry options', () => {
215 it('records canonical option names without retaining flag values', () => {
216 const secret = '--customer-private';
217 const context = commandContext(['execute', 'GMAIL_SEND_EMAIL', '-d', secret], {
218 path: ['execute'],
219 family: 'execute',
220 input: { slug: 'GMAIL_SEND_EMAIL', data: Option.some(secret), dryRun: true },
221 });
222 expect(context.flagNames).toEqual(['--data', '--dry-run']);
223 const event = getPrimaryLifecycleInvokedEvent(context);
224 expect(event?.properties).toMatchObject({ has_data: true, dry_run: true });
225 expect(JSON.stringify(event)).not.toContain(secret);
226 expect(JSON.stringify(context.options)).not.toContain(secret);
227 });
228 it('uses parsed proxy values and counts repeated headers without collecting their content', () => {
229 const context = commandContext(
230 ['proxy', '/path', '-t', 'gmail', '-H', 'Authorization: secret'],
231 {
232 path: ['proxy'],
233 family: 'proxy',
234 input: {
235 endpoint: '/path',
236 toolkit: 'gmail',
237 method: 'POST',
238 headers: ['a: b', 'c: d'],
239 data: Option.none(),
240 },
241 }
242 );
243 const event = getPrimaryLifecycleInvokedEvent(context);
244 expect(event?.properties).toMatchObject({
245 toolkit: 'gmail',
246 method: 'POST',
247 header_count: 2,
248 has_data: false,
249 });
250 expect(JSON.stringify(event)).not.toContain('secret');
251 });
252 it('honors a parsed false boolean even when its flag is present', () => {
253 const context = commandContext(['run', '--dry-run=false', 'code'], {
254 path: ['run'],
255 family: 'run',
256 input: { args: ['code'], dryRun: false },
257 });
258 expect(getPrimaryLifecycleInvokedEvent(context)?.properties).toMatchObject({ dry_run: false });
259 });
260 });
261
262 describe('CLI analytics setup and install lifecycle events', () => {
263 it('tracks setup as a dedicated lifecycle with safe option properties', () => {
264 const context = commandContext(['setup', '--target', 'codex', '--uninstall', '--yes'], {
265 path: ['setup'],
266 family: 'setup',
267 input: { target: 'codex', uninstall: true, yes: true, ifPresent: false },
268 });
269
270 const invoked = getPrimaryLifecycleInvokedEvent(context);
271 const succeeded = getPrimaryLifecycleSucceededEvent(context);
272 const failed = getPrimaryLifecycleFailedEvent(context, new Error('native failure'));
273
274 expect(invoked?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_INVOKED);
275 expect(succeeded?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_SUCCEEDED);
276 expect(failed?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_FAILED);
277 expect(invoked?.properties).toMatchObject({
278 command_path: 'setup',
279 operation: 'uninstall',
280 target: 'codex',
281 yes: true,
282 if_present: false,
283 stdout_is_tty: expect.any(Boolean),
284 });
285 });
286
287 it('tracks shell installation separately from plugin setup', () => {
288 const context = commandContext(['install', '--completions'], {
289 path: ['install'],
290 family: 'install',
291 input: { completions: true, noCompletions: false },
292 });
293
294 expect(getPrimaryLifecycleInvokedEvent(context)).toMatchObject({
295 name: CLI_ANALYTICS_EVENTS.CLI_INSTALL_INVOKED,
296 properties: {
297 command_path: 'install',
298 completions: true,
299 no_completions: false,
300 },
301 });
302 });
303
304 it('summarizes user-controlled search and proxy input without recording it', () => {
305 const endpoint = 'https://user:password@example.com/private?token=small';
306 const search = getPrimaryLifecycleInvokedEvent(
307 commandContext(['search', 'customer@example.com secret'], {
308 path: ['search'],
309 family: 'search',
310 input: { query: ['customer@example.com secret'] },
311 })
312 );
313 const proxy = getPrimaryLifecycleInvokedEvent(
314 commandContext(['proxy', endpoint], { path: ['proxy'], family: 'proxy', input: { endpoint } })
315 );
316
317 expect(search?.properties).toMatchObject({
318 query_length: 'customer@example.com secret'.length,
319 query_term_count: 2,
320 });
321 expect(search?.properties).not.toHaveProperty('query');
322 expect(search?.properties).not.toHaveProperty('search_query');
323 expect(proxy?.properties).toMatchObject({ has_endpoint: true });
324 expect(proxy?.properties).not.toHaveProperty('endpoint');
325 });
326
327 it('tracks a verified per-host plugin change', () => {
328 expect(
329 getPluginLifecycleSucceededEvent({
330 operation: 'setup',
331 target: 'claude',
332 action: 'installed',
333 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
334 cliVersion: APP_VERSION,
335 })
336 ).toMatchObject({
337 name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_SUCCEEDED,
338 properties: {
339 command_path: 'setup',
340 operation: 'setup',
341 agent_host: 'claude',
342 action: 'installed',
343 },
344 });
345 });
346 });
347
348 describe('CLI analytics setup runtime-context events', () => {
349 it('tracks a detected and supported host with its version', () => {
350 expect(
351 getSetupHostDetectedEvent({
352 operation: 'setup',
353 requestedTarget: 'auto',
354 target: 'claude',
355 available: true,
356 supported: true,
357 hostVersion: '2.1.0',
358 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
359 cliVersion: APP_VERSION,
360 })
361 ).toMatchObject({
362 name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
363 properties: {
364 source: 'cli',
365 command_path: 'setup',
366 operation: 'setup',
367 requested_target: 'auto',
368 agent_host: 'claude',
369 available: true,
370 supported: true,
371 host_version: '2.1.0',
372 unsupported_reason_code: undefined,
373 },
374 });
375 });
376
377 it('records host presence signals only for an undetected host', () => {
378 const missing = getSetupHostDetectedEvent({
379 operation: 'setup',
380 requestedTarget: 'auto',
381 target: 'codex',
382 available: false,
383 supported: false,
384 hostConfigDirPresent: true,
385 hostBinaryInKnownPaths: false,
386 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
387 cliVersion: APP_VERSION,
388 });
389 expect(missing?.properties).toMatchObject({
390 agent_host: 'codex',
391 available: false,
392 host_config_dir_present: true,
393 host_binary_in_known_paths: false,
394 });
395
396 const detected = getSetupHostDetectedEvent({
397 operation: 'setup',
398 requestedTarget: 'auto',
399 target: 'claude',
400 available: true,
401 supported: true,
402 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
403 cliVersion: APP_VERSION,
404 });
405 expect(detected?.properties?.host_config_dir_present).toBeUndefined();
406 expect(detected?.properties?.host_binary_in_known_paths).toBeUndefined();
407 });
408
409 it('tracks a printed plugin hint as a setup-stage event', () => {
410 expect(
411 getPluginHintShownEvent({
412 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
413 cliVersion: APP_VERSION,
414 commandPath: 'whoami',
415 agentHost: 'claude',
416 })
417 ).toEqual({
418 name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_HINT_SHOWN,
419 properties: {
420 source: 'cli',
421 invocation_origin: DEFAULT_CLI_INVOCATION_ORIGIN,
422 cli_version: APP_VERSION,
423 command_path: 'whoami',
424 agent_host: 'claude',
425 client_name: '@composio/cli',
426 client_version: APP_VERSION,
427 client_language: 'typescript',
428 client_runtime: 'nodejs',
429 client_runtime_version: process.versions.node,
430 client_library: '@composio/client',
431 client_library_version: clientLibraryVersion,
432 client_framework: 'cli',
433 execution_channel: 'unknown',
434 journey_stage: 'setup',
435 cli_channel: inferSkillReleaseChannel(APP_VERSION),
436 },
437 });
438 });
439
440 // Execute, search, link, connections, listen and proxy each used to report their own class.
441 it.each([
442 { path: ['execute'], family: 'execute' },
443 { path: ['search'], family: 'search' },
444 { path: ['link'], family: 'link' },
445 { path: ['connections', 'list'], family: 'command' },
446 { path: ['listen'], family: 'command' },
447 { path: ['proxy'], family: 'proxy' },
448 ] as const)('reports a missing user on `$path` as MissingActingUserError', ({ path, family }) => {
449 const context = commandContext(path, { path, family });
450 const error = new MissingActingUserError({
451 tree: 'root',
452 projectName: Option.none(),
453 message: 'Missing consumer user id. Run `composio login` and try again.',
454 });
455
456 expect(getPrimaryLifecycleFailedEvent(context, error)?.properties).toMatchObject({
457 error_name: 'commands/MissingActingUserError',
458 });
459 });
460
461 it('records the setup failure reason code carried by SetupCommandError', () => {
462 const context = commandContext(['setup'], { path: ['setup'], family: 'setup', input: {} });
463 const error = new SetupCommandError({
464 message: 'Non-interactive setup requires `--yes` to approve local changes.',
465 operation: 'setup',
466 reasonCode: 'non_interactive_requires_yes',
467 });
468
469 expect(getPrimaryLifecycleFailedEvent(context, error)?.properties).toMatchObject({
470 error_name: 'services/SetupCommandError',
471 failure_reason_code: 'non_interactive_requires_yes',
472 });
473 const unknown = new SetupCommandError({
474 message: 'native failure',
475 operation: 'setup',
476 reasonCode: 'unknown',
477 });
478 expect(getPrimaryLifecycleFailedEvent(context, unknown)?.properties?.failure_reason_code).toBe(
479 'unknown'
480 );
481 expect(
482 getPrimaryLifecycleFailedEvent(context, new Error('boom'))?.properties
483 ).not.toHaveProperty('failure_reason_code');
484 });
485
486 it('records a run whose child exited non-zero as failed with reason child_exit', () => {
487 const context = commandContext(['run', 'process.exit(3)'], {
488 path: ['run'],
489 family: 'run',
490 input: {},
491 });
492
493 const event = getPrimaryLifecycleFailedEvent(context, new ChildProcessExited({ code: 3 }));
494
495 expect(event?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_RUN_FAILED);
496 expect(event?.properties).toMatchObject({
497 error_name: 'effects/ChildProcessExited',
498 failure_reason_code: 'child_exit',
499 });
500 expect(
501 getPrimaryLifecycleFailedEvent(context, new Error('boom'))?.properties
502 ).not.toHaveProperty('failure_reason_code');
503 });
504
505 it('tracks an unsupported host with a normalized reason code', () => {
506 expect(
507 getSetupHostDetectedEvent({
508 operation: 'setup',
509 requestedTarget: 'auto',
510 target: 'codex',
511 available: true,
512 supported: false,
513 hostVersion: 'codex-cli 0.137.0',
514 unsupportedReasonCode: 'codex_too_old',
515 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
516 cliVersion: APP_VERSION,
517 })
518 ).toMatchObject({
519 name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED,
520 properties: {
521 agent_host: 'codex',
522 available: true,
523 supported: false,
524 unsupported_reason_code: 'codex_too_old',
525 },
526 });
527 });
528
529 it('tracks a per-host failure without recording its free-form message', () => {
530 const error = new Error(`Adding the claude marketplace failed${'x'.repeat(600)}`);
531
532 const event = getPluginLifecycleFailedEvent({
533 operation: 'setup',
534 target: 'claude',
535 phase: 'install',
536 error,
537 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
538 cliVersion: APP_VERSION,
539 });
540
541 expect(event).toMatchObject({
542 name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_FAILED,
543 properties: {
544 command_path: 'setup',
545 operation: 'setup',
546 agent_host: 'claude',
547 phase: 'install',
548 error_name: 'Error',
549 },
550 });
551 expect(event?.properties).not.toHaveProperty('error_message');
552 });
553
554 it('tracks user cancellation and installer skips with normalized reasons', () => {
555 expect(
556 getSetupCancelledEvent({
557 operation: 'setup',
558 requestedTarget: 'claude',
559 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
560 cliVersion: APP_VERSION,
561 })
562 ).toMatchObject({
563 name: CLI_ANALYTICS_EVENTS.CLI_SETUP_CANCELLED,
564 properties: {
565 command_path: 'setup',
566 operation: 'setup',
567 requested_target: 'claude',
568 reason: 'user_declined',
569 },
570 });
571
572 expect(
573 getSetupSkippedEvent({
574 operation: 'uninstall',
575 requestedTarget: 'auto',
576 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
577 cliVersion: APP_VERSION,
578 })
579 ).toMatchObject({
580 name: CLI_ANALYTICS_EVENTS.CLI_SETUP_SKIPPED,
581 properties: {
582 command_path: 'setup',
583 operation: 'uninstall',
584 requested_target: 'auto',
585 reason: 'no_host_detected',
586 },
587 });
588 });
589 });
590
591 describe('CLI analytics journey taxonomy', () => {
592 afterEach(() => {
593 configureCliAnalyticsReleaseVersion(APP_VERSION);
594 vi.unstubAllEnvs();
595 });
596
597 /** The context of a root command whose name is its family, or the generic `version`. */
598 const contextFor = (
599 argv: ReadonlyArray<string>,
600 invocation: CliInvocationContext = CLI_INVOCATION
601 ) => {
602 const [name = 'version'] = argv;
603 return commandContext(argv, {
604 path: [name],
605 family: name === 'version' ? 'command' : (name as TelemetryFamily),
606 invocation,
607 });
608 };
609
610 const lifecycleCases: ReadonlyArray<[ReadonlyArray<string>, string]> = [
611 [['execute', 'GMAIL_SEND_EMAIL'], 'execute'],
612 [['search', 'send email'], 'other'],
613 [['link', 'github'], 'connect'],
614 [['login'], 'login'],
615 [['logout'], 'other'],
616 [['proxy', '/api/v3/toolkits'], 'other'],
617 [['run', 'echo hi'], 'other'],
618 [['install'], 'install'],
619 [['setup'], 'setup'],
620 [['version'], 'other'],
621 ];
622
623 it.each(lifecycleCases)('stamps %j lifecycle events with journey_stage %s', (argv, stage) => {
624 const context = contextFor(argv);
625
626 expect(getPrimaryLifecycleInvokedEvent(context)?.properties?.journey_stage).toBe(stage);
627 expect(getPrimaryLifecycleSucceededEvent(context)?.properties?.journey_stage).toBe(stage);
628 expect(
629 getPrimaryLifecycleFailedEvent(context, new Error('boom'))?.properties?.journey_stage
630 ).toBe(stage);
631 });
632
633 it.each(Object.values(CLI_ANALYTICS_EVENTS))(
634 'maps analytics event %s to a declared journey stage',
635 name => {
636 expect(CLI_JOURNEY_STAGES).toContain(CLI_EVENT_JOURNEY_STAGES[name]);
637 }
638 );
639
640 it('stamps standalone setup and tool-invocation events with their stages', () => {
641 expect(
642 getPluginLifecycleSucceededEvent({
643 operation: 'setup',
644 target: 'claude',
645 action: 'installed',
646 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
647 cliVersion: APP_VERSION,
648 })?.properties?.journey_stage
649 ).toBe('setup');
650
651 expect(
652 getSetupSkippedEvent({
653 operation: 'setup',
654 requestedTarget: 'auto',
655 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
656 cliVersion: APP_VERSION,
657 })?.properties?.journey_stage
658 ).toBe('setup');
659
660 expect(
661 getToolExecuteFailedEvent({
662 toolSlug: 'GMAIL_SEND_EMAIL',
663 args: {},
664 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
665 surface: 'root',
666 projectMode: 'consumer',
667 stage: 'execution',
668 failureOrigin: 'main_endpoint',
669 })?.properties?.journey_stage
670 ).toBe('execute');
671 });
672
673 it('stamps events with the release channel of the running build', () => {
674 const properties = getPrimaryLifecycleInvokedEvent(contextFor(['login']))?.properties;
675
676 expect(properties?.cli_channel).toBe(inferSkillReleaseChannel(APP_VERSION));
677 expect(CLI_RELEASE_CHANNELS).toContain(properties?.cli_channel);
678 });
679
680 it.effect('uses beta release metadata even when the package version is stable', () => {
681 const installDir = tempy.temporaryDirectory();
682 const execPath = path.join(installDir, 'composio');
683 writeFileSync(execPath, 'fake binary');
684 writeFileSync(path.join(installDir, 'release-tag.txt'), '@composio/cli@0.3.1-beta.7\n');
685
686 return Effect.gen(function* () {
687 const resolvedVersion = yield* resolveInstalledCliVersion(execPath, APP_VERSION);
688 configureCliAnalyticsReleaseVersion(resolvedVersion);
689
690 const context = commandContext(['login'], {
691 path: ['login'],
692 family: 'login',
693 input: {},
694 cliVersion: resolvedVersion,
695 });
696 const properties = getPrimaryLifecycleInvokedEvent(context)?.properties;
697 const pluginProperties = getPluginLifecycleSucceededEvent({
698 operation: 'setup',
699 target: 'codex',
700 action: 'installed',
701 invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN,
702 cliVersion: APP_VERSION,
703 })?.properties;
704 expect(inferSkillReleaseChannel(APP_VERSION)).toBe('stable');
705 expect(properties?.cli_channel).toBe('beta');
706 expect(pluginProperties?.cli_channel).toBe('beta');
707 }).pipe(Effect.provide(Layer.merge(BunFileSystem.layer, BunPath.layer)));
708 });
709
710 it('propagates the configured installer invocation origin', () => {
711 const installerInvocation = { invocationOrigin: 'installer', parentRunId: undefined };
712
713 expect(
714 getPrimaryLifecycleInvokedEvent(contextFor(['install'], installerInvocation))?.properties
715 ).toMatchObject({
716 invocation_origin: 'installer',
717 journey_stage: 'install',
718 });
719 });
720
721 it('mints a run id for run and reads the execute slug from the parsed input', () => {
722 const run = commandContext(['--telemetry-debug', 'run', 'console.log("hi")'], {
723 path: ['run'],
724 family: 'run',
725 input: {},
726 });
727 expect(run.runId).toEqual(expect.any(String));
728 expect(getPrimaryLifecycleInvokedEvent(run)).toMatchObject({
729 name: CLI_ANALYTICS_EVENTS.CLI_RUN_INVOKED,
730 properties: { run_id: run.runId, command_path: 'run' },
731 });
732
733 const execute = commandContext(['--log-level', 'Debug', 'execute', 'GMAIL_SEND_EMAIL'], {
734 path: ['execute'],
735 family: 'execute',
736 input: { slug: 'GMAIL_SEND_EMAIL', data: Option.none() },
737 });
738 expect(getPrimaryLifecycleInvokedEvent(execute)).toMatchObject({
739 name: CLI_ANALYTICS_EVENTS.CLI_EXECUTE_INVOKED,
740 properties: {
741 command_path: 'execute',
742 tool_slug: 'GMAIL_SEND_EMAIL',
743 toolkit_slug: 'gmail',
744 },
745 });
746 });
747
748 it('reads the link toolkit from the parsed input, and nothing for help or a parse failure', () => {
749 const link = (input?: unknown) =>
750 getPrimaryLifecycleInvokedEvent(
751 commandContext(['link', '--alias', 'work', 'gmail'], {
752 path: ['link'],
753 family: 'link',
754 input,
755 })
756 )?.properties;
757 expect(link({ toolkit: Option.some('gmail'), alias: Option.some('work') })).toMatchObject({
758 toolkit: 'gmail',
759 });
760 expect(link({ toolkit: Option.none() })?.toolkit).toBeUndefined();
761 expect(link()?.toolkit).toBeUndefined();
762 });
763
764 it('records a parse failure with reason parse_error, and a bare group help without one', () => {
765 const context = commandContext(['dev', 'triggers', 'list'], {
766 path: ['dev', 'triggers', 'list'],
767 });
768 const parseError = new CliError.ShowHelp({
769 commandPath: ['composio', 'dev', 'triggers', 'list'],
770 errors: [new CliError.MissingArgument({ argument: 'toolkit' })],
771 });
772 expect(getPrimaryLifecycleFailedEvent(context, parseError)).toMatchObject({
773 name: CLI_ANALYTICS_EVENTS.CLI_COMMAND_FAILED,
774 properties: { command_path: 'dev triggers list', failure_reason_code: 'parse_error' },
775 });
776 const bareGroup = new CliError.ShowHelp({ commandPath: ['composio', 'dev'], errors: [] });
777 expect(getPrimaryLifecycleFailedEvent(context, bareGroup)?.properties).not.toHaveProperty(
778 'failure_reason_code'
779 );
780 });
781
782 it('uses the configured parent run id for nested run telemetry', () => {
783 const context = contextFor(['run', 'console.log("hi")'], {
784 invocationOrigin: 'run',
785 parentRunId: 'run_parent',
786 });
787 expect(context.runId).toBe('run_parent');
788 expect(getPrimaryLifecycleInvokedEvent(context)?.properties).toMatchObject({
789 invocation_origin: 'run',
790 run_id: 'run_parent',
791 });
792 });
793
794 it('keeps the base installer install-only and marks shell-setup delegation as installer-origin', () => {
795 const installScript = readFileSync(
796 new URL('../../../../../install.sh', import.meta.url),
797 'utf8'
798 );
799 // Join backslash-continued lines so multi-line invocations match as one logical line.
800 const logicalLines = installScript.replace(/\\\n\s*/g, ' ').split('\n');
801 const installInvocations = logicalLines.filter(line => line.includes('"$exe" install'));
802
803 // The installer may invoke `composio install` only for the `--shell` capability probe
804 // and the shell-setup delegation; any other invocation would emit install analytics
805 // events without the installer origin attached.
806 const helpProbes = installInvocations.filter(line => line.includes('"$exe" install --help'));
807 const shellDelegations = installInvocations.filter(line =>
808 line.includes('"$exe" install --shell')
809 );
810 expect(helpProbes).toHaveLength(1);
811 expect(shellDelegations).toHaveLength(1);
812 expect(installInvocations).toHaveLength(2);
813 expect(shellDelegations[0]).toContain('COMPOSIO_CLI_INVOCATION_ORIGIN=installer');
814 });
815
816 it.each(['zsh', 'bash', 'fish'])(
817 'keeps the %s installer variant from invoking the CLI itself',
818 shell => {
819 const variantScript = readFileSync(
820 new URL(`../../../../../install/${shell}.sh`, import.meta.url),
821 'utf8'
822 );
823 // Variants never invoke the CLI themselves: they re-exec the base installer with the
824 // shell pinned, so the base installer's shell-setup delegation stays the only CLI
825 // install call.
826 expect(variantScript).not.toContain('"$exe"');
827 expect(variantScript).not.toContain(' install --shell');
828 expect(variantScript).toContain('COMPOSIO_INSTALL_SHELL="$variant_shell"');
829 }
830 );
831 });