Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | import { VERSION as clientLibraryVersion } from '@composio/client/effect'; |
| 2 | import { readFileSync, writeFileSync } from 'node:fs'; |
| 3 | import path from 'node:path'; |
| 4 | import * as tempy from 'tempy'; |
| 5 | import { it } from '@effect/vitest'; |
| 6 | import { afterEach, describe, expect, vi } from 'vitest'; |
| 7 | import * as BunFileSystem from '@effect/platform-bun/BunFileSystem'; |
| 8 | import * as BunPath from '@effect/platform-bun/BunPath'; |
| 9 | import { Effect, Layer, Option } from 'effect'; |
| 10 | import { CliError } from 'effect/cli'; |
| 11 | import { createCliCodactFailureBody } from 'src/analytics/dispatch'; |
| 12 | import { |
| 13 | CLI_ANALYTICS_EVENTS, |
| 14 | CLI_EVENT_JOURNEY_STAGES, |
| 15 | CLI_JOURNEY_STAGES, |
| 16 | configureCliAnalyticsReleaseVersion, |
| 17 | getPluginHintShownEvent, |
| 18 | getPluginLifecycleFailedEvent, |
| 19 | getPluginLifecycleSucceededEvent, |
| 20 | getPrimaryLifecycleFailedEvent, |
| 21 | getPrimaryLifecycleInvokedEvent, |
| 22 | getPrimaryLifecycleSucceededEvent, |
| 23 | getSetupCancelledEvent, |
| 24 | getSetupHostDetectedEvent, |
| 25 | getSetupSkippedEvent, |
| 26 | getToolExecuteFailedEvent, |
| 27 | getToolExecuteToolNotFoundEvent, |
| 28 | getToolExecuteValidationFailedEvent, |
| 29 | isMaybeToolNotFoundError, |
| 30 | isMaybeToolValidationError, |
| 31 | } from 'src/analytics/events'; |
| 32 | import { MissingActingUserError } from 'src/commands/surface'; |
| 33 | import { APP_VERSION } from 'src/constants'; |
| 34 | import { inferSkillReleaseChannel } from 'src/effects/install-skill'; |
| 35 | import { CLI_RELEASE_CHANNELS } from 'src/experimental-features'; |
| 36 | import { |
| 37 | DEFAULT_CLI_INVOCATION_ORIGIN, |
| 38 | type CliInvocationContext, |
| 39 | } from 'src/services/runtime-cli-context'; |
| 40 | import { ChildProcessExited } from 'src/effects/child-process-exited'; |
| 41 | import { SetupCommandError } from 'src/services/setup-command-error'; |
| 42 | import { ToolInputValidationError } from 'src/services/tool-input-validation'; |
| 43 | import { resolveInstalledCliVersion } from 'src/services/run-companion-modules'; |
| 44 | import type { TelemetryFamily } from 'src/analytics/types'; |
| 45 | import { commandTelemetryContext } from 'test/__utils__/telemetry-context'; |
| 46 | |
| 47 | const CLI_INVOCATION = { |
| 48 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 49 | parentRunId: undefined, |
| 50 | } as const; |
| 51 | |
| 52 | /** The context of `args` with this file's defaults: the package version, `cli` origin. */ |
| 53 | const commandContext = ( |
| 54 | args: ReadonlyArray<string>, |
| 55 | options: Parameters<typeof commandTelemetryContext>[1] |
| 56 | ) => commandTelemetryContext(args, { cliVersion: APP_VERSION, ...options }); |
| 57 | |
| 58 | describe('CLI analytics execute failure events', () => { |
| 59 | it('records terminal capabilities supplied by the terminal service', () => { |
| 60 | const context = commandContext([], { |
| 61 | path: [], |
| 62 | cliVersion: '0.3.0', |
| 63 | terminal: { stdoutIsTTY: true, stderrIsTTY: false }, |
| 64 | }); |
| 65 | |
| 66 | const event = getPrimaryLifecycleInvokedEvent(context); |
| 67 | |
| 68 | expect(event?.properties).toMatchObject({ |
| 69 | stdout_is_tty: true, |
| 70 | stderr_is_tty: false, |
| 71 | }); |
| 72 | }); |
| 73 | |
| 74 | it('keeps CLI product and installed library separate on tool-router command events', () => { |
| 75 | const context = commandContext(['execute', 'GITHUB_GET_ME'], { |
| 76 | path: ['execute'], |
| 77 | family: 'execute', |
| 78 | input: { slug: 'GITHUB_GET_ME' }, |
| 79 | cliVersion: '0.4.2', |
| 80 | }); |
| 81 | expect(getPrimaryLifecycleSucceededEvent(context)?.properties).toMatchObject({ |
| 82 | client_name: '@composio/cli', |
| 83 | client_version: '0.4.2', |
| 84 | client_library: '@composio/client', |
| 85 | client_library_version: clientLibraryVersion, |
| 86 | execution_channel: 'tool_router', |
| 87 | duration_ms: expect.any(Number), |
| 88 | }); |
| 89 | }); |
| 90 | |
| 91 | it('marks cached-schema validation failures as fast_fail', () => { |
| 92 | const error = new ToolInputValidationError({ |
| 93 | toolSlug: 'GMAIL_SEND_EMAIL', |
| 94 | schemaPath: '/tmp/schema.json', |
| 95 | issues: ['Unknown key "recipient"'], |
| 96 | }); |
| 97 | |
| 98 | const event = getToolExecuteValidationFailedEvent({ |
| 99 | toolSlug: 'GMAIL_SEND_EMAIL', |
| 100 | args: { recipient: 'a@example.com' }, |
| 101 | error, |
| 102 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 103 | surface: 'root', |
| 104 | projectMode: 'consumer', |
| 105 | stage: 'validation', |
| 106 | failureOrigin: 'fast_fail', |
| 107 | }); |
| 108 | |
| 109 | expect(event).not.toBeNull(); |
| 110 | expect(event!.properties?.failure_origin).toBe('fast_fail'); |
| 111 | expect(event!.properties?.tool_log_id).toBeUndefined(); |
| 112 | }); |
| 113 | |
| 114 | it('marks endpoint tool-not-found failures as main_endpoint and keeps log id', () => { |
| 115 | const event = getToolExecuteToolNotFoundEvent({ |
| 116 | toolSlug: 'FAKE_TOOL', |
| 117 | args: {}, |
| 118 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 119 | surface: 'root', |
| 120 | projectMode: 'consumer', |
| 121 | stage: 'execution', |
| 122 | failureOrigin: 'main_endpoint', |
| 123 | logId: 'log_123', |
| 124 | message: 'Tool not found', |
| 125 | status: 404, |
| 126 | }); |
| 127 | |
| 128 | expect(event).not.toBeNull(); |
| 129 | expect(event!.properties?.failure_origin).toBe('main_endpoint'); |
| 130 | expect(event!.properties?.tool_log_id).toBe('log_123'); |
| 131 | }); |
| 132 | |
| 133 | it('marks endpoint execution failures as main_endpoint and keeps log id', () => { |
| 134 | const event = getToolExecuteFailedEvent({ |
| 135 | toolSlug: 'GMAIL_SEND_EMAIL', |
| 136 | args: { to: 'a@example.com' }, |
| 137 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 138 | surface: 'root', |
| 139 | projectMode: 'consumer', |
| 140 | stage: 'execution', |
| 141 | failureOrigin: 'main_endpoint', |
| 142 | logId: 'log_456', |
| 143 | message: 'Invalid tool arguments', |
| 144 | }); |
| 145 | |
| 146 | expect(event).not.toBeNull(); |
| 147 | expect(event!.properties?.failure_origin).toBe('main_endpoint'); |
| 148 | expect(event!.properties?.tool_log_id).toBe('log_456'); |
| 149 | }); |
| 150 | |
| 151 | it('classifies known Hermes tool-not-found codes', () => { |
| 152 | expect( |
| 153 | isMaybeToolNotFoundError({ |
| 154 | apiCode: 2306, |
| 155 | message: 'random', |
| 156 | }) |
| 157 | ).toBe(true); |
| 158 | expect( |
| 159 | isMaybeToolNotFoundError({ |
| 160 | apiCode: 3703, |
| 161 | message: 'random', |
| 162 | }) |
| 163 | ).toBe(true); |
| 164 | }); |
| 165 | |
| 166 | it('classifies known Hermes validation codes', () => { |
| 167 | expect( |
| 168 | isMaybeToolValidationError({ |
| 169 | apiCode: 3702, |
| 170 | message: 'random', |
| 171 | }) |
| 172 | ).toBe(true); |
| 173 | expect( |
| 174 | isMaybeToolValidationError({ |
| 175 | apiCode: 1149, |
| 176 | message: 'random', |
| 177 | }) |
| 178 | ).toBe(true); |
| 179 | }); |
| 180 | |
| 181 | it('builds the CLI codact failure body using backend field names', () => { |
| 182 | const body = createCliCodactFailureBody({ |
| 183 | failureType: 'wrong_tool_slug', |
| 184 | toolInfo: { |
| 185 | toolkit: 'github', |
| 186 | }, |
| 187 | ctx: { |
| 188 | invalid_tool_slug: 'GITHUB_MAKE_ISSUE', |
| 189 | }, |
| 190 | session: { |
| 191 | command_path: 'execute', |
| 192 | }, |
| 193 | requestId: 'req_123', |
| 194 | }); |
| 195 | |
| 196 | expect(body).toMatchObject({ |
| 197 | failure_type: 'wrong_tool_slug', |
| 198 | tool_info: { |
| 199 | toolkit: 'github', |
| 200 | }, |
| 201 | ctx: { |
| 202 | invalid_tool_slug: 'GITHUB_MAKE_ISSUE', |
| 203 | }, |
| 204 | session: { |
| 205 | source: 'cli', |
| 206 | command_path: 'execute', |
| 207 | cli_version: expect.any(String), |
| 208 | }, |
| 209 | request_id: 'req_123', |
| 210 | }); |
| 211 | }); |
| 212 | }); |
| 213 | |
| 214 | describe('parsed telemetry options', () => { |
| 215 | it('records canonical option names without retaining flag values', () => { |
| 216 | const secret = '--customer-private'; |
| 217 | const context = commandContext(['execute', 'GMAIL_SEND_EMAIL', '-d', secret], { |
| 218 | path: ['execute'], |
| 219 | family: 'execute', |
| 220 | input: { slug: 'GMAIL_SEND_EMAIL', data: Option.some(secret), dryRun: true }, |
| 221 | }); |
| 222 | expect(context.flagNames).toEqual(['--data', '--dry-run']); |
| 223 | const event = getPrimaryLifecycleInvokedEvent(context); |
| 224 | expect(event?.properties).toMatchObject({ has_data: true, dry_run: true }); |
| 225 | expect(JSON.stringify(event)).not.toContain(secret); |
| 226 | expect(JSON.stringify(context.options)).not.toContain(secret); |
| 227 | }); |
| 228 | it('uses parsed proxy values and counts repeated headers without collecting their content', () => { |
| 229 | const context = commandContext( |
| 230 | ['proxy', '/path', '-t', 'gmail', '-H', 'Authorization: secret'], |
| 231 | { |
| 232 | path: ['proxy'], |
| 233 | family: 'proxy', |
| 234 | input: { |
| 235 | endpoint: '/path', |
| 236 | toolkit: 'gmail', |
| 237 | method: 'POST', |
| 238 | headers: ['a: b', 'c: d'], |
| 239 | data: Option.none(), |
| 240 | }, |
| 241 | } |
| 242 | ); |
| 243 | const event = getPrimaryLifecycleInvokedEvent(context); |
| 244 | expect(event?.properties).toMatchObject({ |
| 245 | toolkit: 'gmail', |
| 246 | method: 'POST', |
| 247 | header_count: 2, |
| 248 | has_data: false, |
| 249 | }); |
| 250 | expect(JSON.stringify(event)).not.toContain('secret'); |
| 251 | }); |
| 252 | it('honors a parsed false boolean even when its flag is present', () => { |
| 253 | const context = commandContext(['run', '--dry-run=false', 'code'], { |
| 254 | path: ['run'], |
| 255 | family: 'run', |
| 256 | input: { args: ['code'], dryRun: false }, |
| 257 | }); |
| 258 | expect(getPrimaryLifecycleInvokedEvent(context)?.properties).toMatchObject({ dry_run: false }); |
| 259 | }); |
| 260 | }); |
| 261 | |
| 262 | describe('CLI analytics setup and install lifecycle events', () => { |
| 263 | it('tracks setup as a dedicated lifecycle with safe option properties', () => { |
| 264 | const context = commandContext(['setup', '--target', 'codex', '--uninstall', '--yes'], { |
| 265 | path: ['setup'], |
| 266 | family: 'setup', |
| 267 | input: { target: 'codex', uninstall: true, yes: true, ifPresent: false }, |
| 268 | }); |
| 269 | |
| 270 | const invoked = getPrimaryLifecycleInvokedEvent(context); |
| 271 | const succeeded = getPrimaryLifecycleSucceededEvent(context); |
| 272 | const failed = getPrimaryLifecycleFailedEvent(context, new Error('native failure')); |
| 273 | |
| 274 | expect(invoked?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_INVOKED); |
| 275 | expect(succeeded?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_SUCCEEDED); |
| 276 | expect(failed?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_SETUP_FAILED); |
| 277 | expect(invoked?.properties).toMatchObject({ |
| 278 | command_path: 'setup', |
| 279 | operation: 'uninstall', |
| 280 | target: 'codex', |
| 281 | yes: true, |
| 282 | if_present: false, |
| 283 | stdout_is_tty: expect.any(Boolean), |
| 284 | }); |
| 285 | }); |
| 286 | |
| 287 | it('tracks shell installation separately from plugin setup', () => { |
| 288 | const context = commandContext(['install', '--completions'], { |
| 289 | path: ['install'], |
| 290 | family: 'install', |
| 291 | input: { completions: true, noCompletions: false }, |
| 292 | }); |
| 293 | |
| 294 | expect(getPrimaryLifecycleInvokedEvent(context)).toMatchObject({ |
| 295 | name: CLI_ANALYTICS_EVENTS.CLI_INSTALL_INVOKED, |
| 296 | properties: { |
| 297 | command_path: 'install', |
| 298 | completions: true, |
| 299 | no_completions: false, |
| 300 | }, |
| 301 | }); |
| 302 | }); |
| 303 | |
| 304 | it('summarizes user-controlled search and proxy input without recording it', () => { |
| 305 | const endpoint = 'https://user:password@example.com/private?token=small'; |
| 306 | const search = getPrimaryLifecycleInvokedEvent( |
| 307 | commandContext(['search', 'customer@example.com secret'], { |
| 308 | path: ['search'], |
| 309 | family: 'search', |
| 310 | input: { query: ['customer@example.com secret'] }, |
| 311 | }) |
| 312 | ); |
| 313 | const proxy = getPrimaryLifecycleInvokedEvent( |
| 314 | commandContext(['proxy', endpoint], { path: ['proxy'], family: 'proxy', input: { endpoint } }) |
| 315 | ); |
| 316 | |
| 317 | expect(search?.properties).toMatchObject({ |
| 318 | query_length: 'customer@example.com secret'.length, |
| 319 | query_term_count: 2, |
| 320 | }); |
| 321 | expect(search?.properties).not.toHaveProperty('query'); |
| 322 | expect(search?.properties).not.toHaveProperty('search_query'); |
| 323 | expect(proxy?.properties).toMatchObject({ has_endpoint: true }); |
| 324 | expect(proxy?.properties).not.toHaveProperty('endpoint'); |
| 325 | }); |
| 326 | |
| 327 | it('tracks a verified per-host plugin change', () => { |
| 328 | expect( |
| 329 | getPluginLifecycleSucceededEvent({ |
| 330 | operation: 'setup', |
| 331 | target: 'claude', |
| 332 | action: 'installed', |
| 333 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 334 | cliVersion: APP_VERSION, |
| 335 | }) |
| 336 | ).toMatchObject({ |
| 337 | name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_SUCCEEDED, |
| 338 | properties: { |
| 339 | command_path: 'setup', |
| 340 | operation: 'setup', |
| 341 | agent_host: 'claude', |
| 342 | action: 'installed', |
| 343 | }, |
| 344 | }); |
| 345 | }); |
| 346 | }); |
| 347 | |
| 348 | describe('CLI analytics setup runtime-context events', () => { |
| 349 | it('tracks a detected and supported host with its version', () => { |
| 350 | expect( |
| 351 | getSetupHostDetectedEvent({ |
| 352 | operation: 'setup', |
| 353 | requestedTarget: 'auto', |
| 354 | target: 'claude', |
| 355 | available: true, |
| 356 | supported: true, |
| 357 | hostVersion: '2.1.0', |
| 358 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 359 | cliVersion: APP_VERSION, |
| 360 | }) |
| 361 | ).toMatchObject({ |
| 362 | name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED, |
| 363 | properties: { |
| 364 | source: 'cli', |
| 365 | command_path: 'setup', |
| 366 | operation: 'setup', |
| 367 | requested_target: 'auto', |
| 368 | agent_host: 'claude', |
| 369 | available: true, |
| 370 | supported: true, |
| 371 | host_version: '2.1.0', |
| 372 | unsupported_reason_code: undefined, |
| 373 | }, |
| 374 | }); |
| 375 | }); |
| 376 | |
| 377 | it('records host presence signals only for an undetected host', () => { |
| 378 | const missing = getSetupHostDetectedEvent({ |
| 379 | operation: 'setup', |
| 380 | requestedTarget: 'auto', |
| 381 | target: 'codex', |
| 382 | available: false, |
| 383 | supported: false, |
| 384 | hostConfigDirPresent: true, |
| 385 | hostBinaryInKnownPaths: false, |
| 386 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 387 | cliVersion: APP_VERSION, |
| 388 | }); |
| 389 | expect(missing?.properties).toMatchObject({ |
| 390 | agent_host: 'codex', |
| 391 | available: false, |
| 392 | host_config_dir_present: true, |
| 393 | host_binary_in_known_paths: false, |
| 394 | }); |
| 395 | |
| 396 | const detected = getSetupHostDetectedEvent({ |
| 397 | operation: 'setup', |
| 398 | requestedTarget: 'auto', |
| 399 | target: 'claude', |
| 400 | available: true, |
| 401 | supported: true, |
| 402 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 403 | cliVersion: APP_VERSION, |
| 404 | }); |
| 405 | expect(detected?.properties?.host_config_dir_present).toBeUndefined(); |
| 406 | expect(detected?.properties?.host_binary_in_known_paths).toBeUndefined(); |
| 407 | }); |
| 408 | |
| 409 | it('tracks a printed plugin hint as a setup-stage event', () => { |
| 410 | expect( |
| 411 | getPluginHintShownEvent({ |
| 412 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 413 | cliVersion: APP_VERSION, |
| 414 | commandPath: 'whoami', |
| 415 | agentHost: 'claude', |
| 416 | }) |
| 417 | ).toEqual({ |
| 418 | name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_HINT_SHOWN, |
| 419 | properties: { |
| 420 | source: 'cli', |
| 421 | invocation_origin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 422 | cli_version: APP_VERSION, |
| 423 | command_path: 'whoami', |
| 424 | agent_host: 'claude', |
| 425 | client_name: '@composio/cli', |
| 426 | client_version: APP_VERSION, |
| 427 | client_language: 'typescript', |
| 428 | client_runtime: 'nodejs', |
| 429 | client_runtime_version: process.versions.node, |
| 430 | client_library: '@composio/client', |
| 431 | client_library_version: clientLibraryVersion, |
| 432 | client_framework: 'cli', |
| 433 | execution_channel: 'unknown', |
| 434 | journey_stage: 'setup', |
| 435 | cli_channel: inferSkillReleaseChannel(APP_VERSION), |
| 436 | }, |
| 437 | }); |
| 438 | }); |
| 439 | |
| 440 | // Execute, search, link, connections, listen and proxy each used to report their own class. |
| 441 | it.each([ |
| 442 | { path: ['execute'], family: 'execute' }, |
| 443 | { path: ['search'], family: 'search' }, |
| 444 | { path: ['link'], family: 'link' }, |
| 445 | { path: ['connections', 'list'], family: 'command' }, |
| 446 | { path: ['listen'], family: 'command' }, |
| 447 | { path: ['proxy'], family: 'proxy' }, |
| 448 | ] as const)('reports a missing user on `$path` as MissingActingUserError', ({ path, family }) => { |
| 449 | const context = commandContext(path, { path, family }); |
| 450 | const error = new MissingActingUserError({ |
| 451 | tree: 'root', |
| 452 | projectName: Option.none(), |
| 453 | message: 'Missing consumer user id. Run `composio login` and try again.', |
| 454 | }); |
| 455 | |
| 456 | expect(getPrimaryLifecycleFailedEvent(context, error)?.properties).toMatchObject({ |
| 457 | error_name: 'commands/MissingActingUserError', |
| 458 | }); |
| 459 | }); |
| 460 | |
| 461 | it('records the setup failure reason code carried by SetupCommandError', () => { |
| 462 | const context = commandContext(['setup'], { path: ['setup'], family: 'setup', input: {} }); |
| 463 | const error = new SetupCommandError({ |
| 464 | message: 'Non-interactive setup requires `--yes` to approve local changes.', |
| 465 | operation: 'setup', |
| 466 | reasonCode: 'non_interactive_requires_yes', |
| 467 | }); |
| 468 | |
| 469 | expect(getPrimaryLifecycleFailedEvent(context, error)?.properties).toMatchObject({ |
| 470 | error_name: 'services/SetupCommandError', |
| 471 | failure_reason_code: 'non_interactive_requires_yes', |
| 472 | }); |
| 473 | const unknown = new SetupCommandError({ |
| 474 | message: 'native failure', |
| 475 | operation: 'setup', |
| 476 | reasonCode: 'unknown', |
| 477 | }); |
| 478 | expect(getPrimaryLifecycleFailedEvent(context, unknown)?.properties?.failure_reason_code).toBe( |
| 479 | 'unknown' |
| 480 | ); |
| 481 | expect( |
| 482 | getPrimaryLifecycleFailedEvent(context, new Error('boom'))?.properties |
| 483 | ).not.toHaveProperty('failure_reason_code'); |
| 484 | }); |
| 485 | |
| 486 | it('records a run whose child exited non-zero as failed with reason child_exit', () => { |
| 487 | const context = commandContext(['run', 'process.exit(3)'], { |
| 488 | path: ['run'], |
| 489 | family: 'run', |
| 490 | input: {}, |
| 491 | }); |
| 492 | |
| 493 | const event = getPrimaryLifecycleFailedEvent(context, new ChildProcessExited({ code: 3 })); |
| 494 | |
| 495 | expect(event?.name).toBe(CLI_ANALYTICS_EVENTS.CLI_RUN_FAILED); |
| 496 | expect(event?.properties).toMatchObject({ |
| 497 | error_name: 'effects/ChildProcessExited', |
| 498 | failure_reason_code: 'child_exit', |
| 499 | }); |
| 500 | expect( |
| 501 | getPrimaryLifecycleFailedEvent(context, new Error('boom'))?.properties |
| 502 | ).not.toHaveProperty('failure_reason_code'); |
| 503 | }); |
| 504 | |
| 505 | it('tracks an unsupported host with a normalized reason code', () => { |
| 506 | expect( |
| 507 | getSetupHostDetectedEvent({ |
| 508 | operation: 'setup', |
| 509 | requestedTarget: 'auto', |
| 510 | target: 'codex', |
| 511 | available: true, |
| 512 | supported: false, |
| 513 | hostVersion: 'codex-cli 0.137.0', |
| 514 | unsupportedReasonCode: 'codex_too_old', |
| 515 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 516 | cliVersion: APP_VERSION, |
| 517 | }) |
| 518 | ).toMatchObject({ |
| 519 | name: CLI_ANALYTICS_EVENTS.CLI_SETUP_HOST_DETECTED, |
| 520 | properties: { |
| 521 | agent_host: 'codex', |
| 522 | available: true, |
| 523 | supported: false, |
| 524 | unsupported_reason_code: 'codex_too_old', |
| 525 | }, |
| 526 | }); |
| 527 | }); |
| 528 | |
| 529 | it('tracks a per-host failure without recording its free-form message', () => { |
| 530 | const error = new Error(`Adding the claude marketplace failed${'x'.repeat(600)}`); |
| 531 | |
| 532 | const event = getPluginLifecycleFailedEvent({ |
| 533 | operation: 'setup', |
| 534 | target: 'claude', |
| 535 | phase: 'install', |
| 536 | error, |
| 537 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 538 | cliVersion: APP_VERSION, |
| 539 | }); |
| 540 | |
| 541 | expect(event).toMatchObject({ |
| 542 | name: CLI_ANALYTICS_EVENTS.CLI_PLUGIN_SETUP_FAILED, |
| 543 | properties: { |
| 544 | command_path: 'setup', |
| 545 | operation: 'setup', |
| 546 | agent_host: 'claude', |
| 547 | phase: 'install', |
| 548 | error_name: 'Error', |
| 549 | }, |
| 550 | }); |
| 551 | expect(event?.properties).not.toHaveProperty('error_message'); |
| 552 | }); |
| 553 | |
| 554 | it('tracks user cancellation and installer skips with normalized reasons', () => { |
| 555 | expect( |
| 556 | getSetupCancelledEvent({ |
| 557 | operation: 'setup', |
| 558 | requestedTarget: 'claude', |
| 559 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 560 | cliVersion: APP_VERSION, |
| 561 | }) |
| 562 | ).toMatchObject({ |
| 563 | name: CLI_ANALYTICS_EVENTS.CLI_SETUP_CANCELLED, |
| 564 | properties: { |
| 565 | command_path: 'setup', |
| 566 | operation: 'setup', |
| 567 | requested_target: 'claude', |
| 568 | reason: 'user_declined', |
| 569 | }, |
| 570 | }); |
| 571 | |
| 572 | expect( |
| 573 | getSetupSkippedEvent({ |
| 574 | operation: 'uninstall', |
| 575 | requestedTarget: 'auto', |
| 576 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 577 | cliVersion: APP_VERSION, |
| 578 | }) |
| 579 | ).toMatchObject({ |
| 580 | name: CLI_ANALYTICS_EVENTS.CLI_SETUP_SKIPPED, |
| 581 | properties: { |
| 582 | command_path: 'setup', |
| 583 | operation: 'uninstall', |
| 584 | requested_target: 'auto', |
| 585 | reason: 'no_host_detected', |
| 586 | }, |
| 587 | }); |
| 588 | }); |
| 589 | }); |
| 590 | |
| 591 | describe('CLI analytics journey taxonomy', () => { |
| 592 | afterEach(() => { |
| 593 | configureCliAnalyticsReleaseVersion(APP_VERSION); |
| 594 | vi.unstubAllEnvs(); |
| 595 | }); |
| 596 | |
| 597 | /** The context of a root command whose name is its family, or the generic `version`. */ |
| 598 | const contextFor = ( |
| 599 | argv: ReadonlyArray<string>, |
| 600 | invocation: CliInvocationContext = CLI_INVOCATION |
| 601 | ) => { |
| 602 | const [name = 'version'] = argv; |
| 603 | return commandContext(argv, { |
| 604 | path: [name], |
| 605 | family: name === 'version' ? 'command' : (name as TelemetryFamily), |
| 606 | invocation, |
| 607 | }); |
| 608 | }; |
| 609 | |
| 610 | const lifecycleCases: ReadonlyArray<[ReadonlyArray<string>, string]> = [ |
| 611 | [['execute', 'GMAIL_SEND_EMAIL'], 'execute'], |
| 612 | [['search', 'send email'], 'other'], |
| 613 | [['link', 'github'], 'connect'], |
| 614 | [['login'], 'login'], |
| 615 | [['logout'], 'other'], |
| 616 | [['proxy', '/api/v3/toolkits'], 'other'], |
| 617 | [['run', 'echo hi'], 'other'], |
| 618 | [['install'], 'install'], |
| 619 | [['setup'], 'setup'], |
| 620 | [['version'], 'other'], |
| 621 | ]; |
| 622 | |
| 623 | it.each(lifecycleCases)('stamps %j lifecycle events with journey_stage %s', (argv, stage) => { |
| 624 | const context = contextFor(argv); |
| 625 | |
| 626 | expect(getPrimaryLifecycleInvokedEvent(context)?.properties?.journey_stage).toBe(stage); |
| 627 | expect(getPrimaryLifecycleSucceededEvent(context)?.properties?.journey_stage).toBe(stage); |
| 628 | expect( |
| 629 | getPrimaryLifecycleFailedEvent(context, new Error('boom'))?.properties?.journey_stage |
| 630 | ).toBe(stage); |
| 631 | }); |
| 632 | |
| 633 | it.each(Object.values(CLI_ANALYTICS_EVENTS))( |
| 634 | 'maps analytics event %s to a declared journey stage', |
| 635 | name => { |
| 636 | expect(CLI_JOURNEY_STAGES).toContain(CLI_EVENT_JOURNEY_STAGES[name]); |
| 637 | } |
| 638 | ); |
| 639 | |
| 640 | it('stamps standalone setup and tool-invocation events with their stages', () => { |
| 641 | expect( |
| 642 | getPluginLifecycleSucceededEvent({ |
| 643 | operation: 'setup', |
| 644 | target: 'claude', |
| 645 | action: 'installed', |
| 646 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 647 | cliVersion: APP_VERSION, |
| 648 | })?.properties?.journey_stage |
| 649 | ).toBe('setup'); |
| 650 | |
| 651 | expect( |
| 652 | getSetupSkippedEvent({ |
| 653 | operation: 'setup', |
| 654 | requestedTarget: 'auto', |
| 655 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 656 | cliVersion: APP_VERSION, |
| 657 | })?.properties?.journey_stage |
| 658 | ).toBe('setup'); |
| 659 | |
| 660 | expect( |
| 661 | getToolExecuteFailedEvent({ |
| 662 | toolSlug: 'GMAIL_SEND_EMAIL', |
| 663 | args: {}, |
| 664 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 665 | surface: 'root', |
| 666 | projectMode: 'consumer', |
| 667 | stage: 'execution', |
| 668 | failureOrigin: 'main_endpoint', |
| 669 | })?.properties?.journey_stage |
| 670 | ).toBe('execute'); |
| 671 | }); |
| 672 | |
| 673 | it('stamps events with the release channel of the running build', () => { |
| 674 | const properties = getPrimaryLifecycleInvokedEvent(contextFor(['login']))?.properties; |
| 675 | |
| 676 | expect(properties?.cli_channel).toBe(inferSkillReleaseChannel(APP_VERSION)); |
| 677 | expect(CLI_RELEASE_CHANNELS).toContain(properties?.cli_channel); |
| 678 | }); |
| 679 | |
| 680 | it.effect('uses beta release metadata even when the package version is stable', () => { |
| 681 | const installDir = tempy.temporaryDirectory(); |
| 682 | const execPath = path.join(installDir, 'composio'); |
| 683 | writeFileSync(execPath, 'fake binary'); |
| 684 | writeFileSync(path.join(installDir, 'release-tag.txt'), '@composio/cli@0.3.1-beta.7\n'); |
| 685 | |
| 686 | return Effect.gen(function* () { |
| 687 | const resolvedVersion = yield* resolveInstalledCliVersion(execPath, APP_VERSION); |
| 688 | configureCliAnalyticsReleaseVersion(resolvedVersion); |
| 689 | |
| 690 | const context = commandContext(['login'], { |
| 691 | path: ['login'], |
| 692 | family: 'login', |
| 693 | input: {}, |
| 694 | cliVersion: resolvedVersion, |
| 695 | }); |
| 696 | const properties = getPrimaryLifecycleInvokedEvent(context)?.properties; |
| 697 | const pluginProperties = getPluginLifecycleSucceededEvent({ |
| 698 | operation: 'setup', |
| 699 | target: 'codex', |
| 700 | action: 'installed', |
| 701 | invocationOrigin: DEFAULT_CLI_INVOCATION_ORIGIN, |
| 702 | cliVersion: APP_VERSION, |
| 703 | })?.properties; |
| 704 | expect(inferSkillReleaseChannel(APP_VERSION)).toBe('stable'); |
| 705 | expect(properties?.cli_channel).toBe('beta'); |
| 706 | expect(pluginProperties?.cli_channel).toBe('beta'); |
| 707 | }).pipe(Effect.provide(Layer.merge(BunFileSystem.layer, BunPath.layer))); |
| 708 | }); |
| 709 | |
| 710 | it('propagates the configured installer invocation origin', () => { |
| 711 | const installerInvocation = { invocationOrigin: 'installer', parentRunId: undefined }; |
| 712 | |
| 713 | expect( |
| 714 | getPrimaryLifecycleInvokedEvent(contextFor(['install'], installerInvocation))?.properties |
| 715 | ).toMatchObject({ |
| 716 | invocation_origin: 'installer', |
| 717 | journey_stage: 'install', |
| 718 | }); |
| 719 | }); |
| 720 | |
| 721 | it('mints a run id for run and reads the execute slug from the parsed input', () => { |
| 722 | const run = commandContext(['--telemetry-debug', 'run', 'console.log("hi")'], { |
| 723 | path: ['run'], |
| 724 | family: 'run', |
| 725 | input: {}, |
| 726 | }); |
| 727 | expect(run.runId).toEqual(expect.any(String)); |
| 728 | expect(getPrimaryLifecycleInvokedEvent(run)).toMatchObject({ |
| 729 | name: CLI_ANALYTICS_EVENTS.CLI_RUN_INVOKED, |
| 730 | properties: { run_id: run.runId, command_path: 'run' }, |
| 731 | }); |
| 732 | |
| 733 | const execute = commandContext(['--log-level', 'Debug', 'execute', 'GMAIL_SEND_EMAIL'], { |
| 734 | path: ['execute'], |
| 735 | family: 'execute', |
| 736 | input: { slug: 'GMAIL_SEND_EMAIL', data: Option.none() }, |
| 737 | }); |
| 738 | expect(getPrimaryLifecycleInvokedEvent(execute)).toMatchObject({ |
| 739 | name: CLI_ANALYTICS_EVENTS.CLI_EXECUTE_INVOKED, |
| 740 | properties: { |
| 741 | command_path: 'execute', |
| 742 | tool_slug: 'GMAIL_SEND_EMAIL', |
| 743 | toolkit_slug: 'gmail', |
| 744 | }, |
| 745 | }); |
| 746 | }); |
| 747 | |
| 748 | it('reads the link toolkit from the parsed input, and nothing for help or a parse failure', () => { |
| 749 | const link = (input?: unknown) => |
| 750 | getPrimaryLifecycleInvokedEvent( |
| 751 | commandContext(['link', '--alias', 'work', 'gmail'], { |
| 752 | path: ['link'], |
| 753 | family: 'link', |
| 754 | input, |
| 755 | }) |
| 756 | )?.properties; |
| 757 | expect(link({ toolkit: Option.some('gmail'), alias: Option.some('work') })).toMatchObject({ |
| 758 | toolkit: 'gmail', |
| 759 | }); |
| 760 | expect(link({ toolkit: Option.none() })?.toolkit).toBeUndefined(); |
| 761 | expect(link()?.toolkit).toBeUndefined(); |
| 762 | }); |
| 763 | |
| 764 | it('records a parse failure with reason parse_error, and a bare group help without one', () => { |
| 765 | const context = commandContext(['dev', 'triggers', 'list'], { |
| 766 | path: ['dev', 'triggers', 'list'], |
| 767 | }); |
| 768 | const parseError = new CliError.ShowHelp({ |
| 769 | commandPath: ['composio', 'dev', 'triggers', 'list'], |
| 770 | errors: [new CliError.MissingArgument({ argument: 'toolkit' })], |
| 771 | }); |
| 772 | expect(getPrimaryLifecycleFailedEvent(context, parseError)).toMatchObject({ |
| 773 | name: CLI_ANALYTICS_EVENTS.CLI_COMMAND_FAILED, |
| 774 | properties: { command_path: 'dev triggers list', failure_reason_code: 'parse_error' }, |
| 775 | }); |
| 776 | const bareGroup = new CliError.ShowHelp({ commandPath: ['composio', 'dev'], errors: [] }); |
| 777 | expect(getPrimaryLifecycleFailedEvent(context, bareGroup)?.properties).not.toHaveProperty( |
| 778 | 'failure_reason_code' |
| 779 | ); |
| 780 | }); |
| 781 | |
| 782 | it('uses the configured parent run id for nested run telemetry', () => { |
| 783 | const context = contextFor(['run', 'console.log("hi")'], { |
| 784 | invocationOrigin: 'run', |
| 785 | parentRunId: 'run_parent', |
| 786 | }); |
| 787 | expect(context.runId).toBe('run_parent'); |
| 788 | expect(getPrimaryLifecycleInvokedEvent(context)?.properties).toMatchObject({ |
| 789 | invocation_origin: 'run', |
| 790 | run_id: 'run_parent', |
| 791 | }); |
| 792 | }); |
| 793 | |
| 794 | it('keeps the base installer install-only and marks shell-setup delegation as installer-origin', () => { |
| 795 | const installScript = readFileSync( |
| 796 | new URL('../../../../../install.sh', import.meta.url), |
| 797 | 'utf8' |
| 798 | ); |
| 799 | // Join backslash-continued lines so multi-line invocations match as one logical line. |
| 800 | const logicalLines = installScript.replace(/\\\n\s*/g, ' ').split('\n'); |
| 801 | const installInvocations = logicalLines.filter(line => line.includes('"$exe" install')); |
| 802 | |
| 803 | // The installer may invoke `composio install` only for the `--shell` capability probe |
| 804 | // and the shell-setup delegation; any other invocation would emit install analytics |
| 805 | // events without the installer origin attached. |
| 806 | const helpProbes = installInvocations.filter(line => line.includes('"$exe" install --help')); |
| 807 | const shellDelegations = installInvocations.filter(line => |
| 808 | line.includes('"$exe" install --shell') |
| 809 | ); |
| 810 | expect(helpProbes).toHaveLength(1); |
| 811 | expect(shellDelegations).toHaveLength(1); |
| 812 | expect(installInvocations).toHaveLength(2); |
| 813 | expect(shellDelegations[0]).toContain('COMPOSIO_CLI_INVOCATION_ORIGIN=installer'); |
| 814 | }); |
| 815 | |
| 816 | it.each(['zsh', 'bash', 'fish'])( |
| 817 | 'keeps the %s installer variant from invoking the CLI itself', |
| 818 | shell => { |
| 819 | const variantScript = readFileSync( |
| 820 | new URL(`../../../../../install/${shell}.sh`, import.meta.url), |
| 821 | 'utf8' |
| 822 | ); |
| 823 | // Variants never invoke the CLI themselves: they re-exec the base installer with the |
| 824 | // shell pinned, so the base installer's shell-setup delegation stays the only CLI |
| 825 | // install call. |
| 826 | expect(variantScript).not.toContain('"$exe"'); |
| 827 | expect(variantScript).not.toContain(' install --shell'); |
| 828 | expect(variantScript).toContain('COMPOSIO_INSTALL_SHELL="$variant_shell"'); |
| 829 | } |
| 830 | ); |
| 831 | }); |