Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | import { describe, expect, it, layer } from '@effect/vitest'; |
| 2 | import { Cause, Data, Effect, Option, Ref } from 'effect'; |
| 3 | import type { Command } from 'effect/cli'; |
| 4 | import { |
| 5 | getPrimaryLifecycleFailedEvent, |
| 6 | getPrimaryLifecycleInvokedEvent, |
| 7 | getPrimaryLifecycleSucceededEvent, |
| 8 | } from 'src/analytics/events'; |
| 9 | import type { TelemetryFamily } from 'src/analytics/types'; |
| 10 | import { buildRootCommand, runWithConfig } from 'src/commands'; |
| 11 | import { |
| 12 | CommandInvocation, |
| 13 | type CommandIdentity, |
| 14 | commandPathOf, |
| 15 | } from 'src/invocation/command-identity'; |
| 16 | import { cliExit, recordedInvocationCalls, TestLive } from 'test/__utils__'; |
| 17 | import { RECORDED_BEFORE_IDENTITY } from './command-identity.before'; |
| 18 | |
| 19 | const commandPaths = ( |
| 20 | command: Command.Command.Any, |
| 21 | prefix: ReadonlyArray<string> |
| 22 | ): ReadonlyArray<ReadonlyArray<string>> => |
| 23 | command.subcommands |
| 24 | .flatMap(group => group.commands) |
| 25 | .flatMap(child => { |
| 26 | const path = [...prefix, child.name]; |
| 27 | return [path, ...commandPaths(child, path)]; |
| 28 | }); |
| 29 | |
| 30 | const everyFeature = { isDevModeEnabled: true, isExperimentalFeatureEnabled: () => true }; |
| 31 | const PATHS = [[], ...commandPaths(buildRootCommand(everyFeature), [])].map(path => ({ |
| 32 | args: path, |
| 33 | name: path.join(' ') || 'composio', |
| 34 | })); |
| 35 | |
| 36 | /** The root registrations with their own lifecycle family; every other command is `command`. */ |
| 37 | const FAMILIES: Readonly<Record<string, TelemetryFamily>> = { |
| 38 | execute: 'execute', |
| 39 | install: 'install', |
| 40 | link: 'link', |
| 41 | login: 'login', |
| 42 | logout: 'logout', |
| 43 | proxy: 'proxy', |
| 44 | run: 'run', |
| 45 | search: 'search', |
| 46 | setup: 'setup', |
| 47 | }; |
| 48 | |
| 49 | /** |
| 50 | * The command paths the argv extractor recorded wrongly (`RECORDED_BEFORE_IDENTITY`): a token |
| 51 | * missing from its allowlist stopped it early, so these were recorded as `composio`, as their |
| 52 | * parent (`dev init` as `dev`), or as their parent's family (`setup skill` as `setup`). Every |
| 53 | * other path is recorded as before. |
| 54 | */ |
| 55 | const CORRECTED_PATHS = [ |
| 56 | 'agent', |
| 57 | 'agent claim', |
| 58 | 'agent inbox', |
| 59 | 'agent login', |
| 60 | 'agent signup', |
| 61 | 'agent whoami', |
| 62 | 'config', |
| 63 | 'config experimental', |
| 64 | 'connections', |
| 65 | 'connections list', |
| 66 | 'connections remove', |
| 67 | 'debug', |
| 68 | 'debug api-info', |
| 69 | 'debug generate-graph', |
| 70 | 'debug who-is-my-master', |
| 71 | 'dev init', |
| 72 | 'setup skill', |
| 73 | 'signup', |
| 74 | // `composio help <command>`, all recorded as `composio`. |
| 75 | ...PATHS.filter(({ args }) => args[0] === 'help').map(({ name }) => name), |
| 76 | ].toSorted(); |
| 77 | |
| 78 | class StopAtBegin extends Data.TaggedError('test/StopAtBegin') {} |
| 79 | |
| 80 | /** |
| 81 | * The identity `effect/cli` gives `args` at runtime: the selected registration's, or for help |
| 82 | * and parse failures the one the parser names. Stops at `begin`, so no handler runs. |
| 83 | */ |
| 84 | const identityOf = (args: ReadonlyArray<string>) => |
| 85 | Effect.gen(function* () { |
| 86 | const begun = yield* Ref.make(Option.none<CommandIdentity>()); |
| 87 | const run = yield* runWithConfig; |
| 88 | yield* run(['node', '<CMD>', ...args]).pipe( |
| 89 | Effect.provideService(CommandInvocation, { |
| 90 | begin: identity => |
| 91 | Ref.set(begun, Option.some(identity)).pipe(Effect.andThen(Effect.die(new StopAtBegin()))), |
| 92 | enter: () => Effect.void, |
| 93 | runId: Effect.succeedNone, |
| 94 | }), |
| 95 | Effect.exit |
| 96 | ); |
| 97 | return yield* Ref.get(begun); |
| 98 | }); |
| 99 | |
| 100 | describe('command identity', () => { |
| 101 | layer(TestLive({ cliUserConfig: { experimentalFeatures: { listen: true } } }))(it => { |
| 102 | it.effect.each(PATHS)('`$name` is recorded as itself, with its family', ({ args, name }) => |
| 103 | Effect.gen(function* () { |
| 104 | const identity = yield* identityOf(args); |
| 105 | expect(Option.map(identity, commandPathOf)).toEqual(Option.some(name)); |
| 106 | expect(Option.map(identity, ({ meta }) => meta.telemetryFamily)).toEqual( |
| 107 | Option.some(FAMILIES[name] ?? 'command') |
| 108 | ); |
| 109 | }) |
| 110 | ); |
| 111 | }); |
| 112 | |
| 113 | it('changes the recorded path or family only of the paths the argv extractor got wrong', () => { |
| 114 | const changed = PATHS.filter(({ name }) => { |
| 115 | const before = RECORDED_BEFORE_IDENTITY[name]; |
| 116 | return before === undefined |
| 117 | ? false |
| 118 | : before.path !== name || before.family !== (FAMILIES[name] ?? 'command'); |
| 119 | }).map(({ name }) => name); |
| 120 | expect(changed.toSorted()).toEqual(CORRECTED_PATHS); |
| 121 | expect(PATHS.map(({ name }) => name).toSorted()).toEqual( |
| 122 | Object.keys(RECORDED_BEFORE_IDENTITY).toSorted() |
| 123 | ); |
| 124 | }); |
| 125 | }); |
| 126 | |
| 127 | /** The lifecycle events `cli(args)` recorded, as `<event> <command path> <family>`. */ |
| 128 | const lifecycleOf = (args: ReadonlyArray<string>) => |
| 129 | Effect.gen(function* () { |
| 130 | yield* cliExit(args); |
| 131 | const calls = yield* recordedInvocationCalls; |
| 132 | return calls.flatMap(call => |
| 133 | call.service === 'CommandLifecycleTelemetry' && call.event !== 'pluginHintShown' |
| 134 | ? [{ event: call.event, path: call.context.commandPath, family: call.context.family, call }] |
| 135 | : [] |
| 136 | ); |
| 137 | }).pipe(Effect.provide(TestLive())); |
| 138 | |
| 139 | describe('lifecycle telemetry from the command identity', () => { |
| 140 | it.effect.each([ |
| 141 | { args: ['signup', '--help'], path: 'signup', family: 'command', outcome: 'succeeded' }, |
| 142 | { |
| 143 | args: ['setup', 'skill', '--help'], |
| 144 | path: 'setup skill', |
| 145 | family: 'command', |
| 146 | outcome: 'succeeded', |
| 147 | }, |
| 148 | { args: ['--version'], path: 'composio', family: 'command', outcome: 'succeeded' }, |
| 149 | { |
| 150 | args: ['connections', 'list'], |
| 151 | path: 'connections list', |
| 152 | family: 'command', |
| 153 | outcome: 'failed', |
| 154 | }, |
| 155 | { args: ['tools'], path: 'tools', family: 'command', outcome: 'failed' }, |
| 156 | { args: ['link', 'gmail'], path: 'link', family: 'link', outcome: 'failed' }, |
| 157 | { |
| 158 | args: ['dev', 'connected-accounts', 'link', 'gmail'], |
| 159 | path: 'dev connected-accounts link', |
| 160 | family: 'command', |
| 161 | outcome: 'failed', |
| 162 | }, |
| 163 | { |
| 164 | args: ['dev', '--user-id', 'u', 'triggers', 'list', 'gmail'], |
| 165 | path: 'dev triggers list', |
| 166 | family: 'command', |
| 167 | outcome: 'failed', |
| 168 | }, |
| 169 | { |
| 170 | args: ['dev', 'triggers', 'list', 'gmail', '--user-id', 'u'], |
| 171 | path: 'dev triggers list', |
| 172 | family: 'command', |
| 173 | outcome: 'failed', |
| 174 | }, |
| 175 | // The `logs` group declares --project-name unsupported; its subcommand rejects it once it began. |
| 176 | { |
| 177 | args: ['dev', 'logs', 'tools', '--project-name', 'p'], |
| 178 | path: 'dev logs tools', |
| 179 | family: 'command', |
| 180 | outcome: 'failed', |
| 181 | }, |
| 182 | ])( |
| 183 | '[Given] logged out, `composio $args` [Then] records $path ($family) as $outcome', |
| 184 | ({ args, path, family, outcome }) => |
| 185 | Effect.gen(function* () { |
| 186 | const events = yield* lifecycleOf(args); |
| 187 | expect(events.map(({ event, path, family }) => ({ event, path, family }))).toEqual([ |
| 188 | { event: 'invoked', path, family }, |
| 189 | { event: outcome, path, family }, |
| 190 | ]); |
| 191 | }) |
| 192 | ); |
| 193 | |
| 194 | it.effect('records a typo against the path the parser reached, never the typed token', () => |
| 195 | Effect.gen(function* () { |
| 196 | const events = yield* lifecycleOf(['dve', 'list']); |
| 197 | expect(events.map(({ event, path }) => ({ event, path }))).toEqual([ |
| 198 | { event: 'invoked', path: 'composio' }, |
| 199 | { event: 'failed', path: 'composio' }, |
| 200 | ]); |
| 201 | const failed = events[1]!.call; |
| 202 | expect(failed.event).toBe('failed'); |
| 203 | const properties = |
| 204 | failed.event === 'failed' |
| 205 | ? getPrimaryLifecycleFailedEvent(failed.context, Cause.squash(failed.cause))?.properties |
| 206 | : undefined; |
| 207 | expect(properties).toMatchObject({ |
| 208 | command_path: 'composio', |
| 209 | failure_reason_code: 'parse_error', |
| 210 | }); |
| 211 | expect(JSON.stringify(properties)).not.toContain('dve'); |
| 212 | }) |
| 213 | ); |
| 214 | |
| 215 | it.effect('uses the displayed command identity for redundant help', () => |
| 216 | Effect.gen(function* () { |
| 217 | const events = yield* lifecycleOf(['help', 'execute', '--help']); |
| 218 | expect(events.map(({ event, path, family }) => ({ event, path, family }))).toEqual([ |
| 219 | { event: 'invoked', path: 'execute', family: 'execute' }, |
| 220 | { event: 'succeeded', path: 'execute', family: 'execute' }, |
| 221 | ]); |
| 222 | }) |
| 223 | ); |
| 224 | |
| 225 | it.effect('does not collect a rejected flag name', () => |
| 226 | Effect.gen(function* () { |
| 227 | const events = yield* lifecycleOf(['--customer-secret']); |
| 228 | expect(events.map(({ call }) => call.context.flagNames)).toEqual([[], []]); |
| 229 | }) |
| 230 | ); |
| 231 | |
| 232 | it.effect('does not interpret run script operands as CLI options', () => |
| 233 | Effect.gen(function* () { |
| 234 | // The missing source stops before execution. Parsing and lifecycle telemetry still run. |
| 235 | const events = yield* lifecycleOf([ |
| 236 | 'run', |
| 237 | '--file', |
| 238 | '/missing/cli-telemetry-test.ts', |
| 239 | '--', |
| 240 | '--dry-run', |
| 241 | ]); |
| 242 | const invoked = events[0]!; |
| 243 | expect(invoked.path).toBe('run'); |
| 244 | expect(getPrimaryLifecycleInvokedEvent(invoked.call.context)?.properties).toMatchObject({ |
| 245 | dry_run: false, |
| 246 | file_mode: true, |
| 247 | arg_count: 1, |
| 248 | }); |
| 249 | expect(invoked.call.context.flagNames).toEqual(['--file']); |
| 250 | }) |
| 251 | ); |
| 252 | |
| 253 | it.effect.each([ |
| 254 | { args: ['search', 'send email'], properties: { limit: '10' } }, |
| 255 | { args: ['search', 'send email', '--limit', '7'], properties: { limit: '7' } }, |
| 256 | { |
| 257 | args: ['proxy', '/private-path', '--toolkit', 'gmail'], |
| 258 | properties: { method: 'GET', toolkit: 'gmail', header_count: 0 }, |
| 259 | }, |
| 260 | { |
| 261 | args: ['proxy', '/private-path', '--toolkit', 'gmail', '-X', 'POST'], |
| 262 | properties: { method: 'POST', toolkit: 'gmail', header_count: 0 }, |
| 263 | }, |
| 264 | { args: ['setup', '--if-present'], properties: { target: 'auto', if_present: true } }, |
| 265 | ])('uses effective parser values for `$args`', ({ args, properties }) => |
| 266 | Effect.gen(function* () { |
| 267 | const events = yield* lifecycleOf(args); |
| 268 | expect(events).toHaveLength(2); |
| 269 | events.forEach(({ call }) => { |
| 270 | expect(getPrimaryLifecycleInvokedEvent(call.context)?.properties).toMatchObject(properties); |
| 271 | }); |
| 272 | }) |
| 273 | ); |
| 274 | |
| 275 | it.effect.each( |
| 276 | [ |
| 277 | { |
| 278 | command: 'execute', |
| 279 | invalid: [], |
| 280 | fields: [ |
| 281 | 'dry_run', |
| 282 | 'get_schema', |
| 283 | 'has_data', |
| 284 | 'skip_connection_check', |
| 285 | 'skip_tool_params_check', |
| 286 | 'skip_checks', |
| 287 | ], |
| 288 | }, |
| 289 | { |
| 290 | command: 'search', |
| 291 | invalid: ['--limit', 'invalid'], |
| 292 | fields: ['limit', 'toolkits', 'query_length', 'query_term_count'], |
| 293 | }, |
| 294 | { |
| 295 | command: 'link', |
| 296 | invalid: ['--no-wait=invalid'], |
| 297 | fields: ['no_browser', 'no_wait', 'has_auth_config'], |
| 298 | }, |
| 299 | { |
| 300 | command: 'login', |
| 301 | invalid: ['--no-wait=invalid'], |
| 302 | fields: ['no_browser', 'no_wait', 'has_key', 'skip_org_picker'], |
| 303 | }, |
| 304 | { |
| 305 | command: 'install', |
| 306 | invalid: ['--completions=invalid'], |
| 307 | fields: ['completions', 'no_completions'], |
| 308 | }, |
| 309 | { |
| 310 | command: 'setup', |
| 311 | invalid: ['--target', 'invalid'], |
| 312 | fields: ['operation', 'target', 'yes', 'if_present'], |
| 313 | }, |
| 314 | { |
| 315 | command: 'proxy', |
| 316 | invalid: [], |
| 317 | fields: [ |
| 318 | 'method', |
| 319 | 'toolkit', |
| 320 | 'has_endpoint', |
| 321 | 'has_data', |
| 322 | 'header_count', |
| 323 | 'skip_connection_check', |
| 324 | ], |
| 325 | }, |
| 326 | { |
| 327 | command: 'run', |
| 328 | invalid: ['--file'], |
| 329 | fields: [ |
| 330 | 'debug', |
| 331 | 'dry_run', |
| 332 | 'skip_connection_check', |
| 333 | 'skip_tool_params_check', |
| 334 | 'skip_checks', |
| 335 | 'file_mode', |
| 336 | 'arg_count', |
| 337 | ], |
| 338 | }, |
| 339 | ].flatMap(({ command, invalid, fields }) => [ |
| 340 | { command, fields, args: [command, '--help'], outcome: 'succeeded' }, |
| 341 | { command, fields, args: [command, ...invalid], outcome: 'failed' }, |
| 342 | ]) |
| 343 | )('omits unparsed input for `$args` ($outcome)', ({ command, fields, args, outcome }) => |
| 344 | Effect.gen(function* () { |
| 345 | const events = yield* lifecycleOf(args); |
| 346 | expect(events.map(({ event }) => event)).toEqual(['invoked', outcome]); |
| 347 | events.forEach(({ call }) => { |
| 348 | const event = |
| 349 | call.event === 'failed' |
| 350 | ? getPrimaryLifecycleFailedEvent(call.context, Cause.squash(call.cause)) |
| 351 | : call.event === 'succeeded' |
| 352 | ? getPrimaryLifecycleSucceededEvent(call.context) |
| 353 | : getPrimaryLifecycleInvokedEvent(call.context); |
| 354 | expect(event?.properties).toMatchObject({ source: 'cli', command_path: command }); |
| 355 | fields.forEach(field => expect(event?.properties).not.toHaveProperty(field)); |
| 356 | expect(call.context.options).toEqual({}); |
| 357 | expect(call.context.flagNames).toEqual([]); |
| 358 | if (call.event === 'failed') { |
| 359 | expect(event?.properties).toHaveProperty('failure_reason_code', 'parse_error'); |
| 360 | } |
| 361 | }); |
| 362 | }) |
| 363 | ); |
| 364 | |
| 365 | it.effect('keeps the projection when a shared command is registered under dev', () => |
| 366 | Effect.gen(function* () { |
| 367 | const events = yield* lifecycleOf([ |
| 368 | 'dev', |
| 369 | 'connected-accounts', |
| 370 | 'link', |
| 371 | 'gmail', |
| 372 | '--no-browser', |
| 373 | ]); |
| 374 | expect(events).toHaveLength(2); |
| 375 | events.forEach(({ call }) => { |
| 376 | expect(call.context.family).toBe('command'); |
| 377 | expect(call.context.flagNames).toEqual(['--no-browser']); |
| 378 | expect(getPrimaryLifecycleInvokedEvent(call.context)?.properties).toMatchObject({ |
| 379 | flag_names: ['--no-browser'], |
| 380 | }); |
| 381 | }); |
| 382 | }) |
| 383 | ); |
| 384 | |
| 385 | it.effect('projects only measured fields from parsed proxy input', () => |
| 386 | Effect.gen(function* () { |
| 387 | const events = yield* lifecycleOf([ |
| 388 | 'proxy', |
| 389 | '/private-path', |
| 390 | '--toolkit', |
| 391 | 'gmail', |
| 392 | '--account', |
| 393 | 'private-account', |
| 394 | '-H', |
| 395 | 'Authorization: private-header', |
| 396 | '-d', |
| 397 | 'private-body', |
| 398 | ]); |
| 399 | const context = events[0]!.call.context; |
| 400 | expect(context.options).toEqual({ |
| 401 | toolkit: 'gmail', |
| 402 | method: 'GET', |
| 403 | has_endpoint: true, |
| 404 | has_data: true, |
| 405 | header_count: 1, |
| 406 | skip_connection_check: false, |
| 407 | }); |
| 408 | expect(context.argument).toBeUndefined(); |
| 409 | expect(context.flagNames).toEqual(['--data', '--header']); |
| 410 | expect(JSON.stringify(getPrimaryLifecycleInvokedEvent(context))).not.toContain('private-'); |
| 411 | }) |
| 412 | ); |
| 413 | |
| 414 | it.effect('reads the execute tool slug from the parsed input, and none for --help', () => |
| 415 | Effect.gen(function* () { |
| 416 | const [executed] = yield* lifecycleOf(['execute', 'GMAIL_SEND_EMAIL']); |
| 417 | expect(executed?.call.context).toMatchObject({ |
| 418 | commandPath: 'execute', |
| 419 | family: 'execute', |
| 420 | argument: 'GMAIL_SEND_EMAIL', |
| 421 | toolkitSlug: 'gmail', |
| 422 | }); |
| 423 | const [help] = yield* lifecycleOf(['execute', 'GMAIL_SEND_EMAIL', '--help']); |
| 424 | expect(help?.call.context).toMatchObject({ commandPath: 'execute', family: 'execute' }); |
| 425 | expect(help?.call.context.argument).toBeUndefined(); |
| 426 | }) |
| 427 | ); |
| 428 | }); |