Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 428 lines 14.4 KB
Raw
1 import { describe, expect, it, layer } from '@effect/vitest';
2 import { Cause, Data, Effect, Option, Ref } from 'effect';
3 import type { Command } from 'effect/cli';
4 import {
5 getPrimaryLifecycleFailedEvent,
6 getPrimaryLifecycleInvokedEvent,
7 getPrimaryLifecycleSucceededEvent,
8 } from 'src/analytics/events';
9 import type { TelemetryFamily } from 'src/analytics/types';
10 import { buildRootCommand, runWithConfig } from 'src/commands';
11 import {
12 CommandInvocation,
13 type CommandIdentity,
14 commandPathOf,
15 } from 'src/invocation/command-identity';
16 import { cliExit, recordedInvocationCalls, TestLive } from 'test/__utils__';
17 import { RECORDED_BEFORE_IDENTITY } from './command-identity.before';
18
19 const commandPaths = (
20 command: Command.Command.Any,
21 prefix: ReadonlyArray<string>
22 ): ReadonlyArray<ReadonlyArray<string>> =>
23 command.subcommands
24 .flatMap(group => group.commands)
25 .flatMap(child => {
26 const path = [...prefix, child.name];
27 return [path, ...commandPaths(child, path)];
28 });
29
30 const everyFeature = { isDevModeEnabled: true, isExperimentalFeatureEnabled: () => true };
31 const PATHS = [[], ...commandPaths(buildRootCommand(everyFeature), [])].map(path => ({
32 args: path,
33 name: path.join(' ') || 'composio',
34 }));
35
36 /** The root registrations with their own lifecycle family; every other command is `command`. */
37 const FAMILIES: Readonly<Record<string, TelemetryFamily>> = {
38 execute: 'execute',
39 install: 'install',
40 link: 'link',
41 login: 'login',
42 logout: 'logout',
43 proxy: 'proxy',
44 run: 'run',
45 search: 'search',
46 setup: 'setup',
47 };
48
49 /**
50 * The command paths the argv extractor recorded wrongly (`RECORDED_BEFORE_IDENTITY`): a token
51 * missing from its allowlist stopped it early, so these were recorded as `composio`, as their
52 * parent (`dev init` as `dev`), or as their parent's family (`setup skill` as `setup`). Every
53 * other path is recorded as before.
54 */
55 const CORRECTED_PATHS = [
56 'agent',
57 'agent claim',
58 'agent inbox',
59 'agent login',
60 'agent signup',
61 'agent whoami',
62 'config',
63 'config experimental',
64 'connections',
65 'connections list',
66 'connections remove',
67 'debug',
68 'debug api-info',
69 'debug generate-graph',
70 'debug who-is-my-master',
71 'dev init',
72 'setup skill',
73 'signup',
74 // `composio help <command>`, all recorded as `composio`.
75 ...PATHS.filter(({ args }) => args[0] === 'help').map(({ name }) => name),
76 ].toSorted();
77
78 class StopAtBegin extends Data.TaggedError('test/StopAtBegin') {}
79
80 /**
81 * The identity `effect/cli` gives `args` at runtime: the selected registration's, or for help
82 * and parse failures the one the parser names. Stops at `begin`, so no handler runs.
83 */
84 const identityOf = (args: ReadonlyArray<string>) =>
85 Effect.gen(function* () {
86 const begun = yield* Ref.make(Option.none<CommandIdentity>());
87 const run = yield* runWithConfig;
88 yield* run(['node', '<CMD>', ...args]).pipe(
89 Effect.provideService(CommandInvocation, {
90 begin: identity =>
91 Ref.set(begun, Option.some(identity)).pipe(Effect.andThen(Effect.die(new StopAtBegin()))),
92 enter: () => Effect.void,
93 runId: Effect.succeedNone,
94 }),
95 Effect.exit
96 );
97 return yield* Ref.get(begun);
98 });
99
100 describe('command identity', () => {
101 layer(TestLive({ cliUserConfig: { experimentalFeatures: { listen: true } } }))(it => {
102 it.effect.each(PATHS)('`$name` is recorded as itself, with its family', ({ args, name }) =>
103 Effect.gen(function* () {
104 const identity = yield* identityOf(args);
105 expect(Option.map(identity, commandPathOf)).toEqual(Option.some(name));
106 expect(Option.map(identity, ({ meta }) => meta.telemetryFamily)).toEqual(
107 Option.some(FAMILIES[name] ?? 'command')
108 );
109 })
110 );
111 });
112
113 it('changes the recorded path or family only of the paths the argv extractor got wrong', () => {
114 const changed = PATHS.filter(({ name }) => {
115 const before = RECORDED_BEFORE_IDENTITY[name];
116 return before === undefined
117 ? false
118 : before.path !== name || before.family !== (FAMILIES[name] ?? 'command');
119 }).map(({ name }) => name);
120 expect(changed.toSorted()).toEqual(CORRECTED_PATHS);
121 expect(PATHS.map(({ name }) => name).toSorted()).toEqual(
122 Object.keys(RECORDED_BEFORE_IDENTITY).toSorted()
123 );
124 });
125 });
126
127 /** The lifecycle events `cli(args)` recorded, as `<event> <command path> <family>`. */
128 const lifecycleOf = (args: ReadonlyArray<string>) =>
129 Effect.gen(function* () {
130 yield* cliExit(args);
131 const calls = yield* recordedInvocationCalls;
132 return calls.flatMap(call =>
133 call.service === 'CommandLifecycleTelemetry' && call.event !== 'pluginHintShown'
134 ? [{ event: call.event, path: call.context.commandPath, family: call.context.family, call }]
135 : []
136 );
137 }).pipe(Effect.provide(TestLive()));
138
139 describe('lifecycle telemetry from the command identity', () => {
140 it.effect.each([
141 { args: ['signup', '--help'], path: 'signup', family: 'command', outcome: 'succeeded' },
142 {
143 args: ['setup', 'skill', '--help'],
144 path: 'setup skill',
145 family: 'command',
146 outcome: 'succeeded',
147 },
148 { args: ['--version'], path: 'composio', family: 'command', outcome: 'succeeded' },
149 {
150 args: ['connections', 'list'],
151 path: 'connections list',
152 family: 'command',
153 outcome: 'failed',
154 },
155 { args: ['tools'], path: 'tools', family: 'command', outcome: 'failed' },
156 { args: ['link', 'gmail'], path: 'link', family: 'link', outcome: 'failed' },
157 {
158 args: ['dev', 'connected-accounts', 'link', 'gmail'],
159 path: 'dev connected-accounts link',
160 family: 'command',
161 outcome: 'failed',
162 },
163 {
164 args: ['dev', '--user-id', 'u', 'triggers', 'list', 'gmail'],
165 path: 'dev triggers list',
166 family: 'command',
167 outcome: 'failed',
168 },
169 {
170 args: ['dev', 'triggers', 'list', 'gmail', '--user-id', 'u'],
171 path: 'dev triggers list',
172 family: 'command',
173 outcome: 'failed',
174 },
175 // The `logs` group declares --project-name unsupported; its subcommand rejects it once it began.
176 {
177 args: ['dev', 'logs', 'tools', '--project-name', 'p'],
178 path: 'dev logs tools',
179 family: 'command',
180 outcome: 'failed',
181 },
182 ])(
183 '[Given] logged out, `composio $args` [Then] records $path ($family) as $outcome',
184 ({ args, path, family, outcome }) =>
185 Effect.gen(function* () {
186 const events = yield* lifecycleOf(args);
187 expect(events.map(({ event, path, family }) => ({ event, path, family }))).toEqual([
188 { event: 'invoked', path, family },
189 { event: outcome, path, family },
190 ]);
191 })
192 );
193
194 it.effect('records a typo against the path the parser reached, never the typed token', () =>
195 Effect.gen(function* () {
196 const events = yield* lifecycleOf(['dve', 'list']);
197 expect(events.map(({ event, path }) => ({ event, path }))).toEqual([
198 { event: 'invoked', path: 'composio' },
199 { event: 'failed', path: 'composio' },
200 ]);
201 const failed = events[1]!.call;
202 expect(failed.event).toBe('failed');
203 const properties =
204 failed.event === 'failed'
205 ? getPrimaryLifecycleFailedEvent(failed.context, Cause.squash(failed.cause))?.properties
206 : undefined;
207 expect(properties).toMatchObject({
208 command_path: 'composio',
209 failure_reason_code: 'parse_error',
210 });
211 expect(JSON.stringify(properties)).not.toContain('dve');
212 })
213 );
214
215 it.effect('uses the displayed command identity for redundant help', () =>
216 Effect.gen(function* () {
217 const events = yield* lifecycleOf(['help', 'execute', '--help']);
218 expect(events.map(({ event, path, family }) => ({ event, path, family }))).toEqual([
219 { event: 'invoked', path: 'execute', family: 'execute' },
220 { event: 'succeeded', path: 'execute', family: 'execute' },
221 ]);
222 })
223 );
224
225 it.effect('does not collect a rejected flag name', () =>
226 Effect.gen(function* () {
227 const events = yield* lifecycleOf(['--customer-secret']);
228 expect(events.map(({ call }) => call.context.flagNames)).toEqual([[], []]);
229 })
230 );
231
232 it.effect('does not interpret run script operands as CLI options', () =>
233 Effect.gen(function* () {
234 // The missing source stops before execution. Parsing and lifecycle telemetry still run.
235 const events = yield* lifecycleOf([
236 'run',
237 '--file',
238 '/missing/cli-telemetry-test.ts',
239 '--',
240 '--dry-run',
241 ]);
242 const invoked = events[0]!;
243 expect(invoked.path).toBe('run');
244 expect(getPrimaryLifecycleInvokedEvent(invoked.call.context)?.properties).toMatchObject({
245 dry_run: false,
246 file_mode: true,
247 arg_count: 1,
248 });
249 expect(invoked.call.context.flagNames).toEqual(['--file']);
250 })
251 );
252
253 it.effect.each([
254 { args: ['search', 'send email'], properties: { limit: '10' } },
255 { args: ['search', 'send email', '--limit', '7'], properties: { limit: '7' } },
256 {
257 args: ['proxy', '/private-path', '--toolkit', 'gmail'],
258 properties: { method: 'GET', toolkit: 'gmail', header_count: 0 },
259 },
260 {
261 args: ['proxy', '/private-path', '--toolkit', 'gmail', '-X', 'POST'],
262 properties: { method: 'POST', toolkit: 'gmail', header_count: 0 },
263 },
264 { args: ['setup', '--if-present'], properties: { target: 'auto', if_present: true } },
265 ])('uses effective parser values for `$args`', ({ args, properties }) =>
266 Effect.gen(function* () {
267 const events = yield* lifecycleOf(args);
268 expect(events).toHaveLength(2);
269 events.forEach(({ call }) => {
270 expect(getPrimaryLifecycleInvokedEvent(call.context)?.properties).toMatchObject(properties);
271 });
272 })
273 );
274
275 it.effect.each(
276 [
277 {
278 command: 'execute',
279 invalid: [],
280 fields: [
281 'dry_run',
282 'get_schema',
283 'has_data',
284 'skip_connection_check',
285 'skip_tool_params_check',
286 'skip_checks',
287 ],
288 },
289 {
290 command: 'search',
291 invalid: ['--limit', 'invalid'],
292 fields: ['limit', 'toolkits', 'query_length', 'query_term_count'],
293 },
294 {
295 command: 'link',
296 invalid: ['--no-wait=invalid'],
297 fields: ['no_browser', 'no_wait', 'has_auth_config'],
298 },
299 {
300 command: 'login',
301 invalid: ['--no-wait=invalid'],
302 fields: ['no_browser', 'no_wait', 'has_key', 'skip_org_picker'],
303 },
304 {
305 command: 'install',
306 invalid: ['--completions=invalid'],
307 fields: ['completions', 'no_completions'],
308 },
309 {
310 command: 'setup',
311 invalid: ['--target', 'invalid'],
312 fields: ['operation', 'target', 'yes', 'if_present'],
313 },
314 {
315 command: 'proxy',
316 invalid: [],
317 fields: [
318 'method',
319 'toolkit',
320 'has_endpoint',
321 'has_data',
322 'header_count',
323 'skip_connection_check',
324 ],
325 },
326 {
327 command: 'run',
328 invalid: ['--file'],
329 fields: [
330 'debug',
331 'dry_run',
332 'skip_connection_check',
333 'skip_tool_params_check',
334 'skip_checks',
335 'file_mode',
336 'arg_count',
337 ],
338 },
339 ].flatMap(({ command, invalid, fields }) => [
340 { command, fields, args: [command, '--help'], outcome: 'succeeded' },
341 { command, fields, args: [command, ...invalid], outcome: 'failed' },
342 ])
343 )('omits unparsed input for `$args` ($outcome)', ({ command, fields, args, outcome }) =>
344 Effect.gen(function* () {
345 const events = yield* lifecycleOf(args);
346 expect(events.map(({ event }) => event)).toEqual(['invoked', outcome]);
347 events.forEach(({ call }) => {
348 const event =
349 call.event === 'failed'
350 ? getPrimaryLifecycleFailedEvent(call.context, Cause.squash(call.cause))
351 : call.event === 'succeeded'
352 ? getPrimaryLifecycleSucceededEvent(call.context)
353 : getPrimaryLifecycleInvokedEvent(call.context);
354 expect(event?.properties).toMatchObject({ source: 'cli', command_path: command });
355 fields.forEach(field => expect(event?.properties).not.toHaveProperty(field));
356 expect(call.context.options).toEqual({});
357 expect(call.context.flagNames).toEqual([]);
358 if (call.event === 'failed') {
359 expect(event?.properties).toHaveProperty('failure_reason_code', 'parse_error');
360 }
361 });
362 })
363 );
364
365 it.effect('keeps the projection when a shared command is registered under dev', () =>
366 Effect.gen(function* () {
367 const events = yield* lifecycleOf([
368 'dev',
369 'connected-accounts',
370 'link',
371 'gmail',
372 '--no-browser',
373 ]);
374 expect(events).toHaveLength(2);
375 events.forEach(({ call }) => {
376 expect(call.context.family).toBe('command');
377 expect(call.context.flagNames).toEqual(['--no-browser']);
378 expect(getPrimaryLifecycleInvokedEvent(call.context)?.properties).toMatchObject({
379 flag_names: ['--no-browser'],
380 });
381 });
382 })
383 );
384
385 it.effect('projects only measured fields from parsed proxy input', () =>
386 Effect.gen(function* () {
387 const events = yield* lifecycleOf([
388 'proxy',
389 '/private-path',
390 '--toolkit',
391 'gmail',
392 '--account',
393 'private-account',
394 '-H',
395 'Authorization: private-header',
396 '-d',
397 'private-body',
398 ]);
399 const context = events[0]!.call.context;
400 expect(context.options).toEqual({
401 toolkit: 'gmail',
402 method: 'GET',
403 has_endpoint: true,
404 has_data: true,
405 header_count: 1,
406 skip_connection_check: false,
407 });
408 expect(context.argument).toBeUndefined();
409 expect(context.flagNames).toEqual(['--data', '--header']);
410 expect(JSON.stringify(getPrimaryLifecycleInvokedEvent(context))).not.toContain('private-');
411 })
412 );
413
414 it.effect('reads the execute tool slug from the parsed input, and none for --help', () =>
415 Effect.gen(function* () {
416 const [executed] = yield* lifecycleOf(['execute', 'GMAIL_SEND_EMAIL']);
417 expect(executed?.call.context).toMatchObject({
418 commandPath: 'execute',
419 family: 'execute',
420 argument: 'GMAIL_SEND_EMAIL',
421 toolkitSlug: 'gmail',
422 });
423 const [help] = yield* lifecycleOf(['execute', 'GMAIL_SEND_EMAIL', '--help']);
424 expect(help?.call.context).toMatchObject({ commandPath: 'execute', family: 'execute' });
425 expect(help?.call.context.argument).toBeUndefined();
426 })
427 );
428 });