Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 304 lines 18.8 KB
Raw
1 import { describe, expect, test } from 'bun:test';
2 import { join } from 'node:path';
3 import { checkApiSnippets, loadApiSpecs, parseApiSpec, parseCurlCommands, wordText, type ApiSpecs } from '../../lib/api-snippets';
4 import { listCodeExampleFiles } from '../../lib/code-examples';
5
6 const BASE = 'https://backend.composio.dev/api/v3.1';
7 const KEY = '-H "x-api-key: $COMPOSIO_API_KEY"';
8 const SCHEMES = {
9 ApiKey: { type: 'apiKey', in: 'header', name: 'x-api-key' },
10 UserKey: { type: 'apiKey', in: 'header', name: 'x-user-api-key' },
11 Bearer: { type: 'http', scheme: 'bearer' },
12 };
13 const AUTH = [{ ApiKey: [] }];
14 const json = (schema: unknown, required?: boolean) => ({ required, content: { 'application/json': { schema } } });
15 const EXECUTE = {
16 type: 'object', additionalProperties: false, required: ['arguments'],
17 properties: { arguments: { type: 'object' }, user_id: { type: 'string' } },
18 };
19 const SPECS: ApiSpecs = {
20 v3_1: parseApiSpec('openapi.json', {
21 paths: {
22 '/api/v3.1/tools': { get: { security: AUTH } },
23 '/api/v3.1/tools/execute/{tool_slug}': { post: { security: [{ ApiKey: [] }, { UserKey: [] }], requestBody: json(EXECUTE) } },
24 '/api/v3.1/things/search': { get: { security: AUTH } },
25 '/api/v3.1/things/{id}': { post: { security: AUTH } },
26 '/api/v3.1/health': { get: { security: [{}] } },
27 '/api/v3.1/unknown-security': { get: { security: [{ UndefinedScheme: [] }] } },
28 '/api/v3.1/mixed-security': { get: { security: [{ UndefinedScheme: [] }, { ApiKey: [] }] } },
29 '/api/v3.1/bearer': { get: { security: [{ Bearer: [] }] } },
30 '/api/v3.1/combined-security': { get: { security: [{ ApiKey: [], UserKey: [] }] } },
31 '/api/v3.1/nullable-gap/{id}': { patch: { security: AUTH, requestBody: json({
32 type: 'object', additionalProperties: false,
33 properties: { settings: { type: 'object', properties: { callback_url: { type: 'string' }, enabled: { type: 'boolean' } } } },
34 }) } },
35 },
36 components: { securitySchemes: SCHEMES },
37 }),
38 v3: parseApiSpec('openapi-v3.json', {
39 paths: { '/api/v3/tools': { get: { security: AUTH } } },
40 components: { securitySchemes: SCHEMES },
41 }),
42 };
43 const fence = (...lines: string[]) => ['```bash', ...lines, '```'].join('\n');
44 const check = (source: string, specs = SPECS, path = 'content/docs/page.mdx') => checkApiSnippets([{ path, source }], specs);
45 const rules = (source: string, specs = SPECS, path?: string) => check(source, specs, path).map(({ rule }) => rule);
46 const exception = (rule: string, operation: string, field?: string) =>
47 `{/* code-example: unchecked-api rule="${rule}" operation="${operation}"${field === undefined ? '' : ` field="${field}"`} reason="Published schema differs from the documented endpoint" */}`;
48 const schemaSpecs = (schema: unknown, required?: boolean): ApiSpecs => {
49 const spec = parseApiSpec('schema-fixture', {
50 paths: { '/api/v3.1/check': { post: { security: AUTH, requestBody: json(schema, required) } } },
51 components: { securitySchemes: SCHEMES, schemas: {
52 X: { type: 'object', properties: { a: { type: 'string' } } },
53 Tree: { type: 'object', properties: { label: { type: 'string' }, child: { $ref: '#/components/schemas/Tree' } } },
54 Bounded: { type: 'number', minimum: 0, exclusiveMinimum: true, maximum: 10, exclusiveMaximum: true },
55 } },
56 });
57 return { v3_1: spec, v3: spec };
58 };
59 const bodyCheck = (schema: unknown, value: unknown) => check(
60 fence(`curl ${BASE}/check ${KEY} --json='${JSON.stringify(value)}'`), schemaSpecs(schema),
61 );
62
63 describe('curl request behavior', () => {
64 test('equals-form headers, method, URL, and JSON are actually checked', () => {
65 expect(rules(fence(`curl --url=${BASE}/tools --header="x-api-key: $COMPOSIO_API_KEY"`))).toEqual([]);
66 expect(rules(fence(`curl ${BASE}/tools ${KEY} --request=DELETE`))).toEqual(['api-unknown-operation']);
67 expect(rules(fence(`curl --url=${BASE}/tools/execute/X ${KEY} --json='{"wrong":true}'`))).toEqual(['api-body-schema', 'api-body-schema']);
68 expect(rules(fence(`curl ${BASE}/tools/execute/X ${KEY} --json=`))).toEqual(['api-json-invalid']);
69 });
70
71 test('every positional/--url transfer is checked, including a later invalid URL', () => {
72 expect(check(fence(`curl ${BASE}/tools --url=${BASE}/missing ${KEY}`)).map(({ rule, message }) => [rule, message])).toEqual([
73 ['api-unknown-operation', 'GET /api/v3.1/missing is not an operation in openapi.json'],
74 ]);
75 expect(rules(fence(`curl https://example.com ${BASE}/missing ${KEY}`))).toEqual(['api-unknown-operation']);
76 });
77
78 test('--next resets method, headers, and data instead of leaking them across transfers', () => {
79 const source = fence(`curl ${BASE}/tools/execute/X ${KEY} --json='{"arguments":{}}' --next ${BASE}/tools`);
80 expect(rules(source)).toEqual(['api-key-header']);
81 expect(parseCurlCommands(source.slice(8, -4)).map(({ method, urls, headers, data }) => ({
82 method, urls: urls.map(wordText), headers: headers.length, data: data.length,
83 }))).toEqual([
84 { method: 'POST', urls: [`${BASE}/tools/execute/X`], headers: 1, data: 1 },
85 { method: 'GET', urls: [`${BASE}/tools`], headers: 0, data: 0 },
86 ]);
87 });
88
89 test('upload-file infers PUT and diagnoses its uncheckable body', () => {
90 for (const upload of ['-T payload.json', '--upload-file=payload.json']) {
91 const result = check(fence(`curl ${BASE}/tools ${KEY} ${upload}`));
92 expect(result.some(({ rule, message }) => rule === 'api-unknown-operation' && message.startsWith('PUT '))).toBe(true);
93 expect(result.some(({ rule }) => rule === 'api-body-unchecked')).toBe(true);
94 }
95 });
96
97 test('validates commands inside substitutions and respects quote and continuation semantics', () => {
98 const source = fence(
99 `ID=$(curl -s ${BASE}/missing ${KEY} | jq -r '.id')`,
100 'echo "$(curl ' + BASE + '/tools ' + KEY + ')"',
101 `curl -fsSL --compressed --max-time=30 --retry 2 ${BASE}/tools \\`,
102 ` ${KEY} 2>&1 | jq .`,
103 );
104 expect(check(source).map(({ line, rule }) => `${line} ${rule}`)).toEqual(['2 api-unknown-operation']);
105 expect(check(source.replaceAll('\n', '\r\n')).map(({ line, rule }) => `${line} ${rule}`)).toEqual(['2 api-unknown-operation']);
106 expect(rules(fence(`curl ${BASE}/tools -H 'x-api-key: $COMPOSIO_API_KEY'`))).toEqual(['api-key-header']);
107 expect(rules(fence(`curl ${BASE}/tools -H "x-api-key: \\$COMPOSIO_API_KEY"`))).toEqual(['api-key-header']);
108 expect(rules(fence(`curl ${BASE}/tools -H "x-api-key: \${COMPOSIO_API_KEY}"`))).toEqual([]);
109 expect(rules(fence(`echo "\`curl ${BASE}/missing ${KEY}\`"`))).toEqual(['api-unknown-operation']);
110 });
111
112 test('does not silently accept unknown flags, configs, expanded headers, or malformed shell', () => {
113 for (const options of ['--expand-json=@payload', '--request-target=/hidden', '--config=secrets.cfg', '--header=@headers', '--mystery']) {
114 expect(rules(fence(`curl ${BASE}/tools ${KEY} ${options}`))).toContain('api-curl-unsupported');
115 }
116 expect(rules(fence(`curl ${BASE}/tools ${KEY} -H "unterminated`))).toContain('api-curl-unsupported');
117 });
118
119 test('form, file, URL-encoded, and fully expanded bodies cannot silently pass', () => {
120 for (const data of ["-F wrong=1", '--form-string=wrong=1', '--data=@payload.json', '--json=@payload.json', '--data-urlencode=wrong=1', '-d "$PAYLOAD"']) {
121 expect(rules(fence(`curl ${BASE}/tools/execute/X ${KEY} ${data}`))).toContain('api-body-unchecked');
122 }
123 expect(rules(fence(`curl ${BASE}/tools/execute/X ${KEY} --data-raw=@payload.json`))).toEqual(['api-json-invalid']);
124 });
125
126 test('literal path precedence is resolved before HTTP method selection', () => {
127 expect(check(fence(`curl -X POST ${BASE}/things/search ${KEY}`)).map(({ rule, message }) => [rule, message])).toEqual([
128 ['api-unknown-operation', 'POST /api/v3.1/things/search is not an operation in openapi.json; the path allows GET'],
129 ]);
130 expect(rules(fence(`curl -X POST ${BASE}/things/thing_1 ${KEY}`))).toEqual([]);
131 });
132
133 test('non-API transfers and non-shell fences are not mistaken for Composio calls', () => {
134 expect(rules(fence('curl -fsSL https://composio.dev/install | sh', 'curl http://localhost:8000/x'))).toEqual([]);
135 expect(rules(['```python', `# curl ${BASE}/missing`, '```'].join('\n'))).toEqual([]);
136 });
137
138 test('recognizes self-hosted Thermos variables rather than silently ignoring health requests', () => {
139 for (const base of ['$THERMOS_BASE_URL', '${THERMOS_BASE_URL}']) {
140 expect(rules(fence(`curl "${base}/api/health/deep"`))).toEqual([
141 'api-base-url', 'api-key-header', 'api-legacy-version',
142 ]);
143 }
144 });
145 });
146
147 describe('OpenAPI authentication', () => {
148 test('accepts any fully valid security alternative, not just the first present header', () => {
149 expect(rules(fence(`curl ${BASE}/tools/execute/X -H "x-api-key: literal" -H "x-user-api-key: $COMPOSIO_USER_API_KEY" --json='{"arguments":{}}'`))).toEqual([]);
150 expect(rules(fence(`curl ${BASE}/mixed-security ${KEY}`))).toEqual([]);
151 expect(rules(fence(`curl ${BASE}/combined-security ${KEY}`))).toEqual(['api-key-header']);
152 });
153
154 test('unknown schemes are not public, while an explicit empty requirement is public', () => {
155 expect(rules(fence(`curl ${BASE}/unknown-security`))).toEqual(['api-key-header']);
156 expect(rules(fence(`curl ${BASE}/health`))).toEqual([]);
157 expect(rules(fence(`curl ${BASE}/bearer -H "authorization: Bearer $TOKEN"`))).toEqual([]);
158 expect(rules(fence(`curl ${BASE}/bearer --oauth2-bearer="$TOKEN"`))).toEqual([]);
159 expect(rules(fence(`curl ${BASE}/bearer -H 'authorization: Bearer $TOKEN'`))).toEqual(['api-key-header']);
160 });
161 });
162
163 describe('OpenAPI JSON schema semantics', () => {
164 test('oneOf means exactly one matching branch, while anyOf permits overlap', () => {
165 expect(bodyCheck({ oneOf: [{ type: 'number' }, { type: 'integer' }] }, 1).some(({ message }) => message.includes('(oneOf)'))).toBe(true);
166 expect(bodyCheck({ anyOf: [{ type: 'number' }, { type: 'integer' }] }, 1)).toEqual([]);
167 expect(bodyCheck({ anyOf: [{ type: 'string' }, { type: 'integer' }] }, true).some(({ message }) => message.includes('(anyOf)'))).toBe(true);
168 });
169
170 test('allOf does not merge closed object properties across branches', () => {
171 const schema = { allOf: [
172 { type: 'object', properties: { a: { type: 'string' } }, additionalProperties: false },
173 { type: 'object', properties: { b: { type: 'string' } } },
174 ] };
175 expect(bodyCheck(schema, { a: 'x', b: 'y' }).some(({ message }) => message.includes('"b"') && message.includes('(additionalProperties)'))).toBe(true);
176 });
177
178 test('boolean schemas, nullable, enum null, typed additionalProperties, and refs follow Ajv', () => {
179 expect(bodyCheck(false, {}).map(({ rule }) => rule)).toEqual(['api-body-schema']);
180 expect(bodyCheck({ type: 'string', nullable: true }, null)).toEqual([]);
181 expect(bodyCheck({ enum: ['a', 'b'] }, null).map(({ rule }) => rule)).toEqual(['api-body-schema']);
182 expect(bodyCheck({ type: 'string', nullable: true, enum: ['a'] }, null).map(({ rule }) => rule)).toEqual(['api-body-schema']);
183 expect(bodyCheck({ type: 'object', additionalProperties: { type: 'integer' } }, { a: 'bad' }).map(({ rule }) => rule)).toEqual(['api-body-schema']);
184 expect(bodyCheck({ $ref: '#/components/schemas/X' }, { a: 2 }).map(({ rule }) => rule)).toEqual(['api-body-schema']);
185 expect(bodyCheck({ $ref: '#/components/schemas/Tree' }, { child: { label: 2 } }).map(({ rule }) => rule)).toEqual(['api-body-schema']);
186 });
187
188 test('normalizes OpenAPI 3.0 boolean exclusive bounds without weakening either boundary', () => {
189 for (const [schema, rejected, accepted] of [
190 [{ type: 'number', minimum: 0, exclusiveMinimum: true }, 0, 1],
191 [{ type: 'number', minimum: 0, exclusiveMinimum: false }, -1, 0],
192 [{ type: 'number', maximum: 10, exclusiveMaximum: true }, 10, 9],
193 [{ type: 'number', maximum: 10, exclusiveMaximum: false }, 11, 10],
194 [{ type: 'number', exclusiveMinimum: 0, exclusiveMaximum: 10 }, 0, 5],
195 ] as const) {
196 expect(bodyCheck(schema, rejected).map(({ rule }) => rule)).toEqual(['api-body-schema']);
197 expect(bodyCheck(schema, accepted)).toEqual([]);
198 }
199 for (const value of [0, 10]) {
200 expect(bodyCheck({ $ref: '#/components/schemas/Bounded' }, value).map(({ rule }) => rule)).toEqual(['api-body-schema']);
201 }
202 expect(bodyCheck({ $ref: '#/components/schemas/Bounded' }, 5)).toEqual([]);
203 const schema = { type: 'number', minimum: 0, exclusiveMinimum: true };
204 expect(bodyCheck(schema, 1)).toEqual([]);
205 expect(schema.exclusiveMinimum).toBe(true);
206 expect(schema.minimum).toBe(0);
207 const literal = { minimum: 0, exclusiveMinimum: true };
208 expect(bodyCheck({ type: 'object', const: literal }, literal)).toEqual([]);
209 });
210
211 test('literal abbreviated base64 fails while real shell expansion skips only its unknown value', () => {
212 const schema = {
213 type: 'object',
214 required: ['logo_file'],
215 properties: {
216 logo_file: {
217 type: 'object', required: ['content', 'mime_type'],
218 properties: {
219 content: { type: 'string', pattern: '^[A-Za-z0-9+/]+={0,2}$' },
220 mime_type: { enum: ['image/png', 'image/jpeg'] },
221 },
222 },
223 },
224 };
225 for (const content of ['iVBORw0KGgoAAAANSUhEUgAA...', '${BASE64_CONTENT}']) {
226 expect(bodyCheck(schema, { logo_file: { content, mime_type: 'image/png' } }).some(({ message }) => message.includes('(pattern)'))).toBe(true);
227 }
228 const code = `curl ${BASE}/check ${KEY} -d '{"logo_file":{"content":"'"$BASE64_CONTENT"'","mime_type":"image/png"}}'`;
229 expect(rules(fence(code), schemaSpecs(schema))).toEqual([]);
230 expect(rules(fence(code.replace('image/png', 'invalid/type')), schemaSpecs(schema))).toEqual(['api-body-schema']);
231 });
232
233 test('missing body still reports required fields when requestBody.required is absent', () => {
234 expect(rules(fence(`curl -X POST ${BASE}/tools/execute/X ${KEY}`))).toEqual(['api-body-schema']);
235 expect(rules(fence(`curl -X POST ${BASE}/check ${KEY}`), schemaSpecs({ type: 'object' }, true))).toEqual(['api-body-schema']);
236 expect(rules(fence(`curl -X POST ${BASE}/check ${KEY}`), schemaSpecs({ type: 'object' }))).toEqual([]);
237 });
238
239 test('dynamic values preserve checks on literal surrounding structure and known string types', () => {
240 const specs = schemaSpecs({
241 type: 'object', required: ['count', 'required_literal'], additionalProperties: false,
242 properties: { count: { type: 'number' }, required_literal: { type: 'integer' }, label: { type: 'string', enum: ['fixed'] } },
243 });
244 const valid = `curl ${BASE}/check ${KEY} -d "{\\\"count\\\": $COUNT, \\\"required_literal\\\": 1, \\\"label\\\": \\\"prefix_$LABEL\\\"}"`;
245 expect(rules(fence(valid), specs)).toEqual([]);
246 const invalid = `curl ${BASE}/check ${KEY} -d "{\\\"count\\\": $(date +%s), \\\"wrong\\\": true}"`;
247 const result = check(fence(invalid), specs);
248 expect(result.some(({ message }) => message.includes('required_literal'))).toBe(true);
249 expect(result.some(({ message }) => message.includes('"wrong"'))).toBe(true);
250 const quotedNumber = `curl ${BASE}/check ${KEY} -d "{\\\"count\\\": \\\"$COUNT\\\", \\\"required_literal\\\": 1}"`;
251 expect(check(fence(quotedNumber), specs).some(({ message }) => message.includes('"count"') && message.includes('(type)'))).toBe(true);
252 });
253
254 test('JSON syntax is still checked when an operation forbids a body', () => {
255 expect(rules(fence(`curl -X GET ${BASE}/tools ${KEY} -d '{bad}'`))).toEqual(['api-body-schema', 'api-json-invalid']);
256 expect(rules(fence(`curl -G ${BASE}/tools ${KEY} --data-urlencode=toolkit_slug=gmail`))).toEqual([]);
257 });
258 });
259
260 describe('scoped unchecked-api markers', () => {
261 const patch = `curl -X PATCH ${BASE}/nullable-gap/$ID ${KEY} --json='{"settings":{"callback_url":null}}'`;
262 const op = 'PATCH /api/v3.1/nullable-gap/{id}';
263
264
265 test('stale checks follow request-shape constraints, not proof of runtime endpoint existence', () => {
266 const code = fence(`curl -X POST ${BASE}/things/arbitrary ${KEY}`);
267 expect(rules(code)).toEqual([]);
268 expect(rules([exception('api-unknown-operation', 'POST /api/v3.1/things/arbitrary'), code].join('\n'))).toEqual(['api-exemption']);
269 });
270 test('a body-field gap exempts only that operation, rule, and exact field', () => {
271 const marker = exception('api-body-schema', op, 'settings.callback_url');
272 expect(rules([marker, fence(patch)].join('\n'))).toEqual([]);
273 expect(rules([marker, fence(patch.replace('"callback_url":null', '"callback_url":null,"enabled":"yes"'))].join('\n'))).toEqual(['api-body-schema']);
274 const source = [marker, fence(patch.replace('https:', 'http:').replace(KEY, ''), `curl ${BASE}/missing ${KEY}`)].join('\n');
275 expect(rules(source)).toEqual(['api-base-url', 'api-key-header', 'api-unknown-operation']);
276 });
277
278 test('stale, unmatched, old blanket, and wrong-method markers fail', () => {
279 expect(rules([exception('api-body-schema', op, 'settings.callback_url'), fence(patch.replace('"callback_url":null', '"callback_url":"https://example.com"'))].join('\n'))).toEqual(['api-exemption']);
280 expect(rules([exception('api-body-schema', op.replace('PATCH', 'POST'), 'settings.callback_url'), fence(patch)].join('\n'))).toEqual(['api-exemption', 'api-body-schema']);
281 expect(rules(['{/* code-example: unchecked-api reason="old blanket" */}', fence(`curl ${BASE}/tools ${KEY}`)].join('\n'))).toEqual(['api-exemption']);
282 });
283
284 test('unknown route exceptions retain origin, authentication, and JSON syntax checks', () => {
285 const marker = exception('api-unknown-operation', 'POST /api/v3.1/internal');
286 expect(rules([marker, fence(`curl -X POST ${BASE}/internal ${KEY} --json='{}'`)].join('\n'))).toEqual([]);
287 expect(rules([marker, fence(`curl -X POST ${BASE.replace('https:', 'http:')}/internal --json='{bad}'`)].join('\n'))).toEqual(['api-base-url', 'api-json-invalid', 'api-key-header']);
288 });
289
290 test('version-policy exceptions retain every other check on legacy examples', () => {
291 const marker = exception('api-legacy-version', 'GET /api/v3/tools');
292 expect(rules([marker, fence(`curl https://backend.composio.dev/api/v3/tools ${KEY}`)].join('\n'))).toEqual([]);
293 expect(rules([marker, fence('curl http://backend.composio.dev/api/v3/tools')].join('\n'))).toEqual(['api-base-url', 'api-key-header']);
294 expect(rules(fence(`curl https://backend.composio.dev/api/v3/tools ${KEY}`), SPECS, 'content/reference/v3/authentication.mdx')).toEqual([]);
295 expect(rules(fence('curl http://backend.composio.dev/api/v2/anything --json="{bad}"'))).toEqual(['api-base-url', 'api-json-invalid', 'api-key-header', 'api-legacy-version']);
296 });
297 });
298
299 describe('authored API example inventory', () => {
300 test('all inventory sources match published operations or exact documented exceptions', () => {
301 const docsRoot = join(import.meta.dir, '../..');
302 expect(checkApiSnippets(listCodeExampleFiles(docsRoot), loadApiSpecs(docsRoot))).toEqual([]);
303 });
304 });