@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 954 lines 41.1 KB
Raw
1 /**
2 * Validates Composio curl request shapes against published OpenAPI constraints,
3 * with operation-scoped, stale-checked exceptions. A parameter-route match
4 * alone does not establish that a literal runtime endpoint exists.
5 */
6 import { readFileSync } from 'node:fs';
7 import { join } from 'node:path';
8 import Ajv, { type ErrorObject, type ValidateFunction } from 'ajv';
9 import { z } from 'zod';
10 import { parseCodeExamples, walkCodeExamples, type CodeExampleFile, type MarkerUnit } from './code-examples';
11
12 export type ApiSnippetRule =
13 | 'api-legacy-version'
14 | 'api-base-url'
15 | 'api-unknown-operation'
16 | 'api-key-header'
17 | 'api-json-invalid'
18 | 'api-body-schema'
19 | 'api-body-unchecked'
20 | 'api-curl-unsupported'
21 | 'api-exemption';
22
23 export type ApiSnippetViolation = { path: string; line: number; rule: ApiSnippetRule; message: string };
24
25 // ---------------------------------------------------------------------------
26 // Shell lexing
27 // ---------------------------------------------------------------------------
28
29 /** A run of a shell word: literal text (quotes removed) or an unexpanded `$VAR`/`${..}`/`$(..)`/backtick expansion. */
30 export type ShellSegment = { text: string; expansion: boolean };
31 /** A shell word as its segments, with the 1-based line it starts on. */
32 export type ShellWord = { segments: ShellSegment[]; line: number };
33
34 type Token = { kind: 'word'; word: ShellWord } | { kind: 'op'; op: string };
35
36 type Expansion = { text: string; end: number; substitution?: string };
37
38 const OPERATORS = ['&&', '||', ';;', '|&', '|', '&', ';', '(', ')'];
39 const REDIRECT_RE = /^(?:[<>]{1,2}&?\d*-?|&>>?)/;
40
41 export const wordText = (word: ShellWord) => word.segments.map((segment) => segment.text).join('');
42
43 /** Index just past the `)` closing a `$(` whose body starts at `from`; skips quotes and nested parens. */
44 function closeParen(src: string, from: number): number {
45 let depth = 1;
46 for (let i = from; i < src.length; i++) {
47 const c = src[i];
48 if (c === '\\') i++;
49 else if (c === "'") i = src.indexOf("'", i + 1) < 0 ? src.length : src.indexOf("'", i + 1);
50 else if (c === '`' || (c === '$' && (src[i + 1] === '(' || src[i + 1] === '{'))) {
51 const expansion = readExpansion(src, i);
52 if (expansion) i = expansion.end - 1;
53 } else if (c === '#' && (i === from || /[\s;|&()]/.test(src[i - 1]))) {
54 while (i < src.length && src[i] !== '\n') i++;
55 }
56 else if (c === '"') {
57 for (i++; i < src.length && src[i] !== '"'; i++) {
58 if (src[i] === '\\') i++;
59 else if (src[i] === '$' && src[i + 1] === '(') i = closeParen(src, i + 2) - 1;
60 }
61 } else if (c === '(') depth++;
62 else if (c === ')' && --depth === 0) return i + 1;
63 }
64 return src.length + 1;
65 }
66
67 /** Reads a `$`/backtick expansion at `i`, or returns undefined when `$` is literal. */
68 function readExpansion(src: string, i: number): Expansion | undefined {
69 if (src[i] === '`') {
70 let end = i + 1;
71 while (end < src.length && src[end] !== '`') end += src[end] === '\\' ? 2 : 1;
72 return { text: src.slice(i, end + 1), end: end + 1, substitution: src.slice(i + 1, end) };
73 }
74 if (src.startsWith('$((', i)) {
75 const end = closeParen(src, i + 2);
76 return { text: src.slice(i, end), end };
77 }
78 if (src.startsWith('$(', i)) {
79 const end = closeParen(src, i + 2);
80 return { text: src.slice(i, end), end, substitution: src.slice(i + 2, end - 1) };
81 }
82 if (src.startsWith('${', i)) {
83 let depth = 0;
84 let end = i + 1;
85 for (; end < src.length; end++) {
86 if (src[end] === '{') depth++;
87 else if (src[end] === '}' && --depth === 0) break;
88 }
89 return { text: src.slice(i, end + 1), end: end + 1 };
90 }
91 const name = /^\$(?:[A-Za-z_][A-Za-z0-9_]*|[0-9@*#?$!-])/.exec(src.slice(i, i + 256));
92 return name ? { text: name[0], end: i + name[0].length } : undefined;
93 }
94
95 /**
96 * Tokenizes shell source into words and control operators. Handles quoting,
97 * `\` line continuations, comments, and redirections; command substitutions are
98 * collected (with their starting line) so callers can lex them recursively.
99 */
100 function lexShell(src: string, firstLine: number) {
101 const tokens: Token[] = [];
102 const substitutions: { code: string; line: number }[] = [];
103 const issues: string[] = [];
104 let line = firstLine;
105 let word: ShellSegment[] | undefined;
106 let wordLine = firstLine;
107
108 const startWord = () => {
109 if (!word) {
110 word = [];
111 wordLine = line;
112 }
113 return word;
114 };
115 const push = (text: string, expansion: boolean) => {
116 const segments = startWord();
117 const last = segments.at(-1);
118 if (last && !last.expansion && !expansion) last.text += text;
119 else segments.push({ text, expansion });
120 };
121 const endWord = () => {
122 if (word) tokens.push({ kind: 'word', word: { segments: word, line: wordLine } });
123 word = undefined;
124 };
125 const countLines = (text: string) => {
126 for (const c of text) if (c === '\n') line++;
127 };
128 /** Consumes an expansion at `i`; returns the index after it, or undefined when `$` is literal. */
129 const expand = (i: number): number | undefined => {
130 const expansion = readExpansion(src, i);
131 if (!expansion) return undefined;
132 if (expansion.end > src.length || (src.startsWith('$(', i) && !expansion.text.endsWith(')')) ||
133 (src.startsWith('${', i) && !expansion.text.endsWith('}')) ||
134 (src[i] === '`' && !expansion.text.endsWith('`'))) issues.push('unterminated shell expansion');
135 if (expansion.substitution !== undefined) substitutions.push({ code: expansion.substitution, line });
136 push(expansion.text, true);
137 countLines(expansion.text);
138 return expansion.end;
139 };
140
141 let i = 0;
142 while (i < src.length) {
143 const c = src[i];
144 if (c === '\\') {
145 if (src[i + 1] === '\r' && src[i + 2] === '\n') {
146 line++;
147 i += 3;
148 continue;
149 }
150 if (src[i + 1] === '\n') line++;
151 else if (i + 1 < src.length) push(src[i + 1], false);
152 else issues.push('dangling shell escape');
153 i += 2;
154 } else if (c === ' ' || c === '\t' || c === '\r') {
155 endWord();
156 i++;
157 } else if (c === '\n') {
158 endWord();
159 tokens.push({ kind: 'op', op: '\n' });
160 line++;
161 i++;
162 } else if (c === '#' && !word) {
163 while (i < src.length && src[i] !== '\n') i++;
164 } else if (c === "'") {
165 const close = src.indexOf("'", i + 1);
166 if (close < 0) issues.push('unterminated single quote');
167 const end = close < 0 ? src.length : close;
168 push(src.slice(i + 1, end), false);
169 countLines(src.slice(i + 1, end));
170 i = end + 1;
171 } else if (c === '"') {
172 startWord();
173 for (i++; i < src.length && src[i] !== '"'; ) {
174 const d = src[i];
175 if (d === '\\' && src[i + 1] === '\r' && src[i + 2] === '\n') {
176 line++;
177 i += 3;
178 } else if (d === '\\' && src[i + 1] === '\n') {
179 line++;
180 i += 2;
181 } else if (d === '\\' && '$`"\\'.includes(src[i + 1] ?? '')) {
182 push(src[i + 1], false);
183 i += 2;
184 } else if (d === '$' || d === '`') {
185 const end = expand(i);
186 if (end === undefined) push(d, false);
187 i = end ?? i + 1;
188 } else {
189 if (d === '\n') line++;
190 push(d, false);
191 i++;
192 }
193 }
194 if (i >= src.length) issues.push('unterminated double quote');
195 i++;
196 } else if (c === '$' || c === '`') {
197 const end = expand(i);
198 if (end === undefined) push(c, false);
199 i = end ?? i + 1;
200 } else if ((c === '<' || c === '>' || c === '&') && REDIRECT_RE.test(src.slice(i, i + 8))) {
201 const redirect = REDIRECT_RE.exec(src.slice(i, i + 8))?.[0] ?? c;
202 // A word of digits right before the operator is its file descriptor (`2>`), not an argument.
203 if (word?.length === 1 && !word[0].expansion && /^\d+$/.test(word[0].text)) word = undefined;
204 endWord();
205 tokens.push({ kind: 'op', op: /&\d|&-$/.test(redirect) ? 'dup' : 'redirect' });
206 i += redirect.length;
207 } else {
208 const op = OPERATORS.find((candidate) => src.startsWith(candidate, i));
209 if (op) {
210 endWord();
211 tokens.push({ kind: 'op', op });
212 i += op.length;
213 } else {
214 push(c, false);
215 i++;
216 }
217 }
218 }
219 endWord();
220 return { tokens, substitutions, issues };
221 }
222
223 /** Splits tokens into simple commands (word lists), dropping redirection targets. */
224 function simpleCommands(tokens: Token[]): ShellWord[][] {
225 const commands: ShellWord[][] = [];
226 let current: ShellWord[] = [];
227 let skipNext = false;
228 for (const token of tokens) {
229 if (token.kind === 'word') {
230 if (skipNext) skipNext = false;
231 else current.push(token.word);
232 } else if (token.op === 'redirect') {
233 skipNext = true;
234 } else if (token.op !== 'dup') {
235 if (current.length) commands.push(current);
236 current = [];
237 }
238 }
239 if (current.length) commands.push(current);
240 return commands;
241 }
242
243 // ---------------------------------------------------------------------------
244 // curl parsing
245 // ---------------------------------------------------------------------------
246
247 export type CurlCommand = {
248 /** Line of the `curl` word. */
249 line: number;
250 /** Upper-case HTTP method, explicit (`-X`) or inferred from `-G`/`-I`/data options. */
251 method: string;
252 /** Positional and `--url` arguments, in order. */
253 urls: ShellWord[];
254 headers: { name: string; value: ShellWord }[];
255 /** `-d`/`--data*`/`--json`/`-F` arguments with the option that carried them. */
256 data: { option: string; value: ShellWord }[];
257 /** True for `-G`/`--get`: data is sent as the query string. */
258 get: boolean;
259 upload: boolean;
260 issues: string[];
261 };
262
263 const SHORT_VALUE_OPTIONS = 'AbcCdDeEFHKmoPQrtTuUwxXyYz';
264 const SHORT_FLAGS = 'fsSLsSivNkgORJnj#012346V';
265 const LONG_FLAGS: Record<string, true> = Object.fromEntries([
266 '--get', '--head', '--location', '--location-trusted', '--fail', '--fail-with-body',
267 '--silent', '--show-error', '--compressed', '--verbose', '--include', '--insecure',
268 '--globoff', '--no-buffer', '--no-progress-meter', '--progress-bar', '--http1.0',
269 '--http1.1', '--http2', '--http2-prior-knowledge', '--http3', '--ipv4', '--ipv6',
270 '--retry-all-errors', '--retry-connrefused', '--no-keepalive', '--tcp-nodelay',
271 '--create-dirs', '--remove-on-error', '--remote-name', '--remote-header-name',
272 ].map((option) => [option, true] as const));
273 const LONG_VALUE_OPTIONS: Record<string, boolean> = Object.fromEntries(
274 [
275 '--request', '--header', '--data', '--data-raw', '--data-binary', '--data-ascii', '--data-urlencode', '--json',
276 '--url', '--output', '--user', '--form', '--form-string', '--max-time', '--connect-timeout', '--retry',
277 '--retry-delay', '--retry-max-time', '--write-out', '--user-agent', '--referer', '--cookie', '--cookie-jar',
278 '--upload-file', '--proxy', '--proxy-user', '--config', '--range', '--cert', '--cacert', '--key', '--limit-rate',
279 '--max-redirs', '--resolve', '--dump-header', '--oauth2-bearer', '--variable', '--url-query', '--output-dir',
280 '--connect-to', '--interface', '--continue-at', '--speed-limit', '--speed-time', '--trace', '--trace-ascii',
281 '--stderr', '--expect100-timeout', '--happy-eyeballs-timeout-ms', '--keepalive-time',
282 ].map((option) => [option, true] as const),
283 );
284 const SHORT_LONG: Record<string, string> = {
285 X: '--request', H: '--header', d: '--data', F: '--form', G: '--get', I: '--head',
286 T: '--upload-file', K: '--config',
287 };
288 const DATA_OPTIONS: Record<string, true> = {
289 '--data': true,
290 '--data-raw': true,
291 '--data-binary': true,
292 '--data-ascii': true,
293 '--data-urlencode': true,
294 '--json': true,
295 '--form': true,
296 '--form-string': true,
297 };
298
299 /** Drops the first `count` characters of a word (used for attached short-option values like `-XPOST`). */
300 function dropChars(word: ShellWord, count: number): ShellWord {
301 const segments = [...word.segments];
302 let remaining = count;
303 while (remaining > 0 && segments.length) {
304 const first = segments[0];
305 if (first.text.length > remaining) {
306 segments[0] = { text: first.text.slice(remaining), expansion: first.expansion };
307 remaining = 0;
308 } else {
309 remaining -= first.text.length;
310 segments.shift();
311 }
312 }
313 return { segments, line: word.line };
314 }
315
316 function trimWord(word: ShellWord): ShellWord {
317 const segments = word.segments
318 .map((segment, index, all) => {
319 if (segment.expansion) return segment;
320 let text = segment.text;
321 if (index === 0) text = text.trimStart();
322 if (index === all.length - 1) text = text.trimEnd();
323 return { text, expansion: false };
324 })
325 .filter((segment) => segment.expansion || segment.text);
326 return { segments, line: word.line };
327 }
328
329 function splitHeader(word: ShellWord): { name: string; value: ShellWord } | undefined {
330 const first = word.segments[0];
331 const colon = first && !first.expansion ? first.text.indexOf(':') : -1;
332 if (colon < 0) return undefined;
333 return { name: first.text.slice(0, colon).trim().toLowerCase(), value: trimWord(dropChars(word, colon + 1)) };
334 }
335
336 function parseCurl(words: ShellWord[]): CurlCommand[] {
337 const commands: CurlCommand[] = [];
338 const fresh = (): CurlCommand => ({
339 line: words[0].line, method: '', urls: [], headers: [], data: [], get: false, upload: false, issues: [],
340 });
341 let command = fresh();
342 let head = false;
343 let positional = false;
344 const finish = () => {
345 if (!command.method) command.method = head ? 'HEAD' : command.get ? 'GET' : command.upload ? 'PUT' : command.data.length ? 'POST' : 'GET';
346 commands.push(command);
347 command = fresh();
348 head = false;
349 positional = false;
350 };
351 const apply = (option: string, value?: ShellWord) => {
352 if (option === '--get') command.get = true;
353 else if (option === '--head') head = true;
354 if (LONG_VALUE_OPTIONS[option] && !value) {
355 command.issues.push(`${option} requires a value`);
356 return;
357 }
358 if (!value) return;
359 if (option === '--request') {
360 command.method = wordText(value).toUpperCase();
361 if (value.segments.some((part) => part.expansion)) command.issues.push('expanded HTTP method cannot be checked');
362 if (!/^[!#$%&'*+.^_`|~0-9A-Z-]+$/.test(command.method)) command.issues.push('HTTP method must be a nonempty literal token');
363 } else if (option === '--url') command.urls.push(value);
364 else if (option === '--header') {
365 const header = splitHeader(value);
366 if (header) command.headers.push(header);
367 else command.issues.push('file-backed, expanded, or malformed header cannot be checked');
368 } else if (option === '--oauth2-bearer') {
369 command.headers.push({ name: 'authorization', value: { line: value.line, segments: [{ text: 'Bearer ', expansion: false }, ...value.segments] } });
370 } else if (option === '--upload-file') {
371 command.upload = true;
372 command.data.push({ option, value });
373 } else if (DATA_OPTIONS[option]) command.data.push({ option, value });
374 else if (option === '--config' || option === '--variable') command.issues.push(`${option} may hide request settings and cannot be checked`);
375 };
376 for (let i = 1; i < words.length; i++) {
377 const word = words[i];
378 const text = wordText(word);
379 const first = word.segments[0];
380 if (positional) command.urls.push(word);
381 else if (text === '--') positional = true;
382 else if (text === '--next' || text === '-:') finish();
383 else if (!first || first.expansion || !first.text.startsWith('-') || text === '-') {
384 command.urls.push(word);
385 if (first?.expansion && !BASE_URL_VARS[first.text]) command.issues.push('expanded curl argument may hide request settings');
386 } else if (text.startsWith('--')) {
387 const equals = text.indexOf('=');
388 const option = equals < 0 ? text : text.slice(0, equals);
389 if (!LONG_VALUE_OPTIONS[option] && !LONG_FLAGS[option]) {
390 command.issues.push(`unsupported curl option ${option}`);
391 continue;
392 }
393 if (equals >= 0 && !LONG_VALUE_OPTIONS[option]) command.issues.push(`${option} does not accept a value`);
394 apply(option, LONG_VALUE_OPTIONS[option] ? equals < 0 ? words[++i] : dropChars(word, equals + 1) : undefined);
395 } else {
396 for (let j = 1; j < first.text.length; j++) {
397 const flag = first.text[j];
398 const option = SHORT_LONG[flag] ?? `-${flag}`;
399 if (SHORT_VALUE_OPTIONS.includes(flag)) {
400 const value = j + 1 < text.length ? dropChars(word, j + 1) : words[++i];
401 if (!value) command.issues.push(`${option} requires a value`);
402 apply(option, value);
403 break;
404 }
405 if (!SHORT_LONG[flag] && !SHORT_FLAGS.includes(flag)) command.issues.push(`unsupported curl option -${flag}`);
406 apply(option);
407 }
408 }
409 }
410 finish();
411 return commands;
412 }
413
414 /**
415 * Parses every `curl` invocation in shell source, including those inside
416 * `$(...)`/backtick substitutions and pipelines. `firstLine` is the file line
417 * of the code's first line. Results are ordered by line.
418 */
419 export function parseCurlCommands(code: string, firstLine = 1): CurlCommand[] {
420 const commands: CurlCommand[] = [];
421 const visit = (src: string, line: number) => {
422 const { tokens, substitutions, issues } = lexShell(src, line);
423 for (const words of simpleCommands(tokens)) {
424 let start = 0;
425 while (start < words.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(words[start].segments[0]?.text ?? '')) start++;
426 const name = words[start];
427 if (name && wordText(name) === 'curl' && !name.segments.some((segment) => segment.expansion)) {
428 for (const command of parseCurl(words.slice(start))) {
429 command.issues.push(...issues);
430 commands.push(command);
431 }
432 }
433 }
434 for (const substitution of substitutions) visit(substitution.code, substitution.line);
435 };
436 visit(code, firstLine);
437 return commands.sort((a, b) => a.line - b.line);
438 }
439
440 // ---------------------------------------------------------------------------
441 // OpenAPI specs
442 // ---------------------------------------------------------------------------
443
444 export type ApiOperation = {
445 /** An empty requirement is public; an unknown security scheme is not. */
446 auth: { headers: string[]; unsupported: string[] }[];
447 hasBody: boolean;
448 bodyRequired: boolean;
449 bodySchema: unknown;
450 document: Record<string, unknown>;
451 ajv: Ajv;
452 validator?: ValidateFunction;
453 };
454
455 export type ApiSpec = {
456 name: string;
457 paths: { template: string; parts: string[]; methods: Map<string, ApiOperation> }[];
458 };
459
460 export type ApiSpecs = { v3_1: ApiSpec; v3: ApiSpec };
461
462 const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
463
464 const securityRequirements = z.array(z.record(z.string(), z.array(z.string())));
465
466 const jsonObject = z.record(z.string(), z.unknown());
467 const reference = z.object({ $ref: z.string() });
468 const securityScheme = z.object({ type: z.string(), in: z.string().optional(), name: z.string().optional(), scheme: z.string().optional() });
469 const openApiDocument = z.object({
470 paths: z.record(z.string(), z.record(z.string(), z.unknown())),
471 security: securityRequirements.optional(),
472 components: z.object({
473 securitySchemes: z.record(z.string(), z.unknown()).optional(),
474 }).optional(),
475 });
476 const requestBody = z.object({
477 required: z.boolean().optional(),
478 content: z.record(z.string(), z.object({ schema: z.unknown().optional() })).optional(),
479 });
480 const openApiOperation = z.object({
481 security: securityRequirements.optional(),
482 requestBody: z.unknown().optional(),
483 });
484
485 /**
486 * OpenAPI 3.0 uses draft-04-style boolean exclusive bounds. Ajv's default
487 * dialect uses numeric exclusive bounds. Translate only Schema Objects, not
488 * examples/defaults/enum values or properties that happen to have those names.
489 */
490 function normalizeSchema(value: unknown): unknown {
491 if (Array.isArray(value)) return value.map(normalizeSchema);
492 const object = jsonObject.safeParse(value);
493 if (!object.success) return value;
494 const schema = { ...object.data };
495 for (const [inclusive, exclusive] of [['minimum', 'exclusiveMinimum'], ['maximum', 'exclusiveMaximum']] as const) {
496 const flag = z.boolean().safeParse(schema[exclusive]);
497 if (!flag.success) continue;
498 if (flag.data) {
499 const bound = z.number().safeParse(schema[inclusive]);
500 if (!bound.success) throw new Error(`OpenAPI ${exclusive}: true requires a numeric ${inclusive}`);
501 schema[exclusive] = bound.data;
502 delete schema[inclusive];
503 } else delete schema[exclusive];
504 }
505 for (const key of ['properties', 'patternProperties', 'definitions', '$defs', 'dependentSchemas']) {
506 if (schema[key] === undefined) continue;
507 schema[key] = Object.fromEntries(Object.entries(jsonObject.parse(schema[key])).map(([name, child]) => [name, normalizeSchema(child)]));
508 }
509 for (const key of ['items', 'additionalItems', 'additionalProperties', 'allOf', 'anyOf', 'oneOf', 'not', 'contains', 'if', 'then', 'else', 'propertyNames', 'prefixItems', 'unevaluatedProperties', 'unevaluatedItems']) {
510 if (schema[key] !== undefined) schema[key] = normalizeSchema(schema[key]);
511 }
512 return schema;
513 }
514
515 /** Keep document-relative references intact while normalizing each schema once. */
516 function normalizeOpenApi(value: unknown): unknown {
517 if (Array.isArray(value)) return value.map(normalizeOpenApi);
518 const object = jsonObject.safeParse(value);
519 if (!object.success) return value;
520 return Object.fromEntries(Object.entries(object.data).map(([key, child]) => {
521 if (key === 'schema') return [key, normalizeSchema(child)];
522 if (key === 'schemas') return [key, Object.fromEntries(Object.entries(jsonObject.parse(child)).map(([name, schema]) => [name, normalizeSchema(schema)]))];
523 if (key === 'example' || key === 'examples' || key === 'default' || key === 'enum' || key === 'const') return [key, child];
524 return [key, normalizeOpenApi(child)];
525 }));
526 }
527
528 function resolveReference(value: unknown, document: Record<string, unknown>): unknown {
529 const seen = new Set<string>();
530 for (;;) {
531 const parsed = reference.safeParse(value);
532 if (!parsed.success) return value;
533 const ref = parsed.data.$ref;
534 if (!ref.startsWith('#/') || seen.has(ref)) throw new Error(`Unresolvable OpenAPI reference ${ref}`);
535 seen.add(ref);
536 value = document;
537 for (const part of ref.slice(2).split('/')) {
538 value = jsonObject.parse(value)[part.replace(/~1/g, '/').replace(/~0/g, '~')];
539 }
540 }
541 }
542
543 /** Keeps schema references intact so Ajv, not an approximate schema walker, evaluates them. */
544 export function parseApiSpec(name: string, json: unknown): ApiSpec {
545 const raw = jsonObject.parse(normalizeOpenApi(json));
546 const document = openApiDocument.parse(raw);
547 const schemes = document.components?.securitySchemes ?? {};
548 const ajv = new Ajv({ allErrors: true, strict: false, validateFormats: false, allowUnionTypes: true });
549 const paths = Object.entries(document.paths).map(([template, item]) => {
550 const methods = new Map<string, ApiOperation>();
551 for (const method of HTTP_METHODS) {
552 if (item[method] === undefined) continue;
553 const operation = openApiOperation.parse(resolveReference(item[method], raw));
554 const body = operation.requestBody === undefined ? undefined : requestBody.parse(resolveReference(operation.requestBody, raw));
555 const security = operation.security ?? document.security ?? [];
556 methods.set(method.toUpperCase(), {
557 auth: security.map((requirement) => {
558 const headers: string[] = [];
559 const unsupported: string[] = [];
560 for (const scheme of Object.keys(requirement)) {
561 const definition = schemes[scheme] === undefined ? undefined : securityScheme.parse(resolveReference(schemes[scheme], raw));
562 if (definition?.type === 'apiKey' && definition.in === 'header' && definition.name) headers.push(definition.name.toLowerCase());
563 else if (definition?.type === 'http' && definition.scheme === 'bearer') headers.push('authorization');
564 else unsupported.push(scheme);
565 }
566 return { headers, unsupported };
567 }),
568 hasBody: body !== undefined,
569 bodyRequired: body?.required === true,
570 bodySchema: body?.content?.['application/json']?.schema,
571 document: raw,
572 ajv,
573 });
574 }
575 const parts = template.split('/').filter((part, index, all) => index > 0 && (part || index < all.length - 1));
576 return { template, parts, methods };
577 });
578 return { name, paths };
579 }
580
581 /** Loads `public/openapi.json` (v3.1) and `public/openapi-v3.json` (v3) from the docs root. */
582 export function loadApiSpecs(docsRoot: string): ApiSpecs {
583 const load = (file: string) => parseApiSpec(file, JSON.parse(readFileSync(join(docsRoot, 'public', file), 'utf8')));
584 return { v3_1: load('openapi.json'), v3: load('openapi-v3.json') };
585 }
586
587 // ---------------------------------------------------------------------------
588 // Request body validation
589 // ---------------------------------------------------------------------------
590
591 /** Stands in for shell expansions in JSON bodies; a string equal to it matches any schema. */
592 const SHELL = '\u0000';
593 const SHELL_VALUE = `${SHELL}shell-value`;
594
595 const projectionNode = z.object({
596 properties: z.record(z.string(), z.unknown()).optional(),
597 additionalProperties: z.unknown().optional(),
598 items: z.unknown().optional(),
599 oneOf: z.array(z.unknown()).optional(),
600 anyOf: z.array(z.unknown()).optional(),
601 allOf: z.array(z.unknown()).optional(),
602 }).passthrough();
603
604 function hasUnknown(value: unknown): boolean {
605 if (typeof value === 'string') return value.includes(SHELL);
606 if (Array.isArray(value)) return value.some(hasUnknown);
607 const object = jsonObject.safeParse(value);
608 return object.success && Object.values(object.data).some(hasUnknown);
609 }
610
611 function compileBody(operation: ApiOperation, schema: unknown): ValidateFunction {
612 // Preserve original document-relative and recursive JSON Pointers.
613 return operation.ajv.compile({ ...operation.document, schema, $ref: '#/schema' });
614 }
615
616 /**
617 * Project only unknown instance positions out of a schema. Literal requests use
618 * the original schema verbatim. Ajv still evaluates every remaining constraint.
619 * A partially known oneOf cannot establish branch exclusivity when a passing
620 * branch depends on an unknown value.
621 */
622 function projectSchema(schema: unknown, value: unknown, operation: ApiOperation): { schema: unknown; uncertain: boolean } {
623 if (!hasUnknown(value)) return { schema, uncertain: false };
624 schema = resolveReference(schema, operation.document);
625 if (typeof schema === 'boolean') return { schema, uncertain: false };
626 const node = projectionNode.parse(schema);
627 if (value === SHELL_VALUE) return { schema: true, uncertain: Object.keys(node).length > 0 };
628 const projected: Record<string, unknown> = { ...node };
629 let uncertain = false;
630 for (const keyword of typeof value === 'string' ? ['enum', 'const', 'pattern', 'minLength', 'maxLength', 'format'] : ['enum', 'const']) {
631 if (projected[keyword] !== undefined) {
632 delete projected[keyword];
633 uncertain = true;
634 }
635 }
636 if (node.not !== undefined) {
637 const result = projectSchema(node.not, value, operation);
638 if (result.uncertain) {
639 delete projected.not;
640 uncertain = true;
641 }
642 }
643 const object = jsonObject.safeParse(value);
644 if (object.success) {
645 const properties = { ...node.properties };
646 for (const [key, item] of Object.entries(object.data)) {
647 const itemSchema = properties[key] ?? node.additionalProperties;
648 if (itemSchema === undefined || itemSchema === false) continue;
649 const result = projectSchema(itemSchema, item, operation);
650 if (result.uncertain) {
651 properties[key] = result.schema;
652 uncertain = true;
653 }
654 }
655 if (node.properties || Object.keys(properties).length) projected.properties = properties;
656 }
657 if (Array.isArray(value) && node.items !== undefined) {
658 projected.items = value.map((item, index) => {
659 const itemSchema = Array.isArray(node.items) ? node.items[index] ?? true : node.items;
660 const result = projectSchema(itemSchema, item, operation);
661 uncertain ||= result.uncertain;
662 return result.schema;
663 });
664 }
665 for (const keyword of ['allOf', 'anyOf', 'oneOf'] as const) {
666 const branches = node[keyword];
667 if (!branches) continue;
668 const results = branches.map((branch) => projectSchema(branch, value, operation));
669 projected[keyword] = results.map((result) => result.schema);
670 uncertain ||= results.some((result) => result.uncertain);
671 if (keyword === 'oneOf') {
672 const passing = results.filter((result) => compileBody(operation, result.schema)(value));
673 if (passing.some((result) => result.uncertain)) {
674 delete projected.oneOf;
675 projected.allOf = [...(z.array(z.unknown()).optional().parse(projected.allOf) ?? []), { anyOf: results.map((result) => result.schema) }];
676 }
677 }
678 }
679 return { schema: projected, uncertain };
680 }
681
682 function bodyErrors(operation: ApiOperation, value: unknown): ErrorObject[] {
683 const validate = hasUnknown(value)
684 ? compileBody(operation, projectSchema(operation.bodySchema, value, operation).schema)
685 : operation.validator ??= compileBody(operation, operation.bodySchema);
686 validate(value);
687 return validate.errors ?? [];
688 }
689
690 function errorField(error: ErrorObject): string {
691 const segments = error.instancePath.split('/').slice(1).map((part) => part.replace(/~1/g, '/').replace(/~0/g, '~'));
692 if (error.keyword === 'required') segments.push(z.object({ missingProperty: z.string() }).parse(error.params).missingProperty);
693 if (error.keyword === 'additionalProperties') segments.push(z.object({ additionalProperty: z.string() }).parse(error.params).additionalProperty);
694 return segments.join('.');
695 }
696
697 /**
698 * Renders a data argument as JSON text: shell expansions inside JSON strings
699 * become placeholder characters, bare expansions become a placeholder string.
700 */
701 function bodyJson(word: ShellWord): string {
702 let json = '';
703 let inString = false;
704 let escaped = false;
705 for (const segment of word.segments) {
706 if (segment.expansion) {
707 json += inString ? '\\u0000' : JSON.stringify(SHELL_VALUE);
708 continue;
709 }
710 for (const c of segment.text) {
711 if (escaped) escaped = false;
712 else if (c === '\\') escaped = inString;
713 else if (c === '"') inString = !inString;
714 }
715 json += segment.text;
716 }
717 return json;
718 }
719
720 // ---------------------------------------------------------------------------
721 // Checks
722 // ---------------------------------------------------------------------------
723
724 const SHELL_LANGS: Record<string, true> = { bash: true, sh: true, shell: true, curl: true };
725 const API_HOST_RE = /^(?:[a-z][a-z0-9+.-]*:\/\/)?backend\.composio\.dev(?::\d+)?(?=[/?#]|$)/i;
726 const BASE_URL_VARS: Record<string, true> = {
727 $COMPOSIO_BASE_URL: true, '${COMPOSIO_BASE_URL}': true,
728 $THERMOS_BASE_URL: true, '${THERMOS_BASE_URL}': true,
729 };
730 const CANONICAL_ORIGIN = 'https://backend.composio.dev';
731 const CANONICAL_BASE = `${CANONICAL_ORIGIN}/api/v3.1`;
732 const API_KEY_HEADER = 'x-api-key';
733 const API_KEY_VALUE_RE = /^\$(?:COMPOSIO_[A-Z0-9_]*API_KEY|\{COMPOSIO_[A-Z0-9_]*API_KEY\})$/;
734 const ENV_VAR_RE = /^\$(?:[A-Z][A-Z0-9_]*|\{[A-Z][A-Z0-9_]*\})$/;
735
736 type PathPart = { text: string; dynamic: boolean };
737 type ApiUrl = { origin: string; path: string; parts: PathPart[] };
738 type Report = (rule: ApiSnippetRule, message: string, field?: string) => void;
739
740 /** Splits an API URL into origin and path parts; undefined when the URL is not a Composio API URL. */
741 function apiUrl(word: ShellWord): ApiUrl | undefined {
742 let origin: string;
743 let rest: ShellSegment[];
744 const [first, ...others] = word.segments;
745 if (first?.expansion && BASE_URL_VARS[first.text]) {
746 origin = first.text;
747 rest = others;
748 } else {
749 const match = first && !first.expansion ? API_HOST_RE.exec(first.text) : null;
750 if (!match) return undefined;
751 origin = match[0];
752 rest = [{ text: first.text.slice(match[0].length), expansion: false }, ...others];
753 }
754
755 // Path text with expansions marked, cut at the query string or fragment.
756 let path = '';
757 const dynamic: boolean[] = [];
758 outer: for (const segment of rest) {
759 for (const c of segment.text) {
760 if (!segment.expansion && (c === '?' || c === '#')) break outer;
761 path += c;
762 dynamic.push(segment.expansion);
763 }
764 }
765 const parts: PathPart[] = [];
766 let offset = 0;
767 for (const text of path.split('/')) {
768 parts.push({
769 text,
770 dynamic: dynamic.slice(offset, offset + text.length).some(Boolean) || /^(?:<.+>|\{.+\})$/.test(text),
771 });
772 offset += text.length + 1;
773 }
774 return { origin, path, parts: parts.filter((part, index) => index > 0 && (part.text || index < parts.length - 1)) };
775 }
776
777 function findOperation(spec: ApiSpec, parts: PathPart[], method: string) {
778 const matches = spec.paths.flatMap((entry) => {
779 if (entry.parts.length !== parts.length) return [];
780 let literal = 0;
781 for (const [index, templatePart] of entry.parts.entries()) {
782 const part = parts[index];
783 if (/^\{[^}]+\}$/.test(templatePart)) {
784 if (!part.text) return [];
785 } else if (part.dynamic || part.text !== templatePart) return [];
786 else literal++;
787 }
788 return [{ entry, literal }];
789 });
790 matches.sort((a, b) => b.literal - a.literal);
791 const match = matches[0];
792 if (match) return {
793 template: match.entry.template,
794 operation: match.entry.methods.get(method),
795 allowed: [...match.entry.methods.keys()],
796 };
797 return { allowed: [] };
798 }
799
800 function checkAuth(curl: CurlCommand, operation: Pick<ApiOperation, 'auth'>, report: Report) {
801 if (!operation.auth.length) return;
802 const sent = new Map(curl.headers.map((header) => [header.name, header.value]));
803 const results = operation.auth.map(({ headers, unsupported }) => {
804 const errors: { message: string; field?: string }[] = unsupported.map((name) => ({ message: `cannot validate security scheme "${name}"; it is not public access` }));
805 for (const name of headers) {
806 let value = sent.get(name);
807 if (!value) {
808 errors.push({ message: `missing auth header: send ${name}`, field: name });
809 continue;
810 }
811 if (name === 'authorization') {
812 if (!wordText(value).startsWith('Bearer ')) {
813 errors.push({ message: 'authorization header must use Bearer with an environment variable', field: name });
814 continue;
815 }
816 value = trimWord(dropChars(value, 7));
817 }
818 const [segment, ...others] = value.segments;
819 const single = segment?.expansion && others.length === 0 ? segment.text : undefined;
820 const valid = single && (name === API_KEY_HEADER ? API_KEY_VALUE_RE : ENV_VAR_RE).test(single);
821 if (!valid) errors.push({
822 message: `${name} header must be an expanded environment variable${name === API_KEY_HEADER ? ' named COMPOSIO_API_KEY or COMPOSIO_<NAME>_API_KEY' : ''} (found ${JSON.stringify(wordText(value))})`,
823 field: name,
824 });
825 }
826 return errors;
827 });
828 if (results.some((errors) => errors.length === 0)) return;
829 const closest = results.reduce((best, errors) => errors.length < best.length ? errors : best);
830 for (const error of closest) report('api-key-header', error.message, error.field);
831 }
832
833 function hasDynamicKey(value: unknown): boolean {
834 if (Array.isArray(value)) return value.some(hasDynamicKey);
835 const object = jsonObject.safeParse(value);
836 return object.success && Object.entries(object.data).some(([key, item]) => key.includes(SHELL) || hasDynamicKey(item));
837 }
838
839 function checkBody(curl: CurlCommand, operation: ApiOperation | undefined, operationLabel: string, report: Report) {
840 const data = curl.get ? [] : curl.data;
841 if (!data.length) {
842 if (operation?.bodyRequired) report('api-body-schema', `${operationLabel} requires a JSON request body`);
843 else if (operation?.bodySchema !== undefined) {
844 try {
845 for (const error of bodyErrors(operation, {})) {
846 if (error.keyword === 'required') report('api-body-schema', `${operationLabel} missing JSON body: ${error.message}`, errorField(error));
847 }
848 } catch (error) {
849 report('api-body-unchecked', `${operationLabel} request schema cannot be checked: ${error instanceof Error ? error.message : String(error)}`);
850 }
851 }
852 return;
853 }
854 if (operation && !operation.hasBody) report('api-body-schema', `${operationLabel} takes no request body`);
855 const external = data.find(({ option, value }) =>
856 option === '--upload-file' || option === '--data-urlencode' || option.startsWith('--form') ||
857 (option !== '--data-raw' && wordText(value).startsWith('@')),
858 );
859 if (external) {
860 report('api-body-unchecked', `${operationLabel} ${external.option} uses form, encoded, or file-backed data whose JSON body cannot be checked`);
861 return;
862 }
863 const json = data.map(({ option, value }, index) => `${index > 0 && option !== '--json' ? '&' : ''}${bodyJson(value)}`).join('');
864 let body: unknown;
865 try {
866 body = JSON.parse(json);
867 } catch (error) {
868 const reason = error instanceof Error ? error.message : String(error);
869 report('api-json-invalid', `${operationLabel} body is not valid JSON: ${reason}`);
870 return;
871 }
872 if (body === SHELL_VALUE || hasDynamicKey(body)) {
873 report('api-body-unchecked', `${operationLabel} fully expanded body or expanded JSON property names cannot be checked`);
874 return;
875 }
876 if (operation?.bodySchema === undefined) return;
877 try {
878 for (const error of bodyErrors(operation, body)) {
879 const field = errorField(error);
880 report('api-body-schema', `${operationLabel} ${field ? `"${field}"` : 'body'} ${error.message} (${error.keyword})`, field || undefined);
881 }
882 } catch (error) {
883 report('api-body-unchecked', `${operationLabel} request schema cannot be checked: ${error instanceof Error ? error.message : String(error)}`);
884 }
885 }
886
887 const UNKNOWN_AUTH: Pick<ApiOperation, 'auth'> = {
888 auth: ['x-api-key', 'x-user-api-key', 'x-org-api-key'].map((name) => ({ headers: [name], unsupported: [] })),
889 };
890
891 function checkCurl(curl: CurlCommand, url: ApiUrl, filePath: string, specs: ApiSpecs,
892 emit: (rule: ApiSnippetRule, message: string, operation: string, field?: string) => void) {
893 const version = /^\/api\/(v[^/]+)(?:\/|$)/.exec(url.path)?.[1];
894 const spec = version === 'v3.1' ? specs.v3_1 : version === 'v3' ? specs.v3 : undefined;
895 const found = spec ? findOperation(spec, url.parts, curl.method) : undefined;
896 const operationLabel = `${curl.method} ${found?.template ?? url.path}`;
897 const report: Report = (rule, message, field) => emit(rule, message, operationLabel, field);
898 for (const issue of curl.issues) report('api-curl-unsupported', issue);
899 if (url.origin !== CANONICAL_ORIGIN) report('api-base-url', `origin must be ${CANONICAL_ORIGIN} (found ${JSON.stringify(url.origin)})`);
900 const v3Reference = filePath.startsWith('content/reference/v3/');
901 if (version !== 'v3.1' && !(version === 'v3' && v3Reference)) {
902 report('api-legacy-version', `${operationLabel} uses ${version ? `/api/${version}` : 'a missing API version'}; use ${CANONICAL_BASE}`);
903 }
904 if (spec && !found?.operation) {
905 const hint = found?.allowed?.length ? `; the path allows ${found.allowed.join(', ')}` : '';
906 report('api-unknown-operation', `${curl.method} ${url.path} is not an operation in ${spec.name}${hint}`);
907 }
908 checkAuth(curl, found?.operation ?? UNKNOWN_AUTH, report);
909 checkBody(curl, found?.operation, operationLabel, report);
910 }
911
912 const EXEMPTIBLE_RULES: Record<string, true> = {
913 'api-legacy-version': true, 'api-base-url': true, 'api-unknown-operation': true,
914 'api-key-header': true, 'api-json-invalid': true, 'api-body-schema': true,
915 'api-body-unchecked': true, 'api-curl-unsupported': true,
916 };
917
918 /** Every exception consumes only a matching diagnostic, never a fence or transfer. */
919 export function checkApiSnippets(files: readonly CodeExampleFile[], specs: ApiSpecs): ApiSnippetViolation[] {
920 const violations: ApiSnippetViolation[] = [];
921 for (const file of files) {
922 walkCodeExamples(parseCodeExamples(file.source, file.path).units, (unit) => {
923 if (unit.type !== 'fence') return;
924 const markers = unit.markers.filter((marker) => marker.kind === 'unchecked-api');
925 const used = new Set<MarkerUnit>();
926 const valid = markers.filter((marker) => {
927 if (marker.wellFormed && marker.reason?.trim() && marker.api && EXEMPTIBLE_RULES[marker.api.rule]) return true;
928 violations.push({ path: file.path, line: marker.line, rule: 'api-exemption', message: 'unchecked-api requires a known rule, exact METHOD /operation, and a reason' });
929 return false;
930 });
931 if (SHELL_LANGS[unit.lang]) for (const curl of parseCurlCommands(unit.code, unit.line + 1)) {
932 for (const word of curl.urls) {
933 const url = apiUrl(word);
934 if (!url) continue;
935 checkCurl(curl, url, file.path, specs, (rule, message, operation, field) => {
936 const marker = valid.find((candidate) =>
937 candidate.api?.rule === rule && candidate.api.operation === operation &&
938 (candidate.api.field === undefined || candidate.api.field === field),
939 );
940 if (marker) used.add(marker);
941 else violations.push({ path: file.path, line: curl.line, rule, message });
942 });
943 }
944 }
945 for (const marker of valid) if (!used.has(marker)) violations.push({
946 path: file.path, line: marker.line, rule: 'api-exemption',
947 message: `unchecked-api exception matched no diagnostic: ${marker.api?.rule} ${marker.api?.operation}${marker.api?.field ? ` field="${marker.api.field}"` : ''}`,
948 });
949 });
950 }
951 return violations.sort(
952 (a, b) => a.path.localeCompare(b.path) || a.line - b.line || a.rule.localeCompare(b.rule) || a.message.localeCompare(b.message),
953 );
954 }