Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 129 lines 4.66 KB
Raw
1 /**
2 * Toolkit-version policy for direct-execution samples.
3 *
4 * Direct tool execution requires a toolkit version — `tools.execute()`
5 * without one raises ToolVersionRequiredError at runtime. A 97-run agent
6 * eval of the docs found the top failure (72/97 runs) was readers copying
7 * version-less `tools.execute()` samples.
8 *
9 * Rule: any authored MDX page whose code fences call `tools.execute(` must
10 * also show version configuration somewhere in its code fences — one of
11 * `toolkit_versions` / `toolkitVersions` (constructor), `version=` /
12 * `version:` (per-call), or a `COMPOSIO_TOOLKIT_VERSION_*` env var.
13 *
14 * Additionally: `"latest"` as a toolkit_versions value is rejected by the
15 * SDK for manual execution (runtime-verified 2026-08-03: ToolVersionRequiredError,
16 * '"latest" is not supported in manual execution') unless the execute call
17 * passes `dangerously_skip_version_check` / `dangerouslySkipVersionCheck`.
18 * Pages showing "latest" alongside execute samples must also show the flag.
19 *
20 * Scope: content/docs and content/examples. Excluded: content/reference
21 * (generated upstream), changelog (historical records), and
22 * docs/migration-guide (point-in-time documents that may show old APIs).
23 * The LLM guardrail blocks appended to .md responses are checked too —
24 * they are samples agents copy verbatim.
25 */
26 import { describe, test, expect } from "bun:test";
27 import { readdir, readFile } from "fs/promises";
28 import { join, relative } from "path";
29
30 import {
31 SESSION_GUARDRAILS,
32 DIRECT_EXECUTION_GUARDRAILS,
33 } from "../../lib/llm-guardrails";
34
35 const CONTENT_DIRS = ["docs", "examples"].map((dir) =>
36 join(import.meta.dir, "../../content", dir),
37 );
38 const CONTENT_ROOT = join(import.meta.dir, "../../content");
39 const EXCLUDED_PATH_SEGMENTS = ["docs/migration-guide/"];
40
41 const EXECUTE_CALL_RE = /\btools\.execute\s*\(/;
42 const VERSION_TOKEN_RE =
43 /toolkit_versions|toolkitVersions|version\s*[=:]|COMPOSIO_TOOLKIT_VERSION_|dangerously_skip_version_check|dangerouslySkipVersionCheck/;
44 const LATEST_VALUE_RE = /toolkit_?[vV]ersions[^}\n]{0,120}["']latest["']/;
45 const SKIP_FLAG_RE = /dangerously_skip_version_check|dangerouslySkipVersionCheck/;
46
47 async function findMdxFiles(dir: string): Promise<string[]> {
48 const results: string[] = [];
49 let entries;
50 try {
51 entries = await readdir(dir, { withFileTypes: true });
52 } catch {
53 return results;
54 }
55
56 for (const entry of entries) {
57 const fullPath = join(dir, entry.name);
58 if (entry.isDirectory()) {
59 results.push(...(await findMdxFiles(fullPath)));
60 } else if (entry.name.endsWith(".mdx")) {
61 results.push(fullPath);
62 }
63 }
64 return results;
65 }
66
67 /** Concatenated contents of all fenced code blocks in a document. */
68 function fencedCode(content: string): string {
69 const fences: string[] = [];
70 let inFence = false;
71 for (const line of content.split("\n")) {
72 if (/^\s*(```|~~~)/.test(line)) {
73 inFence = !inFence;
74 continue;
75 }
76 if (inFence) fences.push(line);
77 }
78 return fences.join("\n");
79 }
80
81 function violates(content: string): string | null {
82 const code = fencedCode(content);
83 if (!EXECUTE_CALL_RE.test(code)) return null;
84 if (!VERSION_TOKEN_RE.test(code)) return "no version configuration";
85 if (LATEST_VALUE_RE.test(code) && !SKIP_FLAG_RE.test(code)) {
86 return '"latest" without dangerously_skip_version_check (rejected at runtime for manual execution)';
87 }
88 return null;
89 }
90
91 describe("direct-execution samples show toolkit versions", () => {
92 test("every authored page with a tools.execute() sample shows version configuration", async () => {
93 const files = (
94 await Promise.all(CONTENT_DIRS.map((dir) => findMdxFiles(dir)))
95 ).flat();
96 const failures: string[] = [];
97
98 for (const file of files) {
99 const relPath = relative(CONTENT_ROOT, file);
100 if (EXCLUDED_PATH_SEGMENTS.some((seg) => relPath.startsWith(seg))) {
101 continue;
102 }
103 const content = await readFile(file, "utf-8");
104 const problem = violates(content);
105 if (problem) {
106 failures.push(`${relPath} — ${problem}`);
107 }
108 }
109
110 expect(
111 failures,
112 `Pages with broken tools.execute() version handling ` +
113 `(see /docs/tools-direct/toolkit-versioning):\n` +
114 failures.map((f) => ` - ${f}`).join("\n"),
115 ).toEqual([]);
116 });
117
118 test("LLM guardrail blocks with tools.execute() samples show version configuration", () => {
119 for (const [name, guardrails] of [
120 ["SESSION_GUARDRAILS", SESSION_GUARDRAILS],
121 ["DIRECT_EXECUTION_GUARDRAILS", DIRECT_EXECUTION_GUARDRAILS],
122 ] as const) {
123 expect(
124 violates(guardrails),
125 `${name} has broken tools.execute() version handling: ${violates(guardrails)}`,
126 ).toBeNull();
127 }
128 });
129 });