1
+/**
2
+ * Toolkit-version policy for direct-execution samples.
3
+ *
4
+ * Direct tool execution requires a toolkit version — `tools.execute()`
5
+ * without one raises ToolVersionRequiredError at runtime. A 97-run agent
6
+ * eval of the docs found the top failure (72/97 runs) was readers copying
7
+ * version-less `tools.execute()` samples.
8
+ *
9
+ * Rule: any authored MDX page whose code fences call `tools.execute(` must
10
+ * also show version configuration somewhere in its code fences — one of
11
+ * `toolkit_versions` / `toolkitVersions` (constructor), `version=` /
12
+ * `version:` (per-call), or a `COMPOSIO_TOOLKIT_VERSION_*` env var.
13
+ *
14
+ * Scope: content/docs and content/examples. Excluded: content/reference
15
+ * (generated upstream), changelog (historical records), and
16
+ * docs/migration-guide (point-in-time documents that may show old APIs).
17
+ * The LLM guardrail blocks appended to .md responses are checked too —
18
+ * they are samples agents copy verbatim.
19
+ */
20
+import { describe, test, expect } from "bun:test";
21
+import { readdir, readFile } from "fs/promises";
22
+import { join, relative } from "path";
23
+
24
+import {
25
+ SESSION_GUARDRAILS,
26
+ DIRECT_EXECUTION_GUARDRAILS,
27
+} from "../../lib/llm-guardrails";
28
+
29
+const CONTENT_DIRS = ["docs", "examples"].map((dir) =>
30
+ join(import.meta.dir, "../../content", dir),
31
+);
32
+const CONTENT_ROOT = join(import.meta.dir, "../../content");
33
+const EXCLUDED_PATH_SEGMENTS = ["docs/migration-guide/"];
34
+
35
+const EXECUTE_CALL_RE = /\btools\.execute\s*\(/;
36
+const VERSION_TOKEN_RE =
37
+ /toolkit_versions|toolkitVersions|version\s*[=:]|COMPOSIO_TOOLKIT_VERSION_|dangerously_skip_version_check|dangerouslySkipVersionCheck/;
38
+
39
+async function findMdxFiles(dir: string): Promise<string[]> {
40
+ const results: string[] = [];
41
+ let entries;
42
+ try {
43
+ entries = await readdir(dir, { withFileTypes: true });
44
+ } catch {
45
+ return results;
46
+ }
47
+
48
+ for (const entry of entries) {
49
+ const fullPath = join(dir, entry.name);
50
+ if (entry.isDirectory()) {
51
+ results.push(...(await findMdxFiles(fullPath)));
52
+ } else if (entry.name.endsWith(".mdx")) {
53
+ results.push(fullPath);
54
+ }
55
+ }
56
+ return results;
57
+}
58
+
59
+/** Concatenated contents of all fenced code blocks in a document. */
60
+function fencedCode(content: string): string {
61
+ const fences: string[] = [];
62
+ let inFence = false;
63
+ for (const line of content.split("\n")) {
64
+ if (/^\s*(```|~~~)/.test(line)) {
65
+ inFence = !inFence;
66
+ continue;
67
+ }
68
+ if (inFence) fences.push(line);
69
+ }
70
+ return fences.join("\n");
71
+}
72
+
73
+function violates(content: string): boolean {
74
+ const code = fencedCode(content);
75
+ return EXECUTE_CALL_RE.test(code) && !VERSION_TOKEN_RE.test(code);
76
+}
77
+
78
+describe("direct-execution samples show toolkit versions", () => {
79
+ test("every authored page with a tools.execute() sample shows version configuration", async () => {
80
+ const files = (
81
+ await Promise.all(CONTENT_DIRS.map((dir) => findMdxFiles(dir)))
82
+ ).flat();
83
+ const failures: string[] = [];
84
+
85
+ for (const file of files) {
86
+ const relPath = relative(CONTENT_ROOT, file);
87
+ if (EXCLUDED_PATH_SEGMENTS.some((seg) => relPath.startsWith(seg))) {
88
+ continue;
89
+ }
90
+ const content = await readFile(file, "utf-8");
91
+ if (violates(content)) {
92
+ failures.push(relPath);
93
+ }
94
+ }
95
+
96
+ expect(
97
+ failures,
98
+ `Pages with tools.execute() samples but no version configuration ` +
99
+ `(add toolkit_versions/toolkitVersions to the constructor or ` +
100
+ `version= per call — see /docs/tools-direct/toolkit-versioning):\n` +
101
+ failures.map((f) => ` - ${f}`).join("\n"),
102
+ ).toEqual([]);
103
+ });
104
+
105
+ test("LLM guardrail blocks with tools.execute() samples show version configuration", () => {
106
+ for (const [name, guardrails] of [
107
+ ["SESSION_GUARDRAILS", SESSION_GUARDRAILS],
108
+ ["DIRECT_EXECUTION_GUARDRAILS", DIRECT_EXECUTION_GUARDRAILS],
109
+ ] as const) {
110
+ expect(
111
+ violates(guardrails),
112
+ `${name} contains a tools.execute() sample without version configuration`,
113
+ ).toBe(false);
114
+ }
115
+ });
116
+});