@devries-cameron20 / composio-1 / commits / 70f677203
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

docs: enforce toolkit-version policy on execution samples via static test

Adds tests/static/execute-version.test.ts: any authored page whose code fences call tools.execute() must show version configuration, and the LLM guardrail blocks are held to the same rule. Fixes the five pages the audit caught (sessions-vs-direct-execution, custom-auth-params, before/after-execution-modifiers) which shipped version-less samples. Changelog, generated reference, and migration guides are exempt as historical/point-in-time records. This is the durable half of eval finding 1: the content fix landed in #3999; this test keeps future samples from regressing. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Soumya Medapati committed Aug 3, 2026 at 12:17 UTC 70f6772037e0f44dfb73547f2713e8e31da7b51b
5 files changed +131 -6
docs/content/docs/auth-configuration/custom-auth-params.mdx
+3 -2
@@ -21,7 +21,7 @@ Pass `customAuthParams` in the `tools.execute()` call to inject your own token a
21 ```python
22 from composio import Composio
23
24 -composio = Composio()
24 +composio = Composio(toolkit_versions={"googlecalendar": "latest"})
25
26 result = composio.tools.execute(
27 slug="GOOGLECALENDAR_LIST_EVENTS",
@@ -46,6 +46,7 @@ import { Composio } from "@composio/core";
46
47 const composio = new Composio({
48 apiKey: process.env.COMPOSIO_API_KEY,
49 + toolkitVersions: { googlecalendar: "latest" },
50 });
51
52 const result = await composio.tools.execute(
@@ -100,7 +101,7 @@ Before Execute Modifiers are a way to modify the parameters of a tool before it
101 from composio import Composio, before_execute
102 from composio.types import ToolExecuteParams
103
103 -composio = Composio()
104 +composio = Composio(toolkit_versions={"notion": "latest"})
105
106
107 @before_execute(toolkits=["NOTION"])
docs/content/docs/sessions-vs-direct-execution.mdx
+5 -1
@@ -122,7 +122,10 @@ Fetch tools by slug or toolkit. Pass them to your LLM or call `tools.execute()`
122 from composio import Composio
123 from composio_openai import OpenAIProvider
124
125 -composio = Composio(provider=OpenAIProvider())
125 +composio = Composio(
126 + provider=OpenAIProvider(),
127 + toolkit_versions={"github": "latest"}, # required for direct execution
128 +)
129
130 # Fetch specific tools
131 tools = composio.tools.get(
@@ -153,6 +156,7 @@ import { OpenAIProvider } from '@composio/openai';
156 const composio = new Composio({
157 apiKey: process.env.COMPOSIO_API_KEY,
158 provider: new OpenAIProvider(),
159 + toolkitVersions: { github: 'latest' }, // required for direct execution
160 });
161
162 // Fetch specific tools
docs/content/docs/tools-direct/modify-tool-behavior/after-execution-modifiers.mdx
+5 -1
@@ -122,7 +122,10 @@ from composio import Composio, after_execute
122 from composio.types import ToolExecutionResponse
123 from composio_crewai import CrewAIProvider
124
125 -composio = Composio(provider=CrewAIProvider())
125 +composio = Composio(
126 + provider=CrewAIProvider(),
127 + toolkit_versions={"hackernews": "latest"},
128 +)
129
130 @after_execute(tools=["HACKERNEWS_GET_USER"])
131 def after_execute_modifier(
@@ -156,6 +159,7 @@ import { v4 as uuidv4 } from "uuid";
159 const composio = new Composio({
160 apiKey: process.env.COMPOSIO_API_KEY,
161 provider: new VercelProvider(),
162 + toolkitVersions: { hackernews: "latest" },
163 });
164
165 const userId = uuidv4();
docs/content/docs/tools-direct/modify-tool-behavior/before-execution-modifiers.mdx
+2 -2
@@ -34,7 +34,7 @@ from openai import OpenAI
34 from composio import Composio, before_execute
35 from composio.types import ToolExecuteParams
36
37 -composio = Composio()
37 +composio = Composio(toolkit_versions={"hackernews": "latest"})
38 openai_client = OpenAI()
39 user_id = "user@email.com"
40
@@ -201,7 +201,7 @@ def audit_path(ctx) -> str | bool:
201 # return a new path string, or False to abort the upload
202 return ctx["path"]
203
204 -composio = Composio()
204 +composio = Composio(toolkit_versions={"googledrive": "latest"})
205 composio.tools.execute(
206 "GOOGLEDRIVE_UPLOAD_FILE",
207 {"file_to_upload": "/path/to/document.pdf"},
docs/tests/static/execute-version.test.ts new
+116
@@ -0,0 +1,116 @@
1 +/**
2 + * Toolkit-version policy for direct-execution samples.
3 + *
4 + * Direct tool execution requires a toolkit version — `tools.execute()`
5 + * without one raises ToolVersionRequiredError at runtime. A 97-run agent
6 + * eval of the docs found the top failure (72/97 runs) was readers copying
7 + * version-less `tools.execute()` samples.
8 + *
9 + * Rule: any authored MDX page whose code fences call `tools.execute(` must
10 + * also show version configuration somewhere in its code fences — one of
11 + * `toolkit_versions` / `toolkitVersions` (constructor), `version=` /
12 + * `version:` (per-call), or a `COMPOSIO_TOOLKIT_VERSION_*` env var.
13 + *
14 + * Scope: content/docs and content/examples. Excluded: content/reference
15 + * (generated upstream), changelog (historical records), and
16 + * docs/migration-guide (point-in-time documents that may show old APIs).
17 + * The LLM guardrail blocks appended to .md responses are checked too —
18 + * they are samples agents copy verbatim.
19 + */
20 +import { describe, test, expect } from "bun:test";
21 +import { readdir, readFile } from "fs/promises";
22 +import { join, relative } from "path";
23 +
24 +import {
25 + SESSION_GUARDRAILS,
26 + DIRECT_EXECUTION_GUARDRAILS,
27 +} from "../../lib/llm-guardrails";
28 +
29 +const CONTENT_DIRS = ["docs", "examples"].map((dir) =>
30 + join(import.meta.dir, "../../content", dir),
31 +);
32 +const CONTENT_ROOT = join(import.meta.dir, "../../content");
33 +const EXCLUDED_PATH_SEGMENTS = ["docs/migration-guide/"];
34 +
35 +const EXECUTE_CALL_RE = /\btools\.execute\s*\(/;
36 +const VERSION_TOKEN_RE =
37 + /toolkit_versions|toolkitVersions|version\s*[=:]|COMPOSIO_TOOLKIT_VERSION_|dangerously_skip_version_check|dangerouslySkipVersionCheck/;
38 +
39 +async function findMdxFiles(dir: string): Promise<string[]> {
40 + const results: string[] = [];
41 + let entries;
42 + try {
43 + entries = await readdir(dir, { withFileTypes: true });
44 + } catch {
45 + return results;
46 + }
47 +
48 + for (const entry of entries) {
49 + const fullPath = join(dir, entry.name);
50 + if (entry.isDirectory()) {
51 + results.push(...(await findMdxFiles(fullPath)));
52 + } else if (entry.name.endsWith(".mdx")) {
53 + results.push(fullPath);
54 + }
55 + }
56 + return results;
57 +}
58 +
59 +/** Concatenated contents of all fenced code blocks in a document. */
60 +function fencedCode(content: string): string {
61 + const fences: string[] = [];
62 + let inFence = false;
63 + for (const line of content.split("\n")) {
64 + if (/^\s*(```|~~~)/.test(line)) {
65 + inFence = !inFence;
66 + continue;
67 + }
68 + if (inFence) fences.push(line);
69 + }
70 + return fences.join("\n");
71 +}
72 +
73 +function violates(content: string): boolean {
74 + const code = fencedCode(content);
75 + return EXECUTE_CALL_RE.test(code) && !VERSION_TOKEN_RE.test(code);
76 +}
77 +
78 +describe("direct-execution samples show toolkit versions", () => {
79 + test("every authored page with a tools.execute() sample shows version configuration", async () => {
80 + const files = (
81 + await Promise.all(CONTENT_DIRS.map((dir) => findMdxFiles(dir)))
82 + ).flat();
83 + const failures: string[] = [];
84 +
85 + for (const file of files) {
86 + const relPath = relative(CONTENT_ROOT, file);
87 + if (EXCLUDED_PATH_SEGMENTS.some((seg) => relPath.startsWith(seg))) {
88 + continue;
89 + }
90 + const content = await readFile(file, "utf-8");
91 + if (violates(content)) {
92 + failures.push(relPath);
93 + }
94 + }
95 +
96 + expect(
97 + failures,
98 + `Pages with tools.execute() samples but no version configuration ` +
99 + `(add toolkit_versions/toolkitVersions to the constructor or ` +
100 + `version= per call — see /docs/tools-direct/toolkit-versioning):\n` +
101 + failures.map((f) => ` - ${f}`).join("\n"),
102 + ).toEqual([]);
103 + });
104 +
105 + test("LLM guardrail blocks with tools.execute() samples show version configuration", () => {
106 + for (const [name, guardrails] of [
107 + ["SESSION_GUARDRAILS", SESSION_GUARDRAILS],
108 + ["DIRECT_EXECUTION_GUARDRAILS", DIRECT_EXECUTION_GUARDRAILS],
109 + ] as const) {
110 + expect(
111 + violates(guardrails),
112 + `${name} contains a tools.execute() sample without version configuration`,
113 + ).toBe(false);
114 + }
115 + });
116 +});