Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 266 lines 8.42 KB
Raw
1 /**
2 * Session tool-execution policy for docs samples.
3 *
4 * Provider helpers must receive the Tool Router session that produced the
5 * model-visible tools. Passing a user ID chooses direct execution instead,
6 * so session meta-tools fail at runtime.
7 *
8 * A 102-run agent eval of the docs (August 2026, two-phase build+probe)
9 * found 58/102 runs hit that rejection by copying provider-page samples
10 * that paired `session.tools()` with `handle_tool_calls`. All recovering
11 * runs converged on `session.execute()`, discovered from SDK source. The SDK
12 * helpers now accept the session directly and retain provider normalization.
13 *
14 * Rule: no authored MDX page may pair session tools (`session.tools()` /
15 * `sessions.create` / `composio.create(`) with a provider helper explicitly
16 * bound to a user ID. Session-bound helper calls and direct-path (`tools.get`)
17 * samples are valid.
18 *
19 * Scope: content/docs and content/examples. Excluded: content/reference
20 * (generated upstream), changelog (historical), docs/migration-guide
21 * (point-in-time documents that may show old APIs).
22 * The LLM guardrail blocks appended to .md responses are checked too —
23 * they are samples agents copy verbatim.
24 */
25 import { describe, test, expect } from "bun:test";
26 import { readdir, readFile } from "fs/promises";
27 import { join, relative } from "path";
28
29 import {
30 SESSION_GUARDRAILS,
31 DIRECT_EXECUTION_GUARDRAILS,
32 } from "../../lib/llm-guardrails";
33
34 const CONTENT_DIRS = ["docs", "examples"].map((dir) =>
35 join(import.meta.dir, "../../content", dir),
36 );
37 const CONTENT_ROOT = join(import.meta.dir, "../../content");
38 const EXCLUDED_PATH_SEGMENTS = ["docs/migration-guide/"];
39
40 const SESSION_TOKEN_RE =
41 /session\.tools\s*\(|sessions\.create\s*\(|composio\.create\s*\(/;
42 // Python branch: user_id= must appear inside the helper's argument list —
43 // the bound tolerates one level of nested calls (`response=build_response()`)
44 // but never runs past the helper's closing paren into later code. TS branch:
45 // a string-literal first argument, or a whole identifier that names a user ID
46 // (`userId`, `user_id`, `uid`) — a session argument never does.
47 const DIRECT_HELPER_TOKEN_RE =
48 /(?:handle_tool_calls|handle_response|execute_tool_call)\s*\((?:[^()]*\([^()]*\))*[^()]*\buser_id\s*=|(?:handleToolCalls|handleResponse|executeToolCall)\s*\(\s*(?:["'`]|(?:user_?[iI]d|uid)\b)/;
49 const SAMPLE_BOUNDARY_RE = /^\s*(?:<\/?(?:Tab|Step)\b|#{1,6}\s)/;
50 const FENCE_OPEN_RE = /^\s*(`{3,}|~{3,})/;
51 const FENCE_CLOSE_RE = /^\s*(`{3,}|~{3,})\s*$/;
52
53 async function findMdxFiles(dir: string): Promise<string[]> {
54 const results: string[] = [];
55 let entries;
56 try {
57 entries = await readdir(dir, { withFileTypes: true });
58 } catch {
59 return results;
60 }
61
62 for (const entry of entries) {
63 const fullPath = join(dir, entry.name);
64 if (entry.isDirectory()) {
65 results.push(...(await findMdxFiles(fullPath)));
66 } else if (entry.name.endsWith(".mdx")) {
67 results.push(fullPath);
68 }
69 }
70 return results;
71 }
72
73 function codeSampleGroups(source: string): string[] {
74 const groups: string[] = [];
75 let current: string[] = [];
76 let fenceMarker: string | undefined;
77
78 const flush = () => {
79 if (current.length > 0) groups.push(current.join("\n"));
80 current = [];
81 };
82
83 for (const line of source.split(/\r?\n/)) {
84 if (fenceMarker === undefined && SAMPLE_BOUNDARY_RE.test(line)) flush();
85
86 if (fenceMarker === undefined) {
87 const opening = line.match(FENCE_OPEN_RE)?.[1];
88 if (opening !== undefined) {
89 fenceMarker = opening;
90 current.push(line);
91 }
92 continue;
93 }
94
95 current.push(line);
96 const closing = line.match(FENCE_CLOSE_RE)?.[1];
97 if (
98 closing !== undefined &&
99 closing[0] === fenceMarker[0] &&
100 closing.length >= fenceMarker.length
101 ) {
102 fenceMarker = undefined;
103 }
104 }
105
106 flush();
107 return groups;
108 }
109
110 function hasDirectHelperBoundToSessionTools(source: string): boolean {
111 return codeSampleGroups(source).some(
112 (sample) => SESSION_TOKEN_RE.test(sample) && DIRECT_HELPER_TOKEN_RE.test(sample),
113 );
114 }
115
116 describe("session execution samples", () => {
117 test("detects a direct helper split from session setup within one sample", () => {
118 const source = `
119 <Tab value="Python">
120 ~~~python
121 session = composio.create(user_id="user_123")
122 tools = session.tools()
123 ~~~
124
125 Run the model, then execute its calls:
126
127 ~~~python
128 results = composio.provider.handle_tool_calls(
129 response=response,
130 user_id="user_123",
131 )
132 ~~~
133 </Tab>`;
134
135 expect(hasDirectHelperBoundToSessionTools(source)).toBe(true);
136 });
137
138 test("allows session targets and keeps separate tab samples independent", () => {
139 const sessionTarget = `
140 <Tab value="Python">
141 \`\`\`python
142 session = composio.create(user_id="user_123")
143 tools = session.tools()
144 \`\`\`
145 \`\`\`python
146 results = composio.provider.handle_tool_calls(response=response, session=session)
147 \`\`\`
148 </Tab>`;
149 const separateTargets = `
150 <Tab value="Session">
151 \`\`\`typescript
152 const tools = await session.tools();
153 \`\`\`
154 </Tab>
155 <Tab value="Direct">
156 \`\`\`typescript
157 const results = await composio.provider.handleToolCalls("user_123", response);
158 \`\`\`
159 </Tab>`;
160
161 expect(hasDirectHelperBoundToSessionTools(sessionTarget)).toBe(false);
162 expect(hasDirectHelperBoundToSessionTools(separateTargets)).toBe(false);
163 });
164
165 test("session-bound helper is not flagged by a later unrelated user_id", () => {
166 const source = `
167 <Tab value="Python">
168 \`\`\`python
169 session = composio.create(user_id="user_123")
170 tools = session.tools()
171 results = composio.provider.handle_tool_calls(response=response, session=session)
172 \`\`\`
173 \`\`\`python
174 other_session = composio.create(user_id="user_456")
175 \`\`\`
176 </Tab>`;
177
178 expect(hasDirectHelperBoundToSessionTools(source)).toBe(false);
179 });
180
181 test("detects a user_id hidden behind a nested call in the same argument list", () => {
182 const source = `
183 <Tab value="Python">
184 \`\`\`python
185 session = composio.create(user_id="user_123")
186 tools = session.tools()
187 results = composio.provider.handle_tool_calls(response=build_response(), user_id="user_123")
188 \`\`\`
189 </Tab>`;
190
191 expect(hasDirectHelperBoundToSessionTools(source)).toBe(true);
192 });
193
194 test("session variables that merely start with 'user' are not flagged", () => {
195 const source = `
196 <Tab value="TypeScript">
197 \`\`\`typescript
198 const userSession = await composio.create("user_123");
199 const tools = await userSession.tools();
200 const results = await composio.provider.handleToolCalls(userSession, response);
201 \`\`\`
202 </Tab>`;
203
204 expect(hasDirectHelperBoundToSessionTools(source)).toBe(false);
205 });
206
207 test("detects a TypeScript helper bound to a user-ID variable", () => {
208 const source = `
209 <Tab value="TypeScript">
210 \`\`\`typescript
211 const userId = "user_123";
212 const session = await composio.create(userId);
213 const tools = await session.tools();
214 const results = await composio.provider.handleToolCalls(userId, response);
215 \`\`\`
216 </Tab>`;
217
218 expect(hasDirectHelperBoundToSessionTools(source)).toBe(true);
219 });
220
221 test("detects response helpers bound to a user ID", () => {
222 const python = `
223 \`\`\`python
224 session = composio.create(user_id="user_123")
225 tools = session.tools()
226 results = composio.provider.handle_response(user_id="user_123", response=response)
227 \`\`\``;
228 const typescript = `
229 \`\`\`typescript
230 const session = await composio.create("user_123");
231 const tools = await session.tools();
232 const outputs = await composio.provider.handleResponse("user_123", response);
233 \`\`\``;
234
235 expect(hasDirectHelperBoundToSessionTools(python)).toBe(true);
236 expect(hasDirectHelperBoundToSessionTools(typescript)).toBe(true);
237 });
238
239 test("LLM guardrail blocks never bind provider helpers to a user ID", () => {
240 for (const guardrails of [SESSION_GUARDRAILS, DIRECT_EXECUTION_GUARDRAILS]) {
241 expect(hasDirectHelperBoundToSessionTools(guardrails)).toBe(false);
242 }
243 });
244
245 test("session samples do not bind provider helpers to a user ID", async () => {
246 const offenders: string[] = [];
247
248 for (const dir of CONTENT_DIRS) {
249 for (const file of await findMdxFiles(dir)) {
250 const rel = relative(CONTENT_ROOT, file);
251 if (EXCLUDED_PATH_SEGMENTS.some((seg) => rel.includes(seg))) continue;
252
253 const source = await readFile(file, "utf8");
254 if (hasDirectHelperBoundToSessionTools(source)) {
255 offenders.push(rel);
256 }
257 }
258 }
259
260 expect(
261 offenders,
262 `These pages bind session tools to a direct user-ID execution target. ` +
263 `Pass the session to the provider helper instead:\n ${offenders.join("\n ")}`,
264 ).toEqual([]);
265 });
266 });