Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 487 lines 16.6 KB
Raw
1 import { beforeAll, describe, expect, mock, test } from "bun:test";
2
3 import {
4 REST_VERSION_GUIDANCE,
5 TOOL_VERSION_GUIDANCE,
6 } from "../../lib/api-version-guidance";
7 import { SESSION_GUARDRAILS } from "../../lib/llm-guardrails";
8 import { parseCurlCommands, wordText } from "../../lib/api-snippets";
9
10 let bundledSpec: Record<string, unknown> = {};
11
12 mock.module("next/navigation", () => ({
13 notFound: () => {
14 throw new Error("not found");
15 },
16 usePathname: () => "/reference/api-reference/tasks",
17 useSearchParams: () => new URLSearchParams(),
18 }));
19 // Keep catalog loaders and schema helpers real: Bun module mocks persist across
20 // files and can empty the knowledge catalog or replace transformations in later suites.
21
22 let openapiPageToMarkdown: typeof import("../../app/llms.mdx/[[...slug]]/route").openapiPageToMarkdown;
23 let degradedPageToMarkdown: typeof import("../../app/llms.mdx/[[...slug]]/route").degradedPageToMarkdown;
24
25 beforeAll(async () => {
26 ({ openapiPageToMarkdown, degradedPageToMarkdown } = await import(
27 "../../app/llms.mdx/[[...slug]]/route"
28 ));
29 });
30
31 describe("LLM OpenAPI markdown", () => {
32 test("preserves real toolkit schema helpers for subsequent suites", async () => {
33 const { processSchema, toolFromApi } = await import("../../lib/toolkit-schema");
34 expect(processSchema({
35 properties: { query: { type: "string" } },
36 required: ["query"],
37 })?.query.required).toBe(true);
38 expect(toolFromApi({ slug: "TEST_TOOL" }).name).toBe("TEST_TOOL");
39 });
40
41 test("continues to render API operations with endpoint and cURL details", async () => {
42 bundledSpec = {
43 paths: {
44 "/v3.1/test": {
45 get: {
46 summary: "Get test",
47 responses: {
48 200: { description: "Success" },
49 },
50 },
51 },
52 },
53 servers: [{ url: "https://backend.example.com" }],
54 };
55
56 const markdown = await openapiPageToMarkdown({
57 url: "/reference/api-reference/test/getTest",
58 data: {
59 title: "Get test",
60 getOpenAPIPageProps: () => ({
61 payload: { bundled: bundledSpec },
62 operations: [{ path: "/v3.1/test", method: "GET" }],
63 }),
64 },
65 });
66
67 expect(markdown).toContain("**Endpoint:** `https://backend.example.com/v3.1/test`");
68 for (const [label, language] of [["Python", "python"], ["TypeScript", "typescript"], ["API", "bash"]]) {
69 expect(markdown).toContain(`#### ${label}\n\n\`\`\`${language}\n`);
70 }
71 expect(markdown).toContain(
72 'curl -X GET "https://backend.example.com/v3.1/test"',
73 );
74 });
75
76 test("raw request variants use organization authentication instead of a project key", async () => {
77 const markdown = await openapiPageToMarkdown({
78 url: "/reference/api-reference/organization/test",
79 data: {
80 title: "Organization request",
81 getOpenAPIPageProps: () => ({
82 payload: { bundled: {
83 paths: { "/api/v3.1/org/test": { post: {
84 security: [{ OrgApiKeyAuth: [] }],
85 requestBody: { content: { "application/json": { schema: { oneOf: [{
86 type: "object", required: ["title", "enabled", "value"],
87 properties: { title: { type: "string", example: "O'Reilly" }, enabled: { type: "boolean" }, value: { type: "null" } },
88 }] } } } },
89 responses: { "200": { description: "Success" } },
90 } } },
91 components: { securitySchemes: { OrgApiKeyAuth: { type: "apiKey", in: "header", name: "x-org-api-key" } } },
92 } },
93 operations: [{ path: "/api/v3.1/org/test", method: "POST" }],
94 }),
95 },
96 });
97 const snippets = [...markdown.matchAll(/```(python|typescript|bash)\n([\s\S]*?)```/g)];
98 expect(snippets.map((match) => match[1])).toEqual(["python", "typescript", "bash"]);
99 for (const snippet of snippets) {
100 expect(snippet[2]).toContain("x-org-api-key");
101 expect(snippet[2]).toContain("COMPOSIO_ORG_API_KEY");
102 expect(snippet[2]).not.toContain('"x-api-key"');
103 }
104 const curl = snippets.find((snippet) => snippet[1] === "bash")![2];
105 const request = parseCurlCommands(curl)[0];
106 expect(JSON.parse(wordText(request.data[0].value))).toEqual({ title: "O'Reilly", enabled: true, value: null });
107 });
108
109 test("renders webhook delivery headers and payload schema", async () => {
110 bundledSpec = {
111 openapi: "3.1.0",
112 webhooks: {
113 "composio.test.event": {
114 post: {
115 summary: "Test event",
116 parameters: [
117 {
118 name: "webhook-id",
119 in: "header",
120 required: true,
121 description: "Stable delivery identifier.",
122 schema: { type: "string" },
123 },
124 ],
125 requestBody: {
126 content: {
127 "application/json": {
128 schema: {
129 type: "object",
130 required: ["id"],
131 properties: {
132 id: {
133 type: "string",
134 description: "Unique event identifier.",
135 },
136 },
137 },
138 example: { id: "msg_test" },
139 },
140 },
141 },
142 responses: {
143 200: { description: "Delivery accepted" },
144 },
145 },
146 },
147 },
148 };
149
150 const markdown = await openapiPageToMarkdown({
151 url: "/reference/api-reference/webhook-events/composio_test_event",
152 data: {
153 title: "Test event",
154 getOpenAPIPageProps: () => ({
155 payload: { bundled: bundledSpec },
156 webhooks: [{ name: "composio.test.event", method: "post" }],
157 }),
158 },
159 });
160
161 expect(markdown).toContain("## POST `composio.test.event`");
162 expect(markdown).toContain("### Delivery Headers");
163 expect(markdown).toContain(
164 "- `webhook-id` (string) *(required)*: Stable delivery identifier.",
165 );
166 expect(markdown).toContain("### Request Body");
167 expect(markdown).toContain(
168 "- `id` (string) *(required)*: Unique event identifier.",
169 );
170 expect(markdown).toContain('"id": "msg_test"');
171 expect(markdown).not.toContain("### Example cURL Request");
172 });
173
174 test("bounds recursive array type rendering", async () => {
175 bundledSpec = {
176 paths: {
177 "/v3.1/test": {
178 get: {
179 parameters: [
180 {
181 name: "tree",
182 in: "query",
183 schema: { $ref: "#/components/schemas/RecursiveArray" },
184 },
185 ],
186 responses: {
187 200: { description: "Success" },
188 },
189 },
190 },
191 },
192 components: {
193 schemas: {
194 RecursiveArray: {
195 type: "array",
196 items: { $ref: "#/components/schemas/RecursiveArray" },
197 },
198 },
199 },
200 };
201
202 const markdown = await openapiPageToMarkdown({
203 url: "/reference/api-reference/test/getTest",
204 data: {
205 title: "Get test",
206 getOpenAPIPageProps: () => ({
207 payload: { bundled: bundledSpec },
208 operations: [{ path: "/v3.1/test", method: "GET" }],
209 }),
210 },
211 });
212
213 expect(markdown).toContain(
214 "- `tree` (array<array<array<array<array<...>>>>>):",
215 );
216 });
217
218 test("union request samples keep the parent schema and bound circular unions", async () => {
219 const renderBody = async (schema: Record<string, unknown>, schemas: Record<string, unknown> = {}) => {
220 const markdown = await openapiPageToMarkdown({
221 url: "/reference/api-reference/test/postTest",
222 data: {
223 title: "Post test",
224 getOpenAPIPageProps: () => ({
225 payload: { bundled: {
226 paths: { "/v3.1/test": { post: {
227 requestBody: { content: { "application/json": { schema } } },
228 responses: { 200: { description: "Success" } },
229 } } },
230 components: { schemas },
231 } },
232 operations: [{ path: "/v3.1/test", method: "POST" }],
233 }),
234 },
235 });
236 const curl = [...markdown.matchAll(/```bash\n([\s\S]*?)```/g)][0][1];
237 return JSON.parse(wordText(parseCurlCommands(curl)[0].data[0].value));
238 };
239
240 expect(await renderBody({
241 type: "object",
242 properties: { name: { type: "string" }, id: { type: "integer" } },
243 oneOf: [{ required: ["name"] }, { required: ["id"] }],
244 })).toEqual({ name: "string", id: 1 });
245
246 expect(await renderBody(
247 { $ref: "#/components/schemas/Node" },
248 { Node: { oneOf: [{ $ref: "#/components/schemas/Node" }, { type: "string" }] } },
249 )).toBe("...");
250 });
251 });
252
253 /**
254 * Version identity on operation pages.
255 *
256 * These are the pages that publish a complete working curl example, so they
257 * are the strongest signal an agent acts on — and they never touch
258 * mdxToCleanMarkdown, so a fix confined to lib/source.ts leaves them alone.
259 *
260 * Every fixture below uses production-shaped spec path keys — `/api/v3.1/…`
261 * and `/api/v3/…`, with the `/api` segment — because that is what the
262 * committed specs contain. The synthetic `/v3.1/test` keys in the fixtures
263 * above predate this and are deliberately left alone; do not copy their shape
264 * down here, or these tests will pass against an implementation that
265 * mis-normalizes production paths.
266 */
267 describe("LLM OpenAPI markdown — API version identity", () => {
268 function specWith(pathKey: string) {
269 return {
270 paths: {
271 [pathKey]: {
272 get: {
273 summary: "Fixture operation",
274 responses: { 200: { description: "Success" } },
275 },
276 },
277 },
278 servers: [{ url: "https://backend.composio.dev" }],
279 };
280 }
281
282 function renderOperation(pageUrl: string, pathKey: string) {
283 bundledSpec = specWith(pathKey);
284 return openapiPageToMarkdown({
285 url: pageUrl,
286 data: {
287 title: "Fixture operation",
288 getOpenAPIPageProps: () => ({
289 payload: { bundled: bundledSpec },
290 operations: [{ path: pathKey, method: "GET" }],
291 }),
292 },
293 });
294 }
295
296 const V31_PAGE = "/reference/api-reference/auth-configs/getAuthConfigs";
297 const V30_PAGE = "/reference/v3/api-reference/auth-configs/getAuthConfigs";
298
299 test("a v3.1 operation page carries the current-version pointer above the endpoint line", async () => {
300 const markdown = await renderOperation(V31_PAGE, "/api/v3.1/auth_configs");
301
302 expect(markdown).toContain("**API version:**");
303 // Above **Endpoint:**, so a truncating reader still sees it.
304 expect(markdown.indexOf("**API version:**")).toBeLessThan(
305 markdown.indexOf("**Endpoint:**"),
306 );
307 });
308
309 test("a v3.0 operation page carries the legacy pointer and links its v3.1 page", async () => {
310 const markdown = await renderOperation(V30_PAGE, "/api/v3/auth_configs");
311
312 expect(markdown).toContain("**API version:**");
313 expect(markdown).toContain("v3.0");
314 expect(markdown).toContain(
315 "/reference/api-reference/auth-configs/getAuthConfigs.md",
316 );
317 expect(markdown.indexOf("**API version:**")).toBeLessThan(
318 markdown.indexOf("**Endpoint:**"),
319 );
320 });
321
322 test.each([
323 ["v3.1", V31_PAGE, "/api/v3.1/tools/{tool_slug}"],
324 ["v3.0", V30_PAGE, "/api/v3/tools/{tool_slug}"],
325 ])(
326 "a tool endpoint on %s carries both guidance constants exactly once",
327 async (_label, pageUrl, pathKey) => {
328 const markdown = await renderOperation(pageUrl, pathKey);
329
330 // The v3.1 case fails if the predicate strips /v3.1 without /api, or
331 // tries /api/v3 before /api/v3.1.
332 expect(markdown.split(REST_VERSION_GUIDANCE)).toHaveLength(2);
333 expect(markdown.split(TOOL_VERSION_GUIDANCE)).toHaveLength(2);
334 },
335 );
336
337 test("renders both GET-by-slug version parameters with the v3.1 runtime default", async () => {
338 const pathKey = "/api/v3.1/tools/{tool_slug}";
339 bundledSpec = {
340 paths: {
341 [pathKey]: {
342 get: {
343 parameters: [
344 { name: "version", in: "query", description: "Optional tool version" },
345 {
346 name: "toolkit_versions",
347 in: "query",
348 description: "Toolkit version specification",
349 },
350 ],
351 responses: { 200: { description: "Success" } },
352 },
353 },
354 },
355 };
356
357 const markdown = await openapiPageToMarkdown({
358 url: "/reference/api-reference/tools/getToolsByToolSlug",
359 data: {
360 title: "Get tool",
361 getOpenAPIPageProps: () => ({
362 payload: { bundled: bundledSpec },
363 operations: [{ path: pathKey, method: "GET" }],
364 }),
365 },
366 });
367
368 for (const parameter of ["version", "toolkit_versions"]) {
369 const line = markdown
370 .split("\n")
371 .find(candidate => candidate.startsWith(`- \`${parameter}\``));
372 expect(line).toContain("Defaults to `latest` when omitted on REST API v3.1.");
373 }
374 });
375
376 test.each([
377 [
378 "v3.1 execute",
379 "/reference/api-reference/tools/executeTool",
380 "/api/v3.1/tools/execute/{tool_slug}",
381 'Tool version to execute (defaults to "00000000_00" if not specified)',
382 "latest",
383 "3.1",
384 ],
385 [
386 "v3.1 scopes",
387 "/reference/api-reference/tools/getRequiredScopes",
388 "/api/v3.1/tools/scopes/required",
389 "Toolkit version to resolve scopes against. Defaults to the pinned HTTP version when omitted.",
390 "latest",
391 "3.1",
392 ],
393 [
394 "v3 input generation",
395 "/reference/v3/api-reference/tools/generateToolInputs",
396 "/api/v3/tools/execute/{tool_slug}/input",
397 'Tool version to use when generating inputs (defaults to "latest" if not specified)',
398 "00000000_00",
399 "3.0",
400 ],
401 ])(
402 "replaces the stale %s request-body description",
403 async (_label, pageUrl, pathKey, staleDescription, expectedDefault, apiVersion) => {
404 bundledSpec = {
405 paths: {
406 [pathKey]: {
407 post: {
408 requestBody: {
409 content: {
410 "application/json": {
411 schema: {
412 type: "object",
413 properties: {
414 version: { type: "string", description: staleDescription },
415 },
416 },
417 },
418 },
419 },
420 responses: { 200: { description: "Success" } },
421 },
422 },
423 },
424 };
425
426 const markdown = await openapiPageToMarkdown({
427 url: pageUrl,
428 data: {
429 title: "Tool operation",
430 getOpenAPIPageProps: () => ({
431 payload: { bundled: bundledSpec },
432 operations: [{ path: pathKey, method: "POST" }],
433 }),
434 },
435 });
436 const versionLine = markdown.split("\n").find(line => line.startsWith("- `version`"));
437
438 expect(versionLine).toContain(
439 `Defaults to \`${expectedDefault}\` when omitted on REST API v${apiVersion}.`
440 );
441 expect(versionLine).not.toContain(staleDescription);
442 }
443 );
444
445 test("a non-tool endpoint carries the baseline and NOT the tool-version guidance", async () => {
446 const markdown = await renderOperation(V31_PAGE, "/api/v3.1/auth_configs");
447
448 expect(markdown).toContain(REST_VERSION_GUIDANCE);
449 expect(markdown).not.toContain(TOOL_VERSION_GUIDANCE);
450 });
451
452 test("a near-miss endpoint under /tools behaves like a non-tool endpoint", async () => {
453 // Under /tools, not one of the five — the case a prefix match gets wrong.
454 const markdown = await renderOperation(
455 "/reference/api-reference/tools/getToolsEnum",
456 "/api/v3.1/tools/enum",
457 );
458
459 expect(markdown).toContain(REST_VERSION_GUIDANCE);
460 expect(markdown).not.toContain(TOOL_VERSION_GUIDANCE);
461 });
462
463 test("neither page pulls in SESSION_GUARDRAILS", async () => {
464 // SESSION_GUARDRAILS composes TOOL_VERSION_GUIDANCE. If an operation page
465 // ever appended it, the tool-version text would arrive on every non-tool
466 // page through the back door and the scoping above would be theatre.
467 for (const [pageUrl, pathKey] of [
468 [V31_PAGE, "/api/v3.1/auth_configs"],
469 [V30_PAGE, "/api/v3/auth_configs"],
470 ]) {
471 const markdown = await renderOperation(pageUrl, pathKey);
472 expect(markdown).not.toContain(SESSION_GUARDRAILS);
473 expect(markdown).not.toContain("Instructions for AI Code Generators");
474 }
475 });
476 });
477
478 describe("LLM page degradation", () => {
479 test("keeps a readable response when typed page parsing fails", () => {
480 expect(
481 degradedPageToMarkdown({
482 url: "/docs/example",
483 data: { title: 42, description: "Fallback copy" },
484 })
485 ).toBe("# 42 (/docs/example)\n\nFallback copy");
486 });
487 });