Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { beforeAll, describe, expect, mock, test } from "bun:test"; |
| 2 | |
| 3 | import { |
| 4 | REST_VERSION_GUIDANCE, |
| 5 | TOOL_VERSION_GUIDANCE, |
| 6 | } from "../../lib/api-version-guidance"; |
| 7 | import { SESSION_GUARDRAILS } from "../../lib/llm-guardrails"; |
| 8 | import { parseCurlCommands, wordText } from "../../lib/api-snippets"; |
| 9 | |
| 10 | let bundledSpec: Record<string, unknown> = {}; |
| 11 | |
| 12 | mock.module("next/navigation", () => ({ |
| 13 | notFound: () => { |
| 14 | throw new Error("not found"); |
| 15 | }, |
| 16 | usePathname: () => "/reference/api-reference/tasks", |
| 17 | useSearchParams: () => new URLSearchParams(), |
| 18 | })); |
| 19 | // Keep catalog loaders and schema helpers real: Bun module mocks persist across |
| 20 | // files and can empty the knowledge catalog or replace transformations in later suites. |
| 21 | |
| 22 | let openapiPageToMarkdown: typeof import("../../app/llms.mdx/[[...slug]]/route").openapiPageToMarkdown; |
| 23 | let degradedPageToMarkdown: typeof import("../../app/llms.mdx/[[...slug]]/route").degradedPageToMarkdown; |
| 24 | |
| 25 | beforeAll(async () => { |
| 26 | ({ openapiPageToMarkdown, degradedPageToMarkdown } = await import( |
| 27 | "../../app/llms.mdx/[[...slug]]/route" |
| 28 | )); |
| 29 | }); |
| 30 | |
| 31 | describe("LLM OpenAPI markdown", () => { |
| 32 | test("preserves real toolkit schema helpers for subsequent suites", async () => { |
| 33 | const { processSchema, toolFromApi } = await import("../../lib/toolkit-schema"); |
| 34 | expect(processSchema({ |
| 35 | properties: { query: { type: "string" } }, |
| 36 | required: ["query"], |
| 37 | })?.query.required).toBe(true); |
| 38 | expect(toolFromApi({ slug: "TEST_TOOL" }).name).toBe("TEST_TOOL"); |
| 39 | }); |
| 40 | |
| 41 | test("continues to render API operations with endpoint and cURL details", async () => { |
| 42 | bundledSpec = { |
| 43 | paths: { |
| 44 | "/v3.1/test": { |
| 45 | get: { |
| 46 | summary: "Get test", |
| 47 | responses: { |
| 48 | 200: { description: "Success" }, |
| 49 | }, |
| 50 | }, |
| 51 | }, |
| 52 | }, |
| 53 | servers: [{ url: "https://backend.example.com" }], |
| 54 | }; |
| 55 | |
| 56 | const markdown = await openapiPageToMarkdown({ |
| 57 | url: "/reference/api-reference/test/getTest", |
| 58 | data: { |
| 59 | title: "Get test", |
| 60 | getOpenAPIPageProps: () => ({ |
| 61 | payload: { bundled: bundledSpec }, |
| 62 | operations: [{ path: "/v3.1/test", method: "GET" }], |
| 63 | }), |
| 64 | }, |
| 65 | }); |
| 66 | |
| 67 | expect(markdown).toContain("**Endpoint:** `https://backend.example.com/v3.1/test`"); |
| 68 | for (const [label, language] of [["Python", "python"], ["TypeScript", "typescript"], ["API", "bash"]]) { |
| 69 | expect(markdown).toContain(`#### ${label}\n\n\`\`\`${language}\n`); |
| 70 | } |
| 71 | expect(markdown).toContain( |
| 72 | 'curl -X GET "https://backend.example.com/v3.1/test"', |
| 73 | ); |
| 74 | }); |
| 75 | |
| 76 | test("raw request variants use organization authentication instead of a project key", async () => { |
| 77 | const markdown = await openapiPageToMarkdown({ |
| 78 | url: "/reference/api-reference/organization/test", |
| 79 | data: { |
| 80 | title: "Organization request", |
| 81 | getOpenAPIPageProps: () => ({ |
| 82 | payload: { bundled: { |
| 83 | paths: { "/api/v3.1/org/test": { post: { |
| 84 | security: [{ OrgApiKeyAuth: [] }], |
| 85 | requestBody: { content: { "application/json": { schema: { oneOf: [{ |
| 86 | type: "object", required: ["title", "enabled", "value"], |
| 87 | properties: { title: { type: "string", example: "O'Reilly" }, enabled: { type: "boolean" }, value: { type: "null" } }, |
| 88 | }] } } } }, |
| 89 | responses: { "200": { description: "Success" } }, |
| 90 | } } }, |
| 91 | components: { securitySchemes: { OrgApiKeyAuth: { type: "apiKey", in: "header", name: "x-org-api-key" } } }, |
| 92 | } }, |
| 93 | operations: [{ path: "/api/v3.1/org/test", method: "POST" }], |
| 94 | }), |
| 95 | }, |
| 96 | }); |
| 97 | const snippets = [...markdown.matchAll(/```(python|typescript|bash)\n([\s\S]*?)```/g)]; |
| 98 | expect(snippets.map((match) => match[1])).toEqual(["python", "typescript", "bash"]); |
| 99 | for (const snippet of snippets) { |
| 100 | expect(snippet[2]).toContain("x-org-api-key"); |
| 101 | expect(snippet[2]).toContain("COMPOSIO_ORG_API_KEY"); |
| 102 | expect(snippet[2]).not.toContain('"x-api-key"'); |
| 103 | } |
| 104 | const curl = snippets.find((snippet) => snippet[1] === "bash")![2]; |
| 105 | const request = parseCurlCommands(curl)[0]; |
| 106 | expect(JSON.parse(wordText(request.data[0].value))).toEqual({ title: "O'Reilly", enabled: true, value: null }); |
| 107 | }); |
| 108 | |
| 109 | test("renders webhook delivery headers and payload schema", async () => { |
| 110 | bundledSpec = { |
| 111 | openapi: "3.1.0", |
| 112 | webhooks: { |
| 113 | "composio.test.event": { |
| 114 | post: { |
| 115 | summary: "Test event", |
| 116 | parameters: [ |
| 117 | { |
| 118 | name: "webhook-id", |
| 119 | in: "header", |
| 120 | required: true, |
| 121 | description: "Stable delivery identifier.", |
| 122 | schema: { type: "string" }, |
| 123 | }, |
| 124 | ], |
| 125 | requestBody: { |
| 126 | content: { |
| 127 | "application/json": { |
| 128 | schema: { |
| 129 | type: "object", |
| 130 | required: ["id"], |
| 131 | properties: { |
| 132 | id: { |
| 133 | type: "string", |
| 134 | description: "Unique event identifier.", |
| 135 | }, |
| 136 | }, |
| 137 | }, |
| 138 | example: { id: "msg_test" }, |
| 139 | }, |
| 140 | }, |
| 141 | }, |
| 142 | responses: { |
| 143 | 200: { description: "Delivery accepted" }, |
| 144 | }, |
| 145 | }, |
| 146 | }, |
| 147 | }, |
| 148 | }; |
| 149 | |
| 150 | const markdown = await openapiPageToMarkdown({ |
| 151 | url: "/reference/api-reference/webhook-events/composio_test_event", |
| 152 | data: { |
| 153 | title: "Test event", |
| 154 | getOpenAPIPageProps: () => ({ |
| 155 | payload: { bundled: bundledSpec }, |
| 156 | webhooks: [{ name: "composio.test.event", method: "post" }], |
| 157 | }), |
| 158 | }, |
| 159 | }); |
| 160 | |
| 161 | expect(markdown).toContain("## POST `composio.test.event`"); |
| 162 | expect(markdown).toContain("### Delivery Headers"); |
| 163 | expect(markdown).toContain( |
| 164 | "- `webhook-id` (string) *(required)*: Stable delivery identifier.", |
| 165 | ); |
| 166 | expect(markdown).toContain("### Request Body"); |
| 167 | expect(markdown).toContain( |
| 168 | "- `id` (string) *(required)*: Unique event identifier.", |
| 169 | ); |
| 170 | expect(markdown).toContain('"id": "msg_test"'); |
| 171 | expect(markdown).not.toContain("### Example cURL Request"); |
| 172 | }); |
| 173 | |
| 174 | test("bounds recursive array type rendering", async () => { |
| 175 | bundledSpec = { |
| 176 | paths: { |
| 177 | "/v3.1/test": { |
| 178 | get: { |
| 179 | parameters: [ |
| 180 | { |
| 181 | name: "tree", |
| 182 | in: "query", |
| 183 | schema: { $ref: "#/components/schemas/RecursiveArray" }, |
| 184 | }, |
| 185 | ], |
| 186 | responses: { |
| 187 | 200: { description: "Success" }, |
| 188 | }, |
| 189 | }, |
| 190 | }, |
| 191 | }, |
| 192 | components: { |
| 193 | schemas: { |
| 194 | RecursiveArray: { |
| 195 | type: "array", |
| 196 | items: { $ref: "#/components/schemas/RecursiveArray" }, |
| 197 | }, |
| 198 | }, |
| 199 | }, |
| 200 | }; |
| 201 | |
| 202 | const markdown = await openapiPageToMarkdown({ |
| 203 | url: "/reference/api-reference/test/getTest", |
| 204 | data: { |
| 205 | title: "Get test", |
| 206 | getOpenAPIPageProps: () => ({ |
| 207 | payload: { bundled: bundledSpec }, |
| 208 | operations: [{ path: "/v3.1/test", method: "GET" }], |
| 209 | }), |
| 210 | }, |
| 211 | }); |
| 212 | |
| 213 | expect(markdown).toContain( |
| 214 | "- `tree` (array<array<array<array<array<...>>>>>):", |
| 215 | ); |
| 216 | }); |
| 217 | |
| 218 | test("union request samples keep the parent schema and bound circular unions", async () => { |
| 219 | const renderBody = async (schema: Record<string, unknown>, schemas: Record<string, unknown> = {}) => { |
| 220 | const markdown = await openapiPageToMarkdown({ |
| 221 | url: "/reference/api-reference/test/postTest", |
| 222 | data: { |
| 223 | title: "Post test", |
| 224 | getOpenAPIPageProps: () => ({ |
| 225 | payload: { bundled: { |
| 226 | paths: { "/v3.1/test": { post: { |
| 227 | requestBody: { content: { "application/json": { schema } } }, |
| 228 | responses: { 200: { description: "Success" } }, |
| 229 | } } }, |
| 230 | components: { schemas }, |
| 231 | } }, |
| 232 | operations: [{ path: "/v3.1/test", method: "POST" }], |
| 233 | }), |
| 234 | }, |
| 235 | }); |
| 236 | const curl = [...markdown.matchAll(/```bash\n([\s\S]*?)```/g)][0][1]; |
| 237 | return JSON.parse(wordText(parseCurlCommands(curl)[0].data[0].value)); |
| 238 | }; |
| 239 | |
| 240 | expect(await renderBody({ |
| 241 | type: "object", |
| 242 | properties: { name: { type: "string" }, id: { type: "integer" } }, |
| 243 | oneOf: [{ required: ["name"] }, { required: ["id"] }], |
| 244 | })).toEqual({ name: "string", id: 1 }); |
| 245 | |
| 246 | expect(await renderBody( |
| 247 | { $ref: "#/components/schemas/Node" }, |
| 248 | { Node: { oneOf: [{ $ref: "#/components/schemas/Node" }, { type: "string" }] } }, |
| 249 | )).toBe("..."); |
| 250 | }); |
| 251 | }); |
| 252 | |
| 253 | /** |
| 254 | * Version identity on operation pages. |
| 255 | * |
| 256 | * These are the pages that publish a complete working curl example, so they |
| 257 | * are the strongest signal an agent acts on — and they never touch |
| 258 | * mdxToCleanMarkdown, so a fix confined to lib/source.ts leaves them alone. |
| 259 | * |
| 260 | * Every fixture below uses production-shaped spec path keys — `/api/v3.1/…` |
| 261 | * and `/api/v3/…`, with the `/api` segment — because that is what the |
| 262 | * committed specs contain. The synthetic `/v3.1/test` keys in the fixtures |
| 263 | * above predate this and are deliberately left alone; do not copy their shape |
| 264 | * down here, or these tests will pass against an implementation that |
| 265 | * mis-normalizes production paths. |
| 266 | */ |
| 267 | describe("LLM OpenAPI markdown — API version identity", () => { |
| 268 | function specWith(pathKey: string) { |
| 269 | return { |
| 270 | paths: { |
| 271 | [pathKey]: { |
| 272 | get: { |
| 273 | summary: "Fixture operation", |
| 274 | responses: { 200: { description: "Success" } }, |
| 275 | }, |
| 276 | }, |
| 277 | }, |
| 278 | servers: [{ url: "https://backend.composio.dev" }], |
| 279 | }; |
| 280 | } |
| 281 | |
| 282 | function renderOperation(pageUrl: string, pathKey: string) { |
| 283 | bundledSpec = specWith(pathKey); |
| 284 | return openapiPageToMarkdown({ |
| 285 | url: pageUrl, |
| 286 | data: { |
| 287 | title: "Fixture operation", |
| 288 | getOpenAPIPageProps: () => ({ |
| 289 | payload: { bundled: bundledSpec }, |
| 290 | operations: [{ path: pathKey, method: "GET" }], |
| 291 | }), |
| 292 | }, |
| 293 | }); |
| 294 | } |
| 295 | |
| 296 | const V31_PAGE = "/reference/api-reference/auth-configs/getAuthConfigs"; |
| 297 | const V30_PAGE = "/reference/v3/api-reference/auth-configs/getAuthConfigs"; |
| 298 | |
| 299 | test("a v3.1 operation page carries the current-version pointer above the endpoint line", async () => { |
| 300 | const markdown = await renderOperation(V31_PAGE, "/api/v3.1/auth_configs"); |
| 301 | |
| 302 | expect(markdown).toContain("**API version:**"); |
| 303 | // Above **Endpoint:**, so a truncating reader still sees it. |
| 304 | expect(markdown.indexOf("**API version:**")).toBeLessThan( |
| 305 | markdown.indexOf("**Endpoint:**"), |
| 306 | ); |
| 307 | }); |
| 308 | |
| 309 | test("a v3.0 operation page carries the legacy pointer and links its v3.1 page", async () => { |
| 310 | const markdown = await renderOperation(V30_PAGE, "/api/v3/auth_configs"); |
| 311 | |
| 312 | expect(markdown).toContain("**API version:**"); |
| 313 | expect(markdown).toContain("v3.0"); |
| 314 | expect(markdown).toContain( |
| 315 | "/reference/api-reference/auth-configs/getAuthConfigs.md", |
| 316 | ); |
| 317 | expect(markdown.indexOf("**API version:**")).toBeLessThan( |
| 318 | markdown.indexOf("**Endpoint:**"), |
| 319 | ); |
| 320 | }); |
| 321 | |
| 322 | test.each([ |
| 323 | ["v3.1", V31_PAGE, "/api/v3.1/tools/{tool_slug}"], |
| 324 | ["v3.0", V30_PAGE, "/api/v3/tools/{tool_slug}"], |
| 325 | ])( |
| 326 | "a tool endpoint on %s carries both guidance constants exactly once", |
| 327 | async (_label, pageUrl, pathKey) => { |
| 328 | const markdown = await renderOperation(pageUrl, pathKey); |
| 329 | |
| 330 | // The v3.1 case fails if the predicate strips /v3.1 without /api, or |
| 331 | // tries /api/v3 before /api/v3.1. |
| 332 | expect(markdown.split(REST_VERSION_GUIDANCE)).toHaveLength(2); |
| 333 | expect(markdown.split(TOOL_VERSION_GUIDANCE)).toHaveLength(2); |
| 334 | }, |
| 335 | ); |
| 336 | |
| 337 | test("renders both GET-by-slug version parameters with the v3.1 runtime default", async () => { |
| 338 | const pathKey = "/api/v3.1/tools/{tool_slug}"; |
| 339 | bundledSpec = { |
| 340 | paths: { |
| 341 | [pathKey]: { |
| 342 | get: { |
| 343 | parameters: [ |
| 344 | { name: "version", in: "query", description: "Optional tool version" }, |
| 345 | { |
| 346 | name: "toolkit_versions", |
| 347 | in: "query", |
| 348 | description: "Toolkit version specification", |
| 349 | }, |
| 350 | ], |
| 351 | responses: { 200: { description: "Success" } }, |
| 352 | }, |
| 353 | }, |
| 354 | }, |
| 355 | }; |
| 356 | |
| 357 | const markdown = await openapiPageToMarkdown({ |
| 358 | url: "/reference/api-reference/tools/getToolsByToolSlug", |
| 359 | data: { |
| 360 | title: "Get tool", |
| 361 | getOpenAPIPageProps: () => ({ |
| 362 | payload: { bundled: bundledSpec }, |
| 363 | operations: [{ path: pathKey, method: "GET" }], |
| 364 | }), |
| 365 | }, |
| 366 | }); |
| 367 | |
| 368 | for (const parameter of ["version", "toolkit_versions"]) { |
| 369 | const line = markdown |
| 370 | .split("\n") |
| 371 | .find(candidate => candidate.startsWith(`- \`${parameter}\``)); |
| 372 | expect(line).toContain("Defaults to `latest` when omitted on REST API v3.1."); |
| 373 | } |
| 374 | }); |
| 375 | |
| 376 | test.each([ |
| 377 | [ |
| 378 | "v3.1 execute", |
| 379 | "/reference/api-reference/tools/executeTool", |
| 380 | "/api/v3.1/tools/execute/{tool_slug}", |
| 381 | 'Tool version to execute (defaults to "00000000_00" if not specified)', |
| 382 | "latest", |
| 383 | "3.1", |
| 384 | ], |
| 385 | [ |
| 386 | "v3.1 scopes", |
| 387 | "/reference/api-reference/tools/getRequiredScopes", |
| 388 | "/api/v3.1/tools/scopes/required", |
| 389 | "Toolkit version to resolve scopes against. Defaults to the pinned HTTP version when omitted.", |
| 390 | "latest", |
| 391 | "3.1", |
| 392 | ], |
| 393 | [ |
| 394 | "v3 input generation", |
| 395 | "/reference/v3/api-reference/tools/generateToolInputs", |
| 396 | "/api/v3/tools/execute/{tool_slug}/input", |
| 397 | 'Tool version to use when generating inputs (defaults to "latest" if not specified)', |
| 398 | "00000000_00", |
| 399 | "3.0", |
| 400 | ], |
| 401 | ])( |
| 402 | "replaces the stale %s request-body description", |
| 403 | async (_label, pageUrl, pathKey, staleDescription, expectedDefault, apiVersion) => { |
| 404 | bundledSpec = { |
| 405 | paths: { |
| 406 | [pathKey]: { |
| 407 | post: { |
| 408 | requestBody: { |
| 409 | content: { |
| 410 | "application/json": { |
| 411 | schema: { |
| 412 | type: "object", |
| 413 | properties: { |
| 414 | version: { type: "string", description: staleDescription }, |
| 415 | }, |
| 416 | }, |
| 417 | }, |
| 418 | }, |
| 419 | }, |
| 420 | responses: { 200: { description: "Success" } }, |
| 421 | }, |
| 422 | }, |
| 423 | }, |
| 424 | }; |
| 425 | |
| 426 | const markdown = await openapiPageToMarkdown({ |
| 427 | url: pageUrl, |
| 428 | data: { |
| 429 | title: "Tool operation", |
| 430 | getOpenAPIPageProps: () => ({ |
| 431 | payload: { bundled: bundledSpec }, |
| 432 | operations: [{ path: pathKey, method: "POST" }], |
| 433 | }), |
| 434 | }, |
| 435 | }); |
| 436 | const versionLine = markdown.split("\n").find(line => line.startsWith("- `version`")); |
| 437 | |
| 438 | expect(versionLine).toContain( |
| 439 | `Defaults to \`${expectedDefault}\` when omitted on REST API v${apiVersion}.` |
| 440 | ); |
| 441 | expect(versionLine).not.toContain(staleDescription); |
| 442 | } |
| 443 | ); |
| 444 | |
| 445 | test("a non-tool endpoint carries the baseline and NOT the tool-version guidance", async () => { |
| 446 | const markdown = await renderOperation(V31_PAGE, "/api/v3.1/auth_configs"); |
| 447 | |
| 448 | expect(markdown).toContain(REST_VERSION_GUIDANCE); |
| 449 | expect(markdown).not.toContain(TOOL_VERSION_GUIDANCE); |
| 450 | }); |
| 451 | |
| 452 | test("a near-miss endpoint under /tools behaves like a non-tool endpoint", async () => { |
| 453 | // Under /tools, not one of the five — the case a prefix match gets wrong. |
| 454 | const markdown = await renderOperation( |
| 455 | "/reference/api-reference/tools/getToolsEnum", |
| 456 | "/api/v3.1/tools/enum", |
| 457 | ); |
| 458 | |
| 459 | expect(markdown).toContain(REST_VERSION_GUIDANCE); |
| 460 | expect(markdown).not.toContain(TOOL_VERSION_GUIDANCE); |
| 461 | }); |
| 462 | |
| 463 | test("neither page pulls in SESSION_GUARDRAILS", async () => { |
| 464 | // SESSION_GUARDRAILS composes TOOL_VERSION_GUIDANCE. If an operation page |
| 465 | // ever appended it, the tool-version text would arrive on every non-tool |
| 466 | // page through the back door and the scoping above would be theatre. |
| 467 | for (const [pageUrl, pathKey] of [ |
| 468 | [V31_PAGE, "/api/v3.1/auth_configs"], |
| 469 | [V30_PAGE, "/api/v3/auth_configs"], |
| 470 | ]) { |
| 471 | const markdown = await renderOperation(pageUrl, pathKey); |
| 472 | expect(markdown).not.toContain(SESSION_GUARDRAILS); |
| 473 | expect(markdown).not.toContain("Instructions for AI Code Generators"); |
| 474 | } |
| 475 | }); |
| 476 | }); |
| 477 | |
| 478 | describe("LLM page degradation", () => { |
| 479 | test("keeps a readable response when typed page parsing fails", () => { |
| 480 | expect( |
| 481 | degradedPageToMarkdown({ |
| 482 | url: "/docs/example", |
| 483 | data: { title: 42, description: "Fallback copy" }, |
| 484 | }) |
| 485 | ).toBe("# 42 (/docs/example)\n\nFallback copy"); |
| 486 | }); |
| 487 | }); |