Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { defineEval } from 'eve/evals'; |
| 2 | |
| 3 | const cases = [ |
| 4 | { |
| 5 | description: 'Explains session creation with current TypeScript APIs.', |
| 6 | prompt: |
| 7 | 'Search the Composio docs, then explain in TypeScript how I create a Composio session for user `alice` with GitHub tools. Include the relevant docs link.', |
| 8 | expectedSignals: [/composio\.(create|sessions\.create)/i, /github/i], |
| 9 | }, |
| 10 | { |
| 11 | description: 'Distinguishes auth configs from connected accounts.', |
| 12 | prompt: |
| 13 | 'Search the Composio docs, then explain the difference between an auth config and a connected account in Composio. Keep it short and cite the docs.', |
| 14 | expectedSignals: [/auth config/i, /connected account/i], |
| 15 | }, |
| 16 | { |
| 17 | description: 'Explains the session-based MCP setup path.', |
| 18 | prompt: |
| 19 | 'Search the Composio docs, then explain how I expose Composio session tools over MCP. Include the recommended `mcp: true` shape and cite the docs.', |
| 20 | expectedSignals: [/mcp/i, /mcp\s*:\s*true/i], |
| 21 | }, |
| 22 | ] as const; |
| 23 | |
| 24 | export default cases.map(row => |
| 25 | defineEval({ |
| 26 | description: row.description, |
| 27 | tags: ['docs-agent', 'model-flow', 'grounded-answer'], |
| 28 | metadata: { |
| 29 | suite: 'docs-agent-model-flow', |
| 30 | prompt: row.prompt, |
| 31 | }, |
| 32 | async test(t) { |
| 33 | await t.send(row.prompt); |
| 34 | |
| 35 | t.succeeded(); |
| 36 | t.noFailedActions(); |
| 37 | // The docs channel now injects eager search context before the model step, |
| 38 | // so a good answer may not need an explicit search_docs tool call. |
| 39 | t.maxToolCalls(3); |
| 40 | t.messageIncludes(/\]\(\/(?:docs|reference)\//); |
| 41 | |
| 42 | for (const signal of row.expectedSignals) { |
| 43 | t.messageIncludes(signal); |
| 44 | } |
| 45 | }, |
| 46 | }) |
| 47 | ); |