Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 47 lines 1.64 KB
Raw
1 import { defineEval } from 'eve/evals';
2
3 const cases = [
4 {
5 description: 'Explains session creation with current TypeScript APIs.',
6 prompt:
7 'Search the Composio docs, then explain in TypeScript how I create a Composio session for user `alice` with GitHub tools. Include the relevant docs link.',
8 expectedSignals: [/composio\.(create|sessions\.create)/i, /github/i],
9 },
10 {
11 description: 'Distinguishes auth configs from connected accounts.',
12 prompt:
13 'Search the Composio docs, then explain the difference between an auth config and a connected account in Composio. Keep it short and cite the docs.',
14 expectedSignals: [/auth config/i, /connected account/i],
15 },
16 {
17 description: 'Explains the session-based MCP setup path.',
18 prompt:
19 'Search the Composio docs, then explain how I expose Composio session tools over MCP. Include the recommended `mcp: true` shape and cite the docs.',
20 expectedSignals: [/mcp/i, /mcp\s*:\s*true/i],
21 },
22 ] as const;
23
24 export default cases.map(row =>
25 defineEval({
26 description: row.description,
27 tags: ['docs-agent', 'model-flow', 'grounded-answer'],
28 metadata: {
29 suite: 'docs-agent-model-flow',
30 prompt: row.prompt,
31 },
32 async test(t) {
33 await t.send(row.prompt);
34
35 t.succeeded();
36 t.noFailedActions();
37 // The docs channel now injects eager search context before the model step,
38 // so a good answer may not need an explicit search_docs tool call.
39 t.maxToolCalls(3);
40 t.messageIncludes(/\]\(\/(?:docs|reference)\//);
41
42 for (const signal of row.expectedSignals) {
43 t.messageIncludes(signal);
44 }
45 },
46 })
47 );