8
9
import { Composio } from '@composio/core';
10
import { MastraProvider } from '@composio/mastra';
11
-import { MCPClient } from '@mastra/mcp';
11
+import { getMcpCallToolMeta, MCPClient } from '@mastra/mcp';
12
import { Agent } from '@mastra/core/agent';
13
import { e2e } from '@e2e-tests/utils';
14
import { TIMEOUTS } from '@e2e-tests/utils/const';
18
import { z } from 'zod';
19
20
const MCP_CONNECT_TIMEOUT_MS = 15_000;
21
+const KARMA_SCHEMA = z.object({ karma: z.number() });
22
+const SEARCH_TOOL = 'myHttpClient_COMPOSIO_SEARCH_TOOLS';
23
+const EXECUTE_TOOL = 'myHttpClient_COMPOSIO_MULTI_EXECUTE_TOOL';
24
+
25
+// Allowlist for failure logs: Zod strips every other field.
26
+const idsSchema = z.object({ log_id: z.string().optional(), request_id: z.string().optional() });
27
+const searchResponseSchema = idsSchema.extend({
28
+ data: z
29
+ .object({
30
+ results: z.array(z.object({ primary_tool_slugs: z.array(z.string()).optional() })).optional(),
31
+ tool_schemas: z
32
+ .object({
33
+ HACKERNEWS_GET_USER: z
34
+ .object({
35
+ input_schema: z.object({ required: z.array(z.string()).optional() }).nullish(),
36
+ })
37
+ .optional(),
38
+ })
39
+ .optional(),
40
+ })
41
+ .optional(),
42
+});
43
+const executeArgsSchema = z.object({
44
+ tools: z.array(z.object({ tool_slug: z.string(), arguments: z.record(z.string(), z.unknown()) })),
45
+});
46
+const mcpTextResultSchema = z.object({
47
+ _meta: idsSchema.optional(),
48
+ content: z.array(z.object({ type: z.literal('text'), text: z.string() })),
49
+});
50
+
51
+/** Flat log record for one search result, given as a structured value or an MCP text envelope. */
52
+function searchRecord(payload: { toolCallId: string; isError?: boolean; result: unknown }) {
53
+ const envelope = mcpTextResultSchema.safeParse(payload.result).data;
54
+ let value: unknown = payload.result;
55
+ if (envelope) {
56
+ try {
57
+ value = JSON.parse(envelope.content.map(part => part.text).join(''));
58
+ } catch {
59
+ value = undefined;
60
+ }
61
+ }
62
+ const response = searchResponseSchema.safeParse(value).data;
63
+ const meta = idsSchema.safeParse(getMcpCallToolMeta(payload.result) ?? envelope?._meta).data;
64
+ return {
65
+ toolCallId: payload.toolCallId,
66
+ isError: payload.isError,
67
+ log_id: meta?.log_id ?? response?.log_id,
68
+ request_id: meta?.request_id ?? response?.request_id,
69
+ requiredInput: response?.data?.tool_schemas?.HACKERNEWS_GET_USER?.input_schema?.required,
70
+ matchedSlugs: response?.data?.results?.flatMap(r => r.primary_tool_slugs ?? []),
71
+ };
72
+}
73
+
74
+/** Text of an MCP tool result, or its JSON when the result is structured. */
75
+function toolResultText(result: unknown) {
76
+ const envelope = mcpTextResultSchema.safeParse(result).data;
77
+ return envelope ? envelope.content.map(part => part.text).join('') : JSON.stringify(result);
78
+}
79
80
declare module 'bun' {
81
interface Env {
128
},
129
});
130
73
- const tools = await mcpClient.listTools();
74
- const openai = createOpenAI({ apiKey: Bun.env.OPENAI_API_KEY });
131
+ try {
132
+ const tools = await mcpClient.listTools();
133
+ expect(Object.keys(tools).length).toBeGreaterThan(0);
134
+ const openai = createOpenAI({ apiKey: Bun.env.OPENAI_API_KEY });
135
76
- const hackernewsAgent = new Agent({
77
- id: 'test',
78
- name: 'test',
79
- instructions: `You're a helpful Hackernews agent, able to finding information about users.`,
80
- model: openai('gpt-5.1'),
81
- tools,
82
- });
136
+ const hackernewsAgent = new Agent({
137
+ id: 'test',
138
+ name: 'test',
139
+ instructions: `You're a helpful Hackernews agent, able to finding information about users.`,
140
+ model: openai('gpt-5.5'),
141
+ tools,
142
+ });
143
84
- const result = await hackernewsAgent.generate(
85
- 'Look up user "pg", and tell me their karma score.',
86
- {
87
- structuredOutput: {
88
- schema: z.object({
89
- karma: z.number(),
90
- }),
91
- },
92
- stopWhen: stepCountIs(10),
93
- }
94
- );
144
+ const result = await hackernewsAgent.generate(
145
+ 'Look up the HackerNews user "pg" with HACKERNEWS_GET_USER, then return the exact karma value from that tool result.',
146
+ { structuredOutput: { schema: KARMA_SCHEMA }, stopWhen: stepCountIs(10) }
147
+ );
148
96
- const toolCalls = result.toolCalls.flatMap(toolCall => toolCall.payload.toolName);
97
- const toolCount = Object.keys(tools).length;
149
+ // The karma must come from a HACKERNEWS_GET_USER execution, not from the model's memory.
150
+ const lookupCallIds = new Set(
151
+ result.toolCalls
152
+ .filter(({ payload }) => payload.toolName === EXECUTE_TOOL)
153
+ .filter(({ payload }) =>
154
+ executeArgsSchema
155
+ .safeParse(payload.args)
156
+ .data?.tools.some(
157
+ ({ tool_slug, arguments: args }) =>
158
+ tool_slug === 'HACKERNEWS_GET_USER' && args.username === 'pg'
159
+ )
160
+ )
161
+ .map(({ payload }) => payload.toolCallId)
162
+ );
163
+ const lookupResults = result.toolResults
164
+ .filter(({ payload }) => lookupCallIds.has(payload.toolCallId))
165
+ .map(({ payload }) => toolResultText(payload.result));
166
99
- expect(toolCount).toBeGreaterThan(0);
100
- expect(toolCalls).toBeDefined();
101
- expect(result.error).toBeUndefined();
102
- expect(result.object).toBeDefined();
103
- expect(result.object.karma).toBeGreaterThanOrEqual(0);
167
+ const parsed = KARMA_SCHEMA.safeParse(result.object);
168
+ if (!parsed.success || lookupResults.length === 0) {
169
+ const diagnostics = {
170
+ finishReason: result.finishReason,
171
+ toolCalls: result.toolCalls.map(({ payload }) => payload.toolName),
172
+ searches: result.toolResults
173
+ .filter(({ payload }) => payload.toolName === SEARCH_TOOL)
174
+ .map(({ payload }) => searchRecord(payload)),
175
+ executions: result.toolCalls
176
+ .filter(({ payload }) => payload.toolName === EXECUTE_TOOL)
177
+ .map(({ payload }) => executeArgsSchema.safeParse(payload.args).data?.tools),
178
+ lookupResults: lookupResults.map(text => text.slice(0, 500)),
179
+ steps: result.steps.map(step => ({
180
+ finishReason: step.finishReason,
181
+ usage: step.usage,
182
+ providerMetadata: step.providerMetadata,
183
+ })),
184
+ warnings: result.warnings,
185
+ };
186
+ console.error('Agent diagnostics', JSON.stringify(diagnostics, null, 2));
187
+ }
188
105
- await mcpClient.disconnect();
189
+ expect(result.error).toBeUndefined();
190
+ const { karma } = KARMA_SCHEMA.parse(result.object);
191
+ expect(karma).toBeGreaterThan(150_000);
192
+ expect(lookupResults.join('\n')).toContain(String(karma));
193
+ } finally {
194
+ await mcpClient.disconnect();
195
+ }
196
},
197
{
198
timeout: TIMEOUTS.LLM_LONG,