Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 125 lines 5.02 KB
Raw
1 import { describe, it, expect } from 'vitest';
2 import { applyCompatLayer, convertSchemaToZod } from '@mastra/schema-compat';
3 import { relaxOutputSchema } from '../src/relax-output-schema';
4
5 /**
6 * Integration regression test for issue #3047.
7 *
8 * `mastra.test.ts` mocks `@mastra/schema-compat`, so it verifies *that*
9 * `wrapTool` relaxes the output schema but never exercises the real validation.
10 * This test runs the full production chain instead — the same one `wrapTool`
11 * drives at tool-execution time:
12 *
13 * relaxOutputSchema → applyCompatLayer(mode: 'jsonSchema') → convertSchemaToZod → parse
14 *
15 * `convertSchemaToZod` is the JSON-Schema → Zod conversion Mastra performs
16 * internally before `validateToolOutput` checks a tool result. The test proves
17 * that the relaxation makes a real third-party API response (with `null`
18 * optional fields and extra keys) survive validation instead of being dropped
19 * and replaced by an error — and that no data is truncated in the process.
20 *
21 * See https://github.com/ComposioHQ/composio/issues/3047.
22 */
23
24 // A strict Composio-style output schema: optional fields typed as non-nullable
25 // primitives, objects closed with `additionalProperties: false` — exactly the
26 // shape the Composio API emits for connector tools.
27 const STRICT_OUTPUT_SCHEMA = {
28 type: 'object',
29 additionalProperties: false,
30 properties: {
31 data: {
32 type: 'object',
33 additionalProperties: false,
34 properties: {
35 id: { type: 'string' },
36 title: { type: 'string' },
37 avatarUrl: { type: 'string' },
38 description: { type: 'string' },
39 assignee: {
40 type: 'object',
41 additionalProperties: false,
42 properties: { name: { type: 'string' } },
43 },
44 },
45 },
46 },
47 };
48
49 // A realistic third-party response: `null` for unset optional fields plus an
50 // unexpected extra key — what Linear / Notion / Jira / Slack actually return.
51 const REAL_API_PAYLOAD = {
52 data: {
53 id: 'iss_123',
54 title: 'Fix the thing',
55 avatarUrl: null,
56 description: null,
57 assignee: null,
58 unexpectedExtraKey: 'present',
59 },
60 };
61
62 /** Mirror of the validation Mastra runs on a tool result. */
63 function compileForValidation(schema: object) {
64 const jsonSchema = applyCompatLayer({ schema, compatLayers: [], mode: 'jsonSchema' });
65 return convertSchemaToZod(jsonSchema as object);
66 }
67
68 describe('Mastra output validation against real schema-compat (issue #3047)', () => {
69 it('documents the bug: the strict schema rejects null optional fields', () => {
70 const validator = compileForValidation(STRICT_OUTPUT_SCHEMA);
71 const result = validator.safeParse(REAL_API_PAYLOAD);
72 // Without the relaxation Mastra would drop this data and substitute an error.
73 expect(result.success).toBe(false);
74 });
75
76 it('relaxed schema accepts the payload, preserving null values and extra keys', () => {
77 const validator = compileForValidation(relaxOutputSchema(STRICT_OUTPUT_SCHEMA));
78 const result = validator.safeParse(REAL_API_PAYLOAD);
79
80 expect(result.success).toBe(true);
81 // Nothing is truncated: nulls and the extra key survive untouched.
82 expect(result.data).toEqual(REAL_API_PAYLOAD);
83 });
84
85 it('relaxed schema lets an enum field return null, but still rejects bogus values', () => {
86 const schema = {
87 type: 'object',
88 additionalProperties: false,
89 properties: { status: { type: 'string', enum: ['open', 'closed'] } },
90 };
91 const strict = compileForValidation(schema);
92 const relaxed = compileForValidation(relaxOutputSchema(schema));
93
94 expect(strict.safeParse({ status: null }).success).toBe(false); // the gap
95 expect(relaxed.safeParse({ status: null }).success).toBe(true); // fixed
96 expect(relaxed.safeParse({ status: 'open' }).success).toBe(true); // still valid
97 expect(relaxed.safeParse({ status: 'nope' }).success).toBe(false); // still constrained
98 });
99
100 it('relaxed schema accepts a response that omits a required field', () => {
101 const schema = {
102 type: 'object',
103 additionalProperties: false,
104 required: ['id', 'title'],
105 properties: { id: { type: 'string' }, title: { type: 'string' } },
106 };
107 const strict = compileForValidation(schema);
108 const relaxed = compileForValidation(relaxOutputSchema(schema));
109
110 expect(strict.safeParse({ id: 'iss_1' }).success).toBe(false); // the gap
111 expect(relaxed.safeParse({ id: 'iss_1' }).success).toBe(true); // fixed
112 });
113
114 it('never narrows validation through a `not` keyword', () => {
115 const schema = { not: { type: 'string' } }; // "must not be a string" — null is valid
116 const strict = compileForValidation(schema);
117 const relaxed = compileForValidation(relaxOutputSchema(schema));
118
119 // Relaxing `not`'s subschema would make these previously-valid values fail.
120 expect(strict.safeParse(null).success).toBe(true);
121 expect(relaxed.safeParse(null).success).toBe(true); // still valid, not narrowed
122 expect(relaxed.safeParse(42).success).toBe(true); // still valid
123 expect(relaxed.safeParse('str').success).toBe(false); // constraint preserved
124 });
125 });