Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 246 lines 9.73 KB
Raw
1 /**
2 * Property-based null semantics for strict mode.
3 *
4 * The oracle is Ajv in Draft 2020-12 mode. Strict mode makes every optional
5 * parameter required and uses `null` to mean "omitted", so for every generated
6 * optional property:
7 *
8 * - the strict schema accepts `null`
9 * - the arguments left after `omitNullToolArguments` validate against the
10 * tool's own schema, and a `null` that schema accepts is kept
11 * - the strict schema accepts every non-null value the tool's schema accepts,
12 * and nothing more unless a `oneOf` was converted to `anyOf`
13 * - the rewrite is idempotent
14 *
15 * The Python counterpart is `python/tests/test_strict_schema_properties.py`;
16 * keep the generators and invariants aligned.
17 *
18 * Those generators never produce reference cycles: a cycle that consumes no
19 * input makes the oracle recurse forever. Cycles are checked separately against
20 * the least fixed point of the definitions, computed by plain iteration: a null
21 * is accepted only when a finite chain of branches proves it.
22 */
23 import Ajv2020 from 'ajv/dist/2020';
24 import fc from 'fast-check';
25 import { describe, expect, it } from 'vitest';
26
27 import { omitNullToolArguments, toStrictJsonSchema } from '../../src/utils/jsonSchema';
28
29 type SchemaRecord = Record<string, unknown>;
30
31 const TYPES = ['string', 'integer', 'boolean', 'null'] as const;
32 const LITERALS = ['asc', 'desc', 1, true, null] as const;
33 const SAMPLES = ['asc', 'desc', 'other', 1, 2, true, false] as const;
34 const DEFINITIONS: SchemaRecord = {
35 Direction: { type: 'string', enum: ['asc', 'desc'] },
36 NullableDirection: { enum: ['asc', null] },
37 Text: { type: 'string' },
38 NullableText: { type: ['string', 'null'] },
39 Config: { properties: { url: { type: 'string' } } },
40 };
41 // The rewrite types this definition as an object, so the strict schema rejects
42 // the non-object values the tool's schema lets through.
43 const TYPELESS_OBJECT_REF = '#/$defs/Config';
44 // Compositions strict mode supports, and the ones only the tool's schema uses.
45 const STRICT_COMPOSITIONS = ['anyOf', 'oneOf'] as const;
46 const ALL_COMPOSITIONS = [...STRICT_COMPOSITIONS, 'allOf', 'not', 'if'] as const;
47 type Composition = (typeof ALL_COMPOSITIONS)[number];
48
49 const leafKeywords: fc.Arbitrary<SchemaRecord> = fc
50 .record({
51 type: fc.oneof(
52 fc.constant(undefined),
53 fc.constantFrom(...TYPES),
54 fc.uniqueArray(fc.constantFrom(...TYPES), { minLength: 1, maxLength: 3 })
55 ),
56 literal: fc.oneof(
57 { arbitrary: fc.constant<SchemaRecord>({}), weight: 2 },
58 fc
59 .uniqueArray(fc.constantFrom(...LITERALS), { minLength: 1, maxLength: 3 })
60 .map(values => ({ enum: values })),
61 fc.constantFrom(...LITERALS).map(value => ({ const: value }))
62 ),
63 ref: fc.oneof(
64 { arbitrary: fc.constant(undefined), weight: 3 },
65 fc.constantFrom(...Object.keys(DEFINITIONS)).map(name => `#/$defs/${name}`)
66 ),
67 described: fc.boolean(),
68 })
69 .map(({ type, literal, ref, described }) => ({
70 ...(type === undefined ? {} : { type }),
71 ...literal,
72 ...(ref === undefined ? {} : { $ref: ref }),
73 ...(described ? { description: 'a parameter' } : {}),
74 }));
75
76 function propertySchema(
77 compositions: readonly Composition[],
78 depth = 0
79 ): fc.Arbitrary<SchemaRecord> {
80 if (depth >= 2) return leafKeywords;
81 const nested = propertySchema(compositions, depth + 1);
82 const branches = fc.array(nested, { minLength: 1, maxLength: 3 });
83 const composition: Record<Composition, fc.Arbitrary<SchemaRecord>> = {
84 anyOf: branches.map(anyOf => ({ anyOf })),
85 oneOf: branches.map(oneOf => ({ oneOf })),
86 allOf: branches.map(allOf => ({ allOf })),
87 not: nested.map(not => ({ not })),
88 if: fc
89 .record({ if: nested, then: fc.option(nested), else: fc.option(nested) })
90 .map(({ if: condition, then, else: otherwise }) => ({
91 if: condition,
92 ...(then === null ? {} : { then }),
93 ...(otherwise === null ? {} : { else: otherwise }),
94 })),
95 };
96 return fc
97 .tuple(
98 leafKeywords,
99 fc.oneof(
100 { arbitrary: fc.constant<SchemaRecord>({}), weight: 2 },
101 ...compositions.map(keyword => composition[keyword])
102 )
103 )
104 .map(([leaf, composed]) => ({ ...leaf, ...composed }));
105 }
106
107 const toolSchema = (property: SchemaRecord): SchemaRecord => ({
108 type: 'object',
109 properties: { value: property },
110 $defs: structuredClone(DEFINITIONS),
111 });
112
113 /** Whether any node carries `keyword` (with `value`, when given). */
114 const mentions = (node: unknown, keyword: string, value?: unknown): boolean => {
115 if (Array.isArray(node)) return node.some(item => mentions(item, keyword, value));
116 if (typeof node !== 'object' || node === null) return false;
117 const record = node as SchemaRecord;
118 if (keyword in record && (value === undefined || record[keyword] === value)) return true;
119 return Object.values(record).some(child => mentions(child, keyword, value));
120 };
121
122 const ajv = new Ajv2020({ strict: false });
123 const validatorFor = (schema: SchemaRecord) => ajv.compile(schema);
124 const RUNS = 300;
125 const TIMEOUT_MS = 30_000;
126
127 const LEAVES: readonly SchemaRecord[] = [{ type: 'null' }, { type: 'string' }, {}];
128 const leaf = fc.constantFrom(...LEAVES).map(schema => ({ ...schema }));
129
130 /** Definitions that reference each other freely, cycles included. */
131 const referenceGraph: fc.Arbitrary<Record<string, SchemaRecord>> = fc
132 .integer({ min: 1, max: 7 })
133 .chain(count => {
134 const names = Array.from({ length: count }, (_, index) => `d${index}`);
135 const reference = fc.constantFrom(...names).map(name => ({ $ref: `#/$defs/${name}` }));
136 const composed = fc
137 .record({
138 keyword: fc.constantFrom('anyOf', 'allOf'),
139 references: fc.array(reference, { minLength: 1, maxLength: 3 }),
140 extra: fc.option(leaf),
141 })
142 .map(({ keyword, references, extra }) => ({
143 [keyword]: extra === null ? references : [...references, extra],
144 }));
145 const definition = fc.oneof({ arbitrary: leaf, weight: 1 }, { arbitrary: composed, weight: 4 });
146 return fc
147 .tuple(...names.map(() => definition))
148 .map(definitions =>
149 Object.fromEntries(names.map((name, index) => [name, definitions[index]]))
150 );
151 });
152
153 function leastFixedPoint(definitions: Record<string, SchemaRecord>): Record<string, boolean> {
154 const accepted: Record<string, boolean> = Object.fromEntries(
155 Object.keys(definitions).map(name => [name, false])
156 );
157 const holds = (node: SchemaRecord): boolean => {
158 if (typeof node.$ref === 'string') return accepted[node.$ref.slice('#/$defs/'.length)];
159 if (Array.isArray(node.anyOf)) return node.anyOf.some(holds);
160 if (Array.isArray(node.allOf)) return node.allOf.every(holds);
161 return node.type !== 'string';
162 };
163 for (let changed = true; changed;) {
164 changed = false;
165 for (const [name, definition] of Object.entries(definitions)) {
166 if (!accepted[name] && holds(definition)) accepted[name] = changed = true;
167 }
168 }
169 return accepted;
170 }
171
172 describe('strict-mode null semantics', () => {
173 it('cyclic references resolve to the least fixed point', { timeout: TIMEOUT_MS }, () => {
174 fc.assert(
175 fc.property(referenceGraph, definitions => {
176 const names = Object.keys(definitions);
177 const reference = (name: string) => ({ $ref: `#/$defs/${name}` });
178 const source = {
179 type: 'object',
180 properties: Object.fromEntries(names.map(name => [name, reference(name)])),
181 $defs: definitions,
182 };
183 const expected = leastFixedPoint(definitions);
184 const input = Object.fromEntries(names.map(name => [name, null]));
185 expect(omitNullToolArguments(input, source)).toEqual(
186 Object.fromEntries(names.filter(name => expected[name]).map(name => [name, null]))
187 );
188
189 // `allOf` makes the tool unsupported, but its properties are still widened.
190 const properties = toStrictJsonSchema(source).schema.properties as SchemaRecord;
191 for (const name of names) {
192 expect(properties[name]).toEqual(
193 expected[name] ? reference(name) : { anyOf: [reference(name), { type: 'null' }] }
194 );
195 }
196 }),
197 { numRuns: RUNS }
198 );
199 });
200
201 it('null stands for omission without changing other values', { timeout: TIMEOUT_MS }, () => {
202 fc.assert(
203 fc.property(propertySchema(STRICT_COMPOSITIONS), property => {
204 const source = toolSchema(property);
205 const snapshot = structuredClone(source);
206 const result = toStrictJsonSchema(source);
207 expect(source).toEqual(snapshot);
208 fc.pre(result.unsupported.length === 0);
209
210 const original = validatorFor(source);
211 const strict = validatorFor(result.schema);
212
213 expect(strict({ value: null })).toBe(true);
214 const sent = omitNullToolArguments({ value: null }, result.source);
215 expect(original(sent)).toBe(true);
216 expect(sent).toEqual(original({ value: null }) ? { value: null } : {});
217
218 if (!mentions(property, '$ref', TYPELESS_OBJECT_REF)) {
219 for (const sample of SAMPLES) {
220 const args = { value: sample };
221 if (original(args)) expect(strict(args)).toBe(true);
222 else if (!mentions(property, 'oneOf')) expect(strict(args)).toBe(false);
223 }
224 }
225
226 const again = toStrictJsonSchema(result.schema);
227 expect(again.schema).toEqual(result.schema);
228 expect(again.changes).toEqual([]);
229 }),
230 { numRuns: RUNS }
231 );
232 });
233
234 it('null is kept exactly when the tool schema accepts it', { timeout: TIMEOUT_MS }, () => {
235 fc.assert(
236 fc.property(propertySchema(ALL_COMPOSITIONS), property => {
237 const source = toolSchema(property);
238 const accepted = validatorFor(source)({ value: null });
239 expect(omitNullToolArguments({ value: null }, source)).toEqual(
240 accepted ? { value: null } : {}
241 );
242 }),
243 { numRuns: RUNS }
244 );
245 });
246 });