Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | /** |
| 2 | * Property-based null semantics for strict mode. |
| 3 | * |
| 4 | * The oracle is Ajv in Draft 2020-12 mode. Strict mode makes every optional |
| 5 | * parameter required and uses `null` to mean "omitted", so for every generated |
| 6 | * optional property: |
| 7 | * |
| 8 | * - the strict schema accepts `null` |
| 9 | * - the arguments left after `omitNullToolArguments` validate against the |
| 10 | * tool's own schema, and a `null` that schema accepts is kept |
| 11 | * - the strict schema accepts every non-null value the tool's schema accepts, |
| 12 | * and nothing more unless a `oneOf` was converted to `anyOf` |
| 13 | * - the rewrite is idempotent |
| 14 | * |
| 15 | * The Python counterpart is `python/tests/test_strict_schema_properties.py`; |
| 16 | * keep the generators and invariants aligned. |
| 17 | * |
| 18 | * Those generators never produce reference cycles: a cycle that consumes no |
| 19 | * input makes the oracle recurse forever. Cycles are checked separately against |
| 20 | * the least fixed point of the definitions, computed by plain iteration: a null |
| 21 | * is accepted only when a finite chain of branches proves it. |
| 22 | */ |
| 23 | import Ajv2020 from 'ajv/dist/2020'; |
| 24 | import fc from 'fast-check'; |
| 25 | import { describe, expect, it } from 'vitest'; |
| 26 | |
| 27 | import { omitNullToolArguments, toStrictJsonSchema } from '../../src/utils/jsonSchema'; |
| 28 | |
| 29 | type SchemaRecord = Record<string, unknown>; |
| 30 | |
| 31 | const TYPES = ['string', 'integer', 'boolean', 'null'] as const; |
| 32 | const LITERALS = ['asc', 'desc', 1, true, null] as const; |
| 33 | const SAMPLES = ['asc', 'desc', 'other', 1, 2, true, false] as const; |
| 34 | const DEFINITIONS: SchemaRecord = { |
| 35 | Direction: { type: 'string', enum: ['asc', 'desc'] }, |
| 36 | NullableDirection: { enum: ['asc', null] }, |
| 37 | Text: { type: 'string' }, |
| 38 | NullableText: { type: ['string', 'null'] }, |
| 39 | Config: { properties: { url: { type: 'string' } } }, |
| 40 | }; |
| 41 | // The rewrite types this definition as an object, so the strict schema rejects |
| 42 | // the non-object values the tool's schema lets through. |
| 43 | const TYPELESS_OBJECT_REF = '#/$defs/Config'; |
| 44 | // Compositions strict mode supports, and the ones only the tool's schema uses. |
| 45 | const STRICT_COMPOSITIONS = ['anyOf', 'oneOf'] as const; |
| 46 | const ALL_COMPOSITIONS = [...STRICT_COMPOSITIONS, 'allOf', 'not', 'if'] as const; |
| 47 | type Composition = (typeof ALL_COMPOSITIONS)[number]; |
| 48 | |
| 49 | const leafKeywords: fc.Arbitrary<SchemaRecord> = fc |
| 50 | .record({ |
| 51 | type: fc.oneof( |
| 52 | fc.constant(undefined), |
| 53 | fc.constantFrom(...TYPES), |
| 54 | fc.uniqueArray(fc.constantFrom(...TYPES), { minLength: 1, maxLength: 3 }) |
| 55 | ), |
| 56 | literal: fc.oneof( |
| 57 | { arbitrary: fc.constant<SchemaRecord>({}), weight: 2 }, |
| 58 | fc |
| 59 | .uniqueArray(fc.constantFrom(...LITERALS), { minLength: 1, maxLength: 3 }) |
| 60 | .map(values => ({ enum: values })), |
| 61 | fc.constantFrom(...LITERALS).map(value => ({ const: value })) |
| 62 | ), |
| 63 | ref: fc.oneof( |
| 64 | { arbitrary: fc.constant(undefined), weight: 3 }, |
| 65 | fc.constantFrom(...Object.keys(DEFINITIONS)).map(name => `#/$defs/${name}`) |
| 66 | ), |
| 67 | described: fc.boolean(), |
| 68 | }) |
| 69 | .map(({ type, literal, ref, described }) => ({ |
| 70 | ...(type === undefined ? {} : { type }), |
| 71 | ...literal, |
| 72 | ...(ref === undefined ? {} : { $ref: ref }), |
| 73 | ...(described ? { description: 'a parameter' } : {}), |
| 74 | })); |
| 75 | |
| 76 | function propertySchema( |
| 77 | compositions: readonly Composition[], |
| 78 | depth = 0 |
| 79 | ): fc.Arbitrary<SchemaRecord> { |
| 80 | if (depth >= 2) return leafKeywords; |
| 81 | const nested = propertySchema(compositions, depth + 1); |
| 82 | const branches = fc.array(nested, { minLength: 1, maxLength: 3 }); |
| 83 | const composition: Record<Composition, fc.Arbitrary<SchemaRecord>> = { |
| 84 | anyOf: branches.map(anyOf => ({ anyOf })), |
| 85 | oneOf: branches.map(oneOf => ({ oneOf })), |
| 86 | allOf: branches.map(allOf => ({ allOf })), |
| 87 | not: nested.map(not => ({ not })), |
| 88 | if: fc |
| 89 | .record({ if: nested, then: fc.option(nested), else: fc.option(nested) }) |
| 90 | .map(({ if: condition, then, else: otherwise }) => ({ |
| 91 | if: condition, |
| 92 | ...(then === null ? {} : { then }), |
| 93 | ...(otherwise === null ? {} : { else: otherwise }), |
| 94 | })), |
| 95 | }; |
| 96 | return fc |
| 97 | .tuple( |
| 98 | leafKeywords, |
| 99 | fc.oneof( |
| 100 | { arbitrary: fc.constant<SchemaRecord>({}), weight: 2 }, |
| 101 | ...compositions.map(keyword => composition[keyword]) |
| 102 | ) |
| 103 | ) |
| 104 | .map(([leaf, composed]) => ({ ...leaf, ...composed })); |
| 105 | } |
| 106 | |
| 107 | const toolSchema = (property: SchemaRecord): SchemaRecord => ({ |
| 108 | type: 'object', |
| 109 | properties: { value: property }, |
| 110 | $defs: structuredClone(DEFINITIONS), |
| 111 | }); |
| 112 | |
| 113 | /** Whether any node carries `keyword` (with `value`, when given). */ |
| 114 | const mentions = (node: unknown, keyword: string, value?: unknown): boolean => { |
| 115 | if (Array.isArray(node)) return node.some(item => mentions(item, keyword, value)); |
| 116 | if (typeof node !== 'object' || node === null) return false; |
| 117 | const record = node as SchemaRecord; |
| 118 | if (keyword in record && (value === undefined || record[keyword] === value)) return true; |
| 119 | return Object.values(record).some(child => mentions(child, keyword, value)); |
| 120 | }; |
| 121 | |
| 122 | const ajv = new Ajv2020({ strict: false }); |
| 123 | const validatorFor = (schema: SchemaRecord) => ajv.compile(schema); |
| 124 | const RUNS = 300; |
| 125 | const TIMEOUT_MS = 30_000; |
| 126 | |
| 127 | const LEAVES: readonly SchemaRecord[] = [{ type: 'null' }, { type: 'string' }, {}]; |
| 128 | const leaf = fc.constantFrom(...LEAVES).map(schema => ({ ...schema })); |
| 129 | |
| 130 | /** Definitions that reference each other freely, cycles included. */ |
| 131 | const referenceGraph: fc.Arbitrary<Record<string, SchemaRecord>> = fc |
| 132 | .integer({ min: 1, max: 7 }) |
| 133 | .chain(count => { |
| 134 | const names = Array.from({ length: count }, (_, index) => `d${index}`); |
| 135 | const reference = fc.constantFrom(...names).map(name => ({ $ref: `#/$defs/${name}` })); |
| 136 | const composed = fc |
| 137 | .record({ |
| 138 | keyword: fc.constantFrom('anyOf', 'allOf'), |
| 139 | references: fc.array(reference, { minLength: 1, maxLength: 3 }), |
| 140 | extra: fc.option(leaf), |
| 141 | }) |
| 142 | .map(({ keyword, references, extra }) => ({ |
| 143 | [keyword]: extra === null ? references : [...references, extra], |
| 144 | })); |
| 145 | const definition = fc.oneof({ arbitrary: leaf, weight: 1 }, { arbitrary: composed, weight: 4 }); |
| 146 | return fc |
| 147 | .tuple(...names.map(() => definition)) |
| 148 | .map(definitions => |
| 149 | Object.fromEntries(names.map((name, index) => [name, definitions[index]])) |
| 150 | ); |
| 151 | }); |
| 152 | |
| 153 | function leastFixedPoint(definitions: Record<string, SchemaRecord>): Record<string, boolean> { |
| 154 | const accepted: Record<string, boolean> = Object.fromEntries( |
| 155 | Object.keys(definitions).map(name => [name, false]) |
| 156 | ); |
| 157 | const holds = (node: SchemaRecord): boolean => { |
| 158 | if (typeof node.$ref === 'string') return accepted[node.$ref.slice('#/$defs/'.length)]; |
| 159 | if (Array.isArray(node.anyOf)) return node.anyOf.some(holds); |
| 160 | if (Array.isArray(node.allOf)) return node.allOf.every(holds); |
| 161 | return node.type !== 'string'; |
| 162 | }; |
| 163 | for (let changed = true; changed;) { |
| 164 | changed = false; |
| 165 | for (const [name, definition] of Object.entries(definitions)) { |
| 166 | if (!accepted[name] && holds(definition)) accepted[name] = changed = true; |
| 167 | } |
| 168 | } |
| 169 | return accepted; |
| 170 | } |
| 171 | |
| 172 | describe('strict-mode null semantics', () => { |
| 173 | it('cyclic references resolve to the least fixed point', { timeout: TIMEOUT_MS }, () => { |
| 174 | fc.assert( |
| 175 | fc.property(referenceGraph, definitions => { |
| 176 | const names = Object.keys(definitions); |
| 177 | const reference = (name: string) => ({ $ref: `#/$defs/${name}` }); |
| 178 | const source = { |
| 179 | type: 'object', |
| 180 | properties: Object.fromEntries(names.map(name => [name, reference(name)])), |
| 181 | $defs: definitions, |
| 182 | }; |
| 183 | const expected = leastFixedPoint(definitions); |
| 184 | const input = Object.fromEntries(names.map(name => [name, null])); |
| 185 | expect(omitNullToolArguments(input, source)).toEqual( |
| 186 | Object.fromEntries(names.filter(name => expected[name]).map(name => [name, null])) |
| 187 | ); |
| 188 | |
| 189 | // `allOf` makes the tool unsupported, but its properties are still widened. |
| 190 | const properties = toStrictJsonSchema(source).schema.properties as SchemaRecord; |
| 191 | for (const name of names) { |
| 192 | expect(properties[name]).toEqual( |
| 193 | expected[name] ? reference(name) : { anyOf: [reference(name), { type: 'null' }] } |
| 194 | ); |
| 195 | } |
| 196 | }), |
| 197 | { numRuns: RUNS } |
| 198 | ); |
| 199 | }); |
| 200 | |
| 201 | it('null stands for omission without changing other values', { timeout: TIMEOUT_MS }, () => { |
| 202 | fc.assert( |
| 203 | fc.property(propertySchema(STRICT_COMPOSITIONS), property => { |
| 204 | const source = toolSchema(property); |
| 205 | const snapshot = structuredClone(source); |
| 206 | const result = toStrictJsonSchema(source); |
| 207 | expect(source).toEqual(snapshot); |
| 208 | fc.pre(result.unsupported.length === 0); |
| 209 | |
| 210 | const original = validatorFor(source); |
| 211 | const strict = validatorFor(result.schema); |
| 212 | |
| 213 | expect(strict({ value: null })).toBe(true); |
| 214 | const sent = omitNullToolArguments({ value: null }, result.source); |
| 215 | expect(original(sent)).toBe(true); |
| 216 | expect(sent).toEqual(original({ value: null }) ? { value: null } : {}); |
| 217 | |
| 218 | if (!mentions(property, '$ref', TYPELESS_OBJECT_REF)) { |
| 219 | for (const sample of SAMPLES) { |
| 220 | const args = { value: sample }; |
| 221 | if (original(args)) expect(strict(args)).toBe(true); |
| 222 | else if (!mentions(property, 'oneOf')) expect(strict(args)).toBe(false); |
| 223 | } |
| 224 | } |
| 225 | |
| 226 | const again = toStrictJsonSchema(result.schema); |
| 227 | expect(again.schema).toEqual(result.schema); |
| 228 | expect(again.changes).toEqual([]); |
| 229 | }), |
| 230 | { numRuns: RUNS } |
| 231 | ); |
| 232 | }); |
| 233 | |
| 234 | it('null is kept exactly when the tool schema accepts it', { timeout: TIMEOUT_MS }, () => { |
| 235 | fc.assert( |
| 236 | fc.property(propertySchema(ALL_COMPOSITIONS), property => { |
| 237 | const source = toolSchema(property); |
| 238 | const accepted = validatorFor(source)({ value: null }); |
| 239 | expect(omitNullToolArguments({ value: null }, source)).toEqual( |
| 240 | accepted ? { value: null } : {} |
| 241 | ); |
| 242 | }), |
| 243 | { numRuns: RUNS } |
| 244 | ); |
| 245 | }); |
| 246 | }); |