@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
py 462 lines 14.8 KB
Raw
1 """Schema-supplied regexes must not stall validation (SEC-1178).
2
3 Tool schemas are third-party input, and the strings matched against their
4 patterns (tool arguments, object keys) are attacker-influenced. Every pattern
5 runs on pydantic's linear-time Rust regex, or on Python `re` only when a static
6 check proves it cannot backtrack catastrophically.
7
8 None of these tests may compare a catastrophic input against the stock
9 `jsonschema` oracle: that oracle runs Python `re` and would hang.
10 """
11
12 import time
13 import typing as t
14
15 import pytest
16 from jsonschema import Draft201909Validator, Draft202012Validator
17 from pydantic import TypeAdapter, ValidationError
18
19 from composio.utils import schema_converter
20 from composio.utils.schema_converter import json_schema_to_pydantic_type
21 from composio.utils.shared import json_schema_to_model
22
23 # Look-around forces the `re` fallback; the nested quantifier is exponential.
24 LOOKAROUND_CATASTROPHIC = "^(?=a)(a+)+$"
25 # The Rust engine compiles this, so it must be enforced in linear time.
26 RUST_CATASTROPHIC = "^(a+)+$"
27 EVIL = "a" * 40 + "!"
28 TIME_LIMIT = 1.0
29
30
31 class _Timer:
32 def __enter__(self) -> "_Timer":
33 self.started = time.perf_counter()
34 return self
35
36 def __exit__(self, *_: t.Any) -> None:
37 self.elapsed = time.perf_counter() - self.started
38
39
40 def _adapter(schema: t.Dict[str, t.Any]) -> TypeAdapter:
41 return TypeAdapter(json_schema_to_pydantic_type(schema))
42
43
44 @pytest.fixture
45 def warnings(monkeypatch: pytest.MonkeyPatch) -> t.List[str]:
46 """Capture converter warnings; clear the warn-once matcher cache."""
47 schema_converter._string_pattern_matcher.cache_clear()
48 messages: t.List[str] = []
49
50 def capture(message: str, *args: t.Any, **_: t.Any) -> None:
51 messages.append(message % args)
52
53 monkeypatch.setattr(schema_converter.logger, "warning", capture)
54 return messages
55
56
57 @pytest.mark.unit
58 @pytest.mark.schema
59 @pytest.mark.parametrize("entry_point", ["model", "type"])
60 def test_rust_supported_pattern_property_routes_keys_in_linear_time(
61 entry_point: str,
62 ) -> None:
63 schema = {
64 "type": "object",
65 "title": "Routed",
66 "patternProperties": {RUST_CATASTROPHIC: {"type": "integer"}},
67 "additionalProperties": {"type": "string"},
68 }
69 validate: t.Callable[[t.Any], t.Any]
70 if entry_point == "model":
71 model = json_schema_to_model(schema)
72 validate = model.model_validate
73 else:
74 validate = _adapter(schema).validate_python
75
76 with _Timer() as timer:
77 validate({"aaa": 1, EVIL: "routed to additionalProperties"})
78 with pytest.raises(ValidationError):
79 validate({"aaa": "matching key must be an integer"})
80 with pytest.raises(ValidationError):
81 validate({EVIL: 1})
82 assert timer.elapsed < TIME_LIMIT
83
84
85 @pytest.mark.unit
86 @pytest.mark.schema
87 def test_lookaround_catastrophic_pattern_is_not_enforced(
88 warnings: t.List[str],
89 ) -> None:
90 schema = {
91 "type": "object",
92 "properties": {"value": {"type": "string", "pattern": LOOKAROUND_CATASTROPHIC}},
93 }
94 model = json_schema_to_model(schema)
95 assert any(LOOKAROUND_CATASTROPHIC in message for message in warnings)
96
97 with _Timer() as timer:
98 model.model_validate({"value": EVIL})
99 assert timer.elapsed < TIME_LIMIT
100
101
102 @pytest.mark.unit
103 @pytest.mark.schema
104 def test_forced_fallback_polynomial_pattern_is_not_enforced(
105 warnings: t.List[str],
106 ) -> None:
107 pattern = "(?=a).*.*.*.*.*!"
108 assert not schema_converter._fallback_pattern_is_safe(pattern)
109 adapter = _adapter({"type": "string", "pattern": pattern})
110 assert any(pattern in message for message in warnings)
111
112 with _Timer() as timer:
113 assert adapter.validate_python("a" * 900) == "a" * 900
114 assert timer.elapsed < TIME_LIMIT
115
116
117 @pytest.mark.unit
118 @pytest.mark.schema
119 @pytest.mark.parametrize(
120 "pattern",
121 [
122 "^(?=a){1000000000}a$",
123 "^z|(?=a)" + "(?:a|aa)" * 5 + "a*a*b$",
124 ],
125 )
126 def test_fixed_repeats_and_unanchored_alternatives_are_not_enforced(
127 pattern: str,
128 warnings: t.List[str],
129 ) -> None:
130 # Assert rejection before matching, so a regression cannot execute the attack.
131 assert not schema_converter._fallback_pattern_is_safe(pattern)
132 adapter = _adapter({"type": "string", "pattern": pattern})
133 assert any(pattern in message for message in warnings)
134 value = "a" * schema_converter._BACKTRACKING_FALLBACK_MAX_INPUT
135 assert adapter.validate_python(value) == value
136
137
138 @pytest.mark.unit
139 @pytest.mark.schema
140 @pytest.mark.parametrize("entry_point", ["model", "type"])
141 @pytest.mark.parametrize("pattern", [LOOKAROUND_CATASTROPHIC, "^(?=a)"])
142 def test_lookaround_pattern_property_is_rejected_at_conversion(
143 entry_point: str,
144 pattern: str,
145 ) -> None:
146 # Key patterns decide routing, so they are never dropped and never run on
147 # Python `re`, even when the static check would accept them.
148 schema = {
149 "type": "object",
150 "title": "Keys",
151 "patternProperties": {pattern: {"type": "integer"}},
152 }
153 convert = json_schema_to_model if entry_point == "model" else _adapter
154 with pytest.raises(ValueError, match="Unsupported patternProperties"):
155 convert(schema)
156
157
158 @pytest.mark.unit
159 @pytest.mark.schema
160 def test_nested_lookaround_pattern_property_is_rejected_at_conversion() -> None:
161 schema = {
162 "type": "object",
163 "title": "Nested",
164 "anyOf": [
165 {"patternProperties": {LOOKAROUND_CATASTROPHIC: {"type": "integer"}}},
166 {"required": ["x"]},
167 ],
168 }
169 with pytest.raises(ValueError, match="Unsupported patternProperties"):
170 json_schema_to_model(schema)
171
172
173 @pytest.mark.unit
174 @pytest.mark.schema
175 def test_invalid_pattern_property_keeps_invalid_error() -> None:
176 schema = {"type": "object", "patternProperties": {"(": {"type": "integer"}}}
177 with pytest.raises(ValueError, match="Invalid patternProperties"):
178 _adapter(schema)
179
180
181 @pytest.mark.unit
182 @pytest.mark.schema
183 def test_safe_lookaround_pattern_is_enforced_on_bounded_input() -> None:
184 adapter = _adapter({"type": "string", "pattern": "^(?=.*[a-y])a"})
185 assert adapter.validate_python("ab") == "ab"
186 with pytest.raises(ValidationError):
187 adapter.validate_python("ba")
188 with pytest.raises(ValidationError):
189 adapter.validate_python("z" * 1000)
190
191 # Past the fallback input cap the constraint widens to "satisfied".
192 long_value = "z" * (schema_converter._BACKTRACKING_FALLBACK_MAX_INPUT + 1)
193 assert adapter.validate_python(long_value) == long_value
194
195
196 @pytest.mark.unit
197 @pytest.mark.schema
198 def test_typeless_catastrophic_patterns_complete_in_linear_time() -> None:
199 enforced = _adapter({"pattern": RUST_CATASTROPHIC, "minLength": 1})
200 ignored = _adapter({"pattern": LOOKAROUND_CATASTROPHIC, "minLength": 1})
201
202 with _Timer() as timer:
203 with pytest.raises(ValidationError):
204 enforced.validate_python(EVIL)
205 assert enforced.validate_python("aaa") == "aaa"
206 assert enforced.validate_python(1) == 1
207 assert ignored.validate_python(EVIL) == EVIL
208 assert timer.elapsed < TIME_LIMIT
209
210
211 @pytest.mark.unit
212 @pytest.mark.schema
213 @pytest.mark.parametrize(
214 "dynamic_keyword", ["patternProperties", "additionalProperties"]
215 )
216 def test_catastrophic_pattern_inside_dynamic_key_schema(dynamic_keyword: str) -> None:
217 def schema_for(pattern: str) -> t.Dict[str, t.Any]:
218 value_schema = {"type": "string", "pattern": pattern}
219 dynamic: t.Any = (
220 {"^k": value_schema}
221 if dynamic_keyword == "patternProperties"
222 else value_schema
223 )
224 return {"type": "object", "title": "Dynamic", dynamic_keyword: dynamic}
225
226 enforced = json_schema_to_model(schema_for(RUST_CATASTROPHIC))
227 ignored = json_schema_to_model(schema_for(LOOKAROUND_CATASTROPHIC))
228
229 with _Timer() as timer:
230 with pytest.raises(ValidationError):
231 enforced.model_validate({"k1": EVIL})
232 enforced.model_validate({"k1": "aaa"})
233 ignored.model_validate({"k1": EVIL})
234 assert timer.elapsed < TIME_LIMIT
235
236
237 @pytest.mark.unit
238 @pytest.mark.schema
239 def test_catastrophic_pattern_inside_any_of_branch() -> None:
240 def schema_for(pattern: str) -> t.Dict[str, t.Any]:
241 return {
242 "type": "object",
243 "properties": {
244 "value": {
245 "anyOf": [
246 {"type": "string", "pattern": pattern},
247 {"type": "integer"},
248 ]
249 }
250 },
251 }
252
253 enforced = json_schema_to_model(schema_for(RUST_CATASTROPHIC))
254 ignored = json_schema_to_model(schema_for(LOOKAROUND_CATASTROPHIC))
255
256 with _Timer() as timer:
257 with pytest.raises(ValidationError):
258 enforced.model_validate({"value": EVIL})
259 enforced.model_validate({"value": "aaa"})
260 enforced.model_validate({"value": 1})
261 ignored.model_validate({"value": EVIL})
262 assert timer.elapsed < TIME_LIMIT
263
264
265 @pytest.mark.unit
266 @pytest.mark.schema
267 @pytest.mark.parametrize(
268 ("validator_class", "dialect"),
269 [
270 (Draft201909Validator, "https://json-schema.org/draft/2019-09/schema"),
271 (Draft202012Validator, "https://json-schema.org/draft/2020-12/schema"),
272 ],
273 )
274 def test_unevaluated_properties_match_keys_in_linear_time(
275 validator_class: t.Any,
276 dialect: str,
277 ) -> None:
278 keyword = schema_converter._with_safe_regex_keywords(validator_class).VALIDATORS[
279 "unevaluatedProperties"
280 ]
281 # Fails if a jsonschema upgrade stops the key matching from being rebound.
282 assert keyword is not schema_converter._unsupported_unevaluated_properties
283
284 schema = {
285 "$schema": dialect,
286 "type": "object",
287 "additionalProperties": {
288 "type": "object",
289 "patternProperties": {RUST_CATASTROPHIC: {"type": "integer"}},
290 "unevaluatedProperties": False,
291 },
292 }
293 adapter = _adapter(schema)
294 with _Timer() as timer:
295 adapter.validate_python({"x": {"aaa": 1}})
296 with pytest.raises(ValidationError):
297 adapter.validate_python({"x": {EVIL: 1}})
298 assert timer.elapsed < TIME_LIMIT
299
300
301 @pytest.mark.unit
302 @pytest.mark.schema
303 def test_subschema_dialect_switch_keeps_linear_regex() -> None:
304 # jsonschema switches to its stock validator class for any subschema that
305 # declares `$schema`; the regex overrides must survive that switch.
306 schema = {
307 "type": "object",
308 "title": "DialectSwitch",
309 "properties": {
310 "value": {
311 "$schema": "http://json-schema.org/draft-07/schema#",
312 "type": "object",
313 "properties": {
314 "text": {"type": "string", "pattern": RUST_CATASTROPHIC}
315 },
316 "patternProperties": {RUST_CATASTROPHIC: {"type": "integer"}},
317 "additionalProperties": False,
318 }
319 },
320 }
321 model = json_schema_to_model(schema)
322 with _Timer() as timer:
323 model.model_validate({"value": {"aaa": 1, "text": "aaa"}})
324 with pytest.raises(ValidationError):
325 model.model_validate({"value": {EVIL: 1}})
326 with pytest.raises(ValidationError):
327 model.model_validate({"value": {"text": EVIL}})
328 assert timer.elapsed < TIME_LIMIT
329
330
331 @pytest.mark.unit
332 @pytest.mark.schema
333 @pytest.mark.parametrize("entry_point", ["model", "type"])
334 @pytest.mark.parametrize(
335 ("draft", "keyword"),
336 [
337 (draft, keyword)
338 for draft in ["2019-09", "2020-12"]
339 for keyword in [
340 "dependentSchemas",
341 "unevaluatedProperties",
342 "unevaluatedItems",
343 "prefixItems",
344 ]
345 if keyword != "prefixItems" or draft == "2020-12"
346 ],
347 )
348 def test_switched_dialect_rejects_unsupported_key_patterns_at_conversion(
349 entry_point: str,
350 draft: str,
351 keyword: str,
352 ) -> None:
353 unsupported = {
354 "type": "object",
355 "patternProperties": {"^(?=a)": {"type": "integer"}},
356 }
357 value: t.Any = unsupported
358 if keyword == "dependentSchemas":
359 value = {"trigger": unsupported}
360 elif keyword == "prefixItems":
361 value = [unsupported]
362 schema = {
363 "type": "object",
364 "properties": {
365 "value": {
366 "$schema": f"https://json-schema.org/draft/{draft}/schema",
367 "type": "array"
368 if keyword in {"prefixItems", "unevaluatedItems"}
369 else "object",
370 keyword: value,
371 }
372 },
373 }
374 convert = json_schema_to_model if entry_point == "model" else _adapter
375 with pytest.raises(ValueError, match="Unsupported patternProperties"):
376 convert(schema)
377
378
379 @pytest.mark.unit
380 @pytest.mark.schema
381 def test_draft7_keeps_unsupported_dialect_keywords_as_annotations() -> None:
382 schema = {
383 "type": "object",
384 "dependentSchemas": {
385 "trigger": {"patternProperties": {"^(?=a)": {"type": "integer"}}}
386 },
387 }
388 assert _adapter(schema).validate_python({"trigger": "annotation"}) == {
389 "trigger": "annotation"
390 }
391
392
393 @pytest.mark.unit
394 @pytest.mark.schema
395 def test_shared_reference_is_checked_under_each_effective_dialect() -> None:
396 schema = {
397 "type": "object",
398 "properties": {
399 "old": {"$ref": "#/x"},
400 "new": {
401 "$schema": "https://json-schema.org/draft/2020-12/schema",
402 "$ref": "#/x",
403 },
404 },
405 "x": {
406 "type": "object",
407 "dependentSchemas": {
408 "trigger": {"patternProperties": {"^(?=a)": {"type": "integer"}}}
409 },
410 },
411 }
412 with pytest.raises(ValueError, match="Unsupported patternProperties"):
413 _adapter(schema)
414
415
416 @pytest.mark.unit
417 @pytest.mark.schema
418 @pytest.mark.parametrize(
419 "pattern",
420 [
421 "^(?=.*[a-y])a",
422 "^(?=a){2}ab$",
423 "(?<=a)b",
424 r"^(?=.*[a-z])(?=.*\d)[A-Za-z\d]{8,}$",
425 "^(?=x)(jpg|png|gif)$",
426 ],
427 )
428 def test_static_check_accepts_bounded_patterns(pattern: str) -> None:
429 assert schema_converter._fallback_pattern_is_safe(pattern)
430
431
432 @pytest.mark.unit
433 @pytest.mark.schema
434 @pytest.mark.parametrize(
435 "pattern",
436 [
437 "(a+)+",
438 "(?=(a+)+)",
439 # sre folds `(a|b)` into a character class, which is linear; a
440 # multi-character alternation stays an alternation.
441 "(a|bc)*",
442 r"(a)\1",
443 "(?P<x>a)(?P=x)",
444 "(a)?(?(1)b|c)",
445 "(?=(?=a))",
446 ".*.*x",
447 "(?=.*a.*b)x",
448 "(?=a).*.*.*.*.*!",
449 "(?=a)" + "a?" * 7,
450 "(?<=)" + "(?:a|aa)" * 7 + "b",
451 "^(?=a){1000000000}a$",
452 "^(?=a)a{1000000000}$",
453 "^(?=a)a{1,1000000000}$",
454 "^(?=a)a{1000000000,}$",
455 "^((?=a){64}){64}a$",
456 "^(?=a)b|b+b+c",
457 "^z|(?=a)" + "(?:a|aa)" * 5 + "a*a*b$",
458 "(?m)^a*a*(?=b)b$",
459 ],
460 )
461 def test_static_check_rejects_backtracking_patterns(pattern: str) -> None:
462 assert not schema_converter._fallback_pattern_is_safe(pattern)