@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
py 429 lines 14.4 KB
Raw
1 """Regression tests for URL file-upload SSRF protections."""
2
3 from __future__ import annotations
4
5 import io
6 import socket
7 import typing as t
8 from unittest.mock import MagicMock, call, patch
9
10 import pytest
11
12 from composio.exceptions import BlockedInternalUrlError
13 from composio.utils.url_safety import (
14 assert_safe_fetch_target,
15 is_blocked_ip,
16 safe_request,
17 )
18
19
20 @pytest.mark.parametrize(
21 "address",
22 [
23 "127.0.0.1",
24 "10.0.0.5",
25 "169.254.169.254",
26 "100.64.0.1",
27 "::1",
28 "fc00::1",
29 "::ffff:127.0.0.1",
30 "::127.0.0.1",
31 "::7f00:1",
32 "::169.254.169.254",
33 "64:ff9b::7f00:1",
34 "64:ff9b::a9fe:a9fe",
35 # Ranges the TypeScript guard's CIDR list blocks, so this one does too.
36 "224.0.0.1",
37 "233.252.0.1",
38 "192.88.99.1",
39 "ff02::1",
40 "fec0::1",
41 # Blocked here already; asserted so the two lists stay comparable.
42 "2001::7f00:1",
43 "2002:7f00:1::",
44 "2002:c0a8:1::",
45 "64:ff9b:1::7f00:1",
46 "100::1",
47 "2001:db8::1",
48 ],
49 )
50 def test_blocks_non_public_addresses(address: str) -> None:
51 assert is_blocked_ip(address) is True
52
53
54 @pytest.mark.parametrize(
55 "address",
56 [
57 "8.8.8.8",
58 "93.184.216.34",
59 "2606:4700:4700::1111",
60 "::8.8.8.8",
61 "64:ff9b::8.8.8.8",
62 # Neighbours of the blocked ranges above, so neither list overreaches.
63 "2001:4860:4860::8888",
64 "2a00:1450:4001:80f::200e",
65 "223.255.255.255",
66 "192.88.100.1",
67 ],
68 )
69 def test_allows_public_addresses(address: str) -> None:
70 assert is_blocked_ip(address) is False
71
72
73 @pytest.mark.parametrize(
74 "url",
75 [
76 "file:///etc/passwd",
77 "ftp://example.com/file",
78 "not a url",
79 "http://example.com:invalid/file",
80 ],
81 )
82 def test_rejects_malformed_or_non_http_urls(url: str) -> None:
83 with pytest.raises(BlockedInternalUrlError):
84 assert_safe_fetch_target(url)
85
86
87 @patch("composio.utils.url_safety.socket.getaddrinfo")
88 def test_validates_requests_canonicalized_hostname(mock_getaddrinfo) -> None:
89 def resolve(host: str, _port: int | None):
90 address = "127.0.0.1" if host == "127.0.0.1" else "93.184.216.34"
91 return [
92 (socket.AF_INET, socket.SOCK_STREAM, 6, "", (address, 0)),
93 ]
94
95 mock_getaddrinfo.side_effect = resolve
96
97 with pytest.raises(BlockedInternalUrlError):
98 assert_safe_fetch_target(r"http://127.0.0.1\@example.com/file.pdf")
99
100 mock_getaddrinfo.assert_called_once_with("127.0.0.1", None)
101
102
103 @patch("composio.utils.url_safety.socket.getaddrinfo")
104 def test_rejects_internal_dns_answers(mock_getaddrinfo) -> None:
105 mock_getaddrinfo.return_value = [
106 (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0)),
107 (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0)),
108 ]
109
110 with pytest.raises(BlockedInternalUrlError):
111 assert_safe_fetch_target("https://example.com/file.pdf")
112
113
114 @patch("composio.utils.url_safety.socket.getaddrinfo")
115 def test_allows_public_dns_answers(mock_getaddrinfo) -> None:
116 mock_getaddrinfo.return_value = [
117 (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0)),
118 ]
119
120 assert_safe_fetch_target("https://example.com/file.pdf")
121
122
123 def _response(status_code: int, location: str | None = None) -> MagicMock:
124 response = MagicMock()
125 response.status_code = status_code
126 response.headers = {"Location": location} if location else {}
127 response.close = MagicMock()
128 return response
129
130
131 @patch("composio.utils.url_safety.requests.Session.request")
132 @patch("composio.utils.url_safety.assert_safe_fetch_target")
133 def test_safe_request_validates_before_sending(mock_assert, mock_request) -> None:
134 mock_assert.side_effect = BlockedInternalUrlError("blocked")
135
136 with pytest.raises(BlockedInternalUrlError):
137 safe_request("PUT", "https://s3.example.com/upload", data=b"payload")
138
139 # The point of the guard: a rejected target must never reach the network.
140 mock_request.assert_not_called()
141
142
143 @patch("composio.utils.url_safety.requests.Session.request")
144 @patch("composio.utils.url_safety.assert_safe_fetch_target")
145 def test_safe_request_disables_automatic_redirects(mock_assert, mock_request) -> None:
146 mock_request.return_value = _response(200)
147
148 response = safe_request("PUT", "https://s3.example.com/upload", data=b"payload")
149
150 assert response.status_code == 200
151 mock_assert.assert_called_once_with("https://s3.example.com/upload")
152 # `requests` must not follow redirects on its own, or hops after the first
153 # would be fetched without ever being validated.
154 assert mock_request.call_args.kwargs["allow_redirects"] is False
155
156
157 @patch("composio.utils.url_safety.requests.Session.request")
158 @patch("composio.utils.url_safety.assert_safe_fetch_target")
159 def test_safe_request_revalidates_each_redirect_hop(mock_assert, mock_request) -> None:
160 """A public URL that redirects into private space must be caught at the hop."""
161 mock_assert.side_effect = [["93.184.216.34"], BlockedInternalUrlError("blocked")]
162 mock_request.return_value = _response(
163 307, "http://169.254.169.254/latest/meta-data"
164 )
165
166 with pytest.raises(BlockedInternalUrlError):
167 safe_request("PUT", "https://s3.example.com/upload", data=b"payload")
168
169 assert mock_assert.call_args_list == [
170 call("https://s3.example.com/upload"),
171 call("http://169.254.169.254/latest/meta-data"),
172 ]
173 # Only the first hop was sent; the redirect target never was.
174 assert mock_request.call_count == 1
175
176
177 @patch("composio.utils.url_safety.requests.Session.request")
178 @patch("composio.utils.url_safety.assert_safe_fetch_target")
179 def test_safe_request_follows_validated_redirect(mock_assert, mock_request) -> None:
180 """S3 can answer a PUT with a 307 region redirect; that must still work."""
181 mock_request.side_effect = [
182 _response(307, "https://s3.eu-west-1.example.com/upload"),
183 _response(200),
184 ]
185 body = io.BytesIO(b"payload")
186
187 response = safe_request("PUT", "https://s3.example.com/upload", data=body)
188
189 assert response.status_code == 200
190 assert mock_assert.call_args_list == [
191 call("https://s3.example.com/upload"),
192 call("https://s3.eu-west-1.example.com/upload"),
193 ]
194 # The body was rewound, so the retried hop sends the payload rather than
195 # an already-exhausted stream.
196 assert body.read() == b"payload"
197
198
199 # What the hop after a redirect is expected to carry: either the original body
200 # and the headers describing it, or neither.
201 _REPLAYED: t.Dict[str, t.Any] = {
202 "data": b"payload",
203 "headers": {"Content-Type": "application/octet-stream", "X-Test": "kept"},
204 }
205 _BODILESS: t.Dict[str, t.Any] = {"headers": {"X-Test": "kept"}}
206
207
208 @pytest.mark.parametrize(
209 ("status_code", "method", "expected_method", "expected_kwargs"),
210 [
211 (301, "POST", "GET", _BODILESS),
212 (301, "PUT", "PUT", _REPLAYED),
213 (302, "POST", "GET", _BODILESS),
214 (302, "PUT", "PUT", _REPLAYED),
215 (303, "POST", "GET", _BODILESS),
216 (303, "PUT", "GET", _BODILESS),
217 (303, "HEAD", "HEAD", _BODILESS),
218 (307, "POST", "POST", _REPLAYED),
219 (308, "PUT", "PUT", _REPLAYED),
220 ],
221 )
222 @patch("composio.utils.url_safety.requests.Session.request")
223 @patch("composio.utils.url_safety.assert_safe_fetch_target")
224 def test_safe_request_applies_fetch_redirect_semantics(
225 mock_assert,
226 mock_request,
227 status_code: int,
228 method: str,
229 expected_method: str,
230 expected_kwargs: t.Dict[str, t.Any],
231 ) -> None:
232 """The same rules `ssrfSafeFetch` gets from `fetch` in the TypeScript SDK.
233
234 A 303 sends every method to a bodiless result request, 301/302 do that to a
235 POST only, and 307/308 replay the original method and body.
236 """
237 mock_request.side_effect = [
238 _response(status_code, "https://example.com/result"),
239 _response(200),
240 ]
241
242 safe_request(method, "https://example.com/create", **_REPLAYED)
243
244 assert mock_request.call_args_list[1] == call(
245 expected_method,
246 "https://example.com/result",
247 allow_redirects=False,
248 **expected_kwargs,
249 )
250 assert mock_assert.call_args_list[1] == call("https://example.com/result")
251
252
253 @patch("composio.utils.url_safety.requests.Session.request")
254 @patch("composio.utils.url_safety.assert_safe_fetch_target")
255 def test_safe_request_keeps_the_downgrade_across_later_hops(
256 mock_assert, mock_request
257 ) -> None:
258 """A hop after the 303 must not resurrect the method or the body."""
259 mock_request.side_effect = [
260 _response(303, "https://example.com/result"),
261 _response(307, "https://example.com/final"),
262 _response(200),
263 ]
264
265 safe_request("POST", "https://example.com/create", **_REPLAYED)
266
267 assert mock_request.call_args_list[2] == call(
268 "GET",
269 "https://example.com/final",
270 allow_redirects=False,
271 **_BODILESS,
272 )
273
274
275 @patch("composio.utils.url_safety.requests.Session.request")
276 @patch("composio.utils.url_safety.assert_safe_fetch_target")
277 def test_safe_request_does_not_reapply_params_to_the_redirect_target(
278 mock_assert, mock_request
279 ) -> None:
280 """`Location` carries its own query; re-appending would leak the original."""
281 mock_request.side_effect = [
282 _response(307, "https://other.example.com/elsewhere"),
283 _response(200),
284 ]
285
286 safe_request("GET", "https://example.com/download", params={"token": "secret"})
287
288 assert mock_request.call_args_list[1] == call(
289 "GET",
290 "https://other.example.com/elsewhere",
291 allow_redirects=False,
292 )
293
294
295 _CREDENTIALED: t.Dict[str, str] = {
296 "Authorization": "Bearer token",
297 "Proxy-Authorization": "Basic cHJveHk=",
298 "Cookie": "session=abc",
299 "X-Test": "kept",
300 }
301
302
303 @pytest.mark.parametrize(
304 "location",
305 [
306 "https://other.example.com/elsewhere", # different host
307 "http://example.com/elsewhere", # different scheme
308 "https://example.com:8443/elsewhere", # different port
309 ],
310 )
311 @patch("composio.utils.url_safety.requests.Session.request")
312 @patch("composio.utils.url_safety.assert_safe_fetch_target")
313 def test_safe_request_drops_credentials_on_cross_origin_redirect(
314 mock_assert, mock_request, location: str
315 ) -> None:
316 """Manual redirects bypass `requests`' `rebuild_auth`, so strip here.
317
318 A credential header is addressed to the origin the caller named; replaying
319 it to another origin hands the bearer token to whoever answered the
320 redirect. Mirrors the Fetch rule `ssrfSafeFetch` applies.
321 """
322 mock_request.side_effect = [_response(307, location), _response(200)]
323
324 safe_request("GET", "https://example.com/download", headers=dict(_CREDENTIALED))
325
326 assert mock_request.call_args_list[1] == call(
327 "GET",
328 location,
329 allow_redirects=False,
330 headers={"X-Test": "kept"},
331 )
332
333
334 @patch("composio.utils.url_safety.requests.Session.request")
335 @patch("composio.utils.url_safety.assert_safe_fetch_target")
336 def test_safe_request_keeps_credentials_on_same_origin_redirect(
337 mock_assert, mock_request
338 ) -> None:
339 mock_request.side_effect = [
340 _response(307, "https://example.com:443/moved"),
341 _response(200),
342 ]
343
344 safe_request("GET", "https://example.com/download", headers=dict(_CREDENTIALED))
345
346 assert mock_request.call_args_list[1] == call(
347 "GET",
348 "https://example.com:443/moved",
349 allow_redirects=False,
350 headers=_CREDENTIALED,
351 )
352
353
354 @patch("composio.utils.url_safety.requests.Session.request")
355 @patch("composio.utils.url_safety.assert_safe_fetch_target")
356 def test_safe_request_strips_credentials_case_insensitively(
357 mock_assert, mock_request
358 ) -> None:
359 mock_request.side_effect = [
360 _response(302, "https://other.example.com/elsewhere"),
361 _response(200),
362 ]
363
364 safe_request(
365 "GET",
366 "https://example.com/download",
367 headers={"AUTHORIZATION": "Bearer token", "cookie": "a=b", "X-Test": "kept"},
368 )
369
370 assert mock_request.call_args_list[1].kwargs["headers"] == {"X-Test": "kept"}
371
372
373 @pytest.mark.parametrize(
374 "location",
375 [
376 "https://example.com:99999/elsewhere", # port out of range
377 "https://[::1/elsewhere", # broken IPv6 literal
378 ],
379 )
380 @patch("composio.utils.url_safety.requests.Session.request")
381 @patch("composio.utils.url_safety.assert_safe_fetch_target")
382 def test_safe_request_unparseable_redirect_is_blocked_not_a_crash(
383 mock_assert, mock_request, location: str
384 ) -> None:
385 """`urljoin` / `urlparse(...).port` raise `ValueError` on these.
386
387 Both run on the `Location` before the target is validated, so they must
388 not turn remote input into a stray `ValueError` that call sites catching
389 only `RequestException` would let escape. The hop is rejected the way any
390 malformed URL is, and never sent.
391 """
392 mock_assert.side_effect = [["93.184.216.34"], BlockedInternalUrlError("blocked")]
393 mock_request.return_value = _response(307, location)
394
395 with pytest.raises(BlockedInternalUrlError):
396 safe_request("GET", "https://example.com/download", headers=dict(_CREDENTIALED))
397
398 assert mock_request.call_count == 1
399
400
401 def test_origin_is_none_for_unparseable_urls() -> None:
402 from composio.utils.url_safety import _origin, _same_origin
403
404 assert _origin("https://example.com:99999/x") is None
405 assert _origin("https://[::1/x") is None
406 assert _origin("https://example.com/x") == ("https", "example.com", 443)
407 assert not _same_origin("https://example.com/a", "https://example.com:99999/b")
408 assert _same_origin("https://example.com/a", "https://example.com:443/b")
409
410
411 @patch("composio.utils.url_safety.requests.Session.request")
412 @patch("composio.utils.url_safety.assert_safe_fetch_target")
413 def test_safe_request_relative_redirect_is_resolved(mock_assert, mock_request) -> None:
414 mock_request.side_effect = [_response(302, "/elsewhere"), _response(200)]
415
416 safe_request("GET", "https://files.example.com/a/b")
417
418 assert mock_assert.call_args_list[1] == call("https://files.example.com/elsewhere")
419
420
421 @patch("composio.utils.url_safety.requests.Session.request")
422 @patch("composio.utils.url_safety.assert_safe_fetch_target")
423 def test_safe_request_rejects_endless_redirects(mock_assert, mock_request) -> None:
424 mock_request.return_value = _response(302, "https://s3.example.com/upload")
425
426 with pytest.raises(BlockedInternalUrlError, match="too many redirects"):
427 safe_request("GET", "https://s3.example.com/upload", max_redirects=2)
428
429 assert mock_request.call_count == 3