Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
py 473 lines 19.4 KB
Raw
1 """SSRF protections for URLs the SDK fetches.
2
3 Applies to URLs the caller supplies *and* to URLs an API response supplies:
4 under the trust boundary documented in ``python/AGENTS.md`` the backend may be
5 compromised or the connection MITM'd, so a presigned URL in a response is no
6 more trusted than one typed by a user. Without a guard, either can point the
7 SDK at loopback, RFC1918 space, or a link-local cloud-metadata endpoint
8 (``169.254.169.254``) and turn it into a request proxy for internal
9 infrastructure.
10
11 Validating a hostname is not enough on its own, because the hostname is
12 resolved twice: once here, and once by the HTTP client when it opens the
13 socket. A short-TTL record can answer with a public address for the first
14 lookup and an internal one for the second — a time-of-check/time-of-use
15 window better known as DNS rebinding. So the address validated here is also
16 the address connected to: :func:`safe_get` and :func:`safe_request` pin it
17 onto the connection, keeping the original hostname for the ``Host`` header
18 and TLS SNI/certificate verification. Every fetch in the SDK goes through one
19 of those two, so no call site can reintroduce the gap by calling
20 ``requests.get`` next to a bare check. The TypeScript guard pins the same way.
21
22 Also parses response headers that gate how much of a body the SDK reads:
23 ``parse_content_length`` treats ``Content-Length`` as the untrusted hint it
24 is, so a malformed value degrades to an unknown size under a streamed byte
25 count instead of crashing the fetch.
26 """
27
28 from __future__ import annotations
29
30 import ipaddress
31 import socket
32 import typing as t
33 from urllib.parse import urljoin, urlparse
34
35 import requests
36 import urllib3.exceptions
37
38 from composio.exceptions import BlockedInternalUrlError
39
40 # urllib3 rewraps connect failures into its own hierarchy, none of which
41 # inherits from OSError, so a bare `except OSError` would never see them and
42 # the address fallback below would never run.
43 _CONNECT_ERRORS = (
44 OSError,
45 urllib3.exceptions.NewConnectionError,
46 urllib3.exceptions.ConnectTimeoutError,
47 urllib3.exceptions.NameResolutionError,
48 )
49
50 # Ranges that must not be reachable from a user-supplied URL but that
51 # ``ipaddress.is_global`` does not reject on its own, because it only asks
52 # whether an address is private. The TypeScript guard blocks the same ones from
53 # its explicit CIDR list.
54 _ALSO_BLOCKED_NETWORKS = (
55 ipaddress.ip_network("224.0.0.0/4"), # IPv4 multicast
56 ipaddress.ip_network("192.88.99.0/24"), # 6to4 relay anycast (RFC 7526)
57 ipaddress.ip_network("ff00::/8"), # IPv6 multicast
58 ipaddress.ip_network("fec0::/10"), # IPv6 site-local (deprecated)
59 )
60
61 _REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
62 # Headers that describe a request body, so they have to go when the body does.
63 # The Fetch standard's "request-body-header name" set, plus the two `requests`
64 # purges for the same reason (it recomputes both from the body it sends).
65 _BODY_HEADERS = frozenset(
66 {
67 "content-encoding",
68 "content-language",
69 "content-length",
70 "content-location",
71 "content-type",
72 "transfer-encoding",
73 }
74 )
75 # Headers that carry a credential for the origin the request was addressed
76 # to, so they must not follow a redirect to a different origin. The Fetch
77 # standard strips ``Authorization`` on a cross-origin redirect; ``Cookie`` and
78 # ``Proxy-Authorization`` go with it the way ``requests``' own ``rebuild_auth``
79 # and curl drop them on a host change. Following redirects by hand bypasses
80 # ``rebuild_auth``, so the rule is applied here. The TypeScript guard drops the
81 # same three (``CREDENTIAL_HEADERS``).
82 _CREDENTIAL_HEADERS = frozenset({"authorization", "cookie", "proxy-authorization"})
83 _MAX_REDIRECTS = 5
84
85
86 def _origin(url: str) -> t.Optional[t.Tuple[str, str, int]]:
87 """The ``(scheme, host, port)`` triple two URLs must share to be same-origin.
88
89 ``None`` when the URL cannot be parsed: ``urlparse`` raises ``ValueError``
90 for a broken IPv6 literal, and ``.port`` for an out-of-range port. A
91 redirect ``Location`` is remote input, so that is not an error here: the
92 hop is treated as leaving the origin, and :func:`assert_safe_fetch_target`
93 rejects the URL before anything is sent.
94 """
95 try:
96 parsed = urlparse(url)
97 scheme = parsed.scheme.lower()
98 host = (parsed.hostname or "").lower()
99 port = parsed.port
100 except ValueError:
101 return None
102 if port is None:
103 port = 443 if scheme == "https" else 80
104 return scheme, host, port
105
106
107 def _same_origin(previous_url: str, next_url: str) -> bool:
108 """Whether a redirect hop stays on the origin the caller addressed."""
109 previous_origin = _origin(previous_url)
110 return previous_origin is not None and previous_origin == _origin(next_url)
111
112
113 def is_blocked_ip(value: str) -> bool:
114 """Return whether an address is non-publicly-routable."""
115 try:
116 address = ipaddress.ip_address(value)
117 except ValueError:
118 return True
119
120 if isinstance(address, ipaddress.IPv6Address):
121 embedded_ipv4 = address.ipv4_mapped
122 if embedded_ipv4 is None and address.packed[:12] in {
123 b"\x00" * 12,
124 b"\x00d\xff\x9b" + b"\x00" * 8,
125 }:
126 embedded_ipv4 = ipaddress.IPv4Address(address.packed[-4:])
127 if embedded_ipv4 is not None:
128 return is_blocked_ip(str(embedded_ipv4))
129
130 if any(address in network for network in _ALSO_BLOCKED_NETWORKS):
131 return True
132
133 return not address.is_global
134
135
136 def assert_safe_fetch_target(url: str) -> t.List[str]:
137 """Refuse non-HTTP(S) URLs and hosts that resolve to internal addresses.
138
139 Parse the URL after Requests prepares it so validation uses the same
140 canonical hostname that the eventual connection will use.
141
142 :returns: The validated addresses to connect to, in resolver order.
143 Callers must connect to *these* rather than re-resolving the hostname;
144 see :func:`safe_get`.
145 """
146 try:
147 prepared_url = requests.Request(method="GET", url=url).prepare().url
148 if prepared_url is None:
149 raise ValueError("Prepared URL is missing")
150 parsed = urlparse(prepared_url)
151 if parsed.scheme not in {"http", "https"} or not parsed.hostname:
152 raise ValueError("URL must use HTTP(S) and include a hostname")
153 except (requests.exceptions.RequestException, ValueError):
154 raise BlockedInternalUrlError(
155 "Refusing to fetch a malformed or non-http(s) URL"
156 ) from None
157
158 try:
159 # Resolver order is kept: it encodes the system's address preference
160 # (RFC 6724), and connecting walks it the way urllib3 would.
161 addresses: t.List[str] = []
162 for result in socket.getaddrinfo(parsed.hostname, None):
163 address = result[4][0]
164 if not isinstance(address, str):
165 raise BlockedInternalUrlError(
166 f'Could not resolve host "{parsed.hostname}"'
167 )
168 if address not in addresses:
169 addresses.append(address)
170 except socket.gaierror as error:
171 raise BlockedInternalUrlError(
172 f'Could not resolve host "{parsed.hostname}"'
173 ) from error
174
175 for address in addresses:
176 if is_blocked_ip(address):
177 raise BlockedInternalUrlError(
178 f'Refusing to fetch "{parsed.hostname}" because it resolves to a non-public address'
179 )
180
181 if not addresses:
182 raise BlockedInternalUrlError(f'Could not resolve host "{parsed.hostname}"')
183
184 return addresses
185
186
187 def parse_content_length(value: t.Optional[str]) -> t.Optional[int]:
188 """Parse a ``Content-Length`` header into a non-negative ``int``.
189
190 ``Content-Length`` is supplied by the remote server and is therefore
191 untrusted: it may be absent, non-numeric (``"abc"``), fractional
192 (``"12.5"``), thousands-separated (``"1,024"``) or negative. Anything
193 untrustworthy returns ``None`` so the caller treats the size as unknown
194 and falls through to a streamed byte count, which stays authoritative
195 because the header can also be absent or understated. Mirrors
196 ``readResponseBodyWithLimit`` in the TypeScript SDK, which only trusts
197 values matching ``/^\\d+$/``.
198 """
199 if value is None:
200 return None
201 try:
202 size = int(value.strip())
203 except ValueError:
204 return None
205 return size if size >= 0 else None
206
207
208 def _redirect_rewrite(status_code: int, method: str) -> t.Optional[str]:
209 """The method the hop after a redirect uses, or ``None`` to replay as-is.
210
211 A returned method also means the request body goes: these are the Fetch
212 standard's redirect rules, which the TypeScript guard applies too and which
213 following redirects by hand means applying by hand. ``303`` points at a
214 result URL that has no use for the original body, so every method loses it
215 and everything but ``HEAD`` becomes a ``GET``; ``301``/``302`` do the same
216 to a ``POST`` only; ``307``/``308`` replay both.
217
218 ``requests`` would have applied its own rules in ``resolve_redirects``, and
219 they are not quite these: it downgrades ``302`` for every non-``HEAD``
220 method, matching what browsers did before ``307`` existed. The two SDKs
221 agreeing is worth more than matching that legacy, so the Fetch rules win.
222 """
223 if status_code == 303:
224 return "HEAD" if method == "HEAD" else "GET"
225 if status_code in {301, 302} and method == "POST":
226 return "GET"
227 return None
228
229
230 def safe_request(
231 method: str,
232 url: str,
233 *,
234 max_redirects: int = _MAX_REDIRECTS,
235 **kwargs: t.Any,
236 ) -> requests.Response:
237 """Send a request, validating the target before *every* hop.
238
239 Redirects are followed manually so each new location is validated — and
240 pinned — too. Validating only the first URL is not enough: a target that
241 passes the check and then answers ``302 Location: http://169.254.169.254/``
242 would have the redirect followed by ``requests`` with no further
243 validation. Mirrors ``ssrfSafeFetch`` in the TypeScript SDK.
244
245 Use this where redirects are legitimate (S3 can answer a PUT with a 307
246 region redirect). Call sites that require a direct URL should use
247 :func:`safe_get`, which rejects nothing but simply does not follow them.
248
249 Following a redirect by hand also means rewriting the method and body by
250 hand; see :func:`_redirect_rewrite` for the rules and why they are the
251 Fetch standard's rather than the ones ``requests`` would apply.
252
253 :param max_redirects: Hops to follow before giving up.
254 :raises BlockedInternalUrlError: If any hop fails validation, or the
255 redirect chain is longer than ``max_redirects``.
256 """
257 # `requests` normalizes the method on the prepared request, and the
258 # redirect rules below compare against it, so normalize once up front.
259 method = method.upper()
260 body = kwargs.get("data")
261 current_url = url
262
263 for _ in range(max_redirects + 1):
264 response = _pinned_request(method, current_url, **kwargs)
265
266 location = response.headers.get("Location")
267 if response.status_code not in _REDIRECT_STATUS_CODES or location is None:
268 return response
269
270 response.close()
271 previous_url = current_url
272 try:
273 current_url = urljoin(current_url, location)
274 except ValueError:
275 # `urljoin` refuses a broken IPv6 literal. `Location` is remote
276 # input, so this is a rejected hop, not a crash.
277 raise BlockedInternalUrlError(
278 "Refusing to follow a malformed redirect Location"
279 ) from None
280
281 # The next hop is whatever `Location` says, query string included, so
282 # `params` must not be appended to it a second time — that is how
283 # `requests` builds a redirected request, and it keeps a query-string
284 # credential from being handed to a target that never asked for one.
285 kwargs.pop("params", None)
286
287 # A credential header was addressed to the origin the caller named, so
288 # a hop that leaves that origin must not carry it along.
289 if not _same_origin(previous_url, current_url):
290 if headers := kwargs.get("headers"):
291 kwargs["headers"] = {
292 name: value
293 for name, value in headers.items()
294 if name.lower() not in _CREDENTIAL_HEADERS
295 }
296
297 rewritten_method = _redirect_rewrite(response.status_code, method)
298 if rewritten_method is not None:
299 method = rewritten_method
300 body = None
301 for key in ("data", "json", "files"):
302 kwargs.pop(key, None)
303 if headers := kwargs.get("headers"):
304 kwargs["headers"] = {
305 name: value
306 for name, value in headers.items()
307 if name.lower() not in _BODY_HEADERS
308 }
309 continue
310
311 # `requests` rewinds the body itself when it follows a redirect; doing
312 # it manually means doing that too, or a retried upload sends nothing.
313 seek = getattr(body, "seek", None)
314 if callable(seek):
315 seek(0)
316
317 raise BlockedInternalUrlError(
318 f"Refusing to fetch: too many redirects (max {max_redirects})"
319 )
320
321
322 class _PinnedAddressAdapter(requests.adapters.HTTPAdapter):
323 """Transport adapter that connects to a pre-validated address.
324
325 The hostname is left untouched on the connection, so the ``Host`` header
326 and the TLS SNI/certificate check still use it; only the address the
327 socket dials is replaced. Doing it the other way round — rewriting
328 ``conn._dns_host`` for the whole connection — would also rewrite
329 ``conn.host``, which urllib3 derives from it, and the request would go out
330 with an IP in ``Host`` and an IP in SNI, failing certificate verification
331 against every real origin.
332
333 This reaches into two urllib3 internals, ``HTTPConnection._new_conn`` and
334 ``HTTPConnection._dns_host``. ``test_url_safety_pinning.py`` asserts both
335 exist so a urllib3 upgrade that removes them fails loudly rather than
336 silently un-pinning the connection.
337 """
338
339 def __init__(self, addresses: t.Sequence[str], **kwargs: t.Any) -> None:
340 self._addresses = list(addresses)
341 super().__init__(**kwargs)
342
343 def get_connection_with_tls_context(
344 self,
345 request: requests.PreparedRequest,
346 verify: t.Union[bool, str, None],
347 proxies: t.Optional[t.Mapping[str, str]] = None,
348 cert: t.Union[str, t.Tuple[str, str], None] = None,
349 ) -> t.Any:
350 # `Any`, because the pinning below reaches for urllib3 internals that
351 # the typed `ConnectionPool` surface does not expose.
352 pool: t.Any = super().get_connection_with_tls_context(
353 request, verify, proxies=proxies, cert=cert
354 )
355 addresses = self._addresses
356 build_connection = pool._new_conn
357
358 def _new_conn() -> t.Any:
359 connection = build_connection()
360 open_socket = connection._new_conn
361
362 def _pinned_new_conn() -> t.Any:
363 # Swap the resolution target for the duration of the socket
364 # connect only. urllib3 reads `self.host` for SNI *after*
365 # `_new_conn()` returns, and `http.client` reads it later
366 # still for the `Host` header, so both see the hostname.
367 #
368 # Every validated address is tried in resolver order, the way
369 # urllib3 would have: pinning one address of a dual-stack host
370 # would strand callers whose network cannot reach that family.
371 hostname = connection._dns_host
372 last_error: t.Optional[BaseException] = None
373 for address in addresses:
374 connection._dns_host = address
375 try:
376 sock = open_socket()
377 except _CONNECT_ERRORS as error:
378 last_error = error
379 continue
380 finally:
381 connection._dns_host = hostname
382 _assert_pinned_peer(sock, address, hostname)
383 return sock
384
385 assert last_error is not None
386 raise last_error
387
388 connection._new_conn = _pinned_new_conn
389 return connection
390
391 pool._new_conn = _new_conn
392 return pool
393
394
395 def _assert_pinned_peer(sock: t.Any, address: str, hostname: str) -> None:
396 """Fail closed if the open socket is not connected to the pinned address.
397
398 A redundancy check on the pinning above, run before a single byte is
399 written to the socket — a post-response check would be too late and
400 unreliable, because urllib3 detaches the socket as soon as the server
401 signals ``Connection: close``, while the body stays readable.
402 """
403 try:
404 peer = sock.getpeername()[0]
405 except (AttributeError, OSError, IndexError):
406 return
407
408 try:
409 connected_to_pinned = ipaddress.ip_address(peer) == ipaddress.ip_address(
410 address
411 )
412 except ValueError:
413 connected_to_pinned = peer == address
414
415 if connected_to_pinned:
416 return
417
418 sock.close()
419 raise BlockedInternalUrlError(
420 f'Refusing to talk to "{hostname}": the connection was established to '
421 f"{peer}, not to the validated address {address}"
422 )
423
424
425 def _proxy_applies(url: str, proxies: t.Optional[t.Mapping[str, str]]) -> bool:
426 """Whether Requests would send ``url`` through a proxy.
427
428 Requests honours ``HTTP_PROXY``/``HTTPS_PROXY``/``ALL_PROXY`` (minus
429 ``NO_PROXY``) by default. Through a proxy the socket is dialled to the
430 *proxy*, so pinning the target address would connect to the wrong host
431 entirely.
432
433 Residual: proxied requests keep only the pre-flight check, because the
434 proxy resolves the hostname itself and the SDK cannot see or pin that
435 resolution. A rebinding window therefore remains for callers that run
436 behind a proxy — including one inherited from the environment.
437 """
438 try:
439 environment_proxies = requests.utils.get_environ_proxies(url)
440 merged = {**environment_proxies, **(proxies or {})}
441 return requests.utils.select_proxy(url, merged) is not None
442 except Exception: # pragma: no cover - platform proxy discovery can fail
443 # Unknowable, so assume a proxy rather than pinning onto a connection
444 # that may not go where we think it goes.
445 return True
446
447
448 def safe_get(url: str, **kwargs: t.Any) -> requests.Response:
449 """Validate a URL and fetch it without re-resolving its hostname.
450
451 Redirects are never followed: call sites that need a direct URL treat a
452 3xx as an error, and call sites where redirects are legitimate use
453 :func:`safe_request`.
454 """
455 return _pinned_request("GET", url, **kwargs)
456
457
458 def _pinned_request(method: str, url: str, **kwargs: t.Any) -> requests.Response:
459 addresses = assert_safe_fetch_target(url)
460
461 session = requests.Session()
462 if not _proxy_applies(url, kwargs.get("proxies")):
463 # A fresh Session per request, so a connection pinned to one address is
464 # never reused for a request validated against another.
465 adapter = _PinnedAddressAdapter(addresses)
466 session.mount("http://", adapter)
467 session.mount("https://", adapter)
468
469 response = session.request(method, url, allow_redirects=False, **kwargs)
470 # The response may still be streaming, and closing the session would close
471 # the pool holding its connection, so tie the session's lifetime to it.
472 response._composio_session = session # type: ignore[attr-defined]
473 return response