Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | """SSRF protections for URLs the SDK fetches. |
| 2 | |
| 3 | Applies to URLs the caller supplies *and* to URLs an API response supplies: |
| 4 | under the trust boundary documented in ``python/AGENTS.md`` the backend may be |
| 5 | compromised or the connection MITM'd, so a presigned URL in a response is no |
| 6 | more trusted than one typed by a user. Without a guard, either can point the |
| 7 | SDK at loopback, RFC1918 space, or a link-local cloud-metadata endpoint |
| 8 | (``169.254.169.254``) and turn it into a request proxy for internal |
| 9 | infrastructure. |
| 10 | |
| 11 | Validating a hostname is not enough on its own, because the hostname is |
| 12 | resolved twice: once here, and once by the HTTP client when it opens the |
| 13 | socket. A short-TTL record can answer with a public address for the first |
| 14 | lookup and an internal one for the second — a time-of-check/time-of-use |
| 15 | window better known as DNS rebinding. So the address validated here is also |
| 16 | the address connected to: :func:`safe_get` and :func:`safe_request` pin it |
| 17 | onto the connection, keeping the original hostname for the ``Host`` header |
| 18 | and TLS SNI/certificate verification. Every fetch in the SDK goes through one |
| 19 | of those two, so no call site can reintroduce the gap by calling |
| 20 | ``requests.get`` next to a bare check. The TypeScript guard pins the same way. |
| 21 | |
| 22 | Also parses response headers that gate how much of a body the SDK reads: |
| 23 | ``parse_content_length`` treats ``Content-Length`` as the untrusted hint it |
| 24 | is, so a malformed value degrades to an unknown size under a streamed byte |
| 25 | count instead of crashing the fetch. |
| 26 | """ |
| 27 | |
| 28 | from __future__ import annotations |
| 29 | |
| 30 | import ipaddress |
| 31 | import socket |
| 32 | import typing as t |
| 33 | from urllib.parse import urljoin, urlparse |
| 34 | |
| 35 | import requests |
| 36 | import urllib3.exceptions |
| 37 | |
| 38 | from composio.exceptions import BlockedInternalUrlError |
| 39 | |
| 40 | # urllib3 rewraps connect failures into its own hierarchy, none of which |
| 41 | # inherits from OSError, so a bare `except OSError` would never see them and |
| 42 | # the address fallback below would never run. |
| 43 | _CONNECT_ERRORS = ( |
| 44 | OSError, |
| 45 | urllib3.exceptions.NewConnectionError, |
| 46 | urllib3.exceptions.ConnectTimeoutError, |
| 47 | urllib3.exceptions.NameResolutionError, |
| 48 | ) |
| 49 | |
| 50 | # Ranges that must not be reachable from a user-supplied URL but that |
| 51 | # ``ipaddress.is_global`` does not reject on its own, because it only asks |
| 52 | # whether an address is private. The TypeScript guard blocks the same ones from |
| 53 | # its explicit CIDR list. |
| 54 | _ALSO_BLOCKED_NETWORKS = ( |
| 55 | ipaddress.ip_network("224.0.0.0/4"), # IPv4 multicast |
| 56 | ipaddress.ip_network("192.88.99.0/24"), # 6to4 relay anycast (RFC 7526) |
| 57 | ipaddress.ip_network("ff00::/8"), # IPv6 multicast |
| 58 | ipaddress.ip_network("fec0::/10"), # IPv6 site-local (deprecated) |
| 59 | ) |
| 60 | |
| 61 | _REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308}) |
| 62 | # Headers that describe a request body, so they have to go when the body does. |
| 63 | # The Fetch standard's "request-body-header name" set, plus the two `requests` |
| 64 | # purges for the same reason (it recomputes both from the body it sends). |
| 65 | _BODY_HEADERS = frozenset( |
| 66 | { |
| 67 | "content-encoding", |
| 68 | "content-language", |
| 69 | "content-length", |
| 70 | "content-location", |
| 71 | "content-type", |
| 72 | "transfer-encoding", |
| 73 | } |
| 74 | ) |
| 75 | # Headers that carry a credential for the origin the request was addressed |
| 76 | # to, so they must not follow a redirect to a different origin. The Fetch |
| 77 | # standard strips ``Authorization`` on a cross-origin redirect; ``Cookie`` and |
| 78 | # ``Proxy-Authorization`` go with it the way ``requests``' own ``rebuild_auth`` |
| 79 | # and curl drop them on a host change. Following redirects by hand bypasses |
| 80 | # ``rebuild_auth``, so the rule is applied here. The TypeScript guard drops the |
| 81 | # same three (``CREDENTIAL_HEADERS``). |
| 82 | _CREDENTIAL_HEADERS = frozenset({"authorization", "cookie", "proxy-authorization"}) |
| 83 | _MAX_REDIRECTS = 5 |
| 84 | |
| 85 | |
| 86 | def _origin(url: str) -> t.Optional[t.Tuple[str, str, int]]: |
| 87 | """The ``(scheme, host, port)`` triple two URLs must share to be same-origin. |
| 88 | |
| 89 | ``None`` when the URL cannot be parsed: ``urlparse`` raises ``ValueError`` |
| 90 | for a broken IPv6 literal, and ``.port`` for an out-of-range port. A |
| 91 | redirect ``Location`` is remote input, so that is not an error here: the |
| 92 | hop is treated as leaving the origin, and :func:`assert_safe_fetch_target` |
| 93 | rejects the URL before anything is sent. |
| 94 | """ |
| 95 | try: |
| 96 | parsed = urlparse(url) |
| 97 | scheme = parsed.scheme.lower() |
| 98 | host = (parsed.hostname or "").lower() |
| 99 | port = parsed.port |
| 100 | except ValueError: |
| 101 | return None |
| 102 | if port is None: |
| 103 | port = 443 if scheme == "https" else 80 |
| 104 | return scheme, host, port |
| 105 | |
| 106 | |
| 107 | def _same_origin(previous_url: str, next_url: str) -> bool: |
| 108 | """Whether a redirect hop stays on the origin the caller addressed.""" |
| 109 | previous_origin = _origin(previous_url) |
| 110 | return previous_origin is not None and previous_origin == _origin(next_url) |
| 111 | |
| 112 | |
| 113 | def is_blocked_ip(value: str) -> bool: |
| 114 | """Return whether an address is non-publicly-routable.""" |
| 115 | try: |
| 116 | address = ipaddress.ip_address(value) |
| 117 | except ValueError: |
| 118 | return True |
| 119 | |
| 120 | if isinstance(address, ipaddress.IPv6Address): |
| 121 | embedded_ipv4 = address.ipv4_mapped |
| 122 | if embedded_ipv4 is None and address.packed[:12] in { |
| 123 | b"\x00" * 12, |
| 124 | b"\x00d\xff\x9b" + b"\x00" * 8, |
| 125 | }: |
| 126 | embedded_ipv4 = ipaddress.IPv4Address(address.packed[-4:]) |
| 127 | if embedded_ipv4 is not None: |
| 128 | return is_blocked_ip(str(embedded_ipv4)) |
| 129 | |
| 130 | if any(address in network for network in _ALSO_BLOCKED_NETWORKS): |
| 131 | return True |
| 132 | |
| 133 | return not address.is_global |
| 134 | |
| 135 | |
| 136 | def assert_safe_fetch_target(url: str) -> t.List[str]: |
| 137 | """Refuse non-HTTP(S) URLs and hosts that resolve to internal addresses. |
| 138 | |
| 139 | Parse the URL after Requests prepares it so validation uses the same |
| 140 | canonical hostname that the eventual connection will use. |
| 141 | |
| 142 | :returns: The validated addresses to connect to, in resolver order. |
| 143 | Callers must connect to *these* rather than re-resolving the hostname; |
| 144 | see :func:`safe_get`. |
| 145 | """ |
| 146 | try: |
| 147 | prepared_url = requests.Request(method="GET", url=url).prepare().url |
| 148 | if prepared_url is None: |
| 149 | raise ValueError("Prepared URL is missing") |
| 150 | parsed = urlparse(prepared_url) |
| 151 | if parsed.scheme not in {"http", "https"} or not parsed.hostname: |
| 152 | raise ValueError("URL must use HTTP(S) and include a hostname") |
| 153 | except (requests.exceptions.RequestException, ValueError): |
| 154 | raise BlockedInternalUrlError( |
| 155 | "Refusing to fetch a malformed or non-http(s) URL" |
| 156 | ) from None |
| 157 | |
| 158 | try: |
| 159 | # Resolver order is kept: it encodes the system's address preference |
| 160 | # (RFC 6724), and connecting walks it the way urllib3 would. |
| 161 | addresses: t.List[str] = [] |
| 162 | for result in socket.getaddrinfo(parsed.hostname, None): |
| 163 | address = result[4][0] |
| 164 | if not isinstance(address, str): |
| 165 | raise BlockedInternalUrlError( |
| 166 | f'Could not resolve host "{parsed.hostname}"' |
| 167 | ) |
| 168 | if address not in addresses: |
| 169 | addresses.append(address) |
| 170 | except socket.gaierror as error: |
| 171 | raise BlockedInternalUrlError( |
| 172 | f'Could not resolve host "{parsed.hostname}"' |
| 173 | ) from error |
| 174 | |
| 175 | for address in addresses: |
| 176 | if is_blocked_ip(address): |
| 177 | raise BlockedInternalUrlError( |
| 178 | f'Refusing to fetch "{parsed.hostname}" because it resolves to a non-public address' |
| 179 | ) |
| 180 | |
| 181 | if not addresses: |
| 182 | raise BlockedInternalUrlError(f'Could not resolve host "{parsed.hostname}"') |
| 183 | |
| 184 | return addresses |
| 185 | |
| 186 | |
| 187 | def parse_content_length(value: t.Optional[str]) -> t.Optional[int]: |
| 188 | """Parse a ``Content-Length`` header into a non-negative ``int``. |
| 189 | |
| 190 | ``Content-Length`` is supplied by the remote server and is therefore |
| 191 | untrusted: it may be absent, non-numeric (``"abc"``), fractional |
| 192 | (``"12.5"``), thousands-separated (``"1,024"``) or negative. Anything |
| 193 | untrustworthy returns ``None`` so the caller treats the size as unknown |
| 194 | and falls through to a streamed byte count, which stays authoritative |
| 195 | because the header can also be absent or understated. Mirrors |
| 196 | ``readResponseBodyWithLimit`` in the TypeScript SDK, which only trusts |
| 197 | values matching ``/^\\d+$/``. |
| 198 | """ |
| 199 | if value is None: |
| 200 | return None |
| 201 | try: |
| 202 | size = int(value.strip()) |
| 203 | except ValueError: |
| 204 | return None |
| 205 | return size if size >= 0 else None |
| 206 | |
| 207 | |
| 208 | def _redirect_rewrite(status_code: int, method: str) -> t.Optional[str]: |
| 209 | """The method the hop after a redirect uses, or ``None`` to replay as-is. |
| 210 | |
| 211 | A returned method also means the request body goes: these are the Fetch |
| 212 | standard's redirect rules, which the TypeScript guard applies too and which |
| 213 | following redirects by hand means applying by hand. ``303`` points at a |
| 214 | result URL that has no use for the original body, so every method loses it |
| 215 | and everything but ``HEAD`` becomes a ``GET``; ``301``/``302`` do the same |
| 216 | to a ``POST`` only; ``307``/``308`` replay both. |
| 217 | |
| 218 | ``requests`` would have applied its own rules in ``resolve_redirects``, and |
| 219 | they are not quite these: it downgrades ``302`` for every non-``HEAD`` |
| 220 | method, matching what browsers did before ``307`` existed. The two SDKs |
| 221 | agreeing is worth more than matching that legacy, so the Fetch rules win. |
| 222 | """ |
| 223 | if status_code == 303: |
| 224 | return "HEAD" if method == "HEAD" else "GET" |
| 225 | if status_code in {301, 302} and method == "POST": |
| 226 | return "GET" |
| 227 | return None |
| 228 | |
| 229 | |
| 230 | def safe_request( |
| 231 | method: str, |
| 232 | url: str, |
| 233 | *, |
| 234 | max_redirects: int = _MAX_REDIRECTS, |
| 235 | **kwargs: t.Any, |
| 236 | ) -> requests.Response: |
| 237 | """Send a request, validating the target before *every* hop. |
| 238 | |
| 239 | Redirects are followed manually so each new location is validated — and |
| 240 | pinned — too. Validating only the first URL is not enough: a target that |
| 241 | passes the check and then answers ``302 Location: http://169.254.169.254/`` |
| 242 | would have the redirect followed by ``requests`` with no further |
| 243 | validation. Mirrors ``ssrfSafeFetch`` in the TypeScript SDK. |
| 244 | |
| 245 | Use this where redirects are legitimate (S3 can answer a PUT with a 307 |
| 246 | region redirect). Call sites that require a direct URL should use |
| 247 | :func:`safe_get`, which rejects nothing but simply does not follow them. |
| 248 | |
| 249 | Following a redirect by hand also means rewriting the method and body by |
| 250 | hand; see :func:`_redirect_rewrite` for the rules and why they are the |
| 251 | Fetch standard's rather than the ones ``requests`` would apply. |
| 252 | |
| 253 | :param max_redirects: Hops to follow before giving up. |
| 254 | :raises BlockedInternalUrlError: If any hop fails validation, or the |
| 255 | redirect chain is longer than ``max_redirects``. |
| 256 | """ |
| 257 | # `requests` normalizes the method on the prepared request, and the |
| 258 | # redirect rules below compare against it, so normalize once up front. |
| 259 | method = method.upper() |
| 260 | body = kwargs.get("data") |
| 261 | current_url = url |
| 262 | |
| 263 | for _ in range(max_redirects + 1): |
| 264 | response = _pinned_request(method, current_url, **kwargs) |
| 265 | |
| 266 | location = response.headers.get("Location") |
| 267 | if response.status_code not in _REDIRECT_STATUS_CODES or location is None: |
| 268 | return response |
| 269 | |
| 270 | response.close() |
| 271 | previous_url = current_url |
| 272 | try: |
| 273 | current_url = urljoin(current_url, location) |
| 274 | except ValueError: |
| 275 | # `urljoin` refuses a broken IPv6 literal. `Location` is remote |
| 276 | # input, so this is a rejected hop, not a crash. |
| 277 | raise BlockedInternalUrlError( |
| 278 | "Refusing to follow a malformed redirect Location" |
| 279 | ) from None |
| 280 | |
| 281 | # The next hop is whatever `Location` says, query string included, so |
| 282 | # `params` must not be appended to it a second time — that is how |
| 283 | # `requests` builds a redirected request, and it keeps a query-string |
| 284 | # credential from being handed to a target that never asked for one. |
| 285 | kwargs.pop("params", None) |
| 286 | |
| 287 | # A credential header was addressed to the origin the caller named, so |
| 288 | # a hop that leaves that origin must not carry it along. |
| 289 | if not _same_origin(previous_url, current_url): |
| 290 | if headers := kwargs.get("headers"): |
| 291 | kwargs["headers"] = { |
| 292 | name: value |
| 293 | for name, value in headers.items() |
| 294 | if name.lower() not in _CREDENTIAL_HEADERS |
| 295 | } |
| 296 | |
| 297 | rewritten_method = _redirect_rewrite(response.status_code, method) |
| 298 | if rewritten_method is not None: |
| 299 | method = rewritten_method |
| 300 | body = None |
| 301 | for key in ("data", "json", "files"): |
| 302 | kwargs.pop(key, None) |
| 303 | if headers := kwargs.get("headers"): |
| 304 | kwargs["headers"] = { |
| 305 | name: value |
| 306 | for name, value in headers.items() |
| 307 | if name.lower() not in _BODY_HEADERS |
| 308 | } |
| 309 | continue |
| 310 | |
| 311 | # `requests` rewinds the body itself when it follows a redirect; doing |
| 312 | # it manually means doing that too, or a retried upload sends nothing. |
| 313 | seek = getattr(body, "seek", None) |
| 314 | if callable(seek): |
| 315 | seek(0) |
| 316 | |
| 317 | raise BlockedInternalUrlError( |
| 318 | f"Refusing to fetch: too many redirects (max {max_redirects})" |
| 319 | ) |
| 320 | |
| 321 | |
| 322 | class _PinnedAddressAdapter(requests.adapters.HTTPAdapter): |
| 323 | """Transport adapter that connects to a pre-validated address. |
| 324 | |
| 325 | The hostname is left untouched on the connection, so the ``Host`` header |
| 326 | and the TLS SNI/certificate check still use it; only the address the |
| 327 | socket dials is replaced. Doing it the other way round — rewriting |
| 328 | ``conn._dns_host`` for the whole connection — would also rewrite |
| 329 | ``conn.host``, which urllib3 derives from it, and the request would go out |
| 330 | with an IP in ``Host`` and an IP in SNI, failing certificate verification |
| 331 | against every real origin. |
| 332 | |
| 333 | This reaches into two urllib3 internals, ``HTTPConnection._new_conn`` and |
| 334 | ``HTTPConnection._dns_host``. ``test_url_safety_pinning.py`` asserts both |
| 335 | exist so a urllib3 upgrade that removes them fails loudly rather than |
| 336 | silently un-pinning the connection. |
| 337 | """ |
| 338 | |
| 339 | def __init__(self, addresses: t.Sequence[str], **kwargs: t.Any) -> None: |
| 340 | self._addresses = list(addresses) |
| 341 | super().__init__(**kwargs) |
| 342 | |
| 343 | def get_connection_with_tls_context( |
| 344 | self, |
| 345 | request: requests.PreparedRequest, |
| 346 | verify: t.Union[bool, str, None], |
| 347 | proxies: t.Optional[t.Mapping[str, str]] = None, |
| 348 | cert: t.Union[str, t.Tuple[str, str], None] = None, |
| 349 | ) -> t.Any: |
| 350 | # `Any`, because the pinning below reaches for urllib3 internals that |
| 351 | # the typed `ConnectionPool` surface does not expose. |
| 352 | pool: t.Any = super().get_connection_with_tls_context( |
| 353 | request, verify, proxies=proxies, cert=cert |
| 354 | ) |
| 355 | addresses = self._addresses |
| 356 | build_connection = pool._new_conn |
| 357 | |
| 358 | def _new_conn() -> t.Any: |
| 359 | connection = build_connection() |
| 360 | open_socket = connection._new_conn |
| 361 | |
| 362 | def _pinned_new_conn() -> t.Any: |
| 363 | # Swap the resolution target for the duration of the socket |
| 364 | # connect only. urllib3 reads `self.host` for SNI *after* |
| 365 | # `_new_conn()` returns, and `http.client` reads it later |
| 366 | # still for the `Host` header, so both see the hostname. |
| 367 | # |
| 368 | # Every validated address is tried in resolver order, the way |
| 369 | # urllib3 would have: pinning one address of a dual-stack host |
| 370 | # would strand callers whose network cannot reach that family. |
| 371 | hostname = connection._dns_host |
| 372 | last_error: t.Optional[BaseException] = None |
| 373 | for address in addresses: |
| 374 | connection._dns_host = address |
| 375 | try: |
| 376 | sock = open_socket() |
| 377 | except _CONNECT_ERRORS as error: |
| 378 | last_error = error |
| 379 | continue |
| 380 | finally: |
| 381 | connection._dns_host = hostname |
| 382 | _assert_pinned_peer(sock, address, hostname) |
| 383 | return sock |
| 384 | |
| 385 | assert last_error is not None |
| 386 | raise last_error |
| 387 | |
| 388 | connection._new_conn = _pinned_new_conn |
| 389 | return connection |
| 390 | |
| 391 | pool._new_conn = _new_conn |
| 392 | return pool |
| 393 | |
| 394 | |
| 395 | def _assert_pinned_peer(sock: t.Any, address: str, hostname: str) -> None: |
| 396 | """Fail closed if the open socket is not connected to the pinned address. |
| 397 | |
| 398 | A redundancy check on the pinning above, run before a single byte is |
| 399 | written to the socket — a post-response check would be too late and |
| 400 | unreliable, because urllib3 detaches the socket as soon as the server |
| 401 | signals ``Connection: close``, while the body stays readable. |
| 402 | """ |
| 403 | try: |
| 404 | peer = sock.getpeername()[0] |
| 405 | except (AttributeError, OSError, IndexError): |
| 406 | return |
| 407 | |
| 408 | try: |
| 409 | connected_to_pinned = ipaddress.ip_address(peer) == ipaddress.ip_address( |
| 410 | address |
| 411 | ) |
| 412 | except ValueError: |
| 413 | connected_to_pinned = peer == address |
| 414 | |
| 415 | if connected_to_pinned: |
| 416 | return |
| 417 | |
| 418 | sock.close() |
| 419 | raise BlockedInternalUrlError( |
| 420 | f'Refusing to talk to "{hostname}": the connection was established to ' |
| 421 | f"{peer}, not to the validated address {address}" |
| 422 | ) |
| 423 | |
| 424 | |
| 425 | def _proxy_applies(url: str, proxies: t.Optional[t.Mapping[str, str]]) -> bool: |
| 426 | """Whether Requests would send ``url`` through a proxy. |
| 427 | |
| 428 | Requests honours ``HTTP_PROXY``/``HTTPS_PROXY``/``ALL_PROXY`` (minus |
| 429 | ``NO_PROXY``) by default. Through a proxy the socket is dialled to the |
| 430 | *proxy*, so pinning the target address would connect to the wrong host |
| 431 | entirely. |
| 432 | |
| 433 | Residual: proxied requests keep only the pre-flight check, because the |
| 434 | proxy resolves the hostname itself and the SDK cannot see or pin that |
| 435 | resolution. A rebinding window therefore remains for callers that run |
| 436 | behind a proxy — including one inherited from the environment. |
| 437 | """ |
| 438 | try: |
| 439 | environment_proxies = requests.utils.get_environ_proxies(url) |
| 440 | merged = {**environment_proxies, **(proxies or {})} |
| 441 | return requests.utils.select_proxy(url, merged) is not None |
| 442 | except Exception: # pragma: no cover - platform proxy discovery can fail |
| 443 | # Unknowable, so assume a proxy rather than pinning onto a connection |
| 444 | # that may not go where we think it goes. |
| 445 | return True |
| 446 | |
| 447 | |
| 448 | def safe_get(url: str, **kwargs: t.Any) -> requests.Response: |
| 449 | """Validate a URL and fetch it without re-resolving its hostname. |
| 450 | |
| 451 | Redirects are never followed: call sites that need a direct URL treat a |
| 452 | 3xx as an error, and call sites where redirects are legitimate use |
| 453 | :func:`safe_request`. |
| 454 | """ |
| 455 | return _pinned_request("GET", url, **kwargs) |
| 456 | |
| 457 | |
| 458 | def _pinned_request(method: str, url: str, **kwargs: t.Any) -> requests.Response: |
| 459 | addresses = assert_safe_fetch_target(url) |
| 460 | |
| 461 | session = requests.Session() |
| 462 | if not _proxy_applies(url, kwargs.get("proxies")): |
| 463 | # A fresh Session per request, so a connection pinned to one address is |
| 464 | # never reused for a request validated against another. |
| 465 | adapter = _PinnedAddressAdapter(addresses) |
| 466 | session.mount("http://", adapter) |
| 467 | session.mount("https://", adapter) |
| 468 | |
| 469 | response = session.request(method, url, allow_redirects=False, **kwargs) |
| 470 | # The response may still be streaming, and closing the session would close |
| 471 | # the pool holding its connection, so tie the session's lifetime to it. |
| 472 | response._composio_session = session # type: ignore[attr-defined] |
| 473 | return response |