Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | """Regression tests for the DNS-rebinding window in the SSRF guard. |
| 2 | |
| 3 | The guard resolves a hostname to decide whether a URL is safe. If the HTTP |
| 4 | client then resolves that hostname a *second* time to open the socket, an |
| 5 | attacker who controls the authoritative DNS can answer with a public address |
| 6 | for the check and an internal one for the connect — the check and the use are |
| 7 | about different addresses (issue #4151). |
| 8 | |
| 9 | These tests run against real sockets on loopback rather than mocking the HTTP |
| 10 | client, because a mocked client cannot resolve anything twice and so cannot |
| 11 | express the bug at all. The resolver is the only thing faked: it answers the |
| 12 | first lookup of the hostname with one endpoint and every later lookup with |
| 13 | another, which is exactly what a short-TTL rebinding record does. |
| 14 | """ |
| 15 | |
| 16 | from __future__ import annotations |
| 17 | |
| 18 | import ipaddress |
| 19 | import os |
| 20 | import socket |
| 21 | import threading |
| 22 | import typing as t |
| 23 | from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer |
| 24 | from unittest.mock import MagicMock, patch |
| 25 | |
| 26 | import pytest |
| 27 | import requests |
| 28 | import urllib3.connection |
| 29 | |
| 30 | from composio.exceptions import BlockedInternalUrlError |
| 31 | from composio.utils.url_safety import _assert_pinned_peer, safe_get |
| 32 | |
| 33 | HOSTNAME = "rebind.test" |
| 34 | |
| 35 | |
| 36 | def _is_ip_literal(host: str) -> bool: |
| 37 | try: |
| 38 | ipaddress.ip_address(host) |
| 39 | except ValueError: |
| 40 | return False |
| 41 | return True |
| 42 | |
| 43 | |
| 44 | class _RecordingServer: |
| 45 | """A loopback HTTP server that records what actually reached it.""" |
| 46 | |
| 47 | def __init__(self, body: bytes) -> None: |
| 48 | self.hits: t.List[str] = [] |
| 49 | recorder = self |
| 50 | |
| 51 | class Handler(BaseHTTPRequestHandler): |
| 52 | protocol_version = "HTTP/1.1" |
| 53 | |
| 54 | def do_GET(self) -> None: # noqa: N802 - stdlib naming |
| 55 | recorder.hits.append(self.headers.get("Host", "")) |
| 56 | self.send_response(200) |
| 57 | self.send_header("Content-Type", "text/plain") |
| 58 | self.send_header("Content-Length", str(len(body))) |
| 59 | self.end_headers() |
| 60 | self.wfile.write(body) |
| 61 | |
| 62 | def log_message(self, *args: t.Any) -> None: |
| 63 | pass |
| 64 | |
| 65 | self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) |
| 66 | # Keep-alive connections would otherwise hold `shutdown()` open. |
| 67 | self._server.daemon_threads = True |
| 68 | self.port = self._server.server_address[1] |
| 69 | self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) |
| 70 | self._thread.start() |
| 71 | |
| 72 | def close(self) -> None: |
| 73 | self._server.shutdown() |
| 74 | self._server.server_close() |
| 75 | |
| 76 | |
| 77 | @pytest.fixture |
| 78 | def validated_server() -> t.Iterator[_RecordingServer]: |
| 79 | server = _RecordingServer(b"public payload") |
| 80 | yield server |
| 81 | server.close() |
| 82 | |
| 83 | |
| 84 | @pytest.fixture |
| 85 | def rebound_server() -> t.Iterator[_RecordingServer]: |
| 86 | server = _RecordingServer(b"internal secret") |
| 87 | yield server |
| 88 | server.close() |
| 89 | |
| 90 | |
| 91 | @pytest.fixture |
| 92 | def no_inherited_proxy(monkeypatch: pytest.MonkeyPatch) -> None: |
| 93 | """Make ``_proxy_applies`` deterministic: no proxy, from env or system. |
| 94 | |
| 95 | An inherited ``HTTP_PROXY`` (or a macOS/Windows system proxy, which |
| 96 | ``getproxies`` also reads) would disable the pinning adapter, and the |
| 97 | request would dial the proxy's answer to the hostname instead of the |
| 98 | validated address — failing these tests on machines that carry one. |
| 99 | |
| 100 | ``requests.sessions`` imports ``get_environ_proxies`` by name, so the |
| 101 | session's own proxy lookup needs its own patch: without it the SDK sees no |
| 102 | proxy and pins, while the session still routes through the inherited one. |
| 103 | """ |
| 104 | for module in ("requests.utils", "requests.sessions"): |
| 105 | monkeypatch.setattr( |
| 106 | f"{module}.get_environ_proxies", |
| 107 | lambda url, no_proxy=None: {}, |
| 108 | ) |
| 109 | |
| 110 | |
| 111 | @pytest.fixture |
| 112 | def rebinding_dns( |
| 113 | no_inherited_proxy: None, |
| 114 | validated_server: _RecordingServer, |
| 115 | rebound_server: _RecordingServer, |
| 116 | ) -> t.Iterator[None]: |
| 117 | """Answer the first lookup of the hostname benignly, later ones with the victim. |
| 118 | |
| 119 | Both endpoints live on 127.0.0.1 because a test cannot bind a public |
| 120 | address, so they are told apart by port; `is_blocked_ip` is stubbed out for |
| 121 | the same reason. Neither substitution touches the property under test, |
| 122 | which is whether the fetch connects to the endpoint that was validated or |
| 123 | re-resolves and connects somewhere else. The blocklist itself is covered by |
| 124 | ``test_url_safety.py``. |
| 125 | """ |
| 126 | lookups: t.List[str] = [] |
| 127 | real_getaddrinfo = socket.getaddrinfo |
| 128 | |
| 129 | def fake_getaddrinfo(host, port, *args, **kwargs): # type: ignore[no-untyped-def] |
| 130 | # A real resolver hands an IP literal straight back, which is what the |
| 131 | # fix relies on when it connects to the address it already validated. |
| 132 | if _is_ip_literal(host): |
| 133 | return real_getaddrinfo(host, port, *args, **kwargs) |
| 134 | |
| 135 | lookups.append(host) |
| 136 | victim = rebound_server if len(lookups) > 1 else validated_server |
| 137 | return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", victim.port))] |
| 138 | |
| 139 | with patch("socket.getaddrinfo", side_effect=fake_getaddrinfo): |
| 140 | with patch("composio.utils.url_safety.is_blocked_ip", return_value=False): |
| 141 | yield |
| 142 | |
| 143 | |
| 144 | def test_proxy_environment_keeps_only_the_pre_flight_check( |
| 145 | validated_server: _RecordingServer, |
| 146 | ) -> None: |
| 147 | """Behind a proxy the SDK cannot pin: the proxy resolves the hostname. |
| 148 | |
| 149 | Documented residual — ``_proxy_applies`` disables the pinning adapter, so |
| 150 | the request dials the proxy. A proxy nothing can reach makes that |
| 151 | observable: the request must fail by trying the proxy, never by dialling |
| 152 | the validated address. (This test deliberately does not use |
| 153 | ``no_inherited_proxy`` — it needs ``HTTP_PROXY`` visible — so it patches |
| 154 | the resolver inline instead.) |
| 155 | """ |
| 156 | real_getaddrinfo = socket.getaddrinfo |
| 157 | |
| 158 | def fake_getaddrinfo(host, port, *args, **kwargs): # type: ignore[no-untyped-def] |
| 159 | if _is_ip_literal(host): |
| 160 | return real_getaddrinfo(host, port, *args, **kwargs) |
| 161 | return [ |
| 162 | ( |
| 163 | socket.AF_INET, |
| 164 | socket.SOCK_STREAM, |
| 165 | 6, |
| 166 | "", |
| 167 | ("127.0.0.1", validated_server.port), |
| 168 | ) |
| 169 | ] |
| 170 | |
| 171 | # Empty NO_PROXY so no machine-level bypass list can cover the hostname. |
| 172 | env = {"HTTP_PROXY": "http://127.0.0.1:9", "NO_PROXY": "", "no_proxy": ""} |
| 173 | with patch("socket.getaddrinfo", side_effect=fake_getaddrinfo): |
| 174 | with patch("composio.utils.url_safety.is_blocked_ip", return_value=False): |
| 175 | with patch("composio.utils.url_safety._PinnedAddressAdapter") as adapter: |
| 176 | with patch.dict(os.environ, env): |
| 177 | with pytest.raises(requests.exceptions.RequestException): |
| 178 | safe_get( |
| 179 | f"http://{HOSTNAME}:{validated_server.port}/payload", |
| 180 | timeout=(1, 1), |
| 181 | ) |
| 182 | |
| 183 | adapter.assert_not_called() |
| 184 | assert validated_server.hits == [] |
| 185 | |
| 186 | |
| 187 | @pytest.mark.usefixtures("rebinding_dns") |
| 188 | def test_fetch_connects_to_the_address_it_validated( |
| 189 | validated_server: _RecordingServer, rebound_server: _RecordingServer |
| 190 | ) -> None: |
| 191 | response = safe_get( |
| 192 | f"http://{HOSTNAME}:{validated_server.port}/payload", timeout=(5, 5) |
| 193 | ) |
| 194 | |
| 195 | assert response.content == b"public payload" |
| 196 | assert len(validated_server.hits) == 1 |
| 197 | # The whole point: the rebound endpoint is never even connected to, so |
| 198 | # neither its body nor a bare TCP connection to it is available to the |
| 199 | # attacker. |
| 200 | assert rebound_server.hits == [] |
| 201 | |
| 202 | |
| 203 | @pytest.mark.usefixtures("rebinding_dns") |
| 204 | def test_pinning_ignores_an_inherited_proxy( |
| 205 | monkeypatch: pytest.MonkeyPatch, |
| 206 | validated_server: _RecordingServer, |
| 207 | rebound_server: _RecordingServer, |
| 208 | ) -> None: |
| 209 | """``no_inherited_proxy`` must hold on machines that carry a proxy (#5249). |
| 210 | |
| 211 | A Windows registry or macOS system proxy reaches Requests through the same |
| 212 | ``getproxies`` lookup as these variables, so an unreachable proxy here |
| 213 | stands in for one this suite would otherwise inherit on those machines. |
| 214 | """ |
| 215 | for name in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"): |
| 216 | monkeypatch.setenv(name, "http://127.0.0.1:9") |
| 217 | for name in ("NO_PROXY", "no_proxy"): |
| 218 | monkeypatch.setenv(name, "") |
| 219 | |
| 220 | response = safe_get( |
| 221 | f"http://{HOSTNAME}:{validated_server.port}/payload", timeout=(5, 5) |
| 222 | ) |
| 223 | |
| 224 | assert response.content == b"public payload" |
| 225 | assert len(validated_server.hits) == 1 |
| 226 | assert rebound_server.hits == [] |
| 227 | |
| 228 | |
| 229 | @pytest.mark.usefixtures("rebinding_dns") |
| 230 | def test_pinning_keeps_the_hostname_on_the_wire( |
| 231 | validated_server: _RecordingServer, |
| 232 | ) -> None: |
| 233 | """The address is pinned; the hostname is not replaced by it. |
| 234 | |
| 235 | Pinning by rewriting the connection's host would send ``Host: 127.0.0.1`` |
| 236 | and offer the IP as TLS SNI, which fails certificate verification against |
| 237 | every real origin. |
| 238 | """ |
| 239 | safe_get(f"http://{HOSTNAME}:{validated_server.port}/payload", timeout=(5, 5)) |
| 240 | |
| 241 | assert validated_server.hits == [f"{HOSTNAME}:{validated_server.port}"] |
| 242 | |
| 243 | |
| 244 | def test_pinning_still_has_the_urllib3_internals_it_relies_on() -> None: |
| 245 | """Fail loudly if a urllib3 upgrade removes what the adapter hooks into. |
| 246 | |
| 247 | Silently losing either of these would silently un-pin every connection. |
| 248 | """ |
| 249 | connection = urllib3.connection.HTTPConnection("example.com") |
| 250 | |
| 251 | assert hasattr(connection, "_new_conn") |
| 252 | assert hasattr(connection, "_dns_host") |
| 253 | |
| 254 | |
| 255 | def test_peer_mismatch_fails_closed() -> None: |
| 256 | """The last line of defence, checked before a byte is written to the socket.""" |
| 257 | sock = MagicMock() |
| 258 | sock.getpeername.return_value = ("169.254.169.254", 80) |
| 259 | |
| 260 | with pytest.raises(BlockedInternalUrlError, match="169.254.169.254"): |
| 261 | _assert_pinned_peer(sock, "93.184.216.34", HOSTNAME) |
| 262 | |
| 263 | sock.close.assert_called_once() |
| 264 | |
| 265 | |
| 266 | def test_connect_falls_back_to_the_next_validated_address( |
| 267 | no_inherited_proxy: None, |
| 268 | validated_server: _RecordingServer, |
| 269 | ) -> None: |
| 270 | """A dual-stack host must not be stranded on its first answer. |
| 271 | |
| 272 | ``::1`` is validated but nothing listens there, so the connect has to move |
| 273 | on to the second answer the way the HTTP client would have. urllib3 rewraps |
| 274 | connect failures into its own exception hierarchy, none of which inherits |
| 275 | from ``OSError``, so this also pins down which exceptions the fallback has |
| 276 | to catch. |
| 277 | """ |
| 278 | real_getaddrinfo = socket.getaddrinfo |
| 279 | |
| 280 | def fake_getaddrinfo(host, port, *args, **kwargs): # type: ignore[no-untyped-def] |
| 281 | if _is_ip_literal(host): |
| 282 | return real_getaddrinfo(host, port, *args, **kwargs) |
| 283 | return [ |
| 284 | ( |
| 285 | socket.AF_INET6, |
| 286 | socket.SOCK_STREAM, |
| 287 | 6, |
| 288 | "", |
| 289 | ("::1", validated_server.port, 0, 0), |
| 290 | ), |
| 291 | ( |
| 292 | socket.AF_INET, |
| 293 | socket.SOCK_STREAM, |
| 294 | 6, |
| 295 | "", |
| 296 | ("127.0.0.1", validated_server.port), |
| 297 | ), |
| 298 | ] |
| 299 | |
| 300 | with patch("socket.getaddrinfo", side_effect=fake_getaddrinfo): |
| 301 | with patch("composio.utils.url_safety.is_blocked_ip", return_value=False): |
| 302 | response = safe_get( |
| 303 | f"http://{HOSTNAME}:{validated_server.port}/payload", timeout=(5, 5) |
| 304 | ) |
| 305 | |
| 306 | assert response.content == b"public payload" |
| 307 | assert len(validated_server.hits) == 1 |