@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
py 307 lines 11.3 KB
Raw
1 """Regression tests for the DNS-rebinding window in the SSRF guard.
2
3 The guard resolves a hostname to decide whether a URL is safe. If the HTTP
4 client then resolves that hostname a *second* time to open the socket, an
5 attacker who controls the authoritative DNS can answer with a public address
6 for the check and an internal one for the connect — the check and the use are
7 about different addresses (issue #4151).
8
9 These tests run against real sockets on loopback rather than mocking the HTTP
10 client, because a mocked client cannot resolve anything twice and so cannot
11 express the bug at all. The resolver is the only thing faked: it answers the
12 first lookup of the hostname with one endpoint and every later lookup with
13 another, which is exactly what a short-TTL rebinding record does.
14 """
15
16 from __future__ import annotations
17
18 import ipaddress
19 import os
20 import socket
21 import threading
22 import typing as t
23 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
24 from unittest.mock import MagicMock, patch
25
26 import pytest
27 import requests
28 import urllib3.connection
29
30 from composio.exceptions import BlockedInternalUrlError
31 from composio.utils.url_safety import _assert_pinned_peer, safe_get
32
33 HOSTNAME = "rebind.test"
34
35
36 def _is_ip_literal(host: str) -> bool:
37 try:
38 ipaddress.ip_address(host)
39 except ValueError:
40 return False
41 return True
42
43
44 class _RecordingServer:
45 """A loopback HTTP server that records what actually reached it."""
46
47 def __init__(self, body: bytes) -> None:
48 self.hits: t.List[str] = []
49 recorder = self
50
51 class Handler(BaseHTTPRequestHandler):
52 protocol_version = "HTTP/1.1"
53
54 def do_GET(self) -> None: # noqa: N802 - stdlib naming
55 recorder.hits.append(self.headers.get("Host", ""))
56 self.send_response(200)
57 self.send_header("Content-Type", "text/plain")
58 self.send_header("Content-Length", str(len(body)))
59 self.end_headers()
60 self.wfile.write(body)
61
62 def log_message(self, *args: t.Any) -> None:
63 pass
64
65 self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
66 # Keep-alive connections would otherwise hold `shutdown()` open.
67 self._server.daemon_threads = True
68 self.port = self._server.server_address[1]
69 self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
70 self._thread.start()
71
72 def close(self) -> None:
73 self._server.shutdown()
74 self._server.server_close()
75
76
77 @pytest.fixture
78 def validated_server() -> t.Iterator[_RecordingServer]:
79 server = _RecordingServer(b"public payload")
80 yield server
81 server.close()
82
83
84 @pytest.fixture
85 def rebound_server() -> t.Iterator[_RecordingServer]:
86 server = _RecordingServer(b"internal secret")
87 yield server
88 server.close()
89
90
91 @pytest.fixture
92 def no_inherited_proxy(monkeypatch: pytest.MonkeyPatch) -> None:
93 """Make ``_proxy_applies`` deterministic: no proxy, from env or system.
94
95 An inherited ``HTTP_PROXY`` (or a macOS/Windows system proxy, which
96 ``getproxies`` also reads) would disable the pinning adapter, and the
97 request would dial the proxy's answer to the hostname instead of the
98 validated address — failing these tests on machines that carry one.
99
100 ``requests.sessions`` imports ``get_environ_proxies`` by name, so the
101 session's own proxy lookup needs its own patch: without it the SDK sees no
102 proxy and pins, while the session still routes through the inherited one.
103 """
104 for module in ("requests.utils", "requests.sessions"):
105 monkeypatch.setattr(
106 f"{module}.get_environ_proxies",
107 lambda url, no_proxy=None: {},
108 )
109
110
111 @pytest.fixture
112 def rebinding_dns(
113 no_inherited_proxy: None,
114 validated_server: _RecordingServer,
115 rebound_server: _RecordingServer,
116 ) -> t.Iterator[None]:
117 """Answer the first lookup of the hostname benignly, later ones with the victim.
118
119 Both endpoints live on 127.0.0.1 because a test cannot bind a public
120 address, so they are told apart by port; `is_blocked_ip` is stubbed out for
121 the same reason. Neither substitution touches the property under test,
122 which is whether the fetch connects to the endpoint that was validated or
123 re-resolves and connects somewhere else. The blocklist itself is covered by
124 ``test_url_safety.py``.
125 """
126 lookups: t.List[str] = []
127 real_getaddrinfo = socket.getaddrinfo
128
129 def fake_getaddrinfo(host, port, *args, **kwargs): # type: ignore[no-untyped-def]
130 # A real resolver hands an IP literal straight back, which is what the
131 # fix relies on when it connects to the address it already validated.
132 if _is_ip_literal(host):
133 return real_getaddrinfo(host, port, *args, **kwargs)
134
135 lookups.append(host)
136 victim = rebound_server if len(lookups) > 1 else validated_server
137 return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", victim.port))]
138
139 with patch("socket.getaddrinfo", side_effect=fake_getaddrinfo):
140 with patch("composio.utils.url_safety.is_blocked_ip", return_value=False):
141 yield
142
143
144 def test_proxy_environment_keeps_only_the_pre_flight_check(
145 validated_server: _RecordingServer,
146 ) -> None:
147 """Behind a proxy the SDK cannot pin: the proxy resolves the hostname.
148
149 Documented residual — ``_proxy_applies`` disables the pinning adapter, so
150 the request dials the proxy. A proxy nothing can reach makes that
151 observable: the request must fail by trying the proxy, never by dialling
152 the validated address. (This test deliberately does not use
153 ``no_inherited_proxy`` — it needs ``HTTP_PROXY`` visible — so it patches
154 the resolver inline instead.)
155 """
156 real_getaddrinfo = socket.getaddrinfo
157
158 def fake_getaddrinfo(host, port, *args, **kwargs): # type: ignore[no-untyped-def]
159 if _is_ip_literal(host):
160 return real_getaddrinfo(host, port, *args, **kwargs)
161 return [
162 (
163 socket.AF_INET,
164 socket.SOCK_STREAM,
165 6,
166 "",
167 ("127.0.0.1", validated_server.port),
168 )
169 ]
170
171 # Empty NO_PROXY so no machine-level bypass list can cover the hostname.
172 env = {"HTTP_PROXY": "http://127.0.0.1:9", "NO_PROXY": "", "no_proxy": ""}
173 with patch("socket.getaddrinfo", side_effect=fake_getaddrinfo):
174 with patch("composio.utils.url_safety.is_blocked_ip", return_value=False):
175 with patch("composio.utils.url_safety._PinnedAddressAdapter") as adapter:
176 with patch.dict(os.environ, env):
177 with pytest.raises(requests.exceptions.RequestException):
178 safe_get(
179 f"http://{HOSTNAME}:{validated_server.port}/payload",
180 timeout=(1, 1),
181 )
182
183 adapter.assert_not_called()
184 assert validated_server.hits == []
185
186
187 @pytest.mark.usefixtures("rebinding_dns")
188 def test_fetch_connects_to_the_address_it_validated(
189 validated_server: _RecordingServer, rebound_server: _RecordingServer
190 ) -> None:
191 response = safe_get(
192 f"http://{HOSTNAME}:{validated_server.port}/payload", timeout=(5, 5)
193 )
194
195 assert response.content == b"public payload"
196 assert len(validated_server.hits) == 1
197 # The whole point: the rebound endpoint is never even connected to, so
198 # neither its body nor a bare TCP connection to it is available to the
199 # attacker.
200 assert rebound_server.hits == []
201
202
203 @pytest.mark.usefixtures("rebinding_dns")
204 def test_pinning_ignores_an_inherited_proxy(
205 monkeypatch: pytest.MonkeyPatch,
206 validated_server: _RecordingServer,
207 rebound_server: _RecordingServer,
208 ) -> None:
209 """``no_inherited_proxy`` must hold on machines that carry a proxy (#5249).
210
211 A Windows registry or macOS system proxy reaches Requests through the same
212 ``getproxies`` lookup as these variables, so an unreachable proxy here
213 stands in for one this suite would otherwise inherit on those machines.
214 """
215 for name in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"):
216 monkeypatch.setenv(name, "http://127.0.0.1:9")
217 for name in ("NO_PROXY", "no_proxy"):
218 monkeypatch.setenv(name, "")
219
220 response = safe_get(
221 f"http://{HOSTNAME}:{validated_server.port}/payload", timeout=(5, 5)
222 )
223
224 assert response.content == b"public payload"
225 assert len(validated_server.hits) == 1
226 assert rebound_server.hits == []
227
228
229 @pytest.mark.usefixtures("rebinding_dns")
230 def test_pinning_keeps_the_hostname_on_the_wire(
231 validated_server: _RecordingServer,
232 ) -> None:
233 """The address is pinned; the hostname is not replaced by it.
234
235 Pinning by rewriting the connection's host would send ``Host: 127.0.0.1``
236 and offer the IP as TLS SNI, which fails certificate verification against
237 every real origin.
238 """
239 safe_get(f"http://{HOSTNAME}:{validated_server.port}/payload", timeout=(5, 5))
240
241 assert validated_server.hits == [f"{HOSTNAME}:{validated_server.port}"]
242
243
244 def test_pinning_still_has_the_urllib3_internals_it_relies_on() -> None:
245 """Fail loudly if a urllib3 upgrade removes what the adapter hooks into.
246
247 Silently losing either of these would silently un-pin every connection.
248 """
249 connection = urllib3.connection.HTTPConnection("example.com")
250
251 assert hasattr(connection, "_new_conn")
252 assert hasattr(connection, "_dns_host")
253
254
255 def test_peer_mismatch_fails_closed() -> None:
256 """The last line of defence, checked before a byte is written to the socket."""
257 sock = MagicMock()
258 sock.getpeername.return_value = ("169.254.169.254", 80)
259
260 with pytest.raises(BlockedInternalUrlError, match="169.254.169.254"):
261 _assert_pinned_peer(sock, "93.184.216.34", HOSTNAME)
262
263 sock.close.assert_called_once()
264
265
266 def test_connect_falls_back_to_the_next_validated_address(
267 no_inherited_proxy: None,
268 validated_server: _RecordingServer,
269 ) -> None:
270 """A dual-stack host must not be stranded on its first answer.
271
272 ``::1`` is validated but nothing listens there, so the connect has to move
273 on to the second answer the way the HTTP client would have. urllib3 rewraps
274 connect failures into its own exception hierarchy, none of which inherits
275 from ``OSError``, so this also pins down which exceptions the fallback has
276 to catch.
277 """
278 real_getaddrinfo = socket.getaddrinfo
279
280 def fake_getaddrinfo(host, port, *args, **kwargs): # type: ignore[no-untyped-def]
281 if _is_ip_literal(host):
282 return real_getaddrinfo(host, port, *args, **kwargs)
283 return [
284 (
285 socket.AF_INET6,
286 socket.SOCK_STREAM,
287 6,
288 "",
289 ("::1", validated_server.port, 0, 0),
290 ),
291 (
292 socket.AF_INET,
293 socket.SOCK_STREAM,
294 6,
295 "",
296 ("127.0.0.1", validated_server.port),
297 ),
298 ]
299
300 with patch("socket.getaddrinfo", side_effect=fake_getaddrinfo):
301 with patch("composio.utils.url_safety.is_blocked_ip", return_value=False):
302 response = safe_get(
303 f"http://{HOSTNAME}:{validated_server.port}/payload", timeout=(5, 5)
304 )
305
306 assert response.content == b"public payload"
307 assert len(validated_server.hits) == 1