@samitouri / QOSami_Agents_Zero-2 / commits / 91f43e28

fix: preserve safe remote fetch compatibility for public sites

Restore remote document fetch compatibility for public sites after the CVE-2026-4308 SSRF hardening. The initial security fix correctly blocked non-public destinations, but it also changed the outbound request fingerprint for `document_query` remote fetches. Some public sites, including https://nvd.nist.gov/vuln/detail/CVE-2026-4308, used for testing, responded with HTTP 403 to the default `requests` user agent even though they remained safe and publicly routable. This change keeps the centralized SSRF protections in place while restoring the previous request compatibility behavior by sending the configured `USER_AGENT` header, falling back to the prior `@mixedbread-ai/unstructured` value. What is fixed: - public URLs such as `https://nvd.nist.gov/vuln/detail/CVE-2026-4308` no longer fail with site-specific HTTP 403 due to request fingerprint changes introduced by the SSRF mitigation

Alessandro committed Apr 12, 2026 at 02:08 UTC 91f43e28b4d8b1c557c1591834cc3a1958de2cee

1 file changed +12

helpers/network.py

+12

@@ -2,6 +2,7 @@ from __future__ import annotations

3 from dataclasses import dataclass

4 import ipaddress

5 +import os

6 import socket

7 import struct

8 from urllib.parse import urljoin, urlparse

@@ -11,6 +12,7 @@ import requests

13 SAFE_HTTP_SCHEMES = frozenset({"http", "https"})

14 DEFAULT_FETCH_TIMEOUT = (3.05, 10.0)

15 +DEFAULT_HTTP_USER_AGENT = "@mixedbread-ai/unstructured"

18 @dataclass(frozen=True)

@@ -25,6 +27,15 @@ class UnsafeUrlError(ValueError):

27 """Raised when a remote URL resolves to a non-public destination."""

30 +def _build_request_headers() -> dict[str, str]:

31 + user_agent = (

32 + os.getenv("USER_AGENT")

33 + or os.getenv("user_agent")

34 + or DEFAULT_HTTP_USER_AGENT

35 + ).strip()

36 + return {"User-Agent": user_agent or DEFAULT_HTTP_USER_AGENT}

37 +

38 +

39 def _normalize_content_type(content_type: str | None) -> str | None:

40 if not content_type:

41 return None

@@ -104,6 +115,7 @@ def fetch_public_http_resource(

115 current_url,

116 stream=True,

117 allow_redirects=False,

118 + headers=_build_request_headers(),

119 timeout=timeout,

120 ) as response:

121 if 300 <= response.status_code < 400: