Fix SSRF in document_query remote fetching (CVE-2026-4308)
Address CVE-2026-4308 in the document_query tool remote-fetch path. The issue was originally reported by @YLChen-007. This change replaces ad hoc remote document fetching with a centralized safe fetch flow that validates remote URLs before any network request is used for parsing. It blocks localhost and non-public IPv4/IPv6 targets, validates every redirect hop, disables implicit trust of proxy env settings for this path, and enforces a strict remote document size cap. It also removes direct third-party loader access to attacker-controlled URLs by prefetching remote content first and then parsing only trusted local bytes or temp files for HTML, text, PDF, image, and unstructured document handling. Refs: - CVE-2026-4308 - Report by @YLChen-007
Alessandro committed
Apr 12, 2026 at 02:00 UTC
6397acc092a538594186c6a2bacdcfa516ca0747
2 files changed
+279
-73
helpers/document_query.py
+120
-73
@@ -1,7 +1,6 @@
1
import mimetypes
2
import os
3
import asyncio
4
-import aiohttp
4
import json
5
6
from helpers.vector_db import VectorDB
@@ -13,8 +12,6 @@ from urllib.parse import urlparse
12
from typing import Callable, Sequence, List, Optional, Tuple
13
from datetime import datetime
14
16
-from langchain_community.document_loaders import AsyncHtmlLoader
17
-from langchain_community.document_loaders.text import TextLoader
15
from langchain_community.document_loaders.pdf import PyMuPDFLoader
16
from langchain_community.document_transformers import MarkdownifyTransformer
17
from langchain_community.document_loaders.parsers.images import TesseractBlobParser
@@ -24,12 +21,14 @@ from langchain.schema import SystemMessage, HumanMessage
21
22
from helpers.print_style import PrintStyle
23
from helpers import files, errors
24
+from helpers.network import HttpFetchResult, fetch_public_http_resource
25
from agent import Agent
26
27
from langchain.text_splitter import RecursiveCharacterTextSplitter
28
29
30
DEFAULT_SEARCH_THRESHOLD = 0.5
31
+MAX_REMOTE_DOCUMENT_BYTES = 50 * 1024 * 1024
32
33
34
class DocumentQueryStore:
@@ -450,45 +449,19 @@ class DocumentQueryHelper:
449
scheme = url.scheme or "file"
450
mimetype, encoding = mimetypes.guess_type(document_uri)
451
mimetype = mimetype or "application/octet-stream"
452
+ remote_resource: HttpFetchResult | None = None
453
454
- if mimetype == "application/octet-stream":
455
- if url.scheme in ["http", "https"]:
456
- response: aiohttp.ClientResponse | None = None
457
- retries = 0
458
- last_error = ""
459
- while not response and retries < 3:
460
- try:
461
- async with aiohttp.ClientSession() as session:
462
- response = await session.head(
463
- document_uri,
464
- timeout=aiohttp.ClientTimeout(total=2.0),
465
- allow_redirects=True,
466
- )
467
- if response.status > 399:
468
- raise Exception(response.status)
469
- break
470
- except Exception as e:
471
- await asyncio.sleep(1)
472
- last_error = str(e)
473
- retries += 1
474
- await self.agent.handle_intervention()
475
-
476
- if not response:
477
- raise ValueError(
478
- f"DocumentQueryHelper::document_get_content: Document fetch error: {document_uri} ({last_error})"
479
- )
480
-
481
- mimetype = response.headers["content-type"]
482
- if "content-length" in response.headers:
483
- content_length = (
484
- float(response.headers["content-length"]) / 1024 / 1024
485
- ) # MB
486
- if content_length > 50.0:
487
- raise ValueError(
488
- f"Document content length exceeds max. 50MB: {content_length} MB ({document_uri})"
489
- )
490
- if mimetype and "; charset=" in mimetype:
491
- mimetype = mimetype.split("; charset=")[0]
454
+ if scheme in ["http", "https"]:
455
+ remote_resource = await asyncio.to_thread(
456
+ fetch_public_http_resource,
457
+ document_uri,
458
+ max_bytes=MAX_REMOTE_DOCUMENT_BYTES,
459
+ )
460
+ if (
461
+ remote_resource.content_type
462
+ and remote_resource.content_type != "application/octet-stream"
463
+ ):
464
+ mimetype = remote_resource.content_type
465
466
if scheme == "file":
467
try:
@@ -515,16 +488,24 @@ class DocumentQueryHelper:
488
if not exists:
489
await self.agent.handle_intervention()
490
if mimetype.startswith("image/"):
518
- document_content = self.handle_image_document(document_uri, scheme)
491
+ document_content = self.handle_image_document(
492
+ document_uri, scheme, remote_resource=remote_resource
493
+ )
494
elif mimetype == "text/html":
520
- document_content = self.handle_html_document(document_uri, scheme)
495
+ document_content = self.handle_html_document(
496
+ document_uri, scheme, remote_resource=remote_resource
497
+ )
498
elif mimetype.startswith("text/") or mimetype == "application/json":
522
- document_content = self.handle_text_document(document_uri, scheme)
499
+ document_content = self.handle_text_document(
500
+ document_uri, scheme, remote_resource=remote_resource
501
+ )
502
elif mimetype == "application/pdf":
524
- document_content = self.handle_pdf_document(document_uri, scheme)
503
+ document_content = self.handle_pdf_document(
504
+ document_uri, scheme, remote_resource=remote_resource
505
+ )
506
else:
507
document_content = self.handle_unstructured_document(
527
- document_uri, scheme
508
+ document_uri, scheme, remote_resource=remote_resource
509
)
510
if add_to_db:
511
self.progress_callback(f"Indexing document")
@@ -550,13 +531,53 @@ class DocumentQueryHelper:
531
)
532
return document_content
533
553
- def handle_image_document(self, document: str, scheme: str) -> str:
554
- return self.handle_unstructured_document(document, scheme)
534
+ @staticmethod
535
+ def _decode_remote_text(remote_resource: HttpFetchResult) -> str:
536
+ encoding = remote_resource.encoding or "utf-8"
537
+ try:
538
+ return remote_resource.content.decode(encoding)
539
+ except (LookupError, UnicodeDecodeError):
540
+ return remote_resource.content.decode("utf-8", errors="replace")
541
556
- def handle_html_document(self, document: str, scheme: str) -> str:
542
+ @staticmethod
543
+ def _get_temp_file_suffix(
544
+ document: str, remote_resource: HttpFetchResult | None = None
545
+ ) -> str:
546
+ parsed = urlparse(document)
547
+ _stem, ext = os.path.splitext(parsed.path or document)
548
+ if ext:
549
+ return ext
550
+
551
+ if remote_resource and remote_resource.content_type:
552
+ guessed_ext = mimetypes.guess_extension(
553
+ remote_resource.content_type, strict=False
554
+ )
555
+ if guessed_ext:
556
+ return guessed_ext
557
+
558
+ return ".bin"
559
+
560
+ def handle_image_document(
561
+ self,
562
+ document: str,
563
+ scheme: str,
564
+ remote_resource: HttpFetchResult | None = None,
565
+ ) -> str:
566
+ return self.handle_unstructured_document(
567
+ document, scheme, remote_resource=remote_resource
568
+ )
569
+
570
+ def handle_html_document(
571
+ self,
572
+ document: str,
573
+ scheme: str,
574
+ remote_resource: HttpFetchResult | None = None,
575
+ ) -> str:
576
if scheme in ["http", "https"]:
558
- loader = AsyncHtmlLoader(web_path=document)
559
- parts: list[Document] = loader.load()
577
+ if remote_resource is None:
578
+ raise ValueError("Missing prefetched remote HTML content")
579
+ html_content = self._decode_remote_text(remote_resource)
580
+ parts = [Document(page_content=html_content, metadata={"source": document})]
581
elif scheme == "file":
582
# Use RFC file operations instead of TextLoader
583
file_content_bytes = files.read_file_bin(document)
@@ -573,10 +594,19 @@ class DocumentQueryHelper:
594
]
595
)
596
576
- def handle_text_document(self, document: str, scheme: str) -> str:
597
+ def handle_text_document(
598
+ self,
599
+ document: str,
600
+ scheme: str,
601
+ remote_resource: HttpFetchResult | None = None,
602
+ ) -> str:
603
if scheme in ["http", "https"]:
578
- loader = AsyncHtmlLoader(web_path=document)
579
- elements: list[Document] = loader.load()
604
+ if remote_resource is None:
605
+ raise ValueError("Missing prefetched remote text content")
606
+ file_content = self._decode_remote_text(remote_resource)
607
+ elements = [
608
+ Document(page_content=file_content, metadata={"source": document})
609
+ ]
610
elif scheme == "file":
611
# Use RFC file operations instead of TextLoader
612
file_content_bytes = files.read_file_bin(document)
@@ -590,7 +620,12 @@ class DocumentQueryHelper:
620
621
return "\n".join([element.page_content for element in elements])
622
593
- def handle_pdf_document(self, document: str, scheme: str) -> str:
623
+ def handle_pdf_document(
624
+ self,
625
+ document: str,
626
+ scheme: str,
627
+ remote_resource: HttpFetchResult | None = None,
628
+ ) -> str:
629
temp_file_path = ""
630
if scheme == "file":
631
# Use RFC file operations to read the PDF file as binary
@@ -602,17 +637,12 @@ class DocumentQueryHelper:
637
temp_file.write(file_content_bytes)
638
temp_file_path = temp_file.name
639
elif scheme in ["http", "https"]:
605
- # download the file from the web url to a temporary file using python libraries for downloading
606
- import requests
640
import tempfile
641
642
+ if remote_resource is None:
643
+ raise ValueError("Missing prefetched remote PDF content")
644
with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:
610
- response = requests.get(document, timeout=10.0)
611
- if response.status_code != 200:
612
- raise ValueError(
613
- f"DocumentQueryHelper::handle_pdf_document: Failed to download PDF from {document}: {response.status_code}"
614
- )
615
- temp_file.write(response.content)
645
+ temp_file.write(remote_resource.content)
646
temp_file_path = temp_file.name
647
else:
648
raise ValueError(f"Unsupported scheme: {scheme}")
@@ -658,18 +688,35 @@ class DocumentQueryHelper:
688
finally:
689
os.unlink(temp_file_path)
690
661
- def handle_unstructured_document(self, document: str, scheme: str) -> str:
691
+ def handle_unstructured_document(
692
+ self,
693
+ document: str,
694
+ scheme: str,
695
+ remote_resource: HttpFetchResult | None = None,
696
+ ) -> str:
697
elements: list[Document] = []
698
if scheme in ["http", "https"]:
664
- # loader = UnstructuredURLLoader(urls=[document], mode="single")
665
- loader = UnstructuredLoader(
666
- web_url=document,
667
- mode="single",
668
- partition_via_api=False,
669
- # chunking_strategy="by_page",
670
- strategy="hi_res",
671
- )
672
- elements = loader.load()
699
+ if remote_resource is None:
700
+ raise ValueError("Missing prefetched remote document content")
701
+ import tempfile
702
+
703
+ temp_file_path = ""
704
+ suffix = self._get_temp_file_suffix(document, remote_resource)
705
+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
706
+ temp_file.write(remote_resource.content)
707
+ temp_file_path = temp_file.name
708
+
709
+ try:
710
+ loader = UnstructuredLoader(
711
+ file_path=temp_file_path,
712
+ mode="single",
713
+ partition_via_api=False,
714
+ # chunking_strategy="by_page",
715
+ strategy="hi_res",
716
+ )
717
+ elements = loader.load()
718
+ finally:
719
+ os.unlink(temp_file_path)
720
elif scheme == "file":
721
# Use RFC file operations to read the file as binary
722
file_content_bytes = files.read_file_bin(document)
helpers/network.py
+159
@@ -1,5 +1,164 @@
1
+from __future__ import annotations
2
+
3
+from dataclasses import dataclass
4
+import ipaddress
5
import socket
6
import struct
7
+from urllib.parse import urljoin, urlparse
8
+
9
+import requests
10
+
11
+
12
+SAFE_HTTP_SCHEMES = frozenset({"http", "https"})
13
+DEFAULT_FETCH_TIMEOUT = (3.05, 10.0)
14
+
15
+
16
+@dataclass(frozen=True)
17
+class HttpFetchResult:
18
+ url: str
19
+ content: bytes
20
+ content_type: str | None
21
+ encoding: str | None
22
+
23
+
24
+class UnsafeUrlError(ValueError):
25
+ """Raised when a remote URL resolves to a non-public destination."""
26
+
27
+
28
+def _normalize_content_type(content_type: str | None) -> str | None:
29
+ if not content_type:
30
+ return None
31
+ return content_type.split(";", 1)[0].strip().lower() or None
32
+
33
+
34
+def resolve_host_ips(hostname: str) -> tuple[ipaddress._BaseAddress, ...]:
35
+ try:
36
+ results = socket.getaddrinfo(
37
+ hostname,
38
+ None,
39
+ family=socket.AF_UNSPEC,
40
+ type=socket.SOCK_STREAM,
41
+ )
42
+ except socket.gaierror as exc:
43
+ raise UnsafeUrlError(f"Unable to resolve hostname '{hostname}'") from exc
44
+
45
+ ips: list[ipaddress._BaseAddress] = []
46
+ seen: set[str] = set()
47
+ for _family, _type, _proto, _canonname, sockaddr in results:
48
+ address = sockaddr[0]
49
+ if "%" in address:
50
+ address = address.split("%", 1)[0]
51
+ ip = ipaddress.ip_address(address)
52
+ key = ip.compressed
53
+ if key in seen:
54
+ continue
55
+ seen.add(key)
56
+ ips.append(ip)
57
+
58
+ if not ips:
59
+ raise UnsafeUrlError(f"Hostname '{hostname}' did not resolve to an IP address")
60
+
61
+ return tuple(ips)
62
+
63
+
64
+def validate_public_http_url(url: str) -> tuple[ipaddress._BaseAddress, ...]:
65
+ parsed = urlparse(url)
66
+
67
+ if parsed.scheme not in SAFE_HTTP_SCHEMES:
68
+ raise UnsafeUrlError("Only http:// and https:// URLs are supported")
69
+ if not parsed.hostname:
70
+ raise UnsafeUrlError("URL hostname is required")
71
+ if parsed.username or parsed.password:
72
+ raise UnsafeUrlError("URLs with embedded credentials are not allowed")
73
+
74
+ hostname = parsed.hostname.rstrip(".").lower()
75
+ if hostname == "localhost" or hostname.endswith(".localhost"):
76
+ raise UnsafeUrlError(f"Blocked local hostname '{hostname}'")
77
+
78
+ ips = resolve_host_ips(hostname)
79
+ blocked = [str(ip) for ip in ips if not ip.is_global]
80
+ if blocked:
81
+ raise UnsafeUrlError(
82
+ f"Blocked non-public address resolution for '{hostname}': {', '.join(blocked)}"
83
+ )
84
+
85
+ return ips
86
+
87
+
88
+def fetch_public_http_resource(
89
+ url: str,
90
+ *,
91
+ max_bytes: int,
92
+ max_redirects: int = 5,
93
+ timeout: tuple[float, float] = DEFAULT_FETCH_TIMEOUT,
94
+) -> HttpFetchResult:
95
+ current_url = url
96
+ session = requests.Session()
97
+ session.trust_env = False
98
+
99
+ for redirect_count in range(max_redirects + 1):
100
+ validate_public_http_url(current_url)
101
+
102
+ try:
103
+ with session.get(
104
+ current_url,
105
+ stream=True,
106
+ allow_redirects=False,
107
+ timeout=timeout,
108
+ ) as response:
109
+ if 300 <= response.status_code < 400:
110
+ location = response.headers.get("Location")
111
+ if not location:
112
+ raise ValueError(
113
+ f"Remote URL redirect is missing a Location header: {current_url}"
114
+ )
115
+ if redirect_count >= max_redirects:
116
+ raise ValueError(
117
+ f"Remote URL exceeded redirect limit ({max_redirects}): {url}"
118
+ )
119
+ current_url = urljoin(current_url, location)
120
+ continue
121
+
122
+ if response.status_code >= 400:
123
+ raise ValueError(
124
+ f"Remote URL returned HTTP {response.status_code}: {current_url}"
125
+ )
126
+
127
+ content_length = response.headers.get("Content-Length")
128
+ if content_length:
129
+ try:
130
+ declared_length = int(content_length)
131
+ except ValueError:
132
+ declared_length = None
133
+ if declared_length is not None and declared_length > max_bytes:
134
+ raise ValueError(
135
+ f"Remote document exceeds max size {max_bytes} bytes: {current_url}"
136
+ )
137
+
138
+ body = bytearray()
139
+ for chunk in response.iter_content(chunk_size=64 * 1024):
140
+ if not chunk:
141
+ continue
142
+ body.extend(chunk)
143
+ if len(body) > max_bytes:
144
+ raise ValueError(
145
+ f"Remote document exceeds max size {max_bytes} bytes: {current_url}"
146
+ )
147
+
148
+ return HttpFetchResult(
149
+ url=current_url,
150
+ content=bytes(body),
151
+ content_type=_normalize_content_type(
152
+ response.headers.get("Content-Type")
153
+ ),
154
+ encoding=response.encoding,
155
+ )
156
+ except requests.RequestException as exc:
157
+ raise ValueError(
158
+ f"Remote document fetch failed for {current_url}: {exc}"
159
+ ) from exc
160
+
161
+ raise ValueError(f"Remote URL exceeded redirect limit ({max_redirects}): {url}")
162
163
164
def is_loopback_address(address: str) -> bool: