| 1 | from __future__ import annotations |
| 2 | |
| 3 | import asyncio |
| 4 | import logging |
| 5 | import time |
| 6 | from concurrent.futures import ThreadPoolExecutor |
| 7 | from dataclasses import dataclass |
| 8 | from datetime import datetime, timezone |
| 9 | from email.utils import parsedate_to_datetime |
| 10 | from typing import Protocol |
| 11 | from functools import partial |
| 12 | |
| 13 | import httpx |
| 14 | from io import BytesIO |
| 15 | |
| 16 | from app.settings import Settings |
| 17 | from app.url_security import validate_public_http_url |
| 18 | |
| 19 | logger = logging.getLogger(__name__) |
| 20 | |
| 21 | |
| 22 | class ResearchFetcher(Protocol): |
| 23 | async def fetch(self, url: str) -> "FetchResult": |
| 24 | """Fetch permitted public content while respecting source limits and SSRF controls.""" |
| 25 | |
| 26 | |
| 27 | @dataclass(frozen=True) |
| 28 | class FetchResult: |
| 29 | final_url: str |
| 30 | status_code: int |
| 31 | content_type: str |
| 32 | text: str |
| 33 | bytes_read: int |
| 34 | extraction_status: str = "EXTRACTED" |
| 35 | |
| 36 | |
| 37 | @dataclass(frozen=True) |
| 38 | class NetworkFetchResult: |
| 39 | final_url: str |
| 40 | status_code: int |
| 41 | content_type: str |
| 42 | content: bytes |
| 43 | headers: dict[str, str] |
| 44 | is_redirect: bool |
| 45 | encoding: str | None |
| 46 | headers_elapsed_ms: int |
| 47 | body_elapsed_ms: int |
| 48 | network_elapsed_ms: int |
| 49 | |
| 50 | |
| 51 | class FetchError(RuntimeError): |
| 52 | pass |
| 53 | |
| 54 | |
| 55 | class HttpStatusFetchError(FetchError): |
| 56 | def __init__(self, status_code: int): |
| 57 | self.status_code = status_code |
| 58 | super().__init__(f"Source returned HTTP status {status_code}") |
| 59 | |
| 60 | |
| 61 | class TransportFetchError(FetchError): |
| 62 | pass |
| 63 | |
| 64 | |
| 65 | class PdfExtractionTimeoutError(FetchError): |
| 66 | pass |
| 67 | |
| 68 | |
| 69 | class DocumentSizeLimitExceeded(FetchError): |
| 70 | def __init__(self, content_length: int | None, max_bytes: int): |
| 71 | self.content_length = content_length |
| 72 | self.max_bytes = max_bytes |
| 73 | super().__init__("DOCUMENT_SIZE_LIMIT_EXCEEDED") |
| 74 | |
| 75 | |
| 76 | class RestrictedFetchError(HttpStatusFetchError): |
| 77 | pass |
| 78 | |
| 79 | |
| 80 | class HttpResearchFetcher: |
| 81 | def __init__(self, settings: Settings, client: httpx.AsyncClient | None = None): |
| 82 | self.settings = settings |
| 83 | timeout = httpx.Timeout( |
| 84 | timeout=settings.research_request_timeout_seconds, |
| 85 | connect=settings.research_connect_timeout_seconds, |
| 86 | ) |
| 87 | self._client = client or httpx.AsyncClient( |
| 88 | timeout=timeout, |
| 89 | follow_redirects=False, |
| 90 | headers={"User-Agent": settings.research_user_agent, "Accept-Encoding": "gzip, deflate, br"}, |
| 91 | ) |
| 92 | # PDF parsing is CPU/memory intensive. This gate deliberately covers |
| 93 | # only the blocking parse phase, never async network download. |
| 94 | self._pdf_extraction_semaphore = asyncio.Semaphore(settings.research_pdf_extraction_concurrency) |
| 95 | self._active_pdf_extractions: set[asyncio.Future[FetchResult]] = set() |
| 96 | self._pdf_extraction_executor = ThreadPoolExecutor( |
| 97 | max_workers=settings.research_pdf_extraction_concurrency, |
| 98 | thread_name_prefix="research-pdf-extraction", |
| 99 | ) |
| 100 | |
| 101 | async def fetch(self, url: str) -> FetchResult: |
| 102 | network = await self.fetch_network(url) |
| 103 | return await self.process_network_response_async(network) |
| 104 | |
| 105 | async def fetch_nse_shareholding_xbrl(self, url: str) -> FetchResult: |
| 106 | """Fetch one official NSE XBRL artifact with its required XML context. |
| 107 | |
| 108 | This is intentionally separate from the generic fetch path: NSE's |
| 109 | archive serves these public XML artifacts only when the request looks |
| 110 | like an NSE-site XML navigation. Network validation and response |
| 111 | processing remain exactly the shared secured implementation. |
| 112 | """ |
| 113 | headers = { |
| 114 | "User-Agent": "Mozilla/5.0 (compatible; AIInvestmentResearch/1.0)", |
| 115 | "Accept": "application/xml,text/xml,*/*", |
| 116 | "Referer": "https://www.nseindia.com/", |
| 117 | } |
| 118 | network = await self.fetch_network( |
| 119 | url, |
| 120 | headers=headers, |
| 121 | max_bytes=self.settings.research_max_content_bytes, |
| 122 | ) |
| 123 | return await self.process_network_response_async(network, max_bytes=self.settings.research_max_content_bytes) |
| 124 | |
| 125 | async def process_network_response_async( |
| 126 | self, |
| 127 | response: NetworkFetchResult, |
| 128 | *, |
| 129 | max_bytes: int | None = None, |
| 130 | extraction_timeout_seconds: float | None = None, |
| 131 | ) -> FetchResult: |
| 132 | """Process response without blocking the event loop on PDF parsing. |
| 133 | |
| 134 | A timeout cannot stop a running Python worker thread. The permit is |
| 135 | released only in that worker's done callback, so timed-out parsing |
| 136 | remains globally bounded until the actual underlying work ends. |
| 137 | """ |
| 138 | if getattr(response, "content_type", "") != "application/pdf": |
| 139 | return self.process_network_response(response, max_bytes=max_bytes) |
| 140 | queued_at = time.monotonic() |
| 141 | logger.info( |
| 142 | "pdf_extraction_queued host=%s path=%s documentBytes=%s configuredConcurrency=%s", |
| 143 | _safe_host(response.final_url), _safe_path(response.final_url), len(response.content), |
| 144 | self.settings.research_pdf_extraction_concurrency, |
| 145 | ) |
| 146 | await self._pdf_extraction_semaphore.acquire() |
| 147 | loop = asyncio.get_running_loop() |
| 148 | task = asyncio.ensure_future(loop.run_in_executor( |
| 149 | self._pdf_extraction_executor, |
| 150 | partial(self.process_network_response, response, max_bytes=max_bytes), |
| 151 | )) |
| 152 | self._active_pdf_extractions.add(task) |
| 153 | started_at = time.monotonic() |
| 154 | logger.info( |
| 155 | "pdf_extraction_started host=%s path=%s documentBytes=%s queueWaitMs=%s activeExtractions=%s configuredConcurrency=%s", |
| 156 | _safe_host(response.final_url), _safe_path(response.final_url), len(response.content), |
| 157 | _elapsed_ms(queued_at), len(self._active_pdf_extractions), self.settings.research_pdf_extraction_concurrency, |
| 158 | ) |
| 159 | |
| 160 | def release_when_done(completed: asyncio.Future[FetchResult]) -> None: |
| 161 | self._active_pdf_extractions.discard(completed) |
| 162 | self._pdf_extraction_semaphore.release() |
| 163 | logger.info( |
| 164 | "pdf_extraction_worker_released host=%s path=%s extractionElapsedMs=%s activeExtractions=%s configuredConcurrency=%s", |
| 165 | _safe_host(response.final_url), _safe_path(response.final_url), _elapsed_ms(started_at), |
| 166 | len(self._active_pdf_extractions), self.settings.research_pdf_extraction_concurrency, |
| 167 | ) |
| 168 | # Retrieve late worker exceptions after a caller timeout so they |
| 169 | # do not become unobserved-task warnings. |
| 170 | if not completed.cancelled(): |
| 171 | try: |
| 172 | completed.exception() |
| 173 | except Exception: |
| 174 | pass |
| 175 | |
| 176 | task.add_done_callback(release_when_done) |
| 177 | timeout = extraction_timeout_seconds |
| 178 | if timeout is None: |
| 179 | timeout = self.settings.research_official_document_extraction_timeout_seconds |
| 180 | try: |
| 181 | result = await asyncio.wait_for(asyncio.shield(task), timeout=timeout) |
| 182 | logger.info( |
| 183 | "pdf_extraction_completed host=%s path=%s extractionElapsedMs=%s activeExtractions=%s", |
| 184 | _safe_host(response.final_url), _safe_path(response.final_url), _elapsed_ms(started_at), |
| 185 | len(self._active_pdf_extractions), |
| 186 | ) |
| 187 | return result |
| 188 | except TimeoutError as exc: |
| 189 | logger.warning( |
| 190 | "pdf_extraction_timeout host=%s path=%s timeoutSeconds=%s activeExtractions=%s configuredConcurrency=%s", |
| 191 | _safe_host(response.final_url), |
| 192 | _safe_path(response.final_url), |
| 193 | timeout, |
| 194 | len(self._active_pdf_extractions), |
| 195 | self.settings.research_pdf_extraction_concurrency, |
| 196 | ) |
| 197 | raise PdfExtractionTimeoutError("PDF_EXTRACTION_TIMEOUT") from exc |
| 198 | |
| 199 | async def fetch_network( |
| 200 | self, |
| 201 | url: str, |
| 202 | *, |
| 203 | headers: dict[str, str] | None = None, |
| 204 | max_bytes: int | None = None, |
| 205 | ) -> NetworkFetchResult: |
| 206 | logger.info("fetch_validate_url_start host=%s path=%s", _safe_host(url), _safe_path(url)) |
| 207 | validate_public_http_url(url) |
| 208 | logger.info("fetch_validate_url_complete host=%s path=%s", _safe_host(url), _safe_path(url)) |
| 209 | current_url = url |
| 210 | for redirect_count in range(self.settings.research_max_redirects + 1): |
| 211 | response = await self._request_with_retries(current_url, headers=headers, max_bytes=max_bytes) |
| 212 | if response.is_redirect: |
| 213 | if redirect_count >= self.settings.research_max_redirects: |
| 214 | raise FetchError("Maximum redirects exceeded") |
| 215 | location = response.headers.get("location") |
| 216 | if not location: |
| 217 | raise FetchError("Redirect without Location header") |
| 218 | current_url = str(httpx.URL(response.final_url).join(location)) |
| 219 | validate_public_http_url(current_url) |
| 220 | continue |
| 221 | return response |
| 222 | raise FetchError("Maximum redirects exceeded") |
| 223 | |
| 224 | async def _request_with_retries( |
| 225 | self, |
| 226 | url: str, |
| 227 | *, |
| 228 | headers: dict[str, str] | None = None, |
| 229 | max_bytes: int | None = None, |
| 230 | ) -> NetworkFetchResult: |
| 231 | attempt = 0 |
| 232 | while True: |
| 233 | started = time.monotonic() |
| 234 | try: |
| 235 | logger.info("fetch_http_request_start host=%s path=%s", _safe_host(url), _safe_path(url)) |
| 236 | async with self._client.stream("GET", url, headers=headers) as response: |
| 237 | headers_elapsed_ms = _elapsed_ms(started) |
| 238 | logger.info( |
| 239 | "fetch_http_headers_received host=%s path=%s status=%s elapsedMs=%s", |
| 240 | _safe_host(url), _safe_path(url), response.status_code, headers_elapsed_ms, |
| 241 | ) |
| 242 | if response.status_code in {401, 403, 407, 451}: |
| 243 | raise RestrictedFetchError(response.status_code) |
| 244 | if 400 <= response.status_code < 500 and response.status_code not in {408, 429}: |
| 245 | raise HttpStatusFetchError(response.status_code) |
| 246 | content_length = _content_length(response.headers.get("content-length")) |
| 247 | if max_bytes is not None and content_length is not None and content_length > max_bytes: |
| 248 | raise DocumentSizeLimitExceeded(content_length, max_bytes) |
| 249 | content_parts: list[bytes] = [] |
| 250 | bytes_read = 0 |
| 251 | async for chunk in response.aiter_bytes(): |
| 252 | bytes_read += len(chunk) |
| 253 | if max_bytes is not None and bytes_read > max_bytes: |
| 254 | raise DocumentSizeLimitExceeded(content_length, max_bytes) |
| 255 | content_parts.append(chunk) |
| 256 | content = b"".join(content_parts) |
| 257 | body_elapsed_ms = _elapsed_ms(started) - headers_elapsed_ms |
| 258 | logger.info( |
| 259 | "fetch_http_body_complete host=%s path=%s status=%s bodyBytes=%s bodyElapsedMs=%s totalNetworkElapsedMs=%s", |
| 260 | _safe_host(url), _safe_path(url), response.status_code, len(content), body_elapsed_ms, _elapsed_ms(started), |
| 261 | ) |
| 262 | result = NetworkFetchResult( |
| 263 | final_url=str(response.url), status_code=response.status_code, |
| 264 | content_type=response.headers.get("content-type", "").split(";")[0].lower(), content=content, |
| 265 | headers=dict(response.headers), |
| 266 | is_redirect=response.is_redirect, |
| 267 | encoding=response.encoding, |
| 268 | headers_elapsed_ms=headers_elapsed_ms, body_elapsed_ms=body_elapsed_ms, |
| 269 | network_elapsed_ms=_elapsed_ms(started), |
| 270 | ) |
| 271 | except httpx.TimeoutException as exception: |
| 272 | response = None |
| 273 | last_error: Exception | None = exception |
| 274 | except httpx.TransportError as exception: |
| 275 | response = None |
| 276 | last_error = exception |
| 277 | else: |
| 278 | last_error = None |
| 279 | if result.status_code not in {408, 429, 500, 502, 503, 504}: |
| 280 | return result |
| 281 | response = result |
| 282 | if attempt >= self.settings.research_max_retries: |
| 283 | if response is not None: |
| 284 | return response |
| 285 | if isinstance(last_error, httpx.TimeoutException): |
| 286 | raise TransportFetchError("Fetch timed out") from last_error |
| 287 | raise TransportFetchError("Fetch transport failed") from last_error |
| 288 | retry_after = _retry_after_seconds(response.headers.get("retry-after")) if response else None |
| 289 | delay = retry_after if retry_after is not None else min(2**attempt, 8) |
| 290 | await asyncio.sleep(delay) |
| 291 | attempt += 1 |
| 292 | |
| 293 | def process_network_response(self, response: NetworkFetchResult, *, max_bytes: int | None = None) -> FetchResult: |
| 294 | content_type = response.content_type |
| 295 | allowed = content_type in { |
| 296 | "text/html", |
| 297 | "text/plain", |
| 298 | "application/xml", |
| 299 | "text/xml", |
| 300 | "application/rss+xml", |
| 301 | "application/pdf", |
| 302 | } |
| 303 | if not allowed: |
| 304 | raise FetchError(f"Unsupported content type: {content_type or 'unknown'}") |
| 305 | content = response.content |
| 306 | processing_max_bytes = max_bytes if max_bytes is not None else self.settings.research_max_content_bytes |
| 307 | if len(content) > processing_max_bytes: |
| 308 | raise FetchError("Maximum content size exceeded") |
| 309 | text = content.decode(response.encoding or "utf-8", errors="replace") |
| 310 | extraction_status = "EXTRACTED" |
| 311 | if content_type == "application/pdf": |
| 312 | if not content.startswith(b"%PDF-"): |
| 313 | raise FetchError("PDF_SIGNATURE_INVALID") |
| 314 | logger.info("fetch_pdf_signature_valid host=%s path=%s documentBytes=%s", _safe_host(response.final_url), _safe_path(response.final_url), len(content)) |
| 315 | started = time.monotonic() |
| 316 | logger.info("fetch_extraction_start host=%s path=%s documentBytes=%s", _safe_host(response.final_url), _safe_path(response.final_url), len(content)) |
| 317 | try: |
| 318 | from pypdf import PdfReader |
| 319 | reader = PdfReader(BytesIO(content)) |
| 320 | extracted_pages = [page.extract_text() or "" for page in reader.pages] |
| 321 | text = "\n".join(f"[PDF_PAGE {index}]\n{page}" for index, page in enumerate(extracted_pages, start=1)) |
| 322 | except Exception as exc: |
| 323 | raise FetchError("PDF_TEXT_EXTRACTION_FAILED") from exc |
| 324 | if not any(page.strip() for page in extracted_pages): |
| 325 | extraction_status = "PDF_SCANNED_OCR_REQUIRED" |
| 326 | logger.info( |
| 327 | "fetch_extraction_complete host=%s path=%s pageCount=%s extractionElapsedMs=%s extractionStatus=%s", |
| 328 | _safe_host(response.final_url), _safe_path(response.final_url), len(extracted_pages), _elapsed_ms(started), extraction_status, |
| 329 | ) |
| 330 | return FetchResult( |
| 331 | final_url=response.final_url, |
| 332 | status_code=response.status_code, |
| 333 | content_type=content_type, |
| 334 | text=text, |
| 335 | bytes_read=len(content), |
| 336 | extraction_status=extraction_status, |
| 337 | ) |
| 338 | |
| 339 | |
| 340 | class PlaywrightResearchFetcher: |
| 341 | def __init__(self, settings: Settings): |
| 342 | self.settings = settings |
| 343 | self._semaphore = asyncio.Semaphore(settings.research_playwright_concurrency) |
| 344 | |
| 345 | async def fetch(self, url: str) -> FetchResult: |
| 346 | validate_public_http_url(url) |
| 347 | if not self.settings.research_playwright_enabled: |
| 348 | raise RestrictedFetchError("Playwright fallback is disabled") |
| 349 | async with self._semaphore: |
| 350 | raise RestrictedFetchError("Playwright runtime is not bundled in Phase 3 default image") |
| 351 | |
| 352 | |
| 353 | def _retry_after_seconds(value: str | None) -> float | None: |
| 354 | if not value: |
| 355 | return None |
| 356 | try: |
| 357 | return max(float(value), 0.0) |
| 358 | except ValueError: |
| 359 | try: |
| 360 | delta = parsedate_to_datetime(value) - datetime.now(timezone.utc) |
| 361 | except (TypeError, ValueError): |
| 362 | return None |
| 363 | return max(delta.total_seconds(), 0.0) |
| 364 | |
| 365 | |
| 366 | def _safe_host(url: str) -> str: |
| 367 | try: |
| 368 | from urllib.parse import urlparse |
| 369 | return (urlparse(url).hostname or "").lower() |
| 370 | except ValueError: |
| 371 | return "" |
| 372 | |
| 373 | |
| 374 | def _safe_path(url: str) -> str: |
| 375 | try: |
| 376 | from urllib.parse import urlparse |
| 377 | return urlparse(url).path or "/" |
| 378 | except ValueError: |
| 379 | return "/" |
| 380 | |
| 381 | |
| 382 | def _elapsed_ms(started: float) -> int: |
| 383 | return round((time.monotonic() - started) * 1000) |
| 384 | |
| 385 | |
| 386 | def _content_length(value: str | None) -> int | None: |
| 387 | if not value: |
| 388 | return None |
| 389 | try: |
| 390 | parsed = int(value) |
| 391 | except ValueError: |
| 392 | return None |
| 393 | return parsed if parsed >= 0 else None |