| 1 | """Base parser with built-in thread offload and timeout.""" |
| 2 | |
| 3 | import asyncio |
| 4 | from abc import ABC, abstractmethod |
| 5 | |
| 6 | from helpers.print_style import PrintStyle |
| 7 | from plugins._document_query.helpers.fetch import FetchedDocument |
| 8 | |
| 9 | |
| 10 | class BaseParser(ABC): |
| 11 | """Abstract base for document parsers. |
| 12 | |
| 13 | Every parser runs synchronously but is automatically offloaded to a |
| 14 | thread pool and bounded by a configurable timeout when called through |
| 15 | parse(). This prevents any single parser from blocking the asyncio |
| 16 | event loop. |
| 17 | """ |
| 18 | |
| 19 | mimetypes: list[str] = [] |
| 20 | |
| 21 | def enabled(self, config: dict) -> bool: |
| 22 | return True |
| 23 | |
| 24 | def can_handle(self, mimetype: str) -> bool: |
| 25 | """Return True if this parser supports *mimetype*.""" |
| 26 | for pattern in self.mimetypes: |
| 27 | if pattern == "*": |
| 28 | return True |
| 29 | if pattern.endswith("/"): |
| 30 | if mimetype.startswith(pattern): |
| 31 | return True |
| 32 | elif mimetype == pattern: |
| 33 | return True |
| 34 | return False |
| 35 | |
| 36 | async def parse( |
| 37 | self, |
| 38 | document: FetchedDocument, |
| 39 | config: dict, |
| 40 | timeout: float = 60.0, |
| 41 | thread_offload: bool = True, |
| 42 | ) -> str: |
| 43 | try: |
| 44 | if thread_offload: |
| 45 | return await asyncio.wait_for( |
| 46 | asyncio.to_thread(self._parse_sync, document, config), |
| 47 | timeout=timeout, |
| 48 | ) |
| 49 | else: |
| 50 | return await asyncio.wait_for( |
| 51 | self._parse_async(document, config), |
| 52 | timeout=timeout, |
| 53 | ) |
| 54 | except asyncio.TimeoutError: |
| 55 | PrintStyle.error( |
| 56 | f"Parser {self.__class__.__name__} timed out after {timeout}s on {document.uri}" |
| 57 | ) |
| 58 | raise ValueError(f"Document parsing timed out after {timeout}s: {document.uri}") |
| 59 | |
| 60 | async def _parse_async(self, document: FetchedDocument, config: dict) -> str: |
| 61 | return self._parse_sync(document, config) |
| 62 | |
| 63 | @abstractmethod |
| 64 | def _parse_sync(self, document: FetchedDocument, config: dict) -> str: |
| 65 | ... |