| 1 | """HTML parser using Markdownify transformer.""" |
| 2 | |
| 3 | from plugins._document_query.helpers.fetch import FetchedDocument |
| 4 | from .base import BaseParser |
| 5 | |
| 6 | |
| 7 | class HtmlParser(BaseParser): |
| 8 | mimetypes = ["text/html"] |
| 9 | |
| 10 | def _parse_sync(self, document: FetchedDocument, config: dict) -> str: |
| 11 | from langchain_community.document_transformers import MarkdownifyTransformer |
| 12 | from langchain_core.documents import Document |
| 13 | |
| 14 | parts = [ |
| 15 | Document( |
| 16 | page_content=document.text(), |
| 17 | metadata={"source": document.source_uri or document.uri}, |
| 18 | ) |
| 19 | ] |
| 20 | return "\n".join(e.page_content for e in MarkdownifyTransformer().transform_documents(parts)) |