| 1 | from __future__ import annotations |
| 2 | |
| 3 | from app.models import ResearchDocument |
| 4 | |
| 5 | |
| 6 | class DocumentDeduplicator: |
| 7 | def __init__(self) -> None: |
| 8 | self._by_url: dict[str, ResearchDocument] = {} |
| 9 | self._by_hash: dict[str, ResearchDocument] = {} |
| 10 | |
| 11 | def find_duplicate(self, document: ResearchDocument) -> ResearchDocument | None: |
| 12 | return self._by_url.get(document.canonical_url) or self._by_hash.get(document.content_hash) |
| 13 | |
| 14 | def add(self, document: ResearchDocument) -> ResearchDocument | None: |
| 15 | duplicate = self.find_duplicate(document) |
| 16 | if duplicate: |
| 17 | return duplicate |
| 18 | self._by_url[document.canonical_url] = document |
| 19 | self._by_hash[document.content_hash] = document |
| 20 | return None |