| 1 | from __future__ import annotations |
| 2 | |
| 3 | import asyncio |
| 4 | import logging |
| 5 | import threading |
| 6 | import time |
| 7 | from datetime import datetime, timedelta, timezone |
| 8 | from decimal import Decimal |
| 9 | from uuid import UUID |
| 10 | |
| 11 | import httpx |
| 12 | import pytest |
| 13 | import respx |
| 14 | |
| 15 | from app.deduplication import DocumentDeduplicator |
| 16 | from app.entity_resolution import EntityResolver |
| 17 | from app.extraction import RuleBasedEventExtractor |
| 18 | from app.models import CompanyResearchProfile, DocumentStatus, DocumentSubtype, DocumentType, EntityResolution, EtfResearchProfile, EventImpact, PortfolioResearchCompany, PortfolioResearchSummary, ProvenancedValue, ReliabilityLevel, ResearchDocument, ResearchEvent, ResearchEventType, ShareholdingCategory, ShareholdingSnapshot, ShareholdingSnapshotValue, SourceClassification, SourceMode, SourceType, TimeHorizon |
| 19 | from app.fact_precedence import FactSourceTier, FinancialFact, FinancialFactKey |
| 20 | from app.normalization import canonicalize_url, content_hash, extract_published_at, extract_text, normalize_numbers |
| 21 | from app.repository import ResearchRepository, _InstrumentRefreshGate, _canonical_refresh_category |
| 22 | from app.persistence import SqliteResearchPersistence |
| 23 | from app.portfolio_orchestration import PortfolioResearchOrchestrator, _company_aliases, _hydrate_verified_exchange_mappings, _instrument_asset_type, _instrument_name |
| 24 | from app.research_fetching import DocumentSizeLimitExceeded, FetchError, FetchResult, HttpResearchFetcher, HttpStatusFetchError, NetworkFetchResult, PdfExtractionTimeoutError, RestrictedFetchError, TransportFetchError |
| 25 | from app.scoring import CatalystScorer, canonical_read_model_score |
| 26 | from app.settings import Settings |
| 27 | from app.source_discovery import ( |
| 28 | ApprovedSourceDiscovery, |
| 29 | BraveCompatibleSearchDiscoveryProvider, |
| 30 | CandidateSearchResult, |
| 31 | DiscoveryResult, |
| 32 | GoogleCompatibleSearchDiscoveryProvider, |
| 33 | OfficialFilingDiscovery, |
| 34 | SearchDateWindow, |
| 35 | SearchDiscoveryService, |
| 36 | SearchProviderConfigurationError, |
| 37 | SearchProviderError, |
| 38 | SearxngSearchDiscoveryProvider, |
| 39 | classify_source, |
| 40 | classify_nse_document_subtype, |
| 41 | _candidate_rank, |
| 42 | generate_search_queries, |
| 43 | ) |
| 44 | from app.source_registry import AIXTRON_INSTRUMENT_ID, RegisteredResearchSource, registered_sources_for |
| 45 | import app.structured_research as structured_research |
| 46 | from app.structured_research import enrich_company_research, financial_result_history_from_facts, financial_statement_history_from_facts, latest_quarterly_result, latest_quarterly_result_from_facts, shareholding_changes, shareholding_changes_from_snapshots, source_diversity, valuation_assessment |
| 47 | from app.structured_market import StructuredProviderError |
| 48 | from app.url_security import UnsafeUrlError, validate_public_http_url |
| 49 | |
| 50 | |
| 51 | class _UnavailableStructuredProvider: |
| 52 | provider_name = "test-structured" |
| 53 | |
| 54 | async def collect(self, instrument): |
| 55 | raise StructuredProviderError("STRUCTURED_PROVIDER_UNAVAILABLE:FIXTURE") |
| 56 | |
| 57 | |
| 58 | @pytest.mark.asyncio |
| 59 | async def test_global_instrument_refresh_gate_reuses_fresh_complete_state_and_single_flights_stale_work(monkeypatch) -> None: |
| 60 | repository = ResearchRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=False)) |
| 61 | instrument_id = repository.profiles[0].instrument_id |
| 62 | fresh = _InstrumentRefreshGate(set(), False, False, "FRESH_AND_COMPLETE") |
| 63 | monkeypatch.setattr(repository, "_instrument_refresh_gate", lambda *_args, **_kwargs: fresh) |
| 64 | provider_calls = 0 |
| 65 | |
| 66 | async def should_not_discover(*_args, **_kwargs): |
| 67 | nonlocal provider_calls |
| 68 | provider_calls += 1 |
| 69 | |
| 70 | monkeypatch.setattr(repository._official_filing_discovery, "discover", should_not_discover) |
| 71 | monkeypatch.setattr(repository._discovery, "discover", should_not_discover) |
| 72 | monkeypatch.setattr(repository, "_fetch_registered_source", should_not_discover) |
| 73 | |
| 74 | emitted: list[str] = [] |
| 75 | |
| 76 | class _RepositoryLogCapture(logging.Handler): |
| 77 | def emit(self, record: logging.LogRecord) -> None: |
| 78 | emitted.append(record.getMessage()) |
| 79 | |
| 80 | repository_logger = logging.getLogger("app.repository") |
| 81 | handler = _RepositoryLogCapture() |
| 82 | previous_level = repository_logger.level |
| 83 | repository_logger.setLevel(logging.INFO) |
| 84 | repository_logger.addHandler(handler) |
| 85 | try: |
| 86 | await repository.refresh(instrument_id) |
| 87 | finally: |
| 88 | repository_logger.removeHandler(handler) |
| 89 | repository_logger.setLevel(previous_level) |
| 90 | assert provider_calls == 0 |
| 91 | assert any("research_refresh_gate" in message for message in emitted) |
| 92 | assert any("outcome=REUSE_FRESH" in message for message in emitted) |
| 93 | |
| 94 | stale = _InstrumentRefreshGate({"FINANCIAL_RESULTS"}, False, False, "STALE") |
| 95 | monkeypatch.setattr(repository, "_instrument_refresh_gate", lambda *_args, **_kwargs: stale) |
| 96 | started = asyncio.Event() |
| 97 | release = asyncio.Event() |
| 98 | |
| 99 | live_calls = 0 |
| 100 | |
| 101 | async def refresh_once(*_args): |
| 102 | nonlocal live_calls |
| 103 | live_calls += 1 |
| 104 | started.set() |
| 105 | await release.wait() |
| 106 | |
| 107 | monkeypatch.setattr(repository, "_refresh_live", refresh_once) |
| 108 | first = asyncio.create_task(repository.refresh(instrument_id)) |
| 109 | await started.wait() |
| 110 | second = asyncio.create_task(repository.refresh(instrument_id)) |
| 111 | await asyncio.sleep(0) |
| 112 | release.set() |
| 113 | await asyncio.gather(first, second) |
| 114 | assert live_calls == 1 |
| 115 | |
| 116 | |
| 117 | def test_url_canonicalization_and_hashing_are_deterministic() -> None: |
| 118 | url = canonicalize_url("HTTPS://Example.COM/a//b/?utm_source=x&b=2&a=1#frag") |
| 119 | assert url == "https://example.com/a/b?a=1&b=2" |
| 120 | assert content_hash(" Hello World ") == content_hash("hello world") |
| 121 | |
| 122 | |
| 123 | def test_ssrf_rejects_private_local_and_non_http_urls() -> None: |
| 124 | for url in ["http://localhost/test", "http://127.0.0.1/test", "file:///etc/passwd", "http://169.254.169.254/latest"]: |
| 125 | with pytest.raises(UnsafeUrlError): |
| 126 | validate_public_http_url(url) |
| 127 | |
| 128 | |
| 129 | def test_numeric_normalization_supports_money_capacity_and_percentages() -> None: |
| 130 | values = normalize_numbers("₹2,000 crore, €350 million, $1.2 billion, 73.15 MW, +42% backlog") |
| 131 | assert any(v.currency == "INR" and v.value == Decimal("20000000000") for v in values) |
| 132 | assert any(v.currency == "EUR" and v.value == Decimal("350000000") for v in values) |
| 133 | assert any(v.currency == "USD" and v.value == Decimal("1200000000.0") for v in values) |
| 134 | assert any(v.unit == "MW" and v.value == Decimal("73.15") for v in values) |
| 135 | assert any(v.unit == "PERCENT" and v.value == Decimal("42") for v in values) |
| 136 | |
| 137 | |
| 138 | def test_published_date_extraction_supports_official_formats() -> None: |
| 139 | assert extract_published_at("Herzogenrath, April 14, 2026") == datetime(2026, 4, 14, tzinfo=timezone.utc) |
| 140 | assert extract_published_at("Veröffentlicht am 30.07.2026") == datetime(2026, 7, 30, tzinfo=timezone.utc) |
| 141 | |
| 142 | |
| 143 | def test_entity_resolution_uses_more_than_ticker() -> None: |
| 144 | repo = ResearchRepository() |
| 145 | resolver = EntityResolver(repo.profiles) |
| 146 | resolution = resolver.resolve( |
| 147 | "AIXTRON order", |
| 148 | "AIXTRON SE AIXA XETR DE000A0WMPJ6 receives a new order.", |
| 149 | "https://ir.aixtron.example/releases/order", |
| 150 | ) |
| 151 | assert resolution.instrument_id == UUID("11111111-1111-1111-1111-111111111111") |
| 152 | assert resolution.confidence > 0.8 |
| 153 | assert {"isin", "company_name", "known_domain"}.issubset(set(resolution.matched_on)) |
| 154 | |
| 155 | |
| 156 | def test_entity_resolution_matches_besi_and_reliance_aliases() -> None: |
| 157 | repo = ResearchRepository() |
| 158 | resolver = EntityResolver(repo.profiles) |
| 159 | |
| 160 | besi = resolver.resolve( |
| 161 | "BE Semiconductor Industries N.V. Announces Q2-26 Results", |
| 162 | "Besi reported order growth for XAMS BESI holders.", |
| 163 | "https://www.besi.com/investor-relations/press-releases/details/q2-results", |
| 164 | ) |
| 165 | reliance = resolver.resolve( |
| 166 | "RIL capex update", |
| 167 | "Reliance Industries reported capex for RELIANCE XNSE INE002A01018.", |
| 168 | "https://www.ril.com/ar2025-26/index.html", |
| 169 | ) |
| 170 | |
| 171 | assert besi.instrument_id == UUID("22222222-2222-2222-2222-222222222222") |
| 172 | assert besi.confidence >= 0.30 |
| 173 | assert reliance.instrument_id == UUID("44444444-4444-4444-4444-444444444444") |
| 174 | assert reliance.confidence >= 0.30 |
| 175 | |
| 176 | |
| 177 | def test_document_deduplication_uses_url_and_hash() -> None: |
| 178 | dedupe = DocumentDeduplicator() |
| 179 | doc = _document("https://example.com/a", "same text") |
| 180 | duplicate_url = _document("https://example.com/a", "different text") |
| 181 | duplicate_hash = _document("https://example.com/b", "same text") |
| 182 | assert dedupe.add(doc) is None |
| 183 | assert dedupe.add(duplicate_url) == doc |
| 184 | assert dedupe.add(duplicate_hash) == doc |
| 185 | |
| 186 | |
| 187 | def test_rule_based_extraction_sets_confidence_and_negative_events() -> None: |
| 188 | doc = _document( |
| 189 | "https://example.com/a", |
| 190 | "AIXTRON SE AIXA XETR announced a new order worth €350 million. The factory ramp was delayed.", |
| 191 | ) |
| 192 | doc.instrument_id = UUID("11111111-1111-1111-1111-111111111111") |
| 193 | doc.company_id = UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1") |
| 194 | doc.entity_resolution_confidence = 0.9 |
| 195 | extractor = RuleBasedEventExtractor() |
| 196 | |
| 197 | events = extractor.extract(doc) |
| 198 | |
| 199 | assert any(event.event_type == "NEW_ORDER" and event.monetary_value == Decimal("350000000") for event in events) |
| 200 | assert any(event.event_type == "PROJECT_DELAY" and event.impact == EventImpact.NEGATIVE for event in events) |
| 201 | assert all(0 < event.confidence <= 1 for event in events) |
| 202 | |
| 203 | |
| 204 | def test_rule_based_extraction_classifies_capex_guidance_and_cancellations() -> None: |
| 205 | doc = _document( |
| 206 | "https://example.com/a", |
| 207 | "AIXTRON SE AIXA XETR DE000A0WMPJ6 will invest EUR 120 million in CAPEX. " |
| 208 | "The company raises guidance, later announced a guidance cut, and an order cancelled by a customer.", |
| 209 | ) |
| 210 | doc.instrument_id = AIXTRON_INSTRUMENT_ID |
| 211 | doc.company_id = UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1") |
| 212 | doc.entity_resolution_confidence = 1.0 |
| 213 | |
| 214 | events = RuleBasedEventExtractor().extract(doc) |
| 215 | |
| 216 | assert any(event.event_type == "CAPEX" for event in events) |
| 217 | assert any(event.event_type == "GUIDANCE_RAISED" and event.impact == EventImpact.POSITIVE for event in events) |
| 218 | assert any(event.event_type == "GUIDANCE_CUT" and event.impact == EventImpact.NEGATIVE for event in events) |
| 219 | assert any(event.event_type == "ORDER_CANCELLED" and event.impact == EventImpact.NEGATIVE for event in events) |
| 220 | |
| 221 | |
| 222 | def test_aixtron_extraction_uses_event_specific_fields_and_clean_text() -> None: |
| 223 | title, text = extract_text(_aixtron_mojibake_fixture(), "text/html") |
| 224 | assert title == "Strong momentum in optoelectronics continues" |
| 225 | assert text is not None |
| 226 | assert "Raised full-year 2026 guidance confirmed" in text |
| 227 | assert "Navigation Suche" not in text |
| 228 | assert not text.startswith("AIXTRON Press Information") |
| 229 | |
| 230 | doc = _document("https://www.aixtron.com/en/press/press-releases/strong", text) |
| 231 | doc.title = title |
| 232 | doc.instrument_id = AIXTRON_INSTRUMENT_ID |
| 233 | doc.company_id = UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1") |
| 234 | doc.entity_resolution_confidence = 1.0 |
| 235 | |
| 236 | events = RuleBasedEventExtractor().extract(doc) |
| 237 | by_type = {str(event.event_type): event for event in events} |
| 238 | |
| 239 | assert "ANNUAL_REPORT" not in by_type |
| 240 | assert "ORDER_BACKLOG_CHANGE" in by_type |
| 241 | assert "GUIDANCE_MAINTAINED" in by_type |
| 242 | assert "CAPACITY_EXPANSION" in by_type |
| 243 | assert "EARNINGS_RELEASE" in by_type |
| 244 | |
| 245 | order = by_type["ORDER_BACKLOG_CHANGE"] |
| 246 | assert order.monetary_original == "EUR 214.5 million" |
| 247 | assert order.percentage_original in {"54%", "+81%"} |
| 248 | assert order.customer is None |
| 249 | assert order.counterparty is None |
| 250 | assert "order intake" in order.raw_evidence_reference.lower() |
| 251 | |
| 252 | guidance = by_type["GUIDANCE_MAINTAINED"] |
| 253 | assert guidance.monetary_original is None |
| 254 | assert guidance.percentage_original is None |
| 255 | assert guidance.customer is None |
| 256 | assert guidance.counterparty is None |
| 257 | assert "guidance" in guidance.raw_evidence_reference.lower() |
| 258 | |
| 259 | capacity = by_type["CAPACITY_EXPANSION"] |
| 260 | assert capacity.monetary_original is None |
| 261 | assert capacity.percentage_original is None |
| 262 | assert capacity.customer is None |
| 263 | assert capacity.counterparty is None |
| 264 | assert "production capacity" in capacity.raw_evidence_reference.lower() |
| 265 | |
| 266 | earnings = by_type["EARNINGS_RELEASE"] |
| 267 | assert "q2 results" in earnings.raw_evidence_reference.lower() or "half year results" in earnings.raw_evidence_reference.lower() |
| 268 | |
| 269 | |
| 270 | def test_catalyst_score_uses_negative_events_and_temporal_decay() -> None: |
| 271 | repo = ResearchRepository() |
| 272 | instrument_id = UUID("33333333-3333-3333-3333-333333333333") |
| 273 | events = repo.events_for(instrument_id) |
| 274 | score = CatalystScorer().score(instrument_id, events) |
| 275 | assert 0 <= score.overall_score <= 100 |
| 276 | assert score.overall_score < 60 |
| 277 | |
| 278 | |
| 279 | def test_category_no_evidence_is_not_score_50() -> None: |
| 280 | repo = ResearchRepository() |
| 281 | doc = _document( |
| 282 | "https://www.aixtron.com/en/orders", |
| 283 | "AIXTRON SE AIXA XETR DE000A0WMPJ6 recorded order intake of EUR 214.5 million (+54% yoy).", |
| 284 | ) |
| 285 | doc.instrument_id = AIXTRON_INSTRUMENT_ID |
| 286 | doc.company_id = UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1") |
| 287 | events = RuleBasedEventExtractor().extract(doc) |
| 288 | |
| 289 | score = CatalystScorer().score(AIXTRON_INSTRUMENT_ID, events) |
| 290 | |
| 291 | assert score.category_evidence["Orders & Backlog"].status == "POSITIVE_EVIDENCE" |
| 292 | assert score.category_evidence["Orders & Backlog"].score is not None |
| 293 | assert score.category_evidence["Growth"].status == "NO_EVIDENCE" |
| 294 | assert score.category_evidence["Growth"].score is None |
| 295 | assert score.buckets["Growth"] is None |
| 296 | assert score.buckets["New Customers"] is None |
| 297 | |
| 298 | |
| 299 | def test_canonical_category_evidence_links_scored_events_outside_recent_slice() -> None: |
| 300 | instrument_id = AIXTRON_INSTRUMENT_ID |
| 301 | company_id = UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1") |
| 302 | base = _document( |
| 303 | "https://www.aixtron.com/en/capex", |
| 304 | "AIXTRON SE AIXA XETR DE000A0WMPJ6 will invest EUR 120 million in CAPEX.", |
| 305 | ).model_copy(update={"instrument_id": instrument_id, "company_id": company_id}) |
| 306 | capex_event = next(event for event in RuleBasedEventExtractor().extract(base) if event.event_type == "CAPEX") |
| 307 | newer_events = [ |
| 308 | capex_event.model_copy(update={ |
| 309 | "event_id": UUID(f"00000000-0000-0000-0000-{index:012d}"), |
| 310 | "event_type": ResearchEventType.NEW_ORDER, |
| 311 | "event_date": datetime(2026, 8, 30 - index, tzinfo=timezone.utc), |
| 312 | "independence_key": f"newer-{index}", |
| 313 | }) |
| 314 | for index in range(1, 12) |
| 315 | ] |
| 316 | capex_event = capex_event.model_copy(update={ |
| 317 | "event_date": datetime(2026, 1, 1, tzinfo=timezone.utc), |
| 318 | "independence_key": "capex-source", |
| 319 | "impact": EventImpact.POSITIVE, |
| 320 | }) |
| 321 | canonical = canonical_read_model_score(CatalystScorer().score(instrument_id, newer_events + [capex_event])) |
| 322 | |
| 323 | capex = canonical.category_evidence["CAPEX"] |
| 324 | assert capex.status == "POSITIVE_EVIDENCE" |
| 325 | assert [event.event_id for event in capex.supporting_events] == [capex_event.event_id] |
| 326 | assert capex.source_count == capex.independent_source_count == 1 |
| 327 | assert "CAPEX & Capacity" not in canonical.category_evidence |
| 328 | assert "NEW_FACILITIES" not in canonical.category_evidence |
| 329 | assert canonical.category_evidence["ANALYST_TARGETS"].status == "NO_EVIDENCE" |
| 330 | assert canonical.category_evidence["ANALYST_TARGETS"].supporting_events == [] |
| 331 | |
| 332 | |
| 333 | def test_repository_ingestion_marks_duplicate_and_preserves_demo_summary() -> None: |
| 334 | repo = ResearchRepository() |
| 335 | first = repo.ingest_fixture( |
| 336 | original_url="https://ir.aixtron.example/releases/new-order", |
| 337 | source_type=SourceType.INVESTOR_RELATIONS, |
| 338 | source_name="Fixture", |
| 339 | publisher="DEMO", |
| 340 | content_type="text/html", |
| 341 | body="<html><title>AIXTRON order</title><body>AIXTRON SE AIXA XETR wins a new order worth €10 million.</body></html>", |
| 342 | reliability=ReliabilityLevel.LEVEL_B, |
| 343 | published_at=datetime(2026, 2, 1, tzinfo=timezone.utc), |
| 344 | ) |
| 345 | second = repo.ingest_fixture( |
| 346 | original_url="https://ir.aixtron.example/releases/new-order?utm_source=feed", |
| 347 | source_type=SourceType.RSS, |
| 348 | source_name="Fixture RSS", |
| 349 | publisher="DEMO", |
| 350 | content_type="text/html", |
| 351 | body="<html><title>AIXTRON order</title><body>AIXTRON SE AIXA XETR wins a new order worth €10 million.</body></html>", |
| 352 | reliability=ReliabilityLevel.LEVEL_C, |
| 353 | ) |
| 354 | assert first.status == DocumentStatus.PROCESSED |
| 355 | assert second.status == DocumentStatus.DUPLICATE |
| 356 | assert len([doc for doc in repo.documents_for(UUID("11111111-1111-1111-1111-111111111111")) if doc.canonical_url == first.canonical_url]) == 1 |
| 357 | assert repo.summary(UUID("11111111-1111-1111-1111-111111111111")).demo is True |
| 358 | |
| 359 | |
| 360 | @pytest.mark.asyncio |
| 361 | @respx.mock |
| 362 | async def test_targeted_gap_detection_fetches_only_missing_approved_sources() -> None: |
| 363 | customer_source = _registered_source( |
| 364 | "aixtron-customer-target", |
| 365 | "https://www.aixtron.com/en/press/customer-win", |
| 366 | priority=3, |
| 367 | categories=("Customers",), |
| 368 | ) |
| 369 | growth_source = _registered_source( |
| 370 | "aixtron-growth-target", |
| 371 | "https://www.aixtron.com/en/press/growth", |
| 372 | priority=3, |
| 373 | categories=("Growth",), |
| 374 | ) |
| 375 | discovery = _StaticDiscovery([customer_source, growth_source]) |
| 376 | repo = ResearchRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_max_retries=0), discovery=discovery) |
| 377 | primary = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 378 | respx.get(primary.url).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=_aixtron_mojibake_fixture())) |
| 379 | customer_route = respx.get(customer_source.url).mock( |
| 380 | return_value=httpx.Response( |
| 381 | 200, |
| 382 | headers={"content-type": "text/html"}, |
| 383 | text="<html><title>AIXTRON customer</title><main><p>Herzogenrath, May 2, 2026</p><p>AIXTRON SE AIXA XETR announced a customer win with Infineon Technologies AG for silicon-carbide production tools.</p></main></html>", |
| 384 | ) |
| 385 | ) |
| 386 | growth_route = respx.get(growth_source.url).mock( |
| 387 | return_value=httpx.Response( |
| 388 | 200, |
| 389 | headers={"content-type": "text/html"}, |
| 390 | text="<html><title>AIXTRON growth</title><main><p>Herzogenrath, May 3, 2026</p><p>AIXTRON SE AIXA XETR announced geographic expansion into Japan for optoelectronics service coverage.</p></main></html>", |
| 391 | ) |
| 392 | ) |
| 393 | |
| 394 | summary = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 395 | |
| 396 | assert customer_route.call_count == 1 |
| 397 | assert growth_route.call_count == 1 |
| 398 | assert summary.catalyst_score.category_evidence["Customers"].status == "POSITIVE_EVIDENCE" |
| 399 | assert summary.catalyst_score.category_evidence["Growth"].status == "POSITIVE_EVIDENCE" |
| 400 | urls = {event.source_url for event in summary.recent_events} |
| 401 | assert customer_source.url in urls |
| 402 | assert growth_source.url in urls |
| 403 | |
| 404 | |
| 405 | @pytest.mark.asyncio |
| 406 | @respx.mock |
| 407 | async def test_no_evidence_after_failed_discovery_remains_no_evidence() -> None: |
| 408 | repo = ResearchRepository( |
| 409 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_max_retries=0), |
| 410 | discovery=_StaticDiscovery([]), |
| 411 | ) |
| 412 | primary = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 413 | respx.get(primary.url).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=_aixtron_mojibake_fixture())) |
| 414 | |
| 415 | summary = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 416 | |
| 417 | assert summary.demo is False |
| 418 | assert summary.catalyst_score.category_evidence["Growth"].status == "NO_EVIDENCE" |
| 419 | assert summary.catalyst_score.category_evidence["Growth"].score is None |
| 420 | assert summary.catalyst_score.category_evidence["Customers"].status == "NO_EVIDENCE" |
| 421 | assert summary.catalyst_score.category_evidence["Customers"].score is None |
| 422 | |
| 423 | |
| 424 | @pytest.mark.asyncio |
| 425 | @respx.mock |
| 426 | async def test_cross_document_dedup_and_event_provenance_are_stable() -> None: |
| 427 | source = _registered_source( |
| 428 | "aixtron-duplicate-order", |
| 429 | "https://www.aixtron.com/en/press/order-duplicate", |
| 430 | priority=3, |
| 431 | categories=("Orders & Backlog",), |
| 432 | ) |
| 433 | repo = ResearchRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_max_retries=0), discovery=_StaticDiscovery([source])) |
| 434 | primary = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 435 | body = "<html><title>AIXTRON order</title><main><p>Herzogenrath, May 4, 2026</p><p>AIXTRON SE AIXA XETR reported order intake of EUR 214.5 million (+54% yoy).</p></main></html>" |
| 436 | respx.get(primary.url).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=_aixtron_mojibake_fixture())) |
| 437 | respx.get(source.url).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=body)) |
| 438 | |
| 439 | first = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 440 | second = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 441 | |
| 442 | assert len(repo.documents_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL)) == len(first.documents) == len(second.documents) |
| 443 | assert len(repo.events_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL)) == len(first.recent_events) == len(second.recent_events) |
| 444 | assert {event.source_document_id for event in second.recent_events}.issubset({doc.document_id for doc in second.documents}) |
| 445 | |
| 446 | |
| 447 | def test_conflicting_evidence_is_preserved_in_category_score() -> None: |
| 448 | repo = ResearchRepository() |
| 449 | positive = repo.ingest_fixture( |
| 450 | original_url="https://www.aixtron.com/en/guidance-maintained", |
| 451 | source_type=SourceType.INVESTOR_RELATIONS, |
| 452 | source_name="Official", |
| 453 | publisher="AIXTRON SE", |
| 454 | content_type="text/html", |
| 455 | body="<html><title>AIXTRON guidance</title><body>AIXTRON SE AIXA XETR DE000A0WMPJ6 confirms guidance for the full year 2026.</body></html>", |
| 456 | reliability=ReliabilityLevel.LEVEL_B, |
| 457 | published_at=datetime(2026, 5, 1, tzinfo=timezone.utc), |
| 458 | source_mode=SourceMode.REAL, |
| 459 | ) |
| 460 | negative = repo.ingest_fixture( |
| 461 | original_url="https://www.aixtron.com/en/guidance-cut", |
| 462 | source_type=SourceType.INVESTOR_RELATIONS, |
| 463 | source_name="Official", |
| 464 | publisher="AIXTRON SE", |
| 465 | content_type="text/html", |
| 466 | body="<html><title>AIXTRON guidance cut</title><body>AIXTRON SE AIXA XETR DE000A0WMPJ6 later announced a guidance cut for the full year 2026.</body></html>", |
| 467 | reliability=ReliabilityLevel.LEVEL_B, |
| 468 | published_at=datetime(2026, 6, 1, tzinfo=timezone.utc), |
| 469 | source_mode=SourceMode.REAL, |
| 470 | ) |
| 471 | assert positive.status == DocumentStatus.PROCESSED |
| 472 | assert negative.status == DocumentStatus.PROCESSED |
| 473 | |
| 474 | summary = repo.summary(AIXTRON_INSTRUMENT_ID) |
| 475 | |
| 476 | assert any(event.event_type == "GUIDANCE_MAINTAINED" for event in summary.recent_events) |
| 477 | assert any(event.event_type == "GUIDANCE_CUT" for event in summary.recent_events) |
| 478 | assert summary.catalyst_score.category_evidence["Guidance"].status == "NEGATIVE_EVIDENCE" |
| 479 | |
| 480 | |
| 481 | def test_approved_source_discovery_filters_unapproved_and_seen_sources() -> None: |
| 482 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 483 | seen = canonicalize_url("https://www.aixtron.com/en/press/customer-win") |
| 484 | allowed = _registered_source("allowed", "https://www.aixtron.com/en/press/customer-win", priority=3, categories=("Customers",)) |
| 485 | unapproved = _registered_source("blocked", "https://www.aixtron.com/en/press/growth", priority=3, categories=("Growth",), allowed=False) |
| 486 | discovery = _StaticDiscovery([allowed, unapproved]) |
| 487 | |
| 488 | results = discovery.discover(profile, {"Customers", "Growth"}, {seen}) |
| 489 | |
| 490 | assert results == [] |
| 491 | |
| 492 | |
| 493 | def test_registered_source_rejects_unallowed_source() -> None: |
| 494 | repo = ResearchRepository() |
| 495 | source = _registered_source("blocked", "https://www.aixtron.com/en/blocked", allowed=False) |
| 496 | with pytest.raises(FetchError): |
| 497 | repo._validate_registered_source(repo.profile(AIXTRON_INSTRUMENT_ID), source) |
| 498 | |
| 499 | |
| 500 | def test_search_query_generation_uses_company_ticker_alias_and_category_terms() -> None: |
| 501 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 502 | |
| 503 | queries = generate_search_queries(profile, "Customers", SearchDateWindow(year=2026)) |
| 504 | |
| 505 | assert "AIXTRON SE new customer 2026" in queries |
| 506 | assert "AIXA customer order 2026" in queries |
| 507 | assert "AIXTRON customer qualification 2026" in queries |
| 508 | assert len(queries) == len(set(queries)) |
| 509 | |
| 510 | |
| 511 | def test_search_queries_cover_broad_company_research_topics() -> None: |
| 512 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 513 | |
| 514 | growth = generate_search_queries(profile, "Growth", SearchDateWindow(year=2026)) |
| 515 | orders = generate_search_queries(profile, "Orders & Backlog", SearchDateWindow(year=2026)) |
| 516 | capex = generate_search_queries(profile, "CAPEX & Capacity", SearchDateWindow(year=2026)) |
| 517 | guidance = generate_search_queries(profile, "Guidance", SearchDateWindow(year=2026)) |
| 518 | ownership = generate_search_queries(profile, "Ownership", SearchDateWindow(year=2026)) |
| 519 | analyst = generate_search_queries(profile, "Analyst", SearchDateWindow(year=2026)) |
| 520 | regulatory = generate_search_queries(profile, "Regulatory", SearchDateWindow(year=2026)) |
| 521 | analyst_targets = generate_search_queries(profile, "ANALYST_TARGETS", SearchDateWindow(year=2026)) |
| 522 | institutional = generate_search_queries(profile, "INSTITUTIONAL_ACTIVITY", SearchDateWindow(year=2026)) |
| 523 | |
| 524 | assert any("earnings results" in query for query in growth) |
| 525 | assert any("revenue growth" in query for query in growth) |
| 526 | assert any("profit margins" in query for query in growth) |
| 527 | assert any("order wins" in query for query in orders) |
| 528 | assert any("new plant investment" in query for query in capex) |
| 529 | assert any("management guidance" in query for query in guidance) |
| 530 | assert any("institutional ownership" in query for query in ownership) |
| 531 | assert any("analyst target" in query for query in analyst) |
| 532 | assert any("regulatory announcement" in query for query in regulatory) |
| 533 | assert any("analyst target price" in query for query in analyst_targets) |
| 534 | assert any("institutional ownership" in query for query in institutional) |
| 535 | |
| 536 | |
| 537 | def test_source_classification_prefers_authoritative_publishers() -> None: |
| 538 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 539 | |
| 540 | assert classify_source("www.aixtron.com", profile) == SourceClassification.OFFICIAL_COMPANY |
| 541 | assert classify_source("www.sec.gov", profile) == SourceClassification.REGULATORY |
| 542 | assert classify_source("www.nseindia.com", profile) == SourceClassification.EXCHANGE |
| 543 | assert classify_source("www.bseindia.com", profile) == SourceClassification.EXCHANGE |
| 544 | assert classify_source("www.sebi.gov.in", profile) == SourceClassification.REGULATORY |
| 545 | assert classify_source("www.euronext.com", profile) == SourceClassification.EXCHANGE |
| 546 | assert classify_source("www.afm.nl", profile) == SourceClassification.REGULATORY |
| 547 | assert classify_source("www.deutsche-boerse.com", profile) == SourceClassification.EXCHANGE |
| 548 | assert classify_source("www.infineon.com", profile) == SourceClassification.CUSTOMER |
| 549 | assert classify_source("www.reuters.com", profile) == SourceClassification.REPUTABLE_NEWS |
| 550 | assert classify_source("www.marketscreener.com", profile) == SourceClassification.INVESTMENT_RESEARCH |
| 551 | assert classify_source("www.moneycontrol.com", profile) == SourceClassification.INVESTMENT_RESEARCH |
| 552 | assert classify_source("www.trendlyne.com", profile) == SourceClassification.INVESTMENT_RESEARCH |
| 553 | assert classify_source("www.screener.in", profile) == SourceClassification.INVESTMENT_RESEARCH |
| 554 | assert classify_source("untrusted.example", profile) == SourceClassification.OTHER |
| 555 | |
| 556 | |
| 557 | def test_search_provider_disabled_mode_does_not_require_credentials() -> None: |
| 558 | repo = ResearchRepository(settings=Settings(research_search_enabled=False)) |
| 559 | |
| 560 | assert repo._search_discovery.provider.provider_name == "disabled" |
| 561 | |
| 562 | |
| 563 | def test_google_compatible_provider_complete_configuration_is_accepted() -> None: |
| 564 | repo = ResearchRepository( |
| 565 | settings=Settings( |
| 566 | research_search_enabled=True, |
| 567 | research_search_provider="google-compatible", |
| 568 | research_search_endpoint="https://search.example/v1", |
| 569 | research_search_api_key="secret-key", |
| 570 | research_search_engine_id="engine-id", |
| 571 | ) |
| 572 | ) |
| 573 | |
| 574 | assert repo._search_discovery.provider.provider_name == "google-compatible" |
| 575 | |
| 576 | |
| 577 | def test_searxng_provider_complete_configuration_is_accepted() -> None: |
| 578 | repo = ResearchRepository( |
| 579 | settings=Settings( |
| 580 | research_search_enabled=True, |
| 581 | research_search_provider="searxng", |
| 582 | research_search_endpoint="http://searxng/search", |
| 583 | ) |
| 584 | ) |
| 585 | |
| 586 | assert repo._search_discovery.provider.provider_name == "searxng" |
| 587 | |
| 588 | |
| 589 | def test_search_provider_enabled_missing_api_key_fails_explicitly() -> None: |
| 590 | with pytest.raises(SearchProviderConfigurationError, match="SEARCH_PROVIDER_NOT_CONFIGURED"): |
| 591 | ResearchRepository( |
| 592 | settings=Settings( |
| 593 | research_search_enabled=True, |
| 594 | research_search_provider="google-compatible", |
| 595 | research_search_endpoint="https://search.example/v1", |
| 596 | research_search_engine_id="engine-id", |
| 597 | ) |
| 598 | ) |
| 599 | |
| 600 | |
| 601 | def test_search_provider_enabled_missing_engine_id_fails_explicitly() -> None: |
| 602 | with pytest.raises(SearchProviderConfigurationError, match="engine_id"): |
| 603 | ResearchRepository( |
| 604 | settings=Settings( |
| 605 | research_search_enabled=True, |
| 606 | research_search_provider="google-compatible", |
| 607 | research_search_endpoint="https://search.example/v1", |
| 608 | research_search_api_key="secret-key", |
| 609 | ) |
| 610 | ) |
| 611 | |
| 612 | |
| 613 | def test_brave_compatible_provider_boundary_is_configuration_driven() -> None: |
| 614 | repo = ResearchRepository( |
| 615 | settings=Settings( |
| 616 | research_search_enabled=True, |
| 617 | research_search_provider="brave-compatible", |
| 618 | research_search_endpoint="https://api.search.brave.com/res/v1/web/search", |
| 619 | research_search_api_key="secret-key", |
| 620 | ) |
| 621 | ) |
| 622 | |
| 623 | assert repo._search_discovery.provider.provider_name == "brave-compatible" |
| 624 | |
| 625 | |
| 626 | @pytest.mark.asyncio |
| 627 | async def test_brave_compatible_provider_maps_query_token_header_and_limit() -> None: |
| 628 | client = _RecordingSearchClient( |
| 629 | httpx.Response( |
| 630 | 200, |
| 631 | json={ |
| 632 | "web": { |
| 633 | "results": [ |
| 634 | {"title": "One", "url": "https://www.aixtron.com/en/one", "description": "snippet one"}, |
| 635 | {"title": "Two", "url": "https://www.aixtron.com/en/two", "description": "snippet two"}, |
| 636 | {"title": "Three", "url": "https://www.aixtron.com/en/three", "description": "snippet three"}, |
| 637 | ] |
| 638 | } |
| 639 | }, |
| 640 | ) |
| 641 | ) |
| 642 | provider = BraveCompatibleSearchDiscoveryProvider( |
| 643 | "https://api.search.brave.com/res/v1/web/search", |
| 644 | "secret-key", |
| 645 | max_results_per_query=2, |
| 646 | client=client, |
| 647 | ) |
| 648 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 649 | |
| 650 | results = await provider.discover( |
| 651 | profile, |
| 652 | "Unmapped Category", |
| 653 | SearchDateWindow(year=2026, query_limit=1), |
| 654 | ) |
| 655 | |
| 656 | assert client.calls |
| 657 | assert client.calls[0]["url"] == "https://api.search.brave.com/res/v1/web/search" |
| 658 | assert client.calls[0]["params"]["q"] == "AIXTRON SE Unmapped Category 2026" |
| 659 | assert client.calls[0]["params"]["count"] == 2 |
| 660 | assert client.calls[0]["headers"]["X-Subscription-Token"] == "secret-key" |
| 661 | assert len(client.calls) == 1 |
| 662 | assert len(results) == 2 |
| 663 | assert {result.url for result in results} == {"https://www.aixtron.com/en/one", "https://www.aixtron.com/en/two"} |
| 664 | assert all(result.provider == "brave-compatible" for result in results) |
| 665 | |
| 666 | |
| 667 | @pytest.mark.asyncio |
| 668 | async def test_searxng_provider_maps_json_query_and_limit() -> None: |
| 669 | client = _RecordingSearchClient( |
| 670 | httpx.Response( |
| 671 | 200, |
| 672 | json={ |
| 673 | "results": [ |
| 674 | {"title": "One", "url": "https://www.aixtron.com/en/one", "content": "snippet one"}, |
| 675 | {"title": "Two", "url": "https://www.reuters.com/markets/two", "content": "snippet two"}, |
| 676 | {"title": "Three", "url": "https://example.com/three", "content": "snippet three"}, |
| 677 | ] |
| 678 | }, |
| 679 | ) |
| 680 | ) |
| 681 | provider = SearxngSearchDiscoveryProvider( |
| 682 | "http://searxng/search", |
| 683 | max_results_per_query=2, |
| 684 | client=client, |
| 685 | ) |
| 686 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 687 | |
| 688 | results = await provider.discover(profile, "Unmapped Category", SearchDateWindow(year=2026, query_limit=1)) |
| 689 | |
| 690 | assert client.calls |
| 691 | assert client.calls[0]["url"] == "http://searxng/search" |
| 692 | assert client.calls[0]["params"]["q"] == "AIXTRON SE Unmapped Category 2026" |
| 693 | assert client.calls[0]["params"]["format"] == "json" |
| 694 | assert client.calls[0]["params"]["categories"] == "general" |
| 695 | assert len(results) == 2 |
| 696 | assert {result.provider for result in results} == {"searxng"} |
| 697 | assert {result.url for result in results} == {"https://www.aixtron.com/en/one", "https://www.reuters.com/markets/two"} |
| 698 | |
| 699 | |
| 700 | @pytest.mark.asyncio |
| 701 | async def test_searxng_provider_malformed_response_is_unavailable() -> None: |
| 702 | provider = SearxngSearchDiscoveryProvider( |
| 703 | "http://searxng/search", |
| 704 | client=_RecordingSearchClient(httpx.Response(200, json={"results": "bad"})), |
| 705 | ) |
| 706 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 707 | |
| 708 | with pytest.raises(SearchProviderError) as exc_info: |
| 709 | await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 710 | |
| 711 | assert str(exc_info.value) == "SEARCH_PROVIDER_UNAVAILABLE:invalid_response" |
| 712 | |
| 713 | |
| 714 | @pytest.mark.asyncio |
| 715 | async def test_searxng_empty_healthy_response_remains_a_legitimate_zero_result() -> None: |
| 716 | provider = SearxngSearchDiscoveryProvider("http://searxng/search", client=_RecordingSearchClient(httpx.Response(200, json={"results": [], "unresponsive_engines": []}))) |
| 717 | assert await provider.discover(ResearchRepository().profile(AIXTRON_INSTRUMENT_ID), "Customers", SearchDateWindow(year=2026, query_limit=1)) == [] |
| 718 | |
| 719 | |
| 720 | @pytest.mark.asyncio |
| 721 | async def test_searxng_empty_unresponsive_response_is_retryable_provider_failure() -> None: |
| 722 | provider = SearxngSearchDiscoveryProvider("http://searxng/search", client=_RecordingSearchClient(httpx.Response(200, json={"results": [], "unresponsive_engines": ["google", "bing"]}))) |
| 723 | with pytest.raises(SearchProviderError, match="SEARCH_PROVIDER_DEGRADED"): |
| 724 | await provider.discover(ResearchRepository().profile(AIXTRON_INSTRUMENT_ID), "Customers", SearchDateWindow(year=2026, query_limit=1)) |
| 725 | |
| 726 | |
| 727 | @pytest.mark.asyncio |
| 728 | async def test_brave_compatible_provider_empty_results_are_allowed() -> None: |
| 729 | provider = BraveCompatibleSearchDiscoveryProvider( |
| 730 | "https://api.search.brave.com/res/v1/web/search", |
| 731 | "secret-key", |
| 732 | client=_RecordingSearchClient(httpx.Response(200, json={"web": {"results": []}})), |
| 733 | ) |
| 734 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 735 | |
| 736 | results = await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 737 | |
| 738 | assert results == [] |
| 739 | |
| 740 | |
| 741 | @pytest.mark.asyncio |
| 742 | async def test_brave_compatible_provider_malformed_response_is_unavailable() -> None: |
| 743 | provider = BraveCompatibleSearchDiscoveryProvider( |
| 744 | "https://api.search.brave.com/res/v1/web/search", |
| 745 | "SECRET-KEY-123", |
| 746 | client=_RecordingSearchClient(httpx.Response(200, json={"web": {"results": "bad"}})), |
| 747 | ) |
| 748 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 749 | |
| 750 | with pytest.raises(SearchProviderError) as exc_info: |
| 751 | await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 752 | |
| 753 | assert str(exc_info.value) == "SEARCH_PROVIDER_UNAVAILABLE:invalid_response" |
| 754 | assert "SECRET-KEY-123" not in str(exc_info.value) |
| 755 | |
| 756 | |
| 757 | @pytest.mark.asyncio |
| 758 | @pytest.mark.parametrize( |
| 759 | ("status_code", "expected"), |
| 760 | [ |
| 761 | (401, "SEARCH_PROVIDER_FORBIDDEN"), |
| 762 | (403, "SEARCH_PROVIDER_FORBIDDEN"), |
| 763 | (429, "SEARCH_PROVIDER_RATE_LIMITED"), |
| 764 | ], |
| 765 | ) |
| 766 | async def test_brave_compatible_provider_auth_and_rate_limit_errors_are_sanitized( |
| 767 | status_code: int, expected: str |
| 768 | ) -> None: |
| 769 | provider = BraveCompatibleSearchDiscoveryProvider( |
| 770 | "https://api.search.brave.com/res/v1/web/search", |
| 771 | "SECRET-KEY-123", |
| 772 | client=_RecordingSearchClient(httpx.Response(status_code, json={"error": "bad SECRET-KEY-123"})), |
| 773 | ) |
| 774 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 775 | |
| 776 | with pytest.raises(SearchProviderError) as exc_info: |
| 777 | await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 778 | |
| 779 | assert str(exc_info.value) == expected |
| 780 | assert "SECRET-KEY-123" not in str(exc_info.value) |
| 781 | |
| 782 | |
| 783 | @pytest.mark.asyncio |
| 784 | async def test_brave_compatible_provider_timeout_is_sanitized() -> None: |
| 785 | provider = BraveCompatibleSearchDiscoveryProvider( |
| 786 | "https://api.search.brave.com/res/v1/web/search", |
| 787 | "SECRET-KEY-123", |
| 788 | client=_RecordingSearchClient(httpx.TimeoutException("timeout with SECRET-KEY-123")), |
| 789 | ) |
| 790 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 791 | |
| 792 | with pytest.raises(SearchProviderError) as exc_info: |
| 793 | await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 794 | |
| 795 | assert str(exc_info.value) == "SEARCH_PROVIDER_TIMEOUT" |
| 796 | assert "SECRET-KEY-123" not in str(exc_info.value) |
| 797 | |
| 798 | |
| 799 | @pytest.mark.asyncio |
| 800 | async def test_google_compatible_provider_maps_query_key_cx_and_limit() -> None: |
| 801 | client = _RecordingSearchClient( |
| 802 | httpx.Response( |
| 803 | 200, |
| 804 | json={ |
| 805 | "items": [ |
| 806 | {"title": "One", "link": "https://www.aixtron.com/en/one", "snippet": "snippet one"}, |
| 807 | {"title": "Two", "link": "https://www.aixtron.com/en/two", "snippet": "snippet two"}, |
| 808 | {"title": "Three", "link": "https://www.aixtron.com/en/three", "snippet": "snippet three"}, |
| 809 | ] |
| 810 | }, |
| 811 | ) |
| 812 | ) |
| 813 | provider = GoogleCompatibleSearchDiscoveryProvider( |
| 814 | "https://search.example/v1", |
| 815 | "secret-key", |
| 816 | "engine-id", |
| 817 | max_results_per_query=2, |
| 818 | client=client, |
| 819 | ) |
| 820 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 821 | |
| 822 | results = await provider.discover(profile, "Unmapped Category", SearchDateWindow(year=2026)) |
| 823 | |
| 824 | assert client.calls |
| 825 | assert client.calls[0]["url"] == "https://search.example/v1" |
| 826 | assert client.calls[0]["params"]["q"] == "AIXTRON SE Unmapped Category 2026" |
| 827 | assert client.calls[0]["params"]["key"] == "secret-key" |
| 828 | assert client.calls[0]["params"]["cx"] == "engine-id" |
| 829 | assert client.calls[0]["params"]["num"] == 2 |
| 830 | assert len(results) == 6 |
| 831 | assert {result.url for result in results} == {"https://www.aixtron.com/en/one", "https://www.aixtron.com/en/two"} |
| 832 | |
| 833 | |
| 834 | @pytest.mark.asyncio |
| 835 | async def test_google_compatible_provider_rate_limit_is_sanitized() -> None: |
| 836 | provider = GoogleCompatibleSearchDiscoveryProvider( |
| 837 | "https://search.example/v1", |
| 838 | "SECRET-KEY-123", |
| 839 | "engine-id", |
| 840 | client=_RecordingSearchClient(httpx.Response(429, json={"error": "rate"})), |
| 841 | ) |
| 842 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 843 | |
| 844 | with pytest.raises(SearchProviderError) as exc_info: |
| 845 | await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 846 | |
| 847 | assert str(exc_info.value) == "SEARCH_PROVIDER_RATE_LIMITED" |
| 848 | assert "SECRET-KEY-123" not in str(exc_info.value) |
| 849 | |
| 850 | |
| 851 | @pytest.mark.asyncio |
| 852 | async def test_google_compatible_provider_timeout_is_sanitized() -> None: |
| 853 | provider = GoogleCompatibleSearchDiscoveryProvider( |
| 854 | "https://search.example/v1", |
| 855 | "SECRET-KEY-123", |
| 856 | "engine-id", |
| 857 | client=_RecordingSearchClient(httpx.TimeoutException("timeout with SECRET-KEY-123")), |
| 858 | ) |
| 859 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 860 | |
| 861 | with pytest.raises(SearchProviderError) as exc_info: |
| 862 | await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 863 | |
| 864 | assert str(exc_info.value) == "SEARCH_PROVIDER_TIMEOUT" |
| 865 | assert "SECRET-KEY-123" not in str(exc_info.value) |
| 866 | |
| 867 | |
| 868 | @pytest.mark.asyncio |
| 869 | async def test_google_compatible_provider_403_is_forbidden_and_does_not_leak_api_key() -> None: |
| 870 | provider = GoogleCompatibleSearchDiscoveryProvider( |
| 871 | "https://search.example/v1", |
| 872 | "SECRET-KEY-123", |
| 873 | "engine-id", |
| 874 | client=_RecordingSearchClient(httpx.Response(403, json={"error": "bad SECRET-KEY-123"})), |
| 875 | ) |
| 876 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 877 | |
| 878 | with pytest.raises(SearchProviderError) as exc_info: |
| 879 | await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 880 | |
| 881 | assert str(exc_info.value) == "GOOGLE_PROVIDER_UNAVAILABLE" |
| 882 | assert "SECRET-KEY-123" not in str(exc_info.value) |
| 883 | |
| 884 | |
| 885 | @pytest.mark.asyncio |
| 886 | async def test_google_compatible_provider_500_is_unavailable() -> None: |
| 887 | provider = GoogleCompatibleSearchDiscoveryProvider( |
| 888 | "https://search.example/v1", |
| 889 | "SECRET-KEY-123", |
| 890 | "engine-id", |
| 891 | client=_RecordingSearchClient(httpx.Response(500, json={"error": "server"})), |
| 892 | ) |
| 893 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 894 | |
| 895 | with pytest.raises(SearchProviderError) as exc_info: |
| 896 | await provider.discover(profile, "Customers", SearchDateWindow(year=2026)) |
| 897 | |
| 898 | assert str(exc_info.value) == "SEARCH_PROVIDER_UNAVAILABLE" |
| 899 | |
| 900 | |
| 901 | @pytest.mark.asyncio |
| 902 | @respx.mock |
| 903 | async def test_research_refresh_continues_when_search_provider_forbidden() -> None: |
| 904 | search = SearchDiscoveryService(_FailingSearchProvider("SEARCH_PROVIDER_FORBIDDEN")) |
| 905 | repo = ResearchRepository( |
| 906 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 907 | discovery=_StaticDiscovery([]), |
| 908 | search_discovery=search, |
| 909 | ) |
| 910 | primary = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 911 | respx.get(primary.url).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=_aixtron_mojibake_fixture())) |
| 912 | |
| 913 | summary = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 914 | |
| 915 | assert summary.demo is False |
| 916 | assert summary.data_freshness == "REAL" |
| 917 | assert summary.documents |
| 918 | assert any(event.source_mode == SourceMode.REAL for event in summary.recent_events) |
| 919 | assert summary.catalyst_score.category_evidence["Orders & Backlog"].status == "POSITIVE_EVIDENCE" |
| 920 | assert summary.catalyst_score.category_evidence["Customers"].status == "NO_EVIDENCE" |
| 921 | assert summary.catalyst_score.category_evidence["Customers"].score is None |
| 922 | assert repo.last_live_error[AIXTRON_INSTRUMENT_ID].startswith("SEARCH_PROVIDER_UNAVAILABLE:") |
| 923 | assert "SEARCH_PROVIDER_FORBIDDEN" in repo.last_live_error[AIXTRON_INSTRUMENT_ID] |
| 924 | assert search.last_stats.rejected_reasons["SEARCH_PROVIDER_FORBIDDEN"] == 1 |
| 925 | |
| 926 | |
| 927 | @pytest.mark.asyncio |
| 928 | @respx.mock |
| 929 | async def test_research_refresh_continues_when_search_provider_rate_limited() -> None: |
| 930 | search = SearchDiscoveryService(_FailingSearchProvider("SEARCH_PROVIDER_RATE_LIMITED")) |
| 931 | repo = ResearchRepository( |
| 932 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 933 | discovery=_StaticDiscovery([]), |
| 934 | search_discovery=search, |
| 935 | ) |
| 936 | primary = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 937 | respx.get(primary.url).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=_aixtron_mojibake_fixture())) |
| 938 | |
| 939 | summary = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 940 | |
| 941 | assert summary.demo is False |
| 942 | assert summary.catalyst_score.category_evidence["Customers"].status == "NO_EVIDENCE" |
| 943 | assert repo.last_live_error[AIXTRON_INSTRUMENT_ID].startswith("SEARCH_PROVIDER_UNAVAILABLE:") |
| 944 | assert "SEARCH_PROVIDER_RATE_LIMITED" in repo.last_live_error[AIXTRON_INSTRUMENT_ID] |
| 945 | |
| 946 | |
| 947 | @pytest.mark.asyncio |
| 948 | @respx.mock |
| 949 | async def test_research_refresh_continues_when_search_provider_timeout() -> None: |
| 950 | search = SearchDiscoveryService(_FailingSearchProvider("SEARCH_PROVIDER_TIMEOUT")) |
| 951 | repo = ResearchRepository( |
| 952 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 953 | discovery=_StaticDiscovery([]), |
| 954 | search_discovery=search, |
| 955 | ) |
| 956 | primary = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 957 | respx.get(primary.url).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=_aixtron_mojibake_fixture())) |
| 958 | |
| 959 | summary = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 960 | |
| 961 | assert summary.demo is False |
| 962 | assert summary.catalyst_score.category_evidence["Customers"].status == "NO_EVIDENCE" |
| 963 | assert repo.last_live_error[AIXTRON_INSTRUMENT_ID].startswith("SEARCH_PROVIDER_UNAVAILABLE:") |
| 964 | assert "SEARCH_PROVIDER_TIMEOUT" in repo.last_live_error[AIXTRON_INSTRUMENT_ID] |
| 965 | |
| 966 | |
| 967 | @pytest.mark.asyncio |
| 968 | @respx.mock |
| 969 | async def test_search_result_is_not_evidence_and_original_publisher_page_is_fetched() -> None: |
| 970 | publisher_url = "https://www.infineon.com/cms/en/about-infineon/press/customer-aixtron" |
| 971 | provider = _StaticSearchProvider( |
| 972 | [ |
| 973 | CandidateSearchResult( |
| 974 | title="Search result title", |
| 975 | url=publisher_url, |
| 976 | snippet="AIXTRON customer win snippet cx engine-id that must never become evidence", |
| 977 | discovered_at=datetime(2026, 5, 1, tzinfo=timezone.utc), |
| 978 | provider="test-search", |
| 979 | query_id="Customers:1", |
| 980 | query="AIXTRON customer win 2026", |
| 981 | category="Customers", |
| 982 | ) |
| 983 | ] |
| 984 | ) |
| 985 | search = SearchDiscoveryService(provider, max_documents_per_refresh=2) |
| 986 | repo = ResearchRepository( |
| 987 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 988 | discovery=_StaticDiscovery([]), |
| 989 | search_discovery=search, |
| 990 | ) |
| 991 | primary = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 992 | respx.get(primary.url).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=_aixtron_mojibake_fixture())) |
| 993 | route = respx.get(publisher_url).mock( |
| 994 | return_value=httpx.Response( |
| 995 | 200, |
| 996 | headers={"content-type": "text/html"}, |
| 997 | text="<html><title>Infineon validates AIXTRON tool</title><main><p>May 2, 2026</p><p>AIXTRON SE AIXA XETR was selected by Infineon Technologies AG for silicon-carbide production equipment.</p></main></html>", |
| 998 | ) |
| 999 | ) |
| 1000 | |
| 1001 | summary = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 1002 | |
| 1003 | assert route.call_count == 1 |
| 1004 | assert search.last_stats.accepted_count == 1 |
| 1005 | assert summary.catalyst_score.category_evidence["Customers"].status == "POSITIVE_EVIDENCE" |
| 1006 | customer_events = [event for event in summary.recent_events if event.event_type == "NEW_CUSTOMER"] |
| 1007 | assert customer_events |
| 1008 | assert all("snippet" not in event.raw_evidence_reference.lower() for event in customer_events) |
| 1009 | assert all("engine-id" not in event.raw_evidence_reference.lower() for event in customer_events) |
| 1010 | assert customer_events[0].source_url == publisher_url |
| 1011 | assert customer_events[0].source_classification == SourceClassification.CUSTOMER |
| 1012 | assert customer_events[0].supporting_sources[0].url == publisher_url |
| 1013 | |
| 1014 | |
| 1015 | @pytest.mark.asyncio |
| 1016 | @respx.mock |
| 1017 | async def test_search_refresh_works_for_company_without_registered_sources() -> None: |
| 1018 | besi_id = UUID("22222222-2222-2222-2222-222222222222") |
| 1019 | publisher_url = "https://www.marketscreener.com/quote/stock/BESI-6319/news/besi-capacity-expansion" |
| 1020 | provider = _StaticSearchProvider( |
| 1021 | [ |
| 1022 | CandidateSearchResult( |
| 1023 | title="BESI capacity expansion", |
| 1024 | url=publisher_url, |
| 1025 | snippet="Search result snippet is not evidence", |
| 1026 | discovered_at=datetime(2026, 5, 1, tzinfo=timezone.utc), |
| 1027 | provider="test-search", |
| 1028 | query_id="CAPEX & Capacity:1", |
| 1029 | query="BESI capacity expansion 2026", |
| 1030 | category="CAPEX & Capacity", |
| 1031 | ) |
| 1032 | ] |
| 1033 | ) |
| 1034 | search = SearchDiscoveryService(provider, max_documents_per_refresh=2) |
| 1035 | repo = ResearchRepository( |
| 1036 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 1037 | discovery=_StaticDiscovery([]), |
| 1038 | search_discovery=search, |
| 1039 | ) |
| 1040 | respx.get(publisher_url).mock( |
| 1041 | return_value=httpx.Response( |
| 1042 | 200, |
| 1043 | headers={"content-type": "text/html"}, |
| 1044 | text="<html><title>BESI capacity expansion</title><main><p>May 2, 2026</p><p>BE Semiconductor Industries BESI XAMS will invest EUR 120 million in CAPEX for a new facility and capacity expansion.</p></main></html>", |
| 1045 | ) |
| 1046 | ) |
| 1047 | |
| 1048 | summary = await repo.refresh(besi_id) |
| 1049 | |
| 1050 | assert summary.demo is False |
| 1051 | assert summary.data_freshness == "REAL" |
| 1052 | assert summary.documents |
| 1053 | assert summary.documents[0].canonical_url == publisher_url |
| 1054 | assert summary.documents[0].source_classification == SourceClassification.INVESTMENT_RESEARCH |
| 1055 | assert summary.recent_events |
| 1056 | assert repo.last_live_error.get(besi_id) is None |
| 1057 | |
| 1058 | |
| 1059 | @pytest.mark.asyncio |
| 1060 | @respx.mock |
| 1061 | async def test_search_result_found_but_fetch_rejected_is_categorized() -> None: |
| 1062 | besi_id = UUID("22222222-2222-2222-2222-222222222222") |
| 1063 | restricted_url = "https://seekingalpha.com/article/besi-q2" |
| 1064 | provider = _StaticSearchProvider([_candidate(restricted_url, "CAPEX & Capacity")]) |
| 1065 | search = SearchDiscoveryService(provider, max_documents_per_refresh=2) |
| 1066 | repo = ResearchRepository( |
| 1067 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 1068 | discovery=_StaticDiscovery([]), |
| 1069 | search_discovery=search, |
| 1070 | ) |
| 1071 | respx.get(restricted_url).mock(return_value=httpx.Response(403, headers={"content-type": "text/html"})) |
| 1072 | |
| 1073 | summary = await repo.refresh(besi_id) |
| 1074 | |
| 1075 | assert summary.data_freshness == "DEMO_FALLBACK" |
| 1076 | assert search.last_stats.candidate_count == 1 |
| 1077 | assert search.last_stats.rejected_reasons["ROBOTS_OR_ACCESS_BLOCKED"] == 1 |
| 1078 | assert repo.documents_for(besi_id, source_mode=SourceMode.REAL) == [] |
| 1079 | |
| 1080 | |
| 1081 | @pytest.mark.asyncio |
| 1082 | @respx.mock |
| 1083 | async def test_search_refresh_persists_relevant_document_without_publication_date_or_events() -> None: |
| 1084 | besi_id = UUID("22222222-2222-2222-2222-222222222222") |
| 1085 | publisher_url = "https://www.marketscreener.com/quote/stock/BESI-6319/news/besi-company-profile" |
| 1086 | provider = _StaticSearchProvider([_candidate(publisher_url, "Growth")]) |
| 1087 | search = SearchDiscoveryService(provider, max_documents_per_refresh=2) |
| 1088 | repo = ResearchRepository( |
| 1089 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 1090 | discovery=_StaticDiscovery([]), |
| 1091 | search_discovery=search, |
| 1092 | ) |
| 1093 | respx.get(publisher_url).mock( |
| 1094 | return_value=httpx.Response( |
| 1095 | 200, |
| 1096 | headers={"content-type": "text/html"}, |
| 1097 | text="<html><title>BESI company profile</title><main><p>BE Semiconductor Industries N.V. BESI XAMS supplies assembly equipment for semiconductor manufacturers.</p></main></html>", |
| 1098 | ) |
| 1099 | ) |
| 1100 | |
| 1101 | summary = await repo.refresh(besi_id) |
| 1102 | |
| 1103 | assert summary.demo is False |
| 1104 | assert summary.data_freshness == "REAL" |
| 1105 | assert len(summary.documents) == 1 |
| 1106 | assert summary.documents[0].published_at is None |
| 1107 | assert summary.documents[0].retrieved_at is not None |
| 1108 | assert summary.documents[0].source_classification == SourceClassification.INVESTMENT_RESEARCH |
| 1109 | assert summary.documents[0].discovery_provider == "SEARCH_DISCOVERY" |
| 1110 | assert summary.recent_events == [] |
| 1111 | assert search.last_stats.documents_fetched == 1 |
| 1112 | assert search.last_stats.events_extracted == 0 |
| 1113 | assert search.last_stats.rejected_reasons["SEARCH_RESULT_ACCEPTED"] == 1 |
| 1114 | |
| 1115 | |
| 1116 | @pytest.mark.asyncio |
| 1117 | @respx.mock |
| 1118 | async def test_tier_two_and_three_sources_are_accepted_with_lower_confidence() -> None: |
| 1119 | besi_id = UUID("22222222-2222-2222-2222-222222222222") |
| 1120 | urls = [ |
| 1121 | "https://finance.yahoo.com/technology/articles/semiconductor-industries-n-v-announces-071600996.html", |
| 1122 | "https://www.marketscreener.com/quote/stock/BESI-6319/news/besi-capacity-expansion", |
| 1123 | ] |
| 1124 | provider = _StaticSearchProvider([_candidate(urls[0], "Guidance"), _candidate(urls[1], "CAPEX & Capacity")]) |
| 1125 | search = SearchDiscoveryService(provider, max_documents_per_refresh=3) |
| 1126 | repo = ResearchRepository( |
| 1127 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 1128 | discovery=_StaticDiscovery([]), |
| 1129 | search_discovery=search, |
| 1130 | ) |
| 1131 | respx.get(urls[0]).mock( |
| 1132 | return_value=httpx.Response( |
| 1133 | 200, |
| 1134 | headers={"content-type": "text/html"}, |
| 1135 | text="<html><title>BE Semiconductor Industries N.V. Announces Q2-26 Results</title><main><p>July 23, 2026</p><p>BE Semiconductor Industries N.V. BESI XAMS guidance for the full year remains supported by order intake.</p></main></html>", |
| 1136 | ) |
| 1137 | ) |
| 1138 | respx.get(urls[1]).mock( |
| 1139 | return_value=httpx.Response( |
| 1140 | 200, |
| 1141 | headers={"content-type": "text/html"}, |
| 1142 | text="<html><title>BESI capacity expansion</title><main><p>BE Semiconductor Industries BESI XAMS will invest EUR 120 million in CAPEX and expand capacity.</p></main></html>", |
| 1143 | ) |
| 1144 | ) |
| 1145 | |
| 1146 | summary = await repo.refresh(besi_id) |
| 1147 | |
| 1148 | classifications = {document.source_classification for document in summary.documents} |
| 1149 | assert SourceClassification.REPUTABLE_NEWS in classifications |
| 1150 | assert SourceClassification.INVESTMENT_RESEARCH in classifications |
| 1151 | assert any(event.reliability == ReliabilityLevel.LEVEL_C for event in summary.recent_events) |
| 1152 | assert any(event.reliability == ReliabilityLevel.LEVEL_D for event in summary.recent_events) |
| 1153 | assert all(event.confidence < 0.90 for event in summary.recent_events) |
| 1154 | |
| 1155 | |
| 1156 | @pytest.mark.asyncio |
| 1157 | @respx.mock |
| 1158 | async def test_search_relevance_rejects_wrong_company_result_without_persisting() -> None: |
| 1159 | besi_id = UUID("22222222-2222-2222-2222-222222222222") |
| 1160 | url = "https://finance.yahoo.com/news/nvidia-capex" |
| 1161 | provider = _StaticSearchProvider([_candidate(url, "CAPEX & Capacity")]) |
| 1162 | search = SearchDiscoveryService(provider, max_documents_per_refresh=2) |
| 1163 | repo = ResearchRepository( |
| 1164 | settings=Settings(research_live_enabled=True, research_demo_enabled=True, research_search_enabled=True, research_max_retries=0), |
| 1165 | discovery=_StaticDiscovery([]), |
| 1166 | search_discovery=search, |
| 1167 | ) |
| 1168 | respx.get(url).mock( |
| 1169 | return_value=httpx.Response( |
| 1170 | 200, |
| 1171 | headers={"content-type": "text/html"}, |
| 1172 | text="<html><title>NVIDIA capex</title><main><p>NVIDIA Corporation NVDA XNAS reported large AI capex plans.</p></main></html>", |
| 1173 | ) |
| 1174 | ) |
| 1175 | |
| 1176 | await repo.refresh(besi_id) |
| 1177 | |
| 1178 | assert repo.documents_for(besi_id, source_mode=SourceMode.REAL) == [] |
| 1179 | assert search.last_stats.rejected_reasons["COMPANY_RELEVANCE_FAILED"] == 1 |
| 1180 | |
| 1181 | |
| 1182 | @pytest.mark.asyncio |
| 1183 | async def test_search_discovery_rejects_unsafe_and_seen_candidates_but_accepts_other_public_web() -> None: |
| 1184 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 1185 | provider = _StaticSearchProvider( |
| 1186 | [ |
| 1187 | _candidate("http://127.0.0.1/private", "Customers"), |
| 1188 | _candidate("https://unknown.example/aixtron", "Customers"), |
| 1189 | _candidate("https://www.aixtron.com/en/press/customer-win", "Customers"), |
| 1190 | ] |
| 1191 | ) |
| 1192 | search = SearchDiscoveryService(provider) |
| 1193 | |
| 1194 | results = await search.discover(profile, {"Customers"}, {canonicalize_url("https://www.aixtron.com/en/press/customer-win")}) |
| 1195 | |
| 1196 | assert len(results) == 1 |
| 1197 | assert results[0].source.url == "https://unknown.example/aixtron" |
| 1198 | assert results[0].source.source_classification == SourceClassification.OTHER |
| 1199 | assert search.last_stats.rejected_count == 2 |
| 1200 | assert "DOMAIN_VALIDATION_FAILED" in search.last_stats.rejected_reasons |
| 1201 | assert "DUPLICATE" in search.last_stats.rejected_reasons |
| 1202 | |
| 1203 | |
| 1204 | @pytest.mark.asyncio |
| 1205 | async def test_search_discovery_classifies_validated_company_domain_as_official() -> None: |
| 1206 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 1207 | profile.known_domains = [] |
| 1208 | provider = _StaticSearchProvider( |
| 1209 | [ |
| 1210 | CandidateSearchResult( |
| 1211 | title="AIXTRON SE investor relations", |
| 1212 | url="https://www.aixtron.com/en/investors/results", |
| 1213 | snippet="AIXTRON SE annual report and quarterly results", |
| 1214 | discovered_at=datetime(2026, 5, 1, tzinfo=timezone.utc), |
| 1215 | provider="test-search", |
| 1216 | query_id="FINANCIAL_RESULTS:1", |
| 1217 | query="AIXTRON SE annual report", |
| 1218 | category="FINANCIAL_RESULTS", |
| 1219 | ) |
| 1220 | ] |
| 1221 | ) |
| 1222 | search = SearchDiscoveryService(provider, max_documents_per_refresh=2) |
| 1223 | |
| 1224 | results = await search.discover(profile, {"FINANCIAL_RESULTS"}, set()) |
| 1225 | |
| 1226 | assert results[0].source.source_classification == SourceClassification.OFFICIAL_COMPANY |
| 1227 | assert results[0].source.source_type == SourceType.INVESTOR_RELATIONS |
| 1228 | |
| 1229 | |
| 1230 | @pytest.mark.asyncio |
| 1231 | async def test_search_discovery_is_bounded() -> None: |
| 1232 | profile = ResearchRepository().profile(AIXTRON_INSTRUMENT_ID) |
| 1233 | provider = _StaticSearchProvider([_candidate(f"https://www.aixtron.com/en/press/customer-{index}", "Customers") for index in range(10)]) |
| 1234 | search = SearchDiscoveryService(provider, max_documents_per_refresh=3) |
| 1235 | |
| 1236 | results = await search.discover(profile, {"Customers"}, set()) |
| 1237 | |
| 1238 | assert len(results) == 3 |
| 1239 | assert search.last_stats.accepted_count == 3 |
| 1240 | |
| 1241 | |
| 1242 | def test_source_independence_uses_content_hash_for_syndicated_duplicates() -> None: |
| 1243 | repo = ResearchRepository() |
| 1244 | body = "<html><title>AIXTRON customer</title><body>AIXTRON SE AIXA XETR announced a customer win with Infineon Technologies AG.</body></html>" |
| 1245 | first = repo.ingest_fixture( |
| 1246 | original_url="https://www.aixtron.com/en/customer-win", |
| 1247 | source_type=SourceType.INVESTOR_RELATIONS, |
| 1248 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 1249 | source_name="AIXTRON", |
| 1250 | publisher="AIXTRON SE", |
| 1251 | content_type="text/html", |
| 1252 | body=body, |
| 1253 | reliability=ReliabilityLevel.LEVEL_B, |
| 1254 | source_mode=SourceMode.REAL, |
| 1255 | ) |
| 1256 | duplicate = repo.ingest_fixture( |
| 1257 | original_url="https://www.reuters.com/markets/aixtron-customer-win", |
| 1258 | source_type=SourceType.NEWS, |
| 1259 | source_classification=SourceClassification.REPUTABLE_NEWS, |
| 1260 | source_name="Reuters", |
| 1261 | publisher="Reuters", |
| 1262 | content_type="text/html", |
| 1263 | body=body, |
| 1264 | reliability=ReliabilityLevel.LEVEL_C, |
| 1265 | source_mode=SourceMode.REAL, |
| 1266 | ) |
| 1267 | |
| 1268 | assert first.status == DocumentStatus.PROCESSED |
| 1269 | assert duplicate.status == DocumentStatus.DUPLICATE |
| 1270 | assert duplicate.duplicate_of_document_id == first.document_id |
| 1271 | evidence = repo.summary(AIXTRON_INSTRUMENT_ID).catalyst_score.category_evidence["Customers"] |
| 1272 | assert evidence.independent_source_count == 1 |
| 1273 | |
| 1274 | |
| 1275 | def test_durable_persistence_survives_repository_restart(tmp_path) -> None: |
| 1276 | database = tmp_path / "research.sqlite" |
| 1277 | persistence = SqliteResearchPersistence(database) |
| 1278 | repo = ResearchRepository(settings=Settings(research_demo_enabled=False), persistence=persistence) |
| 1279 | doc = repo.ingest_fixture( |
| 1280 | original_url="https://www.aixtron.com/en/customer-win", |
| 1281 | source_type=SourceType.INVESTOR_RELATIONS, |
| 1282 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 1283 | source_name="AIXTRON", |
| 1284 | publisher="AIXTRON SE", |
| 1285 | content_type="text/html", |
| 1286 | body="<html><title>AIXTRON customer</title><body>AIXTRON SE AIXA XETR DE000A0WMPJ6 announced a customer win with Infineon Technologies AG.</body></html>", |
| 1287 | reliability=ReliabilityLevel.LEVEL_B, |
| 1288 | source_mode=SourceMode.REAL, |
| 1289 | ) |
| 1290 | |
| 1291 | restarted = ResearchRepository( |
| 1292 | settings=Settings(research_demo_enabled=False), |
| 1293 | persistence=SqliteResearchPersistence(database), |
| 1294 | ) |
| 1295 | summary = restarted.summary(AIXTRON_INSTRUMENT_ID) |
| 1296 | |
| 1297 | assert doc.status == DocumentStatus.PROCESSED |
| 1298 | assert summary.demo is False |
| 1299 | assert summary.documents[0].canonical_url == "https://www.aixtron.com/en/customer-win" |
| 1300 | assert summary.recent_events |
| 1301 | assert summary.catalyst_score.category_evidence["Customers"].status == "POSITIVE_EVIDENCE" |
| 1302 | |
| 1303 | |
| 1304 | def test_document_subtype_round_trips_and_deduplication_enriches_metadata(tmp_path) -> None: |
| 1305 | database = tmp_path / "research.sqlite" |
| 1306 | repo = ResearchRepository(settings=Settings(research_demo_enabled=False), persistence=SqliteResearchPersistence(database)) |
| 1307 | kwargs = dict( |
| 1308 | original_url="https://www.aixtron.com/en/investor-presentation", |
| 1309 | source_type=SourceType.INVESTOR_RELATIONS, |
| 1310 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 1311 | source_name="AIXTRON", |
| 1312 | publisher="AIXTRON SE", |
| 1313 | content_type="text/html", |
| 1314 | body="<html><title>AIXTRON presentation</title><body>AIXTRON SE AIXA XETR DE000A0WMPJ6 published a sufficiently detailed investor presentation.</body></html>", |
| 1315 | reliability=ReliabilityLevel.LEVEL_B, |
| 1316 | source_mode=SourceMode.REAL, |
| 1317 | ) |
| 1318 | generic = repo.ingest_fixture(**kwargs) |
| 1319 | enriched = generic.model_copy(update={"document_subtype": DocumentSubtype.INVESTOR_PRESENTATION}) |
| 1320 | assert repo._persistence.upsert_document(enriched) is False |
| 1321 | |
| 1322 | restarted = ResearchRepository(settings=Settings(research_demo_enabled=False), persistence=SqliteResearchPersistence(database)) |
| 1323 | documents = restarted.documents_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL) |
| 1324 | assert len(documents) == 1 |
| 1325 | assert documents[0].document_id == generic.document_id |
| 1326 | assert documents[0].document_subtype == DocumentSubtype.INVESTOR_PRESENTATION |
| 1327 | |
| 1328 | |
| 1329 | def test_durable_document_and_event_refresh_are_idempotent(tmp_path) -> None: |
| 1330 | database = tmp_path / "research.sqlite" |
| 1331 | repo = ResearchRepository(settings=Settings(research_demo_enabled=False), persistence=SqliteResearchPersistence(database)) |
| 1332 | kwargs = dict( |
| 1333 | original_url="https://www.aixtron.com/en/order", |
| 1334 | source_type=SourceType.INVESTOR_RELATIONS, |
| 1335 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 1336 | source_name="AIXTRON", |
| 1337 | publisher="AIXTRON SE", |
| 1338 | content_type="text/html", |
| 1339 | body="<html><title>AIXTRON order</title><body>AIXTRON SE AIXA XETR DE000A0WMPJ6 announced a new order worth EUR 10 million.</body></html>", |
| 1340 | reliability=ReliabilityLevel.LEVEL_B, |
| 1341 | source_mode=SourceMode.REAL, |
| 1342 | ) |
| 1343 | |
| 1344 | first = repo.ingest_fixture(**kwargs) |
| 1345 | second = repo.ingest_fixture(**kwargs) |
| 1346 | restarted = ResearchRepository(settings=Settings(research_demo_enabled=False), persistence=SqliteResearchPersistence(database)) |
| 1347 | |
| 1348 | assert first.status == DocumentStatus.PROCESSED |
| 1349 | assert second.status == DocumentStatus.DUPLICATE |
| 1350 | assert len(restarted.documents_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL)) == 1 |
| 1351 | assert len(restarted.events_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL)) == 1 |
| 1352 | |
| 1353 | |
| 1354 | def test_durable_event_can_link_multiple_sources(tmp_path) -> None: |
| 1355 | persistence = SqliteResearchPersistence(tmp_path / "research.sqlite") |
| 1356 | repo = ResearchRepository(settings=Settings(research_demo_enabled=False), persistence=persistence) |
| 1357 | first = repo.ingest_fixture( |
| 1358 | original_url="https://www.aixtron.com/en/customer-win", |
| 1359 | source_type=SourceType.INVESTOR_RELATIONS, |
| 1360 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 1361 | source_name="AIXTRON", |
| 1362 | publisher="AIXTRON SE", |
| 1363 | content_type="text/html", |
| 1364 | body="<html><title>AIXTRON customer</title><body>AIXTRON SE AIXA XETR DE000A0WMPJ6 announced a customer win with Infineon Technologies AG.</body></html>", |
| 1365 | reliability=ReliabilityLevel.LEVEL_B, |
| 1366 | source_mode=SourceMode.REAL, |
| 1367 | ) |
| 1368 | second = repo.ingest_fixture( |
| 1369 | original_url="https://www.infineon.com/cms/en/aixtron", |
| 1370 | source_type=SourceType.SEARCH_DISCOVERY, |
| 1371 | source_classification=SourceClassification.CUSTOMER, |
| 1372 | source_name="Infineon", |
| 1373 | publisher="Infineon", |
| 1374 | content_type="text/html", |
| 1375 | body="<html><title>Infineon customer</title><body><p>Infineon source confirmation.</p><p>AIXTRON SE AIXA XETR DE000A0WMPJ6 announced a customer win with Infineon Technologies AG.</p></body></html>", |
| 1376 | reliability=ReliabilityLevel.LEVEL_B, |
| 1377 | source_mode=SourceMode.REAL, |
| 1378 | ) |
| 1379 | event = next( |
| 1380 | event |
| 1381 | for event in repo.events_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL) |
| 1382 | if event.source_document_id == first.document_id |
| 1383 | ) |
| 1384 | event.supporting_sources.append( |
| 1385 | event.supporting_sources[0].model_copy( |
| 1386 | update={ |
| 1387 | "document_id": second.document_id, |
| 1388 | "url": second.canonical_url, |
| 1389 | "canonical_url": second.canonical_url, |
| 1390 | "source_name": "Infineon", |
| 1391 | "publisher": "Infineon", |
| 1392 | "source_type": SourceClassification.CUSTOMER, |
| 1393 | } |
| 1394 | ) |
| 1395 | ) |
| 1396 | persistence.upsert_event(event) |
| 1397 | |
| 1398 | restarted = ResearchRepository(settings=Settings(research_demo_enabled=False), persistence=SqliteResearchPersistence(tmp_path / "research.sqlite")) |
| 1399 | loaded_event = next( |
| 1400 | loaded |
| 1401 | for loaded in restarted.events_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL) |
| 1402 | if loaded.event_id == event.event_id |
| 1403 | ) |
| 1404 | |
| 1405 | assert first.status == DocumentStatus.PROCESSED |
| 1406 | assert len(loaded_event.supporting_sources) == 2 |
| 1407 | |
| 1408 | |
| 1409 | def test_refresh_audit_records_success_and_failure_states(tmp_path) -> None: |
| 1410 | persistence = SqliteResearchPersistence(tmp_path / "research.sqlite") |
| 1411 | run = persistence.start_refresh_run( |
| 1412 | instrument_id=AIXTRON_INSTRUMENT_ID, |
| 1413 | company_id=UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1"), |
| 1414 | correlation_id="phase4-test", |
| 1415 | mode="LIVE", |
| 1416 | ) |
| 1417 | persistence.complete_refresh_run( |
| 1418 | run, |
| 1419 | status="FAILED", |
| 1420 | documents_discovered=1, |
| 1421 | documents_accepted=0, |
| 1422 | events_extracted=0, |
| 1423 | events_created=0, |
| 1424 | events_updated=0, |
| 1425 | deduplicated_count=1, |
| 1426 | safe_error_code="FetchError", |
| 1427 | safe_error_message="safe", |
| 1428 | ) |
| 1429 | row = persistence._connection.execute("SELECT * FROM research_refresh_runs WHERE refresh_run_id = ?", (str(run.refresh_run_id),)).fetchone() |
| 1430 | |
| 1431 | assert row["status"] == "FAILED" |
| 1432 | assert row["correlation_id"] == "phase4-test" |
| 1433 | assert row["safe_error_code"] == "FetchError" |
| 1434 | |
| 1435 | |
| 1436 | def test_durable_unique_constraints_and_rollback_on_failure(tmp_path) -> None: |
| 1437 | persistence = SqliteResearchPersistence(tmp_path / "research.sqlite") |
| 1438 | try: |
| 1439 | with persistence._connection: |
| 1440 | persistence._connection.execute( |
| 1441 | "INSERT INTO research_events (event_id, instrument_id, company_id, event_fingerprint, event_type, detected_at, title, summary, impact, time_horizon, confidence, status, source_document_id, source_url, source_type, source_classification, reliability, source_mode, raw_evidence_reference, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", |
| 1442 | ( |
| 1443 | str(UUID("99999999-9999-9999-9999-999999999999")), |
| 1444 | str(AIXTRON_INSTRUMENT_ID), |
| 1445 | "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1", |
| 1446 | "bad", |
| 1447 | "NEW_ORDER", |
| 1448 | "2026-01-01T00:00:00+00:00", |
| 1449 | "bad", |
| 1450 | "bad", |
| 1451 | "POSITIVE", |
| 1452 | "SHORT_TERM", |
| 1453 | 0.9, |
| 1454 | "VALIDATED", |
| 1455 | str(UUID("88888888-8888-8888-8888-888888888888")), |
| 1456 | "https://www.aixtron.com/en/missing", |
| 1457 | "INVESTOR_RELATIONS", |
| 1458 | "OFFICIAL_COMPANY", |
| 1459 | "LEVEL_B", |
| 1460 | "REAL", |
| 1461 | "bad", |
| 1462 | "2026-01-01T00:00:00+00:00", |
| 1463 | "2026-01-01T00:00:00+00:00", |
| 1464 | ), |
| 1465 | ) |
| 1466 | except Exception: |
| 1467 | pass |
| 1468 | |
| 1469 | count = persistence._connection.execute("SELECT count(*) AS count FROM research_events").fetchone()["count"] |
| 1470 | assert count == 0 |
| 1471 | |
| 1472 | |
| 1473 | @pytest.mark.asyncio |
| 1474 | async def test_portfolio_research_handles_supported_source_not_configured_and_unresolved(tmp_path) -> None: |
| 1475 | positions = [ |
| 1476 | _portfolio_position("DE000A0WMPJ6", "AIXA", "XETR", "AIXTRON SE"), |
| 1477 | _portfolio_position("DE000A0WMPJ6", "AIXA", "XETR", "AIXTRON SE"), |
| 1478 | _portfolio_position("NL0012866412", "BESI", "XAMS", "BE Semiconductor Industries"), |
| 1479 | _portfolio_position("UNKNOWN", "UNKNOWN", "XNAS", "Unknown Corp"), |
| 1480 | ] |
| 1481 | client = _RecordingPortfolioClient(positions) |
| 1482 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=False)) |
| 1483 | orchestrator = PortfolioResearchOrchestrator( |
| 1484 | repo, |
| 1485 | Settings( |
| 1486 | portfolio_service_base_url="http://portfolio-service", |
| 1487 | research_live_enabled=True, |
| 1488 | research_search_enabled=True, |
| 1489 | ), |
| 1490 | client=client, |
| 1491 | structured_provider=_UnavailableStructuredProvider(), |
| 1492 | ) |
| 1493 | |
| 1494 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa"), correlation_id="phase4-correlation") |
| 1495 | |
| 1496 | statuses = {company.company_name: company.status for company in result.companies} |
| 1497 | assert result.companies_requested == 3 |
| 1498 | assert statuses["AIXTRON SE"] == "RESOLVED_NO_SOURCES" |
| 1499 | assert statuses["BE Semiconductor Industries"] == "RESOLVED_NO_SOURCES" |
| 1500 | assert statuses["Unknown Corp"] == "COMPANY_NOT_RESOLVED" |
| 1501 | assert client.calls[0]["headers"]["X-Correlation-Id"] == "phase4-correlation" |
| 1502 | |
| 1503 | |
| 1504 | @pytest.mark.asyncio |
| 1505 | async def test_portfolio_research_resolves_ibkr_conid_exchange_alias_and_skips_etf() -> None: |
| 1506 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=True)) |
| 1507 | repo.profiles[0].provider_instrument_ids["IBKR"] = "AIXTRON-CONID" |
| 1508 | client = _RecordingPortfolioClient([ |
| 1509 | _portfolio_position(None, "AIXA", "IBIS2", "AIXA", provider="IBKR", provider_instrument_id="AIXTRON-CONID"), |
| 1510 | _portfolio_position("IE00B4L5Y983", "IUSA", "AEB", "iShares Core S&P 500 UCITS ETF", asset_type="ETF"), |
| 1511 | ]) |
| 1512 | orchestrator = PortfolioResearchOrchestrator( |
| 1513 | repo, |
| 1514 | Settings( |
| 1515 | portfolio_service_base_url="http://portfolio-service", |
| 1516 | research_live_enabled=True, |
| 1517 | research_search_enabled=True, |
| 1518 | ), |
| 1519 | client=client, |
| 1520 | ) |
| 1521 | |
| 1522 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1523 | |
| 1524 | by_ticker = {company.ticker: company for company in result.companies} |
| 1525 | assert by_ticker["AIXA"].instrument_id == AIXTRON_INSTRUMENT_ID |
| 1526 | assert by_ticker["AIXA"].company_id == UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1") |
| 1527 | assert by_ticker["AIXA"].status == "RESOLVED_PARTIAL_DATA" |
| 1528 | assert by_ticker["AIXA"].exchange == "XETR" |
| 1529 | assert by_ticker["IUSA"].status == "ETF_UNSUPPORTED" |
| 1530 | assert by_ticker["IUSA"].asset_type == "ETF" |
| 1531 | assert len([profile for profile in repo.list_profiles() if profile.company_name == "AIXTRON SE"]) == 1 |
| 1532 | |
| 1533 | |
| 1534 | @pytest.mark.asyncio |
| 1535 | async def test_portfolio_research_read_shows_existing_besi_real_research_without_refresh() -> None: |
| 1536 | repo = ResearchRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=True)) |
| 1537 | repo.ingest_fixture( |
| 1538 | original_url="https://www.besi.com/investor-relations/press-releases/details/real-order", |
| 1539 | source_type=SourceType.INVESTOR_RELATIONS, |
| 1540 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 1541 | source_name="BESI official press release", |
| 1542 | publisher="BE Semiconductor Industries", |
| 1543 | content_type="text/html", |
| 1544 | body=( |
| 1545 | "<html><title>BESI capacity expansion</title><body>" |
| 1546 | "BE Semiconductor Industries BESI XAMS announced capacity expansion for hybrid bonding equipment " |
| 1547 | "and a new customer order supporting 2026 backlog." |
| 1548 | "</body></html>" |
| 1549 | ), |
| 1550 | reliability=ReliabilityLevel.LEVEL_B, |
| 1551 | source_mode=SourceMode.REAL, |
| 1552 | ) |
| 1553 | client = _RecordingPortfolioClient([ |
| 1554 | _portfolio_position( |
| 1555 | "NL0012866412", |
| 1556 | "BESI", |
| 1557 | "AEB", |
| 1558 | "BE Semiconductor Industries N.V.", |
| 1559 | provider="IBKR", |
| 1560 | provider_instrument_id="BESI-CONID", |
| 1561 | data_freshness="REAL_BROKER", |
| 1562 | ) |
| 1563 | ]) |
| 1564 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client, |
| 1565 | structured_provider=_UnavailableStructuredProvider()) |
| 1566 | |
| 1567 | result = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1568 | |
| 1569 | company = result.companies[0] |
| 1570 | assert company.company_name == "BE Semiconductor Industries" |
| 1571 | assert company.status == "RESOLVED_RESEARCH_AVAILABLE" |
| 1572 | assert company.freshness == "REAL" |
| 1573 | assert company.mode == "REAL" |
| 1574 | assert company.document_count == 1 |
| 1575 | assert company.event_count > 0 |
| 1576 | assert company.source_count == 1 |
| 1577 | |
| 1578 | |
| 1579 | @pytest.mark.asyncio |
| 1580 | async def test_global_instrument_alias_is_resolvable_before_research_exists_and_reuses_global_research() -> None: |
| 1581 | repo = ResearchRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=False)) |
| 1582 | global_instrument_id = UUID("77777777-7777-7777-7777-777777777777") |
| 1583 | broker_position = _portfolio_position( |
| 1584 | "INE000K01001", "BROKER_ALIAS", "NSE", "Example Components Limited", |
| 1585 | provider="ICICI_DIRECT", provider_instrument_id="broker-alias", data_freshness="REAL_BROKER", |
| 1586 | ) |
| 1587 | broker_position["instrument"].update({ |
| 1588 | "globalInstrumentId": str(global_instrument_id), "country": "IN", |
| 1589 | "providerMappings": [ |
| 1590 | {"provider": "NSE", "providerSymbol": "VERIFIED_NSE", "status": "VERIFIED"}, |
| 1591 | {"provider": "YAHOO_FINANCE", "providerSymbol": "VERIFIED.NS", "status": "VERIFIED"}, |
| 1592 | ], |
| 1593 | }) |
| 1594 | orchestrator = PortfolioResearchOrchestrator( |
| 1595 | repo, Settings(portfolio_service_base_url="http://portfolio-service"), |
| 1596 | client=_RecordingPortfolioClient([broker_position]), structured_provider=_UnavailableStructuredProvider(), |
| 1597 | ) |
| 1598 | |
| 1599 | before_refresh = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-111111111111")) |
| 1600 | assert before_refresh.companies[0].instrument_id == global_instrument_id |
| 1601 | assert before_refresh.companies[0].status == "RESOLVED_NO_SOURCES" |
| 1602 | assert before_refresh.companies[0].provider == "ICICI_DIRECT" |
| 1603 | assert before_refresh.companies[0].provider_instrument_id == "broker-alias" |
| 1604 | assert before_refresh.companies[0].listing_provider == "NSE" |
| 1605 | assert before_refresh.companies[0].listing_symbol == "VERIFIED_NSE" |
| 1606 | assert before_refresh.companies[0].primary_exchange == "NSE" |
| 1607 | assert before_refresh.companies[0].verified_provider_mappings == {"ICICI_DIRECT": "BROKER-ALIAS", "NSE": "VERIFIED_NSE", "YAHOO_FINANCE": "VERIFIED.NS"} |
| 1608 | profile = repo.profile(global_instrument_id) |
| 1609 | assert profile.ticker == "BROKER_ALIAS" |
| 1610 | assert profile.provider_instrument_ids["NSE"] == "VERIFIED_NSE" |
| 1611 | |
| 1612 | repo.ingest_fixture( |
| 1613 | original_url="https://example.test/results", source_type=SourceType.EXCHANGE_ANNOUNCEMENT, |
| 1614 | source_classification=SourceClassification.EXCHANGE, source_name="NSE", publisher="NSE", content_type="text/html", |
| 1615 | body="<main>Example Components Limited VERIFIED_NSE INE000K01001 quarterly financial results revenue 1000 crore PAT 100 crore</main>", |
| 1616 | reliability=ReliabilityLevel.LEVEL_A, source_mode=SourceMode.REAL, expected_profile=profile, |
| 1617 | ) |
| 1618 | manual_alias = _portfolio_position("INE000K01001", "SECOND_BROKER_ALIAS", "NSE", "Example Components Limited") |
| 1619 | manual_alias["instrument"].update({"globalInstrumentId": str(global_instrument_id), "country": "IN", "providerMappings": broker_position["instrument"]["providerMappings"]}) |
| 1620 | assert len(orchestrator._dedupe_instruments([broker_position, manual_alias])) == 1 |
| 1621 | orchestrator._client = _RecordingPortfolioClient([manual_alias]) |
| 1622 | after_refresh = await orchestrator.read_portfolio_summary(UUID("bbbbbbbb-1111-1111-1111-111111111111")) |
| 1623 | assert after_refresh.companies[0].instrument_id == global_instrument_id |
| 1624 | assert after_refresh.companies[0].status == "RESOLVED_PARTIAL_DATA" |
| 1625 | assert after_refresh.companies[0].document_count == 1 |
| 1626 | |
| 1627 | |
| 1628 | @pytest.mark.asyncio |
| 1629 | async def test_portfolio_research_read_resolves_aixa_to_existing_aixtron_without_refresh_or_duplicate() -> None: |
| 1630 | repo = ResearchRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=True)) |
| 1631 | repo.ingest_fixture( |
| 1632 | original_url="https://www.aixtron.com/en/press/press-releases/read-existing", |
| 1633 | source_type=SourceType.INVESTOR_RELATIONS, |
| 1634 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 1635 | source_name="AIXTRON official press release", |
| 1636 | publisher="AIXTRON SE", |
| 1637 | content_type="text/html", |
| 1638 | body=_aixtron_live_fixture(), |
| 1639 | reliability=ReliabilityLevel.LEVEL_B, |
| 1640 | source_mode=SourceMode.REAL, |
| 1641 | ) |
| 1642 | client = _RecordingPortfolioClient([ |
| 1643 | _portfolio_position( |
| 1644 | None, |
| 1645 | "AIXA", |
| 1646 | "IBIS2", |
| 1647 | "AIXA", |
| 1648 | provider="IBKR", |
| 1649 | provider_instrument_id="AIXA-CONID", |
| 1650 | data_freshness="REAL_BROKER", |
| 1651 | ) |
| 1652 | ]) |
| 1653 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client, |
| 1654 | structured_provider=_UnavailableStructuredProvider()) |
| 1655 | |
| 1656 | result = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1657 | |
| 1658 | company = result.companies[0] |
| 1659 | assert company.instrument_id == AIXTRON_INSTRUMENT_ID |
| 1660 | assert company.company_id == UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1") |
| 1661 | assert company.company_name == "AIXTRON SE" |
| 1662 | assert company.status == "RESOLVED_RESEARCH_AVAILABLE" |
| 1663 | assert company.freshness == "REAL" |
| 1664 | assert len([profile for profile in repo.list_profiles() if profile.company_name == "AIXTRON SE"]) == 1 |
| 1665 | |
| 1666 | |
| 1667 | @pytest.mark.asyncio |
| 1668 | async def test_portfolio_research_read_derives_terminal_statuses_without_refresh() -> None: |
| 1669 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=True)) |
| 1670 | client = _RecordingPortfolioClient([ |
| 1671 | _portfolio_position("IE00B4L5Y983", "IUSA", "AEB", "iShares Core S&P 500 UCITS ETF", asset_type="ETF"), |
| 1672 | _portfolio_position("DE000A0WMPJ6", "AIXA", "XETR", "AIXTRON SE", data_freshness="REAL_BROKER"), |
| 1673 | _portfolio_position("UNKNOWN", "UNKNOWN", "XNAS", "Unknown Corp", data_freshness="REAL_BROKER"), |
| 1674 | ]) |
| 1675 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client, |
| 1676 | structured_provider=_UnavailableStructuredProvider()) |
| 1677 | |
| 1678 | result = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1679 | |
| 1680 | by_ticker = {company.ticker: company for company in result.companies} |
| 1681 | assert by_ticker["IUSA"].status == "ETF_UNSUPPORTED" |
| 1682 | assert by_ticker["AIXA"].status == "RESOLVED_NO_SOURCES" |
| 1683 | assert by_ticker["UNKNOWN"].status == "COMPANY_NOT_RESOLVED" |
| 1684 | assert "INSTRUMENT_RESOLVED" not in {company.status for company in result.companies} |
| 1685 | |
| 1686 | |
| 1687 | @pytest.mark.asyncio |
| 1688 | async def test_etf_projects_only_trusted_nse_listing_identity_without_changing_ticker() -> None: |
| 1689 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=True)) |
| 1690 | trusted = _portfolio_position("INF204KB17I5", "GOLDEX", "NSE", "NIPPON INDIA ETF GOLD BEES", asset_type="ETF", |
| 1691 | canonical_symbol="GOLDEX", canonical_name="NIPPON INDIA ETF GOLD BEES", canonical_exchange="NSE", security_type="ETF", |
| 1692 | provider_mappings=[ |
| 1693 | {"provider": "NSE", "providerSymbol": "GOLDBEES", "exchange": "NSE", "status": "VERIFIED", "resolutionSource": "NSE_VALIDATED_RESOLUTION"}, |
| 1694 | {"provider": "YAHOO_FINANCE", "providerSymbol": "GOLDBEES.NS", "exchange": "NSE", "status": "VERIFIED"}, |
| 1695 | ]) |
| 1696 | unverified = _portfolio_position("INF179KC1HS2", "HDFN50", "NSE", "HDFC NIFTY NEXT 50 ETF", asset_type="ETF", |
| 1697 | canonical_symbol="HDFN50", canonical_name="HDFC NIFTY NEXT 50 ETF", canonical_exchange="NSE", security_type="ETF", |
| 1698 | provider_mappings=[{"provider": "NSE", "providerSymbol": "HDFN50", "exchange": "NSE", "status": "INVALID"}]) |
| 1699 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), |
| 1700 | client=_RecordingPortfolioClient([trusted, unverified]), structured_provider=_UnavailableStructuredProvider()) |
| 1701 | |
| 1702 | result = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1703 | by_ticker = {company.ticker: company for company in result.companies} |
| 1704 | gold = by_ticker["GOLDEX"] |
| 1705 | assert gold.listing_provider == "NSE" |
| 1706 | assert gold.listing_symbol == "GOLDBEES" |
| 1707 | assert gold.primary_exchange == "NSE" |
| 1708 | assert gold.verified_provider_mappings == {"NSE": "GOLDBEES", "YAHOO_FINANCE": "GOLDBEES.NS"} |
| 1709 | assert gold.ticker == "GOLDEX" |
| 1710 | hdfc = by_ticker["HDFN50"] |
| 1711 | assert hdfc.listing_provider is None |
| 1712 | assert hdfc.listing_symbol == "HDFN50" |
| 1713 | assert "NSE" not in hdfc.verified_provider_mappings |
| 1714 | |
| 1715 | |
| 1716 | @pytest.mark.asyncio |
| 1717 | async def test_portfolio_research_read_does_not_resolve_from_ticker_only_unknown_exchange() -> None: |
| 1718 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=True)) |
| 1719 | position = _portfolio_position(None, "AIXA", "UNKNOWN", "AIXA", data_freshness="REAL_BROKER") |
| 1720 | position["instrument"]["instrumentId"] = "99999999-9999-9999-9999-999999999999" |
| 1721 | client = _RecordingPortfolioClient([position]) |
| 1722 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client, |
| 1723 | structured_provider=_UnavailableStructuredProvider()) |
| 1724 | |
| 1725 | result = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1726 | |
| 1727 | assert result.companies[0].status == "COMPANY_NOT_RESOLVED" |
| 1728 | |
| 1729 | |
| 1730 | @pytest.mark.asyncio |
| 1731 | async def test_portfolio_research_read_uses_enriched_ibkr_metadata_for_etf_and_equity() -> None: |
| 1732 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=True)) |
| 1733 | client = _RecordingPortfolioClient([ |
| 1734 | _portfolio_position( |
| 1735 | "IE0031442068", |
| 1736 | "IUSA", |
| 1737 | "UNKNOWN", |
| 1738 | "IUSA", |
| 1739 | provider="IBKR", |
| 1740 | provider_instrument_id="789012", |
| 1741 | asset_type="ETF", |
| 1742 | data_freshness="REAL_BROKER", |
| 1743 | broker_symbol="IUSA", |
| 1744 | broker_description="IUSA", |
| 1745 | broker_exchange="AEB", |
| 1746 | canonical_symbol="IUSA", |
| 1747 | canonical_name="iShares Core S&P 500 UCITS ETF", |
| 1748 | canonical_exchange="XAMS", |
| 1749 | canonical_mic="XAMS", |
| 1750 | security_type="ETF", |
| 1751 | ), |
| 1752 | _portfolio_position( |
| 1753 | "DE000RENK730", |
| 1754 | "R3NK", |
| 1755 | "UNKNOWN", |
| 1756 | "R3NK", |
| 1757 | provider="IBKR", |
| 1758 | provider_instrument_id="345678", |
| 1759 | asset_type="EQUITY", |
| 1760 | data_freshness="REAL_BROKER", |
| 1761 | broker_symbol="R3NK", |
| 1762 | broker_description="R3NK", |
| 1763 | broker_exchange="IBIS2", |
| 1764 | canonical_symbol="R3NK", |
| 1765 | canonical_name="RENK Group AG", |
| 1766 | canonical_exchange="XETR", |
| 1767 | canonical_mic="XETR", |
| 1768 | security_type="STK", |
| 1769 | ), |
| 1770 | ]) |
| 1771 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client) |
| 1772 | |
| 1773 | result = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1774 | |
| 1775 | by_ticker = {company.ticker: company for company in result.companies} |
| 1776 | assert by_ticker["IUSA"].company_name == "iShares Core S&P 500 UCITS ETF" |
| 1777 | assert by_ticker["IUSA"].asset_type == "ETF" |
| 1778 | assert by_ticker["IUSA"].status == "ETF_UNSUPPORTED" |
| 1779 | assert by_ticker["R3NK"].company_name == "RENK Group AG" |
| 1780 | assert by_ticker["R3NK"].asset_type == "EQUITY" |
| 1781 | assert by_ticker["R3NK"].exchange == "XETR" |
| 1782 | assert by_ticker["R3NK"].status == "RESOLVED_NO_SOURCES" |
| 1783 | |
| 1784 | |
| 1785 | @pytest.mark.asyncio |
| 1786 | @respx.mock |
| 1787 | async def test_iusa_and_nqse_route_to_etf_pipeline_and_use_canonical_name_search() -> None: |
| 1788 | class RecordingEtfSearchProvider: |
| 1789 | provider_name = "test-search" |
| 1790 | |
| 1791 | def __init__(self) -> None: |
| 1792 | self.names: list[str] = [] |
| 1793 | self.queries: list[str] = [] |
| 1794 | |
| 1795 | async def discover(self, profile, category: str, date_window: SearchDateWindow) -> list[CandidateSearchResult]: |
| 1796 | self.names.append(profile.fund_name) |
| 1797 | query = generate_search_queries(profile, category, date_window)[0] |
| 1798 | self.queries.append(query) |
| 1799 | if category != "ETF_PROFILE": |
| 1800 | return [] |
| 1801 | slug = "iusa" if profile.ticker == "IUSA" else "nqse" |
| 1802 | return [ |
| 1803 | CandidateSearchResult( |
| 1804 | title=f"{profile.fund_name} factsheet", |
| 1805 | url=f"https://www.ishares.com/{slug}/factsheet", |
| 1806 | snippet=f"{profile.fund_name} holdings expense ratio", |
| 1807 | discovered_at=datetime(2026, 5, 1, tzinfo=timezone.utc), |
| 1808 | provider=self.provider_name, |
| 1809 | query_id=f"{category}:1", |
| 1810 | query=query, |
| 1811 | category=category, |
| 1812 | ) |
| 1813 | ] |
| 1814 | |
| 1815 | provider = RecordingEtfSearchProvider() |
| 1816 | search = SearchDiscoveryService(provider, max_documents_per_refresh=4) |
| 1817 | repo = ResearchRepository( |
| 1818 | settings=Settings(research_live_enabled=True, research_search_enabled=True, research_demo_enabled=True, research_max_retries=0), |
| 1819 | search_discovery=search, |
| 1820 | ) |
| 1821 | iusa = _portfolio_position( |
| 1822 | "IE0031442068", |
| 1823 | "IUSA", |
| 1824 | "AEB", |
| 1825 | "ISHARES CORE S&P 500", |
| 1826 | provider="IBKR", |
| 1827 | provider_instrument_id="IUSA-CONID", |
| 1828 | asset_type="ETF", |
| 1829 | data_freshness="REAL_BROKER", |
| 1830 | canonical_symbol="IUSA", |
| 1831 | canonical_name="ISHARES CORE S&P 500", |
| 1832 | canonical_exchange="XAMS", |
| 1833 | canonical_mic="XAMS", |
| 1834 | security_type="ETF", |
| 1835 | ) |
| 1836 | nqse = _portfolio_position( |
| 1837 | "IE00B53SZB19", |
| 1838 | "NQSE", |
| 1839 | "IBIS2", |
| 1840 | "ISHARES NASDAQ 100 EUR-H ACC", |
| 1841 | provider="IBKR", |
| 1842 | provider_instrument_id="NQSE-CONID", |
| 1843 | asset_type="ETF", |
| 1844 | data_freshness="REAL_BROKER", |
| 1845 | canonical_symbol="NQSE", |
| 1846 | canonical_name="ISHARES NASDAQ 100 EUR-H ACC", |
| 1847 | canonical_exchange="XETR", |
| 1848 | canonical_mic="XETR", |
| 1849 | security_type="ETF", |
| 1850 | ) |
| 1851 | respx.get("https://www.ishares.com/iusa/factsheet").mock( |
| 1852 | return_value=httpx.Response( |
| 1853 | 200, |
| 1854 | headers={"content-type": "text/html"}, |
| 1855 | text="<html><title>ISHARES CORE S&P 500 factsheet</title><main><p>iShares ETF factsheet tracks the S&P 500. Expense ratio 0.07%. Holdings 503. AUM USD 85bn. Top holdings Apple Microsoft NVIDIA.</p></main></html>", |
| 1856 | ) |
| 1857 | ) |
| 1858 | respx.get("https://www.ishares.com/nqse/factsheet").mock( |
| 1859 | return_value=httpx.Response( |
| 1860 | 200, |
| 1861 | headers={"content-type": "text/html"}, |
| 1862 | text="<html><title>ISHARES NASDAQ 100 EUR-H ACC factsheet</title><main><p>iShares ETF factsheet tracks the NASDAQ 100. Total expense ratio 0.33%. Holdings 101. Accumulating. Top holdings Apple Microsoft NVIDIA.</p></main></html>", |
| 1863 | ) |
| 1864 | ) |
| 1865 | client = _RecordingPortfolioClient([iusa, nqse]) |
| 1866 | orchestrator = PortfolioResearchOrchestrator( |
| 1867 | repo, |
| 1868 | Settings(portfolio_service_base_url="http://portfolio-service", research_live_enabled=True, research_search_enabled=True), |
| 1869 | client=client, |
| 1870 | ) |
| 1871 | |
| 1872 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1873 | |
| 1874 | by_ticker = {company.ticker: company for company in result.companies} |
| 1875 | assert by_ticker["IUSA"].status == "ETF_UNSUPPORTED" |
| 1876 | assert by_ticker["IUSA"].etf_profile is not None |
| 1877 | assert by_ticker["IUSA"].etf_profile.underlying_index == "S&P 500" |
| 1878 | assert by_ticker["IUSA"].etf_profile.facts["expenseRatio"].value == "0.07%" |
| 1879 | assert by_ticker["NQSE"].status == "ETF_UNSUPPORTED" |
| 1880 | assert by_ticker["NQSE"].etf_profile is not None |
| 1881 | assert by_ticker["NQSE"].etf_profile.underlying_index == "NASDAQ 100" |
| 1882 | assert by_ticker["NQSE"].etf_profile.facts["distributionPolicy"].source_url == "https://www.ishares.com/nqse/factsheet" |
| 1883 | assert "ISHARES CORE S&P 500" in provider.names |
| 1884 | assert "ISHARES NASDAQ 100 EUR-H ACC" in provider.names |
| 1885 | assert all(not query.startswith(("IUSA ", "NQSE ")) for query in provider.queries) |
| 1886 | |
| 1887 | |
| 1888 | @pytest.mark.asyncio |
| 1889 | async def test_etf_read_does_not_call_search_or_mutate_research_db(tmp_path) -> None: |
| 1890 | class FailingSearchDiscovery: |
| 1891 | async def discover(self, company, missing, seen_urls): |
| 1892 | raise AssertionError("search provider must not be called during ETF read") |
| 1893 | |
| 1894 | class CountingPersistence(SqliteResearchPersistence): |
| 1895 | def __init__(self, database_path): |
| 1896 | super().__init__(database_path) |
| 1897 | self.write_count = 0 |
| 1898 | |
| 1899 | def upsert_document(self, document): |
| 1900 | self.write_count += 1 |
| 1901 | return super().upsert_document(document) |
| 1902 | |
| 1903 | def start_refresh_run(self, **kwargs): |
| 1904 | self.write_count += 1 |
| 1905 | return super().start_refresh_run(**kwargs) |
| 1906 | |
| 1907 | def complete_refresh_run(self, run, **kwargs): |
| 1908 | self.write_count += 1 |
| 1909 | return super().complete_refresh_run(run, **kwargs) |
| 1910 | |
| 1911 | persistence = CountingPersistence(tmp_path / "research.sqlite") |
| 1912 | repo = ResearchRepository( |
| 1913 | settings=Settings(research_live_enabled=True, research_search_enabled=True, research_demo_enabled=True), |
| 1914 | search_discovery=FailingSearchDiscovery(), |
| 1915 | persistence=persistence, |
| 1916 | ) |
| 1917 | profile = repo.register_etf_profile( |
| 1918 | repo_etf_profile( |
| 1919 | "IE0031442068", |
| 1920 | "IUSA", |
| 1921 | "XAMS", |
| 1922 | "ISHARES CORE S&P 500", |
| 1923 | "IBKR", |
| 1924 | "IUSA-CONID", |
| 1925 | ) |
| 1926 | ) |
| 1927 | repo.ingest_etf_fixture( |
| 1928 | profile, |
| 1929 | original_url="https://www.ishares.com/iusa/factsheet", |
| 1930 | source_type=SourceType.INVESTOR_RELATIONS, |
| 1931 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 1932 | source_name="iShares factsheet", |
| 1933 | publisher="iShares", |
| 1934 | content_type="text/html", |
| 1935 | body="<html><title>ISHARES CORE S&P 500 factsheet</title><main><p>iShares ETF factsheet tracks the S&P 500. Expense ratio 0.07%. Holdings 503.</p></main></html>", |
| 1936 | reliability=ReliabilityLevel.LEVEL_B, |
| 1937 | ) |
| 1938 | persistence.write_count = 0 |
| 1939 | client = _RecordingPortfolioClient([ |
| 1940 | _portfolio_position( |
| 1941 | "IE0031442068", |
| 1942 | "IUSA", |
| 1943 | "AEB", |
| 1944 | "ISHARES CORE S&P 500", |
| 1945 | provider="IBKR", |
| 1946 | provider_instrument_id="IUSA-CONID", |
| 1947 | asset_type="ETF", |
| 1948 | data_freshness="REAL_BROKER", |
| 1949 | canonical_symbol="IUSA", |
| 1950 | canonical_name="ISHARES CORE S&P 500", |
| 1951 | canonical_exchange="XAMS", |
| 1952 | canonical_mic="XAMS", |
| 1953 | security_type="ETF", |
| 1954 | ) |
| 1955 | ]) |
| 1956 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client) |
| 1957 | |
| 1958 | result = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 1959 | |
| 1960 | assert result.companies[0].status == "ETF_UNSUPPORTED" |
| 1961 | assert persistence.write_count == 0 |
| 1962 | |
| 1963 | |
| 1964 | @pytest.mark.asyncio |
| 1965 | async def test_portfolio_research_refresh_uses_canonical_company_name_for_search() -> None: |
| 1966 | class SearchStats: |
| 1967 | documents_fetched = 0 |
| 1968 | events_extracted = 0 |
| 1969 | |
| 1970 | def reject(self, reason: str) -> None: |
| 1971 | pass |
| 1972 | |
| 1973 | class RecordingSearch: |
| 1974 | def __init__(self) -> None: |
| 1975 | self.last_stats = SearchStats() |
| 1976 | self.company_names: list[str] = [] |
| 1977 | |
| 1978 | async def discover(self, profile, missing, seen_urls): |
| 1979 | self.company_names.append(profile.company_name) |
| 1980 | return [] |
| 1981 | |
| 1982 | search = RecordingSearch() |
| 1983 | repo = ResearchRepository( |
| 1984 | settings=Settings(research_live_enabled=True, research_search_enabled=True, research_demo_enabled=False), |
| 1985 | search_discovery=search, |
| 1986 | ) |
| 1987 | client = _RecordingPortfolioClient([ |
| 1988 | _portfolio_position( |
| 1989 | "DE000RENK730", |
| 1990 | "R3NK", |
| 1991 | "UNKNOWN", |
| 1992 | "R3NK", |
| 1993 | provider="IBKR", |
| 1994 | provider_instrument_id="345678", |
| 1995 | asset_type="EQUITY", |
| 1996 | data_freshness="REAL_BROKER", |
| 1997 | canonical_symbol="R3NK", |
| 1998 | canonical_name="RENK Group AG", |
| 1999 | canonical_exchange="XETR", |
| 2000 | canonical_mic="XETR", |
| 2001 | security_type="STK", |
| 2002 | ) |
| 2003 | ]) |
| 2004 | orchestrator = PortfolioResearchOrchestrator( |
| 2005 | repo, |
| 2006 | Settings(portfolio_service_base_url="http://portfolio-service", research_live_enabled=True, research_search_enabled=True), |
| 2007 | client=client, |
| 2008 | ) |
| 2009 | |
| 2010 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 2011 | |
| 2012 | assert result.companies[0].ticker == "R3NK" |
| 2013 | assert "RENK Group AG" in search.company_names |
| 2014 | assert "R3NK" not in search.company_names |
| 2015 | |
| 2016 | |
| 2017 | @pytest.mark.parametrize( |
| 2018 | ("display_name", "expected"), |
| 2019 | [ |
| 2020 | ("TALAUT / NSE / TALBROS AUTOMOTIVE COMPONENTS", "TALBROS AUTOMOTIVE COMPONENTS"), |
| 2021 | ("TALAUT / NSE / TALBROS AUTOMOTIVE COMPONENTS / EXTRA", "TALBROS AUTOMOTIVE COMPONENTS"), |
| 2022 | ], |
| 2023 | ) |
| 2024 | def test_legacy_holding_display_name_uses_only_company_segment(display_name: str, expected: str) -> None: |
| 2025 | instrument = {"ticker": "TALAUT", "companyName": display_name} |
| 2026 | |
| 2027 | assert _instrument_name(instrument) == expected |
| 2028 | assert instrument["ticker"] == "TALAUT" |
| 2029 | |
| 2030 | |
| 2031 | def test_research_company_name_priority_preserves_identity_fields() -> None: |
| 2032 | instrument = { |
| 2033 | "instrumentId": "11111111-1111-1111-1111-111111111111", |
| 2034 | "provider": "HDFC_SECURITIES", |
| 2035 | "providerInstrumentId": "SYMBOL:TALAUT", |
| 2036 | "isin": None, |
| 2037 | "ticker": "TALAUT", |
| 2038 | "exchange": "NSE", |
| 2039 | "companyName": "Structured Company Limited", |
| 2040 | "canonicalName": "Trusted Resolved Company Limited", |
| 2041 | "_researchDisplayName": "TALAUT / NSE / Legacy Company Limited / EXTRA", |
| 2042 | "_researchCustomDisplayName": " User Company Limited ", |
| 2043 | } |
| 2044 | identity_before = {key: instrument[key] for key in ( |
| 2045 | "instrumentId", "provider", "providerInstrumentId", "isin", "ticker", "exchange" |
| 2046 | )} |
| 2047 | |
| 2048 | assert _instrument_name(instrument) == "User Company Limited" |
| 2049 | instrument["_researchCustomDisplayName"] = None |
| 2050 | assert _instrument_name(instrument) == "Structured Company Limited" |
| 2051 | instrument["companyName"] = "TALAUT / NSE / Legacy Company Limited / EXTRA" |
| 2052 | assert _instrument_name(instrument) == "Trusted Resolved Company Limited" |
| 2053 | instrument["canonicalName"] = None |
| 2054 | assert _instrument_name(instrument) == "Legacy Company Limited" |
| 2055 | assert {key: instrument[key] for key in identity_before} == identity_before |
| 2056 | |
| 2057 | |
| 2058 | @pytest.mark.asyncio |
| 2059 | async def test_portfolio_research_search_receives_clean_legacy_company_name() -> None: |
| 2060 | class SearchStats: |
| 2061 | documents_fetched = 0 |
| 2062 | events_extracted = 0 |
| 2063 | |
| 2064 | def reject(self, reason: str) -> None: |
| 2065 | pass |
| 2066 | |
| 2067 | class RecordingSearch: |
| 2068 | def __init__(self) -> None: |
| 2069 | self.last_stats = SearchStats() |
| 2070 | self.company_names: list[str] = [] |
| 2071 | |
| 2072 | async def discover(self, profile, missing, seen_urls): |
| 2073 | self.company_names.append(profile.company_name) |
| 2074 | return [] |
| 2075 | |
| 2076 | search = RecordingSearch() |
| 2077 | repo = ResearchRepository( |
| 2078 | settings=Settings(research_live_enabled=True, research_search_enabled=True, research_demo_enabled=False), |
| 2079 | search_discovery=search, |
| 2080 | ) |
| 2081 | position = _portfolio_position( |
| 2082 | None, |
| 2083 | "TALAUT", |
| 2084 | "NSE", |
| 2085 | "TALAUT / NSE / TALBROS AUTOMOTIVE COMPONENTS / EXTRA", |
| 2086 | provider="HDFC_SECURITIES", |
| 2087 | provider_instrument_id="SYMBOL:TALAUT", |
| 2088 | data_freshness="REAL_BROKER", |
| 2089 | ) |
| 2090 | identity_before = dict(position["instrument"]) |
| 2091 | orchestrator = PortfolioResearchOrchestrator( |
| 2092 | repo, |
| 2093 | Settings(portfolio_service_base_url="http://portfolio-service", research_live_enabled=True, research_search_enabled=True), |
| 2094 | client=_RecordingPortfolioClient([position]), |
| 2095 | ) |
| 2096 | |
| 2097 | await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 2098 | |
| 2099 | assert search.company_names == ["TALBROS AUTOMOTIVE COMPONENTS"] |
| 2100 | assert position["instrument"] == identity_before |
| 2101 | |
| 2102 | |
| 2103 | @pytest.mark.asyncio |
| 2104 | async def test_portfolio_research_read_does_not_call_search_provider_or_mutate_research_db(tmp_path) -> None: |
| 2105 | class FailingSearchDiscovery: |
| 2106 | async def discover(self, company, missing, seen_urls): |
| 2107 | raise AssertionError("search provider must not be called during read") |
| 2108 | |
| 2109 | class CountingPersistence(SqliteResearchPersistence): |
| 2110 | def __init__(self, database_path): |
| 2111 | super().__init__(database_path) |
| 2112 | self.write_count = 0 |
| 2113 | |
| 2114 | def upsert_document(self, document): |
| 2115 | self.write_count += 1 |
| 2116 | return super().upsert_document(document) |
| 2117 | |
| 2118 | def upsert_event(self, event): |
| 2119 | self.write_count += 1 |
| 2120 | return super().upsert_event(event) |
| 2121 | |
| 2122 | def start_refresh_run(self, **kwargs): |
| 2123 | self.write_count += 1 |
| 2124 | return super().start_refresh_run(**kwargs) |
| 2125 | |
| 2126 | def complete_refresh_run(self, run, **kwargs): |
| 2127 | self.write_count += 1 |
| 2128 | return super().complete_refresh_run(run, **kwargs) |
| 2129 | |
| 2130 | persistence = CountingPersistence(tmp_path / "research.sqlite") |
| 2131 | repo = ResearchRepository( |
| 2132 | settings=Settings(research_live_enabled=True, research_demo_enabled=True), |
| 2133 | search_discovery=FailingSearchDiscovery(), |
| 2134 | persistence=persistence, |
| 2135 | ) |
| 2136 | repo.ingest_fixture( |
| 2137 | original_url="https://www.besi.com/investor-relations/press-releases/details/read-only", |
| 2138 | source_type=SourceType.INVESTOR_RELATIONS, |
| 2139 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 2140 | source_name="BESI official press release", |
| 2141 | publisher="BE Semiconductor Industries", |
| 2142 | content_type="text/html", |
| 2143 | body=( |
| 2144 | "<html><title>BESI backlog</title><body>" |
| 2145 | "BE Semiconductor Industries BESI XAMS announced a new customer order and capacity expansion." |
| 2146 | "</body></html>" |
| 2147 | ), |
| 2148 | reliability=ReliabilityLevel.LEVEL_B, |
| 2149 | source_mode=SourceMode.REAL, |
| 2150 | ) |
| 2151 | persistence.write_count = 0 |
| 2152 | client = _RecordingPortfolioClient([ |
| 2153 | _portfolio_position("NL0012866412", "BESI", "XAMS", "BE Semiconductor Industries", data_freshness="REAL_BROKER") |
| 2154 | ]) |
| 2155 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client) |
| 2156 | |
| 2157 | result = await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 2158 | |
| 2159 | assert result.companies[0].status == "RESOLVED_RESEARCH_AVAILABLE" |
| 2160 | assert persistence.write_count == 0 |
| 2161 | |
| 2162 | |
| 2163 | @pytest.mark.asyncio |
| 2164 | async def test_portfolio_research_read_preserves_user_identity_headers_for_isolation() -> None: |
| 2165 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=False)) |
| 2166 | client = _RecordingPortfolioClient([]) |
| 2167 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client) |
| 2168 | |
| 2169 | await orchestrator.read_portfolio_summary( |
| 2170 | UUID("bbbbbbbb-1111-1111-1111-bbbbbbbbbbbb"), |
| 2171 | identity_headers={"X-AIP-User-Id": "user-b", "X-AIP-User-Email": "b@example.test"}, |
| 2172 | ) |
| 2173 | |
| 2174 | assert client.calls[0]["headers"]["X-AIP-User-Id"] == "user-b" |
| 2175 | assert client.calls[0]["headers"]["X-AIP-User-Email"] == "b@example.test" |
| 2176 | |
| 2177 | |
| 2178 | @pytest.mark.asyncio |
| 2179 | async def test_real_broker_portfolio_research_does_not_use_demo_fallback() -> None: |
| 2180 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=True)) |
| 2181 | repo.profiles[0].provider_instrument_ids["IBKR"] = "AIXTRON-CONID" |
| 2182 | client = _RecordingPortfolioClient([ |
| 2183 | _portfolio_position( |
| 2184 | None, |
| 2185 | "AIXA", |
| 2186 | "IBIS2", |
| 2187 | "AIXA", |
| 2188 | provider="IBKR", |
| 2189 | provider_instrument_id="AIXTRON-CONID", |
| 2190 | data_freshness="REAL_BROKER", |
| 2191 | ), |
| 2192 | ]) |
| 2193 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client, |
| 2194 | structured_provider=_UnavailableStructuredProvider()) |
| 2195 | |
| 2196 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 2197 | |
| 2198 | assert result.companies[0].status == "RESOLVED_NO_SOURCES" |
| 2199 | assert result.companies[0].freshness == "UNAVAILABLE" |
| 2200 | assert result.companies[0].mode == "UNAVAILABLE" |
| 2201 | assert result.companies[0].document_count == 0 |
| 2202 | assert result.companies[0].event_count == 0 |
| 2203 | |
| 2204 | |
| 2205 | @pytest.mark.asyncio |
| 2206 | async def test_portfolio_research_registers_canonical_company_from_resolved_real_equity_metadata() -> None: |
| 2207 | repo = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=True)) |
| 2208 | client = _RecordingPortfolioClient([ |
| 2209 | _portfolio_position( |
| 2210 | "NL0006237562", |
| 2211 | "ARCAD", |
| 2212 | "XAMS", |
| 2213 | "Arcadis NV", |
| 2214 | provider="IBKR", |
| 2215 | provider_instrument_id="ARCAD-CONID", |
| 2216 | data_freshness="REAL_BROKER", |
| 2217 | ), |
| 2218 | _portfolio_position( |
| 2219 | "NL0000852564", |
| 2220 | "AALB", |
| 2221 | "AEB", |
| 2222 | "Aalberts N.V.", |
| 2223 | provider="IBKR", |
| 2224 | provider_instrument_id="AALB-CONID", |
| 2225 | data_freshness="REAL_BROKER", |
| 2226 | ), |
| 2227 | ]) |
| 2228 | orchestrator = PortfolioResearchOrchestrator( |
| 2229 | repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client, |
| 2230 | structured_provider=_UnavailableStructuredProvider(), |
| 2231 | ) |
| 2232 | |
| 2233 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 2234 | |
| 2235 | by_ticker = {company.ticker: company for company in result.companies} |
| 2236 | assert by_ticker["ARCAD"].status == "RESOLVED_NO_SOURCES" |
| 2237 | assert by_ticker["ARCAD"].company_name == "Arcadis NV" |
| 2238 | assert by_ticker["ARCAD"].provider_instrument_id == "ARCAD-CONID" |
| 2239 | assert by_ticker["AALB"].status == "RESOLVED_NO_SOURCES" |
| 2240 | assert by_ticker["AALB"].exchange == "XAMS" |
| 2241 | assert by_ticker["AALB"].company_name == "Aalberts N.V." |
| 2242 | assert {profile.ticker for profile in repo.list_profiles()} >= {"ARCAD", "AALB"} |
| 2243 | |
| 2244 | |
| 2245 | @pytest.mark.asyncio |
| 2246 | @respx.mock |
| 2247 | async def test_portfolio_research_dynamically_discovers_sources_for_unregistered_equities() -> None: |
| 2248 | class DynamicOfficialSearchProvider: |
| 2249 | provider_name = "test-search" |
| 2250 | |
| 2251 | def __init__(self) -> None: |
| 2252 | self.company_names: list[str] = [] |
| 2253 | |
| 2254 | async def discover(self, company, category: str, date_window: SearchDateWindow) -> list[CandidateSearchResult]: |
| 2255 | self.company_names.append(company.company_name) |
| 2256 | if category != "FINANCIAL_RESULTS": |
| 2257 | return [] |
| 2258 | domain = _company_fixture_domain(company.company_name) |
| 2259 | return [ |
| 2260 | CandidateSearchResult( |
| 2261 | title=f"{company.company_name} investor relations quarterly results", |
| 2262 | url=f"https://www.{domain}.com/investor-relations/results", |
| 2263 | snippet=f"{company.company_name} annual report, quarterly results and earnings", |
| 2264 | discovered_at=datetime(2026, 5, 1, tzinfo=timezone.utc), |
| 2265 | provider=self.provider_name, |
| 2266 | query_id=f"{category}:1", |
| 2267 | query=f"{company.company_name} annual report", |
| 2268 | category=category, |
| 2269 | ) |
| 2270 | ] |
| 2271 | |
| 2272 | provider = DynamicOfficialSearchProvider() |
| 2273 | search = SearchDiscoveryService(provider, max_documents_per_refresh=8) |
| 2274 | repo = ResearchRepository( |
| 2275 | settings=Settings(research_live_enabled=True, research_search_enabled=True, research_demo_enabled=True, research_max_retries=0), |
| 2276 | discovery=_StaticDiscovery([]), |
| 2277 | search_discovery=search, |
| 2278 | ) |
| 2279 | positions = [ |
| 2280 | _portfolio_position( |
| 2281 | "DE000RENK730", |
| 2282 | "R3NK", |
| 2283 | "UNKNOWN", |
| 2284 | "R3NK", |
| 2285 | provider="IBKR", |
| 2286 | provider_instrument_id="RENK-CONID", |
| 2287 | asset_type="EQUITY", |
| 2288 | data_freshness="REAL_BROKER", |
| 2289 | canonical_symbol="R3NK", |
| 2290 | canonical_name="RENK Group AG", |
| 2291 | canonical_exchange="XETR", |
| 2292 | canonical_mic="XETR", |
| 2293 | security_type="STK", |
| 2294 | ), |
| 2295 | _portfolio_position( |
| 2296 | "NL0006237562", |
| 2297 | "ARCAD", |
| 2298 | "XAMS", |
| 2299 | "Arcadis NV", |
| 2300 | provider="IBKR", |
| 2301 | provider_instrument_id="ARCAD-CONID", |
| 2302 | asset_type="EQUITY", |
| 2303 | data_freshness="REAL_BROKER", |
| 2304 | ), |
| 2305 | _portfolio_position( |
| 2306 | "NL0000852564", |
| 2307 | "AALB", |
| 2308 | "AEB", |
| 2309 | "Aalberts N.V.", |
| 2310 | provider="IBKR", |
| 2311 | provider_instrument_id="AALB-CONID", |
| 2312 | asset_type="EQUITY", |
| 2313 | data_freshness="REAL_BROKER", |
| 2314 | ), |
| 2315 | _portfolio_position( |
| 2316 | None, |
| 2317 | "SYNTH", |
| 2318 | "XAMS", |
| 2319 | "SynthAlpha Technologies PLC", |
| 2320 | provider="IBKR", |
| 2321 | provider_instrument_id="SYNTH-CONID", |
| 2322 | asset_type="EQUITY", |
| 2323 | data_freshness="REAL_BROKER", |
| 2324 | ), |
| 2325 | ] |
| 2326 | for position in positions: |
| 2327 | instrument = position["instrument"] |
| 2328 | company_name = _instrument_fixture_name(instrument) |
| 2329 | ticker = instrument.get("canonicalSymbol") or instrument.get("ticker") |
| 2330 | exchange = instrument.get("canonicalExchange") or instrument.get("exchange") |
| 2331 | domain = _company_fixture_domain(company_name) |
| 2332 | respx.get(f"https://www.{domain}.com/investor-relations/results").mock( |
| 2333 | return_value=httpx.Response( |
| 2334 | 200, |
| 2335 | headers={"content-type": "text/html"}, |
| 2336 | text=( |
| 2337 | f"<html><title>{company_name} quarterly results</title><main>" |
| 2338 | f"<p>May 2, 2026</p><p>{company_name} {ticker} {exchange} reported quarterly results " |
| 2339 | "with earnings, order intake growth and guidance for the full year.</p></main></html>" |
| 2340 | ), |
| 2341 | ) |
| 2342 | ) |
| 2343 | client = _RecordingPortfolioClient(positions) |
| 2344 | orchestrator = PortfolioResearchOrchestrator( |
| 2345 | repo, |
| 2346 | Settings(portfolio_service_base_url="http://portfolio-service", research_live_enabled=True, research_search_enabled=True), |
| 2347 | client=client, |
| 2348 | ) |
| 2349 | |
| 2350 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 2351 | |
| 2352 | by_ticker = {company.ticker: company for company in result.companies} |
| 2353 | for ticker in ["R3NK", "ARCAD", "AALB", "SYNTH"]: |
| 2354 | company = by_ticker[ticker] |
| 2355 | assert company.status == "RESOLVED_RESEARCH_AVAILABLE" |
| 2356 | assert company.document_count > 0 |
| 2357 | assert company.source_count > 0 |
| 2358 | assert company.freshness == "REAL" |
| 2359 | assert company.mode == "REAL" |
| 2360 | assert company.safe_error_code is None |
| 2361 | assert search.last_stats.documents_fetched > 0 |
| 2362 | assert {profile.ticker for profile in repo.list_profiles()} >= {"R3NK", "ARCAD", "AALB", "SYNTH"} |
| 2363 | assert all(document.source_classification == SourceClassification.OFFICIAL_COMPANY for document in repo.documents.values() if document.source_mode == SourceMode.REAL) |
| 2364 | assert "RENK Group AG" in provider.company_names |
| 2365 | assert "SynthAlpha Technologies PLC" in provider.company_names |
| 2366 | |
| 2367 | |
| 2368 | @pytest.mark.asyncio |
| 2369 | async def test_google_provider_unavailable_maps_to_portfolio_research_provider_unavailable() -> None: |
| 2370 | class GoogleUnavailableRepository(ResearchRepository): |
| 2371 | async def refresh(self, instrument_id: UUID, correlation_id: str | None = None, **kwargs): |
| 2372 | self.last_live_error[instrument_id] = "GOOGLE_PROVIDER_UNAVAILABLE" |
| 2373 | return self.summary(instrument_id, allow_demo=False) |
| 2374 | |
| 2375 | repo = GoogleUnavailableRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=True)) |
| 2376 | repo.profiles[0].provider_instrument_ids["IBKR"] = "AIXTRON-CONID" |
| 2377 | client = _RecordingPortfolioClient([ |
| 2378 | _portfolio_position( |
| 2379 | None, |
| 2380 | "AIXA", |
| 2381 | "IBIS2", |
| 2382 | "AIXA", |
| 2383 | provider="IBKR", |
| 2384 | provider_instrument_id="AIXTRON-CONID", |
| 2385 | data_freshness="REAL_BROKER", |
| 2386 | ), |
| 2387 | ]) |
| 2388 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client, |
| 2389 | structured_provider=_UnavailableStructuredProvider()) |
| 2390 | |
| 2391 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 2392 | |
| 2393 | assert result.companies[0].status == "SEARCH_PROVIDER_UNAVAILABLE" |
| 2394 | assert result.companies[0].safe_error_code == "GOOGLE_PROVIDER_UNAVAILABLE" |
| 2395 | assert result.companies[0].mode == "SOURCE_UNAVAILABLE" |
| 2396 | |
| 2397 | |
| 2398 | @pytest.mark.asyncio |
| 2399 | async def test_real_broker_portfolio_research_never_uses_demo_fallback_when_real_search_fails() -> None: |
| 2400 | class SearchRejectedRepository(ResearchRepository): |
| 2401 | async def refresh(self, instrument_id: UUID, correlation_id: str | None = None, **kwargs): |
| 2402 | self.last_live_error[instrument_id] = "SEARCH_SOURCE_UNAVAILABLE:source:COMPANY_RELEVANCE_FAILED" |
| 2403 | return self.summary(instrument_id, allow_demo=False) |
| 2404 | |
| 2405 | repo = SearchRejectedRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=True)) |
| 2406 | repo.profiles[1].provider_instrument_ids["IBKR"] = "BESI-CONID" |
| 2407 | client = _RecordingPortfolioClient([ |
| 2408 | _portfolio_position( |
| 2409 | None, |
| 2410 | "BESI", |
| 2411 | "AEB", |
| 2412 | "BE Semiconductor Industries N.V.", |
| 2413 | provider="IBKR", |
| 2414 | provider_instrument_id="BESI-CONID", |
| 2415 | data_freshness="REAL_BROKER", |
| 2416 | ), |
| 2417 | ]) |
| 2418 | orchestrator = PortfolioResearchOrchestrator( |
| 2419 | repo, |
| 2420 | Settings( |
| 2421 | portfolio_service_base_url="http://portfolio-service", |
| 2422 | research_live_enabled=True, |
| 2423 | research_search_enabled=True, |
| 2424 | ), |
| 2425 | client=client, |
| 2426 | structured_provider=_UnavailableStructuredProvider(), |
| 2427 | ) |
| 2428 | |
| 2429 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa")) |
| 2430 | |
| 2431 | company = result.companies[0] |
| 2432 | assert company.status == "DOCUMENT_FETCH_FAILED" |
| 2433 | assert company.mode == "SOURCE_UNAVAILABLE" |
| 2434 | assert company.freshness == "SOURCE_UNAVAILABLE" |
| 2435 | assert company.safe_error_code == "SEARCH_SOURCE_UNAVAILABLE:source:COMPANY_RELEVANCE_FAILED" |
| 2436 | assert company.document_count == 0 |
| 2437 | assert company.event_count == 0 |
| 2438 | |
| 2439 | |
| 2440 | @pytest.mark.asyncio |
| 2441 | async def test_portfolio_research_partial_failure_does_not_fail_entire_response() -> None: |
| 2442 | class FailingRefreshRepository(ResearchRepository): |
| 2443 | async def refresh(self, instrument_id: UUID, correlation_id: str | None = None, **kwargs): |
| 2444 | raise FetchError("controlled failure") |
| 2445 | |
| 2446 | repo = FailingRefreshRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=False)) |
| 2447 | client = _RecordingPortfolioClient([_portfolio_position("DE000A0WMPJ6", "AIXA", "XETR", "AIXTRON SE")]) |
| 2448 | orchestrator = PortfolioResearchOrchestrator(repo, Settings(portfolio_service_base_url="http://portfolio-service"), client=client, |
| 2449 | structured_provider=_UnavailableStructuredProvider()) |
| 2450 | |
| 2451 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-aaaaaaaaaaaa"), correlation_id="phase4") |
| 2452 | |
| 2453 | assert result.companies_degraded == 1 |
| 2454 | assert result.companies[0].status == "SEARCH_PROVIDER_UNAVAILABLE" |
| 2455 | assert result.companies[0].safe_error_code == "FetchError" |
| 2456 | |
| 2457 | |
| 2458 | def test_positive_and_negative_sources_show_mixed_evidence() -> None: |
| 2459 | repo = ResearchRepository() |
| 2460 | repo.ingest_fixture( |
| 2461 | original_url="https://www.aixtron.com/en/customer-win", |
| 2462 | source_type=SourceType.INVESTOR_RELATIONS, |
| 2463 | source_classification=SourceClassification.OFFICIAL_COMPANY, |
| 2464 | source_name="AIXTRON", |
| 2465 | publisher="AIXTRON SE", |
| 2466 | content_type="text/html", |
| 2467 | body="<html><title>AIXTRON customer</title><body>AIXTRON SE AIXA XETR announced a customer win with Infineon Technologies AG.</body></html>", |
| 2468 | reliability=ReliabilityLevel.LEVEL_B, |
| 2469 | source_mode=SourceMode.REAL, |
| 2470 | ) |
| 2471 | repo.ingest_fixture( |
| 2472 | original_url="https://www.reuters.com/markets/aixtron-customer-loss", |
| 2473 | source_type=SourceType.NEWS, |
| 2474 | source_classification=SourceClassification.REPUTABLE_NEWS, |
| 2475 | source_name="Reuters", |
| 2476 | publisher="Reuters", |
| 2477 | content_type="text/html", |
| 2478 | body="<html><title>AIXTRON customer loss</title><body>AIXTRON SE AIXA XETR reported a customer loss in silicon-carbide tools.</body></html>", |
| 2479 | reliability=ReliabilityLevel.LEVEL_C, |
| 2480 | source_mode=SourceMode.REAL, |
| 2481 | ) |
| 2482 | |
| 2483 | evidence = repo.summary(AIXTRON_INSTRUMENT_ID).catalyst_score.category_evidence["Customers"] |
| 2484 | |
| 2485 | assert evidence.status == "MIXED_EVIDENCE" |
| 2486 | assert evidence.has_conflict is True |
| 2487 | |
| 2488 | |
| 2489 | @pytest.mark.asyncio |
| 2490 | @respx.mock |
| 2491 | async def test_live_aixtron_refresh_fetches_registered_official_source_and_preserves_provenance() -> None: |
| 2492 | settings = Settings(research_live_enabled=True, research_demo_enabled=True, research_max_retries=0) |
| 2493 | repo = ResearchRepository(settings=settings) |
| 2494 | source = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 2495 | respx.get(source.url).mock( |
| 2496 | return_value=httpx.Response( |
| 2497 | 200, |
| 2498 | headers={"content-type": "text/html"}, |
| 2499 | text=_aixtron_live_fixture(), |
| 2500 | ) |
| 2501 | ) |
| 2502 | |
| 2503 | summary = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 2504 | |
| 2505 | assert summary.demo is False |
| 2506 | assert summary.data_freshness == "REAL" |
| 2507 | assert summary.documents[0].source_mode == SourceMode.REAL |
| 2508 | assert summary.documents[0].source_name == "AIXTRON official press release" |
| 2509 | assert summary.documents[0].published_at == datetime(2026, 4, 14, tzinfo=timezone.utc) |
| 2510 | assert summary.documents[0].content_hash |
| 2511 | assert any(event.source_mode == SourceMode.REAL and event.reliability == ReliabilityLevel.LEVEL_B for event in summary.recent_events) |
| 2512 | assert any(event.event_type == "NEW_ORDER" and event.confidence > 0.8 for event in summary.recent_events) |
| 2513 | |
| 2514 | |
| 2515 | @pytest.mark.asyncio |
| 2516 | @respx.mock |
| 2517 | async def test_live_refresh_is_idempotent_for_documents_events_and_scores() -> None: |
| 2518 | settings = Settings(research_live_enabled=True, research_demo_enabled=True, research_max_retries=0) |
| 2519 | repo = ResearchRepository(settings=settings) |
| 2520 | source = registered_sources_for(AIXTRON_INSTRUMENT_ID)[0] |
| 2521 | respx.get(source.url).mock( |
| 2522 | return_value=httpx.Response(200, headers={"content-type": "text/html"}, text=_aixtron_live_fixture()) |
| 2523 | ) |
| 2524 | |
| 2525 | first = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 2526 | second = await repo.refresh(AIXTRON_INSTRUMENT_ID) |
| 2527 | |
| 2528 | assert len(repo.documents_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL)) == len(first.documents) == len(second.documents) |
| 2529 | assert len(repo.events_for(AIXTRON_INSTRUMENT_ID, source_mode=SourceMode.REAL)) == len(first.recent_events) == len(second.recent_events) |
| 2530 | assert first.catalyst_score.overall_score == second.catalyst_score.overall_score |
| 2531 | |
| 2532 | |
| 2533 | def test_registered_source_rejects_unapproved_host() -> None: |
| 2534 | repo = ResearchRepository() |
| 2535 | profile = repo.profile(AIXTRON_INSTRUMENT_ID) |
| 2536 | source = RegisteredResearchSource( |
| 2537 | source_id="bad", |
| 2538 | instrument_id=AIXTRON_INSTRUMENT_ID, |
| 2539 | url="https://example.com/aixtron", |
| 2540 | source_type=SourceType.INVESTOR_RELATIONS, |
| 2541 | source_name="Bad", |
| 2542 | publisher="Bad", |
| 2543 | reliability_level=ReliabilityLevel.LEVEL_C, |
| 2544 | ) |
| 2545 | with pytest.raises(FetchError): |
| 2546 | repo._validate_registered_source(profile, source) |
| 2547 | |
| 2548 | |
| 2549 | @pytest.mark.asyncio |
| 2550 | @respx.mock |
| 2551 | async def test_http_fetch_validates_content_type_retries_and_size() -> None: |
| 2552 | settings = Settings(research_max_content_bytes=50, research_max_retries=1) |
| 2553 | fetcher = HttpResearchFetcher(settings) |
| 2554 | route = respx.get("https://example.com/release").mock( |
| 2555 | side_effect=[ |
| 2556 | httpx.Response(429, headers={"Retry-After": "0"}), |
| 2557 | httpx.Response(200, headers={"content-type": "text/html"}, text="<html>ok</html>"), |
| 2558 | ] |
| 2559 | ) |
| 2560 | result = await fetcher.fetch("https://example.com/release") |
| 2561 | assert route.call_count == 2 |
| 2562 | assert result.content_type == "text/html" |
| 2563 | |
| 2564 | |
| 2565 | @pytest.mark.asyncio |
| 2566 | @respx.mock |
| 2567 | async def test_http_fetch_timeout_raises_structured_error() -> None: |
| 2568 | settings = Settings(research_max_retries=0) |
| 2569 | fetcher = HttpResearchFetcher(settings) |
| 2570 | respx.get("https://example.com/timeout").mock(side_effect=httpx.TimeoutException("timeout")) |
| 2571 | with pytest.raises(FetchError, match="timed out"): |
| 2572 | await fetcher.fetch("https://example.com/timeout") |
| 2573 | |
| 2574 | |
| 2575 | @pytest.mark.asyncio |
| 2576 | @respx.mock |
| 2577 | async def test_http_fetch_404_raises_structured_error() -> None: |
| 2578 | fetcher = HttpResearchFetcher(Settings(research_max_retries=0)) |
| 2579 | respx.get("https://example.com/missing").mock(return_value=httpx.Response(404, headers={"content-type": "text/html"})) |
| 2580 | with pytest.raises(FetchError, match="404"): |
| 2581 | await fetcher.fetch("https://example.com/missing") |
| 2582 | |
| 2583 | |
| 2584 | @pytest.mark.asyncio |
| 2585 | @respx.mock |
| 2586 | async def test_http_fetch_rejects_unsupported_content_type() -> None: |
| 2587 | fetcher = HttpResearchFetcher(Settings()) |
| 2588 | respx.get("https://example.com/data").mock(return_value=httpx.Response(200, headers={"content-type": "application/json"}, json={"ok": True})) |
| 2589 | with pytest.raises(FetchError, match="Unsupported content type"): |
| 2590 | await fetcher.fetch("https://example.com/data") |
| 2591 | |
| 2592 | |
| 2593 | @pytest.mark.asyncio |
| 2594 | @respx.mock |
| 2595 | async def test_http_fetch_rejects_oversize_response() -> None: |
| 2596 | fetcher = HttpResearchFetcher(Settings(research_max_content_bytes=8)) |
| 2597 | respx.get("https://example.com/large").mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text="0123456789")) |
| 2598 | with pytest.raises(FetchError, match="Maximum content size exceeded"): |
| 2599 | await fetcher.fetch("https://example.com/large") |
| 2600 | |
| 2601 | |
| 2602 | @pytest.mark.asyncio |
| 2603 | @respx.mock |
| 2604 | async def test_http_fetch_does_not_retry_restricted_sources() -> None: |
| 2605 | settings = Settings(research_max_retries=2) |
| 2606 | fetcher = HttpResearchFetcher(settings) |
| 2607 | route = respx.get("https://example.com/private").mock(return_value=httpx.Response(403)) |
| 2608 | with pytest.raises(RestrictedFetchError): |
| 2609 | await fetcher.fetch("https://example.com/private") |
| 2610 | assert route.call_count == 1 |
| 2611 | |
| 2612 | |
| 2613 | @pytest.mark.asyncio |
| 2614 | @respx.mock |
| 2615 | async def test_http_fetch_follows_safe_public_redirect() -> None: |
| 2616 | fetcher = HttpResearchFetcher(Settings(research_max_retries=0)) |
| 2617 | start = "https://nsearchives.nseindia.com/corporate/start.pdf" |
| 2618 | final = "https://nsearchives.nseindia.com/corporate/final.html" |
| 2619 | respx.get(start).mock(return_value=httpx.Response(302, headers={"location": final})) |
| 2620 | respx.get(final).mock(return_value=httpx.Response(200, headers={"content-type": "text/html"}, text="<html>financial results</html>")) |
| 2621 | |
| 2622 | result = await fetcher.fetch(start) |
| 2623 | |
| 2624 | assert result.final_url == final |
| 2625 | |
| 2626 | |
| 2627 | @pytest.mark.asyncio |
| 2628 | @respx.mock |
| 2629 | async def test_http_fetch_rejects_redirect_to_private_target() -> None: |
| 2630 | fetcher = HttpResearchFetcher(Settings(research_max_retries=0)) |
| 2631 | start = "https://nsearchives.nseindia.com/corporate/start.pdf" |
| 2632 | respx.get(start).mock(return_value=httpx.Response(302, headers={"location": "http://127.0.0.1/private.pdf"})) |
| 2633 | |
| 2634 | with pytest.raises(UnsafeUrlError): |
| 2635 | await fetcher.fetch(start) |
| 2636 | |
| 2637 | |
| 2638 | def test_india_equity_queries_are_exchange_first_and_remain_publicly_broad() -> None: |
| 2639 | profiles = [ |
| 2640 | next(profile for profile in ResearchRepository().profiles if profile.ticker == "RELIANCE"), |
| 2641 | CompanyResearchProfile( |
| 2642 | instrument_id=UUID("77777777-7777-7777-7777-777777777777"), |
| 2643 | company_id=UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa7"), |
| 2644 | company_name="Bharat Electronics Limited", |
| 2645 | aliases=["BEL"], isin="INE263A01024", ticker="BEL", exchange="XNSE", mic="XNSE", |
| 2646 | country="IN", currency="INR", known_domains=["bel-india.in"], |
| 2647 | ), |
| 2648 | ] |
| 2649 | for profile in profiles: |
| 2650 | queries = generate_search_queries(profile, "FINANCIAL_RESULTS", SearchDateWindow()) |
| 2651 | assert "site:nseindia.com" in queries[0] |
| 2652 | assert "site:bseindia.com" in queries[1] |
| 2653 | assert any(profile.isin in query for query in queries) |
| 2654 | assert any(profile.company_name in query and "site:" not in query for query in queries[:6]) |
| 2655 | |
| 2656 | |
| 2657 | def test_latest_quarter_result_extracts_only_verified_metrics_with_provenance() -> None: |
| 2658 | document = _document( |
| 2659 | "https://www.bseindia.com/xml-data/corpfiling/q1.pdf", |
| 2660 | "Q1 FY27 revenue 1,250 revenue YoY +18% EBITDA 240 EBITDA margin 19.2% PAT 125 PAT YoY +24% EPS 3.2", |
| 2661 | ).model_copy(update={"source_classification": SourceClassification.EXCHANGE, "source_name": "BSE"}) |
| 2662 | result = latest_quarterly_result([document]) |
| 2663 | assert result is not None |
| 2664 | assert result.period == "Q1 FY27" |
| 2665 | assert result.revenue.value == Decimal("1250") |
| 2666 | assert result.pat_yoy_percent.value == Decimal("24") |
| 2667 | assert result.revenue.source_url == document.canonical_url |
| 2668 | assert result.revenue_qoq_percent is None |
| 2669 | assert result.document_title == document.title |
| 2670 | assert result.yoy_summary == "Revenue 18% YoY; PAT 24% YoY" |
| 2671 | |
| 2672 | |
| 2673 | def test_latest_quarter_result_selects_latest_fiscal_period_not_latest_retrieval() -> None: |
| 2674 | older_period = _document("https://www.nseindia.com/q4.pdf", "Q4 FY26 revenue 900 PAT 90 EPS 2") |
| 2675 | latest_period = _document("https://www.nseindia.com/q1.pdf", "Q1 FY27 revenue 1,000 PAT 110 EPS 2.5") |
| 2676 | older_period = older_period.model_copy(update={"retrieved_at": datetime(2026, 8, 1, tzinfo=timezone.utc)}) |
| 2677 | latest_period = latest_period.model_copy(update={"retrieved_at": datetime(2026, 7, 1, tzinfo=timezone.utc)}) |
| 2678 | result = latest_quarterly_result([older_period, latest_period]) |
| 2679 | assert result is not None |
| 2680 | assert result.period == "Q1 FY27" |
| 2681 | assert result.revenue.value == Decimal("1000") |
| 2682 | |
| 2683 | |
| 2684 | def test_latest_quarter_prefers_consolidated_for_same_fiscal_period_and_preserves_unit() -> None: |
| 2685 | standalone = _document("https://www.nseindia.com/standalone.pdf", "Q1 FY27 standalone revenue 120 crore PAT 12 EPS 1") |
| 2686 | consolidated = _document("https://www.nseindia.com/consolidated.pdf", "Q1 FY27 consolidated revenue 150 crore PAT 15 EPS 1.2") |
| 2687 | result = latest_quarterly_result([standalone, consolidated]) |
| 2688 | assert result is not None |
| 2689 | assert result.reporting_basis == "CONSOLIDATED" |
| 2690 | assert result.revenue.value == Decimal("150") |
| 2691 | assert result.revenue.unit == "crore" |
| 2692 | |
| 2693 | |
| 2694 | def test_official_pdf_filing_candidate_ranks_above_ir_landing_page() -> None: |
| 2695 | profile = next(profile for profile in ResearchRepository().profiles if profile.ticker == "RELIANCE") |
| 2696 | landing = CandidateSearchResult("Investor relations", "https://www.ril.com/investors", "Investor relations home", datetime.now(timezone.utc), "searxng", "1", "q", "FINANCIAL_RESULTS") |
| 2697 | filing = CandidateSearchResult("Quarterly Financial Results Q1 FY27", "https://www.nseindia.com/results/q1.pdf", "Unaudited financial results", datetime.now(timezone.utc), "searxng", "2", "q", "FINANCIAL_RESULTS") |
| 2698 | assert _candidate_rank(profile, filing) > _candidate_rank(profile, landing) |
| 2699 | |
| 2700 | |
| 2701 | @pytest.mark.parametrize(("metadata", "expected"), [ |
| 2702 | (("Investor Presentation Q1", "", "q1-investor-presentation.pdf"), DocumentSubtype.INVESTOR_PRESENTATION), |
| 2703 | (("AGM Presentation", "", "agm-presentation.pdf"), None), |
| 2704 | (("Presentation", "", "presentation.pdf"), None), |
| 2705 | (("Press Release", "Business update", "release.pdf"), DocumentSubtype.INVESTOR_RELEASE), |
| 2706 | (("Compliance notice", "", "notice.pdf"), None), |
| 2707 | (("", "Conference Call Transcript", "transcript.pdf"), DocumentSubtype.CONFERENCE_CALL_MATERIAL), |
| 2708 | (("", "Earnings Call Presentation", "concall.pdf"), DocumentSubtype.CONFERENCE_CALL_MATERIAL), |
| 2709 | (("Conference Call Invitation", "", "invite.pdf"), None), |
| 2710 | (("Receipt of Order", "", "order.pdf"), DocumentSubtype.ORDER_CONTRACT_DISCLOSURE), |
| 2711 | (("Letter of Award", "", "loa.pdf"), DocumentSubtype.ORDER_CONTRACT_DISCLOSURE), |
| 2712 | (("Order update", "", "order.pdf"), None), |
| 2713 | (("Award received", "", "award.pdf"), None), |
| 2714 | (("Capacity expansion", "", "expansion.pdf"), DocumentSubtype.CAPEX_CAPACITY_DISCLOSURE), |
| 2715 | (("New manufacturing facility", "", "facility.pdf"), DocumentSubtype.CAPEX_CAPACITY_DISCLOSURE), |
| 2716 | (("Installed capacity", "", "results.pdf"), None), |
| 2717 | (("", "", "ordinary-attachment.pdf"), None), |
| 2718 | ]) |
| 2719 | def test_nse_document_subtype_classifier_is_conservative(metadata, expected) -> None: |
| 2720 | desc, attachment_text, attachment_file = metadata |
| 2721 | assert classify_nse_document_subtype( |
| 2722 | desc=desc, attachment_text=attachment_text, attachment_file=attachment_file, |
| 2723 | ) == expected |
| 2724 | |
| 2725 | |
| 2726 | def test_nse_document_subtype_classifier_has_single_overlap_precedence() -> None: |
| 2727 | assert classify_nse_document_subtype( |
| 2728 | desc="Investor Presentation and conference call transcript", |
| 2729 | attachment_text="", |
| 2730 | attachment_file="q1.pdf", |
| 2731 | ) == DocumentSubtype.CONFERENCE_CALL_MATERIAL |
| 2732 | |
| 2733 | |
| 2734 | @pytest.mark.asyncio |
| 2735 | async def test_nse_official_discovery_keeps_required_filings_and_confident_high_value_metadata() -> None: |
| 2736 | profile = CompanyResearchProfile( |
| 2737 | instrument_id=UUID("99999999-9999-9999-9999-999999999999"), company_id=UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), |
| 2738 | company_name="Example Components Limited", isin="INE000A01010", ticker="EXAMPLE", exchange="NSE", mic="XNSE", |
| 2739 | country="IN", currency="INR", provider_instrument_ids={"NSE": "EXAMPLE"}, |
| 2740 | ) |
| 2741 | rows = [ |
| 2742 | {"an_dt": "10-Aug-2026 13:52:28", "desc": "Financial results", "attchmntText": "Financial results", "attchmntFile": "https://nsearchives.nseindia.com/corporate/results.pdf"}, |
| 2743 | {"an_dt": "09-Aug-2026 13:52:28", "desc": "Shareholding pattern", "attchmntText": "", "attchmntFile": "https://nsearchives.nseindia.com/corporate/holding.pdf"}, |
| 2744 | {"an_dt": "08-Aug-2026 13:52:28", "desc": "Investor Presentation", "attchmntText": "", "attchmntFile": "https://nsearchives.nseindia.com/corporate/presentation.pdf"}, |
| 2745 | {"an_dt": "07-Aug-2026 13:52:28", "desc": "Receipt of Order", "attchmntText": "", "attchmntFile": "https://nsearchives.nseindia.com/corporate/order.pdf"}, |
| 2746 | {"an_dt": "06-Aug-2026 13:52:28", "desc": "Board meeting notice", "attchmntText": "", "attchmntFile": "https://nsearchives.nseindia.com/corporate/notice.pdf"}, |
| 2747 | ] |
| 2748 | client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=rows, request=request))) |
| 2749 | result = await OfficialFilingDiscovery(client).discover(profile, {"FINANCIAL_RESULTS", "SHAREHOLDING_PATTERN"}, set()) |
| 2750 | assert {item.category for item in result} == { |
| 2751 | "FINANCIAL_RESULTS", "SHAREHOLDING_PATTERN", "INVESTOR_PRESENTATION", "ORDER_CONTRACT_DISCLOSURE", |
| 2752 | } |
| 2753 | assert next(item for item in result if item.category == "INVESTOR_PRESENTATION").source.document_subtype == DocumentSubtype.INVESTOR_PRESENTATION |
| 2754 | assert all("notice.pdf" not in item.source.url for item in result) |
| 2755 | |
| 2756 | |
| 2757 | @pytest.mark.asyncio |
| 2758 | async def test_nse_official_discovery_uses_verified_nse_symbol_and_returns_latest_financial_pdf() -> None: |
| 2759 | profile = CompanyResearchProfile( |
| 2760 | instrument_id=UUID("99999999-9999-9999-9999-999999999999"), company_id=UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), |
| 2761 | company_name="Example Components Limited", isin="INE000A01010", ticker="BROKER_ALIAS", exchange="NSE", mic="XNSE", |
| 2762 | country="IN", currency="INR", provider_instrument_ids={"NSE": "VERIFIED_NSE_SYMBOL"}, |
| 2763 | ) |
| 2764 | rows = [ |
| 2765 | {"an_dt": "10-Aug-2026 13:52:28", "desc": "Outcome of Board Meeting", "attchmntText": "Financial results for the period ended Jun 30, 2026", "attchmntFile": "https://nsearchives.nseindia.com/corporate/latest.pdf"}, |
| 2766 | {"an_dt": "01-Aug-2026 12:00:00", "desc": "AGM notice", "attchmntText": "Annual general meeting", "attchmntFile": "https://nsearchives.nseindia.com/corporate/agm.pdf"}, |
| 2767 | ] |
| 2768 | client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=rows, request=request))) |
| 2769 | result = await OfficialFilingDiscovery(client).discover(profile, {"FINANCIAL_RESULTS"}, set()) |
| 2770 | assert len(result) == 1 |
| 2771 | assert result[0].source.url.endswith("latest.pdf") |
| 2772 | assert result[0].source.source_classification == SourceClassification.EXCHANGE |
| 2773 | assert result[0].source.discovery_method == "NSE_OFFICIAL_API" |
| 2774 | assert result[0].source.official_nse_profile_symbol == "VERIFIED_NSE_SYMBOL" |
| 2775 | |
| 2776 | |
| 2777 | @pytest.mark.asyncio |
| 2778 | async def test_trusted_nse_api_discovery_persists_expected_profile_when_generic_resolution_fails(monkeypatch) -> None: |
| 2779 | profile = CompanyResearchProfile( |
| 2780 | instrument_id=UUID("99999999-9999-9999-9999-999999999999"), company_id=UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), |
| 2781 | company_name="Example Components Limited", isin="INE000A01010", ticker="BROKER_ALIAS", exchange="NSE", mic="XNSE", |
| 2782 | country="IN", currency="INR", provider_instrument_ids={"NSE": "VERIFIED_NSE_SYMBOL"}, |
| 2783 | ) |
| 2784 | rows = [{"an_dt": "10-Aug-2026 13:52:28", "desc": "Financial results", "attchmntText": "Financial results", "attchmntFile": "https://nsearchives.nseindia.com/corporate/latest.pdf"}] |
| 2785 | client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=rows, request=request))) |
| 2786 | discovered = await OfficialFilingDiscovery(client).discover(profile, {"FINANCIAL_RESULTS"}, set()) |
| 2787 | repository = ResearchRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=False)) |
| 2788 | repository._fetcher = _OfficialFetchFixture(FetchResult( |
| 2789 | final_url=discovered[0].source.url, status_code=200, content_type="text/html", bytes_read=100, |
| 2790 | text="<html><title>Board meeting outcome</title><main>Unaudited quarterly financial results were approved by the board today.</main></html>", |
| 2791 | )) |
| 2792 | monkeypatch.setattr(repository._resolver, "resolve", lambda *_args: EntityResolution(instrument_id=None, company_id=None, confidence=0.0)) |
| 2793 | |
| 2794 | await repository._fetch_official_filings(profile, discovered, set()) |
| 2795 | |
| 2796 | document = next(document for document in repository.documents.values() if document.canonical_url == discovered[0].source.url) |
| 2797 | assert document.instrument_id == profile.instrument_id |
| 2798 | assert document.company_id == profile.company_id |
| 2799 | assert document.entity_resolution_confidence == 0.99 |
| 2800 | assert document.discovery_provider == "NSE_OFFICIAL_API" |
| 2801 | |
| 2802 | |
| 2803 | def test_expected_profile_alone_cannot_bypass_generic_relevance(monkeypatch) -> None: |
| 2804 | repository = ResearchRepository() |
| 2805 | profile = repository.profile(AIXTRON_INSTRUMENT_ID) |
| 2806 | documents_before = dict(repository.documents) |
| 2807 | events_before = dict(repository.events) |
| 2808 | event_keys_before = set(repository._event_keys) |
| 2809 | monkeypatch.setattr(repository._resolver, "resolve", lambda *_args: EntityResolution(instrument_id=None, company_id=None, confidence=0.0)) |
| 2810 | |
| 2811 | with pytest.raises(FetchError, match="COMPANY_RELEVANCE_FAILED"): |
| 2812 | repository.ingest_fixture( |
| 2813 | original_url="https://example.com/untrusted", source_type=SourceType.NEWS, source_name="untrusted", |
| 2814 | publisher="untrusted", content_type="text/html", body="<main>unrelated document with sufficient extracted text for validation.</main>", |
| 2815 | reliability=ReliabilityLevel.LEVEL_C, expected_profile=profile, |
| 2816 | ) |
| 2817 | |
| 2818 | |
| 2819 | def test_prepare_ingested_document_does_not_apply_repository_state() -> None: |
| 2820 | repository = ResearchRepository() |
| 2821 | profile = repository.profile(AIXTRON_INSTRUMENT_ID) |
| 2822 | documents_before = dict(repository.documents) |
| 2823 | events_before = dict(repository.events) |
| 2824 | event_keys_before = set(repository._event_keys) |
| 2825 | document = repository._prepare_ingested_document( |
| 2826 | original_url="https://example.com/prepared", source_type=SourceType.NEWS, source_name="fixture", |
| 2827 | publisher="fixture", content_type="text/html", |
| 2828 | body="<main>AIXTRON SE published sufficient fixture content for preparation.</main>", |
| 2829 | reliability=ReliabilityLevel.LEVEL_C, published_at=None, source_mode=SourceMode.DEMO, |
| 2830 | source_classification=SourceClassification.OTHER, discovered_at=None, discovery_provider=None, |
| 2831 | expected_profile=profile, document_status=DocumentStatus.PARSED, allow_empty_content=False, |
| 2832 | trusted_profile_identity=None, |
| 2833 | ) |
| 2834 | assert document.canonical_url == "https://example.com/prepared" |
| 2835 | assert repository.documents == documents_before |
| 2836 | assert repository.events == events_before |
| 2837 | assert repository._event_keys == event_keys_before |
| 2838 | |
| 2839 | |
| 2840 | def test_nse_looking_url_without_official_discovery_provenance_cannot_bypass_relevance(monkeypatch) -> None: |
| 2841 | repository = ResearchRepository() |
| 2842 | profile = repository.profile(AIXTRON_INSTRUMENT_ID) |
| 2843 | source = RegisteredResearchSource( |
| 2844 | source_id="untrusted-nse-lookalike", instrument_id=profile.instrument_id, |
| 2845 | url="https://nsearchives.nseindia.com/corporate/untrusted.pdf", source_type=SourceType.EXCHANGE_ANNOUNCEMENT, |
| 2846 | source_classification=SourceClassification.EXCHANGE, source_name="NSE", publisher="NSE", |
| 2847 | reliability_level=ReliabilityLevel.LEVEL_A, domain="nsearchives.nseindia.com", company_id=profile.company_id, |
| 2848 | discovery_method="NSE_OFFICIAL_API", |
| 2849 | ) |
| 2850 | monkeypatch.setattr(repository._resolver, "resolve", lambda *_args: EntityResolution(instrument_id=None, company_id=None, confidence=0.0)) |
| 2851 | |
| 2852 | with pytest.raises(FetchError, match="COMPANY_RELEVANCE_FAILED"): |
| 2853 | repository._ingest_registered_fetch_result(profile, source, FetchResult( |
| 2854 | final_url=source.url, status_code=200, content_type="text/html", bytes_read=100, |
| 2855 | text="<main>Unrelated document with sufficient extracted text for validation.</main>", |
| 2856 | ), expected_profile=profile) |
| 2857 | |
| 2858 | |
| 2859 | @pytest.mark.asyncio |
| 2860 | async def test_nse_official_discovery_preserves_valid_archive_url_and_logs_terminal_failure(caplog) -> None: |
| 2861 | profile = CompanyResearchProfile( |
| 2862 | instrument_id=UUID("99999999-9999-9999-9999-999999999999"), company_id=UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), |
| 2863 | company_name="Example Components Limited", isin="INE000A01010", ticker="EXAMPLE", exchange="NSE", mic="XNSE", |
| 2864 | country="IN", currency="INR", provider_instrument_ids={"NSE": "EXAMPLE"}, |
| 2865 | ) |
| 2866 | rows = [{"an_dt": "10-Aug-2026 13:52:28", "desc": "Financial results", "attchmntText": "Financial results", "attchmntFile": "HTTPS://nsearchives.nseindia.com/corporate//latest.pdf?b=2&a=1"}] |
| 2867 | client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=rows, request=request))) |
| 2868 | result = await OfficialFilingDiscovery(client).discover(profile, {"FINANCIAL_RESULTS"}, set()) |
| 2869 | assert result[0].source.url == "https://nsearchives.nseindia.com/corporate/latest.pdf?a=1&b=2" |
| 2870 | |
| 2871 | invalid_client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json={}, request=request))) |
| 2872 | with caplog.at_level(logging.WARNING, logger="app.source_discovery"), pytest.raises(SearchProviderError): |
| 2873 | await OfficialFilingDiscovery(invalid_client).discover(profile, {"FINANCIAL_RESULTS"}, set()) |
| 2874 | assert any("official_discovery_result" in record.message and "status=FAILED" in record.message for record in caplog.records) |
| 2875 | |
| 2876 | |
| 2877 | @pytest.mark.asyncio |
| 2878 | async def test_unsafe_or_malformed_nse_attachment_is_rejected_without_aborting_other_candidates(caplog) -> None: |
| 2879 | profile = CompanyResearchProfile( |
| 2880 | instrument_id=UUID("99999999-9999-9999-9999-999999999999"), company_id=UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), |
| 2881 | company_name="Example Components Limited", isin="INE000A01010", ticker="EXAMPLE", exchange="NSE", mic="XNSE", |
| 2882 | country="IN", currency="INR", provider_instrument_ids={"NSE": "EXAMPLE"}, |
| 2883 | ) |
| 2884 | rows = [ |
| 2885 | {"an_dt": "11-Aug-2026 13:52:28", "desc": "Financial results", "attchmntText": "Financial results", "attchmntFile": "http://127.0.0.1/private.pdf"}, |
| 2886 | {"an_dt": "10-Aug-2026 13:52:28", "desc": "Financial results", "attchmntText": "Financial results", "attchmntFile": "/relative.pdf"}, |
| 2887 | {"an_dt": "09-Aug-2026 13:52:28", "desc": "Financial results", "attchmntText": "Financial results", "attchmntFile": "https://nsearchives.nseindia.com/corporate/valid.pdf"}, |
| 2888 | ] |
| 2889 | client = httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=rows, request=request))) |
| 2890 | with caplog.at_level(logging.WARNING, logger="app.source_discovery"): |
| 2891 | result = await OfficialFilingDiscovery(client).discover(profile, {"FINANCIAL_RESULTS"}, set()) |
| 2892 | |
| 2893 | assert [item.source.url for item in result] == ["https://nsearchives.nseindia.com/corporate/valid.pdf"] |
| 2894 | assert sum("official_candidate_rejected" in record.message for record in caplog.records) == 2 |
| 2895 | |
| 2896 | |
| 2897 | def test_public_nse_archive_cdn_url_remains_permitted_by_ssrf_validation() -> None: |
| 2898 | assert validate_public_http_url("https://nsearchives.nseindia.com/corporate/results.pdf") |
| 2899 | |
| 2900 | |
| 2901 | def _official_financial_result_source(profile: CompanyResearchProfile, suffix: str) -> RegisteredResearchSource: |
| 2902 | return RegisteredResearchSource( |
| 2903 | source_id=f"nse-result-{suffix}", instrument_id=profile.instrument_id, |
| 2904 | url=f"https://nsearchives.nseindia.com/corporate/{suffix}.pdf", |
| 2905 | source_type=SourceType.EXCHANGE_ANNOUNCEMENT, source_classification=SourceClassification.EXCHANGE, |
| 2906 | source_name="NSE corporate announcements", publisher="NSE", reliability_level=ReliabilityLevel.LEVEL_A, |
| 2907 | domain="nsearchives.nseindia.com", company_id=profile.company_id, discovery_method="NSE_OFFICIAL_API", |
| 2908 | priority=1, categories=("FINANCIAL_RESULTS",), |
| 2909 | ) |
| 2910 | |
| 2911 | |
| 2912 | class _StaticOfficialDiscovery: |
| 2913 | def __init__(self, results: list[DiscoveryResult]) -> None: |
| 2914 | self.results = results |
| 2915 | self.calls = 0 |
| 2916 | |
| 2917 | async def discover(self, _profile, _categories, _seen_urls) -> list[DiscoveryResult]: |
| 2918 | self.calls += 1 |
| 2919 | return self.results |
| 2920 | |
| 2921 | |
| 2922 | class _OfficialFetchFixture: |
| 2923 | def __init__(self, response: FetchResult | Exception) -> None: |
| 2924 | self.response = response |
| 2925 | self.urls: list[str] = [] |
| 2926 | |
| 2927 | async def fetch(self, url: str) -> FetchResult: |
| 2928 | self.urls.append(url) |
| 2929 | if isinstance(self.response, Exception): |
| 2930 | raise self.response |
| 2931 | return self.response |
| 2932 | |
| 2933 | |
| 2934 | @pytest.mark.asyncio |
| 2935 | async def test_successful_official_financial_result_persists_and_is_removed_from_fallback_missing_categories() -> None: |
| 2936 | settings = Settings(research_live_enabled=True, research_search_enabled=True, research_official_document_max_attempts_per_refresh=3) |
| 2937 | repository = ResearchRepository(settings=settings, search_discovery=SearchDiscoveryService(_StaticSearchProvider([]))) |
| 2938 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 2939 | source = _official_financial_result_source(profile, "latest") |
| 2940 | repository._official_filing_discovery = _StaticOfficialDiscovery([DiscoveryResult("FINANCIAL_RESULTS", source)]) |
| 2941 | repository._fetcher = _OfficialFetchFixture(FetchResult( |
| 2942 | final_url=source.url, status_code=200, content_type="text/html", bytes_read=180, |
| 2943 | text="<html><title>Reliance Industries Limited Quarterly Financial Results</title><main>Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue from operations 1000 crore PAT 100 crore.</main></html>", |
| 2944 | )) |
| 2945 | |
| 2946 | await repository._refresh_targeted(profile, set()) |
| 2947 | |
| 2948 | documents = repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL) |
| 2949 | assert len(documents) == 1 |
| 2950 | assert documents[0].status == DocumentStatus.PROCESSED |
| 2951 | assert "FINANCIAL_RESULTS" not in repository._missing_categories(profile, set(), datetime.now(timezone.utc), {"FINANCIAL_RESULTS"}) |
| 2952 | |
| 2953 | |
| 2954 | @pytest.mark.asyncio |
| 2955 | async def test_official_timeout_uses_host_failure_budget_and_future_refresh_can_retry_without_freshness() -> None: |
| 2956 | settings = Settings( |
| 2957 | research_live_enabled=True, |
| 2958 | research_official_document_max_attempts_per_refresh=3, |
| 2959 | research_official_document_max_transport_failures_per_host=1, |
| 2960 | research_official_document_timeout_seconds=1.0, |
| 2961 | ) |
| 2962 | repository = ResearchRepository(settings=settings) |
| 2963 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 2964 | sources = [_official_financial_result_source(profile, f"newest-{index}") for index in range(5)] |
| 2965 | fetcher = _OfficialFetchFixture(TransportFetchError("Fetch timed out")) |
| 2966 | repository._fetcher = fetcher |
| 2967 | |
| 2968 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source) for source in sources], set()) |
| 2969 | assert fetcher.urls == [sources[0].url] |
| 2970 | assert not repository._category_is_fresh(profile.instrument_id, "FINANCIAL_RESULTS", datetime.now(timezone.utc)) |
| 2971 | |
| 2972 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source) for source in sources], set()) |
| 2973 | assert fetcher.urls == [sources[0].url, sources[0].url] |
| 2974 | |
| 2975 | |
| 2976 | @pytest.mark.asyncio |
| 2977 | async def test_failed_official_fetch_keeps_existing_qualifying_evidence() -> None: |
| 2978 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 2979 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 2980 | existing = repository.ingest_fixture( |
| 2981 | original_url="https://nsearchives.nseindia.com/corporate/existing-result.pdf", |
| 2982 | source_type=SourceType.EXCHANGE_ANNOUNCEMENT, source_classification=SourceClassification.EXCHANGE, |
| 2983 | source_name="NSE corporate announcements", publisher="NSE", content_type="text/html", |
| 2984 | body="<html><title>Reliance Industries Limited Financial Results</title><main>Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue 1000 crore PAT 100 crore.</main></html>", |
| 2985 | reliability=ReliabilityLevel.LEVEL_A, source_mode=SourceMode.REAL, expected_profile=profile, |
| 2986 | ) |
| 2987 | source = _official_financial_result_source(profile, "timeout") |
| 2988 | repository._fetcher = _OfficialFetchFixture(FetchError("Fetch timed out")) |
| 2989 | |
| 2990 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 2991 | |
| 2992 | assert repository.documents[existing.document_id] is existing |
| 2993 | |
| 2994 | |
| 2995 | @pytest.mark.asyncio |
| 2996 | async def test_official_http_document_failures_do_not_trip_host_transport_budget() -> None: |
| 2997 | settings = Settings(research_live_enabled=True, research_official_document_max_attempts_per_refresh=3) |
| 2998 | repository = ResearchRepository(settings=settings) |
| 2999 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3000 | sources = [_official_financial_result_source(profile, f"http-{index}") for index in range(4)] |
| 3001 | fetcher = _OfficialFetchFixture(HttpStatusFetchError(404)) |
| 3002 | repository._fetcher = fetcher |
| 3003 | |
| 3004 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source) for source in sources], set()) |
| 3005 | |
| 3006 | assert fetcher.urls == [source.url for source in sources[:3]] |
| 3007 | |
| 3008 | |
| 3009 | @pytest.mark.asyncio |
| 3010 | async def test_official_restricted_document_failure_does_not_trip_host_transport_budget() -> None: |
| 3011 | settings = Settings(research_live_enabled=True, research_official_document_max_attempts_per_refresh=3) |
| 3012 | repository = ResearchRepository(settings=settings) |
| 3013 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3014 | sources = [_official_financial_result_source(profile, f"restricted-{index}") for index in range(4)] |
| 3015 | fetcher = _OfficialFetchFixture(RestrictedFetchError(403)) |
| 3016 | repository._fetcher = fetcher |
| 3017 | |
| 3018 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source) for source in sources], set()) |
| 3019 | |
| 3020 | assert fetcher.urls == [source.url for source in sources[:3]] |
| 3021 | |
| 3022 | |
| 3023 | @pytest.mark.asyncio |
| 3024 | async def test_pdf_extraction_timeout_does_not_trip_official_host_transport_budget() -> None: |
| 3025 | class SlowPdfFetcher(HttpResearchFetcher): |
| 3026 | def process_network_response(self, response, *, max_bytes=None): |
| 3027 | time.sleep(0.05) |
| 3028 | return FetchResult(response.final_url, 200, "text/html", "<html>unused</html>", len(response.content)) |
| 3029 | |
| 3030 | settings = Settings( |
| 3031 | research_live_enabled=True, |
| 3032 | research_official_document_max_attempts_per_refresh=2, |
| 3033 | research_official_document_timeout_seconds=1.0, |
| 3034 | research_official_document_extraction_timeout_seconds=0.01, |
| 3035 | ) |
| 3036 | requests: list[str] = [] |
| 3037 | def handler(request): |
| 3038 | requests.append(str(request.url)) |
| 3039 | return httpx.Response(200, headers={"content-type": "application/pdf"}, content=b"%PDF-fast", request=request) |
| 3040 | client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) |
| 3041 | repository = ResearchRepository(settings=settings, fetcher=SlowPdfFetcher(settings, client)) |
| 3042 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3043 | sources = [_official_financial_result_source(profile, f"slow-{index}") for index in range(3)] |
| 3044 | |
| 3045 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source) for source in sources], set()) |
| 3046 | |
| 3047 | # Extraction failures are document failures, so the normal attempt budget |
| 3048 | # (two) applies rather than the one-failure host transport circuit. |
| 3049 | assert requests == [source.url for source in sources[:2]] |
| 3050 | |
| 3051 | |
| 3052 | @pytest.mark.asyncio |
| 3053 | async def test_official_filing_single_flight_shares_network_extraction_and_persistence() -> None: |
| 3054 | class CountingFetcher(HttpResearchFetcher): |
| 3055 | def __init__(self, settings): |
| 3056 | super().__init__(settings) |
| 3057 | self.network_calls = 0 |
| 3058 | self.extraction_calls = 0 |
| 3059 | self.network_started = asyncio.Event() |
| 3060 | self.release_network = asyncio.Event() |
| 3061 | |
| 3062 | async def fetch_network(self, url, **_kwargs): |
| 3063 | self.network_calls += 1 |
| 3064 | self.network_started.set() |
| 3065 | await self.release_network.wait() |
| 3066 | return type("Network", (), {"final_url": url, "status_code": 200, "content": b"body"})() |
| 3067 | |
| 3068 | def process_network_response(self, response, *, max_bytes=None): |
| 3069 | self.extraction_calls += 1 |
| 3070 | return FetchResult( |
| 3071 | response.final_url, 200, "text/html", |
| 3072 | "<title>Reliance quarterly financial results</title><main>Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue 1000 crore PAT 100 crore</main>", |
| 3073 | len(response.content), |
| 3074 | ) |
| 3075 | |
| 3076 | settings = Settings(research_live_enabled=True) |
| 3077 | fetcher = CountingFetcher(settings) |
| 3078 | repository = ResearchRepository(settings=settings, fetcher=fetcher) |
| 3079 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3080 | source = _official_financial_result_source(profile, "single-flight") |
| 3081 | persistence_calls = 0 |
| 3082 | original_ingest = repository._ingest_registered_fetch_result_async |
| 3083 | |
| 3084 | async def count_ingest(*args, **kwargs): |
| 3085 | nonlocal persistence_calls |
| 3086 | persistence_calls += 1 |
| 3087 | return await original_ingest(*args, **kwargs) |
| 3088 | |
| 3089 | repository._ingest_registered_fetch_result_async = count_ingest |
| 3090 | diagnostic_records: list[logging.LogRecord] = [] |
| 3091 | |
| 3092 | class DiagnosticHandler(logging.Handler): |
| 3093 | def emit(self, record: logging.LogRecord) -> None: |
| 3094 | diagnostic_records.append(record) |
| 3095 | |
| 3096 | diagnostic_handler = DiagnosticHandler(level=logging.INFO) |
| 3097 | repository_logger = logging.getLogger("app.repository") |
| 3098 | original_level = repository_logger.level |
| 3099 | repository_logger.setLevel(logging.INFO) |
| 3100 | repository_logger.addHandler(diagnostic_handler) |
| 3101 | try: |
| 3102 | leader = asyncio.create_task( |
| 3103 | repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3104 | ) |
| 3105 | await fetcher.network_started.wait() |
| 3106 | follower = asyncio.create_task( |
| 3107 | repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3108 | ) |
| 3109 | # The leader is blocked above. Yielding one event-loop turn lets the |
| 3110 | # follower observe and await the installed flight without timing luck. |
| 3111 | await asyncio.sleep(0) |
| 3112 | assert not follower.done() |
| 3113 | fetcher.release_network.set() |
| 3114 | await asyncio.gather(leader, follower) |
| 3115 | finally: |
| 3116 | repository_logger.removeHandler(diagnostic_handler) |
| 3117 | repository_logger.setLevel(original_level) |
| 3118 | |
| 3119 | assert fetcher.network_calls == fetcher.extraction_calls == persistence_calls == 1 |
| 3120 | assert len(repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL)) == 1 |
| 3121 | assert any( |
| 3122 | "outcome=REUSED" in record.getMessage() and "reason=IN_FLIGHT_SINGLE_FLIGHT" in record.getMessage() |
| 3123 | for record in diagnostic_records |
| 3124 | ) |
| 3125 | assert repository._official_filing_flights == {} |
| 3126 | |
| 3127 | |
| 3128 | @pytest.mark.asyncio |
| 3129 | async def test_async_live_ingestion_preparation_does_not_block_event_loop() -> None: |
| 3130 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3131 | profile = next(item for item in repository.profiles if item.ticker == "RELIANCE") |
| 3132 | source = _official_financial_result_source(profile, "prepare-worker") |
| 3133 | entered, release = threading.Event(), threading.Event() |
| 3134 | worker_thread: list[int] = [] |
| 3135 | original = repository._prepare_ingested_document |
| 3136 | |
| 3137 | def blocked_prepare(*args, **kwargs): |
| 3138 | worker_thread.append(threading.get_ident()) |
| 3139 | entered.set() |
| 3140 | assert release.wait(2) |
| 3141 | return original(*args, **kwargs) |
| 3142 | |
| 3143 | repository._prepare_ingested_document = blocked_prepare |
| 3144 | result = FetchResult(source.url, 200, "text/html", "<main>Reliance Industries Limited RELIANCE INE002A01018 sufficient content</main>", 100) |
| 3145 | loop_thread = threading.get_ident() |
| 3146 | documents_before = dict(repository.documents) |
| 3147 | task = asyncio.create_task(repository._ingest_registered_fetch_result_async(profile, source, result, expected_profile=profile)) |
| 3148 | assert await asyncio.to_thread(entered.wait, 1) |
| 3149 | progressed = False |
| 3150 | async def sibling(): |
| 3151 | nonlocal progressed |
| 3152 | await asyncio.sleep(0) |
| 3153 | progressed = True |
| 3154 | await sibling() |
| 3155 | assert progressed and worker_thread[0] != loop_thread |
| 3156 | assert repository.documents == documents_before |
| 3157 | release.set() |
| 3158 | document = await task |
| 3159 | assert document.status == DocumentStatus.PROCESSED |
| 3160 | assert len([item for item in repository.documents.values() if item.document_id == document.document_id]) == 1 |
| 3161 | |
| 3162 | |
| 3163 | @pytest.mark.asyncio |
| 3164 | async def test_async_live_ingestion_document_persistence_does_not_block_event_loop() -> None: |
| 3165 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3166 | profile = next(item for item in repository.profiles if item.ticker == "RELIANCE") |
| 3167 | source = _official_financial_result_source(profile, "persist-worker") |
| 3168 | started, release = threading.Event(), threading.Event() |
| 3169 | worker: list[int] = []; continued = 0; calls = 0 |
| 3170 | original_persist = repository._persist_ingested_document |
| 3171 | original_continue = repository._continue_ingested_document_after_events_async |
| 3172 | def blocked(document, **kwargs): |
| 3173 | nonlocal calls |
| 3174 | calls += 1; worker.append(threading.get_ident()); started.set(); assert release.wait(2) |
| 3175 | return original_persist(document, **kwargs) |
| 3176 | async def count_continue(*args, **kwargs): |
| 3177 | nonlocal continued |
| 3178 | continued += 1 |
| 3179 | return await original_continue(*args, **kwargs) |
| 3180 | repository._persist_ingested_document = blocked |
| 3181 | repository._continue_ingested_document_after_events_async = count_continue |
| 3182 | result = FetchResult(source.url, 200, "text/html", "<main>Reliance Industries Limited RELIANCE INE002A01018 sufficient content</main>", 100) |
| 3183 | loop_thread = threading.get_ident() |
| 3184 | task = asyncio.create_task(repository._ingest_registered_fetch_result_async(profile, source, result, expected_profile=profile)) |
| 3185 | assert await asyncio.to_thread(started.wait, 1) |
| 3186 | await asyncio.sleep(0) |
| 3187 | assert worker[0] != loop_thread and continued == 0 |
| 3188 | assert any(item.canonical_url == source.url for item in repository.documents.values()) |
| 3189 | release.set(); document = await task |
| 3190 | assert calls == continued == 1 and document.status == DocumentStatus.PROCESSED |
| 3191 | |
| 3192 | |
| 3193 | @pytest.mark.asyncio |
| 3194 | async def test_async_live_ingestion_document_persistence_failure_rolls_back() -> None: |
| 3195 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3196 | profile = next(item for item in repository.profiles if item.ticker == "RELIANCE") |
| 3197 | source = _official_financial_result_source(profile, "persist-failure") |
| 3198 | attempted: list[UUID] = []; continued = 0 |
| 3199 | def fail(document, **_kwargs): |
| 3200 | attempted.append(document.document_id) |
| 3201 | assert document.document_id in repository.documents |
| 3202 | raise RuntimeError("db unavailable") |
| 3203 | repository._persist_ingested_document = fail |
| 3204 | repository._continue_ingested_document_after_persistence = lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("downstream must not run")) |
| 3205 | result = FetchResult(source.url, 200, "text/html", "<main>Reliance Industries Limited RELIANCE INE002A01018 sufficient content</main>", 100) |
| 3206 | with pytest.raises(FetchError, match="DOCUMENT_PERSIST_FAILED"): |
| 3207 | await repository._ingest_registered_fetch_result_async(profile, source, result, expected_profile=profile) |
| 3208 | assert len(attempted) == 1 and attempted[0] not in repository.documents |
| 3209 | |
| 3210 | |
| 3211 | @pytest.mark.asyncio |
| 3212 | async def test_async_live_ingestion_financial_processing_does_not_block_event_loop() -> None: |
| 3213 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3214 | profile = next(item for item in repository.profiles if item.ticker == "RELIANCE") |
| 3215 | source = _official_financial_result_source(profile, "financial-worker") |
| 3216 | started, release = threading.Event(), threading.Event() |
| 3217 | worker: list[int] = []; financial_calls = 0; continued = 0 |
| 3218 | original_financial = repository._persist_official_financial_facts |
| 3219 | original_continue = repository._continue_ingested_document_after_events_async |
| 3220 | def blocked(document): |
| 3221 | nonlocal financial_calls |
| 3222 | financial_calls += 1; worker.append(threading.get_ident()); started.set(); assert release.wait(2) |
| 3223 | return original_financial(document) |
| 3224 | async def count_continue(*args, **kwargs): |
| 3225 | nonlocal continued |
| 3226 | continued += 1 |
| 3227 | return await original_continue(*args, **kwargs) |
| 3228 | repository._persist_official_financial_facts = blocked |
| 3229 | repository._continue_ingested_document_after_events_async = count_continue |
| 3230 | result = FetchResult(source.url, 200, "text/html", "<main>Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue 1000 crore PAT 100 crore</main>", 100) |
| 3231 | loop_thread = threading.get_ident() |
| 3232 | task = asyncio.create_task(repository._ingest_registered_fetch_result_async(profile, source, result, expected_profile=profile)) |
| 3233 | assert await asyncio.to_thread(started.wait, 1) |
| 3234 | await asyncio.sleep(0) |
| 3235 | assert worker[0] != loop_thread and continued == 0 |
| 3236 | assert any(item.canonical_url == source.url for item in repository.documents.values()) |
| 3237 | release.set(); document = await task |
| 3238 | assert document.status == DocumentStatus.PROCESSED |
| 3239 | assert financial_calls == continued == 1 |
| 3240 | |
| 3241 | |
| 3242 | @pytest.mark.asyncio |
| 3243 | async def test_async_live_ingestion_event_extraction_does_not_block_event_loop() -> None: |
| 3244 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3245 | profile = next(item for item in repository.profiles if item.ticker == "RELIANCE") |
| 3246 | source = _official_financial_result_source(profile, "event-worker") |
| 3247 | started, release = threading.Event(), threading.Event() |
| 3248 | worker: list[int] = []; calls = 0 |
| 3249 | original = repository._extract_ingested_event_candidates |
| 3250 | before_keys, before_events = set(repository._event_keys), dict(repository.events) |
| 3251 | def blocked(*args, **kwargs): |
| 3252 | nonlocal calls |
| 3253 | calls += 1; worker.append(threading.get_ident()); started.set(); assert release.wait(2) |
| 3254 | return original(*args, **kwargs) |
| 3255 | repository._extract_ingested_event_candidates = blocked |
| 3256 | result = FetchResult(source.url, 200, "text/html", "<main>Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue 1000 crore PAT 100 crore</main>", 100) |
| 3257 | loop_thread = threading.get_ident() |
| 3258 | task = asyncio.create_task(repository._ingest_registered_fetch_result_async(profile, source, result, expected_profile=profile)) |
| 3259 | assert await asyncio.to_thread(started.wait, 1) |
| 3260 | progressed = False |
| 3261 | async def sibling(): |
| 3262 | nonlocal progressed |
| 3263 | await asyncio.sleep(0); progressed = True |
| 3264 | await sibling() |
| 3265 | assert progressed and worker[0] != loop_thread and calls == 1 |
| 3266 | assert repository._event_keys == before_keys and repository.events == before_events |
| 3267 | assert any(item.canonical_url == source.url for item in repository.documents.values()) |
| 3268 | release.set(); await task |
| 3269 | assert calls == 1 |
| 3270 | |
| 3271 | |
| 3272 | @pytest.mark.asyncio |
| 3273 | async def test_official_filing_single_flight_failure_is_shared_and_cleanup_allows_retry() -> None: |
| 3274 | class FailingFetcher(HttpResearchFetcher): |
| 3275 | def __init__(self, settings): |
| 3276 | super().__init__(settings) |
| 3277 | self.calls = 0 |
| 3278 | |
| 3279 | async def fetch_network(self, _url, **_kwargs): |
| 3280 | self.calls += 1 |
| 3281 | await asyncio.sleep(0.01) |
| 3282 | raise TransportFetchError("Fetch timed out") |
| 3283 | |
| 3284 | settings = Settings(research_live_enabled=True) |
| 3285 | fetcher = FailingFetcher(settings) |
| 3286 | repository = ResearchRepository(settings=settings, fetcher=fetcher) |
| 3287 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3288 | source = _official_financial_result_source(profile, "single-flight-failure") |
| 3289 | |
| 3290 | await asyncio.gather( |
| 3291 | repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()), |
| 3292 | repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()), |
| 3293 | ) |
| 3294 | assert fetcher.calls == 1 |
| 3295 | assert repository._official_filing_flights == {} |
| 3296 | |
| 3297 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3298 | assert fetcher.calls == 2 |
| 3299 | |
| 3300 | |
| 3301 | @pytest.mark.asyncio |
| 3302 | async def test_late_extraction_completion_after_timeout_cannot_persist_or_retain_flight() -> None: |
| 3303 | class SlowFetcher(HttpResearchFetcher): |
| 3304 | def __init__(self, settings): |
| 3305 | super().__init__(settings) |
| 3306 | self.extractions = 0 |
| 3307 | |
| 3308 | async def fetch_network(self, url, **_kwargs): |
| 3309 | return type("Network", (), {"final_url": url, "status_code": 200, "content": b"body"})() |
| 3310 | |
| 3311 | def process_network_response(self, response, *, max_bytes=None): |
| 3312 | self.extractions += 1 |
| 3313 | time.sleep(0.05) |
| 3314 | return FetchResult(response.final_url, 200, "text/html", "<main>late</main>", len(response.content)) |
| 3315 | |
| 3316 | settings = Settings(research_live_enabled=True, research_official_document_extraction_timeout_seconds=0.01) |
| 3317 | fetcher = SlowFetcher(settings) |
| 3318 | repository = ResearchRepository(settings=settings, fetcher=fetcher) |
| 3319 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3320 | source = _official_financial_result_source(profile, "late-thread") |
| 3321 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3322 | await asyncio.sleep(0.07) |
| 3323 | |
| 3324 | assert fetcher.extractions == 1 |
| 3325 | assert repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL) == [] |
| 3326 | assert repository._official_filing_flights == {} |
| 3327 | |
| 3328 | |
| 3329 | @pytest.mark.asyncio |
| 3330 | async def test_official_filing_single_flight_cancellation_cleans_up_and_urls_remain_independent() -> None: |
| 3331 | class BlockingFetcher(HttpResearchFetcher): |
| 3332 | def __init__(self, settings): |
| 3333 | super().__init__(settings) |
| 3334 | self.calls: list[str] = [] |
| 3335 | self.started = asyncio.Event() |
| 3336 | self.release = asyncio.Event() |
| 3337 | |
| 3338 | async def fetch_network(self, url, **_kwargs): |
| 3339 | self.calls.append(url) |
| 3340 | self.started.set() |
| 3341 | await self.release.wait() |
| 3342 | raise TransportFetchError("cancelled test") |
| 3343 | |
| 3344 | settings = Settings(research_live_enabled=True) |
| 3345 | fetcher = BlockingFetcher(settings) |
| 3346 | repository = ResearchRepository(settings=settings, fetcher=fetcher) |
| 3347 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3348 | first = _official_financial_result_source(profile, "cancel-one") |
| 3349 | second = _official_financial_result_source(profile, "cancel-two") |
| 3350 | first_task = asyncio.create_task(repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", first)], set())) |
| 3351 | await fetcher.started.wait() |
| 3352 | second_task = asyncio.create_task(repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", second)], set())) |
| 3353 | for _ in range(10): |
| 3354 | if len(fetcher.calls) == 2: |
| 3355 | break |
| 3356 | await asyncio.sleep(0) |
| 3357 | assert set(fetcher.calls) == {first.url, second.url} |
| 3358 | |
| 3359 | first_task.cancel() |
| 3360 | with pytest.raises(asyncio.CancelledError): |
| 3361 | await first_task |
| 3362 | await asyncio.sleep(0) |
| 3363 | assert (profile.instrument_id, canonicalize_url(first.url)) not in repository._official_filing_flights |
| 3364 | |
| 3365 | fetcher.release.set() |
| 3366 | await second_task |
| 3367 | assert repository._official_filing_flights == {} |
| 3368 | |
| 3369 | |
| 3370 | @pytest.mark.asyncio |
| 3371 | async def test_fast_large_official_pdf_persists_and_marks_financial_results_fresh(monkeypatch) -> None: |
| 3372 | class Page: |
| 3373 | def extract_text(self): |
| 3374 | return "Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue 1000 crore PAT 100 crore" |
| 3375 | |
| 3376 | class Reader: |
| 3377 | def __init__(self, _stream): |
| 3378 | self.pages = [Page()] |
| 3379 | |
| 3380 | monkeypatch.setattr("pypdf.PdfReader", Reader) |
| 3381 | settings = Settings( |
| 3382 | research_live_enabled=True, |
| 3383 | research_max_retries=0, |
| 3384 | research_max_content_bytes=6_000_000, |
| 3385 | research_official_document_timeout_seconds=1.0, |
| 3386 | research_official_document_extraction_timeout_seconds=1.0, |
| 3387 | ) |
| 3388 | client = httpx.AsyncClient(transport=httpx.MockTransport( |
| 3389 | lambda request: httpx.Response(200, headers={"content-type": "application/pdf"}, content=b"%PDF-" + b"x" * 5_000_000, request=request) |
| 3390 | )) |
| 3391 | repository = ResearchRepository(settings=settings, fetcher=HttpResearchFetcher(settings, client)) |
| 3392 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3393 | source = _official_financial_result_source(profile, "large") |
| 3394 | |
| 3395 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3396 | repository._mark_qualifying_categories_fresh(profile.instrument_id, {"FINANCIAL_RESULTS"}, datetime.now(timezone.utc)) |
| 3397 | |
| 3398 | assert repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL)[0].status == DocumentStatus.PROCESSED |
| 3399 | assert repository._category_is_fresh(profile.instrument_id, "FINANCIAL_RESULTS", datetime.now(timezone.utc)) |
| 3400 | |
| 3401 | |
| 3402 | @pytest.mark.asyncio |
| 3403 | async def test_official_document_fetch_uses_configured_provider_neutral_user_agent() -> None: |
| 3404 | settings = Settings(research_max_retries=0, research_official_document_user_agent="Mozilla/5.0 test-agent") |
| 3405 | observed: dict[str, str] = {} |
| 3406 | def handler(request): |
| 3407 | observed["user_agent"] = request.headers["user-agent"] |
| 3408 | return httpx.Response(200, headers={"content-type": "text/html"}, text="<html>official document</html>", request=request) |
| 3409 | fetcher = HttpResearchFetcher(settings, httpx.AsyncClient(transport=httpx.MockTransport(handler))) |
| 3410 | |
| 3411 | result = await fetcher.fetch_network( |
| 3412 | "https://public-official.example/documents/report.html", |
| 3413 | headers={"User-Agent": settings.research_official_document_user_agent}, |
| 3414 | ) |
| 3415 | |
| 3416 | assert result.status_code == 200 |
| 3417 | assert observed["user_agent"] == "Mozilla/5.0 test-agent" |
| 3418 | |
| 3419 | |
| 3420 | @pytest.mark.asyncio |
| 3421 | async def test_official_network_budget_accepts_five_and_eleven_megabyte_pdfs() -> None: |
| 3422 | settings = Settings(research_max_retries=0) |
| 3423 | for size in (5_500_000, 11_000_000): |
| 3424 | client = httpx.AsyncClient(transport=httpx.MockTransport( |
| 3425 | lambda request, size=size: httpx.Response(200, headers={"content-type": "application/pdf"}, content=b"%PDF-" + b"x" * size, request=request) |
| 3426 | )) |
| 3427 | result = await HttpResearchFetcher(settings, client).fetch_network( |
| 3428 | "https://nsearchives.nseindia.com/corporate/result.pdf", max_bytes=settings.research_official_document_max_bytes, |
| 3429 | ) |
| 3430 | assert len(result.content) == size + 5 |
| 3431 | |
| 3432 | |
| 3433 | @pytest.mark.asyncio |
| 3434 | async def test_official_content_length_limit_rejects_before_body_read() -> None: |
| 3435 | settings = Settings(research_max_retries=0, research_official_document_max_bytes=10) |
| 3436 | client = httpx.AsyncClient(transport=httpx.MockTransport( |
| 3437 | lambda request: httpx.Response(200, headers={"content-type": "application/pdf", "content-length": "11"}, content=b"%PDF-body", request=request) |
| 3438 | )) |
| 3439 | with pytest.raises(DocumentSizeLimitExceeded) as exc_info: |
| 3440 | await HttpResearchFetcher(settings, client).fetch_network("https://nsearchives.nseindia.com/corporate/result.pdf", max_bytes=10) |
| 3441 | assert exc_info.value.content_length == 11 |
| 3442 | assert exc_info.value.max_bytes == 10 |
| 3443 | |
| 3444 | |
| 3445 | @pytest.mark.asyncio |
| 3446 | async def test_official_streaming_limit_handles_missing_or_dishonest_content_length() -> None: |
| 3447 | settings = Settings(research_max_retries=0, research_official_document_max_bytes=10) |
| 3448 | for headers in ({"content-type": "application/pdf"}, {"content-type": "application/pdf", "content-length": "1"}): |
| 3449 | client = httpx.AsyncClient(transport=httpx.MockTransport( |
| 3450 | lambda request, headers=headers: httpx.Response(200, headers=headers, content=b"%PDF-" + b"x" * 10, request=request) |
| 3451 | )) |
| 3452 | with pytest.raises(DocumentSizeLimitExceeded): |
| 3453 | await HttpResearchFetcher(settings, client).fetch_network("https://nsearchives.nseindia.com/corporate/result.pdf", max_bytes=10) |
| 3454 | |
| 3455 | |
| 3456 | @pytest.mark.asyncio |
| 3457 | async def test_oversized_official_document_is_retryable_and_does_not_open_transport_circuit_or_freshen() -> None: |
| 3458 | settings = Settings( |
| 3459 | research_live_enabled=True, research_max_retries=0, |
| 3460 | research_official_document_max_bytes=10, |
| 3461 | research_official_document_max_attempts_per_refresh=2, |
| 3462 | ) |
| 3463 | requests: list[str] = [] |
| 3464 | def handler(request): |
| 3465 | requests.append(str(request.url)) |
| 3466 | return httpx.Response(200, headers={"content-type": "application/pdf", "content-length": "11"}, content=b"%PDF-body", request=request) |
| 3467 | repository = ResearchRepository(settings=settings, fetcher=HttpResearchFetcher(settings, httpx.AsyncClient(transport=httpx.MockTransport(handler)))) |
| 3468 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3469 | sources = [_official_financial_result_source(profile, f"oversized-{index}") for index in range(3)] |
| 3470 | |
| 3471 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source) for source in sources], set()) |
| 3472 | |
| 3473 | assert requests == [source.url for source in sources[:2]] |
| 3474 | assert repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL) == [] |
| 3475 | assert not repository._category_is_fresh(profile.instrument_id, "FINANCIAL_RESULTS", datetime.now(timezone.utc)) |
| 3476 | |
| 3477 | |
| 3478 | @pytest.mark.asyncio |
| 3479 | @pytest.mark.parametrize("body_size", [5_500_000, 11_000_000, 16_000_000, 24 * 1024 * 1024]) |
| 3480 | async def test_official_pdf_under_official_budget_reaches_processing_and_persistence(monkeypatch, body_size: int) -> None: |
| 3481 | class Page: |
| 3482 | def extract_text(self): |
| 3483 | return "Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue 1000 crore PAT 100 crore" |
| 3484 | |
| 3485 | class Reader: |
| 3486 | def __init__(self, _stream): |
| 3487 | self.pages = [Page()] |
| 3488 | |
| 3489 | monkeypatch.setattr("pypdf.PdfReader", Reader) |
| 3490 | settings = Settings(research_live_enabled=True, research_max_retries=0) |
| 3491 | client = httpx.AsyncClient(transport=httpx.MockTransport( |
| 3492 | lambda request: httpx.Response(200, headers={"content-type": "application/pdf"}, content=b"%PDF-" + b"x" * body_size, request=request) |
| 3493 | )) |
| 3494 | repository = ResearchRepository(settings=settings, fetcher=HttpResearchFetcher(settings, client)) |
| 3495 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3496 | source = _official_financial_result_source(profile, f"size-{body_size}") |
| 3497 | |
| 3498 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3499 | |
| 3500 | assert repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL)[0].status == DocumentStatus.PROCESSED |
| 3501 | |
| 3502 | |
| 3503 | @pytest.mark.asyncio |
| 3504 | async def test_official_pdf_above_default_budget_fails_before_processing() -> None: |
| 3505 | settings = Settings(research_live_enabled=True, research_max_retries=0) |
| 3506 | requests: list[str] = [] |
| 3507 | def handler(request): |
| 3508 | requests.append(str(request.url)) |
| 3509 | return httpx.Response( |
| 3510 | 200, |
| 3511 | headers={"content-type": "application/pdf", "content-length": str(settings.research_official_document_max_bytes + 1)}, |
| 3512 | content=b"", |
| 3513 | request=request, |
| 3514 | ) |
| 3515 | repository = ResearchRepository(settings=settings, fetcher=HttpResearchFetcher(settings, httpx.AsyncClient(transport=httpx.MockTransport(handler)))) |
| 3516 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3517 | source = _official_financial_result_source(profile, "over-default") |
| 3518 | |
| 3519 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3520 | |
| 3521 | assert requests == [source.url] |
| 3522 | assert repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL) == [] |
| 3523 | |
| 3524 | |
| 3525 | @pytest.mark.asyncio |
| 3526 | async def test_global_official_document_is_reused_without_second_download_or_extraction() -> None: |
| 3527 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3528 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3529 | source = _official_financial_result_source(profile, "reused") |
| 3530 | fetcher = _OfficialFetchFixture(FetchResult( |
| 3531 | final_url=source.url, status_code=200, content_type="text/html", bytes_read=180, |
| 3532 | text="<html><title>Reliance Industries Limited Quarterly Financial Results</title><main>Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue 1000 crore PAT 100 crore.</main></html>", |
| 3533 | )) |
| 3534 | repository._fetcher = fetcher |
| 3535 | |
| 3536 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3537 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3538 | repository._mark_qualifying_categories_fresh(profile.instrument_id, {"FINANCIAL_RESULTS"}, datetime.now(timezone.utc)) |
| 3539 | |
| 3540 | assert fetcher.urls == [source.url] |
| 3541 | assert len(repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL)) == 1 |
| 3542 | assert repository._category_is_fresh(profile.instrument_id, "FINANCIAL_RESULTS", datetime.now(timezone.utc)) |
| 3543 | |
| 3544 | |
| 3545 | @pytest.mark.asyncio |
| 3546 | async def test_failed_or_scanned_official_document_is_not_reused_and_remains_retryable() -> None: |
| 3547 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3548 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3549 | source = _official_financial_result_source(profile, "retryable") |
| 3550 | failed = repository.ingest_fixture( |
| 3551 | original_url=source.url, source_type=source.source_type, source_classification=source.source_classification, |
| 3552 | source_name=source.source_name, publisher=source.publisher, content_type="application/pdf", body="", |
| 3553 | reliability=source.reliability_level, source_mode=SourceMode.REAL, expected_profile=profile, |
| 3554 | document_status=DocumentStatus.FAILED, allow_empty_content=True, |
| 3555 | ) |
| 3556 | assert failed.status == DocumentStatus.FAILED |
| 3557 | fetcher = _OfficialFetchFixture(FetchError("Fetch timed out")) |
| 3558 | repository._fetcher = fetcher |
| 3559 | |
| 3560 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source)], set()) |
| 3561 | |
| 3562 | assert fetcher.urls == [source.url] |
| 3563 | assert not repository._category_is_fresh(profile.instrument_id, "FINANCIAL_RESULTS", datetime.now(timezone.utc)) |
| 3564 | |
| 3565 | |
| 3566 | @pytest.mark.asyncio |
| 3567 | async def test_distinct_official_urls_are_not_collapsed_as_same_quarterly_document() -> None: |
| 3568 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3569 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3570 | first = _official_financial_result_source(profile, "consolidated") |
| 3571 | second = _official_financial_result_source(profile, "standalone") |
| 3572 | repository._fetcher = _OfficialFetchFixture(FetchResult( |
| 3573 | final_url=first.url, status_code=200, content_type="text/html", bytes_read=200, |
| 3574 | text="<html><title>Reliance Industries Limited Financial Results</title><main>Reliance Industries Limited RELIANCE INE002A01018 consolidated quarterly financial results revenue 1000 crore PAT 100 crore.</main></html>", |
| 3575 | )) |
| 3576 | |
| 3577 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", first)], set()) |
| 3578 | repository._fetcher = _OfficialFetchFixture(FetchResult( |
| 3579 | final_url=second.url, status_code=200, content_type="text/html", bytes_read=200, |
| 3580 | text="<html><title>Reliance Industries Limited Financial Results</title><main>Reliance Industries Limited RELIANCE INE002A01018 standalone quarterly financial results revenue 900 crore PAT 90 crore.</main></html>", |
| 3581 | )) |
| 3582 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", second)], set()) |
| 3583 | |
| 3584 | assert len(repository.documents_for(profile.instrument_id, source_mode=SourceMode.REAL)) == 2 |
| 3585 | |
| 3586 | |
| 3587 | def test_existing_durable_financial_result_is_the_only_reason_a_failed_refresh_can_become_fresh() -> None: |
| 3588 | repository = ResearchRepository(settings=Settings(research_live_enabled=True)) |
| 3589 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3590 | now = datetime.now(timezone.utc) |
| 3591 | assert "FINANCIAL_RESULTS" in repository._missing_categories(profile, set(), now, {"FINANCIAL_RESULTS"}) |
| 3592 | document = repository.ingest_fixture( |
| 3593 | original_url="https://nsearchives.nseindia.com/corporate/prior-result.pdf", |
| 3594 | source_type=SourceType.EXCHANGE_ANNOUNCEMENT, source_classification=SourceClassification.EXCHANGE, |
| 3595 | source_name="NSE corporate announcements", publisher="NSE", content_type="text/html", |
| 3596 | body="<html><title>Reliance Industries Limited Quarterly Financial Results</title><main>Reliance Industries Limited RELIANCE INE002A01018 quarterly financial results revenue 1000 crore PAT 100 crore.</main></html>", |
| 3597 | reliability=ReliabilityLevel.LEVEL_A, source_mode=SourceMode.REAL, expected_profile=profile, |
| 3598 | ) |
| 3599 | |
| 3600 | repository._mark_qualifying_categories_fresh(profile.instrument_id, {"FINANCIAL_RESULTS"}, now) |
| 3601 | |
| 3602 | assert document.document_id in repository.documents |
| 3603 | assert "FINANCIAL_RESULTS" not in repository._missing_categories(profile, set(), now, {"FINANCIAL_RESULTS"}) |
| 3604 | |
| 3605 | |
| 3606 | @pytest.mark.asyncio |
| 3607 | async def test_official_fetch_is_newest_first_and_bounded_without_concurrency() -> None: |
| 3608 | settings = Settings(research_live_enabled=True, research_official_document_max_attempts_per_refresh=3) |
| 3609 | repository = ResearchRepository(settings=settings) |
| 3610 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 3611 | sources = [_official_financial_result_source(profile, f"newest-{index}") for index in range(5)] |
| 3612 | class ConcurrentProbe: |
| 3613 | def __init__(self) -> None: |
| 3614 | self.urls: list[str] = [] |
| 3615 | self.active = 0 |
| 3616 | self.max_active = 0 |
| 3617 | |
| 3618 | async def fetch(self, url: str) -> FetchResult: |
| 3619 | self.urls.append(url) |
| 3620 | self.active += 1 |
| 3621 | self.max_active = max(self.max_active, self.active) |
| 3622 | await asyncio.sleep(0) |
| 3623 | self.active -= 1 |
| 3624 | return FetchResult( |
| 3625 | final_url=url, status_code=200, content_type="text/html", bytes_read=200, |
| 3626 | text="<html><title>Reliance Industries Limited Financial Results</title><main>Reliance Industries Limited RELIANCE INE002A01018 financial results revenue 1000 crore PAT 100 crore.</main></html>", |
| 3627 | ) |
| 3628 | |
| 3629 | fetcher = ConcurrentProbe() |
| 3630 | repository._fetcher = fetcher |
| 3631 | |
| 3632 | await repository._fetch_official_filings(profile, [DiscoveryResult("FINANCIAL_RESULTS", source) for source in sources], set()) |
| 3633 | |
| 3634 | assert fetcher.urls == [source.url for source in sources[:3]] |
| 3635 | assert fetcher.max_active == 1 |
| 3636 | |
| 3637 | |
| 3638 | def test_reused_global_profile_hydrates_verified_nse_mapping_over_broker_alias() -> None: |
| 3639 | profile = CompanyResearchProfile( |
| 3640 | instrument_id=UUID("99999999-9999-9999-9999-999999999999"), company_id=UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), |
| 3641 | company_name="Example Components Limited", ticker="BROKER_ALIAS", exchange="NSE", mic="XNSE", country="IN", currency="INR", |
| 3642 | ) |
| 3643 | hydrated = _hydrate_verified_exchange_mappings(profile, {"nseSymbol": "VERIFIED_NSE_SYMBOL"}) |
| 3644 | assert hydrated.provider_instrument_ids["NSE"] == "VERIFIED_NSE_SYMBOL" |
| 3645 | |
| 3646 | |
| 3647 | def test_indian_search_queries_prefer_verified_exchange_mapping_over_broker_ticker() -> None: |
| 3648 | profile = CompanyResearchProfile( |
| 3649 | instrument_id=UUID("99999999-9999-9999-9999-999999999999"), company_id=UUID("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), |
| 3650 | company_name="Example Components Limited", ticker="BROKER_ALIAS", exchange="NSE", mic="XNSE", country="IN", currency="INR", |
| 3651 | provider_instrument_ids={"NSE": "VERIFIED_NSE_SYMBOL"}, |
| 3652 | ) |
| 3653 | queries = generate_search_queries(profile, "FINANCIAL_RESULTS", SearchDateWindow()) |
| 3654 | assert '"VERIFIED_NSE_SYMBOL" NSE' in queries |
| 3655 | assert '"BROKER_ALIAS" NSE' not in queries |
| 3656 | assert '"VERIFIED_NSE_SYMBOL" BSE' in queries |
| 3657 | assert '"BROKER_ALIAS" BSE' not in queries |
| 3658 | |
| 3659 | |
| 3660 | def test_invalid_nse_mapping_is_not_selected_when_verified_mapping_exists() -> None: |
| 3661 | repository = ResearchRepository() |
| 3662 | orchestrator = PortfolioResearchOrchestrator(repository, Settings()) |
| 3663 | position = _portfolio_position("INE000A01010", "BROKER_ALIAS", "NSE", "Example Components Limited") |
| 3664 | position["instrument"]["globalInstrumentId"] = str(UUID("99999999-9999-9999-9999-999999999999")) |
| 3665 | position["instrument"]["providerMappings"] = [ |
| 3666 | {"provider": "NSE", "providerSymbol": "INVALID_ALIAS", "status": "INVALID"}, |
| 3667 | {"provider": "NSE", "providerSymbol": "VERIFIED_NSE_SYMBOL", "status": "VERIFIED"}, |
| 3668 | ] |
| 3669 | try: |
| 3670 | instruments = orchestrator._dedupe_instruments([position]) |
| 3671 | assert instruments[0]["nseSymbol"] == "VERIFIED_NSE_SYMBOL" |
| 3672 | finally: |
| 3673 | import asyncio |
| 3674 | asyncio.run(orchestrator._client.aclose()) |
| 3675 | |
| 3676 | |
| 3677 | @pytest.mark.asyncio |
| 3678 | async def test_portfolio_research_summary_is_read_only_for_structured_market_provider() -> None: |
| 3679 | class MustNotCollect: |
| 3680 | def __init__(self) -> None: |
| 3681 | self.calls = 0 |
| 3682 | |
| 3683 | async def collect(self, _instrument): |
| 3684 | self.calls += 1 |
| 3685 | raise AssertionError("portfolio summary must not invoke a live structured provider") |
| 3686 | |
| 3687 | repository = ResearchRepository(settings=Settings(research_live_enabled=False, research_demo_enabled=False)) |
| 3688 | global_instrument_id = repository.profiles[0].instrument_id |
| 3689 | position = _portfolio_position("DE000A0WMPJ6", "AIXA", "XETR", "AIXTRON SE") |
| 3690 | position["instrument"]["globalInstrumentId"] = str(global_instrument_id) |
| 3691 | provider = MustNotCollect() |
| 3692 | orchestrator = PortfolioResearchOrchestrator( |
| 3693 | repository, Settings(portfolio_service_base_url="http://portfolio-service"), |
| 3694 | client=_RecordingPortfolioClient([position]), structured_provider=provider, |
| 3695 | ) |
| 3696 | |
| 3697 | await orchestrator.read_portfolio_summary(UUID("aaaaaaaa-1111-1111-1111-111111111111")) |
| 3698 | |
| 3699 | assert provider.calls == 0 |
| 3700 | |
| 3701 | |
| 3702 | @pytest.mark.asyncio |
| 3703 | @pytest.mark.parametrize( |
| 3704 | ("broker_symbol", "verified_nse_symbol", "isin"), |
| 3705 | [ |
| 3706 | ("INDR", "IRFC", "INE053F01010"), |
| 3707 | ("CHOINV", "CHOLAFIN", "INE121A01024"), |
| 3708 | ], |
| 3709 | ) |
| 3710 | async def test_portfolio_refresh_uses_verified_nse_mapping_for_official_research_without_static_source( |
| 3711 | broker_symbol: str, verified_nse_symbol: str, isin: str |
| 3712 | ) -> None: |
| 3713 | class RecordingRefreshRepository(ResearchRepository): |
| 3714 | def __init__(self) -> None: |
| 3715 | super().__init__(settings=Settings(research_live_enabled=True, research_search_enabled=False, research_demo_enabled=False)) |
| 3716 | self.refreshed_profiles: list[CompanyResearchProfile] = [] |
| 3717 | |
| 3718 | async def refresh(self, instrument_id: UUID, **_kwargs): |
| 3719 | self.refreshed_profiles.append(self.profile(instrument_id)) |
| 3720 | return self.summary(instrument_id, allow_demo=False) |
| 3721 | |
| 3722 | repository = RecordingRefreshRepository() |
| 3723 | global_instrument_id = UUID(int=uuid_int_from_text(f"{isin}|global")) |
| 3724 | position = _portfolio_position(isin, broker_symbol, "NSE", "Generic Indian Equity", provider="ICICI_DIRECT", |
| 3725 | provider_instrument_id=f"ISIN:{isin}", data_freshness="REAL_BROKER") |
| 3726 | position["instrument"].update({ |
| 3727 | "globalInstrumentId": str(global_instrument_id), |
| 3728 | "country": "IN", |
| 3729 | "providerMappings": [ |
| 3730 | {"provider": "ICICI_DIRECT", "providerSymbol": broker_symbol, "status": "VERIFIED", "resolutionSource": "LEGACY_ADOPTION"}, |
| 3731 | {"provider": "NSE", "providerSymbol": broker_symbol, "status": "INVALID", "resolutionSource": "BROKER_IMPORT_IDENTITY"}, |
| 3732 | {"provider": "NSE", "providerSymbol": verified_nse_symbol, "status": "VERIFIED", "resolutionSource": "NSE_OFFICIAL_ISIN_BOOTSTRAP"}, |
| 3733 | ], |
| 3734 | }) |
| 3735 | orchestrator = PortfolioResearchOrchestrator( |
| 3736 | repository, Settings(portfolio_service_base_url="http://portfolio-service", structured_provider_enabled=False, |
| 3737 | research_live_enabled=True, research_search_enabled=False, research_demo_enabled=False), |
| 3738 | client=_RecordingPortfolioClient([position]), structured_provider=_UnavailableStructuredProvider(), |
| 3739 | ) |
| 3740 | |
| 3741 | await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-111111111111")) |
| 3742 | |
| 3743 | assert [profile.provider_instrument_ids["NSE"] for profile in repository.refreshed_profiles] == [verified_nse_symbol] |
| 3744 | |
| 3745 | |
| 3746 | def test_scanned_report_is_retained_as_document_level_status_without_erasing_other_research() -> None: |
| 3747 | scanned = _document("https://www.nseindia.com/scanned.pdf", "").model_copy(update={ |
| 3748 | "document_type": DocumentType.PDF_REFERENCE, "status": DocumentStatus.FAILED, |
| 3749 | "source_classification": SourceClassification.EXCHANGE, |
| 3750 | }) |
| 3751 | company = PortfolioResearchCompany(company_name="Example", status="RESOLVED_PARTIAL_DATA") |
| 3752 | enrich_company_research(company, [scanned], [], Decimal("0.10")) |
| 3753 | assert company.quarterly_result_status == "PDF_SCANNED_OCR_REQUIRED" |
| 3754 | assert company.status == "RESOLVED_PARTIAL_DATA" |
| 3755 | |
| 3756 | |
| 3757 | def _quarterly_fact(instrument_id: UUID, metric: str, value: str, period: str = "Q1 FY27") -> FinancialFact: |
| 3758 | return FinancialFact( |
| 3759 | FinancialFactKey(instrument_id, metric, period, "QUARTERLY", "CONSOLIDATED"), |
| 3760 | ProvenancedValue( |
| 3761 | value=Decimal(value), unit="INR crore" if metric != "eps" else "INR per share", |
| 3762 | source_url="https://nsearchives.nseindia.com/result.pdf", source_name="NSE", |
| 3763 | source_type="EXCHANGE_ANNOUNCEMENT", retrieved_at=datetime(2026, 8, 1, tzinfo=timezone.utc), |
| 3764 | ), |
| 3765 | FactSourceTier.OFFICIAL_NSE, "NSE", "document-id", SourceMode.REAL, |
| 3766 | ) |
| 3767 | |
| 3768 | |
| 3769 | def _selection_fact( |
| 3770 | instrument_id: UUID, |
| 3771 | metric: str, |
| 3772 | value: str, |
| 3773 | *, |
| 3774 | period: str, |
| 3775 | basis: str | None, |
| 3776 | tier: FactSourceTier, |
| 3777 | provider: str, |
| 3778 | period_type: str = "QUARTERLY", |
| 3779 | ) -> FinancialFact: |
| 3780 | source_name = "NSE" if tier == FactSourceTier.OFFICIAL_NSE else "Yahoo Finance" |
| 3781 | return FinancialFact( |
| 3782 | FinancialFactKey(instrument_id, metric, period, period_type, basis), |
| 3783 | ProvenancedValue( |
| 3784 | value=Decimal(value), |
| 3785 | unit="INR per share" if metric == "eps" else "INR crore", |
| 3786 | source_url=f"https://example.test/{provider.lower()}/{period}/{metric}", |
| 3787 | source_name=source_name, |
| 3788 | source_type="EXCHANGE_ANNOUNCEMENT" if tier == FactSourceTier.OFFICIAL_NSE else "STRUCTURED_FINANCIAL_STATEMENTS", |
| 3789 | published_at=datetime(2026, 8, 1, tzinfo=timezone.utc), |
| 3790 | retrieved_at=datetime(2026, 8, 2, tzinfo=timezone.utc), |
| 3791 | confidence=0.91 if tier == FactSourceTier.OFFICIAL_NSE else 0.78, |
| 3792 | ), |
| 3793 | tier, |
| 3794 | provider, |
| 3795 | f"{provider}:{period}:{metric}", |
| 3796 | SourceMode.REAL, |
| 3797 | ) |
| 3798 | |
| 3799 | |
| 3800 | def test_latest_result_policy_newer_official_partial_period_beats_older_complete_yahoo() -> None: |
| 3801 | instrument_id = UUID("99999999-9999-9999-9999-999999999994") |
| 3802 | facts = [ |
| 3803 | _selection_fact(instrument_id, "revenue", "120", period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3804 | _selection_fact(instrument_id, "eps", "2.4", period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3805 | *[_selection_fact(instrument_id, metric, value, period="2026-03-31", basis="UNKNOWN", tier=FactSourceTier.YAHOO, provider="YAHOO_FINANCE") |
| 3806 | for metric, value in (("revenue", "100"), ("pat", "10"), ("eps", "2.0"))], |
| 3807 | ] |
| 3808 | |
| 3809 | result = latest_quarterly_result_from_facts(facts) |
| 3810 | |
| 3811 | assert result is not None |
| 3812 | assert result.period == "2026-06-30" |
| 3813 | assert result.revenue.value == Decimal("120") |
| 3814 | assert result.eps.value == Decimal("2.4") |
| 3815 | assert result.pat is None |
| 3816 | |
| 3817 | |
| 3818 | def test_latest_result_policy_composes_compatible_unknown_basis_fields_with_field_provenance() -> None: |
| 3819 | instrument_id = UUID("99999999-9999-9999-9999-999999999995") |
| 3820 | facts = [ |
| 3821 | _selection_fact(instrument_id, "revenue", "120", period="2026-06-30", basis="UNKNOWN", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3822 | _selection_fact(instrument_id, "pat", "12", period="2026-06-30", basis="UNKNOWN", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3823 | _selection_fact(instrument_id, "eps", "2.4", period="2026-06-30", basis="UNKNOWN", tier=FactSourceTier.YAHOO, provider="YAHOO_FINANCE"), |
| 3824 | ] |
| 3825 | |
| 3826 | result = latest_quarterly_result_from_facts(facts) |
| 3827 | |
| 3828 | assert result is not None |
| 3829 | assert result.period == "2026-06-30" |
| 3830 | assert result.revenue.source_name == "NSE" |
| 3831 | assert result.pat.source_name == "NSE" |
| 3832 | assert result.eps.source_name == "Yahoo Finance" |
| 3833 | assert result.eps.source_url.endswith("/yahoo_finance/2026-06-30/eps") |
| 3834 | |
| 3835 | |
| 3836 | def test_latest_result_policy_does_not_mix_yahoo_unknown_eps_into_official_consolidated_period() -> None: |
| 3837 | instrument_id = UUID("99999999-9999-9999-9999-999999999996") |
| 3838 | facts = [ |
| 3839 | _selection_fact(instrument_id, "revenue", "120", period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3840 | _selection_fact(instrument_id, "pat", "12", period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3841 | _selection_fact(instrument_id, "eps", "2.4", period="2026-06-30", basis="UNKNOWN", tier=FactSourceTier.YAHOO, provider="YAHOO_FINANCE"), |
| 3842 | ] |
| 3843 | |
| 3844 | result = latest_quarterly_result_from_facts(facts) |
| 3845 | |
| 3846 | assert result is not None |
| 3847 | assert result.period == "2026-06-30" |
| 3848 | assert result.reporting_basis == "CONSOLIDATED" |
| 3849 | assert result.revenue.source_name == "NSE" |
| 3850 | assert result.pat.source_name == "NSE" |
| 3851 | assert result.eps is None |
| 3852 | |
| 3853 | |
| 3854 | def test_latest_result_policy_newer_partial_official_beats_older_complete_official() -> None: |
| 3855 | instrument_id = UUID("99999999-9999-9999-9999-999999999997") |
| 3856 | facts = [ |
| 3857 | _selection_fact(instrument_id, "revenue", "120", period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3858 | *[_selection_fact(instrument_id, metric, value, period="2026-03-31", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE") |
| 3859 | for metric, value in (("revenue", "100"), ("pat", "10"), ("eps", "2.0"))], |
| 3860 | ] |
| 3861 | |
| 3862 | result = latest_quarterly_result_from_facts(facts) |
| 3863 | |
| 3864 | assert result is not None |
| 3865 | assert result.period == "2026-06-30" |
| 3866 | assert result.revenue.value == Decimal("120") |
| 3867 | assert result.pat is None |
| 3868 | assert result.eps is None |
| 3869 | |
| 3870 | |
| 3871 | def test_latest_result_policy_earnings_event_does_not_advance_normalized_fact_period() -> None: |
| 3872 | instrument_id = UUID("99999999-9999-9999-9999-999999999998") |
| 3873 | facts = [_selection_fact(instrument_id, metric, value, period="2026-03-31", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE") |
| 3874 | for metric, value in (("revenue", "100"), ("pat", "10"), ("eps", "2.0"))] |
| 3875 | earnings_document = _document("https://www.nseindia.com/newer-earnings.pdf", "Earnings release.").model_copy(update={"instrument_id": instrument_id}) |
| 3876 | earnings_event = ResearchEvent( |
| 3877 | instrument_id=instrument_id, |
| 3878 | company_id=UUID("99999999-9999-9999-9999-999999999997"), |
| 3879 | event_type=ResearchEventType.EARNINGS_RELEASE, |
| 3880 | event_date=datetime(2026, 6, 30, tzinfo=timezone.utc), |
| 3881 | title="Newer earnings release", |
| 3882 | summary="Earnings release evidence only.", |
| 3883 | source_document_id=earnings_document.document_id, |
| 3884 | source_url=earnings_document.canonical_url, |
| 3885 | source_type=SourceType.EXCHANGE_ANNOUNCEMENT, |
| 3886 | source_classification=SourceClassification.EXCHANGE, |
| 3887 | reliability=ReliabilityLevel.LEVEL_A, |
| 3888 | source_mode=SourceMode.REAL, |
| 3889 | confidence=0.9, |
| 3890 | impact=EventImpact.NEUTRAL, |
| 3891 | time_horizon=TimeHorizon.IMMEDIATE, |
| 3892 | raw_evidence_reference="Quarterly results announced for the quarter ended 30 June 2026.", |
| 3893 | published_at=datetime(2026, 8, 1, tzinfo=timezone.utc), |
| 3894 | retrieved_at=datetime(2026, 8, 1, tzinfo=timezone.utc), |
| 3895 | ) |
| 3896 | company = PortfolioResearchCompany( |
| 3897 | instrument_id=instrument_id, company_name="Example", status="RESOLVED_RESEARCH_AVAILABLE", |
| 3898 | ) |
| 3899 | |
| 3900 | enrich_company_research(company, [earnings_document], [earnings_event], Decimal("0.10"), facts) |
| 3901 | |
| 3902 | assert earnings_event.event_type == ResearchEventType.EARNINGS_RELEASE |
| 3903 | assert company.latest_quarterly_result is not None |
| 3904 | assert company.latest_quarterly_result.period == "2026-03-31" |
| 3905 | |
| 3906 | |
| 3907 | def test_latest_result_policy_financial_institution_does_not_interpret_non_result_metrics() -> None: |
| 3908 | instrument_id = UUID("99999999-9999-9999-9999-999999999999") |
| 3909 | facts = [ |
| 3910 | *[_selection_fact(instrument_id, metric, value, period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE") |
| 3911 | for metric, value in (("revenue", "4"), ("pat", "1176.93"), ("eps", "19.15"))], |
| 3912 | _selection_fact(instrument_id, "debt_or_borrowings", "21710", period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3913 | _selection_fact(instrument_id, "roe", "171", period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3914 | ] |
| 3915 | |
| 3916 | result = latest_quarterly_result_from_facts(facts) |
| 3917 | |
| 3918 | assert result is not None |
| 3919 | assert result.revenue.value == Decimal("4") |
| 3920 | assert result.pat.value == Decimal("1176.93") |
| 3921 | assert result.eps.value == Decimal("19.15") |
| 3922 | assert result.debt_or_borrowings is None |
| 3923 | |
| 3924 | |
| 3925 | def test_financial_history_returns_newest_four_explicit_quarters_without_deleting_history() -> None: |
| 3926 | instrument_id = UUID("99999999-9999-9999-9999-999999999990") |
| 3927 | periods = ["2026-06-30", "2026-03-31", "2025-12-31", "2025-09-30", "2025-06-30"] |
| 3928 | facts = [_selection_fact(instrument_id, metric, value, period=period, basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE") |
| 3929 | for period in periods for metric, value in (("revenue", "100"), ("pat", "10"))] |
| 3930 | history = financial_result_history_from_facts(facts, period_type="QUARTERLY") |
| 3931 | assert [item.period for item in history] == periods[:4] |
| 3932 | assert len({fact.key.period_end for fact in facts}) == 5 |
| 3933 | latest = latest_quarterly_result_from_facts(facts) |
| 3934 | assert latest is not None and latest.period == history[0].period and latest.reporting_basis == history[0].reporting_basis |
| 3935 | |
| 3936 | |
| 3937 | def test_financial_history_returns_newest_four_explicit_annuals_without_basis_composition() -> None: |
| 3938 | instrument_id = UUID("99999999-9999-9999-9999-999999999989") |
| 3939 | annuals = ["2026-03-31", "2025-03-31", "2024-03-31", "2023-03-31", "2022-03-31"] |
| 3940 | facts = [_selection_fact(instrument_id, metric, value, period=period, basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE", period_type="ANNUAL") |
| 3941 | for period in annuals for metric, value in (("revenue", "100"), ("pat", "10"))] |
| 3942 | facts.extend(_selection_fact(instrument_id, "revenue", "9", period="2026-06-30", basis="STANDALONE", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE") for _ in [0]) |
| 3943 | history = financial_result_history_from_facts(facts, period_type="ANNUAL") |
| 3944 | assert [item.period for item in history] == annuals[:4] |
| 3945 | assert {item.reporting_basis for item in history} == {"CONSOLIDATED"} |
| 3946 | |
| 3947 | |
| 3948 | def test_financial_history_mixed_optional_bases_is_deterministic_and_never_composes_fields() -> None: |
| 3949 | instrument_id = UUID("99999999-9999-9999-9999-999999999988") |
| 3950 | facts = [ |
| 3951 | _selection_fact(instrument_id, "revenue", "120", period="2026-06-30", basis=None, tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3952 | _selection_fact(instrument_id, "pat", "12", period="2026-06-30", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3953 | _selection_fact(instrument_id, "revenue", "100", period="2026-03-31", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3954 | _selection_fact(instrument_id, "pat", "10", period="2026-03-31", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE"), |
| 3955 | ] |
| 3956 | history = financial_result_history_from_facts(facts, period_type="QUARTERLY") |
| 3957 | assert [item.period for item in history] == ["2026-06-30", "2026-03-31"] |
| 3958 | assert {item.reporting_basis for item in history} == {"CONSOLIDATED"} |
| 3959 | assert history[0].revenue is None |
| 3960 | assert history[0].pat.value == Decimal("12") |
| 3961 | |
| 3962 | |
| 3963 | def test_statement_history_projects_only_persisted_supported_metrics_without_basis_composition() -> None: |
| 3964 | instrument_id = UUID("99999999-9999-9999-9999-999999999987") |
| 3965 | facts = [ |
| 3966 | _selection_fact(instrument_id, "total_assets", "500", period="2026-03-31", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE", period_type="AS_AT"), |
| 3967 | _selection_fact(instrument_id, "cash_and_cash_equivalents", "80", period="2026-03-31", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE", period_type="AS_AT"), |
| 3968 | _selection_fact(instrument_id, "total_assets", "900", period="2026-03-31", basis="STANDALONE", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE", period_type="AS_AT"), |
| 3969 | _selection_fact(instrument_id, "operating_cash_flow", "100", period="2026-03-31", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE", period_type="ANNUAL"), |
| 3970 | _selection_fact(instrument_id, "pat", "10", period="2026-03-31", basis="CONSOLIDATED", tier=FactSourceTier.OFFICIAL_NSE, provider="NSE", period_type="ANNUAL"), |
| 3971 | ] |
| 3972 | balance = financial_statement_history_from_facts(facts, period_type="AS_AT", metrics={"total_assets", "cash_and_cash_equivalents"}) |
| 3973 | cash_flow = financial_statement_history_from_facts(facts, period_type="ANNUAL", metrics={"operating_cash_flow"}) |
| 3974 | |
| 3975 | assert len(balance) == 1 |
| 3976 | assert balance[0].reporting_basis == "CONSOLIDATED" |
| 3977 | assert balance[0].metrics["total_assets"].value == Decimal("500") |
| 3978 | assert balance[0].metrics["cash_and_cash_equivalents"].value == Decimal("80") |
| 3979 | assert len(cash_flow) == 1 |
| 3980 | assert cash_flow[0].metrics["operating_cash_flow"].value == Decimal("100") |
| 3981 | |
| 3982 | |
| 3983 | def test_enrichment_uses_complete_persisted_quarterly_facts_without_document_parser(monkeypatch) -> None: |
| 3984 | instrument_id = UUID("99999999-9999-9999-9999-999999999991") |
| 3985 | company = PortfolioResearchCompany(instrument_id=instrument_id, company_name="Example", status="RESOLVED_RESEARCH_AVAILABLE") |
| 3986 | facts = [_quarterly_fact(instrument_id, metric, value) for metric, value in ( |
| 3987 | ("revenue", "1000"), ("pat", "110"), ("eps", "2.5"), |
| 3988 | )] |
| 3989 | monkeypatch.setattr( |
| 3990 | structured_research, |
| 3991 | "latest_quarterly_result", |
| 3992 | lambda _documents: (_ for _ in ()).throw(AssertionError("document financial parser must be bypassed")), |
| 3993 | ) |
| 3994 | |
| 3995 | enrich_company_research(company, [_document("https://www.nseindia.com/large.pdf", "large normalized financial statement")], [], Decimal("0.10"), facts) |
| 3996 | |
| 3997 | assert company.latest_quarterly_result is not None |
| 3998 | assert company.latest_quarterly_result.period == "Q1 FY27" |
| 3999 | assert company.latest_quarterly_result.revenue.value == Decimal("1000") |
| 4000 | assert company.latest_quarterly_result.pat.value == Decimal("110") |
| 4001 | assert company.latest_quarterly_result.eps.value == Decimal("2.5") |
| 4002 | |
| 4003 | |
| 4004 | def test_enrichment_uses_document_financial_parser_when_persisted_facts_are_absent(monkeypatch) -> None: |
| 4005 | company = PortfolioResearchCompany(company_name="Example", status="RESOLVED_RESEARCH_AVAILABLE") |
| 4006 | document = _document("https://www.nseindia.com/q1.pdf", "Q1 FY27 revenue 1,000 PAT 110 EPS 2.5") |
| 4007 | original = structured_research.latest_quarterly_result |
| 4008 | calls = 0 |
| 4009 | |
| 4010 | def recording_parser(documents): |
| 4011 | nonlocal calls |
| 4012 | calls += 1 |
| 4013 | return original(documents) |
| 4014 | |
| 4015 | monkeypatch.setattr(structured_research, "latest_quarterly_result", recording_parser) |
| 4016 | enrich_company_research(company, [document], [], Decimal("0.10"), []) |
| 4017 | |
| 4018 | assert calls == 1 |
| 4019 | assert company.latest_quarterly_result is not None |
| 4020 | assert company.latest_quarterly_result.revenue.value == Decimal("1000") |
| 4021 | |
| 4022 | |
| 4023 | def test_partial_persisted_quarterly_facts_remain_primary_when_eps_is_missing(monkeypatch) -> None: |
| 4024 | instrument_id = UUID("99999999-9999-9999-9999-999999999992") |
| 4025 | company = PortfolioResearchCompany(instrument_id=instrument_id, company_name="Example", status="RESOLVED_RESEARCH_AVAILABLE") |
| 4026 | document = _document("https://www.nseindia.com/q1.pdf", "Q1 FY27 revenue 1,000 PAT 110 EPS 2.5") |
| 4027 | original = structured_research.latest_quarterly_result |
| 4028 | calls = 0 |
| 4029 | |
| 4030 | def recording_parser(documents): |
| 4031 | nonlocal calls |
| 4032 | calls += 1 |
| 4033 | return original(documents) |
| 4034 | |
| 4035 | monkeypatch.setattr(structured_research, "latest_quarterly_result", recording_parser) |
| 4036 | facts = [_quarterly_fact(instrument_id, "revenue", "999"), _quarterly_fact(instrument_id, "pat", "99")] |
| 4037 | enrich_company_research(company, [document], [], Decimal("0.10"), facts) |
| 4038 | |
| 4039 | assert calls == 0 |
| 4040 | assert company.latest_quarterly_result is not None |
| 4041 | assert company.latest_quarterly_result.revenue.value == Decimal("999") |
| 4042 | assert company.latest_quarterly_result.pat.value == Decimal("99") |
| 4043 | assert company.latest_quarterly_result.eps is None |
| 4044 | |
| 4045 | |
| 4046 | @pytest.mark.parametrize( |
| 4047 | ("status", "document_type"), |
| 4048 | [ |
| 4049 | (DocumentStatus.FAILED, DocumentType.PDF_REFERENCE), |
| 4050 | ("FAILED", "PDF_REFERENCE"), |
| 4051 | (DocumentStatus.FAILED, "PDF_REFERENCE"), |
| 4052 | ("FAILED", DocumentType.PDF_REFERENCE), |
| 4053 | ], |
| 4054 | ) |
| 4055 | def test_scanned_pdf_enrichment_accepts_enum_and_legacy_string_document_fields(status, document_type) -> None: |
| 4056 | base = _document("https://www.nseindia.com/scanned-legacy.pdf", "") |
| 4057 | legacy = base.model_dump() | {"status": status, "document_type": document_type} |
| 4058 | scanned = ResearchDocument.model_construct(**legacy) |
| 4059 | company = PortfolioResearchCompany(company_name="Example", status="RESOLVED_PARTIAL_DATA") |
| 4060 | |
| 4061 | enrich_company_research(company, [scanned], [], Decimal("0.10")) |
| 4062 | |
| 4063 | assert company.quarterly_result_status == "PDF_SCANNED_OCR_REQUIRED" |
| 4064 | |
| 4065 | |
| 4066 | def test_non_failed_or_non_pdf_legacy_document_fields_preserve_not_available_status() -> None: |
| 4067 | base = _document("https://www.nseindia.com/normal.html", "") |
| 4068 | processed_pdf = ResearchDocument.model_construct(**(base.model_dump() | {"status": "PROCESSED", "document_type": "PDF_REFERENCE"})) |
| 4069 | failed_html = ResearchDocument.model_construct(**(base.model_dump() | {"status": "FAILED", "document_type": "HTML"})) |
| 4070 | company = PortfolioResearchCompany(company_name="Example", status="RESOLVED_PARTIAL_DATA") |
| 4071 | |
| 4072 | enrich_company_research(company, [processed_pdf, failed_html], [], Decimal("0.10")) |
| 4073 | |
| 4074 | assert company.quarterly_result_status == "NOT_AVAILABLE" |
| 4075 | |
| 4076 | |
| 4077 | def test_portfolio_summary_finalization_accepts_legacy_string_document_fields() -> None: |
| 4078 | repository = ResearchRepository() |
| 4079 | instrument_id = UUID("99999999-9999-9999-9999-999999999998") |
| 4080 | base = _document("https://www.nseindia.com/finalized-scanned.pdf", "") |
| 4081 | scanned = ResearchDocument.model_construct(**(base.model_dump() | { |
| 4082 | "instrument_id": instrument_id, "status": "FAILED", "document_type": "PDF_REFERENCE" |
| 4083 | })) |
| 4084 | repository.documents[scanned.document_id] = scanned |
| 4085 | orchestrator = PortfolioResearchOrchestrator(repository, Settings()) |
| 4086 | summary = PortfolioResearchSummary( |
| 4087 | portfolio_id=UUID("99999999-9999-9999-9999-999999999997"), |
| 4088 | companies=[PortfolioResearchCompany( |
| 4089 | instrument_id=instrument_id, company_name="Example", asset_type="EQUITY", status="RESOLVED_PARTIAL_DATA" |
| 4090 | )], |
| 4091 | ) |
| 4092 | try: |
| 4093 | finalized = orchestrator._finalize(summary) |
| 4094 | finally: |
| 4095 | import asyncio |
| 4096 | asyncio.run(orchestrator._client.aclose()) |
| 4097 | |
| 4098 | assert finalized.companies[0].quarterly_result_status == "PDF_SCANNED_OCR_REQUIRED" |
| 4099 | |
| 4100 | |
| 4101 | def test_scanned_official_pdf_is_retained_against_global_instrument_even_without_extractable_metrics() -> None: |
| 4102 | repository = ResearchRepository(Settings(research_live_enabled=True)) |
| 4103 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 4104 | document = repository.ingest_fixture( |
| 4105 | original_url="https://nsearchives.nseindia.com/corporate/scanned.pdf", source_type=SourceType.EXCHANGE_ANNOUNCEMENT, |
| 4106 | source_name="NSE corporate announcements", publisher="NSE", content_type="application/pdf", body="", |
| 4107 | reliability=ReliabilityLevel.LEVEL_A, source_mode=SourceMode.REAL, source_classification=SourceClassification.EXCHANGE, |
| 4108 | expected_profile=profile, document_status=DocumentStatus.FAILED, allow_empty_content=True, |
| 4109 | ) |
| 4110 | assert document.instrument_id == profile.instrument_id |
| 4111 | assert document.status == DocumentStatus.FAILED |
| 4112 | assert document.document_id in repository.documents |
| 4113 | |
| 4114 | |
| 4115 | def test_bank_quarterly_result_extracts_bank_metrics_without_industrial_ebitda_assumptions() -> None: |
| 4116 | document = _document( |
| 4117 | "https://www.nseindia.com/ujjivan-results.pdf", |
| 4118 | "Q1 FY27 total income 2,000 PAT 300 EPS 2.5 NIM 8.2% ROA 1.4% ROE 15.6% Gross NPA 2.1% Net NPA 0.4% deposits 40,000 advances 35,000 capital adequacy ratio 24.1% credit cost 0.8%", |
| 4119 | ).model_copy(update={"source_classification": SourceClassification.EXCHANGE, "source_name": "NSE"}) |
| 4120 | result = latest_quarterly_result([document]) |
| 4121 | assert result is not None |
| 4122 | assert result.nim.value == Decimal("8.2") |
| 4123 | assert result.gross_npa.value == Decimal("2.1") |
| 4124 | assert result.capital_adequacy.value == Decimal("24.1") |
| 4125 | assert result.ebitda is None |
| 4126 | |
| 4127 | |
| 4128 | def test_shareholding_comparison_uses_compatible_periods_and_percentage_points() -> None: |
| 4129 | document = _document( |
| 4130 | "https://www.nseindia.com/shareholding/reliance", |
| 4131 | "Q1 FY27 Q4 FY26 promoter holding 50.20% 50.00% FII/FPI 22.35% 22.20% DII 12.00% 12.05% public holding 15.45% 15.75%", |
| 4132 | ).model_copy(update={"source_classification": SourceClassification.EXCHANGE, "source_name": "NSE"}) |
| 4133 | changes = {change.category: change for change in shareholding_changes([document])} |
| 4134 | assert changes["PROMOTER"].change_percentage_points == Decimal("0.20") |
| 4135 | assert changes["FII_FPI"].change_percentage_points == Decimal("0.15") |
| 4136 | assert changes["DII"].change_percentage_points == Decimal("-0.05") |
| 4137 | assert changes["PROMOTER"].current_period == "Q1 FY27" |
| 4138 | assert changes["PROMOTER"].previous_period == "Q4 FY26" |
| 4139 | assert changes["PROMOTER"].current.source_name == "NSE" |
| 4140 | |
| 4141 | |
| 4142 | def test_shareholding_ignores_unofficial_or_non_comparable_evidence() -> None: |
| 4143 | unofficial = _document("https://blog.example/ownership", "Q1 FY27 Q4 FY26 promoter holding 51% 50%") |
| 4144 | one_period = _document("https://www.nseindia.com/ownership", "Q1 FY27 promoter holding 51%") \ |
| 4145 | .model_copy(update={"source_classification": SourceClassification.EXCHANGE}) |
| 4146 | assert shareholding_changes([unofficial, one_period]) == [] |
| 4147 | |
| 4148 | |
| 4149 | def _structured_shareholding_snapshot( |
| 4150 | period: tuple[int, int, int], |
| 4151 | values: list[tuple[ShareholdingCategory, str]], |
| 4152 | *, |
| 4153 | source_identity: str, |
| 4154 | ) -> ShareholdingSnapshot: |
| 4155 | return ShareholdingSnapshot( |
| 4156 | instrument_id=UUID("99999999-9999-9999-9999-999999999993"), |
| 4157 | period_end=datetime(*period, tzinfo=timezone.utc), |
| 4158 | source_provider="NSE", |
| 4159 | source_type="NSE_SHAREHOLDING_XBRL", |
| 4160 | source_identity_key=source_identity, |
| 4161 | source_url=f"https://nsearchives.nseindia.com/corporate/xbrl/{source_identity}.xml", |
| 4162 | confidence=Decimal("0.90"), |
| 4163 | reliability_level=ReliabilityLevel.LEVEL_A, |
| 4164 | source_mode=SourceMode.REAL, |
| 4165 | values=[ |
| 4166 | ShareholdingSnapshotValue(category=category, percentage=Decimal(value)) |
| 4167 | for category, value in values |
| 4168 | ], |
| 4169 | ) |
| 4170 | |
| 4171 | |
| 4172 | def test_structured_shareholding_changes_use_latest_two_distinct_periods_without_aggregation() -> None: |
| 4173 | snapshots = [ |
| 4174 | _structured_shareholding_snapshot((2026, 6, 30), [ |
| 4175 | (ShareholdingCategory.PROMOTER, "82.90"), |
| 4176 | # A duplicate/nested source row is deliberately not summed. |
| 4177 | (ShareholdingCategory.PROMOTER, "57.68"), |
| 4178 | (ShareholdingCategory.FII_FPI, "8.10"), |
| 4179 | ], source_identity="jun-2026"), |
| 4180 | _structured_shareholding_snapshot((2026, 3, 31), [ |
| 4181 | (ShareholdingCategory.PROMOTER, "84.65"), |
| 4182 | (ShareholdingCategory.FII_FPI, "7.80"), |
| 4183 | ], source_identity="mar-2026"), |
| 4184 | _structured_shareholding_snapshot((2025, 12, 31), [ |
| 4185 | (ShareholdingCategory.PROMOTER, "86.36"), |
| 4186 | ], source_identity="dec-2025"), |
| 4187 | _structured_shareholding_snapshot((2025, 9, 30), [ |
| 4188 | (ShareholdingCategory.PROMOTER, "86.36"), |
| 4189 | ], source_identity="sep-2025"), |
| 4190 | # A second source for the current period must not become "previous". |
| 4191 | _structured_shareholding_snapshot((2026, 6, 30), [ |
| 4192 | (ShareholdingCategory.PROMOTER, "82.90"), |
| 4193 | ], source_identity="jun-2026-duplicate").model_copy(update={ |
| 4194 | "retrieved_at": datetime(2020, 1, 1, tzinfo=timezone.utc), |
| 4195 | }), |
| 4196 | ] |
| 4197 | |
| 4198 | changes = {change.category: change for change in shareholding_changes_from_snapshots(snapshots)} |
| 4199 | |
| 4200 | promoter = changes["PROMOTER"] |
| 4201 | assert promoter.current_period == "2026-06-30" |
| 4202 | assert promoter.previous_period == "2026-03-31" |
| 4203 | assert promoter.current.value == Decimal("82.90") |
| 4204 | assert promoter.previous.value == Decimal("84.65") |
| 4205 | assert promoter.change_percentage_points == Decimal("-1.75") |
| 4206 | assert promoter.current.source_url.endswith("jun-2026.xml") |
| 4207 | assert promoter.previous.source_url.endswith("mar-2026.xml") |
| 4208 | assert changes["FII_FPI"].change_percentage_points == Decimal("0.30") |
| 4209 | assert [ |
| 4210 | snapshot.values[0].percentage |
| 4211 | for snapshot in snapshots[:4] |
| 4212 | ] == [Decimal("82.90"), Decimal("84.65"), Decimal("86.36"), Decimal("86.36")] |
| 4213 | |
| 4214 | |
| 4215 | def test_structured_shareholding_one_or_no_period_uses_legacy_document_fallback() -> None: |
| 4216 | document = _document( |
| 4217 | "https://www.nseindia.com/shareholding/reliance", |
| 4218 | "Q1 FY27 Q4 FY26 promoter holding 50.20% 50.00%", |
| 4219 | ).model_copy(update={"source_classification": SourceClassification.EXCHANGE, "source_name": "NSE"}) |
| 4220 | snapshot = _structured_shareholding_snapshot( |
| 4221 | (2026, 6, 30), [(ShareholdingCategory.PROMOTER, "82.90")], source_identity="only-period", |
| 4222 | ) |
| 4223 | company = PortfolioResearchCompany( |
| 4224 | company_name="Example", status="RESOLVED_RESEARCH_AVAILABLE", shareholding_snapshots=[snapshot], |
| 4225 | ) |
| 4226 | |
| 4227 | enrich_company_research(company, [document], [], Decimal("0.10")) |
| 4228 | |
| 4229 | assert len(company.shareholding_changes) == 1 |
| 4230 | assert company.shareholding_changes[0].current_period == "Q1 FY27" |
| 4231 | assert company.shareholding_changes[0].previous_period == "Q4 FY26" |
| 4232 | |
| 4233 | no_snapshot_company = PortfolioResearchCompany(company_name="Example", status="RESOLVED_RESEARCH_AVAILABLE") |
| 4234 | enrich_company_research(no_snapshot_company, [document], [], Decimal("0.10")) |
| 4235 | assert no_snapshot_company.shareholding_changes == company.shareholding_changes |
| 4236 | |
| 4237 | |
| 4238 | def test_valuation_requires_context_and_source_diversity_counts_domains() -> None: |
| 4239 | official = _document("https://company.example/results", "Q1 FY27 current P/E 16 sector P/E 20 peer P/E 22 ROE 18%") |
| 4240 | exchange = _document("https://www.nseindia.com/results", "Q1 FY27 historical P/E 21").model_copy(update={"source_classification": SourceClassification.EXCHANGE}) |
| 4241 | duplicate = official.model_copy() |
| 4242 | assert valuation_assessment([official, exchange]).state == "CHEAP" |
| 4243 | assert valuation_assessment([_document("https://media.example/story", "Q1 FY27 current P/E 16")]).state == "UNKNOWN" |
| 4244 | diversity = source_diversity([official, exchange, duplicate]) |
| 4245 | assert diversity.sources_found == 2 |
| 4246 | assert diversity.domains_found == 2 |
| 4247 | assert diversity.exchange_sources == 1 |
| 4248 | |
| 4249 | |
| 4250 | @pytest.mark.parametrize(("current", "sector", "state"), [("10", "15", "CHEAP"), ("18", "20", "FAIR"), ("30", "20", "EXPENSIVE"), ("8", "10", "CHEAP"), ("8.01", "10", "FAIR"), ("11.99", "10", "FAIR"), ("12", "10", "EXPENSIVE")]) |
| 4251 | def test_valuation_state_evidence_preserves_existing_boundaries(current, sector, state): |
| 4252 | result = valuation_assessment([_document("https://sector.example/a", f"current P/E {current} sector P/E {sector}")]) |
| 4253 | assert result.state == state and result.state_evidence is not None |
| 4254 | assert result.state_evidence.primary_metric == "CURRENT_PE" |
| 4255 | assert result.state_evidence.benchmark_value == Decimal(sector) |
| 4256 | assert result.state_evidence.comparison_ratio == Decimal(current) / Decimal(sector) |
| 4257 | assert result.state_evidence.benchmarks[0].kind == "SECTOR_PE" |
| 4258 | assert result.state_evidence.benchmarks[0].value.source_url == "https://sector.example/a" |
| 4259 | |
| 4260 | |
| 4261 | def test_valuation_state_evidence_keeps_individual_benchmark_provenance_and_unknown_null(): |
| 4262 | sector = _document("https://sector.example/a", "current P/E 15 sector P/E 20") |
| 4263 | peer = _document("https://peer.example/b", "peer P/E 25") |
| 4264 | historical = _document("https://history.example/c", "historical P/E 30") |
| 4265 | result = valuation_assessment([sector, peer, historical]) |
| 4266 | assert result.state == "CHEAP" and result.state_evidence.benchmark_value == Decimal("25") |
| 4267 | assert result.state_evidence.comparison_ratio == Decimal("0.6") |
| 4268 | assert {(item.kind, item.value.source_url) for item in result.state_evidence.benchmarks} == {("SECTOR_PE", "https://sector.example/a"), ("PEER_PE", "https://peer.example/b"), ("HISTORICAL_PE", "https://history.example/c")} |
| 4269 | assert valuation_assessment([_document("https://x", "sector P/E 20")]).state_evidence is None |
| 4270 | assert valuation_assessment([_document("https://x", "current P/E 20 peer P/E 0")]).state_evidence is None |
| 4271 | |
| 4272 | |
| 4273 | @pytest.mark.parametrize(("text", "expected"), [ |
| 4274 | ("Return Metrics: RoA improved 22 bps YoY to 1.22% and RoE expanded 171 bps YoY to 12.01%.", "12.01"), |
| 4275 | ("ROE increased 150 bps YoY to 14.50%.", "14.50"), |
| 4276 | ("ROE declined 80 bps YoY to 11.20%.", "11.20"), |
| 4277 | ("ROE stood at 13.64%.", "13.64"), |
| 4278 | ("ROE expanded to 13.69%.", "13.69"), |
| 4279 | ]) |
| 4280 | def test_document_roe_prefers_percentage_level_over_basis_point_delta(text, expected) -> None: |
| 4281 | value = valuation_assessment([_document("https://nse.example/federal", text)]).roe |
| 4282 | assert value is not None and value.value == Decimal(expected) |
| 4283 | assert value.value not in {Decimal("171"), Decimal("150"), Decimal("80")} |
| 4284 | |
| 4285 | |
| 4286 | def test_category_freshness_uses_independent_configurable_windows() -> None: |
| 4287 | settings = Settings( |
| 4288 | research_quarterly_freshness_seconds=100, |
| 4289 | research_shareholding_freshness_seconds=200, |
| 4290 | research_catalyst_freshness_seconds=10, |
| 4291 | research_analyst_freshness_seconds=50, |
| 4292 | ) |
| 4293 | repository = ResearchRepository(settings=settings) |
| 4294 | instrument_id = repository.profiles[0].instrument_id |
| 4295 | now = datetime.now(timezone.utc) |
| 4296 | repository._category_refresh[(instrument_id, "FINANCIAL_RESULTS")] = now |
| 4297 | assert not repository._category_is_fresh(instrument_id, "FINANCIAL_RESULTS", now) |
| 4298 | document = _document("https://exchange.example/results", "Quarterly financial results revenue and profit").model_copy(update={ |
| 4299 | "instrument_id": instrument_id, "source_mode": SourceMode.REAL, "status": DocumentStatus.PROCESSED, |
| 4300 | "title": "Quarterly Financial Results", |
| 4301 | }) |
| 4302 | repository.documents[document.document_id] = document |
| 4303 | assert repository._category_is_fresh(instrument_id, "FINANCIAL_RESULTS", now) |
| 4304 | assert repository.documents[document.document_id] is document |
| 4305 | assert not repository._category_is_fresh(instrument_id, "FINANCIAL_RESULTS", now.replace(year=now.year + 1)) |
| 4306 | |
| 4307 | |
| 4308 | def test_quarterly_financial_result_eligibility_uses_explicit_reporting_period_window() -> None: |
| 4309 | repository = ResearchRepository(settings=Settings()) |
| 4310 | instrument_id = repository.profiles[0].instrument_id |
| 4311 | document = _document( |
| 4312 | "https://exchange.example/results", |
| 4313 | "Quarterly financial results for the quarter ended June 30, 2026 revenue 100 profit after tax 10", |
| 4314 | ).model_copy(update={ |
| 4315 | "instrument_id": instrument_id, |
| 4316 | "source_mode": SourceMode.REAL, |
| 4317 | "status": DocumentStatus.PROCESSED, |
| 4318 | "title": "Quarterly Financial Results", |
| 4319 | "retrieved_at": datetime(2026, 7, 15, tzinfo=timezone.utc), |
| 4320 | }) |
| 4321 | repository.documents[document.document_id] = document |
| 4322 | |
| 4323 | eligible, next_eligible = repository._category_is_eligible_to_check( |
| 4324 | instrument_id, "FINANCIAL_RESULTS", datetime(2026, 8, 15, tzinfo=timezone.utc) |
| 4325 | ) |
| 4326 | assert eligible is False |
| 4327 | assert next_eligible == datetime(2026, 9, 1, tzinfo=timezone.utc) |
| 4328 | eligible, _ = repository._category_is_eligible_to_check( |
| 4329 | instrument_id, "FINANCIAL_RESULTS", datetime(2026, 9, 1, tzinfo=timezone.utc) |
| 4330 | ) |
| 4331 | assert eligible is True |
| 4332 | |
| 4333 | |
| 4334 | @pytest.mark.asyncio |
| 4335 | async def test_failed_attempt_states_do_not_make_financial_results_fresh_or_skip_official_retry() -> None: |
| 4336 | class RecordingOfficialDiscovery: |
| 4337 | def __init__(self) -> None: |
| 4338 | self.calls: list[set[str]] = [] |
| 4339 | |
| 4340 | async def discover(self, _profile, categories, _seen_urls): |
| 4341 | self.calls.append(set(categories)) |
| 4342 | return [] |
| 4343 | |
| 4344 | official = RecordingOfficialDiscovery() |
| 4345 | repository = ResearchRepository(settings=Settings(research_search_enabled=False), official_filing_discovery=official) |
| 4346 | profile = next(profile for profile in repository.profiles if profile.exchange in {"NSE", "XNSE"}) |
| 4347 | now = datetime.now(timezone.utc) |
| 4348 | repository._category_refresh[(profile.instrument_id, "FINANCIAL_RESULTS")] = now |
| 4349 | for terminal in ("ZERO_CANDIDATES", "SEARCH_RETURNED_ZERO_RESULTS", "RESULTS_REJECTED", "DOCUMENT_FETCH_FAILED", |
| 4350 | "EXTRACTION_EMPTY", "SEARCH_PROVIDER_UNAVAILABLE", "SKIPPED"): |
| 4351 | repository.last_live_error[profile.instrument_id] = terminal |
| 4352 | assert not repository._category_is_fresh(profile.instrument_id, "FINANCIAL_RESULTS", now) |
| 4353 | |
| 4354 | await repository._refresh_targeted(profile, set()) |
| 4355 | |
| 4356 | assert any("FINANCIAL_RESULTS" in categories for categories in official.calls) |
| 4357 | assert not repository._category_is_fresh(profile.instrument_id, "FINANCIAL_RESULTS", datetime.now(timezone.utc)) |
| 4358 | |
| 4359 | |
| 4360 | def test_refresh_category_aliases_share_one_canonical_obligation() -> None: |
| 4361 | assert _canonical_refresh_category("Guidance") == "GUIDANCE" |
| 4362 | assert _canonical_refresh_category("GUIDANCE") == "GUIDANCE" |
| 4363 | assert _canonical_refresh_category("Customers") == "CLIENTS" |
| 4364 | assert _canonical_refresh_category("Orders & Backlog") == "ORDERS_BACKLOG" |
| 4365 | assert _canonical_refresh_category("Ownership") == "INSTITUTIONAL_ACTIVITY" |
| 4366 | assert _canonical_refresh_category("Regulatory") == "REGULATORY" |
| 4367 | assert _canonical_refresh_category("REGULATORY") == "REGULATORY" |
| 4368 | |
| 4369 | |
| 4370 | def test_successful_no_change_check_throttles_missing_category_without_faking_evidence() -> None: |
| 4371 | repository = ResearchRepository(settings=Settings()) |
| 4372 | instrument_id = repository.profiles[0].instrument_id |
| 4373 | now = datetime(2026, 8, 1, tzinfo=timezone.utc) |
| 4374 | |
| 4375 | repository._mark_successful_categories_checked(instrument_id, {"Guidance"}, now) |
| 4376 | |
| 4377 | assert not repository._category_has_qualifying_evidence(instrument_id, "GUIDANCE") |
| 4378 | assert not repository._category_is_fresh(instrument_id, "GUIDANCE", now) |
| 4379 | eligible, next_eligible = repository._category_is_eligible_to_check(instrument_id, "GUIDANCE", now) |
| 4380 | assert eligible is False |
| 4381 | assert next_eligible == now + timedelta(days=1) |
| 4382 | eligible, _ = repository._category_is_eligible_to_check(instrument_id, "GUIDANCE", now + timedelta(days=1)) |
| 4383 | assert eligible is True |
| 4384 | |
| 4385 | |
| 4386 | @pytest.mark.asyncio |
| 4387 | async def test_backfill_bypasses_due_gate_but_reuses_global_refresh_single_flight(monkeypatch) -> None: |
| 4388 | repository = ResearchRepository(settings=Settings(research_live_enabled=True, research_demo_enabled=False)) |
| 4389 | instrument_id = repository.profiles[0].instrument_id |
| 4390 | calls = 0 |
| 4391 | |
| 4392 | async def record_live(refreshed_instrument_id, _categories, *, force=False): |
| 4393 | nonlocal calls |
| 4394 | assert refreshed_instrument_id == instrument_id |
| 4395 | assert force is True |
| 4396 | calls += 1 |
| 4397 | await asyncio.sleep(0) |
| 4398 | |
| 4399 | monkeypatch.setattr(repository, "_refresh_live", record_live) |
| 4400 | await asyncio.gather(repository.backfill(instrument_id), repository.backfill(instrument_id)) |
| 4401 | |
| 4402 | assert calls == 1 |
| 4403 | |
| 4404 | |
| 4405 | @pytest.mark.asyncio |
| 4406 | async def test_targeted_refresh_constrains_official_discovery_to_precomputed_due_categories(monkeypatch) -> None: |
| 4407 | class RecordingOfficialDiscovery: |
| 4408 | def __init__(self) -> None: |
| 4409 | self.calls: list[set[str]] = [] |
| 4410 | |
| 4411 | async def discover(self, _profile, categories, _seen_urls): |
| 4412 | self.calls.append(set(categories)) |
| 4413 | return [] |
| 4414 | |
| 4415 | official = RecordingOfficialDiscovery() |
| 4416 | repository = ResearchRepository( |
| 4417 | settings=Settings(research_search_enabled=False), |
| 4418 | official_filing_discovery=official, |
| 4419 | ) |
| 4420 | profile = next(profile for profile in repository.profiles if profile.exchange in {"NSE", "XNSE"}) |
| 4421 | guidance_due = _InstrumentRefreshGate({"GUIDANCE"}, False, False, "INCOMPLETE") |
| 4422 | monkeypatch.setattr(repository, "_instrument_refresh_gate", lambda *_args, **_kwargs: guidance_due) |
| 4423 | |
| 4424 | await repository._refresh_targeted(profile, set()) |
| 4425 | |
| 4426 | # A stale/incomplete instrument does not authorize financial filing work |
| 4427 | # when FINANCIAL_RESULTS was filtered out before discovery. |
| 4428 | assert official.calls == [] |
| 4429 | |
| 4430 | |
| 4431 | @pytest.mark.asyncio |
| 4432 | async def test_financial_results_due_category_is_the_only_official_discovery_category(monkeypatch) -> None: |
| 4433 | class RecordingOfficialDiscovery: |
| 4434 | def __init__(self) -> None: |
| 4435 | self.calls: list[set[str]] = [] |
| 4436 | |
| 4437 | async def discover(self, _profile, categories, _seen_urls): |
| 4438 | self.calls.append(set(categories)) |
| 4439 | return [] |
| 4440 | |
| 4441 | official = RecordingOfficialDiscovery() |
| 4442 | repository = ResearchRepository( |
| 4443 | settings=Settings(research_search_enabled=False), |
| 4444 | official_filing_discovery=official, |
| 4445 | ) |
| 4446 | profile = next(profile for profile in repository.profiles if profile.exchange in {"NSE", "XNSE"}) |
| 4447 | financial_due = _InstrumentRefreshGate({"FINANCIAL_RESULTS"}, False, False, "INCOMPLETE") |
| 4448 | monkeypatch.setattr(repository, "_instrument_refresh_gate", lambda *_args, **_kwargs: financial_due) |
| 4449 | |
| 4450 | await repository._refresh_targeted(profile, set()) |
| 4451 | |
| 4452 | assert official.calls == [{"FINANCIAL_RESULTS"}] |
| 4453 | |
| 4454 | |
| 4455 | @pytest.mark.asyncio |
| 4456 | async def test_provider_degraded_zero_result_does_not_record_successful_no_change_check(monkeypatch) -> None: |
| 4457 | class Stats: |
| 4458 | candidate_count = 0 |
| 4459 | accepted_count = 0 |
| 4460 | rejected_reasons: dict[str, int] = {} |
| 4461 | provider_failure_count = 1 |
| 4462 | |
| 4463 | def reject(self, _reason: str) -> None: |
| 4464 | pass |
| 4465 | |
| 4466 | class DegradedSearchDiscovery: |
| 4467 | provider = type("Provider", (), {"provider_name": "fixture-search"})() |
| 4468 | |
| 4469 | def __init__(self) -> None: |
| 4470 | self.last_stats = Stats() |
| 4471 | |
| 4472 | async def discover(self, _profile, _categories, _seen_urls): |
| 4473 | return [] |
| 4474 | |
| 4475 | repository = ResearchRepository(settings=Settings( |
| 4476 | research_search_enabled=True, |
| 4477 | research_search_provider="searxng", |
| 4478 | research_search_endpoint="https://search.example/search", |
| 4479 | )) |
| 4480 | profile = repository.profiles[0] |
| 4481 | repository._search_discovery = DegradedSearchDiscovery() |
| 4482 | guidance_due = _InstrumentRefreshGate({"GUIDANCE"}, False, False, "INCOMPLETE") |
| 4483 | monkeypatch.setattr(repository, "_instrument_refresh_gate", lambda *_args, **_kwargs: guidance_due) |
| 4484 | |
| 4485 | await repository._refresh_targeted(profile, set()) |
| 4486 | |
| 4487 | assert (profile.instrument_id, "GUIDANCE") not in repository._category_successful_no_change_checks |
| 4488 | |
| 4489 | |
| 4490 | @pytest.mark.asyncio |
| 4491 | async def test_regulatory_successful_no_change_is_throttled_under_its_canonical_key(monkeypatch) -> None: |
| 4492 | class Stats: |
| 4493 | candidate_count = 0 |
| 4494 | accepted_count = 0 |
| 4495 | rejected_reasons: dict[str, int] = {} |
| 4496 | provider_failure_count = 0 |
| 4497 | |
| 4498 | def reject(self, _reason: str) -> None: |
| 4499 | pass |
| 4500 | |
| 4501 | class ZeroResultSearchDiscovery: |
| 4502 | provider = type("Provider", (), {"provider_name": "fixture-search"})() |
| 4503 | |
| 4504 | def __init__(self) -> None: |
| 4505 | self.last_stats = Stats() |
| 4506 | self.categories: list[set[str]] = [] |
| 4507 | |
| 4508 | async def discover(self, _profile, categories, _seen_urls): |
| 4509 | self.categories.append(set(categories)) |
| 4510 | return [] |
| 4511 | |
| 4512 | repository = ResearchRepository(settings=Settings( |
| 4513 | research_search_enabled=True, |
| 4514 | research_search_provider="searxng", |
| 4515 | research_search_endpoint="https://search.example/search", |
| 4516 | )) |
| 4517 | profile = repository.profiles[0] |
| 4518 | search = ZeroResultSearchDiscovery() |
| 4519 | repository._search_discovery = search |
| 4520 | regulatory_due = _InstrumentRefreshGate({"REGULATORY"}, False, False, "INCOMPLETE") |
| 4521 | monkeypatch.setattr(repository, "_instrument_refresh_gate", lambda *_args, **_kwargs: regulatory_due) |
| 4522 | |
| 4523 | await repository._refresh_targeted(profile, set()) |
| 4524 | |
| 4525 | assert search.categories == [{"Regulatory"}] |
| 4526 | checked_at = repository._category_successful_no_change_checks[(profile.instrument_id, "REGULATORY")] |
| 4527 | eligible, next_eligible = repository._category_is_eligible_to_check(profile.instrument_id, "Regulatory", checked_at) |
| 4528 | assert eligible is False |
| 4529 | assert next_eligible == checked_at + timedelta(days=1) |
| 4530 | |
| 4531 | |
| 4532 | @pytest.mark.asyncio |
| 4533 | async def test_regulatory_provider_failure_remains_immediately_retryable(monkeypatch) -> None: |
| 4534 | class Stats: |
| 4535 | candidate_count = 0 |
| 4536 | accepted_count = 0 |
| 4537 | rejected_reasons: dict[str, int] = {} |
| 4538 | provider_failure_count = 1 |
| 4539 | |
| 4540 | def reject(self, _reason: str) -> None: |
| 4541 | pass |
| 4542 | |
| 4543 | class FailedSearchDiscovery: |
| 4544 | provider = type("Provider", (), {"provider_name": "fixture-search"})() |
| 4545 | |
| 4546 | def __init__(self) -> None: |
| 4547 | self.last_stats = Stats() |
| 4548 | |
| 4549 | async def discover(self, _profile, _categories, _seen_urls): |
| 4550 | return [] |
| 4551 | |
| 4552 | repository = ResearchRepository(settings=Settings( |
| 4553 | research_search_enabled=True, |
| 4554 | research_search_provider="searxng", |
| 4555 | research_search_endpoint="https://search.example/search", |
| 4556 | )) |
| 4557 | profile = repository.profiles[0] |
| 4558 | repository._search_discovery = FailedSearchDiscovery() |
| 4559 | regulatory_due = _InstrumentRefreshGate({"REGULATORY"}, False, False, "INCOMPLETE") |
| 4560 | monkeypatch.setattr(repository, "_instrument_refresh_gate", lambda *_args, **_kwargs: regulatory_due) |
| 4561 | |
| 4562 | await repository._refresh_targeted(profile, set()) |
| 4563 | |
| 4564 | assert (profile.instrument_id, "REGULATORY") not in repository._category_successful_no_change_checks |
| 4565 | eligible, next_eligible = repository._category_is_eligible_to_check( |
| 4566 | profile.instrument_id, "REGULATORY", datetime.now(timezone.utc) |
| 4567 | ) |
| 4568 | assert eligible is True |
| 4569 | assert next_eligible is None |
| 4570 | |
| 4571 | |
| 4572 | @pytest.mark.asyncio |
| 4573 | async def test_portfolio_refresh_executes_public_research_once_for_duplicate_global_instrument(monkeypatch) -> None: |
| 4574 | repository = ResearchRepository(settings=Settings(research_live_enabled=True, research_search_enabled=False)) |
| 4575 | global_instrument_id = UUID("77777777-7777-7777-7777-777777777778") |
| 4576 | first = _portfolio_position("INE000K01002", "BROKER_ONE", "NSE", "Example Components Limited") |
| 4577 | second = _portfolio_position("INE000K01002", "BROKER_TWO", "NSE", "Example Components Limited") |
| 4578 | for position in (first, second): |
| 4579 | position["instrument"].update({ |
| 4580 | "globalInstrumentId": str(global_instrument_id), |
| 4581 | "country": "IN", |
| 4582 | "providerMappings": [{"provider": "NSE", "providerSymbol": "EXAMPLE", "status": "VERIFIED"}], |
| 4583 | }) |
| 4584 | calls: list[UUID] = [] |
| 4585 | |
| 4586 | async def record_refresh(instrument_id, **_kwargs): |
| 4587 | calls.append(instrument_id) |
| 4588 | return repository.summary(instrument_id, allow_demo=False) |
| 4589 | |
| 4590 | monkeypatch.setattr(repository, "refresh", record_refresh) |
| 4591 | orchestrator = PortfolioResearchOrchestrator( |
| 4592 | repository, |
| 4593 | Settings(portfolio_service_base_url="http://portfolio-service", research_live_enabled=True, research_search_enabled=False), |
| 4594 | client=_RecordingPortfolioClient([first, second]), |
| 4595 | structured_provider=_UnavailableStructuredProvider(), |
| 4596 | ) |
| 4597 | refresh_instrument_calls: list[UUID] = [] |
| 4598 | original_refresh_instrument = orchestrator.refresh_instrument |
| 4599 | |
| 4600 | async def record_orchestration_refresh(instrument_id: UUID, **kwargs): |
| 4601 | refresh_instrument_calls.append(instrument_id) |
| 4602 | return await original_refresh_instrument(instrument_id, **kwargs) |
| 4603 | |
| 4604 | monkeypatch.setattr(orchestrator, "refresh_instrument", record_orchestration_refresh) |
| 4605 | |
| 4606 | await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-111111111111")) |
| 4607 | |
| 4608 | assert calls == [global_instrument_id] |
| 4609 | assert refresh_instrument_calls == [global_instrument_id] |
| 4610 | |
| 4611 | |
| 4612 | @pytest.mark.asyncio |
| 4613 | @pytest.mark.parametrize(("concurrency", "expected_peak"), [(1, 1), (2, 2)]) |
| 4614 | async def test_portfolio_refresh_uses_bounded_instrument_concurrency_and_preserves_order(monkeypatch, concurrency, expected_peak) -> None: |
| 4615 | repository = ResearchRepository(settings=Settings(research_live_enabled=True, research_search_enabled=False)) |
| 4616 | positions = [] |
| 4617 | expected_ids: list[UUID] = [] |
| 4618 | for index in range(3): |
| 4619 | instrument_id = UUID(f"77777777-7777-7777-7777-7777777777{80 + index}") |
| 4620 | expected_ids.append(instrument_id) |
| 4621 | position = _portfolio_position(f"INE000K010{10 + index}", f"ALIAS{index}", "NSE", f"Example {index} Limited") |
| 4622 | position["instrument"].update({ |
| 4623 | "globalInstrumentId": str(instrument_id), |
| 4624 | "country": "IN", |
| 4625 | "providerMappings": [{"provider": "NSE", "providerSymbol": f"EXAMPLE{index}", "status": "VERIFIED"}], |
| 4626 | }) |
| 4627 | positions.append(position) |
| 4628 | active = 0 |
| 4629 | peak = 0 |
| 4630 | calls: list[UUID] = [] |
| 4631 | |
| 4632 | async def delayed_refresh(instrument_id, **_kwargs): |
| 4633 | nonlocal active, peak |
| 4634 | calls.append(instrument_id) |
| 4635 | active += 1 |
| 4636 | peak = max(peak, active) |
| 4637 | await asyncio.sleep(0.02) |
| 4638 | active -= 1 |
| 4639 | return repository.summary(instrument_id, allow_demo=False) |
| 4640 | |
| 4641 | monkeypatch.setattr(repository, "refresh", delayed_refresh) |
| 4642 | orchestrator = PortfolioResearchOrchestrator( |
| 4643 | repository, |
| 4644 | Settings( |
| 4645 | portfolio_service_base_url="http://portfolio-service", |
| 4646 | research_live_enabled=True, |
| 4647 | research_search_enabled=False, |
| 4648 | structured_provider_enabled=False, |
| 4649 | portfolio_refresh_instrument_concurrency=concurrency, |
| 4650 | ), |
| 4651 | client=_RecordingPortfolioClient(positions), |
| 4652 | structured_provider=_UnavailableStructuredProvider(), |
| 4653 | ) |
| 4654 | |
| 4655 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-111111111111")) |
| 4656 | |
| 4657 | assert peak == expected_peak |
| 4658 | assert calls == expected_ids |
| 4659 | assert [company.instrument_id for company in result.companies] == expected_ids |
| 4660 | |
| 4661 | |
| 4662 | @pytest.mark.asyncio |
| 4663 | async def test_portfolio_refresh_isolates_one_instrument_failure_from_other_workers(monkeypatch) -> None: |
| 4664 | repository = ResearchRepository(settings=Settings(research_live_enabled=True, research_search_enabled=False)) |
| 4665 | first_id = UUID("77777777-7777-7777-7777-777777777790") |
| 4666 | second_id = UUID("77777777-7777-7777-7777-777777777791") |
| 4667 | positions = [] |
| 4668 | for instrument_id, ticker, isin in ((first_id, "FAIL", "INE000K01020"), (second_id, "OK", "INE000K01021")): |
| 4669 | position = _portfolio_position(isin, ticker, "NSE", f"Example {ticker} Limited") |
| 4670 | position["instrument"].update({ |
| 4671 | "globalInstrumentId": str(instrument_id), "country": "IN", |
| 4672 | "providerMappings": [{"provider": "NSE", "providerSymbol": ticker, "status": "VERIFIED"}], |
| 4673 | }) |
| 4674 | positions.append(position) |
| 4675 | calls: list[UUID] = [] |
| 4676 | |
| 4677 | async def selectively_fail(instrument_id, **_kwargs): |
| 4678 | calls.append(instrument_id) |
| 4679 | if instrument_id == first_id: |
| 4680 | raise RuntimeError("fixture failure") |
| 4681 | return repository.summary(instrument_id, allow_demo=False) |
| 4682 | |
| 4683 | monkeypatch.setattr(repository, "refresh", selectively_fail) |
| 4684 | orchestrator = PortfolioResearchOrchestrator( |
| 4685 | repository, |
| 4686 | Settings(portfolio_service_base_url="http://portfolio-service", research_live_enabled=True, |
| 4687 | research_search_enabled=False, structured_provider_enabled=False, |
| 4688 | portfolio_refresh_instrument_concurrency=2), |
| 4689 | client=_RecordingPortfolioClient(positions), structured_provider=_UnavailableStructuredProvider(), |
| 4690 | ) |
| 4691 | |
| 4692 | result = await orchestrator.refresh_portfolio(UUID("aaaaaaaa-1111-1111-1111-111111111111")) |
| 4693 | |
| 4694 | assert calls == [first_id, second_id] |
| 4695 | assert result.companies[0].safe_error_code == "RuntimeError" |
| 4696 | assert result.companies[1].instrument_id == second_id |
| 4697 | |
| 4698 | |
| 4699 | def test_indian_etf_identity_is_classified_before_company_orchestration() -> None: |
| 4700 | assert _instrument_asset_type({"assetType": "EQUITY", "securityType": "STK", "ticker": "GOLDBEES", |
| 4701 | "canonicalName": "NIPPON INDIA ETF GOLD BEES"}) == "ETF" |
| 4702 | assert _instrument_asset_type({"assetType": "EQUITY", "securityType": "STK", "ticker": "HDFCN50", |
| 4703 | "companyName": "HDFC NIFTY Next 50 ETF"}) == "ETF" |
| 4704 | |
| 4705 | |
| 4706 | def test_dynamic_eu_legal_names_have_public_page_aliases() -> None: |
| 4707 | assert _company_aliases("Arcadis NV", "ARCAD") == ["Arcadis", "ARCAD"] |
| 4708 | assert _company_aliases("Aalberts N.V.", "AALB") == ["Aalberts", "AALB"] |
| 4709 | assert _company_aliases("RENK Group AG", "R3NK") == ["RENK Group", "R3NK"] |
| 4710 | |
| 4711 | |
| 4712 | def test_exchange_source_is_authoritative_without_being_a_company_domain() -> None: |
| 4713 | repository = ResearchRepository() |
| 4714 | profile = next(profile for profile in repository.profiles if profile.ticker == "RELIANCE") |
| 4715 | source = RegisteredResearchSource( |
| 4716 | source_id="nse-results", instrument_id=profile.instrument_id, |
| 4717 | url="https://www.nseindia.com/companies-listing/corporate-filings-financial-results", |
| 4718 | source_type=SourceType.EXCHANGE_ANNOUNCEMENT, source_classification=SourceClassification.EXCHANGE, |
| 4719 | source_name="NSE", publisher="NSE", reliability_level=ReliabilityLevel.LEVEL_A, |
| 4720 | priority=2, categories=("FINANCIAL_RESULTS",), |
| 4721 | ) |
| 4722 | repository._validate_registered_source(profile, source) |
| 4723 | |
| 4724 | |
| 4725 | def _pdf_network_fixture() -> NetworkFetchResult: |
| 4726 | return NetworkFetchResult( |
| 4727 | final_url="https://nsearchives.nseindia.com/corporate/fixture.pdf", |
| 4728 | status_code=200, |
| 4729 | content_type="application/pdf", |
| 4730 | content=b"%PDF-fixture", |
| 4731 | headers={}, |
| 4732 | is_redirect=False, |
| 4733 | encoding=None, |
| 4734 | headers_elapsed_ms=1, |
| 4735 | body_elapsed_ms=1, |
| 4736 | network_elapsed_ms=2, |
| 4737 | ) |
| 4738 | |
| 4739 | |
| 4740 | @pytest.mark.asyncio |
| 4741 | async def test_pdf_extraction_runs_off_event_loop_and_allows_event_loop_progress(monkeypatch) -> None: |
| 4742 | fetcher = HttpResearchFetcher(Settings(research_pdf_extraction_concurrency=1)) |
| 4743 | main_thread = threading.get_ident() |
| 4744 | worker_started = threading.Event() |
| 4745 | release_worker = threading.Event() |
| 4746 | worker_thread_ids: list[int] = [] |
| 4747 | |
| 4748 | def blocking_extract(_response, *, max_bytes=None): |
| 4749 | worker_thread_ids.append(threading.get_ident()) |
| 4750 | worker_started.set() |
| 4751 | release_worker.wait(timeout=2) |
| 4752 | return FetchResult("https://nsearchives.nseindia.com/corporate/fixture.pdf", 200, "application/pdf", "text", 10) |
| 4753 | |
| 4754 | monkeypatch.setattr(fetcher, "process_network_response", blocking_extract) |
| 4755 | task = asyncio.create_task(fetcher.process_network_response_async(_pdf_network_fixture(), extraction_timeout_seconds=1)) |
| 4756 | while not worker_started.is_set(): |
| 4757 | await asyncio.sleep(0) |
| 4758 | progressed = False |
| 4759 | await asyncio.sleep(0) |
| 4760 | progressed = True |
| 4761 | assert progressed is True |
| 4762 | assert worker_thread_ids == [worker_thread_ids[0]] |
| 4763 | assert worker_thread_ids[0] != main_thread |
| 4764 | release_worker.set() |
| 4765 | assert (await task).text == "text" |
| 4766 | |
| 4767 | |
| 4768 | @pytest.mark.asyncio |
| 4769 | async def test_timed_out_pdf_thread_retains_extraction_permit_until_actual_completion(monkeypatch) -> None: |
| 4770 | fetcher = HttpResearchFetcher(Settings(research_pdf_extraction_concurrency=1)) |
| 4771 | first_started = threading.Event() |
| 4772 | release_first = threading.Event() |
| 4773 | calls = 0 |
| 4774 | active = 0 |
| 4775 | peak = 0 |
| 4776 | lock = threading.Lock() |
| 4777 | |
| 4778 | def blocking_first_then_fast(_response, *, max_bytes=None): |
| 4779 | nonlocal calls, active, peak |
| 4780 | with lock: |
| 4781 | calls += 1 |
| 4782 | ordinal = calls |
| 4783 | active += 1 |
| 4784 | peak = max(peak, active) |
| 4785 | try: |
| 4786 | if ordinal == 1: |
| 4787 | first_started.set() |
| 4788 | release_first.wait(timeout=2) |
| 4789 | return FetchResult("https://nsearchives.nseindia.com/corporate/fixture.pdf", 200, "application/pdf", "text", 10) |
| 4790 | finally: |
| 4791 | with lock: |
| 4792 | active -= 1 |
| 4793 | |
| 4794 | monkeypatch.setattr(fetcher, "process_network_response", blocking_first_then_fast) |
| 4795 | first = asyncio.create_task(fetcher.process_network_response_async(_pdf_network_fixture(), extraction_timeout_seconds=0.01)) |
| 4796 | while not first_started.is_set(): |
| 4797 | await asyncio.sleep(0) |
| 4798 | with pytest.raises(PdfExtractionTimeoutError, match="PDF_EXTRACTION_TIMEOUT"): |
| 4799 | await first |
| 4800 | second = asyncio.create_task(fetcher.process_network_response_async(_pdf_network_fixture(), extraction_timeout_seconds=1)) |
| 4801 | await asyncio.sleep(0.03) |
| 4802 | assert calls == 1 |
| 4803 | assert peak == 1 |
| 4804 | release_first.set() |
| 4805 | assert (await second).text == "text" |
| 4806 | assert calls == 2 |
| 4807 | assert peak == 1 |
| 4808 | |
| 4809 | |
| 4810 | @pytest.mark.asyncio |
| 4811 | async def test_pdf_extraction_concurrency_two_never_runs_more_than_two_workers(monkeypatch) -> None: |
| 4812 | fetcher = HttpResearchFetcher(Settings(research_pdf_extraction_concurrency=2)) |
| 4813 | started = threading.Event() |
| 4814 | release = threading.Event() |
| 4815 | lock = threading.Lock() |
| 4816 | active = 0 |
| 4817 | peak = 0 |
| 4818 | |
| 4819 | def blocking_extract(_response, *, max_bytes=None): |
| 4820 | nonlocal active, peak |
| 4821 | with lock: |
| 4822 | active += 1 |
| 4823 | peak = max(peak, active) |
| 4824 | if active == 2: |
| 4825 | started.set() |
| 4826 | release.wait(timeout=2) |
| 4827 | with lock: |
| 4828 | active -= 1 |
| 4829 | return FetchResult("https://nsearchives.nseindia.com/corporate/fixture.pdf", 200, "application/pdf", "text", 10) |
| 4830 | |
| 4831 | monkeypatch.setattr(fetcher, "process_network_response", blocking_extract) |
| 4832 | tasks = [asyncio.create_task(fetcher.process_network_response_async(_pdf_network_fixture(), extraction_timeout_seconds=1)) for _ in range(3)] |
| 4833 | while not started.is_set(): |
| 4834 | await asyncio.sleep(0) |
| 4835 | await asyncio.sleep(0.02) |
| 4836 | assert peak == 2 |
| 4837 | release.set() |
| 4838 | await asyncio.gather(*tasks) |
| 4839 | assert peak == 2 |
| 4840 | |
| 4841 | |
| 4842 | @pytest.mark.asyncio |
| 4843 | async def test_cancelled_pdf_caller_keeps_permit_until_worker_exits(monkeypatch) -> None: |
| 4844 | fetcher = HttpResearchFetcher(Settings(research_pdf_extraction_concurrency=1)) |
| 4845 | started = threading.Event() |
| 4846 | release = threading.Event() |
| 4847 | calls = 0 |
| 4848 | |
| 4849 | def blocking_first_then_fast(_response, *, max_bytes=None): |
| 4850 | nonlocal calls |
| 4851 | calls += 1 |
| 4852 | if calls == 1: |
| 4853 | started.set() |
| 4854 | release.wait(timeout=2) |
| 4855 | return FetchResult("https://nsearchives.nseindia.com/corporate/fixture.pdf", 200, "application/pdf", "text", 10) |
| 4856 | |
| 4857 | monkeypatch.setattr(fetcher, "process_network_response", blocking_first_then_fast) |
| 4858 | first = asyncio.create_task(fetcher.process_network_response_async(_pdf_network_fixture(), extraction_timeout_seconds=1)) |
| 4859 | while not started.is_set(): |
| 4860 | await asyncio.sleep(0) |
| 4861 | first.cancel() |
| 4862 | with pytest.raises(asyncio.CancelledError): |
| 4863 | await first |
| 4864 | second = asyncio.create_task(fetcher.process_network_response_async(_pdf_network_fixture(), extraction_timeout_seconds=1)) |
| 4865 | await asyncio.sleep(0.02) |
| 4866 | assert calls == 1 |
| 4867 | release.set() |
| 4868 | assert (await second).text == "text" |
| 4869 | assert calls == 2 |
| 4870 | |
| 4871 | |
| 4872 | @pytest.mark.asyncio |
| 4873 | async def test_pdf_network_download_is_not_serialized_by_extraction_admission(monkeypatch) -> None: |
| 4874 | fetcher = HttpResearchFetcher(Settings(research_pdf_extraction_concurrency=1)) |
| 4875 | downloads_active = 0 |
| 4876 | download_peak = 0 |
| 4877 | |
| 4878 | async def concurrent_network(_url, **_kwargs): |
| 4879 | nonlocal downloads_active, download_peak |
| 4880 | downloads_active += 1 |
| 4881 | download_peak = max(download_peak, downloads_active) |
| 4882 | await asyncio.sleep(0.01) |
| 4883 | downloads_active -= 1 |
| 4884 | return _pdf_network_fixture() |
| 4885 | |
| 4886 | monkeypatch.setattr(fetcher, "fetch_network", concurrent_network) |
| 4887 | monkeypatch.setattr(fetcher, "process_network_response", lambda response, *, max_bytes=None: FetchResult(response.final_url, 200, "application/pdf", "text", 10)) |
| 4888 | await asyncio.gather(fetcher.fetch("https://example.test/one.pdf"), fetcher.fetch("https://example.test/two.pdf")) |
| 4889 | assert download_peak == 2 |
| 4890 | |
| 4891 | |
| 4892 | def _document(url: str, text: str) -> ResearchDocument: |
| 4893 | return ResearchDocument( |
| 4894 | canonical_url=canonicalize_url(url), |
| 4895 | original_url=url, |
| 4896 | title="Fixture", |
| 4897 | source_type=SourceType.INVESTOR_RELATIONS, |
| 4898 | source_name="Fixture", |
| 4899 | publisher="DEMO", |
| 4900 | published_at=datetime(2026, 1, 1, tzinfo=timezone.utc), |
| 4901 | content_type="text/html", |
| 4902 | document_type="HTML", |
| 4903 | normalized_text=text, |
| 4904 | content_hash=content_hash(text), |
| 4905 | status=DocumentStatus.PARSED, |
| 4906 | reliability_level=ReliabilityLevel.LEVEL_B, |
| 4907 | ) |
| 4908 | |
| 4909 | |
| 4910 | def _aixtron_live_fixture() -> str: |
| 4911 | return """ |
| 4912 | <html> |
| 4913 | <head><title>Strong momentum in optoelectronics continues</title></head> |
| 4914 | <body> |
| 4915 | <main> |
| 4916 | <p>Herzogenrath, April 14, 2026</p> |
| 4917 | <p>AIXTRON SE AIXA XETR DE000A0WMPJ6 announced a new order worth EUR 350 million from a leading optoelectronics customer.</p> |
| 4918 | <p>The company will invest EUR 120 million in CAPEX to expand production capacity and confirms guidance for the fiscal year.</p> |
| 4919 | </main> |
| 4920 | </body> |
| 4921 | </html> |
| 4922 | """ |
| 4923 | |
| 4924 | |
| 4925 | def _aixtron_mojibake_fixture() -> str: |
| 4926 | return """ |
| 4927 | <html> |
| 4928 | <head><title>Strong momentum in optoelectronics continues</title></head> |
| 4929 | <body> |
| 4930 | <nav>AIXTRON Press Information & Releases :: AIXTRON Navigation Suche EN German English Facebook Instagram linkedIn Xing Close search Search / HOME / press / Press Releases Annual Report</nav> |
| 4931 | <main> |
| 4932 | <h1>Strong momentum in optoelectronics continues</h1> |
| 4933 | <p>Order intake in H1 up 54% yoy / Q2 results in line with guidance / Strong free cash-flow generation / Volume ramp fully on track / Raised fullâyear 2026 guidance confirmed</p> |
| 4934 | <p>Herzogenrath, Germany, July 30, 2026 - AIXTRON SE (FSE: AIXA, ISIN DE000A0WMPJ6) benefitted from a strong order intake of EUR 214.5 million (+81% yoy) in the second quarter 2026, compared with EUR 284.6 million a year earlier and from EUR 257 million in the prior period.</p> |
| 4935 | <p>To serve all customers with shipments at their requested delivery dates, the company is now ramping up production capacity at its own premises and in close collaboration with its suppliers.</p> |
| 4936 | <p>The Executive Board confirms guidance for the full year 2026.</p> |
| 4937 | </main> |
| 4938 | </body> |
| 4939 | </html> |
| 4940 | """ |
| 4941 | |
| 4942 | |
| 4943 | def _registered_source( |
| 4944 | source_id: str, |
| 4945 | url: str, |
| 4946 | *, |
| 4947 | priority: int = 1, |
| 4948 | categories: tuple[str, ...] = ("Customers",), |
| 4949 | allowed: bool = True, |
| 4950 | reliability: ReliabilityLevel = ReliabilityLevel.LEVEL_B, |
| 4951 | ) -> RegisteredResearchSource: |
| 4952 | return RegisteredResearchSource( |
| 4953 | source_id=source_id, |
| 4954 | instrument_id=AIXTRON_INSTRUMENT_ID, |
| 4955 | url=url, |
| 4956 | source_type=SourceType.INVESTOR_RELATIONS, |
| 4957 | source_name="AIXTRON targeted source", |
| 4958 | publisher="AIXTRON SE", |
| 4959 | reliability_level=reliability, |
| 4960 | domain="www.aixtron.com", |
| 4961 | company_id=UUID("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1"), |
| 4962 | allowed=allowed, |
| 4963 | discovery_method="TEST_TARGETED", |
| 4964 | priority=priority, |
| 4965 | categories=categories, |
| 4966 | ) |
| 4967 | |
| 4968 | |
| 4969 | class _StaticDiscovery(ApprovedSourceDiscovery): |
| 4970 | def __init__(self, sources: list[RegisteredResearchSource]) -> None: |
| 4971 | self.sources = sources |
| 4972 | |
| 4973 | def discover(self, profile, missing_categories: set[str], already_seen_urls: set[str]) -> list[DiscoveryResult]: |
| 4974 | results: list[DiscoveryResult] = [] |
| 4975 | for source in self.sources: |
| 4976 | if not source.allowed: |
| 4977 | continue |
| 4978 | if canonicalize_url(source.url) in already_seen_urls: |
| 4979 | continue |
| 4980 | if not set(source.categories) & missing_categories: |
| 4981 | continue |
| 4982 | if source.host != "www.aixtron.com": |
| 4983 | continue |
| 4984 | for category in sorted(set(source.categories) & missing_categories): |
| 4985 | results.append(DiscoveryResult(category=category, source=source)) |
| 4986 | return results |
| 4987 | |
| 4988 | |
| 4989 | class _StaticSearchProvider: |
| 4990 | provider_name = "test-search" |
| 4991 | |
| 4992 | def __init__(self, candidates: list[CandidateSearchResult]) -> None: |
| 4993 | self.candidates = candidates |
| 4994 | |
| 4995 | async def discover(self, company, category: str, date_window: SearchDateWindow) -> list[CandidateSearchResult]: |
| 4996 | return [candidate for candidate in self.candidates if candidate.category == category] |
| 4997 | |
| 4998 | |
| 4999 | class _FailingSearchProvider: |
| 5000 | provider_name = "failing-search" |
Showing first 5,000 of 5,122 lines.
View raw