| 1 | import json |
| 2 | import asyncio |
| 3 | from datetime import datetime, timezone |
| 4 | from decimal import Decimal |
| 5 | from uuid import uuid4 |
| 6 | |
| 7 | from app.models import ProvenancedValue, StructuredInstrumentResolution, StructuredMarketSnapshot, StructuredMarketSnapshotRecord |
| 8 | from app.persistence import SqliteResearchPersistence, _structured_snapshot_from_row |
| 9 | from app.portfolio_orchestration import _market_fundamentals_from_record, _public_analyst_from_record |
| 10 | from app.portfolio_orchestration import PortfolioResearchOrchestrator, _structured_due_classes |
| 11 | from app.repository import ResearchRepository |
| 12 | from app.settings import Settings |
| 13 | from app.structured_market import _is_financial_identity |
| 14 | |
| 15 | |
| 16 | def _record(instrument_id): |
| 17 | now = datetime(2026, 9, 7, 5, tzinfo=timezone.utc) |
| 18 | snapshot = StructuredMarketSnapshot( |
| 19 | resolution=StructuredInstrumentResolution(provider="YAHOO_FINANCE", provider_ticker="ABC.NS", company_name="ABC", exchange="NSE", currency="INR", confidence=.9, resolved_at=now), |
| 20 | status="SUCCESS", retrieved_at=now, market_as_of=now, source_url="https://example.test", facts={ |
| 21 | "latestPrice": ProvenancedValue(value=Decimal("250"), unit="INR", source_url="https://example.test", source_name="Yahoo", retrieved_at=now), |
| 22 | "previousClose": ProvenancedValue(value=Decimal("245"), unit="INR", source_url="https://example.test", source_name="Yahoo", retrieved_at=now), |
| 23 | }, |
| 24 | ) |
| 25 | return StructuredMarketSnapshotRecord(instrument_id=instrument_id, provider="YAHOO_FINANCE", provider_instrument_id="ABC.NS", exchange="NSE", currency="INR", source_url=snapshot.source_url, retrieved_at=now, persisted_at=now, last_price_at=now, last_success_at=now, last_provider_attempt_at=now, snapshot=snapshot) |
| 26 | |
| 27 | |
| 28 | def test_durable_structured_snapshot_survives_reconstruction_and_failure_preserves_payload(tmp_path): |
| 29 | path = tmp_path / "structured.sqlite" |
| 30 | instrument_id = uuid4() |
| 31 | first = SqliteResearchPersistence(path) |
| 32 | first.upsert_structured_market_snapshot(_record(instrument_id)) |
| 33 | first.record_structured_market_failure(instrument_id, "YAHOO_FINANCE", datetime.now(timezone.utc), "HTTP_503", "unavailable") |
| 34 | second = SqliteResearchPersistence(path) |
| 35 | record = second.load_structured_market_snapshots({instrument_id})[0] |
| 36 | assert record.snapshot.facts["latestPrice"].value == Decimal("250") |
| 37 | assert record.acquisition_status == "PROVIDER_UNAVAILABLE" |
| 38 | assert record.last_failure_code == "HTTP_503" |
| 39 | assert second.load_financial_facts() == [] |
| 40 | |
| 41 | |
| 42 | def test_latest_price_snapshot_records_a_durable_market_price_observation(tmp_path): |
| 43 | persistence = SqliteResearchPersistence(tmp_path / "prices.sqlite") |
| 44 | instrument_id = uuid4() |
| 45 | persistence.upsert_structured_market_snapshot(_record(instrument_id)) |
| 46 | observations = persistence.load_market_price_observations({instrument_id}) |
| 47 | assert len(observations) == 1 |
| 48 | assert observations[0].instrument_id == instrument_id |
| 49 | assert observations[0].price == Decimal("250") |
| 50 | assert observations[0].currency == "INR" |
| 51 | |
| 52 | |
| 53 | def test_structured_snapshot_row_accepts_sqlite_json_text_and_psycopg_decoded_jsonb(tmp_path): |
| 54 | persistence = SqliteResearchPersistence(tmp_path / "payload.sqlite") |
| 55 | instrument_id = uuid4() |
| 56 | record = _record(instrument_id).model_copy(update={ |
| 57 | "provider_instrument_id": "FEDERALBNK.NS", |
| 58 | "snapshot": _record(instrument_id).snapshot.model_copy(update={ |
| 59 | "facts": { |
| 60 | "latestPrice": _record(instrument_id).snapshot.facts["latestPrice"].model_copy(update={"value": Decimal("344.6")}), |
| 61 | "previousClose": _record(instrument_id).snapshot.facts["previousClose"], |
| 62 | }, |
| 63 | }), |
| 64 | }) |
| 65 | persistence.upsert_structured_market_snapshot(record) |
| 66 | row = dict(persistence._connection.execute("SELECT * FROM global_structured_market_snapshots").fetchone()) |
| 67 | text_record = _structured_snapshot_from_row(row) |
| 68 | row["facts_json"] = json.loads(row["facts_json"]) |
| 69 | native_record = _structured_snapshot_from_row(row) |
| 70 | for reconstructed in (text_record, native_record): |
| 71 | assert reconstructed.snapshot.facts["latestPrice"].value == Decimal("344.6") |
| 72 | assert reconstructed.snapshot.facts["previousClose"].value == Decimal("245") |
| 73 | assert reconstructed.provider == "YAHOO_FINANCE" |
| 74 | assert reconstructed.provider_instrument_id == "FEDERALBNK.NS" |
| 75 | assert reconstructed.snapshot.status == "SUCCESS" |
| 76 | assert reconstructed.source_url == "https://example.test" |
| 77 | |
| 78 | |
| 79 | def test_durable_public_analyst_projection_is_partial_safe_and_keeps_legacy_targets_unset(): |
| 80 | record = _record(uuid4()) |
| 81 | now = record.retrieved_at |
| 82 | facts = dict(record.snapshot.facts) |
| 83 | for key, value in { |
| 84 | "publicAnalystTargetLowPrice": "284.0", "publicAnalystTargetMedianPrice": "365.0", "publicAnalystTargetMeanPrice": "364.97144", |
| 85 | "publicAnalystTargetHighPrice": "425.0", "publicAnalystCount": "35", "publicAnalystRecommendationMean": "2.02857", |
| 86 | }.items(): |
| 87 | facts[key] = ProvenancedValue(value=Decimal(value), unit="INR" if "Target" in key else "ratio", source_url="https://finance.yahoo.com", source_name="Yahoo Finance", source_type="STRUCTURED_MARKET_PROVIDER", retrieved_at=now) |
| 88 | facts["publicAnalystConsensus"] = ProvenancedValue(value="buy", source_url="https://finance.yahoo.com", source_name="Yahoo Finance", source_type="STRUCTURED_MARKET_PROVIDER", retrieved_at=now) |
| 89 | analyst = _public_analyst_from_record(record.model_copy(update={"provider_instrument_id": "FEDERALBNK.NS", "last_analyst_at": now, "snapshot": record.snapshot.model_copy(update={"facts": facts})}), now, Settings()) |
| 90 | assert analyst.target_mean_price == Decimal("364.97144") |
| 91 | assert analyst.target_low_price == Decimal("284.0") |
| 92 | assert analyst.analyst_count == 35 and analyst.consensus == "buy" |
| 93 | assert analyst.provider == "YAHOO_FINANCE" and analyst.provider_instrument_id == "FEDERALBNK.NS" |
| 94 | assert analyst.source_name == "Yahoo Finance" and analyst.freshness == "FRESH" |
| 95 | partial = _public_analyst_from_record(record.model_copy(update={"snapshot": record.snapshot.model_copy(update={"facts": {"publicAnalystConsensus": facts["publicAnalystConsensus"]}})}), now, Settings()) |
| 96 | assert partial.consensus == "buy" and partial.target_mean_price is None |
| 97 | |
| 98 | |
| 99 | def test_federal_like_durable_market_fundamentals_projection_is_exact_and_partial_safe(): |
| 100 | record = _record(uuid4()).model_copy(update={"provider_instrument_id": "FEDERALBNK.NS"}) |
| 101 | now = record.retrieved_at |
| 102 | values = {"marketCap":"851075006464","enterpriseValue":"924042723328","trailingPE":"18.437668","forwardPE":"12.767692","priceToBook":"2.1213334","bookValue":"162.445","priceToSales":"5.5830674","evToRevenue":"6.062","trailingEps":"18.69","forwardEps":"26.99","profitMargin":"30.721","operatingMargin":"42.963","revenueGrowth":"21.400","earningsGrowth":"32.700","totalCash":"268973195264","totalDebt":"331932499968"} |
| 103 | facts = {key: ProvenancedValue(value=Decimal(value), unit="INR", source_url="https://finance.yahoo.com/quote/FEDERALBNK.NS", source_name="Yahoo Finance", source_type="STRUCTURED_MARKET_PROVIDER", retrieved_at=now) for key, value in values.items()} |
| 104 | full = record.model_copy(update={"last_fundamentals_at": now, "snapshot": record.snapshot.model_copy(update={"facts": facts})}) |
| 105 | projected = _market_fundamentals_from_record(full, now, Settings()) |
| 106 | assert projected.market_cap == Decimal(values["marketCap"]) |
| 107 | assert projected.enterprise_value == Decimal(values["enterpriseValue"]) |
| 108 | assert projected.trailing_pe == Decimal(values["trailingPE"]) |
| 109 | assert projected.book_value_per_share == Decimal(values["bookValue"]) |
| 110 | assert projected.trailing_eps == Decimal(values["trailingEps"]) and projected.forward_eps == Decimal(values["forwardEps"]) |
| 111 | assert projected.provider == "YAHOO_FINANCE" and projected.provider_instrument_id == "FEDERALBNK.NS" |
| 112 | assert projected.source_name == "Yahoo Finance" and projected.freshness == "FRESH" |
| 113 | partial = _market_fundamentals_from_record(full.model_copy(update={"snapshot": full.snapshot.model_copy(update={"facts": {"marketCap": facts["marketCap"], "trailingPE": facts["trailingPE"], "bookValue": facts["bookValue"]}})}), now, Settings()) |
| 114 | assert partial.market_cap == Decimal(values["marketCap"]) and partial.forward_pe is None and partial.book_value_per_share == Decimal(values["bookValue"]) |
| 115 | assert partial.roe is None |
| 116 | |
| 117 | |
| 118 | def test_financial_entity_raw_metrics_are_retained_with_semantic_restrictions(): |
| 119 | record = _record(uuid4()) |
| 120 | now = record.retrieved_at |
| 121 | facts = dict(record.snapshot.facts) |
| 122 | facts.update({ |
| 123 | "sector": ProvenancedValue(value="Financial Services", source_url="https://finance.yahoo.com", source_name="Yahoo Finance", retrieved_at=now), |
| 124 | "debtToEquity": ProvenancedValue(value=Decimal("3"), source_url="https://finance.yahoo.com", source_name="Yahoo Finance", retrieved_at=now), |
| 125 | "evToEbitda": ProvenancedValue(value=Decimal("9"), source_url="https://finance.yahoo.com", source_name="Yahoo Finance", retrieved_at=now), |
| 126 | "operatingMargin": ProvenancedValue(value=Decimal("42"), source_url="https://finance.yahoo.com", source_name="Yahoo Finance", retrieved_at=now), |
| 127 | }) |
| 128 | projected = _market_fundamentals_from_record(record.model_copy(update={"snapshot": record.snapshot.model_copy(update={"facts": facts})}), now, Settings()) |
| 129 | assert projected.debt_to_equity == Decimal("3") and projected.ev_to_ebitda == Decimal("9") |
| 130 | assert projected.metric_semantics["debt_to_equity"] == "BANK_SPECIFIC_INTERPRETATION_REQUIRED" |
| 131 | assert projected.metric_semantics["ev_to_ebitda"] == "NOT_MEANINGFUL_FOR_FINANCIAL_ENTITY" |
| 132 | |
| 133 | |
| 134 | def test_federal_shriram_and_industrial_identity_semantics_are_distinct(): |
| 135 | assert _is_financial_identity({"sector": "Financial Services", "industry": "Banks - Regional", "longName": "Federal Bank Limited"}) |
| 136 | assert _is_financial_identity({"sector": "Financial Services", "industry": "Credit Services", "longName": "Shriram Finance Limited"}) |
| 137 | assert not _is_financial_identity({"sector": "Industrials", "industry": "Aerospace & Defense", "longName": "Zen Technologies Limited"}) |
| 138 | |
| 139 | |
| 140 | def test_durable_structured_due_classes_keep_initial_and_slower_work_eligible_when_market_closed(): |
| 141 | now = datetime(2026, 9, 7, 20, tzinfo=timezone.utc) |
| 142 | settings = Settings() |
| 143 | assert _structured_due_classes(None, "CLOSED", settings, now) == {"PRICE", "VALUATION", "FUNDAMENTALS", "ANALYST"} |
| 144 | fresh = _record(uuid4()).model_copy(update={"last_price_at": now, "last_valuation_at": now, "last_fundamentals_at": now, "last_analyst_at": now, "last_success_at": now}) |
| 145 | assert _structured_due_classes(fresh, "CLOSED", settings, now) == set() |
| 146 | price_stale = fresh.model_copy(update={"last_price_at": now.replace(year=2025)}) |
| 147 | assert _structured_due_classes(price_stale, "CLOSED", settings, now) == set() |
| 148 | valuation_stale = fresh.model_copy(update={"last_valuation_at": now.replace(year=2025)}) |
| 149 | assert _structured_due_classes(valuation_stale, "UNKNOWN", settings, now) == {"VALUATION"} |
| 150 | |
| 151 | |
| 152 | def test_missing_durable_snapshot_reconciles_once_from_verified_mapping_and_fresh_snapshot_reuses_it(): |
| 153 | class Provider: |
| 154 | provider_name = "YAHOO_FINANCE" |
| 155 | def __init__(self): self.calls = 0 |
| 156 | async def collect(self, _instrument): |
| 157 | self.calls += 1 |
| 158 | return _record(instrument_id).snapshot |
| 159 | |
| 160 | instrument_id = uuid4() |
| 161 | repository = ResearchRepository(settings=Settings(research_demo_enabled=False)) |
| 162 | provider = Provider() |
| 163 | orchestrator = PortfolioResearchOrchestrator(repository, Settings(), structured_provider=provider) |
| 164 | instrument = {"instrumentId": str(instrument_id), "assetType": "EQUITY", "companyName": "Example Limited", "ticker": "EXAMPLE", "exchange": "NSE", "structuredProviderTicker": "EXAMPLE.NS", "structuredProviderStatus": "VERIFIED"} |
| 165 | |
| 166 | first = asyncio.run(orchestrator._reconcile_structured_market(instrument_id, instrument)) |
| 167 | fresh_at = datetime.now(timezone.utc) |
| 168 | records = [_record(instrument_id).model_copy(update={ |
| 169 | "last_price_at": fresh_at, "last_valuation_at": fresh_at, |
| 170 | "last_fundamentals_at": fresh_at, "last_analyst_at": fresh_at, |
| 171 | "last_success_at": fresh_at, |
| 172 | })] |
| 173 | second = asyncio.run(orchestrator._reconcile_structured_market(instrument_id, instrument, records=records)) |
| 174 | |
| 175 | assert first.snapshot is not None and provider.calls == 1 |
| 176 | assert records[0].snapshot.facts["latestPrice"].value == Decimal("250") |
| 177 | assert second.due_classes == frozenset() and provider.calls == 1 |