| 1 | """Provider-neutral international fundamentals collection. |
| 2 | |
| 3 | This module deliberately consumes the global instrument-backed research profile |
| 4 | and emits ordinary ``FinancialFact`` values. It never creates an instrument |
| 5 | master or participates in the India/NSE document pipeline. |
| 6 | """ |
| 7 | from __future__ import annotations |
| 8 | |
| 9 | from dataclasses import dataclass |
| 10 | from datetime import datetime, timezone |
| 11 | from decimal import Decimal, InvalidOperation |
| 12 | from typing import Any, Protocol |
| 13 | |
| 14 | import httpx |
| 15 | |
| 16 | from app.fact_precedence import FactSourceTier, FinancialFact, FinancialFactKey |
| 17 | from app.models import CompanyResearchProfile, ProvenancedValue, SourceMode |
| 18 | from app.settings import Settings |
| 19 | |
| 20 | |
| 21 | @dataclass(frozen=True) |
| 22 | class InternationalFundamentalsResult: |
| 23 | facts: list[FinancialFact] |
| 24 | verified_provider_ids: dict[str, str] |
| 25 | |
| 26 | |
| 27 | class InternationalFundamentalProvider(Protocol): |
| 28 | async def collect(self, profile: CompanyResearchProfile) -> InternationalFundamentalsResult: ... |
| 29 | |
| 30 | |
| 31 | def international_provider_for(profile: CompanyResearchProfile, settings: Settings, *, client: httpx.AsyncClient | None = None) -> InternationalFundamentalProvider | None: |
| 32 | if str(profile.country).upper() in {"IN", "IND", "INDIA"} or str(profile.mic).upper() in {"NSE", "XNSE"}: |
| 33 | return None |
| 34 | if str(profile.country).upper() in {"US", "USA"} or str(profile.mic).upper() in {"XNYS", "XNAS", "ARCX", "BATS"}: |
| 35 | return SecEdgarFundamentalProvider(settings, client=client) |
| 36 | if _is_european(profile): |
| 37 | return EodhdFundamentalProvider(settings, client=client) |
| 38 | return None |
| 39 | |
| 40 | |
| 41 | class SecEdgarFundamentalProvider: |
| 42 | provider_name = "SEC_EDGAR" |
| 43 | _CONCEPTS = { |
| 44 | "revenue": ("RevenueFromContractWithCustomerExcludingAssessedTax", "SalesRevenueNet"), |
| 45 | "operating_income": ("OperatingIncomeLoss",), "ebit": ("OperatingIncomeLoss",), |
| 46 | "pat": ("NetIncomeLoss",), "eps": ("EarningsPerShareDiluted", "EarningsPerShareBasic"), |
| 47 | "total_assets": ("Assets",), "total_liabilities": ("Liabilities",), |
| 48 | "equity": ("StockholdersEquity", "StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest"), |
| 49 | "cash_and_cash_equivalents": ("CashAndCashEquivalentsAtCarryingValue",), |
| 50 | "total_debt": ("LongTermDebtAndFinanceLeaseObligationsCurrent", "LongTermDebtCurrent", "LongTermDebtNoncurrent"), |
| 51 | "current_assets": ("AssetsCurrent",), "current_liabilities": ("LiabilitiesCurrent",), |
| 52 | "shares_outstanding": ("CommonStocksIncludingAdditionalPaidInCapitalMember", "EntityCommonStockSharesOutstanding"), |
| 53 | "operating_cash_flow": ("NetCashProvidedByUsedInOperatingActivities",), |
| 54 | "capex": ("PaymentsToAcquirePropertyPlantAndEquipment",), |
| 55 | "investing_cash_flow": ("NetCashProvidedByUsedInInvestingActivities",), |
| 56 | "financing_cash_flow": ("NetCashProvidedByUsedInFinancingActivities",), |
| 57 | } |
| 58 | |
| 59 | def __init__(self, settings: Settings, *, client: httpx.AsyncClient | None = None): |
| 60 | self.settings, self.client = settings, client |
| 61 | |
| 62 | async def collect(self, profile: CompanyResearchProfile) -> InternationalFundamentalsResult: |
| 63 | client, owned = self._client() |
| 64 | try: |
| 65 | cik = profile.provider_instrument_ids.get("SEC_CIK") or await self._resolve_cik(client, profile.ticker) |
| 66 | if not cik: |
| 67 | return InternationalFundamentalsResult([], {}) |
| 68 | response = await client.get(self.settings.sec_edgar_companyfacts_endpoint.format(cik=str(cik).zfill(10))) |
| 69 | response.raise_for_status() |
| 70 | return InternationalFundamentalsResult(self._facts(profile, str(cik).zfill(10), response.json()), {"SEC_CIK": str(cik).zfill(10)}) |
| 71 | finally: |
| 72 | if owned: await client.aclose() |
| 73 | |
| 74 | def _client(self) -> tuple[httpx.AsyncClient, bool]: |
| 75 | return self.client or httpx.AsyncClient(timeout=self.settings.sec_edgar_timeout_seconds, headers={"User-Agent": self.settings.sec_edgar_user_agent, "Accept-Encoding": "gzip, deflate"}), self.client is None |
| 76 | |
| 77 | async def _resolve_cik(self, client: httpx.AsyncClient, ticker: str) -> str | None: |
| 78 | response = await client.get(self.settings.sec_edgar_ticker_endpoint) |
| 79 | response.raise_for_status() |
| 80 | wanted = ticker.upper() |
| 81 | for item in response.json().values() if isinstance(response.json(), dict) else response.json(): |
| 82 | if str(item.get("ticker", "")).upper() == wanted: |
| 83 | return str(item.get("cik_str")) |
| 84 | return None |
| 85 | |
| 86 | def _facts(self, profile: CompanyResearchProfile, cik: str, payload: dict[str, Any]) -> list[FinancialFact]: |
| 87 | facts: list[FinancialFact] = [] |
| 88 | us_gaap = payload.get("facts", {}).get("us-gaap", {}) |
| 89 | for metric, concepts in self._CONCEPTS.items(): |
| 90 | concept = next((us_gaap.get(name) for name in concepts if us_gaap.get(name)), None) |
| 91 | if not concept: continue |
| 92 | units = concept.get("units", {}) |
| 93 | entries = next(iter(units.values()), []) |
| 94 | selected: dict[tuple[str, str], dict[str, Any]] = {} |
| 95 | for entry in entries: |
| 96 | form, end, value = entry.get("form"), entry.get("end"), _decimal(entry.get("val")) |
| 97 | if form not in {"10-Q", "10-K"} or not end or value is None: continue |
| 98 | kind = "ANNUAL" if form == "10-K" else "QUARTERLY" |
| 99 | # Balance facts are point-in-time; flow facts must have an explicit frame. |
| 100 | if metric in {"total_assets", "total_liabilities", "equity", "cash_and_cash_equivalents", "total_debt", "current_assets", "current_liabilities", "shares_outstanding"}: |
| 101 | kind = "AS_AT" |
| 102 | elif not entry.get("frame"): |
| 103 | continue |
| 104 | selected[(kind, end)] = entry |
| 105 | for (kind, end), entry in sorted(selected.items(), reverse=True)[:4]: |
| 106 | value = _decimal(entry.get("val")) |
| 107 | facts.append(_fact(profile, metric, end, kind, value, "UNKNOWN", "SEC_EDGAR", f"SEC:{cik}:{entry.get('accn', end)}", self.settings.sec_edgar_companyfacts_endpoint.format(cik=cik), "SEC EDGAR", "OFFICIAL_REGULATORY_FILING", FactSourceTier.OFFICIAL_REGULATORY, entry.get("filed"))) |
| 108 | return facts |
| 109 | |
| 110 | |
| 111 | class EodhdFundamentalProvider: |
| 112 | provider_name = "EODHD" |
| 113 | _METRICS = {"Revenue": "revenue", "OperatingIncome": "operating_income", "EBIT": "ebit", "EBITDA": "ebitda", "NetIncome": "pat", "EarningsPerShare": "eps", "TotalAssets": "total_assets", "TotalLiab": "total_liabilities", "TotalStockholderEquity": "equity", "CashAndEquivalents": "cash_and_cash_equivalents", "ShortLongTermDebtTotal": "total_debt", "TotalCurrentAssets": "current_assets", "TotalCurrentLiabilities": "current_liabilities", "CommonSharesOutstanding": "shares_outstanding", "TotalCashFromOperatingActivities": "operating_cash_flow", "TotalCashflowsFromInvestingActivities": "investing_cash_flow", "TotalCashFromFinancingActivities": "financing_cash_flow", "CapitalExpenditures": "capex"} |
| 114 | |
| 115 | def __init__(self, settings: Settings, *, client: httpx.AsyncClient | None = None): self.settings, self.client = settings, client |
| 116 | |
| 117 | async def collect(self, profile: CompanyResearchProfile) -> InternationalFundamentalsResult: |
| 118 | if not self.settings.eodhd_api_key: return InternationalFundamentalsResult([], {}) |
| 119 | symbol = profile.provider_instrument_ids.get("EODHD") or profile.ticker |
| 120 | client, owned = self.client or httpx.AsyncClient(timeout=self.settings.eodhd_timeout_seconds), self.client is None |
| 121 | try: |
| 122 | response = await client.get(f"{self.settings.eodhd_base_url.rstrip('/')}/fundamentals/{symbol}", params={"api_token": self.settings.eodhd_api_key, "fmt": "json"}) |
| 123 | response.raise_for_status(); payload = response.json() |
| 124 | if not _eodhd_identity_matches(profile, payload): return InternationalFundamentalsResult([], {}) |
| 125 | return InternationalFundamentalsResult(self._facts(profile, symbol, payload), {"EODHD": symbol}) |
| 126 | finally: |
| 127 | if owned: await client.aclose() |
| 128 | |
| 129 | def _facts(self, profile: CompanyResearchProfile, symbol: str, payload: dict[str, Any]) -> list[FinancialFact]: |
| 130 | out: list[FinancialFact] = [] |
| 131 | for statement_key, annual_kind in (("Income_Statement", "ANNUAL"), ("Balance_Sheet", "AS_AT"), ("Cash_Flow", "ANNUAL")): |
| 132 | statement = payload.get("Financials", {}).get(statement_key, {}) |
| 133 | for frequency, rows in (("yearly", statement.get("yearly", {})), ("quarterly", statement.get("quarterly", {}))): |
| 134 | kind = annual_kind if frequency == "yearly" or annual_kind == "AS_AT" else "QUARTERLY" |
| 135 | for row in rows.values(): |
| 136 | end = row.get("date") |
| 137 | if not end: continue |
| 138 | for raw, metric in self._METRICS.items(): |
| 139 | value = _decimal(row.get(raw)) |
| 140 | if value is not None: out.append(_fact(profile, metric, end, kind, value, "UNKNOWN", "EODHD", f"EODHD:{symbol}:{kind}:{end}", f"{self.settings.eodhd_base_url.rstrip('/')}/fundamentals/{symbol}", "EODHD", "STRUCTURED_FUNDAMENTALS", FactSourceTier.STRUCTURED_FUNDAMENTALS, row.get("filing_date"))) |
| 141 | return _latest_four(out) |
| 142 | |
| 143 | |
| 144 | def _fact(profile, metric, end, kind, value, basis, provider, identity, url, name, source_type, tier, published): |
| 145 | return FinancialFact(FinancialFactKey(profile.instrument_id, metric, str(end), kind, basis), ProvenancedValue(value=value, unit=None, source_url=url, source_name=name, source_type=source_type, published_at=_date(published), retrieved_at=datetime.now(timezone.utc), confidence=.9), tier, provider, identity, SourceMode.REAL) |
| 146 | |
| 147 | def _latest_four(facts): |
| 148 | keep = {}; [keep.setdefault((f.key.metric, f.key.period_type, f.key.period_end), f) for f in sorted(facts, key=lambda f: f.key.period_end or "", reverse=True) if sum(1 for x in keep.values() if x.key.metric == f.key.metric and x.key.period_type == f.key.period_type) < 4]; return list(keep.values()) |
| 149 | def _decimal(value): |
| 150 | try: return Decimal(str(value)) if value not in (None, "") else None |
| 151 | except (InvalidOperation, ValueError): return None |
| 152 | def _date(value): |
| 153 | try: return datetime.fromisoformat(str(value)).replace(tzinfo=timezone.utc) if value else None |
| 154 | except ValueError: return None |
| 155 | def _is_european(profile): return str(profile.country).upper() in {"DE", "GERMANY", "FR", "FRANCE", "GB", "UK", "NL", "ES", "IT", "SE", "CH", "BE", "AT", "DK", "FI", "NO", "IE", "PT"} |
| 156 | def _eodhd_identity_matches(profile, payload): |
| 157 | general = payload.get("General", payload); isin = str(general.get("ISIN") or "").upper(); exchange = str(general.get("Exchange") or general.get("ExchangeCode") or "").upper(); currency = str(general.get("CurrencyCode") or general.get("Currency") or "").upper() |
| 158 | if profile.isin and isin and profile.isin.upper() != isin: return False |
| 159 | if profile.currency and currency and profile.currency.upper() != currency: return False |
| 160 | expected = {str(profile.exchange).upper(), str(profile.mic).upper()}; aliases = {"XETRA", "XFRA", "F", "FRANKFURT"} |
| 161 | if exchange and not (exchange in expected or (expected & aliases and exchange in aliases)): return False |
| 162 | return bool(profile.isin and isin or exchange or not profile.exchange) |