| 1 | from __future__ import annotations |
| 2 | |
| 3 | import logging |
| 4 | import re |
| 5 | import asyncio |
| 6 | from datetime import datetime, timedelta, timezone |
| 7 | from decimal import Decimal, InvalidOperation |
| 8 | from difflib import SequenceMatcher |
| 9 | from typing import Any, Protocol |
| 10 | from urllib.parse import quote |
| 11 | from uuid import UUID |
| 12 | |
| 13 | import httpx |
| 14 | import yfinance as yf |
| 15 | |
| 16 | from app.models import ProvenancedValue, StructuredInstrumentResolution, StructuredMarketSnapshot |
| 17 | from app.settings import Settings |
| 18 | |
| 19 | logger = logging.getLogger(__name__) |
| 20 | |
| 21 | |
| 22 | class StructuredProviderError(RuntimeError): |
| 23 | pass |
| 24 | |
| 25 | |
| 26 | class StructuredResearchProvider(Protocol): |
| 27 | provider_name: str |
| 28 | |
| 29 | async def collect(self, instrument: dict[str, Any]) -> StructuredMarketSnapshot: |
| 30 | ... |
| 31 | |
| 32 | |
| 33 | class YahooFinanceProvider: |
| 34 | """Provider-neutral adapter over Yahoo's public structured market responses.""" |
| 35 | |
| 36 | provider_name = "YAHOO_FINANCE" |
| 37 | SEARCH_URL = "https://query1.finance.yahoo.com/v1/finance/search" |
| 38 | QUOTE_URL = "https://query1.finance.yahoo.com/v7/finance/quote" |
| 39 | SUMMARY_URL = "https://query2.finance.yahoo.com/v10/finance/quoteSummary/{ticker}" |
| 40 | |
| 41 | def __init__(self, settings: Settings, client: httpx.AsyncClient | None = None, |
| 42 | ticker_factory: Any | None = None) -> None: |
| 43 | self.settings = settings |
| 44 | self.search_url = settings.yahoo_search_url |
| 45 | self.quote_url = settings.yahoo_quote_url |
| 46 | self.summary_url = settings.yahoo_summary_url |
| 47 | self.client = client or httpx.AsyncClient( |
| 48 | timeout=httpx.Timeout(settings.structured_provider_timeout_seconds, connect=3.0), |
| 49 | headers={"User-Agent": settings.research_user_agent, "Accept": "application/json"}, |
| 50 | follow_redirects=True, |
| 51 | ) |
| 52 | self._cache: dict[str, tuple[datetime, StructuredMarketSnapshot]] = {} |
| 53 | self.ticker_factory = ticker_factory or yf.Ticker |
| 54 | self.use_yfinance = client is None or ticker_factory is not None |
| 55 | |
| 56 | async def collect(self, instrument: dict[str, Any]) -> StructuredMarketSnapshot: |
| 57 | identity = strongest_company_identity(instrument) |
| 58 | cache_key = _identity_key(instrument, identity) |
| 59 | cached = self._cache.get(cache_key) |
| 60 | now = datetime.now(timezone.utc) |
| 61 | if cached and cached[0] > now: |
| 62 | return cached[1] |
| 63 | durable_ticker = _clean_identity(instrument.get("structuredProviderTicker")) |
| 64 | durable_status = str(instrument.get("structuredProviderStatus") or "").upper() |
| 65 | if durable_ticker and durable_status in {"VERIFIED", "RESOLVED"}: |
| 66 | resolution = StructuredInstrumentResolution( |
| 67 | instrument_id=_uuid(instrument.get("instrumentId")), provider=self.provider_name, |
| 68 | provider_ticker=durable_ticker, company_name=_comparison_company_name(instrument, identity), |
| 69 | exchange=_clean_identity(instrument.get("structuredProviderExchange")) or None, |
| 70 | currency=_clean_identity(instrument.get("structuredProviderCurrency")) or None, |
| 71 | quote_type=str(instrument.get("assetType") or instrument.get("securityType") or "EQUITY").upper(), |
| 72 | confidence=0.95, resolved_at=now, status="VERIFIED_REUSED", |
| 73 | ) |
| 74 | logger.info("structured_mapping_reused instrument=%s provider=%s ticker=%s status=%s resolution_attempted=false", |
| 75 | instrument.get("instrumentId"), self.provider_name, durable_ticker, durable_status) |
| 76 | else: |
| 77 | resolution = await self.resolve_instrument(instrument, identity) |
| 78 | snapshot = await self._collect_resolved(resolution) |
| 79 | # A combined snapshot may contain a quote, so its reuse window follows the shorter market-price TTL. |
| 80 | self._cache[cache_key] = (now + timedelta(seconds=self.settings.structured_market_price_freshness_seconds), snapshot) |
| 81 | return snapshot |
| 82 | |
| 83 | async def resolve_instrument(self, instrument: dict[str, Any], identity: str | None = None) -> StructuredInstrumentResolution: |
| 84 | identity = identity or strongest_company_identity(instrument) |
| 85 | if not identity: |
| 86 | raise StructuredProviderError("COMPANY_NOT_RESOLVED") |
| 87 | trusted_nse_candidate = _trusted_nse_candidate(instrument) |
| 88 | if trusted_nse_candidate: |
| 89 | return await self._resolve_verified_nse_candidate(instrument, identity, trusted_nse_candidate) |
| 90 | raw_overview = _clean_identity(instrument.get("overview") or instrument.get("displayIdentity")) |
| 91 | overview_company = _overview_company(raw_overview) |
| 92 | broker_symbol = _clean_identity(instrument.get("brokerSymbol") or instrument.get("ticker")) |
| 93 | queries = _resolution_queries(instrument, identity) |
| 94 | logger.info( |
| 95 | "yahoo_discovery_identity raw_overview=%r extracted_company=%r normalized_search=%r broker_symbol=%r", |
| 96 | raw_overview, overview_company, queries[0] if queries else "", broker_symbol, |
| 97 | ) |
| 98 | candidates_by_symbol: dict[str, dict[str, Any]] = {} |
| 99 | provider_errors: list[str] = [] |
| 100 | for query_identity in queries: |
| 101 | try: |
| 102 | response = await self.client.get(self.search_url, params={"q": query_identity, "quotesCount": 10, "newsCount": 0}) |
| 103 | if response.status_code >= 400: |
| 104 | provider_errors.append(f"HTTP_{response.status_code}") |
| 105 | continue |
| 106 | payload = response.json() |
| 107 | candidates = payload.get("quotes") if isinstance(payload, dict) else None |
| 108 | if not isinstance(candidates, list): |
| 109 | provider_errors.append("INVALID_RESPONSE") |
| 110 | continue |
| 111 | for candidate in candidates: |
| 112 | if isinstance(candidate, dict) and candidate.get("symbol"): |
| 113 | candidates_by_symbol.setdefault(str(candidate["symbol"]), candidate) |
| 114 | except Exception as exc: |
| 115 | provider_errors.append(type(exc).__name__) |
| 116 | candidates = list(candidates_by_symbol.values()) |
| 117 | if not candidates and provider_errors: |
| 118 | raise StructuredProviderError(f"STRUCTURED_PROVIDER_UNAVAILABLE:{provider_errors[-1]}") |
| 119 | comparison_identity = _comparison_company_name(instrument, identity) |
| 120 | scored = sorted( |
| 121 | ((_candidate_score(instrument, comparison_identity, candidate), candidate) for candidate in candidates), |
| 122 | key=lambda item: item[0], reverse=True, |
| 123 | ) |
| 124 | for score, candidate in scored: |
| 125 | logger.info( |
| 126 | "yahoo_candidate symbol=%r exchange=%r currency=%r quote_type=%r score=%.4f validation=%s", |
| 127 | candidate.get("symbol"), candidate.get("exchange") or candidate.get("exchDisp"), |
| 128 | candidate.get("currency"), candidate.get("quoteType"), score, |
| 129 | _candidate_validation_reason(instrument, candidate, score, |
| 130 | self.settings.structured_resolution_min_confidence), |
| 131 | ) |
| 132 | if not scored or scored[0][0] < self.settings.structured_resolution_min_confidence: |
| 133 | logger.info("yahoo_resolution_rejected reason=LOW_CONFIDENCE") |
| 134 | raise StructuredProviderError("COMPANY_NOT_RESOLVED") |
| 135 | if len(scored) > 1 and scored[0][0] - scored[1][0] < self.settings.structured_resolution_ambiguity_margin: |
| 136 | logger.info("yahoo_resolution_rejected reason=AMBIGUOUS top_score=%.4f runner_up_score=%.4f", |
| 137 | scored[0][0], scored[1][0]) |
| 138 | raise StructuredProviderError("RESOLUTION_AMBIGUOUS") |
| 139 | confidence, candidate = scored[0] |
| 140 | quote_type = str(candidate.get("quoteType") or "").upper() |
| 141 | if quote_type not in {"EQUITY", "STOCK", "ETF", "MUTUALFUND"}: |
| 142 | raise StructuredProviderError("RESOLUTION_UNSUPPORTED_QUOTE_TYPE") |
| 143 | return StructuredInstrumentResolution( |
| 144 | instrument_id=_uuid(instrument.get("instrumentId")), provider=self.provider_name, |
| 145 | provider_ticker=str(candidate.get("symbol")), |
| 146 | company_name=str(candidate.get("longname") or candidate.get("shortname") or identity), |
| 147 | exchange=str(candidate.get("exchange") or candidate.get("exchDisp") or "") or None, |
| 148 | currency=str(candidate.get("currency") or "") or None, quote_type=quote_type, |
| 149 | confidence=confidence, resolved_at=datetime.now(timezone.utc), |
| 150 | ) |
| 151 | |
| 152 | async def _resolve_verified_nse_candidate(self, instrument: dict[str, Any], identity: str, |
| 153 | candidate_symbol: str) -> StructuredInstrumentResolution: |
| 154 | try: |
| 155 | response = await self.client.get(self.search_url, params={"q": candidate_symbol, "quotesCount": 10, "newsCount": 0}) |
| 156 | response.raise_for_status() |
| 157 | quotes = response.json().get("quotes", []) |
| 158 | except Exception as exc: |
| 159 | logger.info("yahoo_mapping_resolution globalInstrumentId=%s candidateSource=VERIFIED_NSE candidate=%s outcome=PROVIDER_UNAVAILABLE reason=%s", |
| 160 | instrument.get("instrumentId"), candidate_symbol, type(exc).__name__) |
| 161 | raise StructuredProviderError("STRUCTURED_PROVIDER_UNAVAILABLE") from exc |
| 162 | matches = [item for item in quotes if isinstance(item, dict) |
| 163 | and str(item.get("symbol") or "").upper() == candidate_symbol] |
| 164 | if len(matches) != 1: |
| 165 | logger.info("yahoo_mapping_resolution globalInstrumentId=%s candidateSource=VERIFIED_NSE candidate=%s outcome=REJECTED reason=EXACT_SYMBOL_NOT_FOUND", |
| 166 | instrument.get("instrumentId"), candidate_symbol) |
| 167 | raise StructuredProviderError("COMPANY_NOT_RESOLVED") |
| 168 | candidate = matches[0] |
| 169 | reason = _trusted_nse_candidate_reason(instrument, identity, candidate, candidate_symbol) |
| 170 | if reason is not None: |
| 171 | logger.info("yahoo_mapping_resolution globalInstrumentId=%s candidateSource=VERIFIED_NSE candidate=%s outcome=REJECTED reason=%s", |
| 172 | instrument.get("instrumentId"), candidate_symbol, reason) |
| 173 | raise StructuredProviderError("COMPANY_NOT_RESOLVED:" + reason) |
| 174 | logger.info("yahoo_mapping_resolution globalInstrumentId=%s candidateSource=VERIFIED_NSE candidate=%s outcome=VALIDATED reason=NONE", |
| 175 | instrument.get("instrumentId"), candidate_symbol) |
| 176 | return StructuredInstrumentResolution( |
| 177 | instrument_id=_uuid(instrument.get("instrumentId")), provider=self.provider_name, |
| 178 | provider_ticker=candidate_symbol, company_name=str(candidate.get("longname") or candidate.get("shortname") or identity), |
| 179 | exchange=str(candidate.get("exchange") or candidate.get("exchDisp") or "") or None, |
| 180 | currency=str(candidate.get("currency") or "") or None, |
| 181 | quote_type=str(candidate.get("quoteType") or "").upper(), confidence=0.95, |
| 182 | resolved_at=datetime.now(timezone.utc), status="VERIFIED_NSE_CANDIDATE", |
| 183 | ) |
| 184 | |
| 185 | async def _collect_resolved(self, resolution: StructuredInstrumentResolution) -> StructuredMarketSnapshot: |
| 186 | if self.use_yfinance: |
| 187 | return await asyncio.to_thread(self._collect_resolved_yfinance, resolution) |
| 188 | return await self._collect_resolved_http(resolution) |
| 189 | |
| 190 | async def collect_verified( |
| 191 | self, resolution: StructuredInstrumentResolution |
| 192 | ) -> StructuredMarketSnapshot: |
| 193 | """Collect through a caller-owned, verified provider mapping. |
| 194 | |
| 195 | This deliberately bypasses Yahoo search and never creates or updates a |
| 196 | provider mapping. It is the narrow acquisition seam used by the |
| 197 | first-party Yahoo MCP service. |
| 198 | """ |
| 199 | if ( |
| 200 | resolution.provider.strip().upper() != self.provider_name |
| 201 | or not resolution.provider_ticker.strip() |
| 202 | ): |
| 203 | raise StructuredProviderError("COMPANY_NOT_RESOLVED") |
| 204 | return await self._collect_resolved(resolution) |
| 205 | |
| 206 | def _collect_resolved_yfinance(self, resolution: StructuredInstrumentResolution) -> StructuredMarketSnapshot: |
| 207 | ticker = resolution.provider_ticker |
| 208 | retrieved = datetime.now(timezone.utc) |
| 209 | try: |
| 210 | provider = self.ticker_factory(ticker) |
| 211 | info = provider.info or {} |
| 212 | raw_news = provider.news or [] |
| 213 | except Exception as exc: |
| 214 | raise StructuredProviderError(f"STRUCTURED_PROVIDER_UNAVAILABLE:{type(exc).__name__}") from exc |
| 215 | if not isinstance(info, dict): |
| 216 | raise StructuredProviderError("STRUCTURED_PROVIDER_UNAVAILABLE:INVALID_INFO") |
| 217 | _validate_returned_identity( |
| 218 | resolution, |
| 219 | returned_symbol=info.get("symbol"), |
| 220 | returned_exchange=info.get("exchange") or info.get("fullExchangeName"), |
| 221 | ) |
| 222 | returned_currency = str(info.get("currency") or "").upper() |
| 223 | returned_type = _normalized_quote_type( |
| 224 | info.get("quoteType") or resolution.quote_type, |
| 225 | info.get("longName") or info.get("shortName") or resolution.company_name, |
| 226 | ) |
| 227 | if resolution.currency and returned_currency and resolution.currency.upper() != returned_currency: |
| 228 | raise StructuredProviderError("PERSISTED_MAPPING_CONFLICT:CURRENCY") |
| 229 | if returned_type not in {"EQUITY", "STOCK", "ETF", "MUTUALFUND"}: |
| 230 | raise StructuredProviderError("PERSISTED_MAPPING_CONFLICT:QUOTE_TYPE") |
| 231 | income_statement = balance_sheet = quarterly_income = quarterly_balance = cashflow = quarterly_cashflow = None |
| 232 | if returned_type in {"EQUITY", "STOCK"}: |
| 233 | try: |
| 234 | income_statement = getattr(provider, "income_stmt", None) |
| 235 | balance_sheet = getattr(provider, "balance_sheet", None) |
| 236 | quarterly_income = getattr(provider, "quarterly_income_stmt", None) |
| 237 | quarterly_balance = getattr(provider, "quarterly_balance_sheet", None) |
| 238 | cashflow = getattr(provider, "cashflow", None) |
| 239 | quarterly_cashflow = getattr(provider, "quarterly_cashflow", None) |
| 240 | except Exception: |
| 241 | # Statement-derived ROCE is optional; quote retrieval remains usable. |
| 242 | pass |
| 243 | market_as_of = _timestamp(info.get("regularMarketTime")) |
| 244 | source_url = f"https://finance.yahoo.com/quote/{quote(ticker, safe='')}" |
| 245 | facts = _normalize_facts(info, ticker, source_url, retrieved, market_as_of) |
| 246 | # Financial issuers still have reported statements; only industrial ROCE is inapplicable. |
| 247 | roce = None if _is_financial_identity(info) else _normalize_roce(income_statement, balance_sheet, source_url, retrieved) |
| 248 | if roce is not None: |
| 249 | facts["roce"] = roce |
| 250 | news = _normalize_yfinance_news(raw_news, retrieved) |
| 251 | if "latestPrice" not in facts: |
| 252 | raise StructuredProviderError("STRUCTURED_PRICE_UNAVAILABLE") |
| 253 | normalized_resolution = resolution.model_copy(update={ |
| 254 | "company_name": str(info.get("longName") or resolution.company_name), |
| 255 | "exchange": str(info.get("exchange") or resolution.exchange or "") or None, |
| 256 | "currency": returned_currency or resolution.currency, |
| 257 | "quote_type": returned_type, |
| 258 | }) |
| 259 | return StructuredMarketSnapshot( |
| 260 | resolution=normalized_resolution, status="STRUCTURED_PROVIDER_AVAILABLE", retrieved_at=retrieved, |
| 261 | market_as_of=market_as_of, source_url=source_url, facts=facts, |
| 262 | statement_facts=_normalize_statement_facts(income_statement, balance_sheet, cashflow, "ANNUAL", source_url, retrieved) |
| 263 | + _normalize_statement_facts(quarterly_income, quarterly_balance, quarterly_cashflow, "QUARTERLY", source_url, retrieved), news=news, |
| 264 | accepted_fields_count=len(facts) + len(news), |
| 265 | ) |
| 266 | |
| 267 | async def _collect_resolved_http(self, resolution: StructuredInstrumentResolution) -> StructuredMarketSnapshot: |
| 268 | ticker = resolution.provider_ticker |
| 269 | retrieved = datetime.now(timezone.utc) |
| 270 | quote_payload: dict[str, Any] = {} |
| 271 | summary_payload: dict[str, Any] = {} |
| 272 | errors: list[str] = [] |
| 273 | news: list[dict[str, Any]] = [] |
| 274 | try: |
| 275 | response = await self.client.get(self.quote_url, params={"symbols": ticker}) |
| 276 | response.raise_for_status() |
| 277 | values = response.json().get("quoteResponse", {}).get("result", []) |
| 278 | if values: |
| 279 | quote_payload = values[0] |
| 280 | except Exception as exc: |
| 281 | errors.append(type(exc).__name__) |
| 282 | try: |
| 283 | modules = "price,summaryDetail,defaultKeyStatistics,financialData,assetProfile,calendarEvents,earnings,earningsHistory,earningsTrend,recommendationTrend,institutionOwnership,majorHoldersBreakdown" |
| 284 | response = await self.client.get(self.summary_url.format(ticker=quote(ticker, safe="")), params={"modules": modules}) |
| 285 | response.raise_for_status() |
| 286 | values = response.json().get("quoteSummary", {}).get("result", []) |
| 287 | if values: |
| 288 | summary_payload = values[0] |
| 289 | except Exception as exc: |
| 290 | errors.append(type(exc).__name__) |
| 291 | try: |
| 292 | response = await self.client.get(self.search_url, params={"q": ticker, "quotesCount": 0, "newsCount": 10}) |
| 293 | response.raise_for_status() |
| 294 | for item in response.json().get("news", []): |
| 295 | if not isinstance(item, dict) or not item.get("title") or not item.get("link"): |
| 296 | continue |
| 297 | news.append({ |
| 298 | "headline": str(item["title"]), "publisher": str(item.get("publisher") or "Yahoo Finance"), |
| 299 | "url": str(item["link"]), "publishedAt": _timestamp(item.get("providerPublishTime")), |
| 300 | "retrievedAt": retrieved, "sourceType": "STRUCTURED_MARKET_PROVIDER", "confidence": 0.72, |
| 301 | }) |
| 302 | except Exception as exc: |
| 303 | errors.append(type(exc).__name__) |
| 304 | combined = _flatten_provider_payload(quote_payload, summary_payload) |
| 305 | _validate_returned_identity( |
| 306 | resolution, |
| 307 | returned_symbol=combined.get("symbol"), |
| 308 | returned_exchange=combined.get("exchange") or combined.get("fullExchangeName"), |
| 309 | ) |
| 310 | returned_currency = str(combined.get("currency") or "").upper() |
| 311 | returned_type = str(combined.get("quoteType") or "EQUITY").upper() |
| 312 | if resolution.currency and returned_currency and resolution.currency.upper() != returned_currency: |
| 313 | raise StructuredProviderError("PERSISTED_MAPPING_CONFLICT:CURRENCY") |
| 314 | if returned_type not in {"EQUITY", "STOCK", "ETF", "MUTUALFUND"}: |
| 315 | raise StructuredProviderError("PERSISTED_MAPPING_CONFLICT:QUOTE_TYPE") |
| 316 | market_as_of = _timestamp(combined.get("regularMarketTime")) |
| 317 | source_url = f"https://finance.yahoo.com/quote/{quote(ticker, safe='')}" |
| 318 | facts = _normalize_facts(combined, ticker, source_url, retrieved, market_as_of) |
| 319 | if not facts: |
| 320 | raise StructuredProviderError("STRUCTURED_PROVIDER_UNAVAILABLE:NO_ACCEPTED_FIELDS") |
| 321 | status = "STRUCTURED_PROVIDER_PARTIAL" if errors or len(facts) < 8 else "STRUCTURED_PROVIDER_AVAILABLE" |
| 322 | return StructuredMarketSnapshot( |
| 323 | resolution=resolution, status=status, retrieved_at=retrieved, market_as_of=market_as_of, |
| 324 | source_url=source_url, facts=facts, news=news, accepted_fields_count=len(facts) + len(news), |
| 325 | safe_error_code=errors[-1] if errors else None, |
| 326 | ) |
| 327 | |
| 328 | |
| 329 | def strongest_company_identity(instrument: dict[str, Any]) -> str: |
| 330 | values = [ |
| 331 | _overview_company(instrument.get("overview") or instrument.get("displayIdentity")), |
| 332 | instrument.get("canonicalName"), instrument.get("companyName"), instrument.get("isin"), |
| 333 | instrument.get("brokerDescription"), instrument.get("displayName"), |
| 334 | instrument.get("brokerSymbol"), |
| 335 | instrument.get("canonicalSymbol"), instrument.get("ticker"), |
| 336 | ] |
| 337 | return next((_clean_identity(value) for value in values if _clean_identity(value)), "") |
| 338 | |
| 339 | |
| 340 | def _resolution_queries(instrument: dict[str, Any], primary: str) -> list[str]: |
| 341 | overview_company = _overview_company(instrument.get("overview") or instrument.get("displayIdentity")) |
| 342 | symbols = {_clean_identity(instrument.get(key)).casefold() for key in |
| 343 | ("canonicalSymbol", "brokerSymbol", "ticker") if _clean_identity(instrument.get(key))} |
| 344 | company_values = [overview_company, instrument.get("canonicalName"), instrument.get("companyName"), |
| 345 | instrument.get("brokerDescription"), instrument.get("displayName")] |
| 346 | result: list[str] = [] |
| 347 | for value in company_values: |
| 348 | cleaned = _clean_identity(value) |
| 349 | if cleaned and cleaned.casefold() not in symbols and cleaned.casefold() not in {item.casefold() for item in result}: |
| 350 | result.append(cleaned) |
| 351 | if result: |
| 352 | return result |
| 353 | # Short symbols are discovery fallback only when no richer company identity exists. |
| 354 | for value in (instrument.get("canonicalSymbol"), instrument.get("brokerSymbol"), instrument.get("ticker"), primary): |
| 355 | cleaned = _clean_identity(value) |
| 356 | if cleaned and cleaned.casefold() not in {item.casefold() for item in result}: |
| 357 | result.append(cleaned) |
| 358 | return result |
| 359 | |
| 360 | |
| 361 | def _comparison_company_name(instrument: dict[str, Any], fallback: str) -> str: |
| 362 | for key in ("canonicalName", "companyName", "brokerDescription", "displayName"): |
| 363 | value = _clean_identity(instrument.get(key)) |
| 364 | if value: |
| 365 | return value |
| 366 | overview = _clean_identity(_overview_company(instrument.get("overview"))) |
| 367 | return overview or fallback |
| 368 | |
| 369 | |
| 370 | def _legacy_overview_company(value: Any) -> str: |
| 371 | cleaned = str(value or "").replace("\u00a0", " ") |
| 372 | return re.split(r"(?:·|\u00c2\u00b7)", cleaned, maxsplit=1)[0] |
| 373 | |
| 374 | |
| 375 | def _clean_identity(value: Any) -> str: |
| 376 | return re.sub(r"\s+", " ", str(value or "").replace("\u00a0", " ")).strip() |
| 377 | |
| 378 | |
| 379 | def _overview_company(value: Any) -> str: |
| 380 | cleaned = str(value or "").replace("\u00a0", " ") |
| 381 | # Handle the actual middle dot and its common UTF-8-as-Latin-1 mojibake form. |
| 382 | return _clean_identity(re.split("(?:\\u00b7|\\u00c2\\u00b7)", cleaned, maxsplit=1)[0]) |
| 383 | |
| 384 | |
| 385 | def _candidate_validation_reason(instrument: dict[str, Any], candidate: dict[str, Any], score: float, |
| 386 | minimum_confidence: float) -> str: |
| 387 | quote_type = str(candidate.get("quoteType") or "").upper() |
| 388 | if quote_type not in {"EQUITY", "STOCK"}: |
| 389 | return "REJECT_QUOTE_TYPE" |
| 390 | expected_currency = str(instrument.get("tradingCurrency") or "").upper() |
| 391 | candidate_currency = str(candidate.get("currency") or "").upper() |
| 392 | if expected_currency and candidate_currency and expected_currency != candidate_currency: |
| 393 | return "REJECT_CURRENCY" |
| 394 | expected_listing = _exchange_family(_held_listing_exchange(instrument), "") |
| 395 | candidate_listing = _exchange_family(candidate.get("exchange") or candidate.get("exchDisp"), candidate.get("symbol")) |
| 396 | if expected_listing and candidate_listing and expected_listing != candidate_listing: |
| 397 | return "REJECT_EXCHANGE" |
| 398 | if score < 0.01: |
| 399 | return "REJECT_IDENTITY_CONFLICT" |
| 400 | return "IDENTITY_SCORE_ACCEPTABLE" if score >= minimum_confidence else "REJECT_LOW_CONFIDENCE" |
| 401 | |
| 402 | |
| 403 | def _trusted_nse_candidate(instrument: dict[str, Any]) -> str | None: |
| 404 | candidate = _clean_identity(instrument.get("structuredNseCandidateTicker")).upper() |
| 405 | if str(instrument.get("structuredNseCandidateSource") or "").upper() != "VERIFIED_NSE": |
| 406 | return None |
| 407 | return candidate if candidate.endswith(".NS") and len(candidate) > 3 else None |
| 408 | |
| 409 | |
| 410 | def _trusted_nse_candidate_reason(instrument: dict[str, Any], identity: str, candidate: dict[str, Any], expected_symbol: str) -> str | None: |
| 411 | if str(candidate.get("symbol") or "").upper() != expected_symbol: |
| 412 | return "SYMBOL_MISMATCH" |
| 413 | if str(candidate.get("quoteType") or "").upper() not in {"EQUITY", "STOCK"}: |
| 414 | return "QUOTE_TYPE_MISMATCH" |
| 415 | candidate_listing = _exchange_family(candidate.get("exchange") or candidate.get("exchDisp"), expected_symbol) |
| 416 | if candidate_listing != "XNSE": |
| 417 | return "EXCHANGE_MISMATCH" |
| 418 | expected_currency = str(instrument.get("tradingCurrency") or instrument.get("currency") or "").upper() |
| 419 | candidate_currency = str(candidate.get("currency") or "").upper() |
| 420 | if expected_currency and candidate_currency and expected_currency != candidate_currency: |
| 421 | return "CURRENCY_MISMATCH" |
| 422 | expected_isin = str(instrument.get("isin") or "").upper() |
| 423 | candidate_isin = str(candidate.get("isin") or "").upper() |
| 424 | if expected_isin and candidate_isin and expected_isin != candidate_isin: |
| 425 | return "ISIN_MISMATCH" |
| 426 | candidate_name = str(candidate.get("longname") or candidate.get("shortname") or "") |
| 427 | if not candidate_name or _name_similarity(identity, candidate_name) < 0.55: |
| 428 | return "COMPANY_NAME_MISMATCH" |
| 429 | return None |
| 430 | |
| 431 | |
| 432 | def _candidate_score(instrument: dict[str, Any], identity: str, candidate: dict[str, Any]) -> float: |
| 433 | if str(candidate.get("quoteType") or "").upper() not in {"EQUITY", "STOCK"}: |
| 434 | return 0.0 |
| 435 | symbol = str(candidate.get("symbol") or "").upper() |
| 436 | expected_symbols = {str(instrument.get(key) or "").upper() for key in ("canonicalSymbol", "brokerSymbol", "ticker")} |
| 437 | expected_symbols.discard("") |
| 438 | base_symbol = symbol.split(".", 1)[0] |
| 439 | score = 0.15 |
| 440 | expected_isin = str(instrument.get("isin") or "").upper() |
| 441 | candidate_isin = str(candidate.get("isin") or "").upper() |
| 442 | if expected_isin and candidate_isin: |
| 443 | if expected_isin != candidate_isin: |
| 444 | return 0.0 |
| 445 | score += 0.60 |
| 446 | if symbol in expected_symbols or base_symbol in expected_symbols: |
| 447 | score += 0.20 |
| 448 | candidate_name = str(candidate.get("longname") or candidate.get("shortname") or "") |
| 449 | # Name establishes company identity, but cannot outweigh a conflicting known listing. |
| 450 | score += 0.35 * _name_similarity(identity, candidate_name) |
| 451 | expected_currency = str(instrument.get("tradingCurrency") or "").upper() |
| 452 | candidate_currency = str(candidate.get("currency") or "").upper() |
| 453 | if expected_currency and candidate_currency: |
| 454 | if expected_currency != candidate_currency: |
| 455 | return 0.0 |
| 456 | score += 0.10 |
| 457 | expected_country = str(instrument.get("country") or "").upper() |
| 458 | expected_listing = _exchange_family(_held_listing_exchange(instrument), "") |
| 459 | candidate_listing = _exchange_family(candidate.get("exchange") or candidate.get("exchDisp"), symbol) |
| 460 | if expected_listing and candidate_listing and expected_listing != candidate_listing: |
| 461 | return 0.0 |
| 462 | if expected_listing and candidate_listing == expected_listing: |
| 463 | score += 0.40 |
| 464 | candidate_country = str(candidate.get("country") or candidate.get("region") or "").upper() |
| 465 | if expected_country and candidate_country: |
| 466 | if expected_country != candidate_country: |
| 467 | return 0.0 |
| 468 | score += 0.05 |
| 469 | return min(score, 1.0) |
| 470 | |
| 471 | |
| 472 | def _held_listing_exchange(instrument: dict[str, Any]) -> str: |
| 473 | # Broker listing exchange outranks SMART or other routing venues. |
| 474 | routing = {"", "SMART", "BEST", "AUTO"} |
| 475 | for key in ("brokerExchange", "primaryExchange", "listingExchange", "canonicalExchange", "exchange"): |
| 476 | value = str(instrument.get(key) or "").upper() |
| 477 | if value not in routing: |
| 478 | return value |
| 479 | return "" |
| 480 | |
| 481 | |
| 482 | def _exchange_family(exchange: Any, symbol: Any) -> str: |
| 483 | value = str(exchange or "").upper().replace(" ", "") |
| 484 | aliases = { |
| 485 | "AEB": "XAMS", "AMS": "XAMS", "XAMS": "XAMS", "EURONEXTAMSTERDAM": "XAMS", |
| 486 | "IBIS": "XETR", "IBIS2": "XETR", "XETR": "XETR", "GER": "XETR", "XETRA": "XETR", |
| 487 | "FRA": "XFRA", "XFRA": "XFRA", "FRANKFURT": "XFRA", |
| 488 | "NSE": "XNSE", "NSI": "XNSE", "XNSE": "XNSE", |
| 489 | "BSE": "XBOM", "BOM": "XBOM", "XBOM": "XBOM", |
| 490 | "NASDAQ": "XNAS", "NMS": "XNAS", "NGM": "XNAS", "NCM": "XNAS", "XNAS": "XNAS", |
| 491 | "NYSE": "XNYS", "NYQ": "XNYS", "XNYS": "XNYS", |
| 492 | "LSE": "XLON", "LONDON": "XLON", "XLON": "XLON", |
| 493 | } |
| 494 | return aliases.get(value, "") |
| 495 | |
| 496 | |
| 497 | def _validate_returned_identity( |
| 498 | resolution: StructuredInstrumentResolution, |
| 499 | *, |
| 500 | returned_symbol: Any, |
| 501 | returned_exchange: Any, |
| 502 | ) -> None: |
| 503 | symbol = _clean_identity(returned_symbol) |
| 504 | if symbol and symbol.upper() != resolution.provider_ticker.upper(): |
| 505 | raise StructuredProviderError("PERSISTED_MAPPING_CONFLICT:SYMBOL") |
| 506 | expected_family = _exchange_family(resolution.exchange, resolution.provider_ticker) |
| 507 | returned_family = _exchange_family(returned_exchange, symbol or resolution.provider_ticker) |
| 508 | if expected_family and returned_family and expected_family != returned_family: |
| 509 | raise StructuredProviderError("PERSISTED_MAPPING_CONFLICT:EXCHANGE") |
| 510 | |
| 511 | |
| 512 | def _name_similarity(left: str, right: str) -> float: |
| 513 | def normalized(value: str) -> str: |
| 514 | value = re.sub(r"\b(limited|ltd|n\.?v\.?|ag|se|plc|inc|corp(?:oration)?)\b", " ", value.lower()) |
| 515 | return re.sub(r"[^a-z0-9]+", " ", value).strip() |
| 516 | return SequenceMatcher(None, normalized(left), normalized(right)).ratio() |
| 517 | |
| 518 | |
| 519 | def _flatten_provider_payload(quote_payload: dict[str, Any], summary_payload: dict[str, Any]) -> dict[str, Any]: |
| 520 | result = dict(quote_payload) |
| 521 | for module in summary_payload.values(): |
| 522 | if isinstance(module, dict): |
| 523 | for key, value in module.items(): |
| 524 | if key not in result or result[key] is None: |
| 525 | result[key] = value |
| 526 | return result |
| 527 | |
| 528 | |
| 529 | FIELD_MAP = { |
| 530 | "regularMarketPreviousClose": ("previousClose", "currency"), |
| 531 | "bid": ("bid", "currency"), "bidSize": ("bidSize", "shares"), |
| 532 | "ask": ("ask", "currency"), "askSize": ("askSize", "shares"), |
| 533 | "regularMarketVolume": ("volume", "shares"), "fiftyTwoWeekHigh": ("fiftyTwoWeekHigh", "currency"), |
| 534 | "fiftyTwoWeekLow": ("fiftyTwoWeekLow", "currency"), "marketCap": ("marketCap", "currency"), |
| 535 | "enterpriseValue": ("enterpriseValue", "currency"), "trailingPE": ("trailingPE", "ratio"), |
| 536 | "forwardPE": ("forwardPE", "ratio"), "priceToBook": ("priceToBook", "ratio"), |
| 537 | "pegRatio": ("pegRatio", "ratio"), "bookValue": ("bookValue", "currency"), |
| 538 | "priceToSalesTrailing12Months": ("priceToSales", "ratio"), |
| 539 | "enterpriseToRevenue": ("evToRevenue", "ratio"), "enterpriseToEbitda": ("evToEbitda", "ratio"), |
| 540 | "trailingEps": ("trailingEps", "currency"), "forwardEps": ("forwardEps", "currency"), |
| 541 | "averageDailyVolume10Day": ("averageVolume10Day", "shares"), "averageVolume": ("averageVolume", "shares"), |
| 542 | "targetLowPrice": ("publicAnalystTargetLowPrice", "currency"), |
| 543 | "targetMedianPrice": ("publicAnalystTargetMedianPrice", "currency"), |
| 544 | "targetHighPrice": ("publicAnalystTargetHighPrice", "currency"), |
| 545 | "recommendationMean": ("publicAnalystRecommendationMean", "ratio"), |
| 546 | "navPrice": ("navPrice", "currency"), "returnOnEquity": ("roe", "percent"), |
| 547 | "returnOnAssets": ("roa", "percent"), "profitMargins": ("profitMargin", "percent"), |
| 548 | "operatingMargins": ("operatingMargin", "percent"), "revenueGrowth": ("revenueGrowth", "percent"), |
| 549 | "earningsGrowth": ("earningsGrowth", "percent"), "totalCash": ("totalCash", "currency"), |
| 550 | "totalDebt": ("totalDebt", "currency"), "debtToEquity": ("debtToEquity", "ratio"), |
| 551 | "freeCashflow": ("freeCashFlow", "currency"), "operatingCashflow": ("operatingCashFlow", "currency"), |
| 552 | "targetMeanPrice": ("publicAnalystTargetMeanPrice", "currency"), "numberOfAnalystOpinions": ("publicAnalystCount", "count"), |
| 553 | "heldPercentInsiders": ("insidersPercent", "percent"), "heldPercentInstitutions": ("institutionsPercent", "percent"), |
| 554 | } |
| 555 | |
| 556 | |
| 557 | def _normalize_facts(payload: dict[str, Any], ticker: str, source_url: str, retrieved: datetime, market_as_of: datetime | None) -> dict[str, ProvenancedValue]: |
| 558 | facts: dict[str, ProvenancedValue] = {} |
| 559 | currency = str(payload.get("currency") or "") or None |
| 560 | price = _decimal(_raw(payload.get("currentPrice"))) |
| 561 | if price is None: |
| 562 | price = _decimal(_raw(payload.get("regularMarketPrice"))) |
| 563 | if price is not None and price > 0: |
| 564 | facts["latestPrice"] = ProvenancedValue(value=price, unit=currency, as_of_date=market_as_of, |
| 565 | source_url=source_url, source_name="Yahoo Finance", source_type="STRUCTURED_MARKET_PROVIDER", |
| 566 | retrieved_at=retrieved, confidence=0.90) |
| 567 | for provider_key, (metric, unit_kind) in FIELD_MAP.items(): |
| 568 | value = _raw(payload.get(provider_key)) |
| 569 | number = _decimal(value) |
| 570 | if number is None: |
| 571 | continue |
| 572 | if metric in {"bid", "ask", "bidSize", "askSize"} and number <= 0: |
| 573 | continue |
| 574 | if unit_kind == "percent" and abs(number) <= 1: |
| 575 | number *= Decimal("100") |
| 576 | unit = currency if unit_kind == "currency" else unit_kind |
| 577 | facts[metric] = ProvenancedValue(value=number, unit=unit, as_of_date=market_as_of, |
| 578 | source_url=source_url, source_name="Yahoo Finance", source_type="STRUCTURED_MARKET_PROVIDER", |
| 579 | retrieved_at=retrieved, confidence=0.82) |
| 580 | text_fields = { |
| 581 | "longName": "providerCompanyName", "sector": "sector", "industry": "industry", |
| 582 | "longBusinessSummary": "businessSummary", "recommendationKey": "publicAnalystConsensus", |
| 583 | } |
| 584 | for provider_key, metric in text_fields.items(): |
| 585 | value = _raw(payload.get(provider_key)) |
| 586 | if value not in (None, ""): |
| 587 | facts[metric] = ProvenancedValue(value=str(value), as_of_date=market_as_of, source_url=source_url, |
| 588 | source_name="Yahoo Finance", source_type="STRUCTURED_MARKET_PROVIDER", retrieved_at=retrieved, confidence=0.78) |
| 589 | earnings = payload.get("quarterly") or payload.get("quarterlyChart") |
| 590 | if isinstance(earnings, list) and earnings: |
| 591 | normalized_earnings = [] |
| 592 | for item in earnings: |
| 593 | if not isinstance(item, dict): |
| 594 | continue |
| 595 | normalized_earnings.append({ |
| 596 | "period": item.get("date") or item.get("period"), |
| 597 | "earnings": _raw(item.get("earnings")), "revenue": _raw(item.get("revenue")), |
| 598 | "epsActual": _raw(item.get("epsActual")), "epsEstimate": _raw(item.get("epsEstimate")), |
| 599 | "epsDifference": _raw(item.get("epsDifference")), "surprisePercent": _raw(item.get("surprisePercent")), |
| 600 | }) |
| 601 | if normalized_earnings: |
| 602 | facts["earningsHistory"] = ProvenancedValue(value=normalized_earnings, source_url=source_url, |
| 603 | source_name="Yahoo Finance", source_type="STRUCTURED_MARKET_PROVIDER", retrieved_at=retrieved, confidence=0.74) |
| 604 | facts["providerTicker"] = ProvenancedValue(value=ticker, source_url=source_url, source_name="Yahoo Finance", |
| 605 | source_type="STRUCTURED_MARKET_PROVIDER", retrieved_at=retrieved, confidence=0.95) |
| 606 | return facts |
| 607 | |
| 608 | |
| 609 | def _normalize_yfinance_news(raw_news: Any, retrieved: datetime) -> list[dict[str, Any]]: |
| 610 | normalized = [] |
| 611 | for item in raw_news if isinstance(raw_news, list) else []: |
| 612 | if not isinstance(item, dict): |
| 613 | continue |
| 614 | content = item.get("content") if isinstance(item.get("content"), dict) else item |
| 615 | title = content.get("title") |
| 616 | provider = content.get("provider") if isinstance(content.get("provider"), dict) else {} |
| 617 | canonical = content.get("canonicalUrl") if isinstance(content.get("canonicalUrl"), dict) else {} |
| 618 | url = content.get("link") or canonical.get("url") |
| 619 | if not title or not url: |
| 620 | continue |
| 621 | normalized.append({"headline": str(title), "publisher": str(content.get("publisher") or provider.get("displayName") or "Yahoo Finance"), |
| 622 | "url": str(url), "publishedAt": _timestamp(content.get("providerPublishTime") or content.get("pubDate")), |
| 623 | "retrievedAt": retrieved, "sourceType": "STRUCTURED_MARKET_PROVIDER", "confidence": 0.72}) |
| 624 | return sorted(normalized, key=lambda article: article.get("publishedAt") or datetime.min.replace(tzinfo=timezone.utc), reverse=True) |
| 625 | |
| 626 | |
| 627 | def _normalize_roce(income_statement: Any, balance_sheet: Any, source_url: str, |
| 628 | retrieved: datetime) -> ProvenancedValue | None: |
| 629 | """Calculate annual ROCE as EBIT / (total assets - current liabilities). |
| 630 | |
| 631 | A value is returned only when all three statement inputs exist for the same |
| 632 | annual period and capital employed is positive. No proxy ratios are used. |
| 633 | """ |
| 634 | if income_statement is None or balance_sheet is None: |
| 635 | return None |
| 636 | try: |
| 637 | income_columns = list(income_statement.columns) |
| 638 | balance_columns = set(balance_sheet.columns) |
| 639 | period = next((column for column in income_columns if column in balance_columns), None) |
| 640 | if period is None: |
| 641 | return None |
| 642 | ebit = _statement_value(income_statement, period, ("EBIT", "Operating Income")) |
| 643 | assets = _statement_value(balance_sheet, period, ("Total Assets",)) |
| 644 | current_liabilities = _statement_value( |
| 645 | balance_sheet, period, ("Current Liabilities", "Total Current Liabilities")) |
| 646 | if ebit is None or assets is None or current_liabilities is None: |
| 647 | return None |
| 648 | capital_employed = assets - current_liabilities |
| 649 | if capital_employed <= 0: |
| 650 | return None |
| 651 | value = (ebit / capital_employed) * Decimal("100") |
| 652 | period_text = period.isoformat() if hasattr(period, "isoformat") else str(period) |
| 653 | return ProvenancedValue( |
| 654 | value=value, unit="percent", as_of_date=_coerce_period_datetime(period), period=period_text, |
| 655 | source_url=source_url, source_name="Yahoo Finance", source_type="STRUCTURED_FINANCIAL_STATEMENTS", |
| 656 | retrieved_at=retrieved, confidence=0.78, |
| 657 | calculation_basis="EBIT / (total assets - current liabilities), latest common annual period", |
| 658 | ) |
| 659 | except (AttributeError, KeyError, TypeError, InvalidOperation, ValueError): |
| 660 | return None |
| 661 | |
| 662 | |
| 663 | def _normalize_statement_facts(income: Any, balance: Any, cashflow: Any, period_type: str, source_url: str, retrieved: datetime) -> list[dict[str, Any]]: |
| 664 | """Normalize only explicit yfinance statement rows and dated columns. |
| 665 | |
| 666 | Yahoo does not expose standalone/consolidated basis here, so UNKNOWN is a |
| 667 | deliberate canonical isolation value rather than an implied equivalence. |
| 668 | """ |
| 669 | mappings = ( |
| 670 | (income, {"Total Revenue": "revenue", "Net Income": "pat", "EBITDA": "ebitda", "Pretax Income": "pbt", "Tax Provision": "tax", "Interest Expense": "finance_cost", "Diluted EPS": "eps"}), |
| 671 | (balance, {"Cash Cash Equivalents And Short Term Investments": "cash_and_equivalents", "Cash And Cash Equivalents": "cash_and_equivalents", "Total Debt": "debt_or_borrowings", "Total Assets": "total_assets", "Total Liabilities Net Minority Interest": "total_liabilities", "Stockholders Equity": "equity", "Accounts Receivable": "receivables", "Inventory": "inventory"}), |
| 672 | (cashflow, {"Operating Cash Flow": "operating_cash_flow", "Investing Cash Flow": "investing_cash_flow", "Financing Cash Flow": "financing_cash_flow", "Capital Expenditure": "capex"}), |
| 673 | ) |
| 674 | normalized: list[dict[str, Any]] = [] |
| 675 | for frame, labels in mappings: |
| 676 | if frame is None or not hasattr(frame, "columns") or not hasattr(frame, "index"): |
| 677 | continue |
| 678 | for period in frame.columns: |
| 679 | period_end = period.isoformat() if hasattr(period, "isoformat") else str(period) |
| 680 | if not period_end: |
| 681 | continue |
| 682 | for label, metric in labels.items(): |
| 683 | if label not in frame.index: |
| 684 | continue |
| 685 | value = _decimal(frame.loc[label, period]) |
| 686 | if value is None: |
| 687 | continue |
| 688 | normalized.append({"metric": metric, "value": value, "periodEnd": period_end, "periodType": period_type, |
| 689 | "reportingBasis": "UNKNOWN", "sourceUrl": source_url, "sourceName": "Yahoo Finance", |
| 690 | "sourceType": "STRUCTURED_FINANCIAL_STATEMENTS", "retrievedAt": retrieved, |
| 691 | "confidence": 0.78, "rawFieldOrigin": label}) |
| 692 | return normalized |
| 693 | |
| 694 | |
| 695 | def _is_financial_identity(info: dict[str, Any]) -> bool: |
| 696 | """Banks and financials do not receive industrial EBIT/ROCE semantics.""" |
| 697 | identity = " ".join(str(info.get(key) or "") for key in ("sector", "industry", "longName", "shortName")).lower() |
| 698 | return bool(re.search(r"\b(bank|financial|insurance|credit|lending|asset management)\b", identity)) |
| 699 | |
| 700 | |
| 701 | def _statement_value(frame: Any, period: Any, labels: tuple[str, ...]) -> Decimal | None: |
| 702 | for label in labels: |
| 703 | try: |
| 704 | value = frame.loc[label, period] |
| 705 | except (KeyError, TypeError): |
| 706 | continue |
| 707 | number = _decimal(value) |
| 708 | if number is not None: |
| 709 | return number |
| 710 | return None |
| 711 | |
| 712 | |
| 713 | def _coerce_period_datetime(period: Any) -> datetime | None: |
| 714 | try: |
| 715 | value = period.to_pydatetime() if hasattr(period, "to_pydatetime") else period |
| 716 | if isinstance(value, datetime): |
| 717 | return value.replace(tzinfo=value.tzinfo or timezone.utc) |
| 718 | except (TypeError, ValueError): |
| 719 | pass |
| 720 | return None |
| 721 | |
| 722 | |
| 723 | def _raw(value: Any) -> Any: |
| 724 | if isinstance(value, dict): |
| 725 | return value.get("raw", value.get("fmt")) |
| 726 | return value |
| 727 | |
| 728 | |
| 729 | def _normalized_quote_type(provider_type: Any, validated_name: Any) -> str: |
| 730 | quote_type = str(provider_type or "").upper() |
| 731 | name_tokens = {token.strip(".,()[]-").upper() for token in str(validated_name or "").split()} |
| 732 | # Yahoo labels some exchange-traded funds as EQUITY. The validated provider |
| 733 | # identity is stronger evidence when its actual security name explicitly says ETF. |
| 734 | if quote_type in {"EQUITY", "STOCK"} and "ETF" in name_tokens: |
| 735 | return "ETF" |
| 736 | return quote_type |
| 737 | |
| 738 | |
| 739 | def _decimal(value: Any) -> Decimal | None: |
| 740 | try: |
| 741 | number = Decimal(str(value)) if value is not None and not isinstance(value, bool) else None |
| 742 | return number if number is not None and number.is_finite() else None |
| 743 | except (InvalidOperation, ValueError): |
| 744 | return None |
| 745 | |
| 746 | |
| 747 | def _timestamp(value: Any) -> datetime | None: |
| 748 | value = _raw(value) |
| 749 | if isinstance(value, str): |
| 750 | try: |
| 751 | parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) |
| 752 | if parsed.tzinfo is not None: |
| 753 | return parsed.astimezone(timezone.utc) |
| 754 | except ValueError: |
| 755 | pass |
| 756 | try: |
| 757 | return datetime.fromtimestamp(float(value), tz=timezone.utc) if value is not None else None |
| 758 | except (ValueError, TypeError, OSError): |
| 759 | return None |
| 760 | |
| 761 | |
| 762 | def _uuid(value: Any) -> UUID | None: |
| 763 | try: |
| 764 | return UUID(str(value)) if value else None |
| 765 | except ValueError: |
| 766 | return None |
| 767 | |
| 768 | |
| 769 | def _identity_key(instrument: dict[str, Any], identity: str) -> str: |
| 770 | return "|".join(str(instrument.get(key) or "").upper() for key in |
| 771 | ("instrumentId", "isin", "provider", "providerInstrumentId", "structuredProviderTicker")) + "|" + identity.upper() |