| 1 | import json |
| 2 | import logging |
| 3 | import re |
| 4 | from contextvars import ContextVar, Token |
| 5 | from datetime import datetime, timezone |
| 6 | from uuid import UUID |
| 7 | |
| 8 | from pydantic import Field, field_validator |
| 9 | from pydantic_settings import BaseSettings, SettingsConfigDict |
| 10 | |
| 11 | |
| 12 | _LOG_LEVELS = {"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"} |
| 13 | _MAX_OFFICIAL_DOCUMENT_BYTES = 50 * 1024 * 1024 |
| 14 | _REQUEST_ID: ContextVar[str | None] = ContextVar("aip_request_id", default=None) |
| 15 | _SENSITIVE_LOG_VALUE = re.compile( |
| 16 | r"(?i)\b(authorization|cookie|password|token|api[_-]?(?:key|token)|" |
| 17 | r"access[_-]?token|refresh[_-]?token|request[_-]?token|client[_-]?secret)" |
| 18 | r"\s*[:=]\s*(?:bearer\s+)?([^\s,;&]+)" |
| 19 | ) |
| 20 | |
| 21 | |
| 22 | class Settings(BaseSettings): |
| 23 | model_config = SettingsConfigDict(env_prefix="AIP_", env_file=".env", extra="ignore") |
| 24 | |
| 25 | service_name: str = "research-engine" |
| 26 | environment: str = "LOCAL" |
| 27 | llm_provider: str = "ollama" |
| 28 | ollama_base_url: str = "http://ollama:11434" |
| 29 | research_live_enabled: bool = False |
| 30 | research_demo_enabled: bool = True |
| 31 | research_log_level: str = "WARNING" |
| 32 | research_user_agent: str = "AIInvestmentResearchBot/0.1 contact=research-compliance@example.invalid" |
| 33 | research_official_document_user_agent: str = "Mozilla/5.0" |
| 34 | research_official_document_max_bytes: int = 25 * 1024 * 1024 |
| 35 | research_request_timeout_seconds: float = 10.0 |
| 36 | research_connect_timeout_seconds: float = 3.0 |
| 37 | research_max_content_bytes: int = 1_500_000 |
| 38 | research_max_redirects: int = 5 |
| 39 | research_max_retries: int = 2 |
| 40 | # Official filings are fetched as part of an interactive refresh. Keep a |
| 41 | # single unhealthy archive host from consuming the whole refresh budget. |
| 42 | research_official_document_max_attempts_per_refresh: int = 3 |
| 43 | research_official_document_timeout_seconds: float = 8.0 |
| 44 | research_official_document_extraction_timeout_seconds: float = 12.0 |
| 45 | research_pdf_extraction_concurrency: int = 1 |
| 46 | research_official_document_max_transport_failures_per_host: int = 1 |
| 47 | research_playwright_enabled: bool = False |
| 48 | research_playwright_concurrency: int = 1 |
| 49 | research_search_enabled: bool = False |
| 50 | research_search_provider: str = "disabled" |
| 51 | research_search_endpoint: str | None = None |
| 52 | research_search_api_key: str | None = None |
| 53 | research_search_engine_id: str | None = None |
| 54 | research_search_window_months: int = 12 |
| 55 | research_search_max_queries_per_category: int = 6 |
| 56 | research_search_max_results_per_query: int = 5 |
| 57 | research_search_max_documents_per_refresh: int = 24 |
| 58 | research_search_refresh_cooldown_seconds: int = 3600 |
| 59 | research_search_cache_ttl_seconds: int = 86400 |
| 60 | research_search_allowed_domains: list[str] = [] |
| 61 | research_ownership_change_threshold_percentage_points: float = 0.10 |
| 62 | research_quarterly_freshness_seconds: int = 2_592_000 |
| 63 | research_shareholding_freshness_seconds: int = 2_592_000 |
| 64 | research_catalyst_freshness_seconds: int = 86_400 |
| 65 | research_annual_report_freshness_seconds: int = 15_552_000 |
| 66 | research_analyst_freshness_seconds: int = 604_800 |
| 67 | structured_provider_enabled: bool = True |
| 68 | structured_provider_timeout_seconds: float = 10.0 |
| 69 | yahoo_search_url: str = "https://query1.finance.yahoo.com/v1/finance/search" |
| 70 | yahoo_quote_url: str = "https://query1.finance.yahoo.com/v7/finance/quote" |
| 71 | yahoo_summary_url: str = "https://query2.finance.yahoo.com/v10/finance/quoteSummary/{ticker}" |
| 72 | nse_announcements_url: str = "https://www.nseindia.com/api/corporate-announcements" |
| 73 | nse_shareholdings_url: str = "https://www.nseindia.com/api/corporate-share-holdings-master" |
| 74 | structured_resolution_min_confidence: float = 0.62 |
| 75 | structured_resolution_ambiguity_margin: float = 0.08 |
| 76 | structured_market_price_freshness_seconds: int = 300 |
| 77 | structured_fundamentals_freshness_seconds: int = 86_400 |
| 78 | sec_edgar_ticker_endpoint: str = "https://www.sec.gov/files/company_tickers.json" |
| 79 | sec_edgar_companyfacts_endpoint: str = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json" |
| 80 | sec_edgar_user_agent: str = "AIInvestmentResearchBot/0.1 contact=research-compliance@example.invalid" |
| 81 | sec_edgar_timeout_seconds: float = 10.0 |
| 82 | eodhd_api_key: str | None = None |
| 83 | eodhd_base_url: str = "https://eodhd.com/api" |
| 84 | eodhd_timeout_seconds: float = 10.0 |
| 85 | structured_valuation_freshness_seconds: int = 86_400 |
| 86 | structured_analyst_freshness_seconds: int = 604_800 |
| 87 | portfolio_refresh_instrument_concurrency: int = 4 |
| 88 | research_persistence_enabled: bool = False |
| 89 | research_database_backend: str = "postgres" |
| 90 | research_database_host: str = "localhost" |
| 91 | research_database_port: int = 5432 |
| 92 | research_database_name: str = "investment" |
| 93 | research_database_user: str = "investment" |
| 94 | research_database_password: str | None = None |
| 95 | research_database_schema: str = "research" |
| 96 | research_database_ssl_mode: str = "disable" |
| 97 | research_database_connect_timeout_seconds: int = 10 |
| 98 | research_database_statement_timeout_seconds: int = 30 |
| 99 | research_distributed_lock_backend: str = "process" |
| 100 | research_distributed_lock_acquire_timeout_seconds: int = 30 |
| 101 | research_mcp_first_enabled: bool = False |
| 102 | research_mcp_gateway_base_url: str = "http://mcp-gateway" |
| 103 | research_mcp_gateway_timeout_seconds: float = 10.0 |
| 104 | research_mcp_service_identity: str = "research-engine" |
| 105 | research_readiness_ensure_timeout_seconds: float = 25.0 |
| 106 | portfolio_service_base_url: str = "http://portfolio-service" |
| 107 | market_data_nifty_refresh_timeout_seconds: float = 30.0 |
| 108 | market_data_population_batch_size: int = 50 |
| 109 | # Operational single-request bound, not a claimed NSE maximum. |
| 110 | nse_historical_request_window_days: int = Field(default=30, ge=1) |
| 111 | nse_historical_max_retries: int = Field(default=2, ge=0, le=3) |
| 112 | market_data_population_request_interval_seconds: float = 0.20 |
| 113 | market_data_population_initial_lookback_days: int = 400 |
| 114 | market_data_nifty_freshness_hours: int = 12 |
| 115 | market_data_historical_freshness_hours: int = 72 |
| 116 | market_data_ensure_retry_cooldown_minutes: int = 15 |
| 117 | market_data_population_retry_cooldown_hours: int = 12 |
| 118 | market_data_internal_user_id: str = "00000000-0000-0000-0000-000000000101" |
| 119 | market_data_internal_issuer: str = "aip-internal" |
| 120 | market_data_internal_subject: str = "research-engine-market-data" |
| 121 | |
| 122 | @field_validator("portfolio_refresh_instrument_concurrency") |
| 123 | @classmethod |
| 124 | def validate_portfolio_refresh_instrument_concurrency(cls, value: int) -> int: |
| 125 | if not 1 <= value <= 8: |
| 126 | raise ValueError("portfolio_refresh_instrument_concurrency must be between 1 and 8") |
| 127 | return value |
| 128 | |
| 129 | @field_validator("market_data_population_batch_size") |
| 130 | @classmethod |
| 131 | def validate_market_data_population_batch_size(cls, value: int) -> int: |
| 132 | if not 1 <= value <= 100: |
| 133 | raise ValueError("market_data_population_batch_size must be between 1 and 100") |
| 134 | return value |
| 135 | |
| 136 | @field_validator("market_data_nifty_refresh_timeout_seconds") |
| 137 | @classmethod |
| 138 | def validate_market_data_nifty_refresh_timeout_seconds(cls, value: float) -> float: |
| 139 | if not 10 <= value <= 120: |
| 140 | raise ValueError("market_data_nifty_refresh_timeout_seconds must be between 10 and 120") |
| 141 | return value |
| 142 | |
| 143 | @field_validator("market_data_population_request_interval_seconds") |
| 144 | @classmethod |
| 145 | def validate_market_data_population_request_interval_seconds(cls, value: float) -> float: |
| 146 | if not 0 <= value <= 10: |
| 147 | raise ValueError("market_data_population_request_interval_seconds must be between 0 and 10") |
| 148 | return value |
| 149 | |
| 150 | @field_validator("market_data_population_initial_lookback_days") |
| 151 | @classmethod |
| 152 | def validate_market_data_population_initial_lookback_days(cls, value: int) -> int: |
| 153 | if not 365 <= value <= 450: |
| 154 | raise ValueError("market_data_population_initial_lookback_days must be between 365 and 450") |
| 155 | return value |
| 156 | |
| 157 | @field_validator("market_data_nifty_freshness_hours", "market_data_historical_freshness_hours") |
| 158 | @classmethod |
| 159 | def validate_market_data_freshness_hours(cls, value: int) -> int: |
| 160 | if not 1 <= value <= 168: |
| 161 | raise ValueError("market-data freshness hours must be between 1 and 168") |
| 162 | return value |
| 163 | |
| 164 | @field_validator("market_data_ensure_retry_cooldown_minutes") |
| 165 | @classmethod |
| 166 | def validate_market_data_ensure_retry_cooldown_minutes(cls, value: int) -> int: |
| 167 | if not 1 <= value <= 1440: |
| 168 | raise ValueError("market_data_ensure_retry_cooldown_minutes must be between 1 and 1440") |
| 169 | return value |
| 170 | |
| 171 | @field_validator("market_data_population_retry_cooldown_hours") |
| 172 | @classmethod |
| 173 | def validate_market_data_population_retry_cooldown_hours(cls, value: int) -> int: |
| 174 | if not 1 <= value <= 168: |
| 175 | raise ValueError("market_data_population_retry_cooldown_hours must be between 1 and 168") |
| 176 | return value |
| 177 | |
| 178 | @field_validator("market_data_internal_user_id") |
| 179 | @classmethod |
| 180 | def validate_market_data_internal_user_id(cls, value: str) -> str: |
| 181 | UUID(value) |
| 182 | return value |
| 183 | |
| 184 | @field_validator("research_pdf_extraction_concurrency") |
| 185 | @classmethod |
| 186 | def validate_research_pdf_extraction_concurrency(cls, value: int) -> int: |
| 187 | if not 1 <= value <= 4: |
| 188 | raise ValueError("research_pdf_extraction_concurrency must be between 1 and 4") |
| 189 | return value |
| 190 | |
| 191 | @field_validator("research_log_level") |
| 192 | @classmethod |
| 193 | def validate_research_log_level(cls, value: str) -> str: |
| 194 | normalized = str(value).upper() |
| 195 | if normalized not in _LOG_LEVELS: |
| 196 | raise ValueError(f"Unsupported research log level: {value}") |
| 197 | return normalized |
| 198 | |
| 199 | @field_validator("environment") |
| 200 | @classmethod |
| 201 | def normalize_environment(cls, value: str) -> str: |
| 202 | normalized = str(value).upper() |
| 203 | if normalized not in {"LOCAL", "TEST", "AZURE", "DEV", "PRD"}: |
| 204 | raise ValueError(f"Unsupported environment profile: {value}") |
| 205 | return normalized |
| 206 | |
| 207 | @field_validator("research_database_ssl_mode") |
| 208 | @classmethod |
| 209 | def validate_database_ssl_mode(cls, value: str) -> str: |
| 210 | normalized = str(value).lower() |
| 211 | if normalized not in {"disable", "allow", "prefer", "require", "verify-ca", "verify-full"}: |
| 212 | raise ValueError(f"Unsupported PostgreSQL sslmode: {value}") |
| 213 | return normalized |
| 214 | |
| 215 | @field_validator("research_database_connect_timeout_seconds", "research_database_statement_timeout_seconds", |
| 216 | "research_distributed_lock_acquire_timeout_seconds") |
| 217 | @classmethod |
| 218 | def validate_positive_timeout(cls, value: int) -> int: |
| 219 | if not 1 <= value <= 300: |
| 220 | raise ValueError("database and lock timeouts must be between 1 and 300 seconds") |
| 221 | return value |
| 222 | |
| 223 | @field_validator("research_distributed_lock_backend") |
| 224 | @classmethod |
| 225 | def validate_distributed_lock_backend(cls, value: str) -> str: |
| 226 | normalized = str(value).lower() |
| 227 | if normalized not in {"process", "postgres", "redis", "disabled"}: |
| 228 | raise ValueError(f"Unsupported distributed lock backend: {value}") |
| 229 | return normalized |
| 230 | |
| 231 | @field_validator("research_mcp_gateway_base_url") |
| 232 | @classmethod |
| 233 | def validate_mcp_gateway_base_url(cls, value: str) -> str: |
| 234 | normalized = value.strip().rstrip("/") |
| 235 | if not normalized.startswith(("http://", "https://")): |
| 236 | raise ValueError("research MCP gateway base URL must be HTTP(S)") |
| 237 | return normalized |
| 238 | |
| 239 | @field_validator("research_mcp_gateway_timeout_seconds") |
| 240 | @classmethod |
| 241 | def validate_mcp_gateway_timeout(cls, value: float) -> float: |
| 242 | if not 0 < value <= 120: |
| 243 | raise ValueError("research MCP gateway timeout must be between 0 and 120 seconds") |
| 244 | return value |
| 245 | |
| 246 | @field_validator("research_readiness_ensure_timeout_seconds") |
| 247 | @classmethod |
| 248 | def validate_readiness_ensure_timeout(cls, value: float) -> float: |
| 249 | if not 1 <= value <= 120: |
| 250 | raise ValueError("research readiness ensure timeout must be between 1 and 120 seconds") |
| 251 | return value |
| 252 | |
| 253 | @field_validator("research_mcp_service_identity") |
| 254 | @classmethod |
| 255 | def validate_mcp_service_identity(cls, value: str) -> str: |
| 256 | normalized = value.strip() |
| 257 | if not normalized: |
| 258 | raise ValueError("research MCP service identity is required") |
| 259 | return normalized |
| 260 | |
| 261 | @field_validator("research_official_document_max_bytes") |
| 262 | @classmethod |
| 263 | def validate_official_document_max_bytes(cls, value: int) -> int: |
| 264 | if value <= 0 or value > _MAX_OFFICIAL_DOCUMENT_BYTES: |
| 265 | raise ValueError(f"Official document max bytes must be between 1 and {_MAX_OFFICIAL_DOCUMENT_BYTES}") |
| 266 | return value |
| 267 | |
| 268 | |
| 269 | def configure_application_logging(settings: Settings) -> None: |
| 270 | """Configure application logging without modifying Uvicorn loggers.""" |
| 271 | level = getattr(logging, settings.research_log_level) |
| 272 | |
| 273 | root_logger = logging.getLogger() |
| 274 | root_logger.setLevel(level) |
| 275 | |
| 276 | app_logger = logging.getLogger("app") |
| 277 | app_logger.setLevel(level) |
| 278 | app_logger.propagate = False |
| 279 | |
| 280 | if not any(getattr(handler, "_aip_application_handler", False) for handler in app_logger.handlers): |
| 281 | handler = logging.StreamHandler() |
| 282 | handler.setLevel(level) |
| 283 | handler.setFormatter(_StructuredLogFormatter(settings.service_name, settings.environment)) |
| 284 | handler._aip_application_handler = True |
| 285 | app_logger.addHandler(handler) |
| 286 | else: |
| 287 | for handler in app_logger.handlers: |
| 288 | if getattr(handler, "_aip_application_handler", False): |
| 289 | handler.setLevel(level) |
| 290 | |
| 291 | |
| 292 | def set_request_id(value: str) -> Token: |
| 293 | return _REQUEST_ID.set(value) |
| 294 | |
| 295 | |
| 296 | def reset_request_id(token: Token) -> None: |
| 297 | _REQUEST_ID.reset(token) |
| 298 | |
| 299 | |
| 300 | class _StructuredLogFormatter(logging.Formatter): |
| 301 | def __init__(self, service: str, environment: str) -> None: |
| 302 | super().__init__() |
| 303 | self.service = service |
| 304 | self.environment = environment |
| 305 | |
| 306 | def format(self, record: logging.LogRecord) -> str: |
| 307 | message = _SENSITIVE_LOG_VALUE.sub(r"\1=<redacted>", record.getMessage()).replace("\r", " ").replace("\n", " ") |
| 308 | payload = { |
| 309 | "timestamp": datetime.fromtimestamp(record.created, timezone.utc).isoformat(), |
| 310 | "service": self.service, |
| 311 | "environment": self.environment, |
| 312 | "level": record.levelname, |
| 313 | "event": record.name, |
| 314 | "requestId": _REQUEST_ID.get(), |
| 315 | "message": message, |
| 316 | } |
| 317 | return json.dumps(payload, separators=(",", ":"), default=str) |