@prakhar-gitlab / ai-investment-platform / commits / e3951c4

feat: add NSE sector benchmark history

prakhar82 committed Sep 13, 2026 at 23:27 UTC e3951c45478330891a3d7cf11119893ad500ec97
18 files changed +985 -25
ai/research-engine/app/global_scanner.py
+5 -7
@@ -314,12 +314,6 @@ class GlobalScanner:
314 candidates = [c for c in scan.candidates if c.eligible_for_deep_analysis]
315 ids = {c.global_instrument_id for c in candidates}
316 daily_histories = defaultdict(list)
317 - candidate_ids = sorted(ids, key=str)
318 - for offset in range(0, len(candidate_ids), self.batch_size):
319 - batch = set(candidate_ids[offset:offset+self.batch_size])
320 - for row in self.persistence.load_daily_market_bars(batch, provider="NSE"):
321 - if row.global_instrument_id in batch:
322 - daily_histories[row.global_instrument_id].append(row)
317 for candidate in candidates:
318 context = contexts.get(candidate.global_instrument_id, SectorContext())
319 for reference in (context.sector_benchmark, context.market_benchmark):
@@ -329,6 +323,9 @@ class GlobalScanner:
323 ordered = sorted(ids, key=str)
324 for offset in range(0, len(ordered), self.batch_size):
325 batch = set(ordered[offset:offset+self.batch_size])
326 + for row in self.persistence.load_daily_market_bars(batch, provider="NSE"):
327 + if row.global_instrument_id in batch:
328 + daily_histories[row.global_instrument_id].append(row)
329 for row in self.persistence.load_market_price_observations(batch):
330 if row.instrument_id in batch:
331 histories[row.instrument_id].append(row)
@@ -339,7 +336,8 @@ class GlobalScanner:
336 technical = technical_engine.compute(key, histories[key], as_of=scan.as_of, currency=candidate.currency,
337 trusted_providers=providers.get(key), daily_bar_history=daily_histories[key])
338 sector = sector_engine.compute(key, histories[key], as_of=scan.as_of, currency=candidate.currency,
342 - context=contexts.get(key), benchmark_histories=histories, trusted_providers=providers.get(key))
339 + context=contexts.get(key), benchmark_histories=histories, trusted_providers=providers.get(key),
340 + daily_bar_histories=daily_histories)
341 scores = [(technical.technical_score, technical_weight), (sector.relative_strength_score, sector_weight)]
342 available = [(score, weight) for score, weight in scores if score is not None and weight > 0]
343 available_weight = sum(weight for _, weight in available)
ai/research-engine/app/market_data_population.py
+31
@@ -79,6 +79,37 @@ class IndiaMarketDataPopulationJobs:
79 correlation_id=correlation_id, offset=offset, instrument_ids=instrument_ids,
80 start=start, end=end, force=force)
81
82 + async def populate_benchmark_history(self, instrument_ids: set[UUID], *, start: date, end: date,
83 + identity_headers: dict[str, str | None], correlation_id=None):
84 + """Explicit bounded index acquisition; registration is a separate master operation."""
85 + from app.nse_index_history import NseIndexHistoryProvider, ENDPOINT
86 + from app.nse_historical_daily import NseHistoricalResult, persist_daily_result
87 + if len(instrument_ids) > self.settings.market_data_population_batch_size:
88 + raise ValueError('BENCHMARK_BATCH_LIMIT')
89 + results = []
90 + async with self._daily_bar_lock:
91 + provider = NseIndexHistoryProvider(self.orchestrator, self.settings)
92 + try:
93 + for key in sorted(instrument_ids, key=str):
94 + now = self._clock()
95 + failures = [t for t in (self._daily_bar_failures.get(key), self._daily_bar_throttled_at) if t]
96 + failed_at = max(failures) if failures else None
97 + if failed_at and now - failed_at < timedelta(hours=self.settings.market_data_population_retry_cooldown_hours):
98 + results.append(NseHistoricalResult(key, start, end, source_url=ENDPOINT, failure_reason='RETRY_COOLDOWN'))
99 + continue
100 + result = await provider.fetch(key, start=start, end=end,
101 + identity_headers=identity_headers, correlation_id=correlation_id)
102 + result = await persist_daily_result(self.repository, result)
103 + results.append(result)
104 + if result.failure_reason:
105 + self._daily_bar_failures[key] = self._clock()
106 + if '429' in result.failure_reason:
107 + self._daily_bar_throttled_at = self._clock()
108 + finally:
109 + await provider.aclose()
110 + await self._sleep(self.settings.market_data_population_request_interval_seconds)
111 + return results
112 +
113 async def populate_daily_bars(
114 self, global_instrument_id: UUID, *, start: date, end: date,
115 identity_headers: dict[str, str | None], correlation_id: str | None = None,
ai/research-engine/app/nse_index_history.py new
+109
@@ -0,0 +1,109 @@
1 +"""Official index JSON acquisition; reuses NSE transport, not equity CSV parsing."""
2 +from datetime import date, datetime, timezone
3 +from decimal import Decimal
4 +import json
5 +
6 +import httpx
7 +from app.models import DailyMarketBar
8 +from app.nse_historical_daily import NseHistoricalDailyProvider, NseHistoricalResult, BOOTSTRAP, number, trading_date
9 +from app.sector_benchmarks import BENCHMARKS, benchmark_id, benchmark_identity
10 +
11 +ENDPOINT = 'https://www.nseindia.com/api/historicalOR/indicesHistory'
12 +HISTORY_VERSION = 'NSE_INDEX_HISTORY_V1'
13 +# Exact response names observed in the checked-in official response fixtures.
14 +# These are contract aliases, never canonical identities or fuzzy matches.
15 +RESPONSE_NAMES = {'NIFTY 500': 'NIFTY 500', 'NIFTY IT': 'NIFTY IT',
16 + 'NIFTY FINANCIAL SERVICES': 'NIFTY FIN SERVICE',
17 + 'NIFTY HEALTHCARE INDEX': 'NIFTY HEALTHCARE'}
18 +REQUIRED = {'EOD_INDEX_NAME', 'EOD_TIMESTAMP', 'EOD_OPEN_INDEX_VAL', 'EOD_HIGH_INDEX_VAL',
19 + 'EOD_LOW_INDEX_VAL', 'EOD_CLOSE_INDEX_VAL'}
20 +
21 +
22 +def parse_index_history(content, result, currency):
23 + try:
24 + payload = json.loads(content.decode('utf-8-sig'), parse_float=Decimal)
25 + except (ValueError, UnicodeError):
26 + raise ValueError('NON_JSON_INDEX_RESPONSE') from None
27 + data = payload.get('data') if isinstance(payload, dict) else None
28 + if not isinstance(data, list):
29 + raise ValueError('INVALID_INDEX_RESPONSE_CONTRACT')
30 + if not data:
31 + raise ValueError('EMPTY_RESPONSE')
32 + accepted = {}
33 + for row in data:
34 + result.rows_parsed += 1
35 + if not isinstance(row, dict) or not REQUIRED.issubset(row):
36 + raise ValueError('MISSING_INDEX_FIELDS')
37 + result.headers = sorted(row)
38 + if ' '.join(str(row['EOD_INDEX_NAME']).upper().split()) != RESPONSE_NAMES.get(result.provider_symbol):
39 + raise ValueError('INDEX_SYMBOL_MISMATCH')
40 + try:
41 + day = trading_date(row['EOD_TIMESTAMP'])
42 + if not result.request_from <= day <= result.request_to or day in accepted:
43 + raise ValueError('INVALID_OR_DUPLICATE_INDEX_DATE')
44 + bar = DailyMarketBar(global_instrument_id=result.global_instrument_id, trading_date=day,
45 + **{name:number(str(row[field]), required=True) for name,field in (
46 + ('open','EOD_OPEN_INDEX_VAL'),('high','EOD_HIGH_INDEX_VAL'),
47 + ('low','EOD_LOW_INDEX_VAL'),('close','EOD_CLOSE_INDEX_VAL'))},
48 + currency=currency, provider='NSE', provider_symbol=result.provider_symbol,
49 + source_mode='REAL', source_url=result.source_url, retrieved_at=result.retrieved_at,
50 + volume=None, turnover=None, previous_close=None)
51 + except (ValueError, TypeError, KeyError):
52 + raise ValueError('INVALID_INDEX_ROW') from None
53 + accepted[day] = bar
54 + result.bars = [accepted[day] for day in sorted(accepted)]
55 +
56 +
57 +class NseIndexHistoryProvider:
58 + def __init__(self, orchestrator, settings, *, client=None, sleep=None):
59 + kwargs = {'client':client}
60 + if sleep is not None:
61 + kwargs['sleep'] = sleep
62 + self.transport = NseHistoricalDailyProvider(orchestrator, settings, **kwargs)
63 + self.orchestrator, self.settings = orchestrator, settings
64 +
65 + async def aclose(self):
66 + await self.transport.aclose()
67 +
68 + async def fetch(self, global_instrument_id, *, start, end, identity_headers=None, correlation_id=None):
69 + result = NseHistoricalResult(global_instrument_id, start, end, source_url=ENDPOINT)
70 + stage = 'IDENTITY'
71 + try:
72 + keys = [key for key in BENCHMARKS if benchmark_id(key) == global_instrument_id]
73 + if len(keys) != 1:
74 + raise ValueError('BENCHMARK_IDENTITY_UNAVAILABLE')
75 + if type(start) is not date or type(end) is not date or start > end or (end-start).days+1 > self.settings.nse_historical_request_window_days:
76 + raise ValueError('INVALID_REQUEST_WINDOW')
77 + key = keys[0]
78 + metadata = await self.orchestrator.global_instrument_metadata(global_instrument_id,
79 + identity_headers=identity_headers, correlation_id=correlation_id)
80 + ref = benchmark_identity(metadata, key)
81 + result.provider_symbol = BENCHMARKS[key]
82 + async with self.transport._lock:
83 + if not self.transport._bootstrapped:
84 + stage = 'BOOTSTRAP'
85 + await self.transport._get(BOOTSTRAP, result)
86 + self.transport._bootstrapped = True
87 + stage = 'HISTORICAL'
88 + response = await self.transport._get(ENDPOINT, result, params={'indexType':result.provider_symbol,
89 + 'from':start.strftime('%d-%m-%Y'),'to':end.strftime('%d-%m-%Y')})
90 + result.retrieved_at = datetime.now(timezone.utc)
91 + result.source_url = str(response.url)
92 + parse_index_history(response.content, result, ref.currency)
93 + result.status = 'SUCCESS'
94 + except httpx.HTTPStatusError as exc:
95 + result.failure_reason = f'{stage}_HTTP_{exc.response.status_code}'
96 + if exc.response.status_code == 403:
97 + self.transport._bootstrapped = False
98 + except httpx.HTTPError:
99 + result.failure_reason = f'{stage}_PROVIDER_UNAVAILABLE'
100 + except ValueError as exc:
101 + safe = {'BENCHMARK_IDENTITY_UNAVAILABLE','INVALID_REQUEST_WINDOW','NON_JSON_INDEX_RESPONSE',
102 + 'INVALID_INDEX_RESPONSE_CONTRACT','EMPTY_RESPONSE','MISSING_INDEX_FIELDS','INDEX_SYMBOL_MISMATCH','INVALID_INDEX_ROW'}
103 + result.failure_reason = str(exc) if str(exc) in safe else f'{stage}_UNAVAILABLE'
104 + except Exception:
105 + result.failure_reason = f'{stage}_UNAVAILABLE'
106 + if result.failure_reason:
107 + result.rows_rejected = result.rows_parsed
108 + result.bars.clear()
109 + return result
ai/research-engine/app/portfolio_orchestration.py
+14
@@ -91,6 +91,20 @@ class PortfolioResearchOrchestrator:
91 except (httpx.HTTPError, ValueError) as exc:
92 raise PortfolioServiceUnavailableError("Portfolio service unavailable for instrument enumeration") from exc
93
94 + async def sector_benchmark_contexts(self, instrument_ids: set[UUID], *, correlation_id=None, identity_headers=None):
95 + """Read-only canonical dependencies, prepared before pure Stage-B computation."""
96 + if not instrument_ids:
97 + return {}
98 + from app.sector_benchmarks import build_sector_contexts
99 + classifications = await self.india_nifty500_universe(correlation_id=correlation_id, identity_headers=identity_headers)
100 + response = await self._client.get(f"{self.settings.portfolio_service_base_url}/api/v1/instruments/benchmarks",
101 + headers={k:v for k,v in (identity_headers or {}).items() if v})
102 + response.raise_for_status()
103 + registered = response.json()
104 + if not isinstance(registered, list):
105 + raise PortfolioServiceUnavailableError('BENCHMARK_IDENTITY_UNAVAILABLE')
106 + return build_sector_contexts(classifications, registered, instrument_ids)
107 +
108 async def india_nifty500_universe(self, *, correlation_id: str | None = None, identity_headers: dict[str, str | None] | None = None) -> list[dict]:
109 """Read portfolio-service owned NSE/Nifty universe; never uses portfolios."""
110 headers = {key: value for key, value in (identity_headers or {}).items() if value}
ai/research-engine/app/sector_benchmarks.py new
+87
@@ -0,0 +1,87 @@
1 +"""Versioned platform benchmark keys resolved only through registered master rows."""
2 +from hashlib import md5
3 +from uuid import UUID
4 +from datetime import datetime
5 +
6 +from app.sector_relative_strength import BenchmarkReference, SectorContext
7 +
8 +MAPPING_VERSION = 'SECTOR_BENCHMARK_MAPPING_V1'
9 +CATALOG_VERSION = 'NSE_BENCHMARK_CATALOG_V1'
10 +BENCHMARKS = {
11 + 'INDIA_BROAD_PRICE': 'NIFTY 500',
12 + 'INDIA_TECHNOLOGY_PRICE': 'NIFTY IT',
13 + 'INDIA_FINANCIALS_PRICE': 'NIFTY FINANCIAL SERVICES',
14 + 'INDIA_HEALTHCARE_PRICE': 'NIFTY HEALTHCARE INDEX',
15 +}
16 +SECTOR_KEYS = {'Technology': 'INDIA_TECHNOLOGY_PRICE', 'Financials': 'INDIA_FINANCIALS_PRICE',
17 + 'Healthcare': 'INDIA_HEALTHCARE_PRICE'}
18 +BROAD_KEY = 'INDIA_BROAD_PRICE'
19 +
20 +
21 +def benchmark_id(key):
22 + """Same platform-key UUIDv3 as portfolio-service; never a provider-symbol UUID."""
23 + if key not in BENCHMARKS:
24 + raise ValueError('UNKNOWN_BENCHMARK_KEY')
25 + return UUID(bytes=md5(('aip:benchmark:' + key).encode(), usedforsecurity=False).digest(), version=3)
26 +
27 +
28 +def benchmark_identity(metadata, key):
29 + if not isinstance(metadata, dict):
30 + raise ValueError('BENCHMARK_IDENTITY_UNAVAILABLE')
31 + mappings = metadata.get('providerMappings') or []
32 + nse = [m for m in mappings if isinstance(m, dict) and m.get('provider') == 'NSE']
33 + if (str(metadata.get('globalInstrumentId')) != str(benchmark_id(key))
34 + or metadata.get('assetType') != 'INDEX' or metadata.get('status') != 'ACTIVE'
35 + or metadata.get('country') != 'IN' or metadata.get('primaryExchange') != 'NSE'
36 + or metadata.get('currency') != 'INR' or len(nse) != 1):
37 + raise ValueError('BENCHMARK_IDENTITY_UNAVAILABLE')
38 + mapping = nse[0]
39 + if (mapping.get('status') != 'VERIFIED' or mapping.get('active') is False
40 + or mapping.get('resolutionSource') != CATALOG_VERSION or mapping.get('currency') != 'INR'
41 + or mapping.get('exchange') != 'NSE' or mapping.get('providerSymbol') != BENCHMARKS[key]):
42 + raise ValueError('BENCHMARK_IDENTITY_UNAVAILABLE')
43 + return BenchmarkReference(benchmark_id(key), 'INR', frozenset({'NSE'}))
44 +
45 +
46 +def build_sector_contexts(classifications, registered, instrument_ids):
47 + """Pure adapter over canonical Nifty500 cache and registered benchmark metadata."""
48 + by_id = {}
49 + for row in registered:
50 + by_id.setdefault(str(row.get('globalInstrumentId')), []).append(row)
51 + def reference(key):
52 + matches = by_id.get(str(benchmark_id(key)), [])
53 + try:
54 + return benchmark_identity(matches[0], key) if len(matches) == 1 else None
55 + except ValueError:
56 + return None
57 + broad = reference(BROAD_KEY)
58 + grouped = {}
59 + for row in classifications:
60 + grouped.setdefault(str(row.get('globalInstrumentId')), []).append(row)
61 + output = {}
62 + for key in sorted(instrument_ids, key=str):
63 + matches = grouped.get(str(key), [])
64 + row = matches[0] if len(matches) == 1 else {}
65 + sector = row.get('canonicalSector')
66 + stamp = row.get('retrievedAt')
67 + try:
68 + stamp = datetime.fromisoformat(stamp.replace('Z', '+00:00')) if isinstance(stamp, str) else stamp
69 + except ValueError:
70 + stamp = None
71 + india_member = (row.get('source') == 'NSE_INDICES_NIFTY500' and row.get('status') == 'ACTIVE'
72 + and row.get('assetType') == 'EQUITY' and row.get('country') == 'IN'
73 + and row.get('exchange') in {'NSE', 'XNSE'})
74 + if not india_member:
75 + sector = None
76 + target = SECTOR_KEYS.get(sector)
77 + sector_ref = reference(target) if target else None
78 + classification_ok = bool(sector and stamp)
79 + status = ('NO_SECTOR_CLASSIFICATION' if not classification_ok else
80 + 'UNMAPPED_SECTOR_BENCHMARK' if not target else
81 + 'BENCHMARK_IDENTITY_UNAVAILABLE' if sector_ref is None else 'AVAILABLE')
82 + market_ref = broad if india_member else None
83 + output[key] = SectorContext(sector=sector, source=row.get('source'), as_of=stamp, region='INDIA' if india_member else None,
84 + sector_benchmark=sector_ref if classification_ok else None, market_benchmark=market_ref,
85 + mapping_version=MAPPING_VERSION, sector_mapping_status=status,
86 + market_mapping_status='AVAILABLE' if market_ref else 'BENCHMARK_IDENTITY_UNAVAILABLE')
87 + return output
ai/research-engine/app/sector_relative_strength.py
+56 -17
@@ -14,9 +14,10 @@ from uuid import UUID
14
15 from pydantic import Field
16
17 -from app.models import MarketPriceObservation, ResearchBaseModel
17 +from app.models import DailyMarketBar, MarketPriceObservation, ResearchBaseModel
18 from app.sector_leaderboard import normalize_sector
19 -from app.technical_features import normalize_price_history, percentage, utc
19 +from app.technical_features import normalize_price_history, normalize_daily_history, percentage, utc
20 +from zoneinfo import ZoneInfo
21
22
23 SECTOR_FEATURE_VERSION = "SECTOR_RELATIVE_STRENGTH_V1"
@@ -43,6 +44,9 @@ class SectorContext:
44 region: str | None = None
45 sector_benchmark: BenchmarkReference | None = None
46 market_benchmark: BenchmarkReference | None = None
47 + mapping_version: str | None = None
48 + sector_mapping_status: str | None = None
49 + market_mapping_status: str | None = None
50
51
52 @dataclass(frozen=True)
@@ -73,6 +77,9 @@ class SectorRelativeStrengthSnapshot(ResearchBaseModel):
77 as_of: datetime
78 feature_version: str = SECTOR_FEATURE_VERSION
79 configuration: dict
80 + benchmark_mapping_version: str | None = None
81 + benchmark_states: dict[str, str] = Field(default_factory=dict)
82 + history_sources: dict[str, str] = Field(default_factory=dict)
83 sector: str | None = None
84 classification_source: str | None = None
85 classification_as_of: datetime | None = None
@@ -116,9 +123,11 @@ class SectorRelativeStrengthEngine:
123 def compute(self, instrument_id: UUID, stock_history: Iterable[MarketPriceObservation], *, as_of: datetime,
124 context: SectorContext | None = None, currency: str | None = None,
125 benchmark_histories: Mapping[UUID, Iterable[MarketPriceObservation]] | None = None,
119 - trusted_providers: frozenset[str] | None = None) -> SectorRelativeStrengthSnapshot:
126 + trusted_providers: frozenset[str] | None = None,
127 + daily_bar_histories: Mapping[UUID, Iterable[DailyMarketBar]] | None = None) -> SectorRelativeStrengthSnapshot:
128 cfg, context = self.config, context or SectorContext()
129 histories = benchmark_histories or {}
130 + daily = daily_bar_histories or {}
131 classification_valid = bool(context.sector and context.source and context.as_of is not None and utc(context.as_of) <= utc(as_of))
132 result = SectorRelativeStrengthSnapshot(global_instrument_id=instrument_id, as_of=utc(as_of), configuration=asdict(cfg),
133 sector=normalize_sector(context.sector)[1] if classification_valid else None,
@@ -126,38 +135,47 @@ class SectorRelativeStrengthEngine:
135 classification_as_of=utc(context.as_of) if classification_valid else None, region=context.region,
136 sector_benchmark_id=context.sector_benchmark.instrument_id if context.sector_benchmark and classification_valid else None,
137 market_benchmark_id=context.market_benchmark.instrument_id if context.market_benchmark else None)
138 + result.benchmark_mapping_version = context.mapping_version
139 if not classification_valid:
140 result.missing_inputs.append("AUTHORITATIVE_SECTOR_CLASSIFICATION")
131 - stock = normalize_price_history(instrument_id, stock_history, as_of=as_of, currency=currency,
132 - trusted_providers=trusted_providers)
133 - stock_stale = bool(stock.observations) and utc(as_of) - utc(stock.observations[-1].observed_at) > timedelta(days=cfg.max_age_days)
141 + stock_dates, stock_conflict, stock_stale, source = _dated_history(instrument_id, stock_history,
142 + daily.get(instrument_id, ()), as_of, currency, trusted_providers, cfg.max_age_days, stock=True)
143 + result.history_sources['stock'] = source
144 if stock_stale:
145 result.stale_inputs.append("STOCK_HISTORY")
136 - if stock.current_conflict:
146 + if stock_conflict:
147 result.missing_inputs.append("CONFLICTING_STOCK_PRICE")
148 benchmark_data = {}
149 for name, reference in (("sector", context.sector_benchmark if classification_valid else None), ("market", context.market_benchmark)):
150 if reference is None:
151 result.missing_inputs.append(f"{name.upper()}_BENCHMARK_MAPPING")
152 + result.benchmark_states[name] = ('NO_SECTOR_CLASSIFICATION' if name == 'sector' and not classification_valid
153 + else getattr(context, f'{name}_mapping_status') or
154 + ('UNMAPPED_SECTOR_BENCHMARK' if name == 'sector' else 'BENCHMARK_IDENTITY_UNAVAILABLE'))
155 continue
156 if reference.instrument_id == instrument_id or not reference.currency:
157 result.missing_inputs.append(f"INVALID_{name.upper()}_BENCHMARK_MAPPING")
158 + result.benchmark_states[name] = 'BENCHMARK_IDENTITY_UNAVAILABLE'
159 continue
146 - history = normalize_price_history(reference.instrument_id, histories.get(reference.instrument_id, ()),
147 - as_of=as_of, currency=reference.currency, trusted_providers=reference.trusted_providers)
148 - if not history.observations or history.current_conflict:
149 - result.missing_inputs.append(f"{name.upper()}_HISTORY" if not history.current_conflict else f"CONFLICTING_{name.upper()}_PRICE")
160 + dates, conflict, stale, source = _dated_history(reference.instrument_id, histories.get(reference.instrument_id, ()),
161 + daily.get(reference.instrument_id, ()), as_of, reference.currency, reference.trusted_providers, cfg.max_age_days)
162 + result.history_sources[name] = source
163 + if not dates or conflict:
164 + result.missing_inputs.append(f"{name.upper()}_HISTORY" if not conflict else f"CONFLICTING_{name.upper()}_PRICE")
165 + result.benchmark_states[name] = 'BENCHMARK_HISTORY_UNAVAILABLE'
166 continue
151 - if utc(as_of) - utc(history.observations[-1].observed_at) > timedelta(days=cfg.max_age_days):
167 + if stale:
168 result.stale_inputs.append(f"{name.upper()}_HISTORY")
169 + result.benchmark_states[name] = 'STALE_BENCHMARK_HISTORY'
170 continue
154 - benchmark_data[name] = {utc(row.observed_at).date(): float(row.price) for row in history.observations}
171 + benchmark_data[name] = dates
172 + result.benchmark_states[name] = 'INSUFFICIENT_OVERLAP'
173 edges = {}
174 for suffix, lookback, weight in zip(("1_w", "1_m", "3_m", "6_m"), cfg.return_lookbacks, cfg.period_weights):
157 - if not stock.current_conflict and len(stock.observations) > lookback:
158 - start, end = stock.observations[-lookback-1], stock.observations[-1]
159 - dates = utc(start.observed_at).date(), utc(end.observed_at).date()
160 - stock_return = percentage(float(end.price), float(start.price))
175 + if not stock_conflict and len(stock_dates) > lookback:
176 + ordered = sorted(stock_dates)
177 + dates = ordered[-lookback-1], ordered[-1]
178 + stock_return = percentage(stock_dates[dates[1]], stock_dates[dates[0]])
179 setattr(result, f"stock_return{suffix}", stock_return)
180 result.comparison_windows[suffix.replace("_", "").upper()] = dates
181 period_edges = []
@@ -168,6 +186,7 @@ class SectorRelativeStrengthEngine:
186 setattr(result, f"{name}_return{suffix}", value)
187 setattr(result, f"relative_vs_{name}{suffix}", stock_return - value)
188 period_edges.append((stock_return - value) / lookback)
189 + result.benchmark_states[name] = 'AVAILABLE'
190 elif rows and not stock_stale:
191 result.missing_inputs.append(f"{name.upper()}_ALIGNED_DATES_{suffix.replace('_', '').upper()}")
192 if period_edges:
@@ -210,3 +229,23 @@ class SectorRelativeStrengthEngine:
229 result.missing_inputs = sorted(set(result.missing_inputs))
230 result.stale_inputs = sorted(set(result.stale_inputs))
231 return result
232 +
233 +
234 +def _dated_history(key, closes, bars, as_of, currency, providers, max_age, stock=False):
235 + """Keep daily DATEs intact; no timestamp shift, filling, or nearest-date match."""
236 + history, present = normalize_daily_history(key, bars, as_of=as_of, currency=currency, trusted_providers=providers)
237 + today = utc(as_of).astimezone(ZoneInfo('Asia/Kolkata')).date()
238 + stale = bool(history.observations) and today - history.observations[-1].trading_date > timedelta(days=max_age)
239 + fallback = None
240 + if present and stock and not history.current_conflict and (len(history.observations) < 20 or stale):
241 + fallback = normalize_price_history(key, closes, as_of=as_of, currency=currency, trusted_providers=providers)
242 + if (len(fallback.observations) >= 20 and not fallback.current_conflict and
243 + utc(as_of) - utc(fallback.observations[-1].observed_at) <= timedelta(days=max_age)):
244 + present = False
245 + if present:
246 + return ({row.trading_date:float(row.close) for row in history.observations}, history.current_conflict,
247 + stale, 'DAILY_MARKET_BAR_NSE')
248 + fallback = fallback or normalize_price_history(key, closes, as_of=as_of, currency=currency, trusted_providers=providers)
249 + stale = bool(fallback.observations) and utc(as_of) - utc(fallback.observations[-1].observed_at) > timedelta(days=max_age)
250 + return ({utc(row.observed_at).date():float(row.price) for row in fallback.observations},
251 + fallback.current_conflict, stale, 'CLOSE_ONLY_FALLBACK')
ai/research-engine/tests/fixtures/nse_index_500.json new
+1
@@ -0,0 +1 @@
1 +{"data":[{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":22774.05,"EOD_HIGH_INDEX_VAL":22894.55,"EOD_CLOSE_INDEX_VAL":22866.5,"EOD_LOW_INDEX_VAL":22672.85,"HIT_TURN_OVER":75777.72,"HIT_TRADED_QTY":2584529506,"EOD_TIMESTAMP":"11-SEP-2026","HI_TIMESTAMP":"2026-09-10T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":22966.65,"EOD_HIGH_INDEX_VAL":22993.5,"EOD_CLOSE_INDEX_VAL":22952.35,"EOD_LOW_INDEX_VAL":22884.6,"HIT_TURN_OVER":68192.4,"HIT_TRADED_QTY":2567863094,"EOD_TIMESTAMP":"10-SEP-2026","HI_TIMESTAMP":"2026-09-09T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23015.8,"EOD_HIGH_INDEX_VAL":23061.7,"EOD_CLOSE_INDEX_VAL":22958.75,"EOD_LOW_INDEX_VAL":22957.6,"HIT_TURN_OVER":85249.17,"HIT_TRADED_QTY":2333313018,"EOD_TIMESTAMP":"09-SEP-2026","HI_TIMESTAMP":"2026-09-08T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23142.45,"EOD_HIGH_INDEX_VAL":23151.45,"EOD_CLOSE_INDEX_VAL":23106.1,"EOD_LOW_INDEX_VAL":23063.75,"HIT_TURN_OVER":69960.9,"HIT_TRADED_QTY":1902827941,"EOD_TIMESTAMP":"08-SEP-2026","HI_TIMESTAMP":"2026-09-07T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23248.05,"EOD_HIGH_INDEX_VAL":23260.4,"EOD_CLOSE_INDEX_VAL":23156.2,"EOD_LOW_INDEX_VAL":23124.1,"HIT_TURN_OVER":64553.48,"HIT_TRADED_QTY":2211977774,"EOD_TIMESTAMP":"07-SEP-2026","HI_TIMESTAMP":"2026-09-06T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23286.5,"EOD_HIGH_INDEX_VAL":23349.9,"EOD_CLOSE_INDEX_VAL":23254.15,"EOD_LOW_INDEX_VAL":23254.15,"HIT_TURN_OVER":82366.28,"HIT_TRADED_QTY":2623135270,"EOD_TIMESTAMP":"04-SEP-2026","HI_TIMESTAMP":"2026-09-03T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23300.75,"EOD_HIGH_INDEX_VAL":23332.85,"EOD_CLOSE_INDEX_VAL":23254.05,"EOD_LOW_INDEX_VAL":23237.25,"HIT_TURN_OVER":73802.82,"HIT_TRADED_QTY":1979780573,"EOD_TIMESTAMP":"03-SEP-2026","HI_TIMESTAMP":"2026-09-02T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23176.95,"EOD_HIGH_INDEX_VAL":23226.2,"EOD_CLOSE_INDEX_VAL":23222.8,"EOD_LOW_INDEX_VAL":23058.3,"HIT_TURN_OVER":81439.03,"HIT_TRADED_QTY":2494416409,"EOD_TIMESTAMP":"02-SEP-2026","HI_TIMESTAMP":"2026-09-01T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23429.65,"EOD_HIGH_INDEX_VAL":23435.65,"EOD_CLOSE_INDEX_VAL":23339.9,"EOD_LOW_INDEX_VAL":23243.25,"HIT_TURN_OVER":84191.15,"HIT_TRADED_QTY":2010695056,"EOD_TIMESTAMP":"01-SEP-2026","HI_TIMESTAMP":"2026-08-31T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23477.75,"EOD_HIGH_INDEX_VAL":23485.15,"EOD_CLOSE_INDEX_VAL":23450.35,"EOD_LOW_INDEX_VAL":23307.2,"HIT_TURN_OVER":144198.89,"HIT_TRADED_QTY":3492858587,"EOD_TIMESTAMP":"31-AUG-2026","HI_TIMESTAMP":"2026-08-30T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23513.05,"EOD_HIGH_INDEX_VAL":23552.3,"EOD_CLOSE_INDEX_VAL":23528.55,"EOD_LOW_INDEX_VAL":23451.1,"HIT_TURN_OVER":74118.09,"HIT_TRADED_QTY":2141439146,"EOD_TIMESTAMP":"28-AUG-2026","HI_TIMESTAMP":"2026-08-27T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23615.65,"EOD_HIGH_INDEX_VAL":23642.85,"EOD_CLOSE_INDEX_VAL":23482,"EOD_LOW_INDEX_VAL":23480.15,"HIT_TURN_OVER":81081.51,"HIT_TRADED_QTY":2071692443,"EOD_TIMESTAMP":"27-AUG-2026","HI_TIMESTAMP":"2026-08-26T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23641.35,"EOD_HIGH_INDEX_VAL":23684.1,"EOD_CLOSE_INDEX_VAL":23559.2,"EOD_LOW_INDEX_VAL":23557.95,"HIT_TURN_OVER":80306.45,"HIT_TRADED_QTY":2453487297,"EOD_TIMESTAMP":"26-AUG-2026","HI_TIMESTAMP":"2026-08-25T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23483.95,"EOD_HIGH_INDEX_VAL":23613.9,"EOD_CLOSE_INDEX_VAL":23611.6,"EOD_LOW_INDEX_VAL":23423.35,"HIT_TURN_OVER":76962.3,"HIT_TRADED_QTY":3210315086,"EOD_TIMESTAMP":"25-AUG-2026","HI_TIMESTAMP":"2026-08-24T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23560.6,"EOD_HIGH_INDEX_VAL":23587.85,"EOD_CLOSE_INDEX_VAL":23511.75,"EOD_LOW_INDEX_VAL":23454.1,"HIT_TURN_OVER":73770.36,"HIT_TRADED_QTY":1957565972,"EOD_TIMESTAMP":"24-AUG-2026","HI_TIMESTAMP":"2026-08-23T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23554.15,"EOD_HIGH_INDEX_VAL":23554.15,"EOD_CLOSE_INDEX_VAL":23530.3,"EOD_LOW_INDEX_VAL":23477,"HIT_TURN_OVER":76597.47,"HIT_TRADED_QTY":1695748201,"EOD_TIMESTAMP":"21-AUG-2026","HI_TIMESTAMP":"2026-08-20T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23519.8,"EOD_HIGH_INDEX_VAL":23565.9,"EOD_CLOSE_INDEX_VAL":23512.95,"EOD_LOW_INDEX_VAL":23491.6,"HIT_TURN_OVER":73080.22,"HIT_TRADED_QTY":1721057248,"EOD_TIMESTAMP":"20-AUG-2026","HI_TIMESTAMP":"2026-08-19T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23475.35,"EOD_HIGH_INDEX_VAL":23488.25,"EOD_CLOSE_INDEX_VAL":23386.2,"EOD_LOW_INDEX_VAL":23348.3,"HIT_TURN_OVER":75345.71,"HIT_TRADED_QTY":1799195227,"EOD_TIMESTAMP":"19-AUG-2026","HI_TIMESTAMP":"2026-08-18T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23521.85,"EOD_HIGH_INDEX_VAL":23555.3,"EOD_CLOSE_INDEX_VAL":23472.4,"EOD_LOW_INDEX_VAL":23470.9,"HIT_TURN_OVER":71834.51,"HIT_TRADED_QTY":2276032124,"EOD_TIMESTAMP":"18-AUG-2026","HI_TIMESTAMP":"2026-08-17T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23581.75,"EOD_HIGH_INDEX_VAL":23617.3,"EOD_CLOSE_INDEX_VAL":23564.45,"EOD_LOW_INDEX_VAL":23488.45,"HIT_TURN_OVER":68597.08,"HIT_TRADED_QTY":1837199203,"EOD_TIMESTAMP":"17-AUG-2026","HI_TIMESTAMP":"2026-08-16T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23635.65,"EOD_HIGH_INDEX_VAL":23638.3,"EOD_CLOSE_INDEX_VAL":23594.9,"EOD_LOW_INDEX_VAL":23564.3,"HIT_TURN_OVER":74739.54,"HIT_TRADED_QTY":2287796107,"EOD_TIMESTAMP":"14-AUG-2026","HI_TIMESTAMP":"2026-08-13T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY 500","EOD_OPEN_INDEX_VAL":23678.7,"EOD_HIGH_INDEX_VAL":23678.7,"EOD_CLOSE_INDEX_VAL":23658.1,"EOD_LOW_INDEX_VAL":23585.6,"HIT_TURN_OVER":80099.77,"HIT_TRADED_QTY":1996962847,"EOD_TIMESTAMP":"13-AUG-2026","HI_TIMESTAMP":"2026-08-12T18:30:00.000Z"}]}
\ No newline at end of file
ai/research-engine/tests/fixtures/nse_index_financials.json new
+1
@@ -0,0 +1 @@
1 +{"data":[{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":25247.3,"EOD_HIGH_INDEX_VAL":25576.95,"EOD_CLOSE_INDEX_VAL":25545.4,"EOD_LOW_INDEX_VAL":25124.25,"HIT_TURN_OVER":10149.97,"HIT_TRADED_QTY":133588753,"EOD_TIMESTAMP":"11-SEP-2026","HI_TIMESTAMP":"2026-09-10T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":25383.9,"EOD_HIGH_INDEX_VAL":25520.3,"EOD_CLOSE_INDEX_VAL":25520.3,"EOD_LOW_INDEX_VAL":25374.75,"HIT_TURN_OVER":8149.08,"HIT_TRADED_QTY":97208606,"EOD_TIMESTAMP":"10-SEP-2026","HI_TIMESTAMP":"2026-09-09T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":25538.05,"EOD_HIGH_INDEX_VAL":25592.4,"EOD_CLOSE_INDEX_VAL":25376.8,"EOD_LOW_INDEX_VAL":25376.8,"HIT_TURN_OVER":11812.77,"HIT_TRADED_QTY":134019127,"EOD_TIMESTAMP":"09-SEP-2026","HI_TIMESTAMP":"2026-09-08T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":25885.15,"EOD_HIGH_INDEX_VAL":25904.95,"EOD_CLOSE_INDEX_VAL":25695.55,"EOD_LOW_INDEX_VAL":25690.15,"HIT_TURN_OVER":8220.76,"HIT_TRADED_QTY":96807672,"EOD_TIMESTAMP":"08-SEP-2026","HI_TIMESTAMP":"2026-09-07T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26069.9,"EOD_HIGH_INDEX_VAL":26080.6,"EOD_CLOSE_INDEX_VAL":25935.6,"EOD_LOW_INDEX_VAL":25910.5,"HIT_TURN_OVER":7178.57,"HIT_TRADED_QTY":78845572,"EOD_TIMESTAMP":"07-SEP-2026","HI_TIMESTAMP":"2026-09-06T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":25988.9,"EOD_HIGH_INDEX_VAL":26174,"EOD_CLOSE_INDEX_VAL":26051,"EOD_LOW_INDEX_VAL":25987.1,"HIT_TURN_OVER":10253.44,"HIT_TRADED_QTY":115831222,"EOD_TIMESTAMP":"04-SEP-2026","HI_TIMESTAMP":"2026-09-03T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":25967.05,"EOD_HIGH_INDEX_VAL":26100.1,"EOD_CLOSE_INDEX_VAL":25923.05,"EOD_LOW_INDEX_VAL":25922.9,"HIT_TURN_OVER":10168.12,"HIT_TRADED_QTY":110286415,"EOD_TIMESTAMP":"03-SEP-2026","HI_TIMESTAMP":"2026-09-02T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":25777.2,"EOD_HIGH_INDEX_VAL":25842.4,"EOD_CLOSE_INDEX_VAL":25813.05,"EOD_LOW_INDEX_VAL":25670.85,"HIT_TURN_OVER":11992.29,"HIT_TRADED_QTY":137931961,"EOD_TIMESTAMP":"02-SEP-2026","HI_TIMESTAMP":"2026-09-01T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26171.35,"EOD_HIGH_INDEX_VAL":26195.9,"EOD_CLOSE_INDEX_VAL":26003.9,"EOD_LOW_INDEX_VAL":25896.5,"HIT_TURN_OVER":10791.1,"HIT_TRADED_QTY":128068094,"EOD_TIMESTAMP":"01-SEP-2026","HI_TIMESTAMP":"2026-08-31T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26204.4,"EOD_HIGH_INDEX_VAL":26293.65,"EOD_CLOSE_INDEX_VAL":26293.65,"EOD_LOW_INDEX_VAL":26052.25,"HIT_TURN_OVER":17729.06,"HIT_TRADED_QTY":224753516,"EOD_TIMESTAMP":"31-AUG-2026","HI_TIMESTAMP":"2026-08-30T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26238.9,"EOD_HIGH_INDEX_VAL":26316.25,"EOD_CLOSE_INDEX_VAL":26286.5,"EOD_LOW_INDEX_VAL":26155.45,"HIT_TURN_OVER":8170.46,"HIT_TRADED_QTY":92992612,"EOD_TIMESTAMP":"28-AUG-2026","HI_TIMESTAMP":"2026-08-27T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26445.35,"EOD_HIGH_INDEX_VAL":26494.9,"EOD_CLOSE_INDEX_VAL":26280.65,"EOD_LOW_INDEX_VAL":26280.65,"HIT_TURN_OVER":12196.08,"HIT_TRADED_QTY":158165854,"EOD_TIMESTAMP":"27-AUG-2026","HI_TIMESTAMP":"2026-08-26T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26293.7,"EOD_HIGH_INDEX_VAL":26478.4,"EOD_CLOSE_INDEX_VAL":26386.75,"EOD_LOW_INDEX_VAL":26286.55,"HIT_TURN_OVER":9165.36,"HIT_TRADED_QTY":102910520,"EOD_TIMESTAMP":"26-AUG-2026","HI_TIMESTAMP":"2026-08-25T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26092,"EOD_HIGH_INDEX_VAL":26246.95,"EOD_CLOSE_INDEX_VAL":26246.95,"EOD_LOW_INDEX_VAL":26034.15,"HIT_TURN_OVER":7570.71,"HIT_TRADED_QTY":89101957,"EOD_TIMESTAMP":"25-AUG-2026","HI_TIMESTAMP":"2026-08-24T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26272.3,"EOD_HIGH_INDEX_VAL":26361.65,"EOD_CLOSE_INDEX_VAL":26158.5,"EOD_LOW_INDEX_VAL":26088.9,"HIT_TURN_OVER":8440.44,"HIT_TRADED_QTY":90319568,"EOD_TIMESTAMP":"24-AUG-2026","HI_TIMESTAMP":"2026-08-23T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26277.35,"EOD_HIGH_INDEX_VAL":26324.1,"EOD_CLOSE_INDEX_VAL":26261,"EOD_LOW_INDEX_VAL":26192.95,"HIT_TURN_OVER":9279.19,"HIT_TRADED_QTY":101826278,"EOD_TIMESTAMP":"21-AUG-2026","HI_TIMESTAMP":"2026-08-20T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26173.4,"EOD_HIGH_INDEX_VAL":26279.3,"EOD_CLOSE_INDEX_VAL":26203.9,"EOD_LOW_INDEX_VAL":26159.9,"HIT_TURN_OVER":9628.37,"HIT_TRADED_QTY":105979433,"EOD_TIMESTAMP":"20-AUG-2026","HI_TIMESTAMP":"2026-08-19T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26106.65,"EOD_HIGH_INDEX_VAL":26129.55,"EOD_CLOSE_INDEX_VAL":26013,"EOD_LOW_INDEX_VAL":25936.15,"HIT_TURN_OVER":8258.74,"HIT_TRADED_QTY":90292804,"EOD_TIMESTAMP":"19-AUG-2026","HI_TIMESTAMP":"2026-08-18T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26132.7,"EOD_HIGH_INDEX_VAL":26220.1,"EOD_CLOSE_INDEX_VAL":26108,"EOD_LOW_INDEX_VAL":26088.15,"HIT_TURN_OVER":8228.01,"HIT_TRADED_QTY":86662090,"EOD_TIMESTAMP":"18-AUG-2026","HI_TIMESTAMP":"2026-08-17T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26148.65,"EOD_HIGH_INDEX_VAL":26322.55,"EOD_CLOSE_INDEX_VAL":26217.15,"EOD_LOW_INDEX_VAL":26051.75,"HIT_TURN_OVER":8481.25,"HIT_TRADED_QTY":85025387,"EOD_TIMESTAMP":"17-AUG-2026","HI_TIMESTAMP":"2026-08-16T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26306.2,"EOD_HIGH_INDEX_VAL":26312.9,"EOD_CLOSE_INDEX_VAL":26213.65,"EOD_LOW_INDEX_VAL":26174.95,"HIT_TURN_OVER":8169.89,"HIT_TRADED_QTY":97623322,"EOD_TIMESTAMP":"14-AUG-2026","HI_TIMESTAMP":"2026-08-13T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY FIN SERVICE","EOD_OPEN_INDEX_VAL":26428.15,"EOD_HIGH_INDEX_VAL":26428.15,"EOD_CLOSE_INDEX_VAL":26327.95,"EOD_LOW_INDEX_VAL":26282.25,"HIT_TURN_OVER":10340.28,"HIT_TRADED_QTY":141343038,"EOD_TIMESTAMP":"13-AUG-2026","HI_TIMESTAMP":"2026-08-12T18:30:00.000Z"}]}
\ No newline at end of file
ai/research-engine/tests/fixtures/nse_index_healthcare.json new
+1
@@ -0,0 +1 @@
1 +{"data":[{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16428.6,"EOD_HIGH_INDEX_VAL":16560.15,"EOD_CLOSE_INDEX_VAL":16492.1,"EOD_LOW_INDEX_VAL":16368.05,"HIT_TURN_OVER":3292.18,"HIT_TRADED_QTY":22990674,"EOD_TIMESTAMP":"11-SEP-2026","HI_TIMESTAMP":"2026-09-10T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16622.45,"EOD_HIGH_INDEX_VAL":16640.05,"EOD_CLOSE_INDEX_VAL":16518.65,"EOD_LOW_INDEX_VAL":16465.35,"HIT_TURN_OVER":2597.13,"HIT_TRADED_QTY":19870008,"EOD_TIMESTAMP":"10-SEP-2026","HI_TIMESTAMP":"2026-09-09T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16619.85,"EOD_HIGH_INDEX_VAL":16720.1,"EOD_CLOSE_INDEX_VAL":16581.15,"EOD_LOW_INDEX_VAL":16548.2,"HIT_TURN_OVER":3232.57,"HIT_TRADED_QTY":28578008,"EOD_TIMESTAMP":"09-SEP-2026","HI_TIMESTAMP":"2026-09-08T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16525.15,"EOD_HIGH_INDEX_VAL":16651.55,"EOD_CLOSE_INDEX_VAL":16624.6,"EOD_LOW_INDEX_VAL":16475.45,"HIT_TURN_OVER":3731.05,"HIT_TRADED_QTY":37938889,"EOD_TIMESTAMP":"08-SEP-2026","HI_TIMESTAMP":"2026-09-07T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16403.25,"EOD_HIGH_INDEX_VAL":16520.35,"EOD_CLOSE_INDEX_VAL":16520.3,"EOD_LOW_INDEX_VAL":16385.1,"HIT_TURN_OVER":2715.46,"HIT_TRADED_QTY":21000792,"EOD_TIMESTAMP":"07-SEP-2026","HI_TIMESTAMP":"2026-09-06T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16548.95,"EOD_HIGH_INDEX_VAL":16573.3,"EOD_CLOSE_INDEX_VAL":16408.05,"EOD_LOW_INDEX_VAL":16400.7,"HIT_TURN_OVER":2140.36,"HIT_TRADED_QTY":15278284,"EOD_TIMESTAMP":"04-SEP-2026","HI_TIMESTAMP":"2026-09-03T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16650.55,"EOD_HIGH_INDEX_VAL":16660.35,"EOD_CLOSE_INDEX_VAL":16552.2,"EOD_LOW_INDEX_VAL":16528.2,"HIT_TURN_OVER":2335.93,"HIT_TRADED_QTY":13625951,"EOD_TIMESTAMP":"03-SEP-2026","HI_TIMESTAMP":"2026-09-02T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16580.95,"EOD_HIGH_INDEX_VAL":16661.35,"EOD_CLOSE_INDEX_VAL":16629.1,"EOD_LOW_INDEX_VAL":16547.5,"HIT_TURN_OVER":2612.61,"HIT_TRADED_QTY":16639917,"EOD_TIMESTAMP":"02-SEP-2026","HI_TIMESTAMP":"2026-09-01T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16816.85,"EOD_HIGH_INDEX_VAL":16816.85,"EOD_CLOSE_INDEX_VAL":16669.85,"EOD_LOW_INDEX_VAL":16616.9,"HIT_TURN_OVER":3684.24,"HIT_TRADED_QTY":23994278,"EOD_TIMESTAMP":"01-SEP-2026","HI_TIMESTAMP":"2026-08-31T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16806.7,"EOD_HIGH_INDEX_VAL":16941.7,"EOD_CLOSE_INDEX_VAL":16941.5,"EOD_LOW_INDEX_VAL":16638.05,"HIT_TURN_OVER":9949.84,"HIT_TRADED_QTY":55642068,"EOD_TIMESTAMP":"31-AUG-2026","HI_TIMESTAMP":"2026-08-30T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16744.45,"EOD_HIGH_INDEX_VAL":16818.5,"EOD_CLOSE_INDEX_VAL":16798.5,"EOD_LOW_INDEX_VAL":16673.2,"HIT_TURN_OVER":3149.96,"HIT_TRADED_QTY":20511402,"EOD_TIMESTAMP":"28-AUG-2026","HI_TIMESTAMP":"2026-08-27T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16626.75,"EOD_HIGH_INDEX_VAL":16728,"EOD_CLOSE_INDEX_VAL":16707.7,"EOD_LOW_INDEX_VAL":16579.9,"HIT_TURN_OVER":3469.78,"HIT_TRADED_QTY":22576990,"EOD_TIMESTAMP":"27-AUG-2026","HI_TIMESTAMP":"2026-08-26T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16592.7,"EOD_HIGH_INDEX_VAL":16689.9,"EOD_CLOSE_INDEX_VAL":16586.65,"EOD_LOW_INDEX_VAL":16486.35,"HIT_TURN_OVER":3296.16,"HIT_TRADED_QTY":20264811,"EOD_TIMESTAMP":"26-AUG-2026","HI_TIMESTAMP":"2026-08-25T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16410.4,"EOD_HIGH_INDEX_VAL":16600.55,"EOD_CLOSE_INDEX_VAL":16597.45,"EOD_LOW_INDEX_VAL":16335,"HIT_TURN_OVER":3237.05,"HIT_TRADED_QTY":20610924,"EOD_TIMESTAMP":"25-AUG-2026","HI_TIMESTAMP":"2026-08-24T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16412.7,"EOD_HIGH_INDEX_VAL":16448.05,"EOD_CLOSE_INDEX_VAL":16418,"EOD_LOW_INDEX_VAL":16306.85,"HIT_TURN_OVER":2980.38,"HIT_TRADED_QTY":19962447,"EOD_TIMESTAMP":"24-AUG-2026","HI_TIMESTAMP":"2026-08-23T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16487.3,"EOD_HIGH_INDEX_VAL":16525.15,"EOD_CLOSE_INDEX_VAL":16423.9,"EOD_LOW_INDEX_VAL":16319.75,"HIT_TURN_OVER":2277.66,"HIT_TRADED_QTY":18648469,"EOD_TIMESTAMP":"21-AUG-2026","HI_TIMESTAMP":"2026-08-20T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16483.2,"EOD_HIGH_INDEX_VAL":16496.1,"EOD_CLOSE_INDEX_VAL":16465.15,"EOD_LOW_INDEX_VAL":16415.1,"HIT_TURN_OVER":2255.12,"HIT_TRADED_QTY":18118432,"EOD_TIMESTAMP":"20-AUG-2026","HI_TIMESTAMP":"2026-08-19T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16459.85,"EOD_HIGH_INDEX_VAL":16500.25,"EOD_CLOSE_INDEX_VAL":16412.8,"EOD_LOW_INDEX_VAL":16363.25,"HIT_TURN_OVER":2869.39,"HIT_TRADED_QTY":19227526,"EOD_TIMESTAMP":"19-AUG-2026","HI_TIMESTAMP":"2026-08-18T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16432.6,"EOD_HIGH_INDEX_VAL":16546,"EOD_CLOSE_INDEX_VAL":16463.7,"EOD_LOW_INDEX_VAL":16403.05,"HIT_TURN_OVER":3046.72,"HIT_TRADED_QTY":21263398,"EOD_TIMESTAMP":"18-AUG-2026","HI_TIMESTAMP":"2026-08-17T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16543.75,"EOD_HIGH_INDEX_VAL":16559,"EOD_CLOSE_INDEX_VAL":16429.35,"EOD_LOW_INDEX_VAL":16424.1,"HIT_TURN_OVER":4481.3,"HIT_TRADED_QTY":29281733,"EOD_TIMESTAMP":"17-AUG-2026","HI_TIMESTAMP":"2026-08-16T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16569.45,"EOD_HIGH_INDEX_VAL":16588.2,"EOD_CLOSE_INDEX_VAL":16505.3,"EOD_LOW_INDEX_VAL":16467.35,"HIT_TURN_OVER":4375.98,"HIT_TRADED_QTY":33043912,"EOD_TIMESTAMP":"14-AUG-2026","HI_TIMESTAMP":"2026-08-13T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY HEALTHCARE","EOD_OPEN_INDEX_VAL":16621.95,"EOD_HIGH_INDEX_VAL":16629.15,"EOD_CLOSE_INDEX_VAL":16581.85,"EOD_LOW_INDEX_VAL":16528.7,"HIT_TURN_OVER":3672.2,"HIT_TRADED_QTY":23560102,"EOD_TIMESTAMP":"13-AUG-2026","HI_TIMESTAMP":"2026-08-12T18:30:00.000Z"}]}
\ No newline at end of file
ai/research-engine/tests/fixtures/nse_index_it.json new
+1
@@ -0,0 +1 @@
1 +{"data":[{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":28672.85,"EOD_HIGH_INDEX_VAL":29124.6,"EOD_CLOSE_INDEX_VAL":28921.5,"EOD_LOW_INDEX_VAL":28671.8,"HIT_TURN_OVER":3112.87,"HIT_TRADED_QTY":25370316,"EOD_TIMESTAMP":"11-SEP-2026","HI_TIMESTAMP":"2026-09-10T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":28919.35,"EOD_HIGH_INDEX_VAL":29264.9,"EOD_CLOSE_INDEX_VAL":28890.9,"EOD_LOW_INDEX_VAL":28740.6,"HIT_TURN_OVER":3378.58,"HIT_TRADED_QTY":28911969,"EOD_TIMESTAMP":"10-SEP-2026","HI_TIMESTAMP":"2026-09-09T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":29299.55,"EOD_HIGH_INDEX_VAL":29299.55,"EOD_CLOSE_INDEX_VAL":28913.95,"EOD_LOW_INDEX_VAL":28779.8,"HIT_TURN_OVER":5841.27,"HIT_TRADED_QTY":45263531,"EOD_TIMESTAMP":"09-SEP-2026","HI_TIMESTAMP":"2026-09-08T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30032.3,"EOD_HIGH_INDEX_VAL":30078.75,"EOD_CLOSE_INDEX_VAL":29883.45,"EOD_LOW_INDEX_VAL":29722.9,"HIT_TURN_OVER":2584.22,"HIT_TRADED_QTY":21723387,"EOD_TIMESTAMP":"08-SEP-2026","HI_TIMESTAMP":"2026-09-07T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30343.2,"EOD_HIGH_INDEX_VAL":30377.1,"EOD_CLOSE_INDEX_VAL":29995.2,"EOD_LOW_INDEX_VAL":29847.25,"HIT_TURN_OVER":3043.8,"HIT_TRADED_QTY":27245871,"EOD_TIMESTAMP":"07-SEP-2026","HI_TIMESTAMP":"2026-09-06T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30956.15,"EOD_HIGH_INDEX_VAL":31263.2,"EOD_CLOSE_INDEX_VAL":30695.1,"EOD_LOW_INDEX_VAL":30656.7,"HIT_TURN_OVER":2603.29,"HIT_TRADED_QTY":20693928,"EOD_TIMESTAMP":"04-SEP-2026","HI_TIMESTAMP":"2026-09-03T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":31180.8,"EOD_HIGH_INDEX_VAL":31185.7,"EOD_CLOSE_INDEX_VAL":30838.85,"EOD_LOW_INDEX_VAL":30621.65,"HIT_TURN_OVER":2783.34,"HIT_TRADED_QTY":20571191,"EOD_TIMESTAMP":"03-SEP-2026","HI_TIMESTAMP":"2026-09-02T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":31164.3,"EOD_HIGH_INDEX_VAL":31241.15,"EOD_CLOSE_INDEX_VAL":31102.9,"EOD_LOW_INDEX_VAL":30668.1,"HIT_TURN_OVER":3673.29,"HIT_TRADED_QTY":31034664,"EOD_TIMESTAMP":"02-SEP-2026","HI_TIMESTAMP":"2026-09-01T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":31006.25,"EOD_HIGH_INDEX_VAL":31529.2,"EOD_CLOSE_INDEX_VAL":31496.7,"EOD_LOW_INDEX_VAL":30834.1,"HIT_TURN_OVER":4598.08,"HIT_TRADED_QTY":33505972,"EOD_TIMESTAMP":"01-SEP-2026","HI_TIMESTAMP":"2026-08-31T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":31225.5,"EOD_HIGH_INDEX_VAL":31277.5,"EOD_CLOSE_INDEX_VAL":31191.45,"EOD_LOW_INDEX_VAL":30634.45,"HIT_TURN_OVER":4767.84,"HIT_TRADED_QTY":38941906,"EOD_TIMESTAMP":"31-AUG-2026","HI_TIMESTAMP":"2026-08-30T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30493.15,"EOD_HIGH_INDEX_VAL":31322.9,"EOD_CLOSE_INDEX_VAL":31281.7,"EOD_LOW_INDEX_VAL":30473.5,"HIT_TURN_OVER":6259.34,"HIT_TRADED_QTY":44244200,"EOD_TIMESTAMP":"28-AUG-2026","HI_TIMESTAMP":"2026-08-27T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30342.55,"EOD_HIGH_INDEX_VAL":30576.95,"EOD_CLOSE_INDEX_VAL":30221.15,"EOD_LOW_INDEX_VAL":30164.5,"HIT_TURN_OVER":2547.2,"HIT_TRADED_QTY":19634787,"EOD_TIMESTAMP":"27-AUG-2026","HI_TIMESTAMP":"2026-08-26T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30809.2,"EOD_HIGH_INDEX_VAL":30841.15,"EOD_CLOSE_INDEX_VAL":30318.85,"EOD_LOW_INDEX_VAL":30307.45,"HIT_TURN_OVER":2561.57,"HIT_TRADED_QTY":22484895,"EOD_TIMESTAMP":"26-AUG-2026","HI_TIMESTAMP":"2026-08-25T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30692.35,"EOD_HIGH_INDEX_VAL":30771.8,"EOD_CLOSE_INDEX_VAL":30771.8,"EOD_LOW_INDEX_VAL":30283.4,"HIT_TURN_OVER":2930.26,"HIT_TRADED_QTY":24339283,"EOD_TIMESTAMP":"25-AUG-2026","HI_TIMESTAMP":"2026-08-24T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30579.6,"EOD_HIGH_INDEX_VAL":30892.9,"EOD_CLOSE_INDEX_VAL":30596.9,"EOD_LOW_INDEX_VAL":30484.8,"HIT_TURN_OVER":2842.25,"HIT_TRADED_QTY":25353286,"EOD_TIMESTAMP":"24-AUG-2026","HI_TIMESTAMP":"2026-08-23T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30676.4,"EOD_HIGH_INDEX_VAL":30693.35,"EOD_CLOSE_INDEX_VAL":30532.25,"EOD_LOW_INDEX_VAL":30337,"HIT_TURN_OVER":2687.41,"HIT_TRADED_QTY":18926312,"EOD_TIMESTAMP":"21-AUG-2026","HI_TIMESTAMP":"2026-08-20T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30850.5,"EOD_HIGH_INDEX_VAL":30942.9,"EOD_CLOSE_INDEX_VAL":30673.05,"EOD_LOW_INDEX_VAL":30637.8,"HIT_TURN_OVER":3831.9,"HIT_TRADED_QTY":25241767,"EOD_TIMESTAMP":"20-AUG-2026","HI_TIMESTAMP":"2026-08-19T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30461.1,"EOD_HIGH_INDEX_VAL":30659.8,"EOD_CLOSE_INDEX_VAL":30433.05,"EOD_LOW_INDEX_VAL":30165,"HIT_TURN_OVER":3638.05,"HIT_TRADED_QTY":25384894,"EOD_TIMESTAMP":"19-AUG-2026","HI_TIMESTAMP":"2026-08-18T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":30584.1,"EOD_HIGH_INDEX_VAL":30615.45,"EOD_CLOSE_INDEX_VAL":30213.45,"EOD_LOW_INDEX_VAL":30212.95,"HIT_TURN_OVER":3074.91,"HIT_TRADED_QTY":25486159,"EOD_TIMESTAMP":"18-AUG-2026","HI_TIMESTAMP":"2026-08-17T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":31242.4,"EOD_HIGH_INDEX_VAL":31331.2,"EOD_CLOSE_INDEX_VAL":30807.8,"EOD_LOW_INDEX_VAL":30722.05,"HIT_TURN_OVER":2434.68,"HIT_TRADED_QTY":17225551,"EOD_TIMESTAMP":"17-AUG-2026","HI_TIMESTAMP":"2026-08-16T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":31419.85,"EOD_HIGH_INDEX_VAL":31529.55,"EOD_CLOSE_INDEX_VAL":31357.75,"EOD_LOW_INDEX_VAL":31052.7,"HIT_TURN_OVER":3159.26,"HIT_TRADED_QTY":22906479,"EOD_TIMESTAMP":"14-AUG-2026","HI_TIMESTAMP":"2026-08-13T18:30:00.000Z"},{"EOD_INDEX_NAME":"NIFTY IT","EOD_OPEN_INDEX_VAL":31277,"EOD_HIGH_INDEX_VAL":31459.3,"EOD_CLOSE_INDEX_VAL":31453.9,"EOD_LOW_INDEX_VAL":31098.6,"HIT_TURN_OVER":2809.83,"HIT_TRADED_QTY":20855020,"EOD_TIMESTAMP":"13-AUG-2026","HI_TIMESTAMP":"2026-08-12T18:30:00.000Z"}]}
\ No newline at end of file
ai/research-engine/tests/test_sector_benchmarks.py new
+314
@@ -0,0 +1,314 @@
1 +"""Canonical identity, official captured contract, and persisted exact-date integration."""
2 +import json
3 +from datetime import date, datetime, timedelta, timezone
4 +from decimal import Decimal
5 +from pathlib import Path
6 +from types import SimpleNamespace
7 +from unittest.mock import AsyncMock
8 +from uuid import UUID
9 +
10 +import httpx
11 +import pytest
12 +
13 +from app.nse_index_history import ENDPOINT, NseIndexHistoryProvider, parse_index_history
14 +from app.nse_historical_daily import BOOTSTRAP, NseHistoricalResult, persist_daily_result
15 +from app.sector_benchmarks import *
16 +from app.sector_relative_strength import SectorRelativeStrengthEngine
17 +from app.persistence import SqliteResearchPersistence
18 +from app.settings import Settings
19 +
20 +START, END = date(2026, 8, 13), date(2026, 9, 11)
21 +NOW = datetime(2026, 9, 13, 20, tzinfo=timezone.utc)
22 +FIXTURES = Path(__file__).parent / 'fixtures'
23 +FILES = dict(zip(BENCHMARKS, ('500', 'it', 'financials', 'healthcare')))
24 +STOCK = UUID(int=1)
25 +
26 +
27 +def metadata(key=BROAD_KEY):
28 + return dict(globalInstrumentId=str(benchmark_id(key)), assetType='INDEX', status='ACTIVE',
29 + country='IN', primaryExchange='NSE', primarySymbol='NOT_IDENTITY', currency='INR',
30 + providerMappings=[dict(provider='NSE', providerSymbol=BENCHMARKS[key], status='VERIFIED',
31 + resolutionSource=CATALOG_VERSION, currency='INR', exchange='NSE')])
32 +
33 +
34 +def classification(sector='Technology', key=STOCK):
35 + return dict(globalInstrumentId=str(key), canonicalSector=sector, source='NSE_INDICES_NIFTY500',
36 + retrievedAt=NOW.isoformat(), status='ACTIVE', assetType='EQUITY', country='IN', exchange='NSE')
37 +
38 +
39 +def parsed(key=BROAD_KEY):
40 + result = NseHistoricalResult(benchmark_id(key), START, END, provider_symbol=BENCHMARKS[key],
41 + source_url=ENDPOINT, retrieved_at=NOW)
42 + parse_index_history((FIXTURES / f'nse_index_{FILES[key]}.json').read_bytes(), result, 'INR')
43 + result.status = 'SUCCESS'
44 + return result
45 +
46 +
47 +@pytest.mark.parametrize('sector', list(SECTOR_KEYS))
48 +def test_exact_mapping(sector):
49 + c = build_sector_contexts([classification(sector)], [metadata(k) for k in BENCHMARKS], {STOCK})[STOCK]
50 + assert c.mapping_version == 'SECTOR_BENCHMARK_MAPPING_V1'
51 + assert c.sector_benchmark.instrument_id == benchmark_id(SECTOR_KEYS[sector])
52 + assert c.market_benchmark.instrument_id == benchmark_id(BROAD_KEY)
53 + assert c.sector_mapping_status == 'AVAILABLE'
54 + with pytest.raises(ValueError): benchmark_id(BENCHMARKS[BROAD_KEY])
55 +
56 +
57 +@pytest.mark.parametrize('sector', ['Industrials', 'Materials', 'Consumer Staples', 'Consumer Discretionary',
58 + 'Energy', 'Utilities', 'Real Estate', 'Communication Services', 'FINANCIAL SERVICES', 'Bank'])
59 +def test_unmapped_not_substituted(sector):
60 + c = build_sector_contexts([classification(sector)], [metadata(k) for k in BENCHMARKS], {STOCK})[STOCK]
61 + assert c.sector_benchmark is None and c.sector_mapping_status == 'UNMAPPED_SECTOR_BENCHMARK'
62 + assert c.market_benchmark is not None
63 +
64 +
65 +@pytest.mark.parametrize('rows', [[], [classification(None)], [classification(), classification('Financials')],
66 + [classification() | dict(source='GUESSED')]])
67 +def test_missing_ambiguous_classification(rows):
68 + c = build_sector_contexts(rows, [], {STOCK})[STOCK]
69 + assert c.sector_mapping_status == 'NO_SECTOR_CLASSIFICATION'
70 + assert c.market_mapping_status == 'BENCHMARK_IDENTITY_UNAVAILABLE'
71 +
72 +
73 +@pytest.mark.parametrize('change', [dict(status='INACTIVE'), dict(assetType='EQUITY'), dict(currency=None),
74 + dict(globalInstrumentId=str(UUID(int=99))), dict(providerMappings=[]),
75 + dict(providerMappings=metadata()['providerMappings'] * 2),
76 + *[dict(providerMappings=[metadata()['providerMappings'][0] | x]) for x in
77 + [dict(active=False), dict(status='UNVERIFIED'), dict(providerSymbol='NIFTY 50'),
78 + dict(resolutionSource='GUESSED'), dict(currency='USD')]]])
79 +def test_identity_rejected(change):
80 + with pytest.raises(ValueError, match='BENCHMARK_IDENTITY_UNAVAILABLE'):
81 + benchmark_identity(metadata() | change, BROAD_KEY)
82 +
83 +
84 +@pytest.mark.parametrize('key', list(BENCHMARKS))
85 +def test_observed_official_contract(key):
86 + r = parsed(key)
87 + assert r.rows_accepted == 22
88 + assert r.first_trading_date == START and r.last_trading_date == END
89 + assert all(b.volume is b.turnover is b.previous_close is None for b in r.bars)
90 + assert all(type(b.close) is Decimal and b.high >= b.low > 0 for b in r.bars)
91 + assert r.bars == sorted(r.bars, key=lambda b: b.trading_date)
92 + # EOD_TIMESTAMP is the trading date; HI_TIMESTAMP is the prior UTC day.
93 + assert r.bars[-1].trading_date == date(2026, 9, 11)
94 +
95 +
96 +@pytest.mark.parametrize('field,value', [('EOD_INDEX_NAME', 'NIFTY 50'), ('EOD_TIMESTAMP', 'bad'),
97 + ('EOD_TIMESTAMP', '12-SEP-2026'), ('EOD_OPEN_INDEX_VAL', 'NaN'), ('EOD_LOW_INDEX_VAL', 0),
98 + ('EOD_HIGH_INDEX_VAL', 1), ('EOD_CLOSE_INDEX_VAL', None)])
99 +def test_invalid_rows_fail_whole_window(field, value):
100 + payload = json.loads((FIXTURES / 'nse_index_500.json').read_bytes())
101 + payload['data'][0][field] = value
102 + r = NseHistoricalResult(benchmark_id(BROAD_KEY), START, END, provider_symbol='NIFTY 500')
103 + with pytest.raises(ValueError): parse_index_history(json.dumps(payload).encode(), r, 'INR')
104 + assert r.bars == []
105 +
106 +
107 +@pytest.mark.parametrize('payload', [b'', b'<html>blocked</html>', b'{}', b'{"data":[]}', b'{"data":[{}]}'])
108 +def test_invalid_contract(payload):
109 + with pytest.raises(ValueError): parse_index_history(payload, parsed(), 'INR')
110 +
111 +
112 +def test_duplicate_and_precision():
113 + payload = json.loads((FIXTURES / 'nse_index_500.json').read_bytes())
114 + payload['data'][0]['EOD_OPEN_INDEX_VAL'] = '12345.123456789012'
115 + r = parsed()
116 + parse_index_history(json.dumps(payload).encode(), r, 'INR')
117 + assert r.bars[-1].open == Decimal('12345.123456789012')
118 + payload['data'].append(payload['data'][0])
119 + with pytest.raises(ValueError): parse_index_history(json.dumps(payload).encode(), parsed(), 'INR')
120 +
121 +
122 +@pytest.mark.asyncio
123 +async def test_session_reuse_query_spacing_no_cookie_logs(caplog):
124 + requests = []
125 + def handler(request):
126 + requests.append(request)
127 + assert request.headers['user-agent'].startswith('Mozilla/')
128 + if str(request.url) == BOOTSTRAP:
129 + return httpx.Response(200, headers={'set-cookie': 'session=private-test-cookie; Path=/; Secure'})
130 + assert request.headers['cookie'] == 'session=private-test-cookie'
131 + assert dict(request.url.params) == {'indexType':'NIFTY 500', 'from':'13-08-2026', 'to':'11-09-2026'}
132 + return httpx.Response(200, content=(FIXTURES / 'nse_index_500.json').read_bytes())
133 + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
134 + sleep = AsyncMock()
135 + p = NseIndexHistoryProvider(SimpleNamespace(global_instrument_metadata=AsyncMock(return_value=metadata())),
136 + Settings(), client=client, sleep=sleep)
137 + for _ in range(2):
138 + r = await p.fetch(benchmark_id(BROAD_KEY), start=START, end=END)
139 + assert r.status == 'SUCCESS' and r.rows_accepted == 22
140 + assert len(requests) == 3 and sleep.await_count >= 2
141 + await p.aclose()
142 + assert 'private-test-cookie' not in caplog.text
143 +
144 +
145 +@pytest.mark.asyncio
146 +@pytest.mark.parametrize('status,attempts', [(403,1), (404,1), (429,3), (500,3)])
147 +async def test_http_failures(status, attempts):
148 + requests = []
149 + def handler(request):
150 + if str(request.url) == BOOTSTRAP: return httpx.Response(200)
151 + requests.append(request)
152 + return httpx.Response(status)
153 + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
154 + p = NseIndexHistoryProvider(SimpleNamespace(global_instrument_metadata=AsyncMock(return_value=metadata())),
155 + Settings(), client=client, sleep=AsyncMock())
156 + r = await p.fetch(benchmark_id(BROAD_KEY), start=START, end=END)
157 + assert r.failure_reason == f'HISTORICAL_HTTP_{status}' and not r.bars
158 + assert len(requests) == attempts
159 +
160 +
161 +@pytest.mark.asyncio
162 +async def test_identity_before_network_and_bounded_dates():
163 + def forbidden(request): pytest.fail('Network before identity/window gate')
164 + async with httpx.AsyncClient(transport=httpx.MockTransport(forbidden)) as client:
165 + p = NseIndexHistoryProvider(SimpleNamespace(global_instrument_metadata=AsyncMock(return_value=None)), Settings(), client=client)
166 + r = await p.fetch(benchmark_id(BROAD_KEY), start=START, end=END)
167 + assert r.failure_reason == 'BENCHMARK_IDENTITY_UNAVAILABLE'
168 + r = await p.fetch(benchmark_id(BROAD_KEY), start=START, end=END+timedelta(days=1))
169 + assert r.failure_reason == 'INVALID_REQUEST_WINDOW'
170 +
171 +
172 +@pytest.mark.asyncio
173 +async def test_persist_repeat_correction_other_provider_and_failure():
174 + store = SqliteResearchPersistence()
175 + repo = SimpleNamespace(upsert_daily_market_bars_async=AsyncMock(side_effect=store.upsert_daily_market_bars))
176 + r = parsed()
177 + await persist_daily_result(repo, r)
178 + await persist_daily_result(repo, r)
179 + corrected = r.bars[-1].model_copy(update={'close': r.bars[-1].close + Decimal('.01')})
180 + store.upsert_daily_market_bar(corrected.model_copy(update={'provider':'OTHER'}))
181 + r.bars[-1] = corrected
182 + await persist_daily_result(repo, r)
183 + rows = store.load_daily_market_bars({r.global_instrument_id}, provider='NSE')
184 + assert len(rows) == 22 and rows[-1].close == corrected.close
185 + assert len(store.load_daily_market_bars({r.global_instrument_id})) == 23
186 + repo.upsert_daily_market_bars_async.side_effect = RuntimeError('do not expose')
187 + await persist_daily_result(repo, r)
188 + assert r.failure_reason == 'DAILY_BAR_PERSISTENCE_UNAVAILABLE'
189 + assert len(store.load_daily_market_bars({r.global_instrument_id}, provider='NSE')) == 22
190 +
191 +
192 +def test_daily_alignment_reversed_no_filling_and_provenance(monkeypatch):
193 + import socket
194 + monkeypatch.setattr(socket, 'create_connection', lambda *a, **k: pytest.fail('network during compute'))
195 + sector_key = SECTOR_KEYS['Technology']
196 + sector, market = parsed(sector_key), parsed()
197 + stock = [b.model_copy(update={'global_instrument_id':STOCK}) for b in sector.bars]
198 + contexts = build_sector_contexts([classification()], [metadata(k) for k in BENCHMARKS], {STOCK})
199 + daily = {STOCK:stock, sector.global_instrument_id:sector.bars, market.global_instrument_id:market.bars}
200 + def compute(rows):
201 + return SectorRelativeStrengthEngine().compute(STOCK, [], as_of=NOW, currency='INR', context=contexts[STOCK], daily_bar_histories=rows)
202 + first = compute(daily)
203 + assert first == compute({k:list(reversed(v)) for k,v in daily.items()})
204 + assert first.benchmark_mapping_version == MAPPING_VERSION
205 + assert first.relative_vs_sector1_m == 0 and first.sector_state != 'INSUFFICIENT_DATA'
206 + assert first.sector_return1_m == pytest.approx(float((sector.bars[-1].close / sector.bars[0].close - 1)*100))
207 + assert first.history_sources['sector'] == 'DAILY_MARKET_BAR_NSE'
208 + daily[sector.global_instrument_id] = sector.bars[1:]
209 + second = compute(daily)
210 + assert second.relative_vs_sector1_m is None and second.relative_vs_sector1_w == 0
211 + daily[sector.global_instrument_id] = [b.model_copy(update={'provider':'OTHER'}) for b in sector.bars]
212 + assert compute(daily).benchmark_states['sector'] == 'BENCHMARK_HISTORY_UNAVAILABLE'
213 +
214 +
215 +@pytest.mark.asyncio
216 +@pytest.mark.parametrize('count,multiple', [(0,False),(1,False),(18,False),(18,True)])
217 +async def test_stage_b_bounded_queries(count, multiple):
218 + from app.global_scanner import GlobalScanner
219 + from test_global_scanner import instrument, persisted, scan
220 + store = SqliteResearchPersistence()
221 + items = [instrument(n) for n in range(10, 10+count)]
222 + for item in items: persisted(store, item)
223 + initial = await scan(items, store, top_n=max(count, 1))
224 + ids = {UUID(item['globalInstrumentId']) for item in items}
225 + sectors = list(SECTOR_KEYS)
226 + rows = [classification(sectors[i%3] if multiple else 'Technology', k) for i,k in enumerate(sorted(ids))]
227 + contexts = build_sector_contexts(rows, [metadata(k) for k in BENCHMARKS], ids)
228 + queries = []
229 + store._connection.set_trace_callback(queries.append)
230 + enriched = GlobalScanner(None, store).enrich_candidates(initial, sector_contexts=contexts)
231 + assert len(enriched) == count
232 + assert len(queries) == (2 if count else 0)
233 + assert all('SELECT' in q for q in queries)
234 +
235 +
236 +@pytest.mark.asyncio
237 +@pytest.mark.parametrize('reason', ['HISTORICAL_HTTP_403', 'INVALID_INDEX_ROW', 'DAILY_BAR_PERSISTENCE_UNAVAILABLE'])
238 +async def test_worker_failure_isolation_session_closed(monkeypatch, reason):
239 + from app.market_data_population import IndiaMarketDataPopulationJobs
240 + import app.nse_index_history as module
241 + keys = sorted([benchmark_id(k) for k in BENCHMARKS], key=str)[:2]
242 + responses = [NseHistoricalResult(keys[0], START, END, failure_reason=reason), parsed()]
243 + provider = SimpleNamespace(fetch=AsyncMock(side_effect=responses), aclose=AsyncMock())
244 + factory = lambda *a, **k: provider
245 + monkeypatch.setattr(module, 'NseIndexHistoryProvider', factory)
246 + repo = SimpleNamespace(upsert_daily_market_bars_async=AsyncMock(return_value=22))
247 + jobs = IndiaMarketDataPopulationJobs(repo, None, None, Settings(), sleep=AsyncMock(), clock=lambda:NOW)
248 + results = await jobs.populate_benchmark_history(set(keys), start=START, end=END, identity_headers={})
249 + assert len(results) == 2 and results[0].failure_reason == reason and results[1].persisted_rows == 22
250 + assert provider.fetch.await_count == 2 and provider.aclose.await_count == 1
251 + assert repo.upsert_daily_market_bars_async.await_count == 1
252 +
253 +
254 +@pytest.mark.asyncio
255 +async def test_worker_throttling_cooldown_and_batch_bound(monkeypatch):
256 + from app.market_data_population import IndiaMarketDataPopulationJobs
257 + import app.nse_index_history as module
258 + keys = {benchmark_id(k) for k in BENCHMARKS}
259 + provider = SimpleNamespace(fetch=AsyncMock(return_value=NseHistoricalResult(next(iter(keys)), START, END,
260 + failure_reason='HISTORICAL_HTTP_429')), aclose=AsyncMock())
261 + monkeypatch.setattr(module, 'NseIndexHistoryProvider', lambda *a, **k:provider)
262 + jobs = IndiaMarketDataPopulationJobs(None, None, None, Settings(), sleep=AsyncMock(), clock=lambda:NOW)
263 + results = await jobs.populate_benchmark_history(keys, start=START, end=END, identity_headers={})
264 + assert provider.fetch.await_count == 1
265 + assert [r.failure_reason for r in results][1:] == ['RETRY_COOLDOWN']*3
266 + jobs.settings.market_data_population_batch_size = 1
267 + with pytest.raises(ValueError, match='BENCHMARK_BATCH_LIMIT'):
268 + await jobs.populate_benchmark_history(keys, start=START, end=END, identity_headers={})
269 +
270 +
271 +def test_hand_calculated_daily_horizons_and_staleness():
272 + from app.models import DailyMarketBar
273 + keys = [STOCK, benchmark_id(SECTOR_KEYS['Technology']), benchmark_id(BROAD_KEY)]
274 + daily = {}
275 + # Independent linear prices: stock +2, sector +1, market flat per observation.
276 + for key, slope in zip(keys, (2,1,0)):
277 + daily[key] = [DailyMarketBar(global_instrument_id=key, trading_date=(NOW-timedelta(days=159-i)).date(),
278 + open=Decimal(1000+slope*i), high=Decimal(1000+slope*i), low=Decimal(1000+slope*i),
279 + close=Decimal(1000+slope*i), currency='INR', provider='NSE', provider_symbol='provenance',
280 + source_mode='REAL', source_url=ENDPOINT, retrieved_at=NOW) for i in range(160)]
281 + ctx = build_sector_contexts([classification()], [metadata(k) for k in BENCHMARKS], {STOCK})[STOCK]
282 + def compute(rows): return SectorRelativeStrengthEngine().compute(STOCK, [], as_of=NOW, context=ctx, daily_bar_histories=rows)
283 + result = compute(daily)
284 + for suffix, n in zip(('1_w','1_m','3_m','6_m'), (5,21,63,126)):
285 + assert getattr(result,'stock_return'+suffix) == pytest.approx(100*2*n/(1318-2*n))
286 + assert getattr(result,'sector_return'+suffix) == pytest.approx(100*n/(1159-n))
287 + assert getattr(result,'market_return'+suffix) == 0
288 + daily[keys[1]] = daily[keys[1]][:-10]
289 + assert compute(daily).benchmark_states['sector'] == 'STALE_BENCHMARK_HISTORY'
290 +
291 +
292 +@pytest.mark.asyncio
293 +async def test_metadata_adapter_is_batched_read_only():
294 + from app.portfolio_orchestration import PortfolioResearchOrchestrator
295 + requests = []
296 + def handler(request):
297 + requests.append(request)
298 + assert request.method == 'GET' and request.url.path == '/api/v1/instruments/benchmarks'
299 + return httpx.Response(200, json=[metadata(k) for k in BENCHMARKS])
300 + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
301 + obj = PortfolioResearchOrchestrator(None, Settings(), client=client)
302 + obj.india_nifty500_universe = AsyncMock(return_value=[classification()])
303 + assert await obj.sector_benchmark_contexts(set()) == {}
304 + assert not requests and obj.india_nifty500_universe.await_count == 0
305 + result = await obj.sector_benchmark_contexts({STOCK})
306 + assert result[STOCK].sector_mapping_status == 'AVAILABLE'
307 + assert len(requests) == obj.india_nifty500_universe.await_count == 1
308 +
309 +
310 +def test_india_market_benchmark_never_assigned_to_foreign_or_unknown_stock():
311 + registered = [metadata(k) for k in BENCHMARKS]
312 + for rows in ([], [classification() | dict(country='US', exchange='NASDAQ')]):
313 + result = build_sector_contexts(rows, registered, {STOCK})[STOCK]
314 + assert result.market_benchmark is None and result.sector_benchmark is None
ai/research-engine/tests/test_sector_relative_strength.py
+1 -1
@@ -167,7 +167,7 @@ async def test_stage_b_batches_only_deep_eligible_and_preserves_phase1(monkeypat
167 assert "instrument_id IN" in close_query
168 assert all(str(UUID(int=5)) not in q for q in queries)
169 assert str(SECTOR) in close_query and str(MARKET) in close_query
170 - assert str(SECTOR) not in daily_query and str(MARKET) not in daily_query
170 + assert str(SECTOR) in daily_query and str(MARKET) in daily_query
171 assert all(c.technical_feature_snapshot.global_instrument_id == c.global_instrument_id for c in enriched)
172 # Private inputs are not part of the enrichment contract.
173 assert enriched == scanner.enrich_candidates(initial, sector_contexts={STOCK: context()})
docs/NSE_SECTOR_BENCHMARKS.md new
+226
@@ -0,0 +1,226 @@
1 +# NSE canonical sector benchmarks and price history
2 +
3 +## Identity and classification
4 +
5 +Baseline: `076215b5ee757061357aa6a71e5a6e85988d8baa` (OHLCV technical features).
6 +The audited master contained 2,568 equities and no indices. Stock classification
7 +comes from `portfolio.nifty500_universe.canonical_sector`, populated by
8 +`Nifty500ReferenceService` and `NiftyIndustrySectorClassifier` from the official
9 +Nifty 500 constituent file. The persisted source is `NSE_INDICES_NIFTY500`.
10 +Portfolio/watchlist membership is irrelevant.
11 +
12 +`SECTOR_BENCHMARK_MAPPING_V1` resolves exact canonical classifications to platform
13 +benchmark keys. `NSE_BENCHMARK_CATALOG_V1` records the authoritative registration
14 +evidence. These are **price indices**, not total-return series.
15 +
16 +| Canonical classification | Platform key | Canonical UUID | Verified NSE request symbol |
17 +|---|---|---|---|
18 +| India broad market | INDIA_BROAD_PRICE | 56623ea0-224c-3481-b6d9-66c448b282a0 | NIFTY 500 |
19 +| Technology | INDIA_TECHNOLOGY_PRICE | 4af7634a-1894-333d-ba68-a1c41a32c71c | NIFTY IT |
20 +| Financials | INDIA_FINANCIALS_PRICE | 13ba7849-43e2-3e1e-a3f3-d93ee32d3d2f | NIFTY FINANCIAL SERVICES |
21 +| Healthcare | INDIA_HEALTHCARE_PRICE | 055c4e9f-5592-3be6-8f87-3d05bb25b95a | NIFTY HEALTHCARE INDEX |
22 +
23 +UUIDs use Java `UUID.nameUUIDFromBytes(UTF8("aip:benchmark:" + platformKey))`;
24 +the Python adapter implements the identical UUIDv3 operation. A calculated UUID
25 +alone is **not** registration or identity evidence. Both acquisition and context
26 +resolution require the actual master row and exactly one verified NSE mapping.
27 +Provider symbols never identify stocks or benchmark instruments.
28 +
29 +The existing master service now registers `AssetType.INDEX` through explicit
30 +`POST /api/v1/instruments/benchmarks/{benchmarkKey}/register`. It uses the existing
31 +transaction/advisory-lock, master, and mapping repositories. Unknown keys and
32 +conflicting identities fail. Repeating a registration is idempotent. The existing
33 +VARCHAR asset-type column accepts INDEX; no migration is required or supplied.
34 +`GET /api/v1/instruments/benchmarks` only reads registered identities and mappings.
35 +It does not register instruments or acquire data. Authentication follows the
36 +existing global-instrument controller convention.
37 +
38 +Acquisition requires ACTIVE / INDEX / IN / NSE / INR master metadata and exactly
39 +one NSE mapping with VERIFIED status, matching exchange/currency/request symbol,
40 +and catalog resolution source. Inactive, missing, conflicting, or untrusted
41 +identity fails closed. Context construction also requires trusted India universe
42 +membership before assigning the India broad benchmark.
43 +
44 +## Mapping evidence and exclusions
45 +
46 +- [Nifty 500](https://www.niftyindices.com/indices/equity/broad-based-indices/nifty-500)
47 + is the broad benchmark: its large/mid/small-cap coverage matches the platform's
48 + existing Nifty 500 classification universe. There is no Nifty 50 fallback.
49 +- [Nifty IT](https://www.niftyindices.com/indices/equity/sectoral-indices/nifty-it)
50 + matches the classifier's Information Technology → Technology mapping.
51 +- [Nifty Financial Services](https://www.niftyindices.com/indices/equity/sectoral-indices/nifty-financial-services)
52 + covers banks and other financial services, matching Financial Services →
53 + Financials. A bank-only index would exclude part of this classification.
54 +- [Nifty Healthcare](https://www.niftyindices.com/indices/equity/sectoral-indices/nifty-healthcare-index)
55 + matches Healthcare; a pharma-only substitution is not made.
56 +
57 +The official NSE `/api/allIndices` catalog and the captured history responses
58 +confirmed these four index contracts on 2026-09-13. Fixtures retain those responses.
59 +
60 +Observed unmapped classifications: Communication Services (14), Consumer
61 +Discretionary (88), Consumer Staples (28), Energy (17), Industrials (75), Materials
62 +(55), Real Estate (11), Utilities (17); 17 records lacked classification. Mapped
63 +counts were Financials 101, Healthcare 48, Technology 27. These counts describe
64 +the audit snapshot, not an ongoing invariant. Broad compound classifications
65 +are not assigned to narrower indices. Raw `FINANCIAL SERVICES`, `Bank`, and other
66 +aliases are not accepted by the mapping adapter; classification must first come
67 +from the canonical classifier. Duplicate classification evidence fails closed.
68 +
69 +## Official history contract
70 +
71 +History version: `NSE_INDEX_HISTORY_V1`.
72 +
73 +`GET https://www.nseindia.com/api/historicalOR/indicesHistory`
74 +with `indexType=<verified mapping>`, `from=DD-MM-YYYY`, `to=DD-MM-YYYY`.
75 +Bootstrap: `https://www.nseindia.com/report-detail/eq_security`.
76 +The older `/api/historical/indicesHistory` returned HTTP 200 HTML during discovery
77 +and is deliberately not used. The equity security CSV endpoint is not reused.
78 +
79 +The index provider composes the existing NSE provider's transport/session,
80 +browser headers, cookies, lock, request spacing and bounded retry behavior; it
81 +does not call its equity identity gate or CSV parser. One transport/session is
82 +reused sequentially per explicit benchmark population invocation and closed in
83 +`finally`. There is no global cookie pool, parallel fetch, proxy, or bypass.
84 +This small composition uses the existing private transport methods; a future
85 +transport refactor must preserve the tested shared-session contract.
86 +
87 +JSON is `{ "data": [ ... ] }`. Required fields:
88 +`EOD_INDEX_NAME`, `EOD_TIMESTAMP`, `EOD_OPEN_INDEX_VAL`, `EOD_HIGH_INDEX_VAL`,
89 +`EOD_LOW_INDEX_VAL`, `EOD_CLOSE_INDEX_VAL`.
90 +
91 +Verified response-name aliases, confined to this parser:
92 +
93 +| Requested symbol | EOD_INDEX_NAME |
94 +|---|---|
95 +| NIFTY 500 | NIFTY 500 |
96 +| NIFTY IT | NIFTY IT |
97 +| NIFTY FINANCIAL SERVICES | NIFTY FIN SERVICE |
98 +| NIFTY HEALTHCARE INDEX | NIFTY HEALTHCARE |
99 +
100 +Case/whitespace normalization is explicit; no other symbol equivalence is inferred.
101 +`EOD_TIMESTAMP` (for example `11-SEP-2026`) is the exchange-local DATE.
102 +`HI_TIMESTAMP` can be the previous UTC day and is never used for trading dates.
103 +JSON decimals are decoded directly to Decimal; domain OHLC validation applies.
104 +Missing fields, malformed values, out-of-window dates, duplicate dates, or wrong
105 +index names reject the complete response before persistence. Empty responses and
106 +HTTP 200 HTML are explicit failures, not zero-valued observations.
107 +
108 +`HIT_TRADED_QTY` and `HIT_TURN_OVER` were observed, but their index aggregation and
109 +units are not normalized in this phase. Volume, turnover, and previous close
110 +remain null. Actual OHLC values are index points; INR is the trusted index market
111 +denomination, not an assertion that each point is a tradable currency amount.
112 +
113 +## Population and persistence
114 +
115 +An explicit worker calls `IndiaMarketDataPopulationJobs.populate_benchmark_history`
116 +with registered canonical IDs and one bounded inclusive start/end window. It
117 +uses `ResearchRepository.upsert_daily_market_bars_async` via `persist_daily_result`.
118 +Primary identity remains `(global_instrument_id, trading_date, provider='NSE')`.
119 +Corrections update that key; another provider remains separate. No deletes,
120 +fabricated OHLC, or close-history dual-writes occur.
121 +
122 +The existing `nse_historical_request_window_days` defaults to 30 inclusive
123 +calendar days. This is an operational bound, **not an NSE guaranteed maximum**.
124 +Invocations are capped by `market_data_population_batch_size`, ordered by UUID,
125 +and persist per successful window. This explicit bounded operation does not
126 +schedule a broad backfill or automatically extend lookbacks.
127 +
128 +Spacing uses `market_data_population_request_interval_seconds` (default 0.20).
129 +The existing default two retries apply to transient network/5xx/429 failures;
130 +ordinary 4xx, including 403, are not retried. Exponential backoff and Retry-After
131 +handling are inherited. Failure cooldown uses the existing population cooldown;
132 +429 also inhibits subsequent instruments in the worker. Cooldowns are process
133 +local, like the existing population jobs. Results reuse `NseHistoricalResult`,
134 +including HTTP status, failure reason, accepted/rejected counts and persisted rows.
135 +
136 +## Computation and batching
137 +
138 +Prepare contexts with
139 +`await orchestrator.sector_benchmark_contexts(candidate_ids, identity_headers=...)`,
140 +then pass them to `GlobalScanner.enrich_candidates(..., sector_contexts=contexts)`.
141 +This keeps the existing explicit Stage-B dependency boundary. No dashboard,
142 +watchlist, portfolio, scanner GET, or feature computation invokes acquisition.
143 +
144 +The context adapter reads the existing paged canonical universe and one benchmark
145 +catalog response. Portfolio-service batches master and mapping reads. Stage B
146 +collects distinct canonical stock/benchmark IDs and performs the existing bounded
147 +close-history read plus one NSE daily-bar read, grouping in memory. With the
148 +default batch bound, history SELECT counts are 0 for empty, 2 for one stock,
149 +2 for 18 same-sector stocks, and 2 for 18 stocks across the three mapped sectors.
150 +Canonical metadata preparation is separate from these history query counts.
151 +
152 +The existing SectorRelativeStrengthEngine consumes persisted NSE daily closes
153 +with DATE semantics, selecting a coherent provider series. Existing close-only
154 +fallback remains supported. Stock short/stale OHLC fallback follows the existing
155 +technical input policy; individual dates/providers are never spliced together.
156 +
157 +Lookbacks preserve the existing 5 / 21 / 63 / 126 **stock observations** for
158 +1W / 1M / 3M / 6M. Both exact stock endpoints must exist in the benchmark history.
159 +Returns are `(end / start - 1) * 100`. Weekends, holidays, and missing intermediate
160 +dates are not synthesized. No forward/back fill, nearest-date matching, or
161 +calendar-session guessing is performed. A missing endpoint makes that horizon
162 +unavailable; at least two horizons are needed for the existing score/state rules.
163 +Thresholds, score weights, Technical Features V2 and Rule Engine V1 are unchanged.
164 +
165 +Snapshots expose mapping version, canonical benchmark IDs, history source and
166 +explicit benchmark states: NO_SECTOR_CLASSIFICATION, UNMAPPED_SECTOR_BENCHMARK,
167 +BENCHMARK_IDENTITY_UNAVAILABLE, BENCHMARK_HISTORY_UNAVAILABLE,
168 +INSUFFICIENT_OVERLAP, STALE_BENCHMARK_HISTORY, AVAILABLE. Per-horizon diagnostics
169 +remain available even when another horizon has overlap. Missing evidence is never
170 +zero or a weak-sector conclusion. Existing seven-day freshness policy remains.
171 +
172 +## Controlled runtime validation and limits
173 +
174 +One four-day Nifty 500 probe established the JSON contract. Subsequently just
175 +four 30-day requests (2026-08-13 through 2026-09-11, one per registered index)
176 +returned HTTP 200 and 22 rows each. No all-sector or all-equity backfill ran.
177 +The captured responses were replayed locally without additional provider calls.
178 +
179 +The real master service registered/read back all four identities in the existing
180 +H2 test schema; its public metadata fed the Python runtime. The existing repository
181 +persisted 88 captured rows to local SQLite. Repeat upserts left 88 rows. Real stock
182 +classification and close history were copied read-only from local PostgreSQL:
183 +
184 +| Stock | Verified canonical sector | Actual overlapping dates | Result |
185 +|---|---|---:|---|
186 +| HDFCBANK | Financials | 22 | Sector and market evidence available |
187 +| SUNPHARMA | Healthcare | 22 | Sector and market evidence available |
188 +| TCS | Technology | 22 | Sector and market evidence available |
189 +| LT | Industrials | 22 with market | Sector explicitly unmapped |
190 +
191 +Repeated/reversed-input results matched, with networking blocked during compute.
192 +The small benchmark capture supports short horizons only; 3M/6M formulas are
193 +validated independently in tests, not claimed as available from this smoke.
194 +These outputs are engineering diagnostics, not production investment conclusions.
195 +
196 +The deployed PostgreSQL instance lacks the pre-existing daily-bar table. No
197 +migration was created/applied, and no benchmark was directly inserted into that
198 +master by SQL. Deployment of the prior daily-bar schema plus this canonical
199 +registration path remains necessary before deployed acquisition can work.
200 +
201 +Unresolved: unmapped sectors, deployed PostgreSQL validation, Global Opportunity
202 +Ranker, short/long action model, recommendation history/lifecycle, news/macro,
203 +prediction/backtesting. None is implemented by this slice.
204 +
205 +## Validation results
206 +
207 +Final research-engine suite: **992 passed**, one existing dependency deprecation
208 +warning. Included: 64 new benchmark tests, 21 sector-relative tests, 25 technical
209 +tests, 42 OHLCV tests, 35 daily-bar persistence tests, 45 backfill tests, 55 NSE
210 +equity provider tests, 22 population tests, 31 Phase-1/batch scanner tests, and
211 +29 Rule Engine V1 tests.
212 +
213 +Java benchmark registration/instrument-master tests: 19 passed, including the
214 +H2 registration/read-back integration test. The broader portfolio reactor run
215 +reported 194 tests with two failures: `CanonicalIdentityBootstrapTest` assumes
216 +an empty shared H2 database (passes when rerun alone), and
217 +`AppUserProvisionerTest` expects an obsolete conflict target/SQL shape. Their
218 +implementation/test files are unchanged by this phase; no unrelated fixes were
219 +included. The benchmark H2 test rolls its writes back after exporting public
220 +metadata for the runtime smoke.
221 +
222 +`git diff --check` passed. No changes to Technical Features V2, Rule Engine V1,
223 +NSE equity provider/backfill, frontend, or migrations. `platform.ps1` is unchanged
224 +by this phase (phase-start SHA256
225 +`4577DA738957A6C35A8035812586344E76A557DFE41A7355F5A4D5F3E9D2E3AF`).
226 +Nothing staged, committed, or pushed. `smtp.password` was not opened.
services/portfolio-service/src/main/java/com/aiinvestment/portfolio/api/GlobalInstrumentController.java
+18
@@ -48,6 +48,24 @@ public class GlobalInstrumentController {
48 .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Global instrument not found"));
49 }
50
51 + @GetMapping("/benchmarks")
52 + public java.util.List<GlobalInstrumentResponse> benchmarks(HttpServletRequest request) {
53 + appUserProvisioner.upsert(AuthenticatedUserResolver.require(request));
54 + return instrumentMasterService.registeredBenchmarks().stream().map(GlobalInstrumentResponse::from).toList();
55 + }
56 +
57 + @PostMapping("/benchmarks/{benchmarkKey}/register")
58 + public GlobalInstrumentResponse registerBenchmark(@PathVariable String benchmarkKey, HttpServletRequest request) {
59 + appUserProvisioner.upsert(AuthenticatedUserResolver.require(request));
60 + try {
61 + return GlobalInstrumentResponse.from(instrumentMasterService.registerBenchmark(benchmarkKey));
62 + } catch (IllegalArgumentException e) {
63 + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "UNKNOWN_BENCHMARK_KEY");
64 + } catch (IllegalStateException e) {
65 + throw new ResponseStatusException(HttpStatus.CONFLICT, "BENCHMARK_IDENTITY_CONFLICT");
66 + }
67 + }
68 +
69 @GetMapping
70 public InstrumentUniverseResponse enumerate(@RequestParam(name = "status", defaultValue = "ACTIVE") String status,
71 @RequestParam(name = "assetType", defaultValue = "EQUITY") AssetType assetType,
services/portfolio-service/src/main/java/com/aiinvestment/portfolio/application/InstrumentMasterService.java
+44
@@ -80,6 +80,50 @@ public class InstrumentMasterService {
80
81 public record GlobalInstrument(InstrumentMasterEntity master, List<InstrumentProviderMappingEntity> providerMappings) {}
82
83 + /** Platform keys are independent of provider symbols; catalog evidence is versioned. */
84 + public static final String BENCHMARK_CATALOG_VERSION = "NSE_BENCHMARK_CATALOG_V1";
85 + public static final Map<String, String> BENCHMARKS = Map.of(
86 + "INDIA_BROAD_PRICE", "NIFTY 500", "INDIA_TECHNOLOGY_PRICE", "NIFTY IT",
87 + "INDIA_FINANCIALS_PRICE", "NIFTY FINANCIAL SERVICES", "INDIA_HEALTHCARE_PRICE", "NIFTY HEALTHCARE INDEX");
88 +
89 + public static UUID benchmarkId(String key) {
90 + if (!BENCHMARKS.containsKey(key)) throw new IllegalArgumentException("UNKNOWN_BENCHMARK_KEY");
91 + return UUID.nameUUIDFromBytes(("aip:benchmark:" + key).getBytes(java.nio.charset.StandardCharsets.UTF_8));
92 + }
93 +
94 + @Transactional
95 + public GlobalInstrument registerBenchmark(String key) {
96 + UUID id = benchmarkId(key);
97 + String symbol = BENCHMARKS.get(key);
98 + lockIdentity("BENCHMARK:" + key);
99 + var bySymbol = mappings.findByProviderAndExchangeIgnoreCaseAndProviderSymbolIgnoreCase("NSE", "NSE", symbol);
100 + if (bySymbol.isPresent() && !id.equals(bySymbol.get().getInstrumentId()))
101 + throw new IllegalStateException("BENCHMARK_IDENTITY_CONFLICT");
102 + var existing = masters.findById(id);
103 + var currentNse = mappings.findByInstrumentId(id).stream().filter(m -> "NSE".equals(m.getProvider())).toList();
104 + if (currentNse.size() > 1 || currentNse.stream().anyMatch(m -> !symbol.equals(m.getProviderSymbol())))
105 + throw new IllegalStateException("BENCHMARK_IDENTITY_CONFLICT");
106 + if (existing.isPresent() && (existing.get().getAssetType() != com.aiinvestment.shared.domain.AssetType.INDEX
107 + || !"ACTIVE".equals(existing.get().getStatus()) || !"INR".equals(existing.get().getCurrency())
108 + || !"NSE".equals(existing.get().getPrimaryExchange()) || !"IN".equals(existing.get().getCountry())
109 + || !symbol.equals(existing.get().getPrimarySymbol())))
110 + throw new IllegalStateException("BENCHMARK_IDENTITY_CONFLICT");
111 + InstrumentMasterEntity master = existing.orElseGet(() -> masters.saveAndFlush(new InstrumentMasterEntity(id, null,
112 + symbol, com.aiinvestment.shared.domain.AssetType.INDEX, "INR", "IN", "NSE", symbol, "ACTIVE", Instant.now())));
113 + persistMapping(id, "NSE", symbol, null, "NSE", "INR", "VERIFIED", BENCHMARK_CATALOG_VERSION, new BigDecimal("0.99"));
114 + return new GlobalInstrument(master, mappings.findByInstrumentId(id));
115 + }
116 +
117 + @Transactional(readOnly=true)
118 + public List<GlobalInstrument> registeredBenchmarks() {
119 + var ids = BENCHMARKS.keySet().stream().sorted().map(InstrumentMasterService::benchmarkId).toList();
120 + var rows = masters.findAllById(ids);
121 + var byId = mappings.findByInstrumentIdIn(ids).stream()
122 + .collect(java.util.stream.Collectors.groupingBy(InstrumentProviderMappingEntity::getInstrumentId));
123 + return rows.stream().sorted(Comparator.comparing(row -> row.getInstrumentId().toString()))
124 + .map(row -> new GlobalInstrument(row, byId.getOrDefault(row.getInstrumentId(), List.of()))).toList();
125 + }
126 +
127 @Transactional(readOnly=true)
128 public Page<InstrumentMasterEntity> enumerate(String status, com.aiinvestment.shared.domain.AssetType assetType, Pageable pageable) {
129 if (status == null || assetType == null) throw new IllegalArgumentException("STATUS_AND_ASSET_TYPE_REQUIRED");
services/portfolio-service/src/test/java/com/aiinvestment/portfolio/application/BenchmarkRegistrationIntegrationTest.java new
+45
@@ -0,0 +1,45 @@
1 +package com.aiinvestment.portfolio.application;
2 +
3 +import com.aiinvestment.portfolio.api.GlobalInstrumentResponse;
4 +import com.aiinvestment.portfolio.infrastructure.persistence.InstrumentMasterRepository;
5 +import com.aiinvestment.portfolio.infrastructure.persistence.InstrumentProviderMappingRepository;
6 +import com.aiinvestment.shared.domain.AssetType;
7 +import com.fasterxml.jackson.databind.ObjectMapper;
8 +import org.junit.jupiter.api.Test;
9 +import org.springframework.beans.factory.annotation.Autowired;
10 +import org.springframework.boot.test.context.SpringBootTest;
11 +import org.springframework.test.context.ActiveProfiles;
12 +
13 +import java.nio.file.Path;
14 +import static org.assertj.core.api.Assertions.assertThat;
15 +
16 +@SpringBootTest
17 +@ActiveProfiles("test")
18 +@org.springframework.transaction.annotation.Transactional
19 +class BenchmarkRegistrationIntegrationTest {
20 + @Autowired InstrumentMasterService service;
21 + @Autowired InstrumentMasterRepository masters;
22 + @Autowired InstrumentProviderMappingRepository mappings;
23 + @Autowired ObjectMapper mapper;
24 +
25 + @Test
26 + void registersAndReadsCanonicalIndicesUsingExistingSchema() throws Exception {
27 + for (String key : InstrumentMasterService.BENCHMARKS.keySet()) {
28 + var first = service.registerBenchmark(key);
29 + var second = service.registerBenchmark(key);
30 + var id = InstrumentMasterService.benchmarkId(key);
31 + assertThat(first.master().getInstrumentId()).isEqualTo(second.master().getInstrumentId()).isEqualTo(id);
32 + assertThat(masters.findById(id).orElseThrow().getAssetType()).isEqualTo(AssetType.INDEX);
33 + assertThat(mappings.findByInstrumentId(id)).singleElement().satisfies(mapping -> {
34 + assertThat(mapping.getProvider()).isEqualTo("NSE");
35 + assertThat(mapping.getStatus()).isEqualTo("VERIFIED");
36 + assertThat(mapping.getResolutionSource()).isEqualTo(InstrumentMasterService.BENCHMARK_CATALOG_VERSION);
37 + });
38 + }
39 + var registered = service.registeredBenchmarks();
40 + assertThat(registered).hasSize(4);
41 + // Public canonical metadata exported for the captured-response local runtime smoke.
42 + mapper.writeValue(Path.of("target", "benchmark-registered-runtime.json").toFile(),
43 + registered.stream().map(GlobalInstrumentResponse::from).toList());
44 + }
45 +}
services/portfolio-service/src/test/java/com/aiinvestment/portfolio/application/InstrumentMasterServiceTest.java
+30
@@ -28,6 +28,36 @@ import static org.mockito.ArgumentMatchers.eq;
28 import static org.mockito.Mockito.*;
29
30 class InstrumentMasterServiceTest {
31 + @Test
32 + void benchmarkRegistrationUsesPlatformKeyAndIsIdempotent() {
33 + String key = "INDIA_FINANCIALS_PRICE";
34 + UUID id = InstrumentMasterService.benchmarkId(key);
35 + assertThat(id.toString()).isEqualTo("13ba7849-43e2-3e1e-a3f3-d93ee32d3d2f");
36 + var first = service.registerBenchmark(key);
37 + assertThat(first.master().getInstrumentId()).isEqualTo(id);
38 + assertThat(first.master().getAssetType()).isEqualTo(AssetType.INDEX);
39 + assertThat(first.master().getPrimarySymbol()).isEqualTo("NIFTY FINANCIAL SERVICES");
40 + assertThat(savedMappings().get(0).getResolutionSource()).isEqualTo("NSE_BENCHMARK_CATALOG_V1");
41 + when(masters.findById(id)).thenReturn(Optional.of(first.master()));
42 + service.registerBenchmark(key);
43 + verify(masters, times(1)).saveAndFlush(any());
44 + verify(legacy, never()).save(any());
45 + verifyNoInteractions(events);
46 + }
47 +
48 + @Test
49 + void benchmarkRegistrationRejectsUnknownKeysAndConflictingIdentity() {
50 + org.assertj.core.api.Assertions.assertThatThrownBy(() -> service.registerBenchmark("NIFTY 500"))
51 + .isInstanceOf(IllegalArgumentException.class);
52 + var mapping = mock(InstrumentProviderMappingEntity.class);
53 + when(mapping.getInstrumentId()).thenReturn(UUID.randomUUID());
54 + when(mappings.findByProviderAndExchangeIgnoreCaseAndProviderSymbolIgnoreCase("NSE", "NSE", "NIFTY 500"))
55 + .thenReturn(Optional.of(mapping));
56 + org.assertj.core.api.Assertions.assertThatThrownBy(() -> service.registerBenchmark("INDIA_BROAD_PRICE"))
57 + .isInstanceOf(IllegalStateException.class).hasMessage("BENCHMARK_IDENTITY_CONFLICT");
58 + verify(masters, never()).saveAndFlush(any());
59 + }
60 +
61 private InstrumentMasterRepository masters;
62 private InstrumentProviderMappingRepository mappings;
63 private InstrumentRepository legacy;
shared/java/domain/src/main/java/com/aiinvestment/shared/domain/AssetType.java
+1
@@ -2,6 +2,7 @@ package com.aiinvestment.shared.domain;
2
3 public enum AssetType {
4 EQUITY,
5 + INDEX,
6 ETF,
7 FUND,
8 BOND,