| 1 | from __future__ import annotations |
| 2 | |
| 3 | import re |
| 4 | from datetime import datetime, timezone |
| 5 | from decimal import Decimal |
| 6 | |
| 7 | from app.models import ( |
| 8 | EventImpact, |
| 9 | ReliabilityLevel, |
| 10 | ResearchDocument, |
| 11 | ResearchEvent, |
| 12 | ResearchEventType, |
| 13 | SourceType, |
| 14 | TimeHorizon, |
| 15 | ) |
| 16 | from app.normalization import normalize_numbers |
| 17 | |
| 18 | |
| 19 | class ResearchEventExtractor: |
| 20 | def extract(self, document: ResearchDocument) -> list[ResearchEvent]: |
| 21 | raise NotImplementedError |
| 22 | |
| 23 | |
| 24 | class RuleBasedEventExtractor(ResearchEventExtractor): |
| 25 | def extract(self, document: ResearchDocument) -> list[ResearchEvent]: |
| 26 | if not document.instrument_id or not document.company_id or not document.normalized_text: |
| 27 | return [] |
| 28 | text = document.normalized_text |
| 29 | lower = text.lower() |
| 30 | events: list[ResearchEvent] = [] |
| 31 | if any(term in lower for term in ["new order", "order worth", "contract worth", "framework agreement", "purchase order"]): |
| 32 | events.append(self._event(document, ResearchEventType.NEW_ORDER, "Order or contract announcement", EventImpact.POSITIVE, TimeHorizon.MEDIUM_TERM, ["new order", "order worth", "contract worth", "framework agreement", "purchase order"])) |
| 33 | if any(term in lower for term in ["backlog", "order intake", "order book"]): |
| 34 | evidence = _evidence(text, ["backlog", "order intake", "order book"]) |
| 35 | impact = EventImpact.POSITIVE if any(term in evidence.lower() for term in ["increase", "grew", "growth", "+", "up "]) else EventImpact.NEUTRAL |
| 36 | events.append(self._event(document, ResearchEventType.ORDER_BACKLOG_CHANGE, "Order backlog update", impact, TimeHorizon.SHORT_TERM, ["backlog", "order intake", "order book"])) |
| 37 | if any(term in lower for term in ["capex", "capital expenditure", "investment of", "invests", "will invest"]): |
| 38 | events.append(self._event(document, ResearchEventType.CAPEX, "CAPEX or investment announcement", EventImpact.UNCERTAIN, TimeHorizon.LONG_TERM, ["capex", "capital expenditure", "investment of", "invests", "will invest"])) |
| 39 | if any(term in lower for term in ["capacity expansion", "expand capacity", "production capacity", "mw"]): |
| 40 | events.append(self._event(document, ResearchEventType.CAPACITY_EXPANSION, "Capacity expansion signal", EventImpact.POSITIVE, TimeHorizon.LONG_TERM, ["capacity expansion", "expand capacity", "production capacity", "volume ramp", "ramping up production"])) |
| 41 | if any(term in lower for term in ["new facility", "new factory", "factory expansion", "new plant"]): |
| 42 | events.append(self._event(document, ResearchEventType.NEW_FACILITY, "Facility expansion signal", EventImpact.POSITIVE, TimeHorizon.LONG_TERM, ["new facility", "new factory", "factory expansion", "new plant"])) |
| 43 | if any(term in lower for term in ["geographic expansion", "market expansion", "revenue growth", "demand growth", "product growth", "new product ramp"]): |
| 44 | events.append(self._event(document, ResearchEventType.GEOGRAPHIC_EXPANSION, "Growth expansion signal", EventImpact.POSITIVE, TimeHorizon.MEDIUM_TERM, ["geographic expansion", "market expansion", "revenue growth", "demand growth", "product growth", "new product ramp"])) |
| 45 | if any(term in lower for term in ["new customer", "customer win", "selected by"]): |
| 46 | events.append(self._event(document, ResearchEventType.NEW_CUSTOMER, "Customer win signal", EventImpact.POSITIVE, TimeHorizon.MEDIUM_TERM, ["new customer", "customer win", "selected by"])) |
| 47 | if any(term in lower for term in ["guidance raised", "raises guidance", "increased guidance"]): |
| 48 | events.append(self._event(document, ResearchEventType.GUIDANCE_RAISED, "Guidance raised", EventImpact.POSITIVE, TimeHorizon.SHORT_TERM, ["guidance raised", "raises guidance", "increased guidance", "raised full-year"])) |
| 49 | if any(term in lower for term in ["guidance confirmed", "confirms guidance", "maintained revenue guidance", "guidance maintained", "guidance for the full year"]): |
| 50 | events.append(self._event(document, ResearchEventType.GUIDANCE_MAINTAINED, "Guidance maintained", EventImpact.NEUTRAL, TimeHorizon.SHORT_TERM, ["guidance confirmed", "confirms guidance", "maintained revenue guidance", "guidance maintained", "guidance for the full year"])) |
| 51 | if any(term in lower for term in ["guidance lowered", "cuts guidance", "lowered guidance", "withdraws guidance", "guidance cut"]): |
| 52 | events.append(self._event(document, ResearchEventType.GUIDANCE_CUT, "Guidance cut", EventImpact.NEGATIVE, TimeHorizon.SHORT_TERM, ["guidance lowered", "cuts guidance", "lowered guidance", "withdraws guidance", "guidance cut"])) |
| 53 | if any(term in lower for term in ["cancelled order", "canceled order", "order cancelled", "order canceled", "contract cancelled", "contract canceled"]): |
| 54 | events.append(self._event(document, ResearchEventType.ORDER_CANCELLED, "Order cancellation", EventImpact.NEGATIVE, TimeHorizon.SHORT_TERM, ["cancelled order", "canceled order", "order cancelled", "order canceled", "contract cancelled", "contract canceled"])) |
| 55 | if any(term in lower for term in ["project delay", "project delayed", "factory ramp was delayed", "ramp was delayed", "delayed by"]): |
| 56 | events.append(self._event(document, ResearchEventType.PROJECT_DELAY, "Project delay", EventImpact.NEGATIVE, TimeHorizon.SHORT_TERM, ["project delay", "project delayed", "factory ramp was delayed", "ramp was delayed", "delayed by"])) |
| 57 | if "customer loss" in lower or "lost customer" in lower: |
| 58 | events.append(self._event(document, ResearchEventType.CUSTOMER_LOSS, "Customer loss", EventImpact.NEGATIVE, TimeHorizon.SHORT_TERM, ["customer loss", "lost customer"])) |
| 59 | if _is_annual_report(document, text): |
| 60 | events.append(self._event(document, ResearchEventType.ANNUAL_REPORT, "Annual report published", EventImpact.NEUTRAL, TimeHorizon.UNKNOWN, ["annual report"])) |
| 61 | if any(term in lower for term in ["earnings", "quarterly results", "half year results", "half-year results", "h1 results", "q2 results"]): |
| 62 | events.append(self._event(document, ResearchEventType.EARNINGS_RELEASE, "Financial results release", EventImpact.NEUTRAL, TimeHorizon.IMMEDIATE, ["earnings", "quarterly results", "half year results", "half-year results", "h1 results", "q2 results"])) |
| 63 | return _deduplicate_events(events) |
| 64 | |
| 65 | def _event( |
| 66 | self, |
| 67 | document: ResearchDocument, |
| 68 | event_type: ResearchEventType, |
| 69 | title: str, |
| 70 | impact: EventImpact, |
| 71 | horizon: TimeHorizon, |
| 72 | evidence_terms: list[str], |
| 73 | ) -> ResearchEvent: |
| 74 | assert document.instrument_id is not None |
| 75 | assert document.company_id is not None |
| 76 | text = document.normalized_text or "" |
| 77 | evidence = _evidence(text, evidence_terms) |
| 78 | numbers = normalize_numbers(evidence) |
| 79 | money = _relevant_money(numbers, event_type, evidence) |
| 80 | pct = _relevant_percent(numbers, event_type, evidence) |
| 81 | capacity = _relevant_capacity(numbers, event_type, evidence) |
| 82 | customer = _extract_customer(evidence) |
| 83 | counterparty = customer if _event_has_counterparty(event_type) else None |
| 84 | confidence = _confidence(document.reliability_level, document.entity_resolution_confidence, bool(numbers), document.published_at is not None) |
| 85 | return ResearchEvent( |
| 86 | instrument_id=document.instrument_id, |
| 87 | company_id=document.company_id, |
| 88 | event_type=event_type, |
| 89 | event_date=document.published_at, |
| 90 | detected_at=datetime.now(timezone.utc), |
| 91 | title=title, |
| 92 | summary=_summary(evidence), |
| 93 | source_document_id=document.document_id, |
| 94 | source_url=document.canonical_url, |
| 95 | source_type=document.source_type, |
| 96 | reliability=document.reliability_level, |
| 97 | confidence=confidence, |
| 98 | impact=impact, |
| 99 | time_horizon=horizon, |
| 100 | currency=money.currency if money else None, |
| 101 | monetary_value=money.value if money else None, |
| 102 | monetary_original=money.original if money else None, |
| 103 | percentage_value=pct.value if pct else None, |
| 104 | percentage_original=pct.original if pct else None, |
| 105 | customer=customer, |
| 106 | counterparty=counterparty, |
| 107 | capacity_value=capacity.value if capacity else None, |
| 108 | capacity_unit=capacity.unit if capacity else None, |
| 109 | raw_evidence_reference=evidence, |
| 110 | ) |
| 111 | |
| 112 | |
| 113 | class LLMEventExtractor(ResearchEventExtractor): |
| 114 | def extract(self, document: ResearchDocument) -> list[ResearchEvent]: |
| 115 | return [] |
| 116 | |
| 117 | |
| 118 | def _confidence(reliability: ReliabilityLevel, entity_confidence: float, has_number: bool, has_date: bool) -> float: |
| 119 | reliability_score = { |
| 120 | ReliabilityLevel.LEVEL_A: 0.95, |
| 121 | ReliabilityLevel.LEVEL_B: 0.85, |
| 122 | ReliabilityLevel.LEVEL_C: 0.70, |
| 123 | ReliabilityLevel.LEVEL_D: 0.55, |
| 124 | ReliabilityLevel.LEVEL_E: 0.35, |
| 125 | }[reliability] |
| 126 | score = reliability_score * 0.45 + entity_confidence * 0.35 + (0.10 if has_number else 0.0) + (0.10 if has_date else 0.0) |
| 127 | return float(min(1.0, round(score, 4))) |
| 128 | |
| 129 | |
| 130 | def _summary(text: str) -> str: |
| 131 | return text[:240].strip() |
| 132 | |
| 133 | |
| 134 | def _evidence(text: str, terms: list[str]) -> str: |
| 135 | sentences = re.split(r"(?<=[.!?])\s+", text) |
| 136 | matches: list[tuple[int, str]] = [] |
| 137 | for index, sentence in enumerate(sentences): |
| 138 | lower = sentence.lower() |
| 139 | if _is_boilerplate_sentence(sentence): |
| 140 | continue |
| 141 | if any(term in lower for term in terms): |
| 142 | matches.append((index, sentence)) |
| 143 | if not matches: |
| 144 | return "" |
| 145 | if any("guidance" in term for term in terms): |
| 146 | pure_guidance = [ |
| 147 | (index, sentence) |
| 148 | for index, sentence in matches |
| 149 | if "guidance" in sentence.lower() |
| 150 | and not any(term in sentence.lower() for term in ["order intake", "backlog", "order book", "new order"]) |
| 151 | ] |
| 152 | if pure_guidance: |
| 153 | return pure_guidance[0][1][:500] |
| 154 | if any(term in {"earnings", "quarterly results", "half year results", "half-year results", "h1 results", "q2 results"} for term in terms): |
| 155 | financial_result = [ |
| 156 | (index, sentence) |
| 157 | for index, sentence in matches |
| 158 | if not any(term in sentence.lower() for term in ["order intake", "backlog", "order book", "new order"]) |
| 159 | ] |
| 160 | if financial_result: |
| 161 | return financial_result[0][1][:500] |
| 162 | result_heading = [ |
| 163 | (index, sentence) |
| 164 | for index, sentence in matches |
| 165 | if any(term in sentence.lower() for term in ["q2 results", "half year results", "half-year results", "h1 results"]) |
| 166 | ] |
| 167 | if result_heading: |
| 168 | return _result_clause(result_heading[0][1])[:500] |
| 169 | if any(term in {"capacity expansion", "expand capacity", "production capacity", "volume ramp", "ramping up production"} for term in terms): |
| 170 | concrete_capacity = [ |
| 171 | (index, sentence) |
| 172 | for index, sentence in matches |
| 173 | if any(term in sentence.lower() for term in ["production capacity", "capacity expansion", "expand capacity", "ramping up production"]) |
| 174 | and not any(term in sentence.lower() for term in ["order intake", "backlog", "order book", "new order"]) |
| 175 | ] |
| 176 | if concrete_capacity: |
| 177 | return concrete_capacity[0][1][:500] |
| 178 | for index, sentence in matches: |
| 179 | if any(term in {"order intake", "backlog"} for term in terms): |
| 180 | prior_heading = sentences[index - 1] if index > 0 and len(sentences[index - 1]) < 180 else "" |
| 181 | combined = f"{prior_heading} {sentence}".strip() |
| 182 | return combined[:500] |
| 183 | return sentence[:500] |
| 184 | return matches[0][1][:500] |
| 185 | |
| 186 | |
| 187 | def _has_article_phrase(text: str, phrases: list[str]) -> bool: |
| 188 | for sentence in re.split(r"(?<=[.!?])\s+", text): |
| 189 | lower = sentence.lower() |
| 190 | if any(phrase in lower for phrase in phrases): |
| 191 | return True |
| 192 | return False |
| 193 | |
| 194 | |
| 195 | def _is_annual_report(document: ResearchDocument, text: str) -> bool: |
| 196 | title = (document.title or "").lower() |
| 197 | url = (document.canonical_url or "").lower() |
| 198 | if "annual report" in title or "annual-report" in url or "annual_reports" in url: |
| 199 | return True |
| 200 | article_sentences = [sentence for sentence in re.split(r"(?<=[.!?])\s+", text) if not _is_boilerplate_sentence(sentence)] |
| 201 | for sentence in article_sentences[:5]: |
| 202 | lower = sentence.lower() |
| 203 | if "annual report" in lower and any(term in lower for term in ["published", "released", "available", "report for fiscal"]): |
| 204 | return True |
| 205 | return False |
| 206 | |
| 207 | |
| 208 | def _is_boilerplate_sentence(sentence: str) -> bool: |
| 209 | lower = sentence.lower() |
| 210 | boilerplate_markers = [ |
| 211 | "the issuer is solely responsible", |
| 212 | "key word(s):", |
| 213 | "cet/cest", |
| 214 | "eqs news", |
| 215 | "dissemination of", |
| 216 | "end of inside information", |
| 217 | "contact investor relations", |
| 218 | "forward-looking statements", |
| 219 | "this could result from a variety of factors", |
| 220 | "actual results may differ", |
| 221 | "juli 2026 | finance news", |
| 222 | ] |
| 223 | return any(marker in lower for marker in boilerplate_markers) |
| 224 | |
| 225 | |
| 226 | def _result_clause(sentence: str) -> str: |
| 227 | parts = [part.strip() for part in re.split(r"\s*/\s*", sentence) if part.strip()] |
| 228 | result_parts = [ |
| 229 | part |
| 230 | for part in parts |
| 231 | if any(term in part.lower() for term in ["q2 results", "half year results", "half-year results", "h1 results"]) |
| 232 | ] |
| 233 | return " / ".join(result_parts) if result_parts else sentence |
| 234 | |
| 235 | |
| 236 | def _relevant_money(numbers, event_type: ResearchEventType, evidence: str): |
| 237 | if event_type in { |
| 238 | ResearchEventType.NEW_ORDER, |
| 239 | ResearchEventType.ORDER_BACKLOG_CHANGE, |
| 240 | ResearchEventType.CAPEX, |
| 241 | ResearchEventType.INVESTMENT, |
| 242 | ResearchEventType.GUIDANCE_RAISED, |
| 243 | ResearchEventType.GUIDANCE_CUT, |
| 244 | }: |
| 245 | return next((n for n in numbers if n.currency), None) |
| 246 | return None |
| 247 | |
| 248 | |
| 249 | def _relevant_percent(numbers, event_type: ResearchEventType, evidence: str): |
| 250 | if event_type in { |
| 251 | ResearchEventType.NEW_ORDER, |
| 252 | ResearchEventType.ORDER_BACKLOG_CHANGE, |
| 253 | ResearchEventType.GUIDANCE_RAISED, |
| 254 | ResearchEventType.GUIDANCE_CUT, |
| 255 | }: |
| 256 | return next((n for n in numbers if n.unit == "PERCENT"), None) |
| 257 | if event_type == ResearchEventType.EARNINGS_RELEASE and any( |
| 258 | term in evidence.lower() for term in ["revenue", "sales", "ebit", "ebitda", "margin", "profit", "cash flow", "cash-flow"] |
| 259 | ): |
| 260 | return next((n for n in numbers if n.unit == "PERCENT"), None) |
| 261 | return None |
| 262 | |
| 263 | |
| 264 | def _relevant_capacity(numbers, event_type: ResearchEventType, evidence: str): |
| 265 | if event_type in {ResearchEventType.CAPACITY_EXPANSION, ResearchEventType.NEW_FACILITY, ResearchEventType.FACTORY_EXPANSION}: |
| 266 | return next((n for n in numbers if n.unit in {"MW", "GW", "UNITS"}), None) |
| 267 | return None |
| 268 | |
| 269 | |
| 270 | def _extract_customer(text: str) -> str | None: |
| 271 | match = re.search(r"(?:with|from|by|for|selected by)\s+([A-Z][A-Za-z0-9&.\- ]{2,80})(?:\.|,| for | to | worth | valued| as )", text) |
| 272 | if not match: |
| 273 | return None |
| 274 | candidate = match.group(1).strip() |
| 275 | return candidate if _valid_organization_candidate(candidate) else None |
| 276 | |
| 277 | |
| 278 | def _event_has_counterparty(event_type: ResearchEventType) -> bool: |
| 279 | return event_type in { |
| 280 | ResearchEventType.NEW_ORDER, |
| 281 | ResearchEventType.MAJOR_CONTRACT, |
| 282 | ResearchEventType.GOVERNMENT_CONTRACT, |
| 283 | ResearchEventType.NEW_CUSTOMER, |
| 284 | ResearchEventType.CUSTOMER_EXPANSION, |
| 285 | ResearchEventType.MAJOR_CUSTOMER, |
| 286 | ResearchEventType.PARTNERSHIP, |
| 287 | ResearchEventType.ACQUISITION, |
| 288 | } |
| 289 | |
| 290 | |
| 291 | def _valid_organization_candidate(candidate: str) -> bool: |
| 292 | candidate = candidate.strip(" ,;:-") |
| 293 | lower = candidate.lower() |
| 294 | if re.search(r"\b(?:eur|usd|inr|million|billion|mn|bn|crore|lakh|percent|yoy|q[1-4]|h[1-2]|fy|fiscal)\b|[%\d]", lower): |
| 295 | return False |
| 296 | blocked = { |
| 297 | "a customer", |
| 298 | "a leading customer", |
| 299 | "a leading optoelectronics customer", |
| 300 | "the company", |
| 301 | "management", |
| 302 | "a year earlier", |
| 303 | "the full year", |
| 304 | "all customers", |
| 305 | "its suppliers", |
| 306 | "requested delivery dates", |
| 307 | } |
| 308 | if lower in blocked: |
| 309 | return False |
| 310 | if any( |
| 311 | phrase in lower |
| 312 | for phrase in [ |
| 313 | "a year earlier", |
| 314 | "from eur", |
| 315 | "from usd", |
| 316 | "compared with", |
| 317 | "quarter", |
| 318 | "fiscal year", |
| 319 | "full year", |
| 320 | "order intake", |
| 321 | "guidance", |
| 322 | "shipment", |
| 323 | "delivery", |
| 324 | "cash-flow", |
| 325 | ] |
| 326 | ): |
| 327 | return False |
| 328 | if lower.startswith(("a ", "an ", "the ", "its ", "their ", "all ")): |
| 329 | return False |
| 330 | if len(candidate.split()) > 6: |
| 331 | return False |
| 332 | return bool(re.search(r"\b(?:AG|SE|Inc|Ltd|Limited|Corporation|Corp|GmbH|NV|S\.A\.|PLC|Group)\b", candidate)) |
| 333 | |
| 334 | |
| 335 | def _deduplicate_events(events: list[ResearchEvent]) -> list[ResearchEvent]: |
| 336 | seen: set[tuple[ResearchEventType, Decimal | None, str | None]] = set() |
| 337 | unique: list[ResearchEvent] = [] |
| 338 | for event in events: |
| 339 | key = (event.event_type, event.monetary_value, event.customer) |
| 340 | if key not in seen: |
| 341 | seen.add(key) |
| 342 | unique.append(event) |
| 343 | return unique |