| 1 | import asyncio |
| 2 | import logging |
| 3 | import os |
| 4 | import signal |
| 5 | import secrets |
| 6 | import subprocess |
| 7 | from dataclasses import dataclass |
| 8 | from datetime import UTC, datetime |
| 9 | from http.cookies import SimpleCookie |
| 10 | from pathlib import Path |
| 11 | from uuid import UUID |
| 12 | |
| 13 | import httpx |
| 14 | |
| 15 | from app.models import ConnectorState, ConnectorStatusResponse |
| 16 | from app.settings import Settings |
| 17 | |
| 18 | logger = logging.getLogger(__name__) |
| 19 | GATEWAY_STOP_TIMEOUT_SECONDS = 10 |
| 20 | GATEWAY_TERMINATE_SIGNAL = signal.SIGTERM |
| 21 | GATEWAY_KILL_SIGNAL = getattr(signal, "SIGKILL", 9) |
| 22 | |
| 23 | |
| 24 | @dataclass |
| 25 | class ConnectorSession: |
| 26 | connector_id: UUID |
| 27 | user_id: UUID |
| 28 | login_token: str |
| 29 | runtime_status: ConnectorState = ConnectorState.STARTING |
| 30 | auth_status: ConnectorState = ConnectorState.AUTHENTICATION_REQUIRED |
| 31 | heartbeat_at: datetime | None = None |
| 32 | authenticated_at: datetime | None = None |
| 33 | process: subprocess.Popen | None = None |
| 34 | gateway_cookies: dict[str, str] = None |
| 35 | # A new Gateway process cannot safely consume a browser session created by |
| 36 | # an earlier process. The first login request clears the Gateway-scoped |
| 37 | # browser state before allowing this process to issue authoritative cookies. |
| 38 | gateway_cookie_names_to_clear: set[str] = None |
| 39 | gateway_browser_reset_pending: bool = False |
| 40 | code: str = "STARTING" |
| 41 | message: str = "IBKR connector runtime is starting." |
| 42 | |
| 43 | def __post_init__(self) -> None: |
| 44 | if self.gateway_cookies is None: |
| 45 | self.gateway_cookies = {} |
| 46 | if self.gateway_cookie_names_to_clear is None: |
| 47 | self.gateway_cookie_names_to_clear = set() |
| 48 | |
| 49 | |
| 50 | class ConnectorError(RuntimeError): |
| 51 | def __init__(self, code: str, message: str) -> None: |
| 52 | super().__init__(message) |
| 53 | self.code = code |
| 54 | self.message = message |
| 55 | |
| 56 | |
| 57 | class ConnectorManager: |
| 58 | def __init__(self, settings: Settings) -> None: |
| 59 | self.settings = settings |
| 60 | self.sessions: dict[UUID, ConnectorSession] = {} |
| 61 | self._lock = asyncio.Lock() |
| 62 | |
| 63 | async def create(self, connector_id: UUID, user_id: UUID) -> ConnectorSession: |
| 64 | async with self._lock: |
| 65 | existing = self.sessions.get(connector_id) |
| 66 | if existing: |
| 67 | self.require_owner(existing, user_id) |
| 68 | if existing.runtime_status == ConnectorState.STOPPED: |
| 69 | existing = ConnectorSession( |
| 70 | connector_id=connector_id, |
| 71 | user_id=user_id, |
| 72 | login_token=secrets.token_urlsafe(32), |
| 73 | gateway_browser_reset_pending=True, |
| 74 | ) |
| 75 | self.sessions[connector_id] = existing |
| 76 | self._start_gateway(existing) |
| 77 | if existing.runtime_status != ConnectorState.ERROR: |
| 78 | await self.refresh_status(existing) |
| 79 | return existing |
| 80 | active = [session for session in self.sessions.values() if session.runtime_status != ConnectorState.STOPPED] |
| 81 | if active: |
| 82 | raise ConnectorError( |
| 83 | "DEV_SINGLE_CONNECTOR_LIMIT", |
| 84 | "This runtime can host one IBKR Gateway session. Use one pod per connector for isolation.", |
| 85 | ) |
| 86 | session = ConnectorSession( |
| 87 | connector_id=connector_id, |
| 88 | user_id=user_id, |
| 89 | login_token=secrets.token_urlsafe(32), |
| 90 | gateway_browser_reset_pending=True, |
| 91 | ) |
| 92 | self.sessions[connector_id] = session |
| 93 | self._start_gateway(session) |
| 94 | if session.runtime_status != ConnectorState.ERROR: |
| 95 | await self.refresh_status(session) |
| 96 | return session |
| 97 | |
| 98 | def get(self, connector_id: UUID, user_id: UUID) -> ConnectorSession: |
| 99 | session = self.sessions.get(connector_id) |
| 100 | if not session: |
| 101 | raise ConnectorError("CONNECTOR_NOT_FOUND", "Connector runtime was not found.") |
| 102 | self.require_owner(session, user_id) |
| 103 | return session |
| 104 | |
| 105 | async def status(self, connector_id: UUID, user_id: UUID) -> ConnectorSession: |
| 106 | session = self.get(connector_id, user_id) |
| 107 | await self.refresh_status(session) |
| 108 | return session |
| 109 | |
| 110 | async def stop(self, connector_id: UUID, user_id: UUID) -> ConnectorSession: |
| 111 | session = self.get(connector_id, user_id) |
| 112 | session.runtime_status = ConnectorState.STOPPING |
| 113 | session.code = "STOPPING" |
| 114 | session.message = "IBKR Gateway process is stopping." |
| 115 | process = session.process |
| 116 | if process and process.poll() is None: |
| 117 | _terminate_gateway_process(process) |
| 118 | session.runtime_status = ConnectorState.STOPPED |
| 119 | session.auth_status = ConnectorState.SESSION_EXPIRED |
| 120 | session.process = None |
| 121 | session.code = "STOPPED" |
| 122 | session.message = "IBKR connector runtime is stopped." |
| 123 | return session |
| 124 | |
| 125 | async def restart(self, connector_id: UUID, user_id: UUID) -> ConnectorSession: |
| 126 | previous_cookie_names = set(self.get(connector_id, user_id).gateway_cookies) |
| 127 | await self.stop(connector_id, user_id) |
| 128 | |
| 129 | async with self._lock: |
| 130 | old_session = self.sessions[connector_id] |
| 131 | self.require_owner(old_session, user_id) |
| 132 | session = ConnectorSession( |
| 133 | connector_id=connector_id, |
| 134 | user_id=user_id, |
| 135 | login_token=secrets.token_urlsafe(32), |
| 136 | gateway_cookie_names_to_clear=previous_cookie_names, |
| 137 | gateway_browser_reset_pending=True, |
| 138 | ) |
| 139 | self.sessions[connector_id] = session |
| 140 | |
| 141 | self._start_gateway(session) |
| 142 | |
| 143 | if session.runtime_status != ConnectorState.ERROR: |
| 144 | await self.refresh_status(session) |
| 145 | |
| 146 | return session |
| 147 | |
| 148 | async def gateway_get(self, connector_id: UUID, user_id: UUID, path: str) -> object: |
| 149 | session = await self.status(connector_id, user_id) |
| 150 | if session.auth_status != ConnectorState.CONNECTED: |
| 151 | raise ConnectorError("SESSION_EXPIRED", "IBKR Gateway session is not authenticated.") |
| 152 | return await self._gateway_get(path, session) |
| 153 | |
| 154 | async def refresh_status(self, session: ConnectorSession) -> None: |
| 155 | if session.runtime_status in {ConnectorState.STOPPED, ConnectorState.STOPPING}: |
| 156 | return |
| 157 | process = session.process |
| 158 | if process and process.poll() is not None: |
| 159 | session.runtime_status = ConnectorState.ERROR |
| 160 | session.auth_status = ConnectorState.SESSION_EXPIRED |
| 161 | session.code = "GATEWAY_PROCESS_EXITED" |
| 162 | session.message = "IBKR Gateway process exited unexpectedly." |
| 163 | return |
| 164 | if not self.settings.gateway_api_base_url: |
| 165 | session.runtime_status = ConnectorState.DEGRADED |
| 166 | session.auth_status = ConnectorState.AUTHENTICATION_REQUIRED |
| 167 | session.code = "GATEWAY_BASE_URL_MISSING" |
| 168 | session.message = "AIP_IBKR_GATEWAY_BASE_URL must be configured." |
| 169 | return |
| 170 | try: |
| 171 | payload = await self._gateway_get("/iserver/auth/status", session) |
| 172 | except ConnectorError as exc: |
| 173 | if session.runtime_status == ConnectorState.STARTING or session.authenticated_at is None: |
| 174 | session.runtime_status = ConnectorState.AUTHENTICATION_REQUIRED |
| 175 | session.auth_status = ConnectorState.AUTHENTICATION_REQUIRED |
| 176 | else: |
| 177 | session.runtime_status = ConnectorState.DEGRADED |
| 178 | session.auth_status = ConnectorState.SESSION_EXPIRED |
| 179 | session.code = exc.code |
| 180 | session.message = exc.message |
| 181 | return |
| 182 | now = datetime.now(UTC) |
| 183 | session.heartbeat_at = now |
| 184 | # Client Portal can report authenticated=true before the popup leaves its MFA document and |
| 185 | # before the auxiliary connected/established flags converge. Authentication is the authority; |
| 186 | # requiring every transport flag caused successful second MFA approvals to remain pending. |
| 187 | authenticated = bool(payload.get("authenticated")) |
| 188 | competing = not authenticated and (bool(payload.get("competing")) or payload.get("connected") is False) |
| 189 | logger.info( |
| 190 | "ibkr_auth_probe connector=%s runtime_exists=%s authenticated=%s connected=%s established=%s competing=%s authenticated_at_present=%s", |
| 191 | session.connector_id, |
| 192 | session.process is not None, |
| 193 | authenticated, |
| 194 | bool(payload.get("connected")), |
| 195 | bool(payload.get("established", True)), |
| 196 | bool(payload.get("competing")), |
| 197 | session.authenticated_at is not None, |
| 198 | ) |
| 199 | if authenticated: |
| 200 | session.runtime_status = ConnectorState.CONNECTED |
| 201 | session.auth_status = ConnectorState.CONNECTED |
| 202 | session.authenticated_at = now |
| 203 | session.code = "CONNECTED" |
| 204 | session.message = "IBKR Gateway session is authenticated." |
| 205 | elif competing: |
| 206 | session.runtime_status = ConnectorState.DEGRADED |
| 207 | session.auth_status = ConnectorState.SESSION_EXPIRED |
| 208 | session.code = "SESSION_EXPIRED" |
| 209 | session.message = "IBKR Gateway reports the session is not connected." |
| 210 | else: |
| 211 | session.runtime_status = ConnectorState.AUTHENTICATION_REQUIRED |
| 212 | session.auth_status = ConnectorState.AUTHENTICATION_REQUIRED |
| 213 | session.code = "AUTHENTICATION_REQUIRED" |
| 214 | session.message = "Interactive IBKR authentication is required." |
| 215 | |
| 216 | def login_url(self, session: ConnectorSession) -> str: |
| 217 | # The Gateway login bundle only submits its final Dispatcher callback |
| 218 | # after mobile approval when forwardTo is present. That callback is |
| 219 | # what establishes the local Client Portal session. |
| 220 | path = ( |
| 221 | f"/connector-sessions/{session.connector_id}/login/sso/Login" |
| 222 | f"?forwardTo=22&RL=1&ip2loc=on&loginToken={session.login_token}" |
| 223 | ) |
| 224 | return f"{self.settings.login_public_base_url}{path}" if self.settings.login_public_base_url else path |
| 225 | |
| 226 | def response(self, session: ConnectorSession) -> ConnectorStatusResponse: |
| 227 | return ConnectorStatusResponse( |
| 228 | connectorId=session.connector_id, |
| 229 | userId=session.user_id, |
| 230 | runtimeStatus=session.runtime_status, |
| 231 | authStatus=session.auth_status, |
| 232 | heartbeatAt=_iso(session.heartbeat_at), |
| 233 | authenticatedAt=_iso(session.authenticated_at), |
| 234 | loginUrl=self.login_url(session), |
| 235 | code=session.code, |
| 236 | message=session.message, |
| 237 | ) |
| 238 | |
| 239 | def require_login_token(self, connector_id: UUID, token: str) -> ConnectorSession: |
| 240 | session = self.sessions.get(connector_id) |
| 241 | if not session or not secrets.compare_digest(session.login_token, token): |
| 242 | raise ConnectorError("LOGIN_TOKEN_INVALID", "Connector login route is invalid or expired.") |
| 243 | return session |
| 244 | |
| 245 | @staticmethod |
| 246 | def remember_gateway_request_cookies(session: ConnectorSession, cookie_header: str) -> None: |
| 247 | if not cookie_header: |
| 248 | return |
| 249 | parsed = SimpleCookie(cookie_header) |
| 250 | for name, morsel in parsed.items(): |
| 251 | session.gateway_cookies[name] = morsel.value |
| 252 | |
| 253 | @staticmethod |
| 254 | def remember_gateway_response_cookies(session: ConnectorSession, set_cookie_headers: list[str]) -> None: |
| 255 | for header in set_cookie_headers: |
| 256 | parsed = SimpleCookie(header) |
| 257 | for name, morsel in parsed.items(): |
| 258 | if morsel.value: |
| 259 | session.gateway_cookies[name] = morsel.value |
| 260 | else: |
| 261 | session.gateway_cookies.pop(name, None) |
| 262 | |
| 263 | |
| 264 | @staticmethod |
| 265 | def require_owner(session: ConnectorSession, user_id: UUID) -> None: |
| 266 | if session.user_id != user_id: |
| 267 | raise ConnectorError("CONNECTOR_FORBIDDEN", "Connector belongs to a different user.") |
| 268 | |
| 269 | def _start_gateway(self, session: ConnectorSession) -> None: |
| 270 | package_path = Path(self.settings.aip_ibkr_gateway_package_path) |
| 271 | run_script = package_path / "bin" / "run.sh" |
| 272 | config_path = package_path / "root" / "conf.yaml" |
| 273 | if not self.settings.aip_ibkr_gateway_package_path: |
| 274 | session.runtime_status = ConnectorState.ERROR |
| 275 | session.auth_status = ConnectorState.SESSION_EXPIRED |
| 276 | session.code = "GATEWAY_PACKAGE_MISSING" |
| 277 | session.message = "AIP_IBKR_GATEWAY_PACKAGE_PATH must point to the official extracted IBKR Gateway package." |
| 278 | return |
| 279 | if not run_script.is_file() or not config_path.is_file(): |
| 280 | session.runtime_status = ConnectorState.ERROR |
| 281 | session.auth_status = ConnectorState.SESSION_EXPIRED |
| 282 | session.code = "GATEWAY_PACKAGE_INVALID" |
| 283 | session.message = "IBKR Gateway package must contain bin/run.sh and root/conf.yaml." |
| 284 | return |
| 285 | session.process = subprocess.Popen( |
| 286 | [str(run_script), "root/conf.yaml"], |
| 287 | cwd=str(package_path), |
| 288 | stdout=subprocess.DEVNULL, |
| 289 | stderr=subprocess.DEVNULL, |
| 290 | **_gateway_process_options(), |
| 291 | ) |
| 292 | session.runtime_status = ConnectorState.STARTING |
| 293 | session.auth_status = ConnectorState.AUTHENTICATION_REQUIRED |
| 294 | session.code = "STARTING" |
| 295 | session.message = "IBKR Gateway process was started." |
| 296 | |
| 297 | async def _gateway_get(self, path: str, session: ConnectorSession | None = None) -> object: |
| 298 | if not self.settings.gateway_api_base_url: |
| 299 | raise ConnectorError("GATEWAY_BASE_URL_MISSING", "AIP_IBKR_GATEWAY_BASE_URL must be configured.") |
| 300 | if _is_trading_path(path): |
| 301 | raise ConnectorError("TRADING_FORBIDDEN", "Trading operations are not exposed by this connector.") |
| 302 | url = f"{self.settings.gateway_api_base_url}/{path.lstrip('/')}" |
| 303 | headers = {} |
| 304 | if session and session.gateway_cookies: |
| 305 | headers["cookie"] = "; ".join(f"{name}={value}" for name, value in session.gateway_cookies.items()) |
| 306 | try: |
| 307 | async with httpx.AsyncClient( |
| 308 | verify=self.settings.aip_ibkr_gateway_tls_verify, |
| 309 | timeout=self.settings.aip_ibkr_heartbeat_timeout_seconds, |
| 310 | ) as client: |
| 311 | response = await client.get(url, headers=headers) |
| 312 | if session: |
| 313 | _trace_gateway_get(path, session, response) |
| 314 | if session: |
| 315 | self.remember_gateway_response_cookies(session, response.headers.get_list("set-cookie")) |
| 316 | if response.status_code in {401, 403}: |
| 317 | raise ConnectorError("SESSION_EXPIRED", "IBKR Gateway session is expired or unauthenticated.") |
| 318 | if response.status_code == 404: |
| 319 | raise ConnectorError("GATEWAY_NOT_FOUND", "IBKR Gateway endpoint was not found.") |
| 320 | response.raise_for_status() |
| 321 | return response.json() |
| 322 | except ConnectorError: |
| 323 | raise |
| 324 | except httpx.HTTPError as exc: |
| 325 | raise ConnectorError("GATEWAY_UNAVAILABLE", "IBKR Gateway is unavailable.") from exc |
| 326 | |
| 327 | |
| 328 | def _gateway_process_options() -> dict[str, object]: |
| 329 | if os.name == "posix": |
| 330 | return {"start_new_session": True} |
| 331 | return {} |
| 332 | |
| 333 | |
| 334 | def _terminate_gateway_process(process: subprocess.Popen) -> None: |
| 335 | if os.name == "posix": |
| 336 | try: |
| 337 | pgid = os.getpgid(process.pid) |
| 338 | os.killpg(pgid, GATEWAY_TERMINATE_SIGNAL) |
| 339 | except ProcessLookupError: |
| 340 | return |
| 341 | try: |
| 342 | process.wait(timeout=GATEWAY_STOP_TIMEOUT_SECONDS) |
| 343 | return |
| 344 | except subprocess.TimeoutExpired: |
| 345 | try: |
| 346 | os.killpg(pgid, GATEWAY_KILL_SIGNAL) |
| 347 | except ProcessLookupError: |
| 348 | return |
| 349 | process.wait(timeout=GATEWAY_STOP_TIMEOUT_SECONDS) |
| 350 | return |
| 351 | |
| 352 | process.terminate() |
| 353 | try: |
| 354 | process.wait(timeout=GATEWAY_STOP_TIMEOUT_SECONDS) |
| 355 | except subprocess.TimeoutExpired: |
| 356 | process.kill() |
| 357 | process.wait(timeout=GATEWAY_STOP_TIMEOUT_SECONDS) |
| 358 | |
| 359 | |
| 360 | def _is_trading_path(path: str) -> bool: |
| 361 | normalized = path.lower() |
| 362 | blocked_terms = ("/orders", "/iserver/account/orders", "/iserver/reply", "/iserver/order") |
| 363 | return any(term in normalized for term in blocked_terms) |
| 364 | |
| 365 | |
| 366 | def _trace_gateway_get(path: str, session: ConnectorSession, response: httpx.Response) -> None: |
| 367 | logger.info( |
| 368 | "ibkr_login_trace timestamp=%s connectorId=%s method=GET proxiedPath=%s upstreamStatus=%s " |
| 369 | "contentType=%s locationPresent=%s setCookieNames=%s cookieRequestNames=%s", |
| 370 | datetime.now(UTC).isoformat(), |
| 371 | session.connector_id, |
| 372 | "/" + path.lstrip("/"), |
| 373 | response.status_code, |
| 374 | response.headers.get("content-type", ""), |
| 375 | bool(response.headers.get("location")), |
| 376 | ",".join(_set_cookie_names(response.headers)), |
| 377 | ",".join(sorted((session.gateway_cookies or {}).keys())), |
| 378 | ) |
| 379 | |
| 380 | |
| 381 | def _set_cookie_names(headers: httpx.Headers) -> list[str]: |
| 382 | names = set() |
| 383 | for header in headers.get_list("set-cookie"): |
| 384 | parsed = SimpleCookie(header) |
| 385 | names.update(parsed.keys()) |
| 386 | return sorted(names) |
| 387 | |
| 388 | |
| 389 | def _iso(value: datetime | None) -> str | None: |
| 390 | return value.isoformat() if value else None |