| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import argparse |
| 5 | import asyncio |
| 6 | import ipaddress |
| 7 | import itertools |
| 8 | import os |
| 9 | import urllib |
| 10 | import uuid |
| 11 | from contextlib import asynccontextmanager |
| 12 | from typing import Any |
| 13 | |
| 14 | import httpx |
| 15 | from fastapi import FastAPI, HTTPException, Request |
| 16 | from fastapi.responses import StreamingResponse |
| 17 | |
| 18 | |
| 19 | def maybe_wrap_ipv6_address(address: str) -> str: |
| 20 | try: |
| 21 | ipaddress.IPv6Address(address) |
| 22 | return f"[{address}]" |
| 23 | except ValueError: |
| 24 | return address |
| 25 | |
| 26 | |
| 27 | def make_http_path(host: str, port: int) -> str: |
| 28 | return f"http://{host}:{port}" |
| 29 | |
| 30 | |
| 31 | def prefiller_cycle(prefill_clients: list[Any]): |
| 32 | while True: |
| 33 | for prefill_client in prefill_clients: |
| 34 | for i in range(prefill_client["dp_size"]): |
| 35 | yield prefill_client, i |
| 36 | |
| 37 | |
| 38 | async def get_prefiller_info(prefill_clients: list, ready: asyncio.Event): |
| 39 | for prefill_client in prefill_clients: |
| 40 | while True: |
| 41 | try: |
| 42 | # Wait for prefill service to be ready |
| 43 | response = await prefill_client["client"].get("/health") |
| 44 | response.raise_for_status() |
| 45 | except Exception: |
| 46 | await asyncio.sleep(1) |
| 47 | continue |
| 48 | |
| 49 | response = await prefill_client["client"].get( |
| 50 | prefill_client["bootstrap_addr"] + "/query" |
| 51 | ) |
| 52 | response.raise_for_status() |
| 53 | data = response.json() |
| 54 | break |
| 55 | |
| 56 | for dp_rank, dp_entry in data.items(): |
| 57 | prefill_client["dp_engine_id"][int(dp_rank)] = dp_entry["engine_id"] |
| 58 | dp_size = len(data) |
| 59 | prefill_client["dp_size"] = dp_size |
| 60 | print(f"Inited prefiller {prefill_client['url']} with dp_size={dp_size}") |
| 61 | |
| 62 | ready.set() |
| 63 | print("All prefiller instances are ready.") |
| 64 | |
| 65 | |
| 66 | @asynccontextmanager |
| 67 | async def lifespan(app: FastAPI): |
| 68 | """Lifespan context manager to handle startup and shutdown events.""" |
| 69 | # Startup: Initialize client pools for prefiller and decoder services |
| 70 | app.state.prefill_clients = [] |
| 71 | app.state.decode_clients = [] |
| 72 | app.state.ready = asyncio.Event() |
| 73 | |
| 74 | # Create prefill clients |
| 75 | for i, (url, bootstrap_port) in enumerate(global_args.prefill): |
| 76 | parsed_url = urllib.parse.urlparse(url) |
| 77 | hostname = maybe_wrap_ipv6_address(parsed_url.hostname) |
| 78 | app.state.prefill_clients.append( |
| 79 | { |
| 80 | "client": httpx.AsyncClient( |
| 81 | timeout=None, |
| 82 | base_url=url, |
| 83 | limits=httpx.Limits( |
| 84 | max_connections=None, |
| 85 | max_keepalive_connections=None, |
| 86 | ), |
| 87 | ), |
| 88 | "url": url, |
| 89 | "bootstrap_addr": make_http_path(hostname, bootstrap_port or 8998), |
| 90 | "dp_engine_id": {}, |
| 91 | } |
| 92 | ) |
| 93 | |
| 94 | # Create decode clients |
| 95 | for i, url in enumerate(global_args.decode): |
| 96 | parsed_url = urllib.parse.urlparse(url) |
| 97 | hostname = maybe_wrap_ipv6_address(parsed_url.hostname) |
| 98 | app.state.decode_clients.append( |
| 99 | { |
| 100 | "client": httpx.AsyncClient( |
| 101 | timeout=None, |
| 102 | base_url=url, |
| 103 | limits=httpx.Limits( |
| 104 | max_connections=None, |
| 105 | max_keepalive_connections=None, |
| 106 | ), |
| 107 | ), |
| 108 | } |
| 109 | ) |
| 110 | |
| 111 | asyncio.create_task(get_prefiller_info(app.state.prefill_clients, app.state.ready)) |
| 112 | |
| 113 | # Initialize round-robin iterators |
| 114 | app.state.prefill_iterator = prefiller_cycle(app.state.prefill_clients) |
| 115 | app.state.decode_iterator = itertools.cycle(range(len(app.state.decode_clients))) |
| 116 | |
| 117 | print( |
| 118 | f"Got {len(app.state.prefill_clients)} prefill clients " |
| 119 | f"and {len(app.state.decode_clients)} decode clients." |
| 120 | ) |
| 121 | |
| 122 | yield |
| 123 | |
| 124 | # Shutdown: Close all clients |
| 125 | for client_info in app.state.prefill_clients: |
| 126 | await client_info["client"].aclose() |
| 127 | |
| 128 | for client_info in app.state.decode_clients: |
| 129 | await client_info["client"].aclose() |
| 130 | |
| 131 | |
| 132 | # Update FastAPI app initialization to use lifespan |
| 133 | app = FastAPI(lifespan=lifespan) |
| 134 | |
| 135 | |
| 136 | def parse_args(): |
| 137 | parser = argparse.ArgumentParser() |
| 138 | |
| 139 | parser.add_argument("--port", type=int, default=8000) |
| 140 | # Always use 127.0.0.1 as localhost binds to IPv6 which is blocked on CI |
| 141 | parser.add_argument("--host", type=str, default="127.0.0.1") |
| 142 | |
| 143 | # For prefiller instances |
| 144 | parser.add_argument( |
| 145 | "--prefill", |
| 146 | nargs="+", |
| 147 | action="append", |
| 148 | dest="prefill_raw", |
| 149 | metavar=("URL", "bootstrap_port"), |
| 150 | help=( |
| 151 | "Prefill server URL and optional bootstrap port. " |
| 152 | "Can be specified multiple times. " |
| 153 | "Format: --prefill URL [BOOTSTRAP_PORT]. " |
| 154 | "BOOTSTRAP_PORT can be a port number, " |
| 155 | "'none', or omitted (defaults to none)." |
| 156 | ), |
| 157 | ) |
| 158 | |
| 159 | # For decoder instances |
| 160 | parser.add_argument( |
| 161 | "--decode", |
| 162 | nargs=1, |
| 163 | action="append", |
| 164 | dest="decode_raw", |
| 165 | metavar=("URL",), |
| 166 | help="Decode server URL. Can be specified multiple times.", |
| 167 | ) |
| 168 | |
| 169 | args = parser.parse_args() |
| 170 | args.prefill = _parse_prefill_urls(args.prefill_raw) |
| 171 | args.decode = _parse_decode_urls(args.decode_raw) |
| 172 | |
| 173 | return args |
| 174 | |
| 175 | |
| 176 | # From sglang router_args.py |
| 177 | def _parse_prefill_urls(prefill_list): |
| 178 | """Parse prefill URLs from --prefill arguments. |
| 179 | |
| 180 | Format: --prefill URL [BOOTSTRAP_PORT] |
| 181 | |
| 182 | Example: |
| 183 | --prefill http://prefill1:8080 9000 # With bootstrap port |
| 184 | --prefill http://prefill2:8080 none # Explicitly no bootstrap port |
| 185 | --prefill http://prefill3:8080 # Defaults to no bootstrap port |
| 186 | |
| 187 | """ |
| 188 | if not prefill_list: |
| 189 | return [] |
| 190 | |
| 191 | prefill_urls = [] |
| 192 | for prefill_args in prefill_list: |
| 193 | url = prefill_args[0] |
| 194 | |
| 195 | # Handle optional bootstrap port |
| 196 | if len(prefill_args) >= 2: |
| 197 | bootstrap_port_str = prefill_args[1] |
| 198 | # Handle 'none' as None |
| 199 | if bootstrap_port_str.lower() == "none": |
| 200 | bootstrap_port = None |
| 201 | else: |
| 202 | try: |
| 203 | bootstrap_port = int(bootstrap_port_str) |
| 204 | except ValueError as e: |
| 205 | raise ValueError( |
| 206 | f"Invalid bootstrap port: {bootstrap_port_str}. Must be a number or 'none'" # noqa: E501 |
| 207 | ) from e |
| 208 | else: |
| 209 | # No bootstrap port specified, default to None |
| 210 | bootstrap_port = None |
| 211 | |
| 212 | prefill_urls.append((url, bootstrap_port)) |
| 213 | |
| 214 | return prefill_urls |
| 215 | |
| 216 | |
| 217 | def _parse_decode_urls(decode_list): |
| 218 | """Parse decode URLs from --decode arguments. |
| 219 | |
| 220 | Format: --decode URL |
| 221 | Example: --decode http://decode1:8081 --decode http://decode2:8081 |
| 222 | """ |
| 223 | if not decode_list: |
| 224 | return [] |
| 225 | |
| 226 | # decode_list is a list of single-element lists due to nargs=1 |
| 227 | return [url[0] for url in decode_list] |
| 228 | |
| 229 | |
| 230 | def get_next_client(app, service_type: str): |
| 231 | """Get the next client in round-robin fashion. |
| 232 | |
| 233 | Args: |
| 234 | app: The FastAPI app instance |
| 235 | service_type: Either 'prefill' or 'decode' |
| 236 | |
| 237 | Returns: |
| 238 | The next client to use |
| 239 | |
| 240 | """ |
| 241 | if service_type == "prefill": |
| 242 | return next(app.state.prefill_iterator) |
| 243 | elif service_type == "decode": |
| 244 | client_idx = next(app.state.decode_iterator) |
| 245 | return app.state.decode_clients[client_idx] |
| 246 | else: |
| 247 | raise ValueError(f"Unknown service type: {service_type}") |
| 248 | |
| 249 | |
| 250 | async def send_request_to_service( |
| 251 | client_info: dict, dp_rank: int, endpoint: str, req_data: dict, request_id: str |
| 252 | ): |
| 253 | """Send a request to a service using a client from the pool.""" |
| 254 | req_data = req_data.copy() |
| 255 | req_data["kv_transfer_params"] = { |
| 256 | "do_remote_decode": True, |
| 257 | "do_remote_prefill": False, |
| 258 | "transfer_id": f"xfer-{request_id}", |
| 259 | } |
| 260 | req_data["stream"] = False |
| 261 | req_data["max_tokens"] = 1 |
| 262 | if "max_completion_tokens" in req_data: |
| 263 | req_data["max_completion_tokens"] = 1 |
| 264 | if "stream_options" in req_data: |
| 265 | del req_data["stream_options"] |
| 266 | headers = { |
| 267 | "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", |
| 268 | "X-Request-Id": request_id, |
| 269 | "X-data-parallel-rank": str(dp_rank), |
| 270 | } |
| 271 | |
| 272 | response = await client_info["client"].post( |
| 273 | endpoint, json=req_data, headers=headers |
| 274 | ) |
| 275 | response.raise_for_status() |
| 276 | |
| 277 | # CRITICAL: Release connection back to pool |
| 278 | await response.aclose() |
| 279 | |
| 280 | |
| 281 | async def stream_service_response( |
| 282 | prefill_client_info: dict, |
| 283 | prefill_dp_rank: int, |
| 284 | decode_client_info: dict, |
| 285 | endpoint: str, |
| 286 | req_data: dict, |
| 287 | request_id: str, |
| 288 | ): |
| 289 | """Asynchronously stream response from a service using a client from the pool.""" |
| 290 | headers = { |
| 291 | "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", |
| 292 | "X-Request-Id": request_id, |
| 293 | } |
| 294 | |
| 295 | req_data["kv_transfer_params"] = { |
| 296 | "do_remote_decode": False, |
| 297 | "do_remote_prefill": True, |
| 298 | "remote_bootstrap_addr": prefill_client_info["bootstrap_addr"], |
| 299 | "remote_engine_id": prefill_client_info["dp_engine_id"][prefill_dp_rank], |
| 300 | "transfer_id": f"xfer-{request_id}", |
| 301 | } |
| 302 | |
| 303 | async with decode_client_info["client"].stream( |
| 304 | "POST", endpoint, json=req_data, headers=headers |
| 305 | ) as response: |
| 306 | response.raise_for_status() |
| 307 | async for chunk in response.aiter_bytes(): |
| 308 | yield chunk |
| 309 | |
| 310 | |
| 311 | async def _handle_completions(api: str, request: Request): |
| 312 | if not app.state.ready.is_set(): |
| 313 | raise HTTPException(status_code=503, detail="Service Unavailable") |
| 314 | |
| 315 | try: |
| 316 | req_data = await request.json() |
| 317 | request_id = str(uuid.uuid4()) |
| 318 | |
| 319 | # Get the next prefill client in round-robin fashion |
| 320 | prefill_client_info, prefill_dp_rank = get_next_client(request.app, "prefill") |
| 321 | |
| 322 | # Send request to prefill service |
| 323 | asyncio.create_task( |
| 324 | send_request_to_service( |
| 325 | prefill_client_info, prefill_dp_rank, api, req_data, request_id |
| 326 | ) |
| 327 | ) |
| 328 | |
| 329 | decode_client_info = get_next_client(request.app, "decode") |
| 330 | |
| 331 | # Stream response from decode service |
| 332 | async def generate_stream(): |
| 333 | async for chunk in stream_service_response( |
| 334 | prefill_client_info, |
| 335 | prefill_dp_rank, |
| 336 | decode_client_info, |
| 337 | api, |
| 338 | req_data, |
| 339 | request_id=request_id, |
| 340 | ): |
| 341 | yield chunk |
| 342 | |
| 343 | return StreamingResponse(generate_stream(), media_type="application/json") |
| 344 | |
| 345 | except Exception as e: |
| 346 | import sys |
| 347 | import traceback |
| 348 | |
| 349 | exc_info = sys.exc_info() |
| 350 | print(f"Error occurred in disagg prefill proxy server - {api} endpoint") |
| 351 | print(e) |
| 352 | print("".join(traceback.format_exception(*exc_info))) |
| 353 | raise |
| 354 | |
| 355 | |
| 356 | @app.post("/v1/completions") |
| 357 | async def handle_completions(request: Request): |
| 358 | return await _handle_completions("/v1/completions", request) |
| 359 | |
| 360 | |
| 361 | @app.post("/v1/chat/completions") |
| 362 | async def handle_chat_completions(request: Request): |
| 363 | return await _handle_completions("/v1/chat/completions", request) |
| 364 | |
| 365 | |
| 366 | if __name__ == "__main__": |
| 367 | global global_args |
| 368 | global_args = parse_args() |
| 369 | |
| 370 | import uvicorn |
| 371 | |
| 372 | uvicorn.run(app, host=global_args.host, port=global_args.port) |