| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """P2PConnector proxy server for OffloadingConnector + TieringOffloadingSpec. |
| 4 | |
| 5 | Unlike NixlConnector (which returns remote_host/remote_port in the prefill |
| 6 | response), OffloadingConnector does not embed connector coordinates in its |
| 7 | response. This proxy injects the prefiller's P2PConnector address into |
| 8 | kv_transfer_params before forwarding the decode request so the decoder knows |
| 9 | where to pull KV blocks from. |
| 10 | |
| 11 | Usage: |
| 12 | .venv/bin/python p2p_connector_proxy.py \ |
| 13 | --port 8192 \ |
| 14 | --prefiller-host 127.0.0.1 --prefiller-port 8100 \ |
| 15 | --decoder-host 127.0.0.1 --decoder-port 8200 \ |
| 16 | --p2p-connector-host 127.0.0.1 --p2p-connector-port 5710 |
| 17 | """ |
| 18 | |
| 19 | import argparse |
| 20 | import asyncio |
| 21 | import itertools |
| 22 | import logging |
| 23 | import os |
| 24 | import uuid |
| 25 | from contextlib import asynccontextmanager |
| 26 | |
| 27 | import httpx |
| 28 | from fastapi import FastAPI, Request |
| 29 | from fastapi.responses import StreamingResponse |
| 30 | |
| 31 | logger = logging.getLogger(__name__) |
| 32 | logger.setLevel(logging.DEBUG) |
| 33 | |
| 34 | |
| 35 | @asynccontextmanager |
| 36 | async def lifespan(app: FastAPI): |
| 37 | app.state.prefill_clients = [] |
| 38 | app.state.decode_clients = [] |
| 39 | |
| 40 | for i, (host, port) in enumerate(global_args.prefiller_instances): |
| 41 | app.state.prefill_clients.append( |
| 42 | { |
| 43 | "client": httpx.AsyncClient( |
| 44 | timeout=None, |
| 45 | base_url=f"http://{host}:{port}/v1", |
| 46 | limits=httpx.Limits( |
| 47 | max_connections=None, max_keepalive_connections=None |
| 48 | ), |
| 49 | ), |
| 50 | "host": host, |
| 51 | "port": port, |
| 52 | "id": i, |
| 53 | } |
| 54 | ) |
| 55 | |
| 56 | for i, (host, port) in enumerate(global_args.decoder_instances): |
| 57 | app.state.decode_clients.append( |
| 58 | { |
| 59 | "client": httpx.AsyncClient( |
| 60 | timeout=None, |
| 61 | base_url=f"http://{host}:{port}/v1", |
| 62 | limits=httpx.Limits( |
| 63 | max_connections=None, max_keepalive_connections=None |
| 64 | ), |
| 65 | ), |
| 66 | "host": host, |
| 67 | "port": port, |
| 68 | "id": i, |
| 69 | } |
| 70 | ) |
| 71 | |
| 72 | app.state.prefill_iterator = itertools.cycle(range(len(app.state.prefill_clients))) |
| 73 | app.state.decode_iterator = itertools.cycle(range(len(app.state.decode_clients))) |
| 74 | |
| 75 | # Round-robin over each role's data-parallel replicas (independent of the |
| 76 | # instance iterators above). For dp_size=1 these yield only rank 0. |
| 77 | app.state.prefill_dp_iterator = itertools.cycle( |
| 78 | range(global_args.prefiller_dp_size) |
| 79 | ) |
| 80 | app.state.decode_dp_iterator = itertools.cycle(range(global_args.decoder_dp_size)) |
| 81 | |
| 82 | mode = "decoder-first" if global_args.decoder_first else "prefiller-first" |
| 83 | if global_args.p2p: |
| 84 | mode += ",p2p" |
| 85 | pd_host = global_args.p2p_connector_host |
| 86 | pd_port = global_args.p2p_connector_port |
| 87 | n_pref = len(app.state.prefill_clients) |
| 88 | n_dec = len(app.state.decode_clients) |
| 89 | print( |
| 90 | f"Proxy ready [{mode}]: " |
| 91 | f"{n_pref} prefiller(s) x dp={global_args.prefiller_dp_size}, " |
| 92 | f"{n_dec} decoder(s) x dp={global_args.decoder_dp_size}. " |
| 93 | f"P2PConnector base at {pd_host}:{pd_port}" |
| 94 | ) |
| 95 | yield |
| 96 | |
| 97 | for ci in app.state.prefill_clients: |
| 98 | await ci["client"].aclose() |
| 99 | for ci in app.state.decode_clients: |
| 100 | await ci["client"].aclose() |
| 101 | |
| 102 | |
| 103 | app = FastAPI(lifespan=lifespan) |
| 104 | |
| 105 | |
| 106 | def parse_args(): |
| 107 | p = argparse.ArgumentParser() |
| 108 | p.add_argument("--port", type=int, default=8192) |
| 109 | p.add_argument("--host", type=str, default="127.0.0.1") |
| 110 | p.add_argument("--prefiller-hosts", type=str, nargs="+", default=["127.0.0.1"]) |
| 111 | p.add_argument("--prefiller-ports", type=int, nargs="+", default=[8100]) |
| 112 | p.add_argument("--decoder-hosts", type=str, nargs="+", default=["127.0.0.1"]) |
| 113 | p.add_argument("--decoder-ports", type=int, nargs="+", default=[8200]) |
| 114 | # P2PConnector coordinates of the prefiller — injected into decode requests. |
| 115 | p.add_argument( |
| 116 | "--p2p-connector-host", |
| 117 | type=str, |
| 118 | default="127.0.0.1", |
| 119 | help="Host of the prefiller's P2PConnector ZMQ socket", |
| 120 | ) |
| 121 | p.add_argument( |
| 122 | "--p2p-connector-port", |
| 123 | type=int, |
| 124 | default=int(os.getenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5710")), |
| 125 | help="Port of the prefiller's P2PConnector ZMQ socket " |
| 126 | "(default: $VLLM_P2P_SIDE_CHANNEL_PORT or 5710)", |
| 127 | ) |
| 128 | # P2PConnector coordinates of the decoder — injected into prefill requests |
| 129 | # so the prefiller's submit_store can resolve the peer to push KV to. |
| 130 | p.add_argument( |
| 131 | "--decoder-p2p-connector-host", |
| 132 | type=str, |
| 133 | default="127.0.0.1", |
| 134 | help="Host of the decoder's P2PConnector ZMQ socket", |
| 135 | ) |
| 136 | p.add_argument( |
| 137 | "--decoder-p2p-connector-port", |
| 138 | type=int, |
| 139 | default=int(os.getenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5710")) + 1, |
| 140 | help="Port of the decoder's P2PConnector ZMQ socket " |
| 141 | "(default: $VLLM_P2P_SIDE_CHANNEL_PORT + 1 or 5711)", |
| 142 | ) |
| 143 | p.add_argument( |
| 144 | "--decoder-first", |
| 145 | action="store_true", |
| 146 | help="Send decode request before prefill so decoder is already " |
| 147 | "waiting when KV blocks arrive (decoder-first mode)", |
| 148 | ) |
| 149 | p.add_argument( |
| 150 | "--p2p", |
| 151 | action="store_true", |
| 152 | help="P2P mode: do not inject kv_transfer_params on the prefiller; " |
| 153 | "on the decoder, inject kv_transfer_params with a top-level " |
| 154 | "'remote_kv_source' block ({kv_request_id, remote_host, remote_port}) " |
| 155 | "instead of the default 'remote_prefiller' block.", |
| 156 | ) |
| 157 | p.add_argument( |
| 158 | "--prefiller-dp-size", |
| 159 | type=int, |
| 160 | default=1, |
| 161 | help="Data-parallel replica count of the prefiller. When >1 the proxy " |
| 162 | "round-robins prefill across ranks via the X-data-parallel-rank header " |
| 163 | "and injects remote_port = p2p-connector-port + rank into the decode " |
| 164 | "request so the decoder pulls KV from that replica's control socket. " |
| 165 | "Assumes a single prefiller instance (one host/port fronting N replicas).", |
| 166 | ) |
| 167 | p.add_argument( |
| 168 | "--decoder-dp-size", |
| 169 | type=int, |
| 170 | default=1, |
| 171 | help="Data-parallel replica count of the decoder. When >1 the proxy " |
| 172 | "round-robins decode across ranks via the X-data-parallel-rank header.", |
| 173 | ) |
| 174 | args = p.parse_args() |
| 175 | if len(args.prefiller_hosts) != len(args.prefiller_ports): |
| 176 | raise ValueError("Prefiller host/port count mismatch") |
| 177 | if len(args.decoder_hosts) != len(args.decoder_ports): |
| 178 | raise ValueError("Decoder host/port count mismatch") |
| 179 | args.prefiller_instances = list(zip(args.prefiller_hosts, args.prefiller_ports)) |
| 180 | args.decoder_instances = list(zip(args.decoder_hosts, args.decoder_ports)) |
| 181 | return args |
| 182 | |
| 183 | |
| 184 | def _get_next(app, service: str): |
| 185 | if service == "prefill": |
| 186 | return app.state.prefill_clients[next(app.state.prefill_iterator)] |
| 187 | return app.state.decode_clients[next(app.state.decode_iterator)] |
| 188 | |
| 189 | |
| 190 | def _next_dp_rank(app, service: str): |
| 191 | """Advance the round-robin DP cursor for a role. |
| 192 | |
| 193 | Returns (rank, header_rank). ``rank`` (0..dp_size-1) is always used for the |
| 194 | P2P remote_port arithmetic; ``header_rank`` is the same value when the role |
| 195 | has dp_size>1 and None otherwise (so dp=1 sends no header, matching the |
| 196 | single-replica behavior). |
| 197 | """ |
| 198 | if service == "prefill": |
| 199 | rank = next(app.state.prefill_dp_iterator) |
| 200 | dp_size = global_args.prefiller_dp_size |
| 201 | else: |
| 202 | rank = next(app.state.decode_dp_iterator) |
| 203 | dp_size = global_args.decoder_dp_size |
| 204 | return rank, (rank if dp_size > 1 else None) |
| 205 | |
| 206 | |
| 207 | def _auth_headers(request_id: str) -> dict: |
| 208 | headers: dict = {"X-Request-Id": request_id} |
| 209 | api_key = os.environ.get("OPENAI_API_KEY", "") |
| 210 | if api_key: |
| 211 | headers["Authorization"] = f"Bearer {api_key}" |
| 212 | return headers |
| 213 | |
| 214 | |
| 215 | async def _prefill(client_info, endpoint, req_data, request_id, dp_rank=None): |
| 216 | """Send a prefill-only request (max_tokens=1) to the prefiller.""" |
| 217 | data = req_data.copy() |
| 218 | if global_args.p2p: |
| 219 | data.pop("kv_transfer_params", None) |
| 220 | else: |
| 221 | data["kv_transfer_params"] = { |
| 222 | "remote_decoder": { |
| 223 | "kv_request_id": request_id, |
| 224 | }, |
| 225 | } |
| 226 | data["stream"] = False |
| 227 | data["max_tokens"] = 1 |
| 228 | data.pop("max_completion_tokens", None) |
| 229 | data.pop("stream_options", None) |
| 230 | data.pop("min_tokens", None) |
| 231 | data.pop("min_completion_tokens", None) |
| 232 | |
| 233 | headers = _auth_headers(request_id) |
| 234 | if dp_rank is not None: |
| 235 | # Pin this prefill to a specific DP replica so its P2PConnector (bound |
| 236 | # at base_port + data_parallel_index) holds the produced KV blocks. |
| 237 | headers["X-data-parallel-rank"] = str(dp_rank) |
| 238 | resp = await client_info["client"].post(endpoint, json=data, headers=headers) |
| 239 | resp.raise_for_status() |
| 240 | await resp.aread() |
| 241 | return resp |
| 242 | |
| 243 | |
| 244 | async def _stream_decode(client_info, endpoint, req_data, request_id, dp_rank=None): |
| 245 | headers = _auth_headers(request_id) |
| 246 | if dp_rank is not None: |
| 247 | # Round-robin this decode onto a specific DP replica of the decoder. |
| 248 | headers["X-data-parallel-rank"] = str(dp_rank) |
| 249 | async with client_info["client"].stream( |
| 250 | "POST", endpoint, json=req_data, headers=headers |
| 251 | ) as resp: |
| 252 | resp.raise_for_status() |
| 253 | async for chunk in resp.aiter_bytes(): |
| 254 | yield chunk |
| 255 | |
| 256 | |
| 257 | async def _handle_completions(api: str, request: Request): |
| 258 | try: |
| 259 | req_data = await request.json() |
| 260 | request_id = str(uuid.uuid4()) |
| 261 | |
| 262 | # Pick DP replicas for both roles up front (before any await) so the |
| 263 | # two round-robin advances are atomic together. Header rank is None for |
| 264 | # a role with dp_size==1 (no header → single replica, unchanged). |
| 265 | prefill_rank, prefill_hdr = _next_dp_rank(request.app, "prefill") |
| 266 | decode_rank, decode_hdr = _next_dp_rank(request.app, "decode") |
| 267 | |
| 268 | prefill_client = _get_next(request.app, "prefill") |
| 269 | await _prefill(prefill_client, api, req_data, request_id, dp_rank=prefill_hdr) |
| 270 | |
| 271 | # Inject the prefiller's P2PConnector address so the decoder can pull |
| 272 | # KV blocks from it. remote_port = base + prefill_rank targets the |
| 273 | # replica that produced the KV (base+0 == base when dp=1). |
| 274 | decoder_key = "remote_kv_source" if global_args.p2p else "remote_prefiller" |
| 275 | req_data["kv_transfer_params"] = { |
| 276 | decoder_key: { |
| 277 | "kv_request_id": request_id, |
| 278 | "remote_host": global_args.p2p_connector_host, |
| 279 | "remote_port": global_args.p2p_connector_port + prefill_rank, |
| 280 | }, |
| 281 | } |
| 282 | |
| 283 | decode_client = _get_next(request.app, "decode") |
| 284 | logger.debug( |
| 285 | "prefill=%s dp=%s decode=%s dp=%s", |
| 286 | prefill_client, |
| 287 | prefill_rank, |
| 288 | decode_client, |
| 289 | decode_rank, |
| 290 | ) |
| 291 | |
| 292 | async def generate(): |
| 293 | async for chunk in _stream_decode( |
| 294 | decode_client, api, req_data, request_id, dp_rank=decode_hdr |
| 295 | ): |
| 296 | yield chunk |
| 297 | |
| 298 | return StreamingResponse(generate(), media_type="application/json") |
| 299 | |
| 300 | except Exception as e: |
| 301 | import sys |
| 302 | import traceback |
| 303 | |
| 304 | print(f"Proxy error on {api}: {e}") |
| 305 | print("".join(traceback.format_exception(*sys.exc_info()))) |
| 306 | raise |
| 307 | |
| 308 | |
| 309 | async def _handle_completions_decoder_first(api: str, request: Request): |
| 310 | """Decoder-first mode: send decode request before prefill. |
| 311 | |
| 312 | The decoder establishes its request and starts polling for KV blocks |
| 313 | immediately. The prefill is then sent so the prefiller computes and |
| 314 | pushes blocks to the already-waiting decoder. |
| 315 | """ |
| 316 | try: |
| 317 | req_data = await request.json() |
| 318 | request_id = str(uuid.uuid4()) |
| 319 | |
| 320 | # Pick DP replicas up front (see _handle_completions for rationale). |
| 321 | prefill_rank, prefill_hdr = _next_dp_rank(request.app, "prefill") |
| 322 | decode_rank, decode_hdr = _next_dp_rank(request.app, "decode") |
| 323 | |
| 324 | prefill_client = _get_next(request.app, "prefill") |
| 325 | decode_client = _get_next(request.app, "decode") |
| 326 | |
| 327 | decoder_key = "remote_kv_source" if global_args.p2p else "remote_prefiller" |
| 328 | decode_data = req_data.copy() |
| 329 | decode_data["kv_transfer_params"] = { |
| 330 | decoder_key: { |
| 331 | "kv_request_id": request_id, |
| 332 | "remote_host": global_args.p2p_connector_host, |
| 333 | "remote_port": global_args.p2p_connector_port + prefill_rank, |
| 334 | }, |
| 335 | } |
| 336 | |
| 337 | async def generate(): |
| 338 | queue: asyncio.Queue = asyncio.Queue() |
| 339 | |
| 340 | async def _run_decode(): |
| 341 | try: |
| 342 | async for chunk in _stream_decode( |
| 343 | decode_client, api, decode_data, request_id, dp_rank=decode_hdr |
| 344 | ): |
| 345 | await queue.put(("data", chunk)) |
| 346 | except Exception as exc: |
| 347 | await queue.put(("error", exc)) |
| 348 | finally: |
| 349 | await queue.put(("done", None)) |
| 350 | |
| 351 | # 1. Start decode request — decoder is now waiting for KV blocks |
| 352 | asyncio.create_task(_run_decode()) |
| 353 | |
| 354 | # 2. Send prefill — blocks are computed and pushed to the decoder |
| 355 | try: |
| 356 | await _prefill( |
| 357 | prefill_client, api, req_data, request_id, dp_rank=prefill_hdr |
| 358 | ) |
| 359 | except Exception as exc: |
| 360 | logger.warning("decoder-first: prefill failed: %s", exc) |
| 361 | |
| 362 | logger.debug( |
| 363 | "decoder-first: prefill done, streaming decode prefill=%s decode=%s", |
| 364 | prefill_client, |
| 365 | decode_client, |
| 366 | ) |
| 367 | |
| 368 | # 3. Stream the decode response |
| 369 | while True: |
| 370 | kind, value = await queue.get() |
| 371 | if kind == "done": |
| 372 | break |
| 373 | if kind == "error": |
| 374 | raise value # type: ignore[misc] |
| 375 | yield value |
| 376 | |
| 377 | return StreamingResponse(generate(), media_type="application/json") |
| 378 | |
| 379 | except Exception as e: |
| 380 | import sys |
| 381 | import traceback |
| 382 | |
| 383 | print(f"Proxy error on {api}: {e}") |
| 384 | print("".join(traceback.format_exception(*sys.exc_info()))) |
| 385 | raise |
| 386 | |
| 387 | |
| 388 | def _route_handler(api: str): |
| 389 | if global_args.decoder_first: |
| 390 | return lambda req: _handle_completions_decoder_first(api, req) |
| 391 | return lambda req: _handle_completions(api, req) |
| 392 | |
| 393 | |
| 394 | @app.post("/v1/completions") |
| 395 | async def completions(request: Request): |
| 396 | return await _route_handler("/completions")(request) |
| 397 | |
| 398 | |
| 399 | @app.post("/v1/chat/completions") |
| 400 | async def chat_completions(request: Request): |
| 401 | return await _route_handler("/chat/completions")(request) |
| 402 | |
| 403 | |
| 404 | @app.get("/healthcheck") |
| 405 | async def healthcheck(): |
| 406 | return { |
| 407 | "status": "ok", |
| 408 | "prefill_instances": len(app.state.prefill_clients), |
| 409 | "decode_instances": len(app.state.decode_clients), |
| 410 | } |
| 411 | |
| 412 | |
| 413 | if __name__ == "__main__": |
| 414 | global global_args |
| 415 | global_args = parse_args() |
| 416 | import uvicorn |
| 417 | |
| 418 | uvicorn.run(app, host=global_args.host, port=global_args.port) |