@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 47f5e03b5b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Bind api server port before starting engine (#8491)

Kevin Lin committed Sep 16, 2024 at 15:56 UTC 47f5e03b5b9fc719b7e5ee00cbd6d1e79627f105
1 file changed +6
vllm/entrypoints/openai/api_server.py
+6
@@ -5,6 +5,7 @@ import multiprocessing
5 import os
6 import re
7 import signal
8 +import socket
9 import tempfile
10 from argparse import Namespace
11 from contextlib import asynccontextmanager
@@ -525,6 +526,9 @@ async def run_server(args, **uvicorn_kwargs) -> None:
526 logger.info("vLLM API server version %s", VLLM_VERSION)
527 logger.info("args: %s", args)
528
529 + temp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
530 + temp_socket.bind(("", args.port))
531 +
532 def signal_handler(*_) -> None:
533 # Interrupt server on sigterm while initializing
534 raise KeyboardInterrupt("terminated")
@@ -541,6 +545,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
545 model_config = await async_engine_client.get_model_config()
546 init_app_state(async_engine_client, model_config, app.state, args)
547
548 + temp_socket.close()
549 +
550 shutdown_task = await serve_http(
551 app,
552 limit_concurrency=async_engine_client.limit_concurrency,