A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Bind api server port before starting engine (#8491)
Kevin Lin committed
Sep 16, 2024 at 15:56 UTC
47f5e03b5b9fc719b7e5ee00cbd6d1e79627f105
1 file changed
+6
vllm/entrypoints/openai/api_server.py
+6
@@ -5,6 +5,7 @@ import multiprocessing
5
import os
6
import re
7
import signal
8
+import socket
9
import tempfile
10
from argparse import Namespace
11
from contextlib import asynccontextmanager
@@ -525,6 +526,9 @@ async def run_server(args, **uvicorn_kwargs) -> None:
526
logger.info("vLLM API server version %s", VLLM_VERSION)
527
logger.info("args: %s", args)
528
529
+ temp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
530
+ temp_socket.bind(("", args.port))
531
+
532
def signal_handler(*_) -> None:
533
# Interrupt server on sigterm while initializing
534
raise KeyboardInterrupt("terminated")
@@ -541,6 +545,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
545
model_config = await async_engine_client.get_model_config()
546
init_app_state(async_engine_client, model_config, app.state, args)
547
548
+ temp_socket.close()
549
+
550
shutdown_task = await serve_http(
551
app,
552
limit_concurrency=async_engine_client.limit_concurrency,