A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Frontend] Fix tcp port reservation for api server (#10012)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Nov 5, 2024 at 10:50 UTC
5952d811398d3a22f30d72d2d2943787a78f66ea
1 file changed
+4
-2
vllm/entrypoints/openai/api_server.py
+4
-2
@@ -569,7 +569,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
569
# This avoids race conditions with ray.
570
# see https://github.com/vllm-project/vllm/issues/8204
571
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
572
- sock.bind(("", args.port))
572
+ sock.bind((args.host or "", args.port))
573
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
574
575
def signal_handler(*_) -> None:
576
# Interrupt server on sigterm while initializing
@@ -593,13 +594,14 @@ async def run_server(args, **uvicorn_kwargs) -> None:
594
ssl_certfile=args.ssl_certfile,
595
ssl_ca_certs=args.ssl_ca_certs,
596
ssl_cert_reqs=args.ssl_cert_reqs,
596
- fd=sock.fileno(),
597
**uvicorn_kwargs,
598
)
599
600
# NB: Await server shutdown only after the backend context is exited
601
await shutdown_task
602
603
+ sock.close()
604
+
605
606
if __name__ == "__main__":
607
# NOTE(simon):