@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5952d81139
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Frontend] Fix tcp port reservation for api server (#10012)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Nov 5, 2024 at 10:50 UTC 5952d811398d3a22f30d72d2d2943787a78f66ea
1 file changed +4 -2
vllm/entrypoints/openai/api_server.py
+4 -2
@@ -569,7 +569,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
569 # This avoids race conditions with ray.
570 # see https://github.com/vllm-project/vllm/issues/8204
571 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
572 - sock.bind(("", args.port))
572 + sock.bind((args.host or "", args.port))
573 + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
574
575 def signal_handler(*_) -> None:
576 # Interrupt server on sigterm while initializing
@@ -593,13 +594,14 @@ async def run_server(args, **uvicorn_kwargs) -> None:
594 ssl_certfile=args.ssl_certfile,
595 ssl_ca_certs=args.ssl_ca_certs,
596 ssl_cert_reqs=args.ssl_cert_reqs,
596 - fd=sock.fileno(),
597 **uvicorn_kwargs,
598 )
599
600 # NB: Await server shutdown only after the backend context is exited
601 await shutdown_task
602
603 + sock.close()
604 +
605
606 if __name__ == "__main__":
607 # NOTE(simon):