@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9587b050fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Use uvloop with zmq-decoupled front-end (#7570)

Nick Hill committed Aug 15, 2024 at 22:48 UTC 9587b050fba00c3c35da05d3512bf7e351914a50
1 file changed +2 -1
vllm/entrypoints/openai/rpc/server.py
+2 -1
@@ -3,6 +3,7 @@ import signal
3 from typing import Any, Coroutine
4
5 import cloudpickle
6 +import uvloop
7 import zmq
8 import zmq.asyncio
9 from typing_extensions import Never
@@ -217,4 +218,4 @@ async def run_server(server: AsyncEngineRPCServer):
218 def run_rpc_server(async_engine_args: AsyncEngineArgs,
219 usage_context: UsageContext, rpc_path: str):
220 server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
220 - asyncio.run(run_server(server))
221 + uvloop.run(run_server(server))