A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Use uvloop with zmq-decoupled front-end (#7570)
Nick Hill committed
Aug 15, 2024 at 22:48 UTC
9587b050fba00c3c35da05d3512bf7e351914a50
1 file changed
+2
-1
vllm/entrypoints/openai/rpc/server.py
+2
-1
@@ -3,6 +3,7 @@ import signal
3
from typing import Any, Coroutine
4
5
import cloudpickle
6
+import uvloop
7
import zmq
8
import zmq.asyncio
9
from typing_extensions import Never
@@ -217,4 +218,4 @@ async def run_server(server: AsyncEngineRPCServer):
218
def run_rpc_server(async_engine_args: AsyncEngineArgs,
219
usage_context: UsageContext, rpc_path: str):
220
server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
220
- asyncio.run(run_server(server))
221
+ uvloop.run(run_server(server))