A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] fix race condition that leads to wrong order of token returned (#10802)
Signed-off-by: Jannis Schönleber <joennlae@gmail.com>
Jannis Schönleber committed
Jan 21, 2025 at 18:47 UTC
9705b90bcf66ba6316e70bef442074df7ee6cebf
1 file changed
+11
-3
vllm/engine/multiprocessing/client.py
+11
-3
@@ -262,7 +262,14 @@ class MQLLMEngineClient(EngineClient):
262
"""Setup the client before it starts sending server requests."""
263
264
# Start output_loop
265
- self.output_loop = asyncio.create_task(self.run_output_handler_loop())
265
+ if self.output_loop is None:
266
+ # only generate once to avoid multiple concurrent output_loops
267
+ # this will lead to race conditions and wrong orders of tokens
268
+ # returned by the engine
269
+ # setup will be called multiple times during the startup of
270
+ # the engine
271
+ self.output_loop = asyncio.create_task(
272
+ self.run_output_handler_loop())
273
274
with self.get_data_socket() as socket:
275
# Wait until server is ready.
@@ -271,8 +278,9 @@ class MQLLMEngineClient(EngineClient):
278
self.tracing_flag = response.tracing_enabled
279
280
# Start health_loop.
274
- self.health_loop = asyncio.create_task(
275
- self.run_heartbeat_loop(timeout=VLLM_RPC_TIMEOUT))
281
+ if self.health_loop is None:
282
+ self.health_loop = asyncio.create_task(
283
+ self.run_heartbeat_loop(timeout=VLLM_RPC_TIMEOUT))
284
285
def close(self):
286
"""Destroy the ZeroMQ Context."""