@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9705b90bcf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] fix race condition that leads to wrong order of token returned (#10802)

Signed-off-by: Jannis Schönleber <joennlae@gmail.com>

Jannis Schönleber committed Jan 21, 2025 at 18:47 UTC 9705b90bcf66ba6316e70bef442074df7ee6cebf
1 file changed +11 -3
vllm/engine/multiprocessing/client.py
+11 -3
@@ -262,7 +262,14 @@ class MQLLMEngineClient(EngineClient):
262 """Setup the client before it starts sending server requests."""
263
264 # Start output_loop
265 - self.output_loop = asyncio.create_task(self.run_output_handler_loop())
265 + if self.output_loop is None:
266 + # only generate once to avoid multiple concurrent output_loops
267 + # this will lead to race conditions and wrong orders of tokens
268 + # returned by the engine
269 + # setup will be called multiple times during the startup of
270 + # the engine
271 + self.output_loop = asyncio.create_task(
272 + self.run_output_handler_loop())
273
274 with self.get_data_socket() as socket:
275 # Wait until server is ready.
@@ -271,8 +278,9 @@ class MQLLMEngineClient(EngineClient):
278 self.tracing_flag = response.tracing_enabled
279
280 # Start health_loop.
274 - self.health_loop = asyncio.create_task(
275 - self.run_heartbeat_loop(timeout=VLLM_RPC_TIMEOUT))
281 + if self.health_loop is None:
282 + self.health_loop = asyncio.create_task(
283 + self.run_heartbeat_loop(timeout=VLLM_RPC_TIMEOUT))
284
285 def close(self):
286 """Destroy the ZeroMQ Context."""