A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[BugFix] Avoid error traceback in logs when V1 `LLM` terminates (#13565)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Feb 19, 2025 at 16:49 UTC
a4c402a756fa3213caf9d2cde0e4ceb2d57727f2
1 file changed
+12
-8
vllm/v1/engine/core_client.py
+12
-8
@@ -252,14 +252,18 @@ class SyncMPClient(MPClient):
252
outputs_queue = self.outputs_queue
253
254
def process_outputs_socket():
255
- while True:
256
- (frame, ) = output_socket.recv_multipart(copy=False)
257
- outputs = decoder.decode(frame.buffer)
258
- if outputs.utility_output:
259
- _process_utility_output(outputs.utility_output,
260
- utility_results)
261
- else:
262
- outputs_queue.put_nowait(outputs)
255
+ try:
256
+ while True:
257
+ (frame, ) = output_socket.recv_multipart(copy=False)
258
+ outputs = decoder.decode(frame.buffer)
259
+ if outputs.utility_output:
260
+ _process_utility_output(outputs.utility_output,
261
+ utility_results)
262
+ else:
263
+ outputs_queue.put_nowait(outputs)
264
+ except zmq.error.ContextTerminated:
265
+ # Expected when the class is GC'd / during process termination.
266
+ pass
267
268
# Process outputs from engine in separate thread.
269
Thread(target=process_outputs_socket, daemon=True).start()