@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 27cd36e6e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] PicklingError on RayTaskError (#9934)

Signed-off-by: Gene Su <e870252314@gmail.com>

Gene Der Su committed Nov 1, 2024 at 15:08 UTC 27cd36e6e2e808464c8343066b03db5db2d15413
1 file changed +6
vllm/engine/multiprocessing/engine.py
+6
@@ -5,6 +5,7 @@ from typing import Iterator, List, Optional, Union
5
6 import cloudpickle
7 import zmq
8 +from ray.exceptions import RayTaskError
9
10 from vllm import AsyncEngineArgs, SamplingParams
11 # yapf conflicts with isort for this block
@@ -305,6 +306,11 @@ class MQLLMEngine:
306 def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
307 """Send List of RequestOutput to RPCClient."""
308 if outputs:
309 + # RayTaskError might not pickelable here. We need to unpack the
310 + # underlying exception as the real exception in the output.
311 + if (isinstance(outputs, RPCError)
312 + and isinstance(outputs.exception, RayTaskError)):
313 + outputs.exception = outputs.exception.cause
314 output_bytes = pickle.dumps(outputs)
315 self.output_socket.send_multipart((output_bytes, ), copy=False)
316