A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] PicklingError on RayTaskError (#9934)
Signed-off-by: Gene Su <e870252314@gmail.com>
Gene Der Su committed
Nov 1, 2024 at 15:08 UTC
27cd36e6e2e808464c8343066b03db5db2d15413
1 file changed
+6
vllm/engine/multiprocessing/engine.py
+6
@@ -5,6 +5,7 @@ from typing import Iterator, List, Optional, Union
5
6
import cloudpickle
7
import zmq
8
+from ray.exceptions import RayTaskError
9
10
from vllm import AsyncEngineArgs, SamplingParams
11
# yapf conflicts with isort for this block
@@ -305,6 +306,11 @@ class MQLLMEngine:
306
def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
307
"""Send List of RequestOutput to RPCClient."""
308
if outputs:
309
+ # RayTaskError might not pickelable here. We need to unpack the
310
+ # underlying exception as the real exception in the output.
311
+ if (isinstance(outputs, RPCError)
312
+ and isinstance(outputs.exception, RayTaskError)):
313
+ outputs.exception = outputs.exception.cause
314
output_bytes = pickle.dumps(outputs)
315
self.output_socket.send_multipart((output_bytes, ), copy=False)
316