A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[BugFix] Lazy import ray (#10021)
Gene Der Su committed
Nov 5, 2024 at 02:04 UTC
7a83b1aec06834e58174694042105e365828507a
1 file changed
+11
-6
vllm/engine/multiprocessing/engine.py
+11
-6
@@ -5,7 +5,6 @@ from typing import Iterator, List, Optional, Union
5
6
import cloudpickle
7
import zmq
8
-from ray.exceptions import RayTaskError
8
9
from vllm import AsyncEngineArgs, SamplingParams
10
# yapf conflicts with isort for this block
@@ -306,11 +305,17 @@ class MQLLMEngine:
305
def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
306
"""Send List of RequestOutput to RPCClient."""
307
if outputs:
309
- # RayTaskError might not pickelable here. We need to unpack the
310
- # underlying exception as the real exception in the output.
311
- if (isinstance(outputs, RPCError)
312
- and isinstance(outputs.exception, RayTaskError)):
313
- outputs.exception = outputs.exception.cause
308
+ try:
309
+ from ray.exceptions import RayTaskError
310
+
311
+ # RayTaskError might not pickelable here. We need to unpack the
312
+ # underlying exception as the real exception in the output.
313
+ if (isinstance(outputs, RPCError)
314
+ and isinstance(outputs.exception, RayTaskError)):
315
+ outputs.exception = outputs.exception.cause
316
+ except ImportError:
317
+ pass
318
+
319
output_bytes = pickle.dumps(outputs)
320
self.output_socket.send_multipart((output_bytes, ), copy=False)
321