@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a83b1aec0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[BugFix] Lazy import ray (#10021)

Gene Der Su committed Nov 5, 2024 at 02:04 UTC 7a83b1aec06834e58174694042105e365828507a
1 file changed +11 -6
vllm/engine/multiprocessing/engine.py
+11 -6
@@ -5,7 +5,6 @@ from typing import Iterator, List, Optional, Union
5
6 import cloudpickle
7 import zmq
8 -from ray.exceptions import RayTaskError
8
9 from vllm import AsyncEngineArgs, SamplingParams
10 # yapf conflicts with isort for this block
@@ -306,11 +305,17 @@ class MQLLMEngine:
305 def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
306 """Send List of RequestOutput to RPCClient."""
307 if outputs:
309 - # RayTaskError might not pickelable here. We need to unpack the
310 - # underlying exception as the real exception in the output.
311 - if (isinstance(outputs, RPCError)
312 - and isinstance(outputs.exception, RayTaskError)):
313 - outputs.exception = outputs.exception.cause
308 + try:
309 + from ray.exceptions import RayTaskError
310 +
311 + # RayTaskError might not pickelable here. We need to unpack the
312 + # underlying exception as the real exception in the output.
313 + if (isinstance(outputs, RPCError)
314 + and isinstance(outputs.exception, RayTaskError)):
315 + outputs.exception = outputs.exception.cause
316 + except ImportError:
317 + pass
318 +
319 output_bytes = pickle.dumps(outputs)
320 self.output_socket.send_multipart((output_bytes, ), copy=False)
321