A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix V1 engine serialization error with Ray distributed executor (#26148)
Signed-off-by: Nikhil Ghosh <nikhil@anyscale.com>
Nikhil G committed
Oct 3, 2025 at 11:39 UTC
cd9e5b8340b4c9dc093b7bd6f960fc2fdddf2b98
1 file changed
+6
vllm/executor/ray_utils.py
+6
@@ -16,6 +16,7 @@ from vllm.logger import init_logger
16
from vllm.platforms import current_platform
17
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
18
from vllm.utils import get_ip
19
+from vllm.v1.outputs import AsyncModelRunnerOutput
20
from vllm.v1.worker.worker_base import WorkerWrapperBase
21
22
if TYPE_CHECKING:
@@ -142,6 +143,11 @@ try:
143
# but may still be finished requests.
144
assert not output or not output.req_ids
145
output = scheduler_output, None
146
+ # Ensure outputs crossing Ray compiled DAG are serializable.
147
+ # AsyncModelRunnerOutput holds CUDA events and cannot be
148
+ # pickled.
149
+ if isinstance(output, AsyncModelRunnerOutput):
150
+ output = output.get_output()
151
return output
152
153
def override_env_vars(self, vars: Dict[str, str]):