@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1adddb14bf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Fix ray forward_dag error mssg (#6792)

SangBin Cho committed Jul 25, 2024 at 16:53 UTC 1adddb14bf0e1a603581bca49e8d29e8bfb337dc
1 file changed +1 -2
vllm/executor/ray_gpu_executor.py
+1 -2
@@ -29,6 +29,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
29 uses_ray: bool = True
30
31 def _init_executor(self) -> None:
32 + self.forward_dag: Optional["ray.dag.CompiledDAG"] = None
33 # If the env var is set, it uses the Ray's compiled DAG API
34 # which optimizes the control plane overhead.
35 # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
@@ -60,8 +61,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
61 # Create the parallel GPU workers.
62 self._init_workers_ray(placement_group)
63
63 - self.forward_dag: Optional["ray.dag.CompiledDAG"] = None
64 -
64 def _configure_ray_workers_use_nsight(self,
65 ray_remote_kwargs) -> Dict[str, Any]:
66 # If nsight profiling is enabled, we need to set the profiling