A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] Fix ray forward_dag error mssg (#6792)
SangBin Cho committed
Jul 25, 2024 at 16:53 UTC
1adddb14bf0e1a603581bca49e8d29e8bfb337dc
1 file changed
+1
-2
vllm/executor/ray_gpu_executor.py
+1
-2
@@ -29,6 +29,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
29
uses_ray: bool = True
30
31
def _init_executor(self) -> None:
32
+ self.forward_dag: Optional["ray.dag.CompiledDAG"] = None
33
# If the env var is set, it uses the Ray's compiled DAG API
34
# which optimizes the control plane overhead.
35
# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
@@ -60,8 +61,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
61
# Create the parallel GPU workers.
62
self._init_workers_ray(placement_group)
63
63
- self.forward_dag: Optional["ray.dag.CompiledDAG"] = None
64
-
64
def _configure_ray_workers_use_nsight(self,
65
ray_remote_kwargs) -> Dict[str, Any]:
66
# If nsight profiling is enabled, we need to set the profiling