@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7be15d9356
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[core][misc] remove use_dummy driver for _run_workers (#10920)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 7, 2024 at 12:06 UTC 7be15d9356a10c6ae3537565548e4f8bf46f35dd
4 files changed +43 -44
vllm/executor/ray_gpu_executor.py
+12 -15
@@ -188,8 +188,14 @@ class RayGPUExecutor(DistributedGPUExecutor):
188 self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
189
190 # Get the set of GPU IDs used on each node.
191 - worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
192 - use_dummy_driver=True)
191 + worker_node_and_gpu_ids = []
192 + for worker in [self.driver_dummy_worker] + self.workers:
193 + if worker is None:
194 + # driver_dummy_worker can be None when using ray spmd worker.
195 + continue
196 + worker_node_and_gpu_ids.append(
197 + ray.get(worker.get_node_and_gpu_ids.remote()) \
198 + ) # type: ignore
199
200 node_workers = defaultdict(list) # node id -> list of worker ranks
201 node_gpus = defaultdict(list) # node id -> list of gpu ids
@@ -329,7 +335,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
335 async_run_tensor_parallel_workers_only: bool = False,
336 all_args: Optional[List[Tuple[Any, ...]]] = None,
337 all_kwargs: Optional[List[Dict[str, Any]]] = None,
332 - use_dummy_driver: bool = False,
338 max_concurrent_workers: Optional[int] = None,
339 **kwargs,
340 ) -> Any:
@@ -389,18 +394,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
394 driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
395
396 # Start the driver worker after all the ray workers.
392 - if not use_dummy_driver:
393 - driver_worker_output = [
394 - self.driver_worker.execute_method(method, *driver_args,
395 - **driver_kwargs)
396 - ]
397 - else:
398 - assert self.driver_dummy_worker is not None
399 - driver_worker_output = [
400 - ray.get(
401 - self.driver_dummy_worker.execute_method.remote(
402 - method, *driver_args, **driver_kwargs))
403 - ]
397 + driver_worker_output = [
398 + self.driver_worker.execute_method(method, *driver_args,
399 + **driver_kwargs)
400 + ]
401
402 # Get the results of the ray workers.
403 if self.workers:
vllm/executor/ray_hpu_executor.py
+12 -16
@@ -163,9 +163,14 @@ class RayHPUExecutor(DistributedGPUExecutor):
163 # node will be placed first.
164 self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
165
166 - # Get the set of GPU IDs used on each node.
167 - worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
168 - use_dummy_driver=True)
166 + worker_node_and_gpu_ids = []
167 + for worker in [self.driver_dummy_worker] + self.workers:
168 + if worker is None:
169 + # driver_dummy_worker can be None when using ray spmd worker.
170 + continue
171 + worker_node_and_gpu_ids.append(
172 + ray.get(worker.get_node_and_gpu_ids.remote()) \
173 + ) # type: ignore
174
175 node_workers = defaultdict(list) # node id -> list of worker ranks
176 node_gpus = defaultdict(list) # node id -> list of gpu ids
@@ -296,7 +301,6 @@ class RayHPUExecutor(DistributedGPUExecutor):
301 async_run_tensor_parallel_workers_only: bool = False,
302 all_args: Optional[List[Tuple[Any, ...]]] = None,
303 all_kwargs: Optional[List[Dict[str, Any]]] = None,
299 - use_dummy_driver: bool = False,
304 max_concurrent_workers: Optional[int] = None,
305 **kwargs,
306 ) -> Any:
@@ -356,18 +360,10 @@ class RayHPUExecutor(DistributedGPUExecutor):
360 driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
361
362 # Start the driver worker after all the ray workers.
359 - if not use_dummy_driver:
360 - driver_worker_output = [
361 - self.driver_worker.execute_method(method, *driver_args,
362 - **driver_kwargs)
363 - ]
364 - else:
365 - assert self.driver_dummy_worker is not None
366 - driver_worker_output = [
367 - ray.get(
368 - self.driver_dummy_worker.execute_method.remote(
369 - method, *driver_args, **driver_kwargs))
370 - ]
363 + driver_worker_output = [
364 + self.driver_worker.execute_method(method, *driver_args,
365 + **driver_kwargs)
366 + ]
367
368 # Get the results of the ray workers.
369 if self.workers:
vllm/executor/ray_tpu_executor.py
+10 -11
@@ -137,8 +137,14 @@ class RayTPUExecutor(TPUExecutor):
137 self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
138
139 # Get the set of TPU IDs used on each node.
140 - worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
141 - use_dummy_driver=True)
140 + worker_node_and_gpu_ids = []
141 + for worker in [self.driver_dummy_worker] + self.workers:
142 + if worker is None:
143 + # driver_dummy_worker can be None when using ray spmd worker.
144 + continue
145 + worker_node_and_gpu_ids.append(
146 + ray.get(worker.get_node_and_gpu_ids.remote()) \
147 + ) # type: ignore
148
149 node_workers = defaultdict(list)
150 for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
@@ -199,7 +205,6 @@ class RayTPUExecutor(TPUExecutor):
205 async_run_remote_workers_only: bool = False,
206 all_args: Optional[List[Tuple[Any, ...]]] = None,
207 all_kwargs: Optional[List[Dict[str, Any]]] = None,
202 - use_dummy_driver: bool = False,
208 max_concurrent_workers: Optional[int] = None,
209 use_ray_compiled_dag: bool = False,
210 **kwargs,
@@ -241,14 +246,8 @@ class RayTPUExecutor(TPUExecutor):
246 driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
247
248 # Start the driver worker after all the ray workers.
244 - if not use_dummy_driver:
245 - driver_worker_output = self.driver_worker.execute_method(
246 - method, *driver_args, **driver_kwargs)
247 - else:
248 - assert self.driver_dummy_worker is not None
249 - driver_worker_output = ray.get(
250 - self.driver_dummy_worker.execute_method.remote(
251 - method, *driver_args, **driver_kwargs))
249 + driver_worker_output = self.driver_worker.execute_method(
250 + method, *driver_args, **driver_kwargs)
251 # Get the results of the ray workers.
252 if self.workers:
253 ray_worker_outputs = ray.get(ray_worker_outputs)
vllm/executor/ray_xpu_executor.py
+9 -2
@@ -1,6 +1,8 @@
1 import asyncio
2 from typing import List, Optional
3
4 +import ray
5 +
6 import vllm.envs as envs
7 from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync
8 from vllm.executor.xpu_executor import XPUExecutor
@@ -14,8 +16,13 @@ class RayXPUExecutor(RayGPUExecutor, XPUExecutor):
16
17 def _get_env_vars_to_be_updated(self):
18 # Get the set of GPU IDs used on each node.
17 - worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
18 - use_dummy_driver=True)
19 + worker_node_and_gpu_ids = []
20 + for worker in [self.driver_dummy_worker] + self.workers:
21 + if worker is None:
22 + # driver_dummy_worker can be None when using ray spmd worker.
23 + continue
24 + worker_node_and_gpu_ids.append(
25 + ray.get(worker.get_node_and_gpu_ids.remote())) # type: ignore
26
27 # Set environment variables for the driver and workers.
28 all_args_to_update_environment_variables = [({