A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[core][misc] remove use_dummy driver for _run_workers (#10920)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 7, 2024 at 12:06 UTC
7be15d9356a10c6ae3537565548e4f8bf46f35dd
4 files changed
+43
-44
vllm/executor/ray_gpu_executor.py
+12
-15
@@ -188,8 +188,14 @@ class RayGPUExecutor(DistributedGPUExecutor):
188
self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
189
190
# Get the set of GPU IDs used on each node.
191
- worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
192
- use_dummy_driver=True)
191
+ worker_node_and_gpu_ids = []
192
+ for worker in [self.driver_dummy_worker] + self.workers:
193
+ if worker is None:
194
+ # driver_dummy_worker can be None when using ray spmd worker.
195
+ continue
196
+ worker_node_and_gpu_ids.append(
197
+ ray.get(worker.get_node_and_gpu_ids.remote()) \
198
+ ) # type: ignore
199
200
node_workers = defaultdict(list) # node id -> list of worker ranks
201
node_gpus = defaultdict(list) # node id -> list of gpu ids
@@ -329,7 +335,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
335
async_run_tensor_parallel_workers_only: bool = False,
336
all_args: Optional[List[Tuple[Any, ...]]] = None,
337
all_kwargs: Optional[List[Dict[str, Any]]] = None,
332
- use_dummy_driver: bool = False,
338
max_concurrent_workers: Optional[int] = None,
339
**kwargs,
340
) -> Any:
@@ -389,18 +394,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
394
driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
395
396
# Start the driver worker after all the ray workers.
392
- if not use_dummy_driver:
393
- driver_worker_output = [
394
- self.driver_worker.execute_method(method, *driver_args,
395
- **driver_kwargs)
396
- ]
397
- else:
398
- assert self.driver_dummy_worker is not None
399
- driver_worker_output = [
400
- ray.get(
401
- self.driver_dummy_worker.execute_method.remote(
402
- method, *driver_args, **driver_kwargs))
403
- ]
397
+ driver_worker_output = [
398
+ self.driver_worker.execute_method(method, *driver_args,
399
+ **driver_kwargs)
400
+ ]
401
402
# Get the results of the ray workers.
403
if self.workers:
vllm/executor/ray_hpu_executor.py
+12
-16
@@ -163,9 +163,14 @@ class RayHPUExecutor(DistributedGPUExecutor):
163
# node will be placed first.
164
self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
165
166
- # Get the set of GPU IDs used on each node.
167
- worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
168
- use_dummy_driver=True)
166
+ worker_node_and_gpu_ids = []
167
+ for worker in [self.driver_dummy_worker] + self.workers:
168
+ if worker is None:
169
+ # driver_dummy_worker can be None when using ray spmd worker.
170
+ continue
171
+ worker_node_and_gpu_ids.append(
172
+ ray.get(worker.get_node_and_gpu_ids.remote()) \
173
+ ) # type: ignore
174
175
node_workers = defaultdict(list) # node id -> list of worker ranks
176
node_gpus = defaultdict(list) # node id -> list of gpu ids
@@ -296,7 +301,6 @@ class RayHPUExecutor(DistributedGPUExecutor):
301
async_run_tensor_parallel_workers_only: bool = False,
302
all_args: Optional[List[Tuple[Any, ...]]] = None,
303
all_kwargs: Optional[List[Dict[str, Any]]] = None,
299
- use_dummy_driver: bool = False,
304
max_concurrent_workers: Optional[int] = None,
305
**kwargs,
306
) -> Any:
@@ -356,18 +360,10 @@ class RayHPUExecutor(DistributedGPUExecutor):
360
driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
361
362
# Start the driver worker after all the ray workers.
359
- if not use_dummy_driver:
360
- driver_worker_output = [
361
- self.driver_worker.execute_method(method, *driver_args,
362
- **driver_kwargs)
363
- ]
364
- else:
365
- assert self.driver_dummy_worker is not None
366
- driver_worker_output = [
367
- ray.get(
368
- self.driver_dummy_worker.execute_method.remote(
369
- method, *driver_args, **driver_kwargs))
370
- ]
363
+ driver_worker_output = [
364
+ self.driver_worker.execute_method(method, *driver_args,
365
+ **driver_kwargs)
366
+ ]
367
368
# Get the results of the ray workers.
369
if self.workers:
vllm/executor/ray_tpu_executor.py
+10
-11
@@ -137,8 +137,14 @@ class RayTPUExecutor(TPUExecutor):
137
self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
138
139
# Get the set of TPU IDs used on each node.
140
- worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
141
- use_dummy_driver=True)
140
+ worker_node_and_gpu_ids = []
141
+ for worker in [self.driver_dummy_worker] + self.workers:
142
+ if worker is None:
143
+ # driver_dummy_worker can be None when using ray spmd worker.
144
+ continue
145
+ worker_node_and_gpu_ids.append(
146
+ ray.get(worker.get_node_and_gpu_ids.remote()) \
147
+ ) # type: ignore
148
149
node_workers = defaultdict(list)
150
for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
@@ -199,7 +205,6 @@ class RayTPUExecutor(TPUExecutor):
205
async_run_remote_workers_only: bool = False,
206
all_args: Optional[List[Tuple[Any, ...]]] = None,
207
all_kwargs: Optional[List[Dict[str, Any]]] = None,
202
- use_dummy_driver: bool = False,
208
max_concurrent_workers: Optional[int] = None,
209
use_ray_compiled_dag: bool = False,
210
**kwargs,
@@ -241,14 +246,8 @@ class RayTPUExecutor(TPUExecutor):
246
driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
247
248
# Start the driver worker after all the ray workers.
244
- if not use_dummy_driver:
245
- driver_worker_output = self.driver_worker.execute_method(
246
- method, *driver_args, **driver_kwargs)
247
- else:
248
- assert self.driver_dummy_worker is not None
249
- driver_worker_output = ray.get(
250
- self.driver_dummy_worker.execute_method.remote(
251
- method, *driver_args, **driver_kwargs))
249
+ driver_worker_output = self.driver_worker.execute_method(
250
+ method, *driver_args, **driver_kwargs)
251
# Get the results of the ray workers.
252
if self.workers:
253
ray_worker_outputs = ray.get(ray_worker_outputs)
vllm/executor/ray_xpu_executor.py
+9
-2
@@ -1,6 +1,8 @@
1
import asyncio
2
from typing import List, Optional
3
4
+import ray
5
+
6
import vllm.envs as envs
7
from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync
8
from vllm.executor.xpu_executor import XPUExecutor
@@ -14,8 +16,13 @@ class RayXPUExecutor(RayGPUExecutor, XPUExecutor):
16
17
def _get_env_vars_to_be_updated(self):
18
# Get the set of GPU IDs used on each node.
17
- worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
18
- use_dummy_driver=True)
19
+ worker_node_and_gpu_ids = []
20
+ for worker in [self.driver_dummy_worker] + self.workers:
21
+ if worker is None:
22
+ # driver_dummy_worker can be None when using ray spmd worker.
23
+ continue
24
+ worker_node_and_gpu_ids.append(
25
+ ray.get(worker.get_node_and_gpu_ids.remote())) # type: ignore
26
27
# Set environment variables for the driver and workers.
28
all_args_to_update_environment_variables = [({