A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Some simplification of WorkerWrapper changes (#4183)
Nick Hill committed
Apr 23, 2024 at 00:49 UTC
8f2ea22bde67161895152e7f7ad602ac96ea326e
2 files changed
+45
-54
vllm/executor/ray_gpu_executor.py
+41
-49
@@ -2,6 +2,7 @@ import asyncio
2
import os
3
import pickle
4
from collections import defaultdict
5
+from itertools import islice, repeat
6
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
7
8
from vllm.engine.ray_utils import RayWorkerWrapper, ray
@@ -136,16 +137,14 @@ class RayGPUExecutor(ExecutorBase):
137
VLLM_INSTANCE_ID = get_vllm_instance_id()
138
139
# Set environment variables for the driver and workers.
139
- all_args_to_update_environment_variables = []
140
- for (node_id, _) in worker_node_and_gpu_ids:
141
- all_args_to_update_environment_variables.append([{
142
- "CUDA_VISIBLE_DEVICES":
143
- ",".join(map(str, node_gpus[node_id])),
144
- "VLLM_INSTANCE_ID":
145
- VLLM_INSTANCE_ID,
146
- "VLLM_TRACE_FUNCTION":
147
- os.getenv("VLLM_TRACE_FUNCTION", "0"),
148
- }])
140
+ all_args_to_update_environment_variables = [({
141
+ "CUDA_VISIBLE_DEVICES":
142
+ ",".join(map(str, node_gpus[node_id])),
143
+ "VLLM_INSTANCE_ID":
144
+ VLLM_INSTANCE_ID,
145
+ "VLLM_TRACE_FUNCTION":
146
+ os.getenv("VLLM_TRACE_FUNCTION", "0"),
147
+ }, ) for (node_id, _) in worker_node_and_gpu_ids]
148
self._run_workers("update_environment_variables",
149
all_args=all_args_to_update_environment_variables)
150
@@ -156,10 +155,9 @@ class RayGPUExecutor(ExecutorBase):
155
# avoid writing `{"name": value}` manually
156
return kwargs
157
159
- init_worker_all_kwargs = []
160
-
158
# Initialize the actual workers inside worker wrapper.
162
- for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids, ):
159
+ init_worker_all_kwargs = []
160
+ for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids):
161
local_rank = node_workers[node_id].index(rank)
162
init_worker_all_kwargs.append(
163
collect_arg_helper_func(
@@ -265,40 +263,40 @@ class RayGPUExecutor(ExecutorBase):
263
self,
264
method: str,
265
*args,
268
- driver_args: Optional[Tuple[Any]] = None,
266
+ driver_args: Optional[Tuple[Any, ...]] = None,
267
driver_kwargs: Optional[Dict[str, Any]] = None,
270
- all_args: Optional[List[List[Any]]] = None,
268
+ all_args: Optional[List[Tuple[Any, ...]]] = None,
269
all_kwargs: Optional[List[Dict[str, Any]]] = None,
270
use_dummy_driver: bool = False,
271
max_concurrent_workers: Optional[int] = None,
272
use_ray_compiled_dag: bool = False,
273
**kwargs,
274
) -> Any:
277
- """Runs the given method on all workers.
278
- all_args and all_kwargs are used to pass heterogeneous arguments,
279
- i.e. different arguments for each worker.
275
+ """Runs the given method on all workers. Can be used in the following
276
+ ways:
277
+
278
+ - args/kwargs: All workers share the same args/kwargs
279
+ - args/kwargs and driver_args/driver_kwargs: Driver worker has
280
+ different args
281
+ - all_args/all_kwargs: args/kwargs for each worker are specified
282
+ individually
283
"""
281
- if driver_args is None:
282
- driver_args = args
283
- if driver_kwargs is None:
284
- driver_kwargs = kwargs
285
-
286
- # for mypy type checking
287
- assert driver_args is not None
288
- assert driver_kwargs is not None
289
- if all_args is None:
290
- all_args = [driver_args] + [args] * len(self.workers)
291
- if all_kwargs is None:
292
- all_kwargs = [driver_kwargs] + [kwargs] * len(self.workers)
293
-
294
- # for mypy type checking
295
- assert all_args is not None
296
- assert all_kwargs is not None
284
285
if max_concurrent_workers:
286
raise NotImplementedError(
287
"max_concurrent_workers is not supported yet.")
288
289
+ if driver_args is None:
290
+ driver_args = args if all_args is None else all_args[0]
291
+ if driver_kwargs is None:
292
+ driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
293
+
294
+ count = len(self.workers)
295
+ all_worker_args = repeat(args, count) if all_args is None \
296
+ else islice(all_args, 1, None)
297
+ all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
298
+ else islice(all_kwargs, 1, None)
299
+
300
if use_ray_compiled_dag:
301
# Right now, compiled DAG can only accept a single
302
# input. TODO(sang): Fix it.
@@ -310,22 +308,17 @@ class RayGPUExecutor(ExecutorBase):
308
worker.execute_method.remote(method, *worker_args,
309
**worker_kwargs)
310
for (worker, worker_args, worker_kwargs
313
- ) in zip(self.workers, all_args[1:], all_kwargs[1:])
311
+ ) in zip(self.workers, all_worker_args, all_worker_kwargs)
312
]
313
316
- if driver_args is None:
317
- driver_args = args
318
- if driver_kwargs is None:
319
- driver_kwargs = kwargs
320
-
314
# Start the driver worker after all the ray workers.
315
if not use_dummy_driver:
316
driver_worker_output = self.driver_worker.execute_method(
324
- method, *all_args[0], **all_kwargs[0])
317
+ method, *driver_args, **driver_kwargs)
318
else:
319
driver_worker_output = ray.get(
320
self.driver_dummy_worker.execute_method.remote(
328
- method, *all_args[0], **all_kwargs[0]))
321
+ method, *driver_args, **driver_kwargs))
322
# Get the results of the ray workers.
323
if self.workers:
324
if use_ray_compiled_dag:
@@ -383,6 +376,10 @@ class RayGPUExecutor(ExecutorBase):
376
377
class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
378
379
+ def __init__(self, *args, **kwargs):
380
+ super().__init__(*args, **kwargs)
381
+ self.driver_executor = make_async(self.driver_worker.execute_method)
382
+
383
async def _run_workers_async(
384
self,
385
method: str,
@@ -399,13 +396,8 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
396
if driver_kwargs is None:
397
driver_kwargs = kwargs
398
402
- # Run the driver worker asynchronously.
403
- def helper():
404
- return self.driver_worker.execute_method(method, *driver_args,
405
- **driver_kwargs)
406
-
407
- driver_executor = make_async(helper)
408
- coros.append(driver_executor())
399
+ coros.append(
400
+ self.driver_executor(method, *driver_args, **driver_kwargs))
401
402
# Run the ray workers asynchronously.
403
for worker in self.workers:
vllm/worker/worker_base.py
+4
-5
@@ -108,7 +108,8 @@ class WorkerWrapperBase:
108
self.worker_class_name = worker_class_name
109
self.worker = None
110
111
- def update_environment_variables(self, envs: Dict[str, str]) -> None:
111
+ @staticmethod
112
+ def update_environment_variables(envs: Dict[str, str]) -> None:
113
key = 'CUDA_VISIBLE_DEVICES'
114
if key in envs and key in os.environ:
115
# overwriting CUDA_VISIBLE_DEVICES is desired behavior
@@ -138,10 +139,8 @@ class WorkerWrapperBase:
139
140
def execute_method(self, method, *args, **kwargs):
141
try:
141
- if hasattr(self, method):
142
- executor = getattr(self, method)
143
- else:
144
- executor = getattr(self.worker, method)
142
+ target = self if self.worker is None else self.worker
143
+ executor = getattr(target, method)
144
return executor(*args, **kwargs)
145
except Exception as e:
146
# if the driver worker also execute methods,