A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] RayWorkerVllm --> WorkerWrapper to reduce duplication (#4024)
[Core] replace narrow-usage RayWorkerVllm to general WorkerWrapper to reduce code duplication (#4024)
youkaichao committed
Apr 17, 2024 at 01:34 UTC
8438e0569eaf8496aa3d41deb808f2c831b64ecf
8 files changed
+176
-116
tests/distributed/test_pynccl.py
+3
-4
@@ -1,18 +1,18 @@
1
import multiprocessing
2
-import os
2
3
import pytest
4
import torch
5
6
from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
7
ncclGetUniqueId)
8
+from vllm.utils import update_environment_variables
9
10
11
def distributed_run(fn, world_size):
12
number_of_processes = world_size
13
processes = []
14
for i in range(number_of_processes):
15
- env = os.environ.copy()
15
+ env = {}
16
env['RANK'] = str(i)
17
env['LOCAL_RANK'] = str(i)
18
env['WORLD_SIZE'] = str(number_of_processes)
@@ -32,8 +32,7 @@ def update_env(fn):
32
# so we need to pass the environment variables as arguments
33
# and update the environment variables in the function
34
def wrapper(env):
35
- import os
36
- os.environ.update(env)
35
+ update_environment_variables(env)
36
fn()
37
38
return wrapper
vllm/engine/ray_utils.py
+7
-37
@@ -1,55 +1,28 @@
1
import pickle
2
-from typing import Callable, List, Optional, Tuple
2
+from typing import List, Optional, Tuple
3
4
from vllm.config import ParallelConfig
5
from vllm.logger import init_logger
6
-from vllm.utils import get_ip, is_hip, set_cuda_visible_devices
7
-from vllm.worker.worker import Worker
6
+from vllm.utils import get_ip, is_hip
7
+from vllm.worker.worker_base import WorkerWrapperBase
8
9
logger = init_logger(__name__)
10
11
try:
12
import ray
13
14
- class RayWorkerVllm:
14
+ class RayWorkerWrapper(WorkerWrapperBase):
15
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
16
lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
17
18
- def __init__(self, init_cached_hf_modules=False) -> None:
19
- if init_cached_hf_modules:
20
- from transformers.dynamic_module_utils import init_hf_modules
21
- init_hf_modules()
22
- self._worker: Optional[Worker] = None
18
+ def __init__(self, *args, **kwargs) -> None:
19
+ super().__init__(*args, **kwargs)
20
# Since the compiled DAG runs a main execution
21
# in a different thread that calls cuda.set_device.
22
# The flag indicates is set_device is called on
23
# that thread.
24
self.compiled_dag_cuda_device_set = False
25
29
- def init_worker(self, worker_init_fn: Callable[[], Worker]):
30
- self._worker = worker_init_fn()
31
-
32
- @property
33
- def worker(self) -> Worker:
34
- assert self._worker is not None
35
- return self._worker
36
-
37
- def __getattr__(self, name):
38
- return getattr(self.worker, name)
39
-
40
- def execute_method(self, method, *args, **kwargs):
41
- try:
42
- executor = getattr(self, method)
43
- return executor(*args, **kwargs)
44
- except Exception as e:
45
- # exceptions in ray worker may cause deadlock
46
- # see https://github.com/vllm-project/vllm/issues/3455
47
- # print the error and inform the user to solve the error
48
- msg = (f"Error executing method {method}. "
49
- "This might cause deadlock in distributed execution.")
50
- logger.exception(msg)
51
- raise e
52
-
26
def get_node_ip(self) -> str:
27
return get_ip()
28
@@ -58,9 +31,6 @@ try:
31
gpu_ids = ray.get_gpu_ids()
32
return node_id, gpu_ids
33
61
- def set_cuda_visible_devices(self, device_ids) -> None:
62
- set_cuda_visible_devices(device_ids)
63
-
34
def execute_model_compiled_dag_remote(self, ignored):
35
"""Used only when compiled DAG is enabled."""
36
import torch
@@ -77,7 +47,7 @@ except ImportError as e:
47
"For distributed inference, please install Ray with "
48
"`pip install ray`.")
49
ray = None # type: ignore
80
- RayWorkerVllm = None # type: ignore
50
+ RayWorkerWrapper = None # type: ignore
51
52
53
def initialize_ray_cluster(
vllm/executor/ray_gpu_executor.py
+84
-72
@@ -1,17 +1,16 @@
1
import asyncio
2
-import copy
2
import os
3
import pickle
4
from collections import defaultdict
5
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
6
8
-from vllm.engine.ray_utils import RayWorkerVllm, ray
7
+from vllm.engine.ray_utils import RayWorkerWrapper, ray
8
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
9
from vllm.logger import init_logger
10
from vllm.lora.request import LoRARequest
11
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
14
- make_async, set_cuda_visible_devices)
13
+ make_async)
14
15
if ray is not None:
16
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -74,9 +73,9 @@ class RayGPUExecutor(ExecutorBase):
73
74
# The driver dummy worker does not actually use any resources.
75
# It holds the resource for the driver worker.
77
- self.driver_dummy_worker: RayWorkerVllm = None
76
+ self.driver_dummy_worker: RayWorkerWrapper = None
77
# The remaining workers are the actual ray actors.
79
- self.workers: List[RayWorkerVllm] = []
78
+ self.workers: List[RayWorkerWrapper] = []
79
80
if self.parallel_config.ray_workers_use_nsight:
81
ray_remote_kwargs = self._configure_ray_workers_use_nsight(
@@ -97,13 +96,20 @@ class RayGPUExecutor(ExecutorBase):
96
num_gpus=num_gpus,
97
scheduling_strategy=scheduling_strategy,
98
**ray_remote_kwargs,
100
- )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
99
+ )(RayWorkerWrapper).remote(
100
+ worker_module_name="vllm.worker.worker",
101
+ worker_class_name="Worker",
102
+ )
103
104
worker_ip = ray.get(worker.get_node_ip.remote())
105
if worker_ip == driver_ip and self.driver_dummy_worker is None:
106
# If the worker is on the same node as the driver, we use it
107
# as the resource holder for the driver process.
108
self.driver_dummy_worker = worker
109
+ self.driver_worker = RayWorkerWrapper(
110
+ worker_module_name="vllm.worker.worker",
111
+ worker_class_name="Worker",
112
+ )
113
else:
114
# Else, added to the list of workers.
115
self.workers.append(worker)
@@ -115,82 +121,56 @@ class RayGPUExecutor(ExecutorBase):
121
"GPU node.")
122
123
# Get the set of GPU IDs used on each node.
118
- driver_node_id, driver_gpu_ids = ray.get(
119
- self.driver_dummy_worker.get_node_and_gpu_ids.remote())
120
- worker_node_and_gpu_ids = ray.get(
121
- [worker.get_node_and_gpu_ids.remote() for worker in self.workers])
124
+ worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
125
+ use_dummy_driver=True)
126
127
node_workers = defaultdict(list)
128
node_gpus = defaultdict(list)
129
126
- node_workers[driver_node_id].append(0)
127
- node_gpus[driver_node_id].extend(driver_gpu_ids)
128
- for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids,
129
- start=1):
130
+ for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
131
node_workers[node_id].append(i)
132
node_gpus[node_id].extend(gpu_ids)
133
for node_id, gpu_ids in node_gpus.items():
134
node_gpus[node_id] = sorted(gpu_ids)
135
136
# Set CUDA_VISIBLE_DEVICES for the driver and workers.
136
- set_cuda_visible_devices(node_gpus[driver_node_id])
137
- for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
138
- worker.set_cuda_visible_devices.remote(node_gpus[node_id])
137
+ all_args_to_update_environment_variables = []
138
+ for (node_id, _) in worker_node_and_gpu_ids:
139
+ all_args_to_update_environment_variables.append([{
140
+ "CUDA_VISIBLE_DEVICES":
141
+ ",".join(map(str, node_gpus[node_id]))
142
+ }])
143
+ self._run_workers("update_environment_variables",
144
+ all_args=all_args_to_update_environment_variables)
145
146
distributed_init_method = get_distributed_init_method(
147
driver_ip, get_open_port())
148
143
- # Lazy import the Worker to avoid importing torch.cuda/xformers
144
- # before CUDA_VISIBLE_DEVICES is set in the Worker
145
- from vllm.worker.worker import Worker
146
-
147
- model_config = copy.deepcopy(self.model_config)
148
- parallel_config = copy.deepcopy(self.parallel_config)
149
- scheduler_config = copy.deepcopy(self.scheduler_config)
150
- load_config = copy.deepcopy(self.load_config)
151
- device_config = copy.deepcopy(self.device_config)
152
- lora_config = copy.deepcopy(self.lora_config)
153
- cache_config = copy.deepcopy(self.cache_config)
154
- vision_language_config = copy.deepcopy(self.vision_language_config)
155
-
156
- # Initialize the actual workers with the Worker class.
157
- for rank, (worker, (node_id, _)) in enumerate(
158
- zip(self.workers, worker_node_and_gpu_ids),
159
- start=1,
160
- ):
149
+ def collect_arg_helper_func(**kwargs):
150
+ # avoid writing `{"name": value}` manually
151
+ return kwargs
152
+
153
+ init_worker_all_kwargs = []
154
+
155
+ # Initialize the actual workers inside worker wrapper.
156
+ for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids, ):
157
local_rank = node_workers[node_id].index(rank)
162
- worker.init_worker.remote(
163
- lambda rank=rank, local_rank=local_rank: Worker(
164
- model_config=model_config,
165
- parallel_config=parallel_config,
166
- scheduler_config=scheduler_config,
167
- device_config=device_config,
168
- cache_config=cache_config,
169
- load_config=load_config,
158
+ init_worker_all_kwargs.append(
159
+ collect_arg_helper_func(
160
+ model_config=self.model_config,
161
+ parallel_config=self.parallel_config,
162
+ scheduler_config=self.scheduler_config,
163
+ device_config=self.device_config,
164
+ cache_config=self.cache_config,
165
+ load_config=self.load_config,
166
local_rank=local_rank,
167
rank=rank,
168
distributed_init_method=distributed_init_method,
173
- lora_config=lora_config,
174
- vision_language_config=vision_language_config,
169
+ lora_config=self.lora_config,
170
+ vision_language_config=self.vision_language_config,
171
+ is_driver_worker=rank == 0,
172
))
176
-
177
- # Initialize the driver worker with the Worker class.
178
- driver_rank = 0
179
- driver_local_rank = node_workers[driver_node_id].index(driver_rank)
180
- self.driver_worker = Worker(
181
- model_config=self.model_config,
182
- parallel_config=self.parallel_config,
183
- scheduler_config=self.scheduler_config,
184
- device_config=self.device_config,
185
- cache_config=self.cache_config,
186
- local_rank=driver_local_rank,
187
- rank=driver_rank,
188
- distributed_init_method=distributed_init_method,
189
- lora_config=self.lora_config,
190
- vision_language_config=self.vision_language_config,
191
- load_config=self.load_config,
192
- is_driver_worker=True,
193
- )
173
+ self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
174
175
self._run_workers("init_device")
176
self._run_workers(
@@ -279,13 +259,35 @@ class RayGPUExecutor(ExecutorBase):
259
self,
260
method: str,
261
*args,
282
- driver_args: Optional[Tuple[Any, ...]] = None,
262
+ driver_args: Optional[Tuple[Any]] = None,
263
driver_kwargs: Optional[Dict[str, Any]] = None,
264
+ all_args: Optional[List[List[Any]]] = None,
265
+ all_kwargs: Optional[List[Dict[str, Any]]] = None,
266
+ use_dummy_driver: bool = False,
267
max_concurrent_workers: Optional[int] = None,
268
use_ray_compiled_dag: bool = False,
269
**kwargs,
270
) -> Any:
288
- """Runs the given method on all workers."""
271
+ """Runs the given method on all workers.
272
+ all_args and all_kwargs are used to pass heterogeneous arguments,
273
+ i.e. different arguments for each worker.
274
+ """
275
+ if driver_args is None:
276
+ driver_args = args
277
+ if driver_kwargs is None:
278
+ driver_kwargs = kwargs
279
+
280
+ # for mypy type checking
281
+ assert driver_args is not None
282
+ assert driver_kwargs is not None
283
+ if all_args is None:
284
+ all_args = [driver_args] + [args] * len(self.workers)
285
+ if all_kwargs is None:
286
+ all_kwargs = [driver_kwargs] + [kwargs] * len(self.workers)
287
+
288
+ # for mypy type checking
289
+ assert all_args is not None
290
+ assert all_kwargs is not None
291
292
if max_concurrent_workers:
293
raise NotImplementedError(
@@ -299,8 +301,10 @@ class RayGPUExecutor(ExecutorBase):
301
else:
302
# Start the ray workers first.
303
ray_worker_outputs = [
302
- worker.execute_method.remote(method, *args, **kwargs)
303
- for worker in self.workers
304
+ worker.execute_method.remote(method, *worker_args,
305
+ **worker_kwargs)
306
+ for (worker, worker_args, worker_kwargs
307
+ ) in zip(self.workers, all_args[1:], all_kwargs[1:])
308
]
309
310
if driver_args is None:
@@ -309,9 +313,13 @@ class RayGPUExecutor(ExecutorBase):
313
driver_kwargs = kwargs
314
315
# Start the driver worker after all the ray workers.
312
- driver_worker_output = getattr(self.driver_worker,
313
- method)(*driver_args, **driver_kwargs)
314
-
316
+ if not use_dummy_driver:
317
+ driver_worker_output = self.driver_worker.execute_method(
318
+ method, *all_args[0], **all_kwargs[0])
319
+ else:
320
+ driver_worker_output = ray.get(
321
+ self.driver_dummy_worker.execute_method.remote(
322
+ method, *all_args[0], **all_kwargs[0]))
323
# Get the results of the ray workers.
324
if self.workers:
325
if use_ray_compiled_dag:
@@ -386,8 +394,12 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
394
driver_kwargs = kwargs
395
396
# Run the driver worker asynchronously.
389
- driver_executor = make_async(getattr(self.driver_worker, method))
390
- coros.append(driver_executor(*driver_args, **driver_kwargs))
397
+ def helper():
398
+ return self.driver_worker.execute_method(method, *driver_args,
399
+ **driver_kwargs)
400
+
401
+ driver_executor = make_async(helper)
402
+ coros.append(driver_executor())
403
404
# Run the ray workers asynchronously.
405
for worker in self.workers:
vllm/utils.py
+14
-2
@@ -271,8 +271,12 @@ def get_open_port() -> int:
271
return s.getsockname()[1]
272
273
274
-def set_cuda_visible_devices(device_ids: List[int]) -> None:
275
- os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
274
+def update_environment_variables(envs: Dict[str, str]):
275
+ for k, v in envs.items():
276
+ if k in os.environ:
277
+ logger.warning(f"Overwriting environment variable {k} "
278
+ f"from '{os.environ[k]}' to '{v}'")
279
+ os.environ[k] = v
280
281
282
def chunk_list(lst, chunk_size):
@@ -505,3 +509,11 @@ def merge_dicts(dict1: Dict[Any, List[Any]],
509
merged_dict[key].extend(value)
510
511
return dict(merged_dict)
512
+
513
+
514
+def init_cached_hf_modules():
515
+ """
516
+ Lazy initialization of the Hugging Face modules.
517
+ """
518
+ from transformers.dynamic_module_utils import init_hf_modules
519
+ init_hf_modules()
vllm/worker/cpu_worker.py
+4
-1
@@ -138,7 +138,10 @@ class CPUWorker(LoraNotSupportedWorkerBase):
138
self.is_driver_worker = is_driver_worker
139
if self.is_driver_worker:
140
assert self.rank == 0, "The driver worker must have rank 0."
141
-
141
+ if self.model_config.trust_remote_code:
142
+ # note: lazy import to avoid importing torch before initializing
143
+ from vllm.utils import init_cached_hf_modules
144
+ init_cached_hf_modules()
145
self.model_runner = CPUModelRunner(model_config,
146
parallel_config,
147
scheduler_config,
vllm/worker/neuron_worker.py
+4
@@ -29,6 +29,10 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
29
self.scheduler_config = scheduler_config
30
self.device_config = device_config
31
self.cache_config = cache_config
32
+ if self.model_config.trust_remote_code:
33
+ # note: lazy import to avoid importing torch before initializing
34
+ from vllm.utils import init_cached_hf_modules
35
+ init_cached_hf_modules()
36
37
self.model_runner = NeuronModelRunner(model_config, parallel_config,
38
scheduler_config, device_config)
vllm/worker/worker.py
+4
@@ -60,6 +60,10 @@ class Worker(WorkerBase):
60
if self.is_driver_worker:
61
assert self.rank == 0, "The driver worker must have rank 0."
62
63
+ if self.model_config.trust_remote_code:
64
+ # note: lazy import to avoid importing torch before initializing
65
+ from vllm.utils import init_cached_hf_modules
66
+ init_cached_hf_modules()
67
self.vision_language_config = vision_language_config
68
if self.vision_language_config:
69
assert not self.lora_config, (
vllm/worker/worker_base.py
+56
@@ -1,8 +1,14 @@
1
+import importlib
2
+import os
3
from abc import ABC, abstractmethod
4
from typing import Dict, List, Tuple
5
6
+from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
8
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
+from vllm.utils import update_environment_variables
10
+
11
+logger = init_logger(__name__)
12
13
14
class WorkerBase(ABC):
@@ -82,3 +88,53 @@ class LoraNotSupportedWorkerBase(WorkerBase):
88
89
def list_loras(self) -> List[int]:
90
raise ValueError(f"{type(self)} does not support LoRA")
91
+
92
+
93
+class WorkerWrapperBase:
94
+ """
95
+ The whole point of this class is to lazily initialize the worker.
96
+ We first instantiate the WorkerWrapper, which remembers the worker module
97
+ and class name. Then, when we call `update_environment_variables`, and the
98
+ real initialization happens in `init_worker`.
99
+ """
100
+
101
+ def __init__(self,
102
+ worker_module_name=None,
103
+ worker_class_name=None) -> None:
104
+ self.worker_module_name = worker_module_name
105
+ self.worker_class_name = worker_class_name
106
+ self.worker = None
107
+
108
+ def update_environment_variables(self, envs: Dict[str, str]) -> None:
109
+ key = 'CUDA_VISIBLE_DEVICES'
110
+ if key in envs and key in os.environ:
111
+ # overwriting CUDA_VISIBLE_DEVICES is desired behavior
112
+ # suppress the warning in `update_environment_variables`
113
+ del os.environ[key]
114
+ update_environment_variables(envs)
115
+
116
+ def init_worker(self, *args, **kwargs):
117
+ """
118
+ Actual initialization of the worker class.
119
+ Arguments are passed to the worker class constructor.
120
+ """
121
+ mod = importlib.import_module(self.worker_module_name)
122
+ worker_class = getattr(mod, self.worker_class_name)
123
+ self.worker = worker_class(*args, **kwargs)
124
+
125
+ def execute_method(self, method, *args, **kwargs):
126
+ try:
127
+ if hasattr(self, method):
128
+ executor = getattr(self, method)
129
+ else:
130
+ executor = getattr(self.worker, method)
131
+ return executor(*args, **kwargs)
132
+ except Exception as e:
133
+ # if the driver worker also execute methods,
134
+ # exceptions in the rest worker may cause deadlock in rpc like ray
135
+ # see https://github.com/vllm-project/vllm/issues/3455
136
+ # print the error and inform the user to solve the error
137
+ msg = (f"Error executing method {method}. "
138
+ "This might cause deadlock in distributed execution.")
139
+ logger.exception(msg)
140
+ raise e