@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8438e0569e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] RayWorkerVllm --> WorkerWrapper to reduce duplication (#4024)

[Core] replace narrow-usage RayWorkerVllm to general WorkerWrapper to reduce code duplication (#4024)

youkaichao committed Apr 17, 2024 at 01:34 UTC 8438e0569eaf8496aa3d41deb808f2c831b64ecf
8 files changed +176 -116
tests/distributed/test_pynccl.py
+3 -4
@@ -1,18 +1,18 @@
1 import multiprocessing
2 -import os
2
3 import pytest
4 import torch
5
6 from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
7 ncclGetUniqueId)
8 +from vllm.utils import update_environment_variables
9
10
11 def distributed_run(fn, world_size):
12 number_of_processes = world_size
13 processes = []
14 for i in range(number_of_processes):
15 - env = os.environ.copy()
15 + env = {}
16 env['RANK'] = str(i)
17 env['LOCAL_RANK'] = str(i)
18 env['WORLD_SIZE'] = str(number_of_processes)
@@ -32,8 +32,7 @@ def update_env(fn):
32 # so we need to pass the environment variables as arguments
33 # and update the environment variables in the function
34 def wrapper(env):
35 - import os
36 - os.environ.update(env)
35 + update_environment_variables(env)
36 fn()
37
38 return wrapper
vllm/engine/ray_utils.py
+7 -37
@@ -1,55 +1,28 @@
1 import pickle
2 -from typing import Callable, List, Optional, Tuple
2 +from typing import List, Optional, Tuple
3
4 from vllm.config import ParallelConfig
5 from vllm.logger import init_logger
6 -from vllm.utils import get_ip, is_hip, set_cuda_visible_devices
7 -from vllm.worker.worker import Worker
6 +from vllm.utils import get_ip, is_hip
7 +from vllm.worker.worker_base import WorkerWrapperBase
8
9 logger = init_logger(__name__)
10
11 try:
12 import ray
13
14 - class RayWorkerVllm:
14 + class RayWorkerWrapper(WorkerWrapperBase):
15 """Ray wrapper for vllm.worker.Worker, allowing Worker to be
16 lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
17
18 - def __init__(self, init_cached_hf_modules=False) -> None:
19 - if init_cached_hf_modules:
20 - from transformers.dynamic_module_utils import init_hf_modules
21 - init_hf_modules()
22 - self._worker: Optional[Worker] = None
18 + def __init__(self, *args, **kwargs) -> None:
19 + super().__init__(*args, **kwargs)
20 # Since the compiled DAG runs a main execution
21 # in a different thread that calls cuda.set_device.
22 # The flag indicates is set_device is called on
23 # that thread.
24 self.compiled_dag_cuda_device_set = False
25
29 - def init_worker(self, worker_init_fn: Callable[[], Worker]):
30 - self._worker = worker_init_fn()
31 -
32 - @property
33 - def worker(self) -> Worker:
34 - assert self._worker is not None
35 - return self._worker
36 -
37 - def __getattr__(self, name):
38 - return getattr(self.worker, name)
39 -
40 - def execute_method(self, method, *args, **kwargs):
41 - try:
42 - executor = getattr(self, method)
43 - return executor(*args, **kwargs)
44 - except Exception as e:
45 - # exceptions in ray worker may cause deadlock
46 - # see https://github.com/vllm-project/vllm/issues/3455
47 - # print the error and inform the user to solve the error
48 - msg = (f"Error executing method {method}. "
49 - "This might cause deadlock in distributed execution.")
50 - logger.exception(msg)
51 - raise e
52 -
26 def get_node_ip(self) -> str:
27 return get_ip()
28
@@ -58,9 +31,6 @@ try:
31 gpu_ids = ray.get_gpu_ids()
32 return node_id, gpu_ids
33
61 - def set_cuda_visible_devices(self, device_ids) -> None:
62 - set_cuda_visible_devices(device_ids)
63 -
34 def execute_model_compiled_dag_remote(self, ignored):
35 """Used only when compiled DAG is enabled."""
36 import torch
@@ -77,7 +47,7 @@ except ImportError as e:
47 "For distributed inference, please install Ray with "
48 "`pip install ray`.")
49 ray = None # type: ignore
80 - RayWorkerVllm = None # type: ignore
50 + RayWorkerWrapper = None # type: ignore
51
52
53 def initialize_ray_cluster(
vllm/executor/ray_gpu_executor.py
+84 -72
@@ -1,17 +1,16 @@
1 import asyncio
2 -import copy
2 import os
3 import pickle
4 from collections import defaultdict
5 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
6
8 -from vllm.engine.ray_utils import RayWorkerVllm, ray
7 +from vllm.engine.ray_utils import RayWorkerWrapper, ray
8 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
9 from vllm.logger import init_logger
10 from vllm.lora.request import LoRARequest
11 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
14 - make_async, set_cuda_visible_devices)
13 + make_async)
14
15 if ray is not None:
16 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -74,9 +73,9 @@ class RayGPUExecutor(ExecutorBase):
73
74 # The driver dummy worker does not actually use any resources.
75 # It holds the resource for the driver worker.
77 - self.driver_dummy_worker: RayWorkerVllm = None
76 + self.driver_dummy_worker: RayWorkerWrapper = None
77 # The remaining workers are the actual ray actors.
79 - self.workers: List[RayWorkerVllm] = []
78 + self.workers: List[RayWorkerWrapper] = []
79
80 if self.parallel_config.ray_workers_use_nsight:
81 ray_remote_kwargs = self._configure_ray_workers_use_nsight(
@@ -97,13 +96,20 @@ class RayGPUExecutor(ExecutorBase):
96 num_gpus=num_gpus,
97 scheduling_strategy=scheduling_strategy,
98 **ray_remote_kwargs,
100 - )(RayWorkerVllm).remote(self.model_config.trust_remote_code)
99 + )(RayWorkerWrapper).remote(
100 + worker_module_name="vllm.worker.worker",
101 + worker_class_name="Worker",
102 + )
103
104 worker_ip = ray.get(worker.get_node_ip.remote())
105 if worker_ip == driver_ip and self.driver_dummy_worker is None:
106 # If the worker is on the same node as the driver, we use it
107 # as the resource holder for the driver process.
108 self.driver_dummy_worker = worker
109 + self.driver_worker = RayWorkerWrapper(
110 + worker_module_name="vllm.worker.worker",
111 + worker_class_name="Worker",
112 + )
113 else:
114 # Else, added to the list of workers.
115 self.workers.append(worker)
@@ -115,82 +121,56 @@ class RayGPUExecutor(ExecutorBase):
121 "GPU node.")
122
123 # Get the set of GPU IDs used on each node.
118 - driver_node_id, driver_gpu_ids = ray.get(
119 - self.driver_dummy_worker.get_node_and_gpu_ids.remote())
120 - worker_node_and_gpu_ids = ray.get(
121 - [worker.get_node_and_gpu_ids.remote() for worker in self.workers])
124 + worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
125 + use_dummy_driver=True)
126
127 node_workers = defaultdict(list)
128 node_gpus = defaultdict(list)
129
126 - node_workers[driver_node_id].append(0)
127 - node_gpus[driver_node_id].extend(driver_gpu_ids)
128 - for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids,
129 - start=1):
130 + for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
131 node_workers[node_id].append(i)
132 node_gpus[node_id].extend(gpu_ids)
133 for node_id, gpu_ids in node_gpus.items():
134 node_gpus[node_id] = sorted(gpu_ids)
135
136 # Set CUDA_VISIBLE_DEVICES for the driver and workers.
136 - set_cuda_visible_devices(node_gpus[driver_node_id])
137 - for worker, (node_id, _) in zip(self.workers, worker_node_and_gpu_ids):
138 - worker.set_cuda_visible_devices.remote(node_gpus[node_id])
137 + all_args_to_update_environment_variables = []
138 + for (node_id, _) in worker_node_and_gpu_ids:
139 + all_args_to_update_environment_variables.append([{
140 + "CUDA_VISIBLE_DEVICES":
141 + ",".join(map(str, node_gpus[node_id]))
142 + }])
143 + self._run_workers("update_environment_variables",
144 + all_args=all_args_to_update_environment_variables)
145
146 distributed_init_method = get_distributed_init_method(
147 driver_ip, get_open_port())
148
143 - # Lazy import the Worker to avoid importing torch.cuda/xformers
144 - # before CUDA_VISIBLE_DEVICES is set in the Worker
145 - from vllm.worker.worker import Worker
146 -
147 - model_config = copy.deepcopy(self.model_config)
148 - parallel_config = copy.deepcopy(self.parallel_config)
149 - scheduler_config = copy.deepcopy(self.scheduler_config)
150 - load_config = copy.deepcopy(self.load_config)
151 - device_config = copy.deepcopy(self.device_config)
152 - lora_config = copy.deepcopy(self.lora_config)
153 - cache_config = copy.deepcopy(self.cache_config)
154 - vision_language_config = copy.deepcopy(self.vision_language_config)
155 -
156 - # Initialize the actual workers with the Worker class.
157 - for rank, (worker, (node_id, _)) in enumerate(
158 - zip(self.workers, worker_node_and_gpu_ids),
159 - start=1,
160 - ):
149 + def collect_arg_helper_func(**kwargs):
150 + # avoid writing `{"name": value}` manually
151 + return kwargs
152 +
153 + init_worker_all_kwargs = []
154 +
155 + # Initialize the actual workers inside worker wrapper.
156 + for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids, ):
157 local_rank = node_workers[node_id].index(rank)
162 - worker.init_worker.remote(
163 - lambda rank=rank, local_rank=local_rank: Worker(
164 - model_config=model_config,
165 - parallel_config=parallel_config,
166 - scheduler_config=scheduler_config,
167 - device_config=device_config,
168 - cache_config=cache_config,
169 - load_config=load_config,
158 + init_worker_all_kwargs.append(
159 + collect_arg_helper_func(
160 + model_config=self.model_config,
161 + parallel_config=self.parallel_config,
162 + scheduler_config=self.scheduler_config,
163 + device_config=self.device_config,
164 + cache_config=self.cache_config,
165 + load_config=self.load_config,
166 local_rank=local_rank,
167 rank=rank,
168 distributed_init_method=distributed_init_method,
173 - lora_config=lora_config,
174 - vision_language_config=vision_language_config,
169 + lora_config=self.lora_config,
170 + vision_language_config=self.vision_language_config,
171 + is_driver_worker=rank == 0,
172 ))
176 -
177 - # Initialize the driver worker with the Worker class.
178 - driver_rank = 0
179 - driver_local_rank = node_workers[driver_node_id].index(driver_rank)
180 - self.driver_worker = Worker(
181 - model_config=self.model_config,
182 - parallel_config=self.parallel_config,
183 - scheduler_config=self.scheduler_config,
184 - device_config=self.device_config,
185 - cache_config=self.cache_config,
186 - local_rank=driver_local_rank,
187 - rank=driver_rank,
188 - distributed_init_method=distributed_init_method,
189 - lora_config=self.lora_config,
190 - vision_language_config=self.vision_language_config,
191 - load_config=self.load_config,
192 - is_driver_worker=True,
193 - )
173 + self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
174
175 self._run_workers("init_device")
176 self._run_workers(
@@ -279,13 +259,35 @@ class RayGPUExecutor(ExecutorBase):
259 self,
260 method: str,
261 *args,
282 - driver_args: Optional[Tuple[Any, ...]] = None,
262 + driver_args: Optional[Tuple[Any]] = None,
263 driver_kwargs: Optional[Dict[str, Any]] = None,
264 + all_args: Optional[List[List[Any]]] = None,
265 + all_kwargs: Optional[List[Dict[str, Any]]] = None,
266 + use_dummy_driver: bool = False,
267 max_concurrent_workers: Optional[int] = None,
268 use_ray_compiled_dag: bool = False,
269 **kwargs,
270 ) -> Any:
288 - """Runs the given method on all workers."""
271 + """Runs the given method on all workers.
272 + all_args and all_kwargs are used to pass heterogeneous arguments,
273 + i.e. different arguments for each worker.
274 + """
275 + if driver_args is None:
276 + driver_args = args
277 + if driver_kwargs is None:
278 + driver_kwargs = kwargs
279 +
280 + # for mypy type checking
281 + assert driver_args is not None
282 + assert driver_kwargs is not None
283 + if all_args is None:
284 + all_args = [driver_args] + [args] * len(self.workers)
285 + if all_kwargs is None:
286 + all_kwargs = [driver_kwargs] + [kwargs] * len(self.workers)
287 +
288 + # for mypy type checking
289 + assert all_args is not None
290 + assert all_kwargs is not None
291
292 if max_concurrent_workers:
293 raise NotImplementedError(
@@ -299,8 +301,10 @@ class RayGPUExecutor(ExecutorBase):
301 else:
302 # Start the ray workers first.
303 ray_worker_outputs = [
302 - worker.execute_method.remote(method, *args, **kwargs)
303 - for worker in self.workers
304 + worker.execute_method.remote(method, *worker_args,
305 + **worker_kwargs)
306 + for (worker, worker_args, worker_kwargs
307 + ) in zip(self.workers, all_args[1:], all_kwargs[1:])
308 ]
309
310 if driver_args is None:
@@ -309,9 +313,13 @@ class RayGPUExecutor(ExecutorBase):
313 driver_kwargs = kwargs
314
315 # Start the driver worker after all the ray workers.
312 - driver_worker_output = getattr(self.driver_worker,
313 - method)(*driver_args, **driver_kwargs)
314 -
316 + if not use_dummy_driver:
317 + driver_worker_output = self.driver_worker.execute_method(
318 + method, *all_args[0], **all_kwargs[0])
319 + else:
320 + driver_worker_output = ray.get(
321 + self.driver_dummy_worker.execute_method.remote(
322 + method, *all_args[0], **all_kwargs[0]))
323 # Get the results of the ray workers.
324 if self.workers:
325 if use_ray_compiled_dag:
@@ -386,8 +394,12 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
394 driver_kwargs = kwargs
395
396 # Run the driver worker asynchronously.
389 - driver_executor = make_async(getattr(self.driver_worker, method))
390 - coros.append(driver_executor(*driver_args, **driver_kwargs))
397 + def helper():
398 + return self.driver_worker.execute_method(method, *driver_args,
399 + **driver_kwargs)
400 +
401 + driver_executor = make_async(helper)
402 + coros.append(driver_executor())
403
404 # Run the ray workers asynchronously.
405 for worker in self.workers:
vllm/utils.py
+14 -2
@@ -271,8 +271,12 @@ def get_open_port() -> int:
271 return s.getsockname()[1]
272
273
274 -def set_cuda_visible_devices(device_ids: List[int]) -> None:
275 - os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
274 +def update_environment_variables(envs: Dict[str, str]):
275 + for k, v in envs.items():
276 + if k in os.environ:
277 + logger.warning(f"Overwriting environment variable {k} "
278 + f"from '{os.environ[k]}' to '{v}'")
279 + os.environ[k] = v
280
281
282 def chunk_list(lst, chunk_size):
@@ -505,3 +509,11 @@ def merge_dicts(dict1: Dict[Any, List[Any]],
509 merged_dict[key].extend(value)
510
511 return dict(merged_dict)
512 +
513 +
514 +def init_cached_hf_modules():
515 + """
516 + Lazy initialization of the Hugging Face modules.
517 + """
518 + from transformers.dynamic_module_utils import init_hf_modules
519 + init_hf_modules()
vllm/worker/cpu_worker.py
+4 -1
@@ -138,7 +138,10 @@ class CPUWorker(LoraNotSupportedWorkerBase):
138 self.is_driver_worker = is_driver_worker
139 if self.is_driver_worker:
140 assert self.rank == 0, "The driver worker must have rank 0."
141 -
141 + if self.model_config.trust_remote_code:
142 + # note: lazy import to avoid importing torch before initializing
143 + from vllm.utils import init_cached_hf_modules
144 + init_cached_hf_modules()
145 self.model_runner = CPUModelRunner(model_config,
146 parallel_config,
147 scheduler_config,
vllm/worker/neuron_worker.py
+4
@@ -29,6 +29,10 @@ class NeuronWorker(LoraNotSupportedWorkerBase):
29 self.scheduler_config = scheduler_config
30 self.device_config = device_config
31 self.cache_config = cache_config
32 + if self.model_config.trust_remote_code:
33 + # note: lazy import to avoid importing torch before initializing
34 + from vllm.utils import init_cached_hf_modules
35 + init_cached_hf_modules()
36
37 self.model_runner = NeuronModelRunner(model_config, parallel_config,
38 scheduler_config, device_config)
vllm/worker/worker.py
+4
@@ -60,6 +60,10 @@ class Worker(WorkerBase):
60 if self.is_driver_worker:
61 assert self.rank == 0, "The driver worker must have rank 0."
62
63 + if self.model_config.trust_remote_code:
64 + # note: lazy import to avoid importing torch before initializing
65 + from vllm.utils import init_cached_hf_modules
66 + init_cached_hf_modules()
67 self.vision_language_config = vision_language_config
68 if self.vision_language_config:
69 assert not self.lora_config, (
vllm/worker/worker_base.py
+56
@@ -1,8 +1,14 @@
1 +import importlib
2 +import os
3 from abc import ABC, abstractmethod
4 from typing import Dict, List, Tuple
5
6 +from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9 +from vllm.utils import update_environment_variables
10 +
11 +logger = init_logger(__name__)
12
13
14 class WorkerBase(ABC):
@@ -82,3 +88,53 @@ class LoraNotSupportedWorkerBase(WorkerBase):
88
89 def list_loras(self) -> List[int]:
90 raise ValueError(f"{type(self)} does not support LoRA")
91 +
92 +
93 +class WorkerWrapperBase:
94 + """
95 + The whole point of this class is to lazily initialize the worker.
96 + We first instantiate the WorkerWrapper, which remembers the worker module
97 + and class name. Then, when we call `update_environment_variables`, and the
98 + real initialization happens in `init_worker`.
99 + """
100 +
101 + def __init__(self,
102 + worker_module_name=None,
103 + worker_class_name=None) -> None:
104 + self.worker_module_name = worker_module_name
105 + self.worker_class_name = worker_class_name
106 + self.worker = None
107 +
108 + def update_environment_variables(self, envs: Dict[str, str]) -> None:
109 + key = 'CUDA_VISIBLE_DEVICES'
110 + if key in envs and key in os.environ:
111 + # overwriting CUDA_VISIBLE_DEVICES is desired behavior
112 + # suppress the warning in `update_environment_variables`
113 + del os.environ[key]
114 + update_environment_variables(envs)
115 +
116 + def init_worker(self, *args, **kwargs):
117 + """
118 + Actual initialization of the worker class.
119 + Arguments are passed to the worker class constructor.
120 + """
121 + mod = importlib.import_module(self.worker_module_name)
122 + worker_class = getattr(mod, self.worker_class_name)
123 + self.worker = worker_class(*args, **kwargs)
124 +
125 + def execute_method(self, method, *args, **kwargs):
126 + try:
127 + if hasattr(self, method):
128 + executor = getattr(self, method)
129 + else:
130 + executor = getattr(self.worker, method)
131 + return executor(*args, **kwargs)
132 + except Exception as e:
133 + # if the driver worker also execute methods,
134 + # exceptions in the rest worker may cause deadlock in rpc like ray
135 + # see https://github.com/vllm-project/vllm/issues/3455
136 + # print the error and inform the user to solve the error
137 + msg = (f"Error executing method {method}. "
138 + "This might cause deadlock in distributed execution.")
139 + logger.exception(msg)
140 + raise e