A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Centralize GPU Worker construction (#4419)
Nick Hill committed
Apr 30, 2024 at 18:06 UTC
2e240c69a9874743abc8b0b681e8c13d675beda3
2 files changed
+47
-68
vllm/executor/gpu_executor.py
+39
-44
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Set, Tuple
1
+from typing import Any, Dict, List, Optional, Set, Tuple
2
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
@@ -6,6 +6,7 @@ from vllm.lora.request import LoRARequest
6
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
8
make_async)
9
+from vllm.worker.worker_base import WorkerWrapperBase
10
11
logger = init_logger(__name__)
12
@@ -23,30 +24,47 @@ class GPUExecutor(ExecutorBase):
24
else:
25
self._init_spec_worker()
26
26
- def _init_non_spec_worker(self):
27
- # Lazy import the Worker to avoid importing torch.cuda/xformers
28
- # before CUDA_VISIBLE_DEVICES is set in the Worker
29
- from vllm.worker.worker import Worker
30
-
31
- assert self.parallel_config.world_size == 1, (
32
- "GPUExecutor only supports single GPU.")
33
-
34
- distributed_init_method = get_distributed_init_method(
35
- get_ip(), get_open_port())
36
- self.driver_worker = Worker(
27
+ def _get_worker_kwargs(
28
+ self,
29
+ local_rank: int = 0,
30
+ rank: int = 0,
31
+ distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
32
+ """Return worker init args for a given rank."""
33
+ if distributed_init_method is None:
34
+ distributed_init_method = get_distributed_init_method(
35
+ get_ip(), get_open_port())
36
+ return dict(
37
model_config=self.model_config,
38
parallel_config=self.parallel_config,
39
scheduler_config=self.scheduler_config,
40
device_config=self.device_config,
41
cache_config=self.cache_config,
42
load_config=self.load_config,
43
- local_rank=0,
44
- rank=0,
43
+ local_rank=local_rank,
44
+ rank=rank,
45
distributed_init_method=distributed_init_method,
46
lora_config=self.lora_config,
47
vision_language_config=self.vision_language_config,
48
- is_driver_worker=True,
48
+ is_driver_worker=rank == 0,
49
+ )
50
+
51
+ def _create_worker(self,
52
+ local_rank: int = 0,
53
+ rank: int = 0,
54
+ distributed_init_method: Optional[str] = None):
55
+ wrapper = WorkerWrapperBase(
56
+ worker_module_name="vllm.worker.worker",
57
+ worker_class_name="Worker",
58
)
59
+ wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
60
+ distributed_init_method))
61
+ return wrapper.worker
62
+
63
+ def _init_non_spec_worker(self):
64
+ assert self.parallel_config.world_size == 1, (
65
+ "GPUExecutor only supports single GPU.")
66
+
67
+ self.driver_worker = self._create_worker()
68
self.driver_worker.init_device()
69
self.driver_worker.load_model()
70
@@ -57,41 +75,18 @@ class GPUExecutor(ExecutorBase):
75
76
from vllm.spec_decode.multi_step_worker import MultiStepWorker
77
from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
60
- from vllm.worker.worker import Worker
78
62
- distributed_init_method = get_distributed_init_method(
63
- get_ip(), get_open_port())
64
-
65
- target_worker = Worker(
66
- model_config=self.model_config,
67
- parallel_config=self.parallel_config,
68
- scheduler_config=self.scheduler_config,
69
- device_config=self.device_config,
70
- cache_config=self.cache_config,
71
- load_config=self.load_config,
72
- local_rank=0,
73
- rank=0,
74
- distributed_init_method=distributed_init_method,
75
- lora_config=self.lora_config,
76
- vision_language_config=self.vision_language_config,
77
- is_driver_worker=True,
78
- )
79
+ target_worker = self._create_worker()
80
80
- draft_worker = MultiStepWorker(
81
+ draft_worker_kwargs = self._get_worker_kwargs()
82
+ # Override draft-model specific worker args.
83
+ draft_worker_kwargs.update(
84
model_config=self.speculative_config.draft_model_config,
85
parallel_config=self.speculative_config.draft_parallel_config,
83
- scheduler_config=self.scheduler_config,
84
- device_config=self.device_config,
85
- cache_config=self.cache_config,
86
# TODO allow draft-model specific load config.
87
- load_config=self.load_config,
88
- local_rank=0,
89
- rank=0,
90
- distributed_init_method=distributed_init_method,
91
- lora_config=self.lora_config,
92
- vision_language_config=self.vision_language_config,
93
- is_driver_worker=True,
87
+ #load_config=self.load_config,
88
)
89
+ draft_worker = MultiStepWorker(**draft_worker_kwargs)
90
91
spec_decode_worker = SpecDecodeWorker.from_workers(
92
proposer_worker=draft_worker, scorer_worker=target_worker)
vllm/executor/ray_gpu_executor.py
+8
-24
@@ -153,29 +153,14 @@ class RayGPUExecutor(DistributedGPUExecutor):
153
distributed_init_method = get_distributed_init_method(
154
driver_ip, get_open_port())
155
156
- def collect_arg_helper_func(**kwargs):
157
- # avoid writing `{"name": value}` manually
158
- return kwargs
159
-
156
# Initialize the actual workers inside worker wrapper.
161
- init_worker_all_kwargs = []
162
- for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids):
163
- local_rank = node_workers[node_id].index(rank)
164
- init_worker_all_kwargs.append(
165
- collect_arg_helper_func(
166
- model_config=self.model_config,
167
- parallel_config=self.parallel_config,
168
- scheduler_config=self.scheduler_config,
169
- device_config=self.device_config,
170
- cache_config=self.cache_config,
171
- load_config=self.load_config,
172
- local_rank=local_rank,
173
- rank=rank,
174
- distributed_init_method=distributed_init_method,
175
- lora_config=self.lora_config,
176
- vision_language_config=self.vision_language_config,
177
- is_driver_worker=rank == 0,
178
- ))
157
+ init_worker_all_kwargs = [
158
+ self._get_worker_kwargs(
159
+ local_rank=node_workers[node_id].index(rank),
160
+ rank=rank,
161
+ distributed_init_method=distributed_init_method,
162
+ ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
163
+ ]
164
self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
165
166
self._run_workers("init_device")
@@ -201,8 +186,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
186
use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
187
188
# Only the driver worker returns the sampling results.
204
- output = all_outputs[0]
205
- return output
189
+ return all_outputs[0]
190
191
def _run_workers(
192
self,