@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2e240c69a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Centralize GPU Worker construction (#4419)

Nick Hill committed Apr 30, 2024 at 18:06 UTC 2e240c69a9874743abc8b0b681e8c13d675beda3
2 files changed +47 -68
vllm/executor/gpu_executor.py
+39 -44
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Set, Tuple
1 +from typing import Any, Dict, List, Optional, Set, Tuple
2
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
@@ -6,6 +6,7 @@ from vllm.lora.request import LoRARequest
6 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
8 make_async)
9 +from vllm.worker.worker_base import WorkerWrapperBase
10
11 logger = init_logger(__name__)
12
@@ -23,30 +24,47 @@ class GPUExecutor(ExecutorBase):
24 else:
25 self._init_spec_worker()
26
26 - def _init_non_spec_worker(self):
27 - # Lazy import the Worker to avoid importing torch.cuda/xformers
28 - # before CUDA_VISIBLE_DEVICES is set in the Worker
29 - from vllm.worker.worker import Worker
30 -
31 - assert self.parallel_config.world_size == 1, (
32 - "GPUExecutor only supports single GPU.")
33 -
34 - distributed_init_method = get_distributed_init_method(
35 - get_ip(), get_open_port())
36 - self.driver_worker = Worker(
27 + def _get_worker_kwargs(
28 + self,
29 + local_rank: int = 0,
30 + rank: int = 0,
31 + distributed_init_method: Optional[str] = None) -> Dict[str, Any]:
32 + """Return worker init args for a given rank."""
33 + if distributed_init_method is None:
34 + distributed_init_method = get_distributed_init_method(
35 + get_ip(), get_open_port())
36 + return dict(
37 model_config=self.model_config,
38 parallel_config=self.parallel_config,
39 scheduler_config=self.scheduler_config,
40 device_config=self.device_config,
41 cache_config=self.cache_config,
42 load_config=self.load_config,
43 - local_rank=0,
44 - rank=0,
43 + local_rank=local_rank,
44 + rank=rank,
45 distributed_init_method=distributed_init_method,
46 lora_config=self.lora_config,
47 vision_language_config=self.vision_language_config,
48 - is_driver_worker=True,
48 + is_driver_worker=rank == 0,
49 + )
50 +
51 + def _create_worker(self,
52 + local_rank: int = 0,
53 + rank: int = 0,
54 + distributed_init_method: Optional[str] = None):
55 + wrapper = WorkerWrapperBase(
56 + worker_module_name="vllm.worker.worker",
57 + worker_class_name="Worker",
58 )
59 + wrapper.init_worker(**self._get_worker_kwargs(local_rank, rank,
60 + distributed_init_method))
61 + return wrapper.worker
62 +
63 + def _init_non_spec_worker(self):
64 + assert self.parallel_config.world_size == 1, (
65 + "GPUExecutor only supports single GPU.")
66 +
67 + self.driver_worker = self._create_worker()
68 self.driver_worker.init_device()
69 self.driver_worker.load_model()
70
@@ -57,41 +75,18 @@ class GPUExecutor(ExecutorBase):
75
76 from vllm.spec_decode.multi_step_worker import MultiStepWorker
77 from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker
60 - from vllm.worker.worker import Worker
78
62 - distributed_init_method = get_distributed_init_method(
63 - get_ip(), get_open_port())
64 -
65 - target_worker = Worker(
66 - model_config=self.model_config,
67 - parallel_config=self.parallel_config,
68 - scheduler_config=self.scheduler_config,
69 - device_config=self.device_config,
70 - cache_config=self.cache_config,
71 - load_config=self.load_config,
72 - local_rank=0,
73 - rank=0,
74 - distributed_init_method=distributed_init_method,
75 - lora_config=self.lora_config,
76 - vision_language_config=self.vision_language_config,
77 - is_driver_worker=True,
78 - )
79 + target_worker = self._create_worker()
80
80 - draft_worker = MultiStepWorker(
81 + draft_worker_kwargs = self._get_worker_kwargs()
82 + # Override draft-model specific worker args.
83 + draft_worker_kwargs.update(
84 model_config=self.speculative_config.draft_model_config,
85 parallel_config=self.speculative_config.draft_parallel_config,
83 - scheduler_config=self.scheduler_config,
84 - device_config=self.device_config,
85 - cache_config=self.cache_config,
86 # TODO allow draft-model specific load config.
87 - load_config=self.load_config,
88 - local_rank=0,
89 - rank=0,
90 - distributed_init_method=distributed_init_method,
91 - lora_config=self.lora_config,
92 - vision_language_config=self.vision_language_config,
93 - is_driver_worker=True,
87 + #load_config=self.load_config,
88 )
89 + draft_worker = MultiStepWorker(**draft_worker_kwargs)
90
91 spec_decode_worker = SpecDecodeWorker.from_workers(
92 proposer_worker=draft_worker, scorer_worker=target_worker)
vllm/executor/ray_gpu_executor.py
+8 -24
@@ -153,29 +153,14 @@ class RayGPUExecutor(DistributedGPUExecutor):
153 distributed_init_method = get_distributed_init_method(
154 driver_ip, get_open_port())
155
156 - def collect_arg_helper_func(**kwargs):
157 - # avoid writing `{"name": value}` manually
158 - return kwargs
159 -
156 # Initialize the actual workers inside worker wrapper.
161 - init_worker_all_kwargs = []
162 - for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids):
163 - local_rank = node_workers[node_id].index(rank)
164 - init_worker_all_kwargs.append(
165 - collect_arg_helper_func(
166 - model_config=self.model_config,
167 - parallel_config=self.parallel_config,
168 - scheduler_config=self.scheduler_config,
169 - device_config=self.device_config,
170 - cache_config=self.cache_config,
171 - load_config=self.load_config,
172 - local_rank=local_rank,
173 - rank=rank,
174 - distributed_init_method=distributed_init_method,
175 - lora_config=self.lora_config,
176 - vision_language_config=self.vision_language_config,
177 - is_driver_worker=rank == 0,
178 - ))
157 + init_worker_all_kwargs = [
158 + self._get_worker_kwargs(
159 + local_rank=node_workers[node_id].index(rank),
160 + rank=rank,
161 + distributed_init_method=distributed_init_method,
162 + ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
163 + ]
164 self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
165
166 self._run_workers("init_device")
@@ -201,8 +186,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
186 use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
187
188 # Only the driver worker returns the sampling results.
204 - output = all_outputs[0]
205 - return output
189 + return all_outputs[0]
190
191 def _run_workers(
192 self,