A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
fix RAM OOM when load large models in tensor parallel mode. (#1395)
Co-authored-by: ran_lin <rlin@thoughtworks.com>
boydfd committed
Nov 21, 2023 at 11:02 UTC
4bb6b67188f1073419555a2c214ca8d2b571467c
4 files changed
+52
-7
vllm/config.py
+2
@@ -285,10 +285,12 @@ class ParallelConfig:
285
pipeline_parallel_size: int,
286
tensor_parallel_size: int,
287
worker_use_ray: bool,
288
+ max_parallel_loading_workers: Optional[int] = None,
289
) -> None:
290
self.pipeline_parallel_size = pipeline_parallel_size
291
self.tensor_parallel_size = tensor_parallel_size
292
self.worker_use_ray = worker_use_ray
293
+ self.max_parallel_loading_workers = max_parallel_loading_workers
294
295
self.world_size = pipeline_parallel_size * tensor_parallel_size
296
if self.world_size > 1:
vllm/engine/arg_utils.py
+9
-1
@@ -22,6 +22,7 @@ class EngineArgs:
22
worker_use_ray: bool = False
23
pipeline_parallel_size: int = 1
24
tensor_parallel_size: int = 1
25
+ max_parallel_loading_workers: Optional[int] = None
26
block_size: int = 16
27
swap_space: int = 4 # GiB
28
gpu_memory_utilization: float = 0.90
@@ -128,6 +129,12 @@ class EngineArgs:
129
type=int,
130
default=EngineArgs.tensor_parallel_size,
131
help='number of tensor parallel replicas')
132
+ parser.add_argument(
133
+ '--max-parallel-loading-workers',
134
+ type=int,
135
+ help='load model sequentially in multiple batches, '
136
+ 'to avoid RAM OOM when using tensor '
137
+ 'parallel and large models')
138
# KV cache arguments
139
parser.add_argument('--block-size',
140
type=int,
@@ -195,7 +202,8 @@ class EngineArgs:
202
getattr(model_config.hf_config, 'sliding_window', None))
203
parallel_config = ParallelConfig(self.pipeline_parallel_size,
204
self.tensor_parallel_size,
198
- self.worker_use_ray)
205
+ self.worker_use_ray,
206
+ self.max_parallel_loading_workers)
207
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
208
self.max_num_seqs,
209
model_config.max_model_len,
vllm/engine/llm_engine.py
+39
-6
@@ -143,6 +143,12 @@ class LLMEngine:
143
"init_model",
144
get_all_outputs=True,
145
)
146
+ self._run_workers(
147
+ "load_model",
148
+ get_all_outputs=True,
149
+ max_concurrent_workers=self.parallel_config.
150
+ max_parallel_loading_workers,
151
+ )
152
153
def _init_workers_ray(self, placement_group: "PlacementGroup",
154
**ray_remote_kwargs):
@@ -182,6 +188,12 @@ class LLMEngine:
188
"init_model",
189
get_all_outputs=True,
190
)
191
+ self._run_workers(
192
+ "load_model",
193
+ get_all_outputs=True,
194
+ max_concurrent_workers=self.parallel_config.
195
+ max_parallel_loading_workers,
196
+ )
197
198
def _verify_args(self) -> None:
199
self.model_config.verify_with_parallel_config(self.parallel_config)
@@ -682,16 +694,15 @@ class LLMEngine:
694
seq.status = SequenceStatus.FINISHED_STOPPED
695
return
696
685
- def _run_workers(
697
+ def _run_workers_in_batch(
698
self,
699
+ workers,
700
method: str,
701
*args,
689
- get_all_outputs: bool = False,
702
**kwargs,
691
- ) -> Any:
692
- """Runs the given method on all workers."""
703
+ ):
704
all_outputs = []
694
- for worker in self.workers:
705
+ for worker in workers:
706
if self.parallel_config.worker_use_ray:
707
executor = partial(worker.execute_method.remote, method)
708
else:
@@ -699,9 +710,31 @@ class LLMEngine:
710
711
output = executor(*args, **kwargs)
712
all_outputs.append(output)
702
-
713
if self.parallel_config.worker_use_ray:
714
all_outputs = ray.get(all_outputs)
715
+ return all_outputs
716
+
717
+ def _run_workers(
718
+ self,
719
+ method: str,
720
+ *args,
721
+ get_all_outputs: bool = False,
722
+ max_concurrent_workers: Optional[int] = None,
723
+ **kwargs,
724
+ ) -> Any:
725
+ """Runs the given method on all workers."""
726
+ all_outputs = []
727
+ if max_concurrent_workers:
728
+ work_groups = [
729
+ self.workers[i:i + max_concurrent_workers]
730
+ for i in range(0, len(self.workers), max_concurrent_workers)
731
+ ]
732
+ else:
733
+ work_groups = [self.workers]
734
+
735
+ for workers in work_groups:
736
+ all_outputs.extend(
737
+ self._run_workers_in_batch(workers, method, *args, **kwargs))
738
739
if get_all_outputs:
740
return all_outputs
vllm/worker/worker.py
+2
@@ -67,6 +67,8 @@ class Worker:
67
68
# Initialize the model.
69
set_random_seed(self.model_config.seed)
70
+
71
+ def load_model(self):
72
self.model = get_model(self.model_config)
73
74
@torch.inference_mode()