@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4bb6b67188
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

fix RAM OOM when load large models in tensor parallel mode. (#1395)

Co-authored-by: ran_lin <rlin@thoughtworks.com>

boydfd committed Nov 21, 2023 at 11:02 UTC 4bb6b67188f1073419555a2c214ca8d2b571467c
4 files changed +52 -7
vllm/config.py
+2
@@ -285,10 +285,12 @@ class ParallelConfig:
285 pipeline_parallel_size: int,
286 tensor_parallel_size: int,
287 worker_use_ray: bool,
288 + max_parallel_loading_workers: Optional[int] = None,
289 ) -> None:
290 self.pipeline_parallel_size = pipeline_parallel_size
291 self.tensor_parallel_size = tensor_parallel_size
292 self.worker_use_ray = worker_use_ray
293 + self.max_parallel_loading_workers = max_parallel_loading_workers
294
295 self.world_size = pipeline_parallel_size * tensor_parallel_size
296 if self.world_size > 1:
vllm/engine/arg_utils.py
+9 -1
@@ -22,6 +22,7 @@ class EngineArgs:
22 worker_use_ray: bool = False
23 pipeline_parallel_size: int = 1
24 tensor_parallel_size: int = 1
25 + max_parallel_loading_workers: Optional[int] = None
26 block_size: int = 16
27 swap_space: int = 4 # GiB
28 gpu_memory_utilization: float = 0.90
@@ -128,6 +129,12 @@ class EngineArgs:
129 type=int,
130 default=EngineArgs.tensor_parallel_size,
131 help='number of tensor parallel replicas')
132 + parser.add_argument(
133 + '--max-parallel-loading-workers',
134 + type=int,
135 + help='load model sequentially in multiple batches, '
136 + 'to avoid RAM OOM when using tensor '
137 + 'parallel and large models')
138 # KV cache arguments
139 parser.add_argument('--block-size',
140 type=int,
@@ -195,7 +202,8 @@ class EngineArgs:
202 getattr(model_config.hf_config, 'sliding_window', None))
203 parallel_config = ParallelConfig(self.pipeline_parallel_size,
204 self.tensor_parallel_size,
198 - self.worker_use_ray)
205 + self.worker_use_ray,
206 + self.max_parallel_loading_workers)
207 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
208 self.max_num_seqs,
209 model_config.max_model_len,
vllm/engine/llm_engine.py
+39 -6
@@ -143,6 +143,12 @@ class LLMEngine:
143 "init_model",
144 get_all_outputs=True,
145 )
146 + self._run_workers(
147 + "load_model",
148 + get_all_outputs=True,
149 + max_concurrent_workers=self.parallel_config.
150 + max_parallel_loading_workers,
151 + )
152
153 def _init_workers_ray(self, placement_group: "PlacementGroup",
154 **ray_remote_kwargs):
@@ -182,6 +188,12 @@ class LLMEngine:
188 "init_model",
189 get_all_outputs=True,
190 )
191 + self._run_workers(
192 + "load_model",
193 + get_all_outputs=True,
194 + max_concurrent_workers=self.parallel_config.
195 + max_parallel_loading_workers,
196 + )
197
198 def _verify_args(self) -> None:
199 self.model_config.verify_with_parallel_config(self.parallel_config)
@@ -682,16 +694,15 @@ class LLMEngine:
694 seq.status = SequenceStatus.FINISHED_STOPPED
695 return
696
685 - def _run_workers(
697 + def _run_workers_in_batch(
698 self,
699 + workers,
700 method: str,
701 *args,
689 - get_all_outputs: bool = False,
702 **kwargs,
691 - ) -> Any:
692 - """Runs the given method on all workers."""
703 + ):
704 all_outputs = []
694 - for worker in self.workers:
705 + for worker in workers:
706 if self.parallel_config.worker_use_ray:
707 executor = partial(worker.execute_method.remote, method)
708 else:
@@ -699,9 +710,31 @@ class LLMEngine:
710
711 output = executor(*args, **kwargs)
712 all_outputs.append(output)
702 -
713 if self.parallel_config.worker_use_ray:
714 all_outputs = ray.get(all_outputs)
715 + return all_outputs
716 +
717 + def _run_workers(
718 + self,
719 + method: str,
720 + *args,
721 + get_all_outputs: bool = False,
722 + max_concurrent_workers: Optional[int] = None,
723 + **kwargs,
724 + ) -> Any:
725 + """Runs the given method on all workers."""
726 + all_outputs = []
727 + if max_concurrent_workers:
728 + work_groups = [
729 + self.workers[i:i + max_concurrent_workers]
730 + for i in range(0, len(self.workers), max_concurrent_workers)
731 + ]
732 + else:
733 + work_groups = [self.workers]
734 +
735 + for workers in work_groups:
736 + all_outputs.extend(
737 + self._run_workers_in_batch(workers, method, *args, **kwargs))
738
739 if get_all_outputs:
740 return all_outputs
vllm/worker/worker.py
+2
@@ -67,6 +67,8 @@ class Worker:
67
68 # Initialize the model.
69 set_random_seed(self.model_config.seed)
70 +
71 + def load_model(self):
72 self.model = get_model(self.model_config)
73
74 @torch.inference_mode()