@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3de6e6a30e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[core][distributed] support n layers % pp size != 0 (#6115)

youkaichao committed Jul 3, 2024 at 16:40 UTC 3de6e6a30e33406c2b3bb81fb5a82a2966cebd87
7 files changed +19 -10
.buildkite/test-pipeline.yaml
+1
@@ -80,6 +80,7 @@ steps:
80 commands:
81 - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
82 - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
83 + - TP_SIZE=1 PP_SIZE=3 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
84 - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
85 - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
86
vllm/config.py
+6 -9
@@ -265,8 +265,6 @@ class ModelConfig:
265 " must be divisible by tensor parallel size "
266 f"({tensor_parallel_size}).")
267
268 - total_num_hidden_layers = getattr(self.hf_text_config,
269 - "num_hidden_layers", 0)
268 pipeline_parallel_size = parallel_config.pipeline_parallel_size
269 architectures = getattr(self.hf_config, "architectures", [])
270 if not all(arch in _PP_SUPPORTED_MODELS
@@ -275,12 +273,6 @@ class ModelConfig:
273 "Pipeline parallelism is only supported for the following "
274 f" architectures: {_PP_SUPPORTED_MODELS}.")
275
278 - if total_num_hidden_layers % pipeline_parallel_size != 0:
279 - raise ValueError(
280 - f"Total number of hidden layers ({total_num_hidden_layers}) "
281 - "must be divisible by pipeline parallel size "
282 - f"({pipeline_parallel_size}).")
283 -
276 if self.quantization == "bitsandbytes" and (
277 parallel_config.tensor_parallel_size > 1
278 or parallel_config.pipeline_parallel_size > 1):
@@ -385,9 +377,13 @@ class ModelConfig:
377 return num_heads // parallel_config.tensor_parallel_size
378
379 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
380 + from vllm.distributed.utils import get_pp_indices
381 total_num_hidden_layers = getattr(self.hf_text_config,
382 "num_hidden_layers", 0)
390 - return total_num_hidden_layers // parallel_config.pipeline_parallel_size
383 + pp_rank = parallel_config.rank // parallel_config.tensor_parallel_size
384 + pp_size = parallel_config.pipeline_parallel_size
385 + start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size)
386 + return end - start
387
388 def contains_seqlen_agnostic_layers(
389 self, parallel_config: "ParallelConfig") -> bool:
@@ -709,6 +705,7 @@ class ParallelConfig:
705 {"CUDA_VISIBLE_DEVICES": envs.CUDA_VISIBLE_DEVICES})
706
707 self._verify_args()
708 + self.rank = 0
709
710 def _verify_args(self) -> None:
711 if (self.pipeline_parallel_size > 1
vllm/distributed/utils.py
+8 -1
@@ -50,8 +50,15 @@ def split_tensor_along_last_dim(
50
51 def get_pp_indices(num_hidden_layers: int, pp_rank: int,
52 pp_size: int) -> Tuple[int, int]:
53 - layers_per_partition = divide(num_hidden_layers, pp_size)
53 + """Try to evenly distribute layers across partitions.
54 + If the number of layers is not divisible by the number of partitions,
55 + the last partition will have the remaining layers.
56 + """
57 + layers_per_partition = num_hidden_layers // pp_size
58 start_layer = pp_rank * layers_per_partition
59 end_layer = start_layer + layers_per_partition
60
61 + if pp_rank == pp_size - 1:
62 + end_layer = num_hidden_layers
63 +
64 return (start_layer, end_layer)
vllm/worker/openvino_worker.py
+1
@@ -154,6 +154,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
154 ) -> None:
155 self.model_config = model_config
156 self.parallel_config = parallel_config
157 + self.parallel_config.rank = rank
158 self.scheduler_config = scheduler_config
159 self.device_config = device_config
160 self.cache_config = cache_config
vllm/worker/tpu_worker.py
+1
@@ -39,6 +39,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
39 ) -> None:
40 self.model_config = model_config
41 self.parallel_config = parallel_config
42 + self.parallel_config.rank = rank
43 self.scheduler_config = scheduler_config
44 self.device_config = device_config
45 self.cache_config = cache_config
vllm/worker/worker.py
+1
@@ -50,6 +50,7 @@ class Worker(LocalOrDistributedWorkerBase):
50 ) -> None:
51 self.model_config = model_config
52 self.parallel_config = parallel_config
53 + self.parallel_config.rank = rank
54 self.scheduler_config = scheduler_config
55 self.device_config = device_config
56 self.cache_config = cache_config
vllm/worker/xpu_worker.py
+1
@@ -54,6 +54,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
54
55 self.model_config = model_config
56 self.parallel_config = parallel_config
57 + self.parallel_config.rank = rank
58 self.scheduler_config = scheduler_config
59 self.device_config = device_config
60 self.cache_config = cache_config