A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[core][distributed] support n layers % pp size != 0 (#6115)
youkaichao committed
Jul 3, 2024 at 16:40 UTC
3de6e6a30e33406c2b3bb81fb5a82a2966cebd87
7 files changed
+19
-10
.buildkite/test-pipeline.yaml
+1
@@ -80,6 +80,7 @@ steps:
80
commands:
81
- TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
82
- TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
83
+ - TP_SIZE=1 PP_SIZE=3 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
84
- PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
85
- PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
86
vllm/config.py
+6
-9
@@ -265,8 +265,6 @@ class ModelConfig:
265
" must be divisible by tensor parallel size "
266
f"({tensor_parallel_size}).")
267
268
- total_num_hidden_layers = getattr(self.hf_text_config,
269
- "num_hidden_layers", 0)
268
pipeline_parallel_size = parallel_config.pipeline_parallel_size
269
architectures = getattr(self.hf_config, "architectures", [])
270
if not all(arch in _PP_SUPPORTED_MODELS
@@ -275,12 +273,6 @@ class ModelConfig:
273
"Pipeline parallelism is only supported for the following "
274
f" architectures: {_PP_SUPPORTED_MODELS}.")
275
278
- if total_num_hidden_layers % pipeline_parallel_size != 0:
279
- raise ValueError(
280
- f"Total number of hidden layers ({total_num_hidden_layers}) "
281
- "must be divisible by pipeline parallel size "
282
- f"({pipeline_parallel_size}).")
283
-
276
if self.quantization == "bitsandbytes" and (
277
parallel_config.tensor_parallel_size > 1
278
or parallel_config.pipeline_parallel_size > 1):
@@ -385,9 +377,13 @@ class ModelConfig:
377
return num_heads // parallel_config.tensor_parallel_size
378
379
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
380
+ from vllm.distributed.utils import get_pp_indices
381
total_num_hidden_layers = getattr(self.hf_text_config,
382
"num_hidden_layers", 0)
390
- return total_num_hidden_layers // parallel_config.pipeline_parallel_size
383
+ pp_rank = parallel_config.rank // parallel_config.tensor_parallel_size
384
+ pp_size = parallel_config.pipeline_parallel_size
385
+ start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size)
386
+ return end - start
387
388
def contains_seqlen_agnostic_layers(
389
self, parallel_config: "ParallelConfig") -> bool:
@@ -709,6 +705,7 @@ class ParallelConfig:
705
{"CUDA_VISIBLE_DEVICES": envs.CUDA_VISIBLE_DEVICES})
706
707
self._verify_args()
708
+ self.rank = 0
709
710
def _verify_args(self) -> None:
711
if (self.pipeline_parallel_size > 1
vllm/distributed/utils.py
+8
-1
@@ -50,8 +50,15 @@ def split_tensor_along_last_dim(
50
51
def get_pp_indices(num_hidden_layers: int, pp_rank: int,
52
pp_size: int) -> Tuple[int, int]:
53
- layers_per_partition = divide(num_hidden_layers, pp_size)
53
+ """Try to evenly distribute layers across partitions.
54
+ If the number of layers is not divisible by the number of partitions,
55
+ the last partition will have the remaining layers.
56
+ """
57
+ layers_per_partition = num_hidden_layers // pp_size
58
start_layer = pp_rank * layers_per_partition
59
end_layer = start_layer + layers_per_partition
60
61
+ if pp_rank == pp_size - 1:
62
+ end_layer = num_hidden_layers
63
+
64
return (start_layer, end_layer)
vllm/worker/openvino_worker.py
+1
@@ -154,6 +154,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
154
) -> None:
155
self.model_config = model_config
156
self.parallel_config = parallel_config
157
+ self.parallel_config.rank = rank
158
self.scheduler_config = scheduler_config
159
self.device_config = device_config
160
self.cache_config = cache_config
vllm/worker/tpu_worker.py
+1
@@ -39,6 +39,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
39
) -> None:
40
self.model_config = model_config
41
self.parallel_config = parallel_config
42
+ self.parallel_config.rank = rank
43
self.scheduler_config = scheduler_config
44
self.device_config = device_config
45
self.cache_config = cache_config
vllm/worker/worker.py
+1
@@ -50,6 +50,7 @@ class Worker(LocalOrDistributedWorkerBase):
50
) -> None:
51
self.model_config = model_config
52
self.parallel_config = parallel_config
53
+ self.parallel_config.rank = rank
54
self.scheduler_config = scheduler_config
55
self.device_config = device_config
56
self.cache_config = cache_config
vllm/worker/xpu_worker.py
+1
@@ -54,6 +54,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
54
55
self.model_config = model_config
56
self.parallel_config = parallel_config
57
+ self.parallel_config.rank = rank
58
self.scheduler_config = scheduler_config
59
self.device_config = device_config
60
self.cache_config = cache_config