A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[MISC] Introduce pipeline parallelism partition strategies (#6920)
Co-authored-by: youkaichao <youkaichao@126.com>
Cody Yu committed
Jul 31, 2024 at 12:02 UTC
bd700134072d9513902b42f3ef20a7cd8a1c6377
3 files changed
+66
-5
tests/distributed/test_pipeline_partition.py
new
+34
@@ -0,0 +1,34 @@
1
+import os
2
+
3
+import pytest
4
+
5
+from vllm.distributed.utils import get_pp_indices
6
+
7
+
8
+def test_custom_layer_partition():
9
+
10
+ def _verify(partition_str, num_layers, pp_size, goldens):
11
+ bak = os.environ.get("VLLM_PP_LAYER_PARTITION", None)
12
+ os.environ["VLLM_PP_LAYER_PARTITION"] = partition_str
13
+ for pp_rank, golden in enumerate(goldens):
14
+ assert get_pp_indices(num_layers, pp_rank, pp_size) == golden
15
+ if bak is not None:
16
+ os.environ["VLLM_PP_LAYER_PARTITION"] = bak
17
+
18
+ # Even partition
19
+ _verify("5,5,5,5", 20, 4, [(0, 5), (5, 10), (10, 15), (15, 20)])
20
+ # Balanced partition
21
+ _verify("4,6,6,4", 20, 4, [(0, 4), (4, 10), (10, 16), (16, 20)])
22
+ # Put reminder somewhere
23
+ _verify("5,6,5,6", 22, 4, [(0, 5), (5, 11), (11, 16), (16, 22)])
24
+ # Invalid partition strings
25
+ with pytest.raises(ValueError):
26
+ _verify("5,5,5,5,", 20, 4, [(0, 5), (5, 10), (10, 15), (15, 20)])
27
+ with pytest.raises(ValueError):
28
+ _verify("5,5,5,a", 20, 4, [(0, 5), (5, 10), (10, 15), (15, 20)])
29
+ # Wrong number of partitions
30
+ with pytest.raises(ValueError):
31
+ _verify("5,5,5", 20, 4, [(0, 5), (5, 10), (10, 15), (15, 20)])
32
+ # Wrong number of layers
33
+ with pytest.raises(ValueError):
34
+ _verify("5,5,5,5", 21, 4, [(0, 5), (5, 10), (10, 15), (15, 20)])
vllm/distributed/utils.py
+27
-5
@@ -6,6 +6,11 @@ from typing import Sequence, Tuple
6
7
import torch
8
9
+import vllm.envs as envs
10
+from vllm.logger import init_logger
11
+
12
+logger = init_logger(__name__)
13
+
14
15
def ensure_divisibility(numerator, denominator):
16
"""Ensure that numerator is divisible by the denominator."""
@@ -54,11 +59,28 @@ def get_pp_indices(num_hidden_layers: int, pp_rank: int,
59
If the number of layers is not divisible by the number of partitions,
60
the last partition will have the remaining layers.
61
"""
57
- layers_per_partition = num_hidden_layers // pp_size
58
- start_layer = pp_rank * layers_per_partition
59
- end_layer = start_layer + layers_per_partition
62
+ partition_list_str = envs.VLLM_PP_LAYER_PARTITION
63
+ if partition_list_str is not None:
64
+ try:
65
+ partitions = [
66
+ int(layer) for layer in partition_list_str.split(",")
67
+ ]
68
+ except ValueError as err:
69
+ raise ValueError("Invalid partition string: {}".format(
70
+ partition_list_str)) from err
71
+ if len(partitions) != pp_size:
72
+ raise ValueError(f"{len(partitions)=} does not match {pp_size=}.")
73
+ if sum(partitions) != num_hidden_layers:
74
+ raise ValueError(
75
+ f"{sum(partitions)=} does not match {num_hidden_layers=}.")
76
+ start_layer = sum(partitions[:pp_rank])
77
+ end_layer = start_layer + partitions[pp_rank]
78
+ else:
79
+ layers_per_partition = num_hidden_layers // pp_size
80
+ start_layer = pp_rank * layers_per_partition
81
+ end_layer = start_layer + layers_per_partition
82
61
- if pp_rank == pp_size - 1:
62
- end_layer = num_hidden_layers
83
+ if pp_rank == pp_size - 1:
84
+ end_layer = num_hidden_layers
85
86
return (start_layer, end_layer)
vllm/envs.py
+5
@@ -28,6 +28,7 @@ if TYPE_CHECKING:
28
VLLM_LOGGING_CONFIG_PATH: Optional[str] = None
29
VLLM_TRACE_FUNCTION: int = 0
30
VLLM_ATTENTION_BACKEND: Optional[str] = None
31
+ VLLM_PP_LAYER_PARTITION: Optional[str] = None
32
VLLM_CPU_KVCACHE_SPACE: int = 0
33
VLLM_CPU_OMP_THREADS_BIND: str = ""
34
VLLM_OPENVINO_KVCACHE_SPACE: int = 0
@@ -242,6 +243,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
243
"VLLM_ATTENTION_BACKEND":
244
lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
245
246
+ # Pipeline stage partition strategy
247
+ "VLLM_PP_LAYER_PARTITION":
248
+ lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),
249
+
250
# (CPU backend only) CPU key-value cache space.
251
# default is 4GB
252
"VLLM_CPU_KVCACHE_SPACE":