A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Model] Pipeline parallel support for Mixtral (#6516)
Cody Yu committed
Jul 17, 2024 at 19:26 UTC
b5af8c223c3d70557e7d73ba3c4c2e9d56fc9694
3 files changed
+60
-19
tests/distributed/test_pipeline_parallel.py
+11
-6
@@ -1,4 +1,5 @@
1
import pytest
2
+from transformers import AutoTokenizer
3
4
from ..utils import RemoteOpenAIServer
5
@@ -12,6 +13,8 @@ from ..utils import RemoteOpenAIServer
13
(1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
14
])
15
def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
16
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
17
+
18
pp_args = [
19
# use half precision for speed and memory savings in CI environment
20
"--dtype",
@@ -34,7 +37,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
37
"--dtype",
38
"bfloat16",
39
"--tensor-parallel-size",
37
- str(max(TP_SIZE, 2)), # use at least TP_SIZE=2 to hold the model
40
+ str(max(TP_SIZE, 2)), # We only use 2 GPUs in the CI.
41
"--distributed-executor-backend",
42
"mp",
43
]
@@ -45,8 +48,10 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
48
pp_args.append("--enforce-eager")
49
tp_args.append("--enforce-eager")
50
51
+ prompt = "Hello, my name is"
52
+ token_ids = tokenizer(prompt)["input_ids"]
53
results = []
49
- for args in [pp_args, tp_args]:
54
+ for args in (pp_args, tp_args):
55
with RemoteOpenAIServer(MODEL_NAME, args) as server:
56
client = server.get_client()
57
@@ -62,7 +67,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
67
68
# test with text prompt
69
completion = client.completions.create(model=MODEL_NAME,
65
- prompt="Hello, my name is",
70
+ prompt=prompt,
71
max_tokens=5,
72
temperature=0.0)
73
@@ -76,7 +81,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
81
# test using token IDs
82
completion = client.completions.create(
83
model=MODEL_NAME,
79
- prompt=[0, 0, 0, 0, 0],
84
+ prompt=token_ids,
85
max_tokens=5,
86
temperature=0.0,
87
)
@@ -91,7 +96,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
96
# test simple list
97
batch = client.completions.create(
98
model=MODEL_NAME,
94
- prompt=["Hello, my name is", "Hello, my name is"],
99
+ prompt=[prompt, prompt],
100
max_tokens=5,
101
temperature=0.0,
102
)
@@ -105,7 +110,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
110
# test streaming
111
batch = client.completions.create(
112
model=MODEL_NAME,
108
- prompt=["Hello, my name is", "Hello, my name is"],
113
+ prompt=[prompt, prompt],
114
max_tokens=5,
115
temperature=0.0,
116
stream=True,
vllm/config.py
+1
@@ -34,6 +34,7 @@ _PP_SUPPORTED_MODELS = [
34
"MistralForCausalLM",
35
"Phi3ForCausalLM",
36
"GPT2LMHeadModel",
37
+ "MixtralForCausalLM",
38
]
39
40
vllm/model_executor/models/mixtral.py
+48
-13
@@ -29,7 +29,7 @@ from transformers import MixtralConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import CacheConfig, LoRAConfig
32
-from vllm.distributed import get_tensor_model_parallel_world_size
32
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.fused_moe import FusedMoE
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (QKVParallelLinear,
@@ -48,6 +48,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.sequence import IntermediateTensors, SamplerOutput
49
50
from .interfaces import SupportsLoRA
51
+from .utils import is_pp_missing_parameter, make_layers
52
53
54
class MixtralMoE(nn.Module):
@@ -255,12 +256,11 @@ class MixtralModel(nn.Module):
256
config.hidden_size,
257
org_num_embeddings=config.vocab_size,
258
)
258
- self.layers = nn.ModuleList([
259
- MixtralDecoderLayer(config,
260
- cache_config,
261
- quant_config=quant_config)
262
- for _ in range(config.num_hidden_layers)
263
- ])
259
+
260
+ self.start_layer, self.end_layer, self.layers = make_layers(
261
+ config.num_hidden_layers, lambda: MixtralDecoderLayer(
262
+ config, cache_config, quant_config=quant_config))
263
+
264
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
265
266
def forward(
@@ -269,14 +269,25 @@ class MixtralModel(nn.Module):
269
positions: torch.Tensor,
270
kv_caches: List[torch.Tensor],
271
attn_metadata: AttentionMetadata,
272
+ intermediate_tensors: Optional[IntermediateTensors],
273
) -> torch.Tensor:
273
- hidden_states = self.embed_tokens(input_ids)
274
- residual = None
275
- for i in range(len(self.layers)):
274
+ if get_pp_group().is_first_rank:
275
+ hidden_states = self.embed_tokens(input_ids)
276
+ residual = None
277
+ else:
278
+ assert intermediate_tensors is not None
279
+ hidden_states = intermediate_tensors["hidden_states"]
280
+ residual = intermediate_tensors["residual"]
281
+ for i in range(self.start_layer, self.end_layer):
282
layer = self.layers[i]
283
hidden_states, residual = layer(positions, hidden_states,
278
- kv_caches[i], attn_metadata,
279
- residual)
284
+ kv_caches[i - self.start_layer],
285
+ attn_metadata, residual)
286
+ if not get_pp_group().is_last_rank:
287
+ return IntermediateTensors({
288
+ "hidden_states": hidden_states,
289
+ "residual": residual
290
+ })
291
hidden_states, _ = self.norm(hidden_states, residual)
292
return hidden_states
293
@@ -347,7 +358,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
358
intermediate_tensors: Optional[IntermediateTensors] = None,
359
) -> torch.Tensor:
360
hidden_states = self.model(input_ids, positions, kv_caches,
350
- attn_metadata)
361
+ attn_metadata, intermediate_tensors)
362
return hidden_states
363
364
def compute_logits(self, hidden_states: torch.Tensor,
@@ -356,6 +367,20 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
367
sampling_metadata)
368
return logits
369
370
+ def make_empty_intermediate_tensors(
371
+ self, batch_size: int, dtype: torch.dtype,
372
+ device: torch.device) -> IntermediateTensors:
373
+ return IntermediateTensors({
374
+ "hidden_states":
375
+ torch.zeros((batch_size, self.config.hidden_size),
376
+ dtype=dtype,
377
+ device=device),
378
+ "residual":
379
+ torch.zeros((batch_size, self.config.hidden_size),
380
+ dtype=dtype,
381
+ device=device),
382
+ })
383
+
384
def sample(
385
self,
386
logits: Optional[torch.Tensor],
@@ -392,6 +417,10 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
417
# Skip loading extra bias for GPTQ models.
418
if name.endswith(".bias") and name not in params_dict:
419
continue
420
+ # Skip layers on other devices.
421
+ if is_pp_missing_parameter(name, self):
422
+ continue
423
+
424
param = params_dict[name]
425
weight_loader = param.weight_loader
426
weight_loader(param, loaded_weight, shard_id)
@@ -402,6 +431,9 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
431
if weight_name not in name:
432
continue
433
name = name.replace(weight_name, param_name)
434
+ # Skip layers on other devices.
435
+ if is_pp_missing_parameter(name, self):
436
+ continue
437
param = params_dict[name]
438
weight_loader = param.weight_loader
439
weight_loader(param,
@@ -414,6 +446,9 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
446
# Skip loading extra bias for GPTQ models.
447
if name.endswith(".bias") and name not in params_dict:
448
continue
449
+ # Skip layers on other devices.
450
+ if is_pp_missing_parameter(name, self):
451
+ continue
452
# Remapping the name of FP8 kv-scale.
453
name = maybe_remap_kv_scale_name(name, params_dict)
454
if name is None: