@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5af8c223c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Model] Pipeline parallel support for Mixtral (#6516)

Cody Yu committed Jul 17, 2024 at 19:26 UTC b5af8c223c3d70557e7d73ba3c4c2e9d56fc9694
3 files changed +60 -19
tests/distributed/test_pipeline_parallel.py
+11 -6
@@ -1,4 +1,5 @@
1 import pytest
2 +from transformers import AutoTokenizer
3
4 from ..utils import RemoteOpenAIServer
5
@@ -12,6 +13,8 @@ from ..utils import RemoteOpenAIServer
13 (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
14 ])
15 def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
16 + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
17 +
18 pp_args = [
19 # use half precision for speed and memory savings in CI environment
20 "--dtype",
@@ -34,7 +37,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
37 "--dtype",
38 "bfloat16",
39 "--tensor-parallel-size",
37 - str(max(TP_SIZE, 2)), # use at least TP_SIZE=2 to hold the model
40 + str(max(TP_SIZE, 2)), # We only use 2 GPUs in the CI.
41 "--distributed-executor-backend",
42 "mp",
43 ]
@@ -45,8 +48,10 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
48 pp_args.append("--enforce-eager")
49 tp_args.append("--enforce-eager")
50
51 + prompt = "Hello, my name is"
52 + token_ids = tokenizer(prompt)["input_ids"]
53 results = []
49 - for args in [pp_args, tp_args]:
54 + for args in (pp_args, tp_args):
55 with RemoteOpenAIServer(MODEL_NAME, args) as server:
56 client = server.get_client()
57
@@ -62,7 +67,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
67
68 # test with text prompt
69 completion = client.completions.create(model=MODEL_NAME,
65 - prompt="Hello, my name is",
70 + prompt=prompt,
71 max_tokens=5,
72 temperature=0.0)
73
@@ -76,7 +81,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
81 # test using token IDs
82 completion = client.completions.create(
83 model=MODEL_NAME,
79 - prompt=[0, 0, 0, 0, 0],
84 + prompt=token_ids,
85 max_tokens=5,
86 temperature=0.0,
87 )
@@ -91,7 +96,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
96 # test simple list
97 batch = client.completions.create(
98 model=MODEL_NAME,
94 - prompt=["Hello, my name is", "Hello, my name is"],
99 + prompt=[prompt, prompt],
100 max_tokens=5,
101 temperature=0.0,
102 )
@@ -105,7 +110,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
110 # test streaming
111 batch = client.completions.create(
112 model=MODEL_NAME,
108 - prompt=["Hello, my name is", "Hello, my name is"],
113 + prompt=[prompt, prompt],
114 max_tokens=5,
115 temperature=0.0,
116 stream=True,
vllm/config.py
+1
@@ -34,6 +34,7 @@ _PP_SUPPORTED_MODELS = [
34 "MistralForCausalLM",
35 "Phi3ForCausalLM",
36 "GPT2LMHeadModel",
37 + "MixtralForCausalLM",
38 ]
39
40
vllm/model_executor/models/mixtral.py
+48 -13
@@ -29,7 +29,7 @@ from transformers import MixtralConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import CacheConfig, LoRAConfig
32 -from vllm.distributed import get_tensor_model_parallel_world_size
32 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.fused_moe import FusedMoE
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (QKVParallelLinear,
@@ -48,6 +48,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
48 from vllm.sequence import IntermediateTensors, SamplerOutput
49
50 from .interfaces import SupportsLoRA
51 +from .utils import is_pp_missing_parameter, make_layers
52
53
54 class MixtralMoE(nn.Module):
@@ -255,12 +256,11 @@ class MixtralModel(nn.Module):
256 config.hidden_size,
257 org_num_embeddings=config.vocab_size,
258 )
258 - self.layers = nn.ModuleList([
259 - MixtralDecoderLayer(config,
260 - cache_config,
261 - quant_config=quant_config)
262 - for _ in range(config.num_hidden_layers)
263 - ])
259 +
260 + self.start_layer, self.end_layer, self.layers = make_layers(
261 + config.num_hidden_layers, lambda: MixtralDecoderLayer(
262 + config, cache_config, quant_config=quant_config))
263 +
264 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
265
266 def forward(
@@ -269,14 +269,25 @@ class MixtralModel(nn.Module):
269 positions: torch.Tensor,
270 kv_caches: List[torch.Tensor],
271 attn_metadata: AttentionMetadata,
272 + intermediate_tensors: Optional[IntermediateTensors],
273 ) -> torch.Tensor:
273 - hidden_states = self.embed_tokens(input_ids)
274 - residual = None
275 - for i in range(len(self.layers)):
274 + if get_pp_group().is_first_rank:
275 + hidden_states = self.embed_tokens(input_ids)
276 + residual = None
277 + else:
278 + assert intermediate_tensors is not None
279 + hidden_states = intermediate_tensors["hidden_states"]
280 + residual = intermediate_tensors["residual"]
281 + for i in range(self.start_layer, self.end_layer):
282 layer = self.layers[i]
283 hidden_states, residual = layer(positions, hidden_states,
278 - kv_caches[i], attn_metadata,
279 - residual)
284 + kv_caches[i - self.start_layer],
285 + attn_metadata, residual)
286 + if not get_pp_group().is_last_rank:
287 + return IntermediateTensors({
288 + "hidden_states": hidden_states,
289 + "residual": residual
290 + })
291 hidden_states, _ = self.norm(hidden_states, residual)
292 return hidden_states
293
@@ -347,7 +358,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
358 intermediate_tensors: Optional[IntermediateTensors] = None,
359 ) -> torch.Tensor:
360 hidden_states = self.model(input_ids, positions, kv_caches,
350 - attn_metadata)
361 + attn_metadata, intermediate_tensors)
362 return hidden_states
363
364 def compute_logits(self, hidden_states: torch.Tensor,
@@ -356,6 +367,20 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
367 sampling_metadata)
368 return logits
369
370 + def make_empty_intermediate_tensors(
371 + self, batch_size: int, dtype: torch.dtype,
372 + device: torch.device) -> IntermediateTensors:
373 + return IntermediateTensors({
374 + "hidden_states":
375 + torch.zeros((batch_size, self.config.hidden_size),
376 + dtype=dtype,
377 + device=device),
378 + "residual":
379 + torch.zeros((batch_size, self.config.hidden_size),
380 + dtype=dtype,
381 + device=device),
382 + })
383 +
384 def sample(
385 self,
386 logits: Optional[torch.Tensor],
@@ -392,6 +417,10 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
417 # Skip loading extra bias for GPTQ models.
418 if name.endswith(".bias") and name not in params_dict:
419 continue
420 + # Skip layers on other devices.
421 + if is_pp_missing_parameter(name, self):
422 + continue
423 +
424 param = params_dict[name]
425 weight_loader = param.weight_loader
426 weight_loader(param, loaded_weight, shard_id)
@@ -402,6 +431,9 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
431 if weight_name not in name:
432 continue
433 name = name.replace(weight_name, param_name)
434 + # Skip layers on other devices.
435 + if is_pp_missing_parameter(name, self):
436 + continue
437 param = params_dict[name]
438 weight_loader = param.weight_loader
439 weight_loader(param,
@@ -414,6 +446,9 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
446 # Skip loading extra bias for GPTQ models.
447 if name.endswith(".bias") and name not in params_dict:
448 continue
449 + # Skip layers on other devices.
450 + if is_pp_missing_parameter(name, self):
451 + continue
452 # Remapping the name of FP8 kv-scale.
453 name = maybe_remap_kv_scale_name(name, params_dict)
454 if name is None: