A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Model] Pipeline Parallel Support for DeepSeek v2 (#6519)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Jul 23, 2024 at 13:22 UTC
507ef787d85dec24490069ffceacbd6b161f4f72
2 files changed
+115
-39
vllm/config.py
+1
@@ -31,6 +31,7 @@ _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
31
_PP_SUPPORTED_MODELS = [
32
"AquilaModel",
33
"AquilaForCausalLM",
34
+ "DeepseekV2ForCausalLM",
35
"InternLMForCausalLM",
36
"LlamaForCausalLM",
37
"LLaMAForCausalLM",
vllm/model_executor/models/deepseek_v2.py
+114
-39
@@ -29,7 +29,8 @@ from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import CacheConfig
32
-from vllm.distributed import (get_tensor_model_parallel_world_size,
32
+from vllm.distributed import (get_pp_group,
33
+ get_tensor_model_parallel_world_size,
34
tensor_model_parallel_all_reduce)
35
from vllm.model_executor.layers.activation import SiluAndMul
36
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -49,6 +50,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
51
from vllm.sequence import IntermediateTensors, SamplerOutput
52
53
+from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
54
+
55
56
class DeepseekV2MLP(nn.Module):
57
@@ -59,17 +62,20 @@ class DeepseekV2MLP(nn.Module):
62
hidden_act: str,
63
quant_config: Optional[QuantizationConfig] = None,
64
reduce_results: bool = True,
65
+ prefix: str = "",
66
) -> None:
67
super().__init__()
68
self.gate_up_proj = MergedColumnParallelLinear(
69
hidden_size, [intermediate_size] * 2,
70
bias=False,
67
- quant_config=quant_config)
71
+ quant_config=quant_config,
72
+ prefix=f"{prefix}.gate_up_proj")
73
self.down_proj = RowParallelLinear(intermediate_size,
74
hidden_size,
75
bias=False,
76
quant_config=quant_config,
72
- reduce_results=reduce_results)
77
+ reduce_results=reduce_results,
78
+ prefix=f"{prefix}.down_proj")
79
if hidden_act != "silu":
80
raise ValueError(f"Unsupported activation: {hidden_act}. "
81
"Only silu is supported for now.")
@@ -88,6 +94,7 @@ class DeepseekV2MoE(nn.Module):
94
self,
95
config: PretrainedConfig,
96
quant_config: Optional[QuantizationConfig] = None,
97
+ prefix: str = "",
98
):
99
super().__init__()
100
self.tp_size = get_tensor_model_parallel_world_size()
@@ -112,12 +119,14 @@ class DeepseekV2MoE(nn.Module):
119
quant_config=quant_config,
120
use_grouped_topk=True,
121
num_expert_group=config.n_group,
115
- topk_group=config.topk_group)
122
+ topk_group=config.topk_group,
123
+ prefix=f"{prefix}.experts")
124
125
self.gate = ReplicatedLinear(config.hidden_size,
126
config.n_routed_experts,
127
bias=False,
120
- quant_config=None)
128
+ quant_config=None,
129
+ prefix=f"{prefix}.gate")
130
if config.n_shared_experts is not None:
131
intermediate_size = (config.moe_intermediate_size *
132
config.n_shared_experts)
@@ -172,10 +181,9 @@ class DeepseekV2Attention(nn.Module):
181
max_position_embeddings: int = 8192,
182
cache_config: Optional[CacheConfig] = None,
183
quant_config: Optional[QuantizationConfig] = None,
175
- layer_idx=None,
184
+ prefix: str = "",
185
) -> None:
186
super().__init__()
178
- self.layer_idx = layer_idx
187
self.hidden_size = hidden_size
188
self.qk_nope_head_dim = qk_nope_head_dim
189
self.qk_rope_head_dim = qk_rope_head_dim
@@ -195,38 +203,44 @@ class DeepseekV2Attention(nn.Module):
203
self.q_a_proj = ReplicatedLinear(self.hidden_size,
204
self.q_lora_rank,
205
bias=False,
198
- quant_config=quant_config)
206
+ quant_config=quant_config,
207
+ prefix=f"{prefix}.q_a_proj")
208
self.q_a_layernorm = RMSNorm(self.q_lora_rank,
209
eps=config.rms_norm_eps)
210
self.q_b_proj = ColumnParallelLinear(q_lora_rank,
211
self.num_heads *
212
self.qk_head_dim,
213
bias=False,
205
- quant_config=quant_config)
214
+ quant_config=quant_config,
215
+ prefix=f"{prefix}.q_b_proj")
216
else:
217
self.q_proj = ColumnParallelLinear(self.hidden_size,
218
self.num_heads *
219
self.qk_head_dim,
220
bias=False,
211
- quant_config=quant_config)
221
+ quant_config=quant_config,
222
+ prefix=f"{prefix}.q_proj")
223
213
- self.kv_a_proj_with_mqa = ReplicatedLinear(self.hidden_size,
214
- self.kv_lora_rank +
215
- self.qk_rope_head_dim,
216
- bias=False,
217
- quant_config=quant_config)
224
+ self.kv_a_proj_with_mqa = ReplicatedLinear(
225
+ self.hidden_size,
226
+ self.kv_lora_rank + self.qk_rope_head_dim,
227
+ bias=False,
228
+ quant_config=quant_config,
229
+ prefix=f"{prefix}.kv_a_proj_with_mqa")
230
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
231
eps=config.rms_norm_eps)
232
self.kv_b_proj = ColumnParallelLinear(
233
self.kv_lora_rank,
234
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
235
bias=False,
224
- quant_config=quant_config)
236
+ quant_config=quant_config,
237
+ prefix=f"{prefix}.kv_b_proj")
238
# O projection.
239
self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
240
self.hidden_size,
241
bias=False,
229
- quant_config=quant_config)
242
+ quant_config=quant_config,
243
+ prefix=f"{prefix}.o_proj")
244
rope_scaling['type'] = 'deepseek_yarn'
245
self.rotary_emb = get_rope(qk_rope_head_dim,
246
rotary_dim=qk_rope_head_dim,
@@ -308,7 +322,7 @@ class DeepseekV2DecoderLayer(nn.Module):
322
def __init__(
323
self,
324
config: PretrainedConfig,
311
- layer_idx: int,
325
+ prefix: str,
326
cache_config: Optional[CacheConfig] = None,
327
quant_config: Optional[QuantizationConfig] = None,
328
) -> None:
@@ -318,6 +332,9 @@ class DeepseekV2DecoderLayer(nn.Module):
332
rope_scaling = getattr(config, "rope_scaling", None)
333
max_position_embeddings = getattr(config, "max_position_embeddings",
334
8192)
335
+ # DecoderLayers are created with `make_layers` which passes the prefix
336
+ # with the layer's index.
337
+ layer_idx = int(prefix.split(sep='.')[-1])
338
self.self_attn = DeepseekV2Attention(
339
config=config,
340
hidden_size=self.hidden_size,
@@ -333,18 +350,23 @@ class DeepseekV2DecoderLayer(nn.Module):
350
max_position_embeddings=max_position_embeddings,
351
cache_config=cache_config,
352
quant_config=quant_config,
336
- layer_idx=layer_idx,
353
+ prefix=f"{prefix}.self_attn",
354
)
355
if (config.n_routed_experts is not None
356
and layer_idx >= config.first_k_dense_replace
357
and layer_idx % config.moe_layer_freq == 0):
341
- self.mlp = DeepseekV2MoE(config=config, quant_config=quant_config)
358
+ self.mlp = DeepseekV2MoE(
359
+ config=config,
360
+ quant_config=quant_config,
361
+ prefix=f"{prefix}.mlp",
362
+ )
363
else:
364
self.mlp = DeepseekV2MLP(
365
hidden_size=config.hidden_size,
366
intermediate_size=config.intermediate_size,
367
hidden_act=config.hidden_act,
368
quant_config=quant_config,
369
+ prefix=f"{prefix}.mlp",
370
)
371
self.input_layernorm = RMSNorm(config.hidden_size,
372
eps=config.rms_norm_eps)
@@ -389,23 +411,34 @@ class DeepseekV2Model(nn.Module):
411
config: PretrainedConfig,
412
cache_config: Optional[CacheConfig] = None,
413
quant_config: Optional[QuantizationConfig] = None,
414
+ prefix: str = "",
415
) -> None:
416
super().__init__()
417
self.padding_idx = config.pad_token_id
418
self.vocab_size = config.vocab_size
419
397
- self.embed_tokens = VocabParallelEmbedding(
398
- config.vocab_size,
399
- config.hidden_size,
400
- )
401
- self.layers = nn.ModuleList([
402
- DeepseekV2DecoderLayer(config,
403
- layer_idx,
404
- cache_config=cache_config,
405
- quant_config=quant_config)
406
- for layer_idx in range(config.num_hidden_layers)
407
- ])
408
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
420
+ if get_pp_group().is_first_rank:
421
+ self.embed_tokens = VocabParallelEmbedding(
422
+ config.vocab_size,
423
+ config.hidden_size,
424
+ )
425
+ else:
426
+ self.embed_tokens = PPMissingLayer()
427
+
428
+ self.start_layer, self.end_layer, self.layers = make_layers(
429
+ config.num_hidden_layers,
430
+ lambda prefix: DeepseekV2DecoderLayer(
431
+ config,
432
+ prefix,
433
+ cache_config=cache_config,
434
+ quant_config=quant_config,
435
+ ),
436
+ prefix=f"{prefix}.layers")
437
+
438
+ if get_pp_group().is_last_rank:
439
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
440
+ else:
441
+ self.norm = PPMissingLayer()
442
443
def forward(
444
self,
@@ -413,14 +446,28 @@ class DeepseekV2Model(nn.Module):
446
positions: torch.Tensor,
447
kv_caches: List[torch.Tensor],
448
attn_metadata: AttentionMetadata,
449
+ intermediate_tensors: Optional[IntermediateTensors],
450
) -> torch.Tensor:
417
- hidden_states = self.embed_tokens(input_ids)
418
- residual = None
419
- for i in range(len(self.layers)):
451
+ if get_pp_group().is_first_rank:
452
+ hidden_states = self.embed_tokens(input_ids)
453
+ residual = None
454
+ else:
455
+ assert intermediate_tensors is not None
456
+ hidden_states = intermediate_tensors["hidden_states"]
457
+ residual = intermediate_tensors["residual"]
458
+
459
+ for i in range(self.start_layer, self.end_layer):
460
layer = self.layers[i]
461
hidden_states, residual = layer(positions, hidden_states,
422
- kv_caches[i], attn_metadata,
423
- residual)
462
+ kv_caches[i - self.start_layer],
463
+ attn_metadata, residual)
464
+
465
+ if not get_pp_group().is_last_rank:
466
+ return IntermediateTensors({
467
+ "hidden_states": hidden_states,
468
+ "residual": residual
469
+ })
470
+
471
hidden_states, _ = self.norm(hidden_states, residual)
472
return hidden_states
473
@@ -436,7 +483,10 @@ class DeepseekV2ForCausalLM(nn.Module):
483
super().__init__()
484
self.config = config
485
self.quant_config = quant_config
439
- self.model = DeepseekV2Model(config, cache_config, quant_config)
486
+ self.model = DeepseekV2Model(config,
487
+ cache_config,
488
+ quant_config,
489
+ prefix="model")
490
self.lm_head = ParallelLMHead(config.vocab_size,
491
config.hidden_size,
492
quant_config=quant_config)
@@ -452,7 +502,7 @@ class DeepseekV2ForCausalLM(nn.Module):
502
intermediate_tensors: Optional[IntermediateTensors] = None,
503
) -> torch.Tensor:
504
hidden_states = self.model(input_ids, positions, kv_caches,
455
- attn_metadata)
505
+ attn_metadata, intermediate_tensors)
506
return hidden_states
507
508
def compute_logits(self, hidden_states: torch.Tensor,
@@ -469,6 +519,20 @@ class DeepseekV2ForCausalLM(nn.Module):
519
next_tokens = self.sampler(logits, sampling_metadata)
520
return next_tokens
521
522
+ def make_empty_intermediate_tensors(
523
+ self, batch_size: int, dtype: torch.dtype,
524
+ device: torch.device) -> IntermediateTensors:
525
+ return IntermediateTensors({
526
+ "hidden_states":
527
+ torch.zeros((batch_size, self.config.hidden_size),
528
+ dtype=dtype,
529
+ device=device),
530
+ "residual":
531
+ torch.zeros((batch_size, self.config.hidden_size),
532
+ dtype=dtype,
533
+ device=device),
534
+ })
535
+
536
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
537
stacked_params_mapping = [
538
# (param_name, shard_name, shard_id)
@@ -504,6 +568,10 @@ class DeepseekV2ForCausalLM(nn.Module):
568
# Skip loading extra bias for GPTQ models.
569
if name.endswith(".bias") and name not in params_dict:
570
continue
571
+
572
+ if is_pp_missing_parameter(name, self):
573
+ continue
574
+
575
param = params_dict[name]
576
weight_loader = param.weight_loader
577
weight_loader(param, loaded_weight, shard_id)
@@ -514,6 +582,10 @@ class DeepseekV2ForCausalLM(nn.Module):
582
if weight_name not in name:
583
continue
584
name = name.replace(weight_name, param_name)
585
+
586
+ if is_pp_missing_parameter(name, self):
587
+ continue
588
+
589
param = params_dict[name]
590
weight_loader = param.weight_loader
591
weight_loader(param,
@@ -527,6 +599,9 @@ class DeepseekV2ForCausalLM(nn.Module):
599
if name.endswith(".bias") and name not in params_dict:
600
continue
601
602
+ if is_pp_missing_parameter(name, self):
603
+ continue
604
+
605
param = params_dict[name]
606
weight_loader = getattr(param, "weight_loader",
607
default_weight_loader)