@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 507ef787d8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Model] Pipeline Parallel Support for DeepSeek v2 (#6519)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Jul 23, 2024 at 13:22 UTC 507ef787d85dec24490069ffceacbd6b161f4f72
2 files changed +115 -39
vllm/config.py
+1
@@ -31,6 +31,7 @@ _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
31 _PP_SUPPORTED_MODELS = [
32 "AquilaModel",
33 "AquilaForCausalLM",
34 + "DeepseekV2ForCausalLM",
35 "InternLMForCausalLM",
36 "LlamaForCausalLM",
37 "LLaMAForCausalLM",
vllm/model_executor/models/deepseek_v2.py
+114 -39
@@ -29,7 +29,8 @@ from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import CacheConfig
32 -from vllm.distributed import (get_tensor_model_parallel_world_size,
32 +from vllm.distributed import (get_pp_group,
33 + get_tensor_model_parallel_world_size,
34 tensor_model_parallel_all_reduce)
35 from vllm.model_executor.layers.activation import SiluAndMul
36 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -49,6 +50,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 from vllm.model_executor.sampling_metadata import SamplingMetadata
51 from vllm.sequence import IntermediateTensors, SamplerOutput
52
53 +from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
54 +
55
56 class DeepseekV2MLP(nn.Module):
57
@@ -59,17 +62,20 @@ class DeepseekV2MLP(nn.Module):
62 hidden_act: str,
63 quant_config: Optional[QuantizationConfig] = None,
64 reduce_results: bool = True,
65 + prefix: str = "",
66 ) -> None:
67 super().__init__()
68 self.gate_up_proj = MergedColumnParallelLinear(
69 hidden_size, [intermediate_size] * 2,
70 bias=False,
67 - quant_config=quant_config)
71 + quant_config=quant_config,
72 + prefix=f"{prefix}.gate_up_proj")
73 self.down_proj = RowParallelLinear(intermediate_size,
74 hidden_size,
75 bias=False,
76 quant_config=quant_config,
72 - reduce_results=reduce_results)
77 + reduce_results=reduce_results,
78 + prefix=f"{prefix}.down_proj")
79 if hidden_act != "silu":
80 raise ValueError(f"Unsupported activation: {hidden_act}. "
81 "Only silu is supported for now.")
@@ -88,6 +94,7 @@ class DeepseekV2MoE(nn.Module):
94 self,
95 config: PretrainedConfig,
96 quant_config: Optional[QuantizationConfig] = None,
97 + prefix: str = "",
98 ):
99 super().__init__()
100 self.tp_size = get_tensor_model_parallel_world_size()
@@ -112,12 +119,14 @@ class DeepseekV2MoE(nn.Module):
119 quant_config=quant_config,
120 use_grouped_topk=True,
121 num_expert_group=config.n_group,
115 - topk_group=config.topk_group)
122 + topk_group=config.topk_group,
123 + prefix=f"{prefix}.experts")
124
125 self.gate = ReplicatedLinear(config.hidden_size,
126 config.n_routed_experts,
127 bias=False,
120 - quant_config=None)
128 + quant_config=None,
129 + prefix=f"{prefix}.gate")
130 if config.n_shared_experts is not None:
131 intermediate_size = (config.moe_intermediate_size *
132 config.n_shared_experts)
@@ -172,10 +181,9 @@ class DeepseekV2Attention(nn.Module):
181 max_position_embeddings: int = 8192,
182 cache_config: Optional[CacheConfig] = None,
183 quant_config: Optional[QuantizationConfig] = None,
175 - layer_idx=None,
184 + prefix: str = "",
185 ) -> None:
186 super().__init__()
178 - self.layer_idx = layer_idx
187 self.hidden_size = hidden_size
188 self.qk_nope_head_dim = qk_nope_head_dim
189 self.qk_rope_head_dim = qk_rope_head_dim
@@ -195,38 +203,44 @@ class DeepseekV2Attention(nn.Module):
203 self.q_a_proj = ReplicatedLinear(self.hidden_size,
204 self.q_lora_rank,
205 bias=False,
198 - quant_config=quant_config)
206 + quant_config=quant_config,
207 + prefix=f"{prefix}.q_a_proj")
208 self.q_a_layernorm = RMSNorm(self.q_lora_rank,
209 eps=config.rms_norm_eps)
210 self.q_b_proj = ColumnParallelLinear(q_lora_rank,
211 self.num_heads *
212 self.qk_head_dim,
213 bias=False,
205 - quant_config=quant_config)
214 + quant_config=quant_config,
215 + prefix=f"{prefix}.q_b_proj")
216 else:
217 self.q_proj = ColumnParallelLinear(self.hidden_size,
218 self.num_heads *
219 self.qk_head_dim,
220 bias=False,
211 - quant_config=quant_config)
221 + quant_config=quant_config,
222 + prefix=f"{prefix}.q_proj")
223
213 - self.kv_a_proj_with_mqa = ReplicatedLinear(self.hidden_size,
214 - self.kv_lora_rank +
215 - self.qk_rope_head_dim,
216 - bias=False,
217 - quant_config=quant_config)
224 + self.kv_a_proj_with_mqa = ReplicatedLinear(
225 + self.hidden_size,
226 + self.kv_lora_rank + self.qk_rope_head_dim,
227 + bias=False,
228 + quant_config=quant_config,
229 + prefix=f"{prefix}.kv_a_proj_with_mqa")
230 self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
231 eps=config.rms_norm_eps)
232 self.kv_b_proj = ColumnParallelLinear(
233 self.kv_lora_rank,
234 self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
235 bias=False,
224 - quant_config=quant_config)
236 + quant_config=quant_config,
237 + prefix=f"{prefix}.kv_b_proj")
238 # O projection.
239 self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
240 self.hidden_size,
241 bias=False,
229 - quant_config=quant_config)
242 + quant_config=quant_config,
243 + prefix=f"{prefix}.o_proj")
244 rope_scaling['type'] = 'deepseek_yarn'
245 self.rotary_emb = get_rope(qk_rope_head_dim,
246 rotary_dim=qk_rope_head_dim,
@@ -308,7 +322,7 @@ class DeepseekV2DecoderLayer(nn.Module):
322 def __init__(
323 self,
324 config: PretrainedConfig,
311 - layer_idx: int,
325 + prefix: str,
326 cache_config: Optional[CacheConfig] = None,
327 quant_config: Optional[QuantizationConfig] = None,
328 ) -> None:
@@ -318,6 +332,9 @@ class DeepseekV2DecoderLayer(nn.Module):
332 rope_scaling = getattr(config, "rope_scaling", None)
333 max_position_embeddings = getattr(config, "max_position_embeddings",
334 8192)
335 + # DecoderLayers are created with `make_layers` which passes the prefix
336 + # with the layer's index.
337 + layer_idx = int(prefix.split(sep='.')[-1])
338 self.self_attn = DeepseekV2Attention(
339 config=config,
340 hidden_size=self.hidden_size,
@@ -333,18 +350,23 @@ class DeepseekV2DecoderLayer(nn.Module):
350 max_position_embeddings=max_position_embeddings,
351 cache_config=cache_config,
352 quant_config=quant_config,
336 - layer_idx=layer_idx,
353 + prefix=f"{prefix}.self_attn",
354 )
355 if (config.n_routed_experts is not None
356 and layer_idx >= config.first_k_dense_replace
357 and layer_idx % config.moe_layer_freq == 0):
341 - self.mlp = DeepseekV2MoE(config=config, quant_config=quant_config)
358 + self.mlp = DeepseekV2MoE(
359 + config=config,
360 + quant_config=quant_config,
361 + prefix=f"{prefix}.mlp",
362 + )
363 else:
364 self.mlp = DeepseekV2MLP(
365 hidden_size=config.hidden_size,
366 intermediate_size=config.intermediate_size,
367 hidden_act=config.hidden_act,
368 quant_config=quant_config,
369 + prefix=f"{prefix}.mlp",
370 )
371 self.input_layernorm = RMSNorm(config.hidden_size,
372 eps=config.rms_norm_eps)
@@ -389,23 +411,34 @@ class DeepseekV2Model(nn.Module):
411 config: PretrainedConfig,
412 cache_config: Optional[CacheConfig] = None,
413 quant_config: Optional[QuantizationConfig] = None,
414 + prefix: str = "",
415 ) -> None:
416 super().__init__()
417 self.padding_idx = config.pad_token_id
418 self.vocab_size = config.vocab_size
419
397 - self.embed_tokens = VocabParallelEmbedding(
398 - config.vocab_size,
399 - config.hidden_size,
400 - )
401 - self.layers = nn.ModuleList([
402 - DeepseekV2DecoderLayer(config,
403 - layer_idx,
404 - cache_config=cache_config,
405 - quant_config=quant_config)
406 - for layer_idx in range(config.num_hidden_layers)
407 - ])
408 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
420 + if get_pp_group().is_first_rank:
421 + self.embed_tokens = VocabParallelEmbedding(
422 + config.vocab_size,
423 + config.hidden_size,
424 + )
425 + else:
426 + self.embed_tokens = PPMissingLayer()
427 +
428 + self.start_layer, self.end_layer, self.layers = make_layers(
429 + config.num_hidden_layers,
430 + lambda prefix: DeepseekV2DecoderLayer(
431 + config,
432 + prefix,
433 + cache_config=cache_config,
434 + quant_config=quant_config,
435 + ),
436 + prefix=f"{prefix}.layers")
437 +
438 + if get_pp_group().is_last_rank:
439 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
440 + else:
441 + self.norm = PPMissingLayer()
442
443 def forward(
444 self,
@@ -413,14 +446,28 @@ class DeepseekV2Model(nn.Module):
446 positions: torch.Tensor,
447 kv_caches: List[torch.Tensor],
448 attn_metadata: AttentionMetadata,
449 + intermediate_tensors: Optional[IntermediateTensors],
450 ) -> torch.Tensor:
417 - hidden_states = self.embed_tokens(input_ids)
418 - residual = None
419 - for i in range(len(self.layers)):
451 + if get_pp_group().is_first_rank:
452 + hidden_states = self.embed_tokens(input_ids)
453 + residual = None
454 + else:
455 + assert intermediate_tensors is not None
456 + hidden_states = intermediate_tensors["hidden_states"]
457 + residual = intermediate_tensors["residual"]
458 +
459 + for i in range(self.start_layer, self.end_layer):
460 layer = self.layers[i]
461 hidden_states, residual = layer(positions, hidden_states,
422 - kv_caches[i], attn_metadata,
423 - residual)
462 + kv_caches[i - self.start_layer],
463 + attn_metadata, residual)
464 +
465 + if not get_pp_group().is_last_rank:
466 + return IntermediateTensors({
467 + "hidden_states": hidden_states,
468 + "residual": residual
469 + })
470 +
471 hidden_states, _ = self.norm(hidden_states, residual)
472 return hidden_states
473
@@ -436,7 +483,10 @@ class DeepseekV2ForCausalLM(nn.Module):
483 super().__init__()
484 self.config = config
485 self.quant_config = quant_config
439 - self.model = DeepseekV2Model(config, cache_config, quant_config)
486 + self.model = DeepseekV2Model(config,
487 + cache_config,
488 + quant_config,
489 + prefix="model")
490 self.lm_head = ParallelLMHead(config.vocab_size,
491 config.hidden_size,
492 quant_config=quant_config)
@@ -452,7 +502,7 @@ class DeepseekV2ForCausalLM(nn.Module):
502 intermediate_tensors: Optional[IntermediateTensors] = None,
503 ) -> torch.Tensor:
504 hidden_states = self.model(input_ids, positions, kv_caches,
455 - attn_metadata)
505 + attn_metadata, intermediate_tensors)
506 return hidden_states
507
508 def compute_logits(self, hidden_states: torch.Tensor,
@@ -469,6 +519,20 @@ class DeepseekV2ForCausalLM(nn.Module):
519 next_tokens = self.sampler(logits, sampling_metadata)
520 return next_tokens
521
522 + def make_empty_intermediate_tensors(
523 + self, batch_size: int, dtype: torch.dtype,
524 + device: torch.device) -> IntermediateTensors:
525 + return IntermediateTensors({
526 + "hidden_states":
527 + torch.zeros((batch_size, self.config.hidden_size),
528 + dtype=dtype,
529 + device=device),
530 + "residual":
531 + torch.zeros((batch_size, self.config.hidden_size),
532 + dtype=dtype,
533 + device=device),
534 + })
535 +
536 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
537 stacked_params_mapping = [
538 # (param_name, shard_name, shard_id)
@@ -504,6 +568,10 @@ class DeepseekV2ForCausalLM(nn.Module):
568 # Skip loading extra bias for GPTQ models.
569 if name.endswith(".bias") and name not in params_dict:
570 continue
571 +
572 + if is_pp_missing_parameter(name, self):
573 + continue
574 +
575 param = params_dict[name]
576 weight_loader = param.weight_loader
577 weight_loader(param, loaded_weight, shard_id)
@@ -514,6 +582,10 @@ class DeepseekV2ForCausalLM(nn.Module):
582 if weight_name not in name:
583 continue
584 name = name.replace(weight_name, param_name)
585 +
586 + if is_pp_missing_parameter(name, self):
587 + continue
588 +
589 param = params_dict[name]
590 weight_loader = param.weight_loader
591 weight_loader(param,
@@ -527,6 +599,9 @@ class DeepseekV2ForCausalLM(nn.Module):
599 if name.endswith(".bias") and name not in params_dict:
600 continue
601
602 + if is_pp_missing_parameter(name, self):
603 + continue
604 +
605 param = params_dict[name]
606 weight_loader = getattr(param, "weight_loader",
607 default_weight_loader)