A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
Support Deepseek-V2 (#4650)
Co-authored-by: Philipp Moritz <pcmoritz@gmail.com>
wangding zeng committed
Jun 29, 2024 at 04:24 UTC
be0b3af9e068418726fa2b1dccef966e39024fd5
6 files changed
+700
-1
vllm/config.py
+6
@@ -297,6 +297,12 @@ class ModelConfig:
297
return self.hf_text_config.hidden_size
298
299
def get_head_size(self) -> int:
300
+ # TODO remove hard code
301
+ if hasattr(self.hf_text_config, "model_type"
302
+ ) and self.hf_text_config.model_type == 'deepseek_v2':
303
+ # FlashAttention supports only head_size 32, 64, 128, 256,
304
+ # we need to pad head_size 192 to 256
305
+ return 256
306
if hasattr(self.hf_text_config, "head_dim"):
307
return self.hf_text_config.head_dim
308
# FIXME(woosuk): This may not be true for all models.
vllm/model_executor/layers/fused_moe/__init__.py
+2
-1
@@ -1,9 +1,10 @@
1
from vllm.model_executor.layers.fused_moe.fused_moe import (
2
- fused_experts, fused_moe, fused_topk, get_config_file_name)
2
+ fused_experts, fused_moe, fused_topk, get_config_file_name, grouped_topk)
3
4
__all__ = [
5
"fused_moe",
6
"fused_topk",
7
"fused_experts",
8
"get_config_file_name",
9
+ "grouped_topk",
10
]
vllm/model_executor/layers/fused_moe/fused_moe.py
+31
@@ -367,6 +367,37 @@ def fused_topk(
367
return topk_weights, topk_ids
368
369
370
+# This is used by the Deepseek-V2 model
371
+def grouped_topk(
372
+ hidden_states: torch.Tensor,
373
+ gating_output: torch.Tensor,
374
+ topk: int,
375
+ renormalize: bool,
376
+ num_expert_group: int = 0,
377
+ topk_group: int = 0,
378
+):
379
+ scores = torch.softmax(gating_output, dim=-1)
380
+ num_token = scores.shape[0]
381
+ group_scores = scores.view(num_token, num_expert_group,
382
+ -1).max(dim=-1).values # [n, n_group]
383
+ group_idx = torch.topk(group_scores, k=topk_group, dim=-1,
384
+ sorted=False)[1] # [n, top_k_group]
385
+ group_mask = torch.zeros_like(group_scores) # [n, n_group]
386
+ group_mask.scatter_(1, group_idx, 1) # [n, n_group]
387
+ score_mask = group_mask.unsqueeze(-1).expand(
388
+ num_token, num_expert_group,
389
+ scores.shape[-1] // num_expert_group).reshape(num_token, -1) # [n, e]
390
+ tmp_scores = scores.masked_fill(~score_mask.bool(), 0.0) # [n, e]
391
+ topk_weights, topk_ids = torch.topk(tmp_scores,
392
+ k=topk,
393
+ dim=-1,
394
+ sorted=False)
395
+
396
+ if renormalize:
397
+ topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
398
+ return topk_weights, topk_ids
399
+
400
+
401
def fused_experts(hidden_states: torch.Tensor,
402
w1: torch.Tensor,
403
w2: torch.Tensor,
vllm/model_executor/layers/rotary_embedding.py
+126
@@ -610,6 +610,119 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
610
return query.flatten(-2), key.flatten(-2)
611
612
613
+def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
614
+ if scale <= 1:
615
+ return 1.0
616
+ return 0.1 * mscale * math.log(scale) + 1.0
617
+
618
+
619
+class DeepseekScalingRotaryEmbedding(RotaryEmbedding):
620
+ """RotaryEmbedding extended with YaRN method.
621
+
622
+ Credits to Peng et al. github.com/jquesnelle/yarn
623
+ """
624
+
625
+ def __init__(
626
+ self,
627
+ head_size: int,
628
+ rotary_dim: int,
629
+ max_position_embeddings: int,
630
+ base: int,
631
+ is_neox_style: bool,
632
+ scaling_factor: float,
633
+ dtype: torch.dtype,
634
+ *,
635
+ extrapolation_factor: float = 1,
636
+ attn_factor: float = 1,
637
+ beta_fast: int = 32,
638
+ beta_slow: int = 1,
639
+ mscale: float = 1,
640
+ mscale_all_dim: float = 0,
641
+ ) -> None:
642
+ self.scaling_factor = scaling_factor
643
+ self.extrapolation_factor = extrapolation_factor
644
+ self.attn_factor = attn_factor
645
+ self.beta_fast = beta_fast
646
+ self.beta_slow = beta_slow
647
+ # Get n-d magnitude scaling corrected for interpolation.
648
+ self.mscale = float(
649
+ yarn_get_mscale(self.scaling_factor, float(mscale)) /
650
+ yarn_get_mscale(self.scaling_factor, float(mscale_all_dim)) *
651
+ attn_factor)
652
+ super().__init__(head_size, rotary_dim, max_position_embeddings, base,
653
+ is_neox_style, dtype)
654
+
655
+ def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
656
+ pos_freqs = self.base**(torch.arange(
657
+ 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
658
+ self.rotary_dim)
659
+ inv_freq_extrapolation = 1.0 / pos_freqs
660
+ inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs)
661
+
662
+ low, high = _yarn_find_correction_range(self.beta_fast, self.beta_slow,
663
+ self.rotary_dim, self.base,
664
+ self.max_position_embeddings)
665
+ # Get n-d rotational scaling corrected for extrapolation
666
+ inv_freq_mask = (1 - _yarn_linear_ramp_mask(
667
+ low, high, self.rotary_dim // 2,
668
+ dtype=torch.float)) * self.extrapolation_factor
669
+ inv_freq = inv_freq_interpolation * (
670
+ 1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask
671
+ return inv_freq
672
+
673
+ def _compute_cos_sin_cache(self) -> torch.Tensor:
674
+ inv_freq = self._compute_inv_freq(self.scaling_factor)
675
+ t = torch.arange(self.max_position_embeddings * self.scaling_factor,
676
+ device="cuda",
677
+ dtype=torch.float32)
678
+ freqs = torch.einsum("i,j -> ij", t, inv_freq)
679
+ cos = (freqs.cos() * self.mscale)
680
+ sin = (freqs.sin() * self.mscale)
681
+ cache = torch.cat((cos, sin), dim=-1)
682
+ print("Cache shape", cache.shape)
683
+ return cache
684
+
685
+ def forward(
686
+ self,
687
+ positions: torch.Tensor,
688
+ query: torch.Tensor,
689
+ key: torch.Tensor,
690
+ offsets: Optional[torch.Tensor] = None,
691
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
692
+ """PyTorch-native implementation equivalent to forward()."""
693
+ query_rot = query[..., :self.rotary_dim]
694
+ key_rot = key[..., :self.rotary_dim]
695
+ if self.rotary_dim < self.head_size:
696
+ query_pass = query[..., self.rotary_dim:]
697
+ key_pass = key[..., self.rotary_dim:]
698
+
699
+ self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(
700
+ positions.device)
701
+ cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
702
+ if offsets is not None else positions]
703
+ cos, sin = cos_sin.chunk(2, dim=-1)
704
+ if self.is_neox_style:
705
+ # NOTE(woosuk): Here we assume that the positions tensor has the
706
+ # shape [batch_size, seq_len].
707
+ cos = cos.repeat(1, 1, 2).unsqueeze(-2)
708
+ sin = sin.repeat(1, 1, 2).unsqueeze(-2)
709
+ else:
710
+ cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
711
+ sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
712
+
713
+ rotate_fn = _rotate_neox if self.is_neox_style else _rotate_gptj
714
+ query_rot = query_rot * cos + rotate_fn(query_rot) * sin
715
+ key_rot = key_rot * cos + rotate_fn(key_rot) * sin
716
+
717
+ if self.rotary_dim < self.head_size:
718
+ query = torch.cat((query_rot, query_pass), dim=-1)
719
+ key = torch.cat((key_rot, key_pass), dim=-1)
720
+ else:
721
+ query = query_rot
722
+ key = key_rot
723
+ return query, key
724
+
725
+
726
class GemmaRotaryEmbedding(RotaryEmbedding):
727
728
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
@@ -679,6 +792,19 @@ def get_rope(
792
base, is_neox_style,
793
scaling_factor, dtype,
794
**extra_kwargs)
795
+ elif scaling_type == "deepseek_yarn":
796
+ original_max_position = rope_scaling[
797
+ "original_max_position_embeddings"]
798
+ # assert max_position == original_max_position * scaling_factor
799
+ extra_kwargs = {
800
+ k: v
801
+ for k, v in rope_scaling.items()
802
+ if k in ("extrapolation_factor", "attn_factor", "beta_fast",
803
+ "beta_slow", "mscale", "mscale_all_dim")
804
+ }
805
+ rotary_emb = DeepseekScalingRotaryEmbedding(
806
+ head_size, rotary_dim, original_max_position, base,
807
+ is_neox_style, scaling_factor, dtype, **extra_kwargs)
808
# The correct one should be "longrope" but keep "su" here
809
# for backward compatible
810
elif scaling_type == "su" or scaling_type == "longrope":
vllm/model_executor/models/__init__.py
+1
@@ -21,6 +21,7 @@ _GENERATION_MODELS = {
21
"DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
22
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
23
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24
+ "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
25
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
26
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
27
"Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
vllm/model_executor/models/deepseek_v2.py
new
+534
@@ -0,0 +1,534 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only DeepseekV2 model."""
24
+from typing import Any, Dict, Iterable, List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+from transformers import PretrainedConfig
29
+
30
+from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import CacheConfig
32
+from vllm.distributed import (get_tensor_model_parallel_rank,
33
+ get_tensor_model_parallel_world_size,
34
+ tensor_model_parallel_all_reduce)
35
+from vllm.model_executor.layers.activation import SiluAndMul
36
+from vllm.model_executor.layers.fused_moe import fused_experts, grouped_topk
37
+from vllm.model_executor.layers.layernorm import RMSNorm
38
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
39
+ MergedColumnParallelLinear,
40
+ ReplicatedLinear,
41
+ RowParallelLinear)
42
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
43
+from vllm.model_executor.layers.quantization.base_config import (
44
+ QuantizationConfig)
45
+from vllm.model_executor.layers.rotary_embedding import get_rope
46
+from vllm.model_executor.layers.sampler import Sampler
47
+from vllm.model_executor.layers.vocab_parallel_embedding import (
48
+ ParallelLMHead, VocabParallelEmbedding)
49
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50
+from vllm.model_executor.sampling_metadata import SamplingMetadata
51
+from vllm.sequence import SamplerOutput
52
+
53
+
54
+class DeepseekV2MLP(nn.Module):
55
+
56
+ def __init__(
57
+ self,
58
+ hidden_size: int,
59
+ intermediate_size: int,
60
+ hidden_act: str,
61
+ quant_config: Optional[QuantizationConfig] = None,
62
+ reduce_results: bool = True,
63
+ ) -> None:
64
+ super().__init__()
65
+ self.gate_up_proj = MergedColumnParallelLinear(
66
+ hidden_size, [intermediate_size] * 2,
67
+ bias=False,
68
+ quant_config=quant_config)
69
+ self.down_proj = RowParallelLinear(intermediate_size,
70
+ hidden_size,
71
+ bias=False,
72
+ quant_config=quant_config,
73
+ reduce_results=reduce_results)
74
+ if hidden_act != "silu":
75
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
76
+ "Only silu is supported for now.")
77
+ self.act_fn = SiluAndMul()
78
+
79
+ def forward(self, x):
80
+ gate_up, _ = self.gate_up_proj(x)
81
+ x = self.act_fn(gate_up)
82
+ x, _ = self.down_proj(x)
83
+ return x
84
+
85
+
86
+class DeepseekV2MoE(nn.Module):
87
+
88
+ def __init__(
89
+ self,
90
+ config: PretrainedConfig,
91
+ quant_config: Optional[QuantizationConfig] = None,
92
+ ):
93
+ super().__init__()
94
+ self.config = config
95
+ self.rank = get_tensor_model_parallel_rank()
96
+ self.tp_size = get_tensor_model_parallel_world_size()
97
+ self.n_routed_experts = config.n_routed_experts
98
+ self.top_k = config.num_experts_per_tok
99
+ self.routed_scaling_factor = config.routed_scaling_factor
100
+ if self.tp_size > self.n_routed_experts:
101
+ raise ValueError(
102
+ f"Tensor parallel size {self.tp_size} is greater than "
103
+ f"the number of experts {self.n_routed_experts}.")
104
+
105
+ self.experts = nn.ModuleList([
106
+ DeepseekV2MLP(hidden_size=config.hidden_size,
107
+ intermediate_size=config.moe_intermediate_size,
108
+ hidden_act=config.hidden_act,
109
+ quant_config=quant_config,
110
+ reduce_results=False)
111
+ for idx in range(self.n_routed_experts)
112
+ ])
113
+ self.pack_params()
114
+
115
+ self.gate = ReplicatedLinear(config.hidden_size,
116
+ self.n_routed_experts,
117
+ bias=False,
118
+ quant_config=None)
119
+
120
+ if config.n_shared_experts is not None:
121
+ intermediate_size = (config.moe_intermediate_size *
122
+ config.n_shared_experts)
123
+ self.shared_experts = DeepseekV2MLP(
124
+ hidden_size=config.hidden_size,
125
+ intermediate_size=intermediate_size,
126
+ hidden_act=config.hidden_act,
127
+ quant_config=quant_config,
128
+ reduce_results=False,
129
+ )
130
+
131
+ def pack_params(self):
132
+ w1 = []
133
+ w2 = []
134
+ for expert in self.experts:
135
+ w1.append(expert.gate_up_proj.weight)
136
+ w2.append(expert.down_proj.weight)
137
+ self.w1 = torch._utils._flatten_dense_tensors(w1)
138
+ w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
139
+ for data, param in zip(w1s, w1):
140
+ param.data = data
141
+ self.w1 = self.w1.view(len(w1), *w1s[0].shape)
142
+
143
+ self.w2 = torch._utils._flatten_dense_tensors(w2)
144
+ w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
145
+ for data, param in zip(w2s, w2):
146
+ param.data = data
147
+
148
+ self.w2 = self.w2.view(len(w2), *w2s[0].shape)
149
+
150
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
151
+ num_tokens, hidden_dim = hidden_states.shape
152
+ hidden_states = hidden_states.view(-1, hidden_dim)
153
+ if self.config.n_shared_experts is not None:
154
+ shared_output = self.shared_experts(hidden_states)
155
+ # router_logits: (num_tokens, n_experts)
156
+ router_logits, _ = self.gate(hidden_states)
157
+ topk_weights, topk_ids = grouped_topk(
158
+ hidden_states,
159
+ router_logits,
160
+ self.top_k,
161
+ renormalize=self.config.norm_topk_prob,
162
+ num_expert_group=self.config.n_group,
163
+ topk_group=self.config.topk_group)
164
+ final_hidden_states = fused_experts(
165
+ hidden_states,
166
+ self.w1,
167
+ self.w2,
168
+ topk_weights,
169
+ topk_ids,
170
+ inplace=True) * self.routed_scaling_factor
171
+ if self.config.n_shared_experts is not None:
172
+ final_hidden_states = final_hidden_states + shared_output
173
+ final_hidden_states = tensor_model_parallel_all_reduce(
174
+ final_hidden_states)
175
+
176
+ return final_hidden_states.view(num_tokens, hidden_dim)
177
+
178
+
179
+def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
180
+ import math
181
+ if scale <= 1:
182
+ return 1.0
183
+ return 0.1 * mscale * math.log(scale) + 1.0
184
+
185
+
186
+class DeepseekV2Attention(nn.Module):
187
+
188
+ def __init__(
189
+ self,
190
+ config: PretrainedConfig,
191
+ hidden_size: int,
192
+ num_heads: int,
193
+ qk_nope_head_dim: int,
194
+ qk_rope_head_dim: int,
195
+ v_head_dim: int,
196
+ q_lora_rank: int,
197
+ kv_lora_rank: int,
198
+ rope_theta: float = 10000,
199
+ rope_scaling: Optional[Dict[str, Any]] = None,
200
+ max_position_embeddings: int = 8192,
201
+ cache_config: Optional[CacheConfig] = None,
202
+ quant_config: Optional[QuantizationConfig] = None,
203
+ layer_idx=None,
204
+ ) -> None:
205
+ super().__init__()
206
+ self.layer_idx = layer_idx
207
+ self.hidden_size = hidden_size
208
+ self.qk_nope_head_dim = qk_nope_head_dim
209
+ self.qk_rope_head_dim = qk_rope_head_dim
210
+ self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
211
+ self.v_head_dim = v_head_dim
212
+ self.q_lora_rank = q_lora_rank
213
+ self.kv_lora_rank = kv_lora_rank
214
+ self.num_heads = num_heads
215
+ tp_size = get_tensor_model_parallel_world_size()
216
+ assert num_heads % tp_size == 0
217
+ self.num_local_heads = num_heads // tp_size
218
+ self.scaling = self.qk_head_dim**-0.5
219
+ self.rope_theta = rope_theta
220
+ self.max_position_embeddings = max_position_embeddings
221
+
222
+ if self.q_lora_rank is not None:
223
+ self.q_a_proj = ReplicatedLinear(self.hidden_size,
224
+ self.q_lora_rank,
225
+ bias=False,
226
+ quant_config=quant_config)
227
+ self.q_a_layernorm = RMSNorm(self.q_lora_rank,
228
+ eps=config.rms_norm_eps)
229
+ self.q_b_proj = ColumnParallelLinear(q_lora_rank,
230
+ self.num_heads *
231
+ self.qk_head_dim,
232
+ bias=False,
233
+ quant_config=quant_config)
234
+ else:
235
+ self.q_proj = ColumnParallelLinear(self.hidden_size,
236
+ self.num_heads *
237
+ self.qk_head_dim,
238
+ bias=False,
239
+ quant_config=quant_config)
240
+
241
+ self.kv_a_proj_with_mqa = ReplicatedLinear(self.hidden_size,
242
+ self.kv_lora_rank +
243
+ self.qk_rope_head_dim,
244
+ bias=False,
245
+ quant_config=quant_config)
246
+ self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
247
+ eps=config.rms_norm_eps)
248
+ self.kv_b_proj = ColumnParallelLinear(
249
+ self.kv_lora_rank,
250
+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
251
+ bias=False,
252
+ quant_config=quant_config)
253
+ # O projection.
254
+ self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
255
+ self.hidden_size,
256
+ bias=False,
257
+ quant_config=quant_config)
258
+ rope_scaling['type'] = 'deepseek_yarn'
259
+ self.rotary_emb = get_rope(qk_rope_head_dim,
260
+ rotary_dim=qk_rope_head_dim,
261
+ max_position=max_position_embeddings,
262
+ base=rope_theta,
263
+ rope_scaling=rope_scaling,
264
+ is_neox_style=False)
265
+
266
+ if rope_scaling:
267
+ mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
268
+ scaling_factor = rope_scaling["factor"]
269
+ mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
270
+ self.scaling = self.scaling * mscale * mscale
271
+
272
+ # self.attn = Attention(self.num_heads,
273
+ # self.qk_head_dim,
274
+ # self.scaling,
275
+ # num_kv_heads=self.num_heads)
276
+
277
+ # TODO, support head_size 192
278
+ self.attn = Attention(self.num_local_heads,
279
+ 256,
280
+ self.scaling,
281
+ num_kv_heads=self.num_local_heads,
282
+ cache_config=cache_config,
283
+ quant_config=quant_config)
284
+
285
+ def forward(
286
+ self,
287
+ positions: torch.Tensor,
288
+ hidden_states: torch.Tensor,
289
+ kv_cache: torch.Tensor,
290
+ attn_metadata: AttentionMetadata,
291
+ ) -> torch.Tensor:
292
+ if self.q_lora_rank is not None:
293
+ q = self.q_a_proj(hidden_states)[0]
294
+ q = self.q_a_layernorm(q)
295
+ q = self.q_b_proj(q)[0].view(-1, self.num_local_heads,
296
+ self.qk_head_dim)
297
+ else:
298
+ q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads,
299
+ self.qk_head_dim)
300
+ q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim],
301
+ dim=-1)
302
+ latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0]
303
+ kv_a, _ = latent_cache.split(
304
+ [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
305
+ latent_cache = latent_cache.unsqueeze(1)
306
+ kv_a = self.kv_a_layernorm(kv_a.contiguous())
307
+ kv = self.kv_b_proj(kv_a)[0]
308
+ kv = kv.view(-1, self.num_local_heads,
309
+ self.qk_nope_head_dim + self.v_head_dim)
310
+ k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
311
+ k_pe = latent_cache[:, :, self.kv_lora_rank:]
312
+ q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
313
+ q[..., self.qk_nope_head_dim:] = q_pe
314
+ k = torch.empty_like(q)
315
+ k[..., :self.qk_nope_head_dim] = k_nope
316
+ k[..., self.qk_nope_head_dim:] = k_pe
317
+ q = torch.nn.functional.pad(q, [0, 256 - self.qk_head_dim],
318
+ value=0).view(-1,
319
+ self.num_local_heads * 256)
320
+ k = torch.nn.functional.pad(k, [0, 256 - self.qk_head_dim],
321
+ value=0).view(-1,
322
+ self.num_local_heads * 256)
323
+ v = torch.nn.functional.pad(v, [0, 256 - self.v_head_dim],
324
+ value=0).view(-1,
325
+ self.num_local_heads * 256)
326
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
327
+ attn_output = attn_output.view(
328
+ -1, self.num_local_heads, 256)[..., :self.v_head_dim].reshape(
329
+ -1, self.num_local_heads * self.v_head_dim)
330
+ output, _ = self.o_proj(attn_output)
331
+ return output
332
+
333
+
334
+class DeepseekV2DecoderLayer(nn.Module):
335
+
336
+ def __init__(
337
+ self,
338
+ config: PretrainedConfig,
339
+ layer_idx: int,
340
+ cache_config: Optional[CacheConfig] = None,
341
+ quant_config: Optional[QuantizationConfig] = None,
342
+ ) -> None:
343
+ super().__init__()
344
+ self.hidden_size = config.hidden_size
345
+ rope_theta = getattr(config, "rope_theta", 10000)
346
+ rope_scaling = getattr(config, "rope_scaling", None)
347
+ max_position_embeddings = getattr(config, "max_position_embeddings",
348
+ 8192)
349
+ self.self_attn = DeepseekV2Attention(
350
+ config=config,
351
+ hidden_size=self.hidden_size,
352
+ num_heads=config.num_attention_heads,
353
+ qk_nope_head_dim=config.qk_nope_head_dim,
354
+ qk_rope_head_dim=config.qk_rope_head_dim,
355
+ v_head_dim=config.v_head_dim,
356
+ q_lora_rank=config.q_lora_rank
357
+ if hasattr(config, "q_lora_rank") else None,
358
+ kv_lora_rank=config.kv_lora_rank,
359
+ rope_theta=rope_theta,
360
+ rope_scaling=rope_scaling,
361
+ max_position_embeddings=max_position_embeddings,
362
+ cache_config=cache_config,
363
+ quant_config=quant_config,
364
+ layer_idx=layer_idx,
365
+ )
366
+ if (config.n_routed_experts is not None
367
+ and layer_idx >= config.first_k_dense_replace
368
+ and layer_idx % config.moe_layer_freq == 0):
369
+ self.mlp = DeepseekV2MoE(config=config, quant_config=quant_config)
370
+ else:
371
+ self.mlp = DeepseekV2MLP(
372
+ hidden_size=config.hidden_size,
373
+ intermediate_size=config.intermediate_size,
374
+ hidden_act=config.hidden_act,
375
+ quant_config=quant_config,
376
+ )
377
+ self.input_layernorm = RMSNorm(config.hidden_size,
378
+ eps=config.rms_norm_eps)
379
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
380
+ eps=config.rms_norm_eps)
381
+
382
+ def forward(
383
+ self,
384
+ positions: torch.Tensor,
385
+ hidden_states: torch.Tensor,
386
+ kv_cache: torch.Tensor,
387
+ attn_metadata: AttentionMetadata,
388
+ residual: Optional[torch.Tensor],
389
+ ) -> torch.Tensor:
390
+ # Self Attention
391
+ if residual is None:
392
+ residual = hidden_states
393
+ hidden_states = self.input_layernorm(hidden_states)
394
+ else:
395
+ hidden_states, residual = self.input_layernorm(
396
+ hidden_states, residual)
397
+ hidden_states = self.self_attn(
398
+ positions=positions,
399
+ hidden_states=hidden_states,
400
+ kv_cache=kv_cache,
401
+ attn_metadata=attn_metadata,
402
+ )
403
+
404
+ # Fully Connected
405
+ hidden_states, residual = self.post_attention_layernorm(
406
+ hidden_states, residual)
407
+ hidden_states = self.mlp(hidden_states)
408
+ return hidden_states, residual
409
+
410
+
411
+class DeepseekV2Model(nn.Module):
412
+
413
+ fall_back_to_pt_during_load = False
414
+
415
+ def __init__(
416
+ self,
417
+ config: PretrainedConfig,
418
+ cache_config: Optional[CacheConfig] = None,
419
+ quant_config: Optional[QuantizationConfig] = None,
420
+ ) -> None:
421
+ super().__init__()
422
+ self.padding_idx = config.pad_token_id
423
+ self.vocab_size = config.vocab_size
424
+
425
+ self.embed_tokens = VocabParallelEmbedding(
426
+ config.vocab_size,
427
+ config.hidden_size,
428
+ )
429
+ self.layers = nn.ModuleList([
430
+ DeepseekV2DecoderLayer(config,
431
+ layer_idx,
432
+ cache_config=cache_config,
433
+ quant_config=quant_config)
434
+ for layer_idx in range(config.num_hidden_layers)
435
+ ])
436
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
437
+
438
+ def forward(
439
+ self,
440
+ input_ids: torch.Tensor,
441
+ positions: torch.Tensor,
442
+ kv_caches: List[torch.Tensor],
443
+ attn_metadata: AttentionMetadata,
444
+ ) -> torch.Tensor:
445
+ hidden_states = self.embed_tokens(input_ids)
446
+ residual = None
447
+ for i in range(len(self.layers)):
448
+ layer = self.layers[i]
449
+ hidden_states, residual = layer(positions, hidden_states,
450
+ kv_caches[i], attn_metadata,
451
+ residual)
452
+ hidden_states, _ = self.norm(hidden_states, residual)
453
+ return hidden_states
454
+
455
+
456
+class DeepseekV2ForCausalLM(nn.Module):
457
+
458
+ def __init__(
459
+ self,
460
+ config: PretrainedConfig,
461
+ cache_config: Optional[CacheConfig] = None,
462
+ quant_config: Optional[QuantizationConfig] = None,
463
+ ) -> None:
464
+ super().__init__()
465
+ self.config = config
466
+ self.quant_config = quant_config
467
+ self.model = DeepseekV2Model(config, cache_config, quant_config)
468
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
469
+ self.logits_processor = LogitsProcessor(config.vocab_size)
470
+ self.sampler = Sampler()
471
+
472
+ def forward(
473
+ self,
474
+ input_ids: torch.Tensor,
475
+ positions: torch.Tensor,
476
+ kv_caches: List[torch.Tensor],
477
+ attn_metadata: AttentionMetadata,
478
+ ) -> torch.Tensor:
479
+ hidden_states = self.model(input_ids, positions, kv_caches,
480
+ attn_metadata)
481
+ return hidden_states
482
+
483
+ def compute_logits(self, hidden_states: torch.Tensor,
484
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
485
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
486
+ sampling_metadata)
487
+ return logits
488
+
489
+ def sample(
490
+ self,
491
+ logits: Optional[torch.Tensor],
492
+ sampling_metadata: SamplingMetadata,
493
+ ) -> Optional[SamplerOutput]:
494
+ next_tokens = self.sampler(logits, sampling_metadata)
495
+ return next_tokens
496
+
497
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
498
+ stacked_params_mapping = [
499
+ # (param_name, shard_name, shard_id)
500
+ ("gate_up_proj", "gate_proj", 0),
501
+ ("gate_up_proj", "up_proj", 1),
502
+ ]
503
+
504
+ params_dict = dict(self.named_parameters())
505
+ for name, loaded_weight in weights:
506
+ if "rotary_emb.inv_freq" in name:
507
+ continue
508
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
509
+ if weight_name not in name:
510
+ continue
511
+ name = name.replace(weight_name, param_name)
512
+ # Skip loading extra bias for GPTQ models.
513
+ if name.endswith(".bias") and name not in params_dict:
514
+ continue
515
+ # Skip experts that are not assigned to this worker.
516
+ if (("mlp.experts." in name or "mlp.shared_experts." in name)
517
+ and name not in params_dict):
518
+ continue
519
+ param = params_dict[name]
520
+ weight_loader = param.weight_loader
521
+ weight_loader(param, loaded_weight, shard_id)
522
+ break
523
+ else:
524
+ # Skip loading extra bias for GPTQ models.
525
+ if name.endswith(".bias") and name not in params_dict:
526
+ continue
527
+ # Skip experts that are not assigned to this worker.
528
+ if (("mlp.experts." in name or "mlp.shared_experts." in name)
529
+ and name not in params_dict):
530
+ continue
531
+ param = params_dict[name]
532
+ weight_loader = getattr(param, "weight_loader",
533
+ default_weight_loader)
534
+ weight_loader(param, loaded_weight)