@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / be0b3af9e0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

Support Deepseek-V2 (#4650)

Co-authored-by: Philipp Moritz <pcmoritz@gmail.com>

wangding zeng committed Jun 29, 2024 at 04:24 UTC be0b3af9e068418726fa2b1dccef966e39024fd5
6 files changed +700 -1
vllm/config.py
+6
@@ -297,6 +297,12 @@ class ModelConfig:
297 return self.hf_text_config.hidden_size
298
299 def get_head_size(self) -> int:
300 + # TODO remove hard code
301 + if hasattr(self.hf_text_config, "model_type"
302 + ) and self.hf_text_config.model_type == 'deepseek_v2':
303 + # FlashAttention supports only head_size 32, 64, 128, 256,
304 + # we need to pad head_size 192 to 256
305 + return 256
306 if hasattr(self.hf_text_config, "head_dim"):
307 return self.hf_text_config.head_dim
308 # FIXME(woosuk): This may not be true for all models.
vllm/model_executor/layers/fused_moe/__init__.py
+2 -1
@@ -1,9 +1,10 @@
1 from vllm.model_executor.layers.fused_moe.fused_moe import (
2 - fused_experts, fused_moe, fused_topk, get_config_file_name)
2 + fused_experts, fused_moe, fused_topk, get_config_file_name, grouped_topk)
3
4 __all__ = [
5 "fused_moe",
6 "fused_topk",
7 "fused_experts",
8 "get_config_file_name",
9 + "grouped_topk",
10 ]
vllm/model_executor/layers/fused_moe/fused_moe.py
+31
@@ -367,6 +367,37 @@ def fused_topk(
367 return topk_weights, topk_ids
368
369
370 +# This is used by the Deepseek-V2 model
371 +def grouped_topk(
372 + hidden_states: torch.Tensor,
373 + gating_output: torch.Tensor,
374 + topk: int,
375 + renormalize: bool,
376 + num_expert_group: int = 0,
377 + topk_group: int = 0,
378 +):
379 + scores = torch.softmax(gating_output, dim=-1)
380 + num_token = scores.shape[0]
381 + group_scores = scores.view(num_token, num_expert_group,
382 + -1).max(dim=-1).values # [n, n_group]
383 + group_idx = torch.topk(group_scores, k=topk_group, dim=-1,
384 + sorted=False)[1] # [n, top_k_group]
385 + group_mask = torch.zeros_like(group_scores) # [n, n_group]
386 + group_mask.scatter_(1, group_idx, 1) # [n, n_group]
387 + score_mask = group_mask.unsqueeze(-1).expand(
388 + num_token, num_expert_group,
389 + scores.shape[-1] // num_expert_group).reshape(num_token, -1) # [n, e]
390 + tmp_scores = scores.masked_fill(~score_mask.bool(), 0.0) # [n, e]
391 + topk_weights, topk_ids = torch.topk(tmp_scores,
392 + k=topk,
393 + dim=-1,
394 + sorted=False)
395 +
396 + if renormalize:
397 + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
398 + return topk_weights, topk_ids
399 +
400 +
401 def fused_experts(hidden_states: torch.Tensor,
402 w1: torch.Tensor,
403 w2: torch.Tensor,
vllm/model_executor/layers/rotary_embedding.py
+126
@@ -610,6 +610,119 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
610 return query.flatten(-2), key.flatten(-2)
611
612
613 +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
614 + if scale <= 1:
615 + return 1.0
616 + return 0.1 * mscale * math.log(scale) + 1.0
617 +
618 +
619 +class DeepseekScalingRotaryEmbedding(RotaryEmbedding):
620 + """RotaryEmbedding extended with YaRN method.
621 +
622 + Credits to Peng et al. github.com/jquesnelle/yarn
623 + """
624 +
625 + def __init__(
626 + self,
627 + head_size: int,
628 + rotary_dim: int,
629 + max_position_embeddings: int,
630 + base: int,
631 + is_neox_style: bool,
632 + scaling_factor: float,
633 + dtype: torch.dtype,
634 + *,
635 + extrapolation_factor: float = 1,
636 + attn_factor: float = 1,
637 + beta_fast: int = 32,
638 + beta_slow: int = 1,
639 + mscale: float = 1,
640 + mscale_all_dim: float = 0,
641 + ) -> None:
642 + self.scaling_factor = scaling_factor
643 + self.extrapolation_factor = extrapolation_factor
644 + self.attn_factor = attn_factor
645 + self.beta_fast = beta_fast
646 + self.beta_slow = beta_slow
647 + # Get n-d magnitude scaling corrected for interpolation.
648 + self.mscale = float(
649 + yarn_get_mscale(self.scaling_factor, float(mscale)) /
650 + yarn_get_mscale(self.scaling_factor, float(mscale_all_dim)) *
651 + attn_factor)
652 + super().__init__(head_size, rotary_dim, max_position_embeddings, base,
653 + is_neox_style, dtype)
654 +
655 + def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
656 + pos_freqs = self.base**(torch.arange(
657 + 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
658 + self.rotary_dim)
659 + inv_freq_extrapolation = 1.0 / pos_freqs
660 + inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs)
661 +
662 + low, high = _yarn_find_correction_range(self.beta_fast, self.beta_slow,
663 + self.rotary_dim, self.base,
664 + self.max_position_embeddings)
665 + # Get n-d rotational scaling corrected for extrapolation
666 + inv_freq_mask = (1 - _yarn_linear_ramp_mask(
667 + low, high, self.rotary_dim // 2,
668 + dtype=torch.float)) * self.extrapolation_factor
669 + inv_freq = inv_freq_interpolation * (
670 + 1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask
671 + return inv_freq
672 +
673 + def _compute_cos_sin_cache(self) -> torch.Tensor:
674 + inv_freq = self._compute_inv_freq(self.scaling_factor)
675 + t = torch.arange(self.max_position_embeddings * self.scaling_factor,
676 + device="cuda",
677 + dtype=torch.float32)
678 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
679 + cos = (freqs.cos() * self.mscale)
680 + sin = (freqs.sin() * self.mscale)
681 + cache = torch.cat((cos, sin), dim=-1)
682 + print("Cache shape", cache.shape)
683 + return cache
684 +
685 + def forward(
686 + self,
687 + positions: torch.Tensor,
688 + query: torch.Tensor,
689 + key: torch.Tensor,
690 + offsets: Optional[torch.Tensor] = None,
691 + ) -> Tuple[torch.Tensor, torch.Tensor]:
692 + """PyTorch-native implementation equivalent to forward()."""
693 + query_rot = query[..., :self.rotary_dim]
694 + key_rot = key[..., :self.rotary_dim]
695 + if self.rotary_dim < self.head_size:
696 + query_pass = query[..., self.rotary_dim:]
697 + key_pass = key[..., self.rotary_dim:]
698 +
699 + self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(
700 + positions.device)
701 + cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
702 + if offsets is not None else positions]
703 + cos, sin = cos_sin.chunk(2, dim=-1)
704 + if self.is_neox_style:
705 + # NOTE(woosuk): Here we assume that the positions tensor has the
706 + # shape [batch_size, seq_len].
707 + cos = cos.repeat(1, 1, 2).unsqueeze(-2)
708 + sin = sin.repeat(1, 1, 2).unsqueeze(-2)
709 + else:
710 + cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
711 + sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
712 +
713 + rotate_fn = _rotate_neox if self.is_neox_style else _rotate_gptj
714 + query_rot = query_rot * cos + rotate_fn(query_rot) * sin
715 + key_rot = key_rot * cos + rotate_fn(key_rot) * sin
716 +
717 + if self.rotary_dim < self.head_size:
718 + query = torch.cat((query_rot, query_pass), dim=-1)
719 + key = torch.cat((key_rot, key_pass), dim=-1)
720 + else:
721 + query = query_rot
722 + key = key_rot
723 + return query, key
724 +
725 +
726 class GemmaRotaryEmbedding(RotaryEmbedding):
727
728 def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
@@ -679,6 +792,19 @@ def get_rope(
792 base, is_neox_style,
793 scaling_factor, dtype,
794 **extra_kwargs)
795 + elif scaling_type == "deepseek_yarn":
796 + original_max_position = rope_scaling[
797 + "original_max_position_embeddings"]
798 + # assert max_position == original_max_position * scaling_factor
799 + extra_kwargs = {
800 + k: v
801 + for k, v in rope_scaling.items()
802 + if k in ("extrapolation_factor", "attn_factor", "beta_fast",
803 + "beta_slow", "mscale", "mscale_all_dim")
804 + }
805 + rotary_emb = DeepseekScalingRotaryEmbedding(
806 + head_size, rotary_dim, original_max_position, base,
807 + is_neox_style, scaling_factor, dtype, **extra_kwargs)
808 # The correct one should be "longrope" but keep "su" here
809 # for backward compatible
810 elif scaling_type == "su" or scaling_type == "longrope":
vllm/model_executor/models/__init__.py
+1
@@ -21,6 +21,7 @@ _GENERATION_MODELS = {
21 "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
22 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
23 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24 + "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
25 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
26 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
27 "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
vllm/model_executor/models/deepseek_v2.py new
+534
@@ -0,0 +1,534 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only DeepseekV2 model."""
24 +from typing import Any, Dict, Iterable, List, Optional, Tuple
25 +
26 +import torch
27 +from torch import nn
28 +from transformers import PretrainedConfig
29 +
30 +from vllm.attention import Attention, AttentionMetadata
31 +from vllm.config import CacheConfig
32 +from vllm.distributed import (get_tensor_model_parallel_rank,
33 + get_tensor_model_parallel_world_size,
34 + tensor_model_parallel_all_reduce)
35 +from vllm.model_executor.layers.activation import SiluAndMul
36 +from vllm.model_executor.layers.fused_moe import fused_experts, grouped_topk
37 +from vllm.model_executor.layers.layernorm import RMSNorm
38 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
39 + MergedColumnParallelLinear,
40 + ReplicatedLinear,
41 + RowParallelLinear)
42 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
43 +from vllm.model_executor.layers.quantization.base_config import (
44 + QuantizationConfig)
45 +from vllm.model_executor.layers.rotary_embedding import get_rope
46 +from vllm.model_executor.layers.sampler import Sampler
47 +from vllm.model_executor.layers.vocab_parallel_embedding import (
48 + ParallelLMHead, VocabParallelEmbedding)
49 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 +from vllm.model_executor.sampling_metadata import SamplingMetadata
51 +from vllm.sequence import SamplerOutput
52 +
53 +
54 +class DeepseekV2MLP(nn.Module):
55 +
56 + def __init__(
57 + self,
58 + hidden_size: int,
59 + intermediate_size: int,
60 + hidden_act: str,
61 + quant_config: Optional[QuantizationConfig] = None,
62 + reduce_results: bool = True,
63 + ) -> None:
64 + super().__init__()
65 + self.gate_up_proj = MergedColumnParallelLinear(
66 + hidden_size, [intermediate_size] * 2,
67 + bias=False,
68 + quant_config=quant_config)
69 + self.down_proj = RowParallelLinear(intermediate_size,
70 + hidden_size,
71 + bias=False,
72 + quant_config=quant_config,
73 + reduce_results=reduce_results)
74 + if hidden_act != "silu":
75 + raise ValueError(f"Unsupported activation: {hidden_act}. "
76 + "Only silu is supported for now.")
77 + self.act_fn = SiluAndMul()
78 +
79 + def forward(self, x):
80 + gate_up, _ = self.gate_up_proj(x)
81 + x = self.act_fn(gate_up)
82 + x, _ = self.down_proj(x)
83 + return x
84 +
85 +
86 +class DeepseekV2MoE(nn.Module):
87 +
88 + def __init__(
89 + self,
90 + config: PretrainedConfig,
91 + quant_config: Optional[QuantizationConfig] = None,
92 + ):
93 + super().__init__()
94 + self.config = config
95 + self.rank = get_tensor_model_parallel_rank()
96 + self.tp_size = get_tensor_model_parallel_world_size()
97 + self.n_routed_experts = config.n_routed_experts
98 + self.top_k = config.num_experts_per_tok
99 + self.routed_scaling_factor = config.routed_scaling_factor
100 + if self.tp_size > self.n_routed_experts:
101 + raise ValueError(
102 + f"Tensor parallel size {self.tp_size} is greater than "
103 + f"the number of experts {self.n_routed_experts}.")
104 +
105 + self.experts = nn.ModuleList([
106 + DeepseekV2MLP(hidden_size=config.hidden_size,
107 + intermediate_size=config.moe_intermediate_size,
108 + hidden_act=config.hidden_act,
109 + quant_config=quant_config,
110 + reduce_results=False)
111 + for idx in range(self.n_routed_experts)
112 + ])
113 + self.pack_params()
114 +
115 + self.gate = ReplicatedLinear(config.hidden_size,
116 + self.n_routed_experts,
117 + bias=False,
118 + quant_config=None)
119 +
120 + if config.n_shared_experts is not None:
121 + intermediate_size = (config.moe_intermediate_size *
122 + config.n_shared_experts)
123 + self.shared_experts = DeepseekV2MLP(
124 + hidden_size=config.hidden_size,
125 + intermediate_size=intermediate_size,
126 + hidden_act=config.hidden_act,
127 + quant_config=quant_config,
128 + reduce_results=False,
129 + )
130 +
131 + def pack_params(self):
132 + w1 = []
133 + w2 = []
134 + for expert in self.experts:
135 + w1.append(expert.gate_up_proj.weight)
136 + w2.append(expert.down_proj.weight)
137 + self.w1 = torch._utils._flatten_dense_tensors(w1)
138 + w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
139 + for data, param in zip(w1s, w1):
140 + param.data = data
141 + self.w1 = self.w1.view(len(w1), *w1s[0].shape)
142 +
143 + self.w2 = torch._utils._flatten_dense_tensors(w2)
144 + w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
145 + for data, param in zip(w2s, w2):
146 + param.data = data
147 +
148 + self.w2 = self.w2.view(len(w2), *w2s[0].shape)
149 +
150 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
151 + num_tokens, hidden_dim = hidden_states.shape
152 + hidden_states = hidden_states.view(-1, hidden_dim)
153 + if self.config.n_shared_experts is not None:
154 + shared_output = self.shared_experts(hidden_states)
155 + # router_logits: (num_tokens, n_experts)
156 + router_logits, _ = self.gate(hidden_states)
157 + topk_weights, topk_ids = grouped_topk(
158 + hidden_states,
159 + router_logits,
160 + self.top_k,
161 + renormalize=self.config.norm_topk_prob,
162 + num_expert_group=self.config.n_group,
163 + topk_group=self.config.topk_group)
164 + final_hidden_states = fused_experts(
165 + hidden_states,
166 + self.w1,
167 + self.w2,
168 + topk_weights,
169 + topk_ids,
170 + inplace=True) * self.routed_scaling_factor
171 + if self.config.n_shared_experts is not None:
172 + final_hidden_states = final_hidden_states + shared_output
173 + final_hidden_states = tensor_model_parallel_all_reduce(
174 + final_hidden_states)
175 +
176 + return final_hidden_states.view(num_tokens, hidden_dim)
177 +
178 +
179 +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
180 + import math
181 + if scale <= 1:
182 + return 1.0
183 + return 0.1 * mscale * math.log(scale) + 1.0
184 +
185 +
186 +class DeepseekV2Attention(nn.Module):
187 +
188 + def __init__(
189 + self,
190 + config: PretrainedConfig,
191 + hidden_size: int,
192 + num_heads: int,
193 + qk_nope_head_dim: int,
194 + qk_rope_head_dim: int,
195 + v_head_dim: int,
196 + q_lora_rank: int,
197 + kv_lora_rank: int,
198 + rope_theta: float = 10000,
199 + rope_scaling: Optional[Dict[str, Any]] = None,
200 + max_position_embeddings: int = 8192,
201 + cache_config: Optional[CacheConfig] = None,
202 + quant_config: Optional[QuantizationConfig] = None,
203 + layer_idx=None,
204 + ) -> None:
205 + super().__init__()
206 + self.layer_idx = layer_idx
207 + self.hidden_size = hidden_size
208 + self.qk_nope_head_dim = qk_nope_head_dim
209 + self.qk_rope_head_dim = qk_rope_head_dim
210 + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
211 + self.v_head_dim = v_head_dim
212 + self.q_lora_rank = q_lora_rank
213 + self.kv_lora_rank = kv_lora_rank
214 + self.num_heads = num_heads
215 + tp_size = get_tensor_model_parallel_world_size()
216 + assert num_heads % tp_size == 0
217 + self.num_local_heads = num_heads // tp_size
218 + self.scaling = self.qk_head_dim**-0.5
219 + self.rope_theta = rope_theta
220 + self.max_position_embeddings = max_position_embeddings
221 +
222 + if self.q_lora_rank is not None:
223 + self.q_a_proj = ReplicatedLinear(self.hidden_size,
224 + self.q_lora_rank,
225 + bias=False,
226 + quant_config=quant_config)
227 + self.q_a_layernorm = RMSNorm(self.q_lora_rank,
228 + eps=config.rms_norm_eps)
229 + self.q_b_proj = ColumnParallelLinear(q_lora_rank,
230 + self.num_heads *
231 + self.qk_head_dim,
232 + bias=False,
233 + quant_config=quant_config)
234 + else:
235 + self.q_proj = ColumnParallelLinear(self.hidden_size,
236 + self.num_heads *
237 + self.qk_head_dim,
238 + bias=False,
239 + quant_config=quant_config)
240 +
241 + self.kv_a_proj_with_mqa = ReplicatedLinear(self.hidden_size,
242 + self.kv_lora_rank +
243 + self.qk_rope_head_dim,
244 + bias=False,
245 + quant_config=quant_config)
246 + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
247 + eps=config.rms_norm_eps)
248 + self.kv_b_proj = ColumnParallelLinear(
249 + self.kv_lora_rank,
250 + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
251 + bias=False,
252 + quant_config=quant_config)
253 + # O projection.
254 + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
255 + self.hidden_size,
256 + bias=False,
257 + quant_config=quant_config)
258 + rope_scaling['type'] = 'deepseek_yarn'
259 + self.rotary_emb = get_rope(qk_rope_head_dim,
260 + rotary_dim=qk_rope_head_dim,
261 + max_position=max_position_embeddings,
262 + base=rope_theta,
263 + rope_scaling=rope_scaling,
264 + is_neox_style=False)
265 +
266 + if rope_scaling:
267 + mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
268 + scaling_factor = rope_scaling["factor"]
269 + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
270 + self.scaling = self.scaling * mscale * mscale
271 +
272 + # self.attn = Attention(self.num_heads,
273 + # self.qk_head_dim,
274 + # self.scaling,
275 + # num_kv_heads=self.num_heads)
276 +
277 + # TODO, support head_size 192
278 + self.attn = Attention(self.num_local_heads,
279 + 256,
280 + self.scaling,
281 + num_kv_heads=self.num_local_heads,
282 + cache_config=cache_config,
283 + quant_config=quant_config)
284 +
285 + def forward(
286 + self,
287 + positions: torch.Tensor,
288 + hidden_states: torch.Tensor,
289 + kv_cache: torch.Tensor,
290 + attn_metadata: AttentionMetadata,
291 + ) -> torch.Tensor:
292 + if self.q_lora_rank is not None:
293 + q = self.q_a_proj(hidden_states)[0]
294 + q = self.q_a_layernorm(q)
295 + q = self.q_b_proj(q)[0].view(-1, self.num_local_heads,
296 + self.qk_head_dim)
297 + else:
298 + q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads,
299 + self.qk_head_dim)
300 + q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim],
301 + dim=-1)
302 + latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0]
303 + kv_a, _ = latent_cache.split(
304 + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
305 + latent_cache = latent_cache.unsqueeze(1)
306 + kv_a = self.kv_a_layernorm(kv_a.contiguous())
307 + kv = self.kv_b_proj(kv_a)[0]
308 + kv = kv.view(-1, self.num_local_heads,
309 + self.qk_nope_head_dim + self.v_head_dim)
310 + k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
311 + k_pe = latent_cache[:, :, self.kv_lora_rank:]
312 + q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
313 + q[..., self.qk_nope_head_dim:] = q_pe
314 + k = torch.empty_like(q)
315 + k[..., :self.qk_nope_head_dim] = k_nope
316 + k[..., self.qk_nope_head_dim:] = k_pe
317 + q = torch.nn.functional.pad(q, [0, 256 - self.qk_head_dim],
318 + value=0).view(-1,
319 + self.num_local_heads * 256)
320 + k = torch.nn.functional.pad(k, [0, 256 - self.qk_head_dim],
321 + value=0).view(-1,
322 + self.num_local_heads * 256)
323 + v = torch.nn.functional.pad(v, [0, 256 - self.v_head_dim],
324 + value=0).view(-1,
325 + self.num_local_heads * 256)
326 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
327 + attn_output = attn_output.view(
328 + -1, self.num_local_heads, 256)[..., :self.v_head_dim].reshape(
329 + -1, self.num_local_heads * self.v_head_dim)
330 + output, _ = self.o_proj(attn_output)
331 + return output
332 +
333 +
334 +class DeepseekV2DecoderLayer(nn.Module):
335 +
336 + def __init__(
337 + self,
338 + config: PretrainedConfig,
339 + layer_idx: int,
340 + cache_config: Optional[CacheConfig] = None,
341 + quant_config: Optional[QuantizationConfig] = None,
342 + ) -> None:
343 + super().__init__()
344 + self.hidden_size = config.hidden_size
345 + rope_theta = getattr(config, "rope_theta", 10000)
346 + rope_scaling = getattr(config, "rope_scaling", None)
347 + max_position_embeddings = getattr(config, "max_position_embeddings",
348 + 8192)
349 + self.self_attn = DeepseekV2Attention(
350 + config=config,
351 + hidden_size=self.hidden_size,
352 + num_heads=config.num_attention_heads,
353 + qk_nope_head_dim=config.qk_nope_head_dim,
354 + qk_rope_head_dim=config.qk_rope_head_dim,
355 + v_head_dim=config.v_head_dim,
356 + q_lora_rank=config.q_lora_rank
357 + if hasattr(config, "q_lora_rank") else None,
358 + kv_lora_rank=config.kv_lora_rank,
359 + rope_theta=rope_theta,
360 + rope_scaling=rope_scaling,
361 + max_position_embeddings=max_position_embeddings,
362 + cache_config=cache_config,
363 + quant_config=quant_config,
364 + layer_idx=layer_idx,
365 + )
366 + if (config.n_routed_experts is not None
367 + and layer_idx >= config.first_k_dense_replace
368 + and layer_idx % config.moe_layer_freq == 0):
369 + self.mlp = DeepseekV2MoE(config=config, quant_config=quant_config)
370 + else:
371 + self.mlp = DeepseekV2MLP(
372 + hidden_size=config.hidden_size,
373 + intermediate_size=config.intermediate_size,
374 + hidden_act=config.hidden_act,
375 + quant_config=quant_config,
376 + )
377 + self.input_layernorm = RMSNorm(config.hidden_size,
378 + eps=config.rms_norm_eps)
379 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
380 + eps=config.rms_norm_eps)
381 +
382 + def forward(
383 + self,
384 + positions: torch.Tensor,
385 + hidden_states: torch.Tensor,
386 + kv_cache: torch.Tensor,
387 + attn_metadata: AttentionMetadata,
388 + residual: Optional[torch.Tensor],
389 + ) -> torch.Tensor:
390 + # Self Attention
391 + if residual is None:
392 + residual = hidden_states
393 + hidden_states = self.input_layernorm(hidden_states)
394 + else:
395 + hidden_states, residual = self.input_layernorm(
396 + hidden_states, residual)
397 + hidden_states = self.self_attn(
398 + positions=positions,
399 + hidden_states=hidden_states,
400 + kv_cache=kv_cache,
401 + attn_metadata=attn_metadata,
402 + )
403 +
404 + # Fully Connected
405 + hidden_states, residual = self.post_attention_layernorm(
406 + hidden_states, residual)
407 + hidden_states = self.mlp(hidden_states)
408 + return hidden_states, residual
409 +
410 +
411 +class DeepseekV2Model(nn.Module):
412 +
413 + fall_back_to_pt_during_load = False
414 +
415 + def __init__(
416 + self,
417 + config: PretrainedConfig,
418 + cache_config: Optional[CacheConfig] = None,
419 + quant_config: Optional[QuantizationConfig] = None,
420 + ) -> None:
421 + super().__init__()
422 + self.padding_idx = config.pad_token_id
423 + self.vocab_size = config.vocab_size
424 +
425 + self.embed_tokens = VocabParallelEmbedding(
426 + config.vocab_size,
427 + config.hidden_size,
428 + )
429 + self.layers = nn.ModuleList([
430 + DeepseekV2DecoderLayer(config,
431 + layer_idx,
432 + cache_config=cache_config,
433 + quant_config=quant_config)
434 + for layer_idx in range(config.num_hidden_layers)
435 + ])
436 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
437 +
438 + def forward(
439 + self,
440 + input_ids: torch.Tensor,
441 + positions: torch.Tensor,
442 + kv_caches: List[torch.Tensor],
443 + attn_metadata: AttentionMetadata,
444 + ) -> torch.Tensor:
445 + hidden_states = self.embed_tokens(input_ids)
446 + residual = None
447 + for i in range(len(self.layers)):
448 + layer = self.layers[i]
449 + hidden_states, residual = layer(positions, hidden_states,
450 + kv_caches[i], attn_metadata,
451 + residual)
452 + hidden_states, _ = self.norm(hidden_states, residual)
453 + return hidden_states
454 +
455 +
456 +class DeepseekV2ForCausalLM(nn.Module):
457 +
458 + def __init__(
459 + self,
460 + config: PretrainedConfig,
461 + cache_config: Optional[CacheConfig] = None,
462 + quant_config: Optional[QuantizationConfig] = None,
463 + ) -> None:
464 + super().__init__()
465 + self.config = config
466 + self.quant_config = quant_config
467 + self.model = DeepseekV2Model(config, cache_config, quant_config)
468 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
469 + self.logits_processor = LogitsProcessor(config.vocab_size)
470 + self.sampler = Sampler()
471 +
472 + def forward(
473 + self,
474 + input_ids: torch.Tensor,
475 + positions: torch.Tensor,
476 + kv_caches: List[torch.Tensor],
477 + attn_metadata: AttentionMetadata,
478 + ) -> torch.Tensor:
479 + hidden_states = self.model(input_ids, positions, kv_caches,
480 + attn_metadata)
481 + return hidden_states
482 +
483 + def compute_logits(self, hidden_states: torch.Tensor,
484 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
485 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
486 + sampling_metadata)
487 + return logits
488 +
489 + def sample(
490 + self,
491 + logits: Optional[torch.Tensor],
492 + sampling_metadata: SamplingMetadata,
493 + ) -> Optional[SamplerOutput]:
494 + next_tokens = self.sampler(logits, sampling_metadata)
495 + return next_tokens
496 +
497 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
498 + stacked_params_mapping = [
499 + # (param_name, shard_name, shard_id)
500 + ("gate_up_proj", "gate_proj", 0),
501 + ("gate_up_proj", "up_proj", 1),
502 + ]
503 +
504 + params_dict = dict(self.named_parameters())
505 + for name, loaded_weight in weights:
506 + if "rotary_emb.inv_freq" in name:
507 + continue
508 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
509 + if weight_name not in name:
510 + continue
511 + name = name.replace(weight_name, param_name)
512 + # Skip loading extra bias for GPTQ models.
513 + if name.endswith(".bias") and name not in params_dict:
514 + continue
515 + # Skip experts that are not assigned to this worker.
516 + if (("mlp.experts." in name or "mlp.shared_experts." in name)
517 + and name not in params_dict):
518 + continue
519 + param = params_dict[name]
520 + weight_loader = param.weight_loader
521 + weight_loader(param, loaded_weight, shard_id)
522 + break
523 + else:
524 + # Skip loading extra bias for GPTQ models.
525 + if name.endswith(".bias") and name not in params_dict:
526 + continue
527 + # Skip experts that are not assigned to this worker.
528 + if (("mlp.experts." in name or "mlp.shared_experts." in name)
529 + and name not in params_dict):
530 + continue
531 + param = params_dict[name]
532 + weight_loader = getattr(param, "weight_loader",
533 + default_weight_loader)
534 + weight_loader(param, loaded_weight)