@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 449d1bce02
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc] Remove duplicated DeepSeek V2/V3 model definition (#12793)

Michael Goin committed Feb 6, 2025 at 02:16 UTC 449d1bce029f87e0d1cf3f30483687ff659268f2
4 files changed +36 -821
vllm/config.py
-1
@@ -754,7 +754,6 @@ class ModelConfig:
754
755 @property
756 def is_deepseek_mla(self) -> bool:
757 - # TODO add deepseek_v3
757 return (hasattr(self.hf_text_config, "model_type")) \
758 and (self.hf_text_config.model_type in \
759 ('deepseek_v2', 'deepseek_v3'))\
vllm/model_executor/models/deepseek_v2.py
+35 -13
@@ -21,7 +21,7 @@
21 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 # See the License for the specific language governing permissions and
23 # limitations under the License.
24 -"""Inference-only DeepseekV2 model."""
24 +"""Inference-only DeepseekV2/DeepseekV3 model."""
25 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
26
27 import torch
@@ -115,23 +115,32 @@ class DeepseekV2MoE(nn.Module):
115 raise ValueError(f"Unsupported activation: {config.hidden_act}. "
116 "Only silu is supported for now.")
117
118 - self.experts = FusedMoE(num_experts=config.n_routed_experts,
119 - top_k=config.num_experts_per_tok,
120 - hidden_size=config.hidden_size,
121 - intermediate_size=config.moe_intermediate_size,
122 - reduce_results=False,
123 - renormalize=config.norm_topk_prob,
124 - quant_config=quant_config,
125 - use_grouped_topk=True,
126 - num_expert_group=config.n_group,
127 - topk_group=config.topk_group,
128 - prefix=f"{prefix}.experts")
129 -
118 self.gate = ReplicatedLinear(config.hidden_size,
119 config.n_routed_experts,
120 bias=False,
121 quant_config=None,
122 prefix=f"{prefix}.gate")
123 + if config.topk_method == "noaux_tc":
124 + self.gate.e_score_correction_bias = nn.Parameter(
125 + torch.empty(config.n_routed_experts))
126 + else:
127 + self.gate.e_score_correction_bias = None
128 +
129 + self.experts = FusedMoE(
130 + num_experts=config.n_routed_experts,
131 + top_k=config.num_experts_per_tok,
132 + hidden_size=config.hidden_size,
133 + intermediate_size=config.moe_intermediate_size,
134 + reduce_results=False,
135 + renormalize=config.norm_topk_prob,
136 + quant_config=quant_config,
137 + use_grouped_topk=True,
138 + num_expert_group=config.n_group,
139 + topk_group=config.topk_group,
140 + prefix=f"{prefix}.experts",
141 + scoring_func=config.scoring_func,
142 + e_score_correction_bias=self.gate.e_score_correction_bias)
143 +
144 if config.n_shared_experts is not None:
145 intermediate_size = (config.moe_intermediate_size *
146 config.n_shared_experts)
@@ -732,6 +741,15 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
741 for name, loaded_weight in weights:
742 if "rotary_emb.inv_freq" in name:
743 continue
744 +
745 + # TODO(simon): support nextn predict layers
746 + if hasattr(self.config, "num_nextn_predict_layers"
747 + ) and self.config.num_nextn_predict_layers > 0:
748 + assert self.config.num_nextn_predict_layers == 1
749 + layer_idx = self.config.num_hidden_layers
750 + if name.startswith(f"model.layers.{layer_idx}"):
751 + continue
752 +
753 for (param_name, weight_name, shard_id) in stacked_params_mapping:
754 # Skip non-stacked layers and experts (experts handled below).
755 if weight_name not in name:
@@ -793,3 +811,7 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
811 weight_loader(param, loaded_weight)
812 loaded_params.add(name)
813 return loaded_params
814 +
815 +
816 +class DeepseekV3ForCausalLM(DeepseekV2ForCausalLM):
817 + pass
vllm/model_executor/models/deepseek_v3.py deleted
-806
@@ -1,806 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -# Adapted from
4 -# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
5 -# Copyright 2023 The vLLM team.
6 -# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
7 -#
8 -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9 -# and OPT implementations in this library. It has been modified from its
10 -# original forms to accommodate minor architectural differences compared
11 -# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 -#
13 -# Licensed under the Apache License, Version 2.0 (the "License");
14 -# you may not use this file except in compliance with the License.
15 -# You may obtain a copy of the License at
16 -#
17 -# http://www.apache.org/licenses/LICENSE-2.0
18 -#
19 -# Unless required by applicable law or agreed to in writing, software
20 -# distributed under the License is distributed on an "AS IS" BASIS,
21 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 -# See the License for the specific language governing permissions and
23 -# limitations under the License.
24 -"""Inference-only DeepseekV3 model."""
25 -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
26 -
27 -import torch
28 -from torch import nn
29 -from transformers import PretrainedConfig
30 -
31 -from vllm.attention import Attention, AttentionMetadata
32 -from vllm.compilation.decorators import support_torch_compile
33 -from vllm.config import CacheConfig, ModelConfig, VllmConfig
34 -from vllm.distributed import (get_pp_group,
35 - get_tensor_model_parallel_world_size,
36 - tensor_model_parallel_all_reduce)
37 -from vllm.model_executor.layers.activation import SiluAndMul
38 -from vllm.model_executor.layers.fused_moe import FusedMoE
39 -from vllm.model_executor.layers.layernorm import RMSNorm
40 -from vllm.model_executor.layers.linear import (ColumnParallelLinear,
41 - MergedColumnParallelLinear,
42 - ReplicatedLinear,
43 - RowParallelLinear)
44 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
45 -from vllm.model_executor.layers.quantization import QuantizationConfig
46 -from vllm.model_executor.layers.rotary_embedding import get_rope
47 -from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
48 -from vllm.model_executor.layers.vocab_parallel_embedding import (
49 - ParallelLMHead, VocabParallelEmbedding)
50 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51 -from vllm.model_executor.sampling_metadata import SamplingMetadata
52 -from vllm.sequence import IntermediateTensors
53 -
54 -from .interfaces import SupportsPP
55 -from .utils import (PPMissingLayer, is_pp_missing_parameter,
56 - make_empty_intermediate_tensors_factory, make_layers,
57 - maybe_prefix)
58 -
59 -
60 -class DeepseekV3MLP(nn.Module):
61 -
62 - def __init__(
63 - self,
64 - hidden_size: int,
65 - intermediate_size: int,
66 - hidden_act: str,
67 - quant_config: Optional[QuantizationConfig] = None,
68 - reduce_results: bool = True,
69 - prefix: str = "",
70 - ) -> None:
71 - super().__init__()
72 - self.gate_up_proj = MergedColumnParallelLinear(
73 - hidden_size, [intermediate_size] * 2,
74 - bias=False,
75 - quant_config=quant_config,
76 - prefix=f"{prefix}.gate_up_proj")
77 - self.down_proj = RowParallelLinear(intermediate_size,
78 - hidden_size,
79 - bias=False,
80 - quant_config=quant_config,
81 - reduce_results=reduce_results,
82 - prefix=f"{prefix}.down_proj")
83 - if hidden_act != "silu":
84 - raise ValueError(f"Unsupported activation: {hidden_act}. "
85 - "Only silu is supported for now.")
86 - self.act_fn = SiluAndMul()
87 -
88 - def forward(self, x):
89 - gate_up, _ = self.gate_up_proj(x)
90 - x = self.act_fn(gate_up)
91 - x, _ = self.down_proj(x)
92 - return x
93 -
94 -
95 -class DeepseekV3MoE(nn.Module):
96 -
97 - def __init__(
98 - self,
99 - config: PretrainedConfig,
100 - quant_config: Optional[QuantizationConfig] = None,
101 - prefix: str = "",
102 - ):
103 - super().__init__()
104 - self.tp_size = get_tensor_model_parallel_world_size()
105 - self.routed_scaling_factor = config.routed_scaling_factor
106 - self.n_shared_experts = config.n_shared_experts
107 - self.routed_scaling_factor = config.routed_scaling_factor
108 - if self.tp_size > config.n_routed_experts:
109 - raise ValueError(
110 - f"Tensor parallel size {self.tp_size} is greater than "
111 - f"the number of experts {config.n_routed_experts}.")
112 -
113 - if config.hidden_act != "silu":
114 - raise ValueError(f"Unsupported activation: {config.hidden_act}. "
115 - "Only silu is supported for now.")
116 -
117 - self.gate = ReplicatedLinear(config.hidden_size,
118 - config.n_routed_experts,
119 - bias=False,
120 - quant_config=None,
121 - prefix=f"{prefix}.gate")
122 - if config.topk_method == "noaux_tc":
123 - self.gate.e_score_correction_bias = nn.Parameter(
124 - torch.empty(config.n_routed_experts))
125 - else:
126 - self.gate.e_score_correction_bias = None
127 -
128 - self.experts = FusedMoE(
129 - num_experts=config.n_routed_experts,
130 - top_k=config.num_experts_per_tok,
131 - hidden_size=config.hidden_size,
132 - intermediate_size=config.moe_intermediate_size,
133 - reduce_results=False,
134 - renormalize=config.norm_topk_prob,
135 - quant_config=quant_config,
136 - use_grouped_topk=True,
137 - num_expert_group=config.n_group,
138 - topk_group=config.topk_group,
139 - prefix=f"{prefix}.experts",
140 - scoring_func=config.scoring_func,
141 - e_score_correction_bias=self.gate.e_score_correction_bias)
142 -
143 - if config.n_shared_experts is not None:
144 - intermediate_size = (config.moe_intermediate_size *
145 - config.n_shared_experts)
146 - self.shared_experts = DeepseekV3MLP(
147 - hidden_size=config.hidden_size,
148 - intermediate_size=intermediate_size,
149 - hidden_act=config.hidden_act,
150 - quant_config=quant_config,
151 - reduce_results=False,
152 - )
153 -
154 - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
155 - num_tokens, hidden_dim = hidden_states.shape
156 - hidden_states = hidden_states.view(-1, hidden_dim)
157 - if self.n_shared_experts is not None:
158 - shared_output = self.shared_experts(hidden_states)
159 - # router_logits: (num_tokens, n_experts)
160 - router_logits, _ = self.gate(hidden_states)
161 - final_hidden_states = self.experts(
162 - hidden_states=hidden_states,
163 - router_logits=router_logits) * self.routed_scaling_factor
164 - if shared_output is not None:
165 - final_hidden_states = final_hidden_states + shared_output
166 - if self.tp_size > 1:
167 - final_hidden_states = tensor_model_parallel_all_reduce(
168 - final_hidden_states)
169 -
170 - return final_hidden_states.view(num_tokens, hidden_dim)
171 -
172 -
173 -def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
174 - import math
175 - if scale <= 1:
176 - return 1.0
177 - return 0.1 * mscale * math.log(scale) + 1.0
178 -
179 -
180 -class DeepseekV3Attention(nn.Module):
181 -
182 - def __init__(
183 - self,
184 - config: PretrainedConfig,
185 - hidden_size: int,
186 - num_heads: int,
187 - qk_nope_head_dim: int,
188 - qk_rope_head_dim: int,
189 - v_head_dim: int,
190 - q_lora_rank: int,
191 - kv_lora_rank: int,
192 - rope_theta: float = 10000,
193 - rope_scaling: Optional[Dict[str, Any]] = None,
194 - max_position_embeddings: int = 8192,
195 - cache_config: Optional[CacheConfig] = None,
196 - quant_config: Optional[QuantizationConfig] = None,
197 - prefix: str = "",
198 - ) -> None:
199 - super().__init__()
200 - self.hidden_size = hidden_size
201 - self.qk_nope_head_dim = qk_nope_head_dim
202 - self.qk_rope_head_dim = qk_rope_head_dim
203 - self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
204 - self.v_head_dim = v_head_dim
205 - self.q_lora_rank = q_lora_rank
206 - self.kv_lora_rank = kv_lora_rank
207 - self.num_heads = num_heads
208 - tp_size = get_tensor_model_parallel_world_size()
209 - assert num_heads % tp_size == 0
210 - self.num_local_heads = num_heads // tp_size
211 - self.scaling = self.qk_head_dim**-0.5
212 - self.rope_theta = rope_theta
213 - self.max_position_embeddings = max_position_embeddings
214 -
215 - if self.q_lora_rank is not None:
216 - self.q_a_proj = ReplicatedLinear(self.hidden_size,
217 - self.q_lora_rank,
218 - bias=False,
219 - quant_config=quant_config,
220 - prefix=f"{prefix}.q_a_proj")
221 - self.q_a_layernorm = RMSNorm(self.q_lora_rank,
222 - eps=config.rms_norm_eps)
223 - self.q_b_proj = ColumnParallelLinear(q_lora_rank,
224 - self.num_heads *
225 - self.qk_head_dim,
226 - bias=False,
227 - quant_config=quant_config,
228 - prefix=f"{prefix}.q_b_proj")
229 - else:
230 - self.q_proj = ColumnParallelLinear(self.hidden_size,
231 - self.num_heads *
232 - self.qk_head_dim,
233 - bias=False,
234 - quant_config=quant_config,
235 - prefix=f"{prefix}.q_proj")
236 -
237 - self.kv_a_proj_with_mqa = ReplicatedLinear(
238 - self.hidden_size,
239 - self.kv_lora_rank + self.qk_rope_head_dim,
240 - bias=False,
241 - quant_config=quant_config,
242 - prefix=f"{prefix}.kv_a_proj_with_mqa")
243 - self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
244 - eps=config.rms_norm_eps)
245 - self.kv_b_proj = ColumnParallelLinear(
246 - self.kv_lora_rank,
247 - self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
248 - bias=False,
249 - quant_config=quant_config,
250 - prefix=f"{prefix}.kv_b_proj")
251 - # O projection.
252 - self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
253 - self.hidden_size,
254 - bias=False,
255 - quant_config=quant_config,
256 - prefix=f"{prefix}.o_proj")
257 - if rope_scaling:
258 - rope_scaling["rope_type"] = 'deepseek_yarn'
259 - self.use_normal_rope = False
260 - else:
261 - self.use_normal_rope = True
262 - self.rotary_emb = get_rope(qk_rope_head_dim,
263 - rotary_dim=qk_rope_head_dim,
264 - max_position=max_position_embeddings,
265 - base=rope_theta,
266 - rope_scaling=rope_scaling,
267 - is_neox_style=False)
268 -
269 - if rope_scaling:
270 - mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
271 - scaling_factor = rope_scaling["factor"]
272 - mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
273 - self.scaling = self.scaling * mscale * mscale
274 -
275 - self.attn = Attention(self.num_local_heads,
276 - self.qk_head_dim,
277 - self.scaling,
278 - num_kv_heads=self.num_local_heads,
279 - cache_config=cache_config,
280 - quant_config=quant_config,
281 - prefix=f"{prefix}.attn")
282 -
283 - def forward(
284 - self,
285 - positions: torch.Tensor,
286 - hidden_states: torch.Tensor,
287 - kv_cache: torch.Tensor,
288 - attn_metadata: AttentionMetadata,
289 - ) -> torch.Tensor:
290 - if self.q_lora_rank is not None:
291 - q = self.q_a_proj(hidden_states)[0]
292 - q = self.q_a_layernorm(q)
293 - q = self.q_b_proj(q)[0].view(-1, self.num_local_heads,
294 - self.qk_head_dim)
295 - else:
296 - q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads,
297 - self.qk_head_dim)
298 - q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim],
299 - dim=-1)
300 - latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0]
301 - kv_a, _ = latent_cache.split(
302 - [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
303 - latent_cache = latent_cache.unsqueeze(1)
304 - kv_a = self.kv_a_layernorm(kv_a.contiguous())
305 - kv = self.kv_b_proj(kv_a)[0]
306 - kv = kv.view(-1, self.num_local_heads,
307 - self.qk_nope_head_dim + self.v_head_dim)
308 - k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
309 - k_pe = latent_cache[:, :, self.kv_lora_rank:]
310 -
311 - if self.use_normal_rope:
312 - seq_len = positions.size(0)
313 - ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
314 - q_pe = q_pe.reshape(seq_len, -1)
315 - k_pe = k_pe.reshape(seq_len, -1)
316 -
317 - q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
318 -
319 - if self.use_normal_rope:
320 - q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
321 -
322 - q[..., self.qk_nope_head_dim:] = q_pe
323 - k = torch.empty_like(q)
324 - k[..., :self.qk_nope_head_dim] = k_nope
325 - k[..., self.qk_nope_head_dim:] = k_pe
326 - # padding value to qk_head_dim for alignment
327 - v = torch.nn.functional.pad(
328 - v, [0, self.qk_head_dim - self.v_head_dim],
329 - value=0).view(-1, self.num_local_heads * self.qk_head_dim)
330 - attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
331 - attn_output = attn_output.view(
332 - -1, self.num_local_heads,
333 - self.qk_head_dim)[..., :self.v_head_dim].reshape(
334 - -1, self.num_local_heads * self.v_head_dim)
335 - output, _ = self.o_proj(attn_output)
336 - return output
337 -
338 -
339 -class DeepseekV3MLAAttention(nn.Module):
340 - """
341 - Main reference: DeepseekV2 paper, and FlashInfer Implementation
342 - (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
343 -
344 - For more info see MLACommonImpl in: vllm/attention/backends/mla/utils.py
345 - """
346 -
347 - def __init__(
348 - self,
349 - config: PretrainedConfig,
350 - hidden_size: int,
351 - num_heads: int,
352 - qk_nope_head_dim: int,
353 - qk_rope_head_dim: int,
354 - v_head_dim: int,
355 - q_lora_rank: Optional[int],
356 - kv_lora_rank: int,
357 - rope_theta: float = 10000,
358 - rope_scaling: Optional[Dict[str, Any]] = None,
359 - max_position_embeddings: int = 8192,
360 - cache_config: Optional[CacheConfig] = None,
361 - quant_config: Optional[QuantizationConfig] = None,
362 - prefix: str = "",
363 - ) -> None:
364 - super().__init__()
365 - self.hidden_size = hidden_size
366 - self.qk_nope_head_dim = qk_nope_head_dim
367 - self.qk_rope_head_dim = qk_rope_head_dim
368 - self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
369 - self.v_head_dim = v_head_dim
370 -
371 - self.q_lora_rank = q_lora_rank
372 - self.kv_lora_rank = kv_lora_rank
373 -
374 - self.num_heads = num_heads
375 - tp_size = get_tensor_model_parallel_world_size()
376 - assert num_heads % tp_size == 0
377 - self.num_local_heads = num_heads // tp_size
378 -
379 - self.scaling = self.qk_head_dim**-0.5
380 - self.rope_theta = rope_theta
381 - self.max_position_embeddings = max_position_embeddings
382 -
383 - if self.q_lora_rank is not None:
384 - self.q_a_proj = ReplicatedLinear(self.hidden_size,
385 - self.q_lora_rank,
386 - bias=False,
387 - quant_config=quant_config,
388 - prefix=f"{prefix}.q_a_proj")
389 - self.q_a_layernorm = RMSNorm(self.q_lora_rank,
390 - eps=config.rms_norm_eps)
391 - self.q_b_proj = ColumnParallelLinear(q_lora_rank,
392 - self.num_heads *
393 - self.qk_head_dim,
394 - bias=False,
395 - quant_config=quant_config,
396 - prefix=f"{prefix}.q_b_proj")
397 - else:
398 - self.q_proj = ColumnParallelLinear(self.hidden_size,
399 - self.num_heads *
400 - self.qk_head_dim,
401 - bias=False,
402 - quant_config=quant_config,
403 - prefix=f"{prefix}.q_proj")
404 -
405 - self.kv_a_proj_with_mqa = ReplicatedLinear(
406 - self.hidden_size,
407 - self.kv_lora_rank + self.qk_rope_head_dim,
408 - bias=False,
409 - quant_config=quant_config,
410 - prefix=f"{prefix}.kv_a_proj_with_mqa")
411 - self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
412 - eps=config.rms_norm_eps)
413 - self.kv_b_proj = ColumnParallelLinear(
414 - self.kv_lora_rank,
415 - self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
416 - bias=False,
417 - quant_config=quant_config,
418 - prefix=f"{prefix}.kv_b_proj")
419 - self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
420 - self.hidden_size,
421 - bias=False,
422 - quant_config=quant_config,
423 - prefix=f"{prefix}.o_proj")
424 -
425 - if rope_scaling:
426 - rope_scaling["rope_type"] = 'deepseek_yarn'
427 - self.rotary_emb = get_rope(qk_rope_head_dim,
428 - rotary_dim=qk_rope_head_dim,
429 - max_position=max_position_embeddings,
430 - base=rope_theta,
431 - rope_scaling=rope_scaling,
432 - is_neox_style=False)
433 - if rope_scaling:
434 - mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
435 - scaling_factor = rope_scaling["factor"]
436 - mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
437 - self.scaling = self.scaling * mscale * mscale
438 -
439 - self.mla_attn = Attention(
440 - num_heads=self.num_local_heads,
441 - head_size=self.kv_lora_rank,
442 - scale=self.scaling,
443 - num_kv_heads=1,
444 - cache_config=cache_config,
445 - quant_config=quant_config,
446 - prefix=f"{prefix}.attn",
447 - use_mla=True,
448 - # MLA Args
449 - q_lora_rank=self.q_lora_rank,
450 - kv_lora_rank=self.kv_lora_rank,
451 - qk_nope_head_dim=self.qk_nope_head_dim,
452 - qk_rope_head_dim=self.qk_rope_head_dim,
453 - qk_head_dim=self.qk_head_dim,
454 - v_head_dim=self.v_head_dim,
455 - rotary_emb=self.rotary_emb,
456 - q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj,
457 - kv_b_proj=self.kv_b_proj,
458 - o_proj=self.o_proj,
459 - )
460 -
461 - self.prefix = prefix
462 - self.debug_layer_idx = int(self.prefix.split(".")[-2])
463 -
464 - def forward(
465 - self,
466 - positions: torch.Tensor,
467 - hidden_states: torch.Tensor,
468 - kv_cache: torch.Tensor,
469 - attn_metadata: AttentionMetadata,
470 - ) -> torch.Tensor:
471 - if self.q_lora_rank is not None:
472 - ckq = self.q_a_proj(hidden_states)[0]
473 - hidden_states_or_q_c = self.q_a_layernorm(ckq)
474 - else:
475 - hidden_states_or_q_c = hidden_states
476 - kv_c, k_pe = self.kv_a_proj_with_mqa(hidden_states)[0].split(
477 - [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
478 - kv_c_normed = self.kv_a_layernorm(kv_c.contiguous())
479 - return self.mla_attn(hidden_states_or_q_c, kv_c_normed, k_pe, kv_cache,
480 - attn_metadata)
481 -
482 -
483 -class DeepseekV3DecoderLayer(nn.Module):
484 -
485 - def __init__(
486 - self,
487 - config: PretrainedConfig,
488 - prefix: str,
489 - model_config: ModelConfig,
490 - cache_config: Optional[CacheConfig] = None,
491 - quant_config: Optional[QuantizationConfig] = None,
492 - ) -> None:
493 - super().__init__()
494 - self.hidden_size = config.hidden_size
495 - rope_theta = getattr(config, "rope_theta", 10000)
496 - rope_scaling = getattr(config, "rope_scaling", None)
497 - max_position_embeddings = getattr(config, "max_position_embeddings",
498 - 8192)
499 - # DecoderLayers are created with `make_layers` which passes the prefix
500 - # with the layer's index.
501 - layer_idx = int(prefix.split(sep='.')[-1])
502 - if model_config.use_mla:
503 - attn_cls = DeepseekV3MLAAttention
504 - else:
505 - attn_cls = DeepseekV3Attention
506 - self.self_attn = attn_cls(
507 - config=config,
508 - hidden_size=self.hidden_size,
509 - num_heads=config.num_attention_heads,
510 - qk_nope_head_dim=config.qk_nope_head_dim,
511 - qk_rope_head_dim=config.qk_rope_head_dim,
512 - v_head_dim=config.v_head_dim,
513 - q_lora_rank=config.q_lora_rank
514 - if hasattr(config, "q_lora_rank") else None,
515 - kv_lora_rank=config.kv_lora_rank,
516 - rope_theta=rope_theta,
517 - rope_scaling=rope_scaling,
518 - max_position_embeddings=max_position_embeddings,
519 - cache_config=cache_config,
520 - quant_config=quant_config,
521 - prefix=f"{prefix}.self_attn",
522 - )
523 - if (config.n_routed_experts is not None
524 - and layer_idx >= config.first_k_dense_replace
525 - and layer_idx % config.moe_layer_freq == 0):
526 - self.mlp = DeepseekV3MoE(
527 - config=config,
528 - quant_config=quant_config,
529 - prefix=f"{prefix}.mlp",
530 - )
531 - else:
532 - self.mlp = DeepseekV3MLP(
533 - hidden_size=config.hidden_size,
534 - intermediate_size=config.intermediate_size,
535 - hidden_act=config.hidden_act,
536 - quant_config=quant_config,
537 - prefix=f"{prefix}.mlp",
538 - )
539 - self.input_layernorm = RMSNorm(config.hidden_size,
540 - eps=config.rms_norm_eps)
541 - self.post_attention_layernorm = RMSNorm(config.hidden_size,
542 - eps=config.rms_norm_eps)
543 -
544 - def forward(
545 - self,
546 - positions: torch.Tensor,
547 - hidden_states: torch.Tensor,
548 - kv_cache: torch.Tensor,
549 - attn_metadata: AttentionMetadata,
550 - residual: Optional[torch.Tensor],
551 - ) -> torch.Tensor:
552 - # Self Attention
553 - if residual is None:
554 - residual = hidden_states
555 - hidden_states = self.input_layernorm(hidden_states)
556 - else:
557 - hidden_states, residual = self.input_layernorm(
558 - hidden_states, residual)
559 - hidden_states = self.self_attn(
560 - positions=positions,
561 - hidden_states=hidden_states,
562 - kv_cache=kv_cache,
563 - attn_metadata=attn_metadata,
564 - )
565 -
566 - # Fully Connected
567 - hidden_states, residual = self.post_attention_layernorm(
568 - hidden_states, residual)
569 - hidden_states = self.mlp(hidden_states)
570 - return hidden_states, residual
571 -
572 -
573 -@support_torch_compile
574 -class DeepseekV3Model(nn.Module):
575 -
576 - fall_back_to_pt_during_load = False
577 -
578 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
579 - super().__init__()
580 -
581 - config = vllm_config.model_config.hf_config
582 - model_config = vllm_config.model_config
583 - cache_config = vllm_config.cache_config
584 - quant_config = vllm_config.quant_config
585 -
586 - self.padding_idx = config.pad_token_id
587 - self.vocab_size = config.vocab_size
588 -
589 - if get_pp_group().is_first_rank:
590 - self.embed_tokens = VocabParallelEmbedding(
591 - config.vocab_size,
592 - config.hidden_size,
593 - )
594 - else:
595 - self.embed_tokens = PPMissingLayer()
596 -
597 - self.start_layer, self.end_layer, self.layers = make_layers(
598 - config.num_hidden_layers,
599 - lambda prefix: DeepseekV3DecoderLayer(
600 - config,
601 - prefix,
602 - model_config=model_config,
603 - cache_config=cache_config,
604 - quant_config=quant_config,
605 - ),
606 - prefix=f"{prefix}.layers")
607 -
608 - if get_pp_group().is_last_rank:
609 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
610 - else:
611 - self.norm = PPMissingLayer()
612 - self.make_empty_intermediate_tensors = (
613 - make_empty_intermediate_tensors_factory(
614 - ["hidden_states", "residual"], config.hidden_size))
615 -
616 - def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
617 - return self.embed_tokens(input_ids)
618 -
619 - def forward(
620 - self,
621 - input_ids: torch.Tensor,
622 - positions: torch.Tensor,
623 - kv_caches: List[torch.Tensor],
624 - attn_metadata: AttentionMetadata,
625 - intermediate_tensors: Optional[IntermediateTensors],
626 - inputs_embeds: Optional[torch.Tensor] = None,
627 - ) -> Union[torch.Tensor, IntermediateTensors]:
628 - if get_pp_group().is_first_rank:
629 - if inputs_embeds is not None:
630 - hidden_states = inputs_embeds
631 - else:
632 - hidden_states = self.get_input_embeddings(input_ids)
633 - residual = None
634 - else:
635 - assert intermediate_tensors is not None
636 - hidden_states = intermediate_tensors["hidden_states"]
637 - residual = intermediate_tensors["residual"]
638 -
639 - for i in range(self.start_layer, self.end_layer):
640 - layer = self.layers[i]
641 - hidden_states, residual = layer(positions, hidden_states,
642 - kv_caches[i - self.start_layer],
643 - attn_metadata, residual)
644 -
645 - if not get_pp_group().is_last_rank:
646 - return IntermediateTensors({
647 - "hidden_states": hidden_states,
648 - "residual": residual
649 - })
650 -
651 - hidden_states, _ = self.norm(hidden_states, residual)
652 - return hidden_states
653 -
654 -
655 -class DeepseekV3ForCausalLM(nn.Module, SupportsPP):
656 -
657 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
658 - super().__init__()
659 - config = vllm_config.model_config.hf_config
660 - quant_config = vllm_config.quant_config
661 - self.config = config
662 - self.quant_config = quant_config
663 - self.model = DeepseekV3Model(vllm_config=vllm_config,
664 - prefix=maybe_prefix(prefix, "model"))
665 - self.lm_head = ParallelLMHead(config.vocab_size,
666 - config.hidden_size,
667 - quant_config=quant_config)
668 - self.logits_processor = LogitsProcessor(config.vocab_size)
669 - self.sampler = get_sampler()
670 - self.make_empty_intermediate_tensors = (
671 - self.model.make_empty_intermediate_tensors)
672 -
673 - def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
674 - return self.model.get_input_embeddings(input_ids)
675 -
676 - def forward(
677 - self,
678 - input_ids: torch.Tensor,
679 - positions: torch.Tensor,
680 - kv_caches: List[torch.Tensor],
681 - attn_metadata: AttentionMetadata,
682 - intermediate_tensors: Optional[IntermediateTensors] = None,
683 - inputs_embeds: Optional[torch.Tensor] = None,
684 - ) -> Union[torch.Tensor, IntermediateTensors]:
685 - hidden_states = self.model(input_ids, positions, kv_caches,
686 - attn_metadata, intermediate_tensors,
687 - inputs_embeds)
688 - return hidden_states
689 -
690 - def compute_logits(
691 - self,
692 - hidden_states: torch.Tensor,
693 - sampling_metadata: SamplingMetadata,
694 - ) -> Optional[torch.Tensor]:
695 - logits = self.logits_processor(self.lm_head, hidden_states,
696 - sampling_metadata)
697 - return logits
698 -
699 - def sample(
700 - self,
701 - logits: Optional[torch.Tensor],
702 - sampling_metadata: SamplingMetadata,
703 - ) -> Optional[SamplerOutput]:
704 - next_tokens = self.sampler(logits, sampling_metadata)
705 - return next_tokens
706 -
707 - def make_empty_intermediate_tensors(
708 - self, batch_size: int, dtype: torch.dtype,
709 - device: torch.device) -> IntermediateTensors:
710 - return IntermediateTensors({
711 - "hidden_states":
712 - torch.zeros((batch_size, self.config.hidden_size),
713 - dtype=dtype,
714 - device=device),
715 - "residual":
716 - torch.zeros((batch_size, self.config.hidden_size),
717 - dtype=dtype,
718 - device=device),
719 - })
720 -
721 - def load_weights(self, weights: Iterable[Tuple[str,
722 - torch.Tensor]]) -> Set[str]:
723 - stacked_params_mapping = [
724 - # (param_name, shard_name, shard_id)
725 - ("gate_up_proj", "gate_proj", 0),
726 - ("gate_up_proj", "up_proj", 1),
727 - ]
728 -
729 - # Params for weights, fp8 weight scales, fp8 activation scales
730 - # (param_name, weight_name, expert_id, shard_id)
731 - expert_params_mapping = FusedMoE.make_expert_params_mapping(
732 - ckpt_gate_proj_name="gate_proj",
733 - ckpt_down_proj_name="down_proj",
734 - ckpt_up_proj_name="up_proj",
735 - num_experts=self.config.n_routed_experts)
736 -
737 - params_dict = dict(self.named_parameters())
738 - loaded_params: Set[str] = set()
739 - for name, loaded_weight in weights:
740 - if "rotary_emb.inv_freq" in name:
741 - continue
742 -
743 - # TODO(simon): support nextn predict layers
744 - if hasattr(self.config, "num_nextn_predict_layers"
745 - ) and self.config.num_nextn_predict_layers > 0:
746 - assert self.config.num_nextn_predict_layers == 1
747 - layer_idx = self.config.num_hidden_layers
748 - if name.startswith(f"model.layers.{layer_idx}"):
749 - continue
750 -
751 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
752 - # Skip non-stacked layers and experts (experts handled below).
753 - if weight_name not in name:
754 - continue
755 - # We have mlp.experts[0].gate_proj in the checkpoint.
756 - # Since we handle the experts below in expert_params_mapping,
757 - # we need to skip here BEFORE we update the name, otherwise
758 - # name will be updated to mlp.experts[0].gate_up_proj, which
759 - # will then be updated below in expert_params_mapping
760 - # for mlp.experts[0].gate_gate_up_proj, which breaks load.
761 - if (("mlp.experts." in name) and name not in params_dict):
762 - continue
763 - name = name.replace(weight_name, param_name)
764 - # Skip loading extra bias for GPTQ models.
765 - if name.endswith(".bias") and name not in params_dict:
766 - continue
767 -
768 - if is_pp_missing_parameter(name, self):
769 - continue
770 -
771 - param = params_dict[name]
772 - weight_loader = param.weight_loader
773 - weight_loader(param, loaded_weight, shard_id)
774 - break
775 - else:
776 - for mapping in expert_params_mapping:
777 - param_name, weight_name, expert_id, shard_id = mapping
778 - if weight_name not in name:
779 - continue
780 - name = name.replace(weight_name, param_name)
781 -
782 - if is_pp_missing_parameter(name, self):
783 - continue
784 -
785 - param = params_dict[name]
786 - weight_loader = param.weight_loader
787 - weight_loader(param,
788 - loaded_weight,
789 - name,
790 - shard_id=shard_id,
791 - expert_id=expert_id)
792 - break
793 - else:
794 - # Skip loading extra bias for GPTQ models.
795 - if name.endswith(".bias") and name not in params_dict:
796 - continue
797 -
798 - if is_pp_missing_parameter(name, self):
799 - continue
800 -
801 - param = params_dict[name]
802 - weight_loader = getattr(param, "weight_loader",
803 - default_weight_loader)
804 - weight_loader(param, loaded_weight)
805 - loaded_params.add(name)
806 - return loaded_params
vllm/model_executor/models/registry.py
+1 -1
@@ -45,7 +45,7 @@ _TEXT_GENERATION_MODELS = {
45 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
46 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
47 "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
48 - "DeepseekV3ForCausalLM": ("deepseek_v3", "DeepseekV3ForCausalLM"),
48 + "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
49 "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
50 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
51 "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),