A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Misc] Remove duplicated DeepSeek V2/V3 model definition (#12793)
Michael Goin committed
Feb 6, 2025 at 02:16 UTC
449d1bce029f87e0d1cf3f30483687ff659268f2
4 files changed
+36
-821
vllm/config.py
-1
@@ -754,7 +754,6 @@ class ModelConfig:
754
755
@property
756
def is_deepseek_mla(self) -> bool:
757
- # TODO add deepseek_v3
757
return (hasattr(self.hf_text_config, "model_type")) \
758
and (self.hf_text_config.model_type in \
759
('deepseek_v2', 'deepseek_v3'))\
vllm/model_executor/models/deepseek_v2.py
+35
-13
@@ -21,7 +21,7 @@
21
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
# See the License for the specific language governing permissions and
23
# limitations under the License.
24
-"""Inference-only DeepseekV2 model."""
24
+"""Inference-only DeepseekV2/DeepseekV3 model."""
25
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
26
27
import torch
@@ -115,23 +115,32 @@ class DeepseekV2MoE(nn.Module):
115
raise ValueError(f"Unsupported activation: {config.hidden_act}. "
116
"Only silu is supported for now.")
117
118
- self.experts = FusedMoE(num_experts=config.n_routed_experts,
119
- top_k=config.num_experts_per_tok,
120
- hidden_size=config.hidden_size,
121
- intermediate_size=config.moe_intermediate_size,
122
- reduce_results=False,
123
- renormalize=config.norm_topk_prob,
124
- quant_config=quant_config,
125
- use_grouped_topk=True,
126
- num_expert_group=config.n_group,
127
- topk_group=config.topk_group,
128
- prefix=f"{prefix}.experts")
129
-
118
self.gate = ReplicatedLinear(config.hidden_size,
119
config.n_routed_experts,
120
bias=False,
121
quant_config=None,
122
prefix=f"{prefix}.gate")
123
+ if config.topk_method == "noaux_tc":
124
+ self.gate.e_score_correction_bias = nn.Parameter(
125
+ torch.empty(config.n_routed_experts))
126
+ else:
127
+ self.gate.e_score_correction_bias = None
128
+
129
+ self.experts = FusedMoE(
130
+ num_experts=config.n_routed_experts,
131
+ top_k=config.num_experts_per_tok,
132
+ hidden_size=config.hidden_size,
133
+ intermediate_size=config.moe_intermediate_size,
134
+ reduce_results=False,
135
+ renormalize=config.norm_topk_prob,
136
+ quant_config=quant_config,
137
+ use_grouped_topk=True,
138
+ num_expert_group=config.n_group,
139
+ topk_group=config.topk_group,
140
+ prefix=f"{prefix}.experts",
141
+ scoring_func=config.scoring_func,
142
+ e_score_correction_bias=self.gate.e_score_correction_bias)
143
+
144
if config.n_shared_experts is not None:
145
intermediate_size = (config.moe_intermediate_size *
146
config.n_shared_experts)
@@ -732,6 +741,15 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
741
for name, loaded_weight in weights:
742
if "rotary_emb.inv_freq" in name:
743
continue
744
+
745
+ # TODO(simon): support nextn predict layers
746
+ if hasattr(self.config, "num_nextn_predict_layers"
747
+ ) and self.config.num_nextn_predict_layers > 0:
748
+ assert self.config.num_nextn_predict_layers == 1
749
+ layer_idx = self.config.num_hidden_layers
750
+ if name.startswith(f"model.layers.{layer_idx}"):
751
+ continue
752
+
753
for (param_name, weight_name, shard_id) in stacked_params_mapping:
754
# Skip non-stacked layers and experts (experts handled below).
755
if weight_name not in name:
@@ -793,3 +811,7 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP):
811
weight_loader(param, loaded_weight)
812
loaded_params.add(name)
813
return loaded_params
814
+
815
+
816
+class DeepseekV3ForCausalLM(DeepseekV2ForCausalLM):
817
+ pass
vllm/model_executor/models/deepseek_v3.py
deleted
-806
@@ -1,806 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
3
-# Adapted from
4
-# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
5
-# Copyright 2023 The vLLM team.
6
-# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
7
-#
8
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9
-# and OPT implementations in this library. It has been modified from its
10
-# original forms to accommodate minor architectural differences compared
11
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12
-#
13
-# Licensed under the Apache License, Version 2.0 (the "License");
14
-# you may not use this file except in compliance with the License.
15
-# You may obtain a copy of the License at
16
-#
17
-# http://www.apache.org/licenses/LICENSE-2.0
18
-#
19
-# Unless required by applicable law or agreed to in writing, software
20
-# distributed under the License is distributed on an "AS IS" BASIS,
21
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
-# See the License for the specific language governing permissions and
23
-# limitations under the License.
24
-"""Inference-only DeepseekV3 model."""
25
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
26
-
27
-import torch
28
-from torch import nn
29
-from transformers import PretrainedConfig
30
-
31
-from vllm.attention import Attention, AttentionMetadata
32
-from vllm.compilation.decorators import support_torch_compile
33
-from vllm.config import CacheConfig, ModelConfig, VllmConfig
34
-from vllm.distributed import (get_pp_group,
35
- get_tensor_model_parallel_world_size,
36
- tensor_model_parallel_all_reduce)
37
-from vllm.model_executor.layers.activation import SiluAndMul
38
-from vllm.model_executor.layers.fused_moe import FusedMoE
39
-from vllm.model_executor.layers.layernorm import RMSNorm
40
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
41
- MergedColumnParallelLinear,
42
- ReplicatedLinear,
43
- RowParallelLinear)
44
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
45
-from vllm.model_executor.layers.quantization import QuantizationConfig
46
-from vllm.model_executor.layers.rotary_embedding import get_rope
47
-from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
48
-from vllm.model_executor.layers.vocab_parallel_embedding import (
49
- ParallelLMHead, VocabParallelEmbedding)
50
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
-from vllm.model_executor.sampling_metadata import SamplingMetadata
52
-from vllm.sequence import IntermediateTensors
53
-
54
-from .interfaces import SupportsPP
55
-from .utils import (PPMissingLayer, is_pp_missing_parameter,
56
- make_empty_intermediate_tensors_factory, make_layers,
57
- maybe_prefix)
58
-
59
-
60
-class DeepseekV3MLP(nn.Module):
61
-
62
- def __init__(
63
- self,
64
- hidden_size: int,
65
- intermediate_size: int,
66
- hidden_act: str,
67
- quant_config: Optional[QuantizationConfig] = None,
68
- reduce_results: bool = True,
69
- prefix: str = "",
70
- ) -> None:
71
- super().__init__()
72
- self.gate_up_proj = MergedColumnParallelLinear(
73
- hidden_size, [intermediate_size] * 2,
74
- bias=False,
75
- quant_config=quant_config,
76
- prefix=f"{prefix}.gate_up_proj")
77
- self.down_proj = RowParallelLinear(intermediate_size,
78
- hidden_size,
79
- bias=False,
80
- quant_config=quant_config,
81
- reduce_results=reduce_results,
82
- prefix=f"{prefix}.down_proj")
83
- if hidden_act != "silu":
84
- raise ValueError(f"Unsupported activation: {hidden_act}. "
85
- "Only silu is supported for now.")
86
- self.act_fn = SiluAndMul()
87
-
88
- def forward(self, x):
89
- gate_up, _ = self.gate_up_proj(x)
90
- x = self.act_fn(gate_up)
91
- x, _ = self.down_proj(x)
92
- return x
93
-
94
-
95
-class DeepseekV3MoE(nn.Module):
96
-
97
- def __init__(
98
- self,
99
- config: PretrainedConfig,
100
- quant_config: Optional[QuantizationConfig] = None,
101
- prefix: str = "",
102
- ):
103
- super().__init__()
104
- self.tp_size = get_tensor_model_parallel_world_size()
105
- self.routed_scaling_factor = config.routed_scaling_factor
106
- self.n_shared_experts = config.n_shared_experts
107
- self.routed_scaling_factor = config.routed_scaling_factor
108
- if self.tp_size > config.n_routed_experts:
109
- raise ValueError(
110
- f"Tensor parallel size {self.tp_size} is greater than "
111
- f"the number of experts {config.n_routed_experts}.")
112
-
113
- if config.hidden_act != "silu":
114
- raise ValueError(f"Unsupported activation: {config.hidden_act}. "
115
- "Only silu is supported for now.")
116
-
117
- self.gate = ReplicatedLinear(config.hidden_size,
118
- config.n_routed_experts,
119
- bias=False,
120
- quant_config=None,
121
- prefix=f"{prefix}.gate")
122
- if config.topk_method == "noaux_tc":
123
- self.gate.e_score_correction_bias = nn.Parameter(
124
- torch.empty(config.n_routed_experts))
125
- else:
126
- self.gate.e_score_correction_bias = None
127
-
128
- self.experts = FusedMoE(
129
- num_experts=config.n_routed_experts,
130
- top_k=config.num_experts_per_tok,
131
- hidden_size=config.hidden_size,
132
- intermediate_size=config.moe_intermediate_size,
133
- reduce_results=False,
134
- renormalize=config.norm_topk_prob,
135
- quant_config=quant_config,
136
- use_grouped_topk=True,
137
- num_expert_group=config.n_group,
138
- topk_group=config.topk_group,
139
- prefix=f"{prefix}.experts",
140
- scoring_func=config.scoring_func,
141
- e_score_correction_bias=self.gate.e_score_correction_bias)
142
-
143
- if config.n_shared_experts is not None:
144
- intermediate_size = (config.moe_intermediate_size *
145
- config.n_shared_experts)
146
- self.shared_experts = DeepseekV3MLP(
147
- hidden_size=config.hidden_size,
148
- intermediate_size=intermediate_size,
149
- hidden_act=config.hidden_act,
150
- quant_config=quant_config,
151
- reduce_results=False,
152
- )
153
-
154
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
155
- num_tokens, hidden_dim = hidden_states.shape
156
- hidden_states = hidden_states.view(-1, hidden_dim)
157
- if self.n_shared_experts is not None:
158
- shared_output = self.shared_experts(hidden_states)
159
- # router_logits: (num_tokens, n_experts)
160
- router_logits, _ = self.gate(hidden_states)
161
- final_hidden_states = self.experts(
162
- hidden_states=hidden_states,
163
- router_logits=router_logits) * self.routed_scaling_factor
164
- if shared_output is not None:
165
- final_hidden_states = final_hidden_states + shared_output
166
- if self.tp_size > 1:
167
- final_hidden_states = tensor_model_parallel_all_reduce(
168
- final_hidden_states)
169
-
170
- return final_hidden_states.view(num_tokens, hidden_dim)
171
-
172
-
173
-def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
174
- import math
175
- if scale <= 1:
176
- return 1.0
177
- return 0.1 * mscale * math.log(scale) + 1.0
178
-
179
-
180
-class DeepseekV3Attention(nn.Module):
181
-
182
- def __init__(
183
- self,
184
- config: PretrainedConfig,
185
- hidden_size: int,
186
- num_heads: int,
187
- qk_nope_head_dim: int,
188
- qk_rope_head_dim: int,
189
- v_head_dim: int,
190
- q_lora_rank: int,
191
- kv_lora_rank: int,
192
- rope_theta: float = 10000,
193
- rope_scaling: Optional[Dict[str, Any]] = None,
194
- max_position_embeddings: int = 8192,
195
- cache_config: Optional[CacheConfig] = None,
196
- quant_config: Optional[QuantizationConfig] = None,
197
- prefix: str = "",
198
- ) -> None:
199
- super().__init__()
200
- self.hidden_size = hidden_size
201
- self.qk_nope_head_dim = qk_nope_head_dim
202
- self.qk_rope_head_dim = qk_rope_head_dim
203
- self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
204
- self.v_head_dim = v_head_dim
205
- self.q_lora_rank = q_lora_rank
206
- self.kv_lora_rank = kv_lora_rank
207
- self.num_heads = num_heads
208
- tp_size = get_tensor_model_parallel_world_size()
209
- assert num_heads % tp_size == 0
210
- self.num_local_heads = num_heads // tp_size
211
- self.scaling = self.qk_head_dim**-0.5
212
- self.rope_theta = rope_theta
213
- self.max_position_embeddings = max_position_embeddings
214
-
215
- if self.q_lora_rank is not None:
216
- self.q_a_proj = ReplicatedLinear(self.hidden_size,
217
- self.q_lora_rank,
218
- bias=False,
219
- quant_config=quant_config,
220
- prefix=f"{prefix}.q_a_proj")
221
- self.q_a_layernorm = RMSNorm(self.q_lora_rank,
222
- eps=config.rms_norm_eps)
223
- self.q_b_proj = ColumnParallelLinear(q_lora_rank,
224
- self.num_heads *
225
- self.qk_head_dim,
226
- bias=False,
227
- quant_config=quant_config,
228
- prefix=f"{prefix}.q_b_proj")
229
- else:
230
- self.q_proj = ColumnParallelLinear(self.hidden_size,
231
- self.num_heads *
232
- self.qk_head_dim,
233
- bias=False,
234
- quant_config=quant_config,
235
- prefix=f"{prefix}.q_proj")
236
-
237
- self.kv_a_proj_with_mqa = ReplicatedLinear(
238
- self.hidden_size,
239
- self.kv_lora_rank + self.qk_rope_head_dim,
240
- bias=False,
241
- quant_config=quant_config,
242
- prefix=f"{prefix}.kv_a_proj_with_mqa")
243
- self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
244
- eps=config.rms_norm_eps)
245
- self.kv_b_proj = ColumnParallelLinear(
246
- self.kv_lora_rank,
247
- self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
248
- bias=False,
249
- quant_config=quant_config,
250
- prefix=f"{prefix}.kv_b_proj")
251
- # O projection.
252
- self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
253
- self.hidden_size,
254
- bias=False,
255
- quant_config=quant_config,
256
- prefix=f"{prefix}.o_proj")
257
- if rope_scaling:
258
- rope_scaling["rope_type"] = 'deepseek_yarn'
259
- self.use_normal_rope = False
260
- else:
261
- self.use_normal_rope = True
262
- self.rotary_emb = get_rope(qk_rope_head_dim,
263
- rotary_dim=qk_rope_head_dim,
264
- max_position=max_position_embeddings,
265
- base=rope_theta,
266
- rope_scaling=rope_scaling,
267
- is_neox_style=False)
268
-
269
- if rope_scaling:
270
- mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
271
- scaling_factor = rope_scaling["factor"]
272
- mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
273
- self.scaling = self.scaling * mscale * mscale
274
-
275
- self.attn = Attention(self.num_local_heads,
276
- self.qk_head_dim,
277
- self.scaling,
278
- num_kv_heads=self.num_local_heads,
279
- cache_config=cache_config,
280
- quant_config=quant_config,
281
- prefix=f"{prefix}.attn")
282
-
283
- def forward(
284
- self,
285
- positions: torch.Tensor,
286
- hidden_states: torch.Tensor,
287
- kv_cache: torch.Tensor,
288
- attn_metadata: AttentionMetadata,
289
- ) -> torch.Tensor:
290
- if self.q_lora_rank is not None:
291
- q = self.q_a_proj(hidden_states)[0]
292
- q = self.q_a_layernorm(q)
293
- q = self.q_b_proj(q)[0].view(-1, self.num_local_heads,
294
- self.qk_head_dim)
295
- else:
296
- q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads,
297
- self.qk_head_dim)
298
- q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim],
299
- dim=-1)
300
- latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0]
301
- kv_a, _ = latent_cache.split(
302
- [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
303
- latent_cache = latent_cache.unsqueeze(1)
304
- kv_a = self.kv_a_layernorm(kv_a.contiguous())
305
- kv = self.kv_b_proj(kv_a)[0]
306
- kv = kv.view(-1, self.num_local_heads,
307
- self.qk_nope_head_dim + self.v_head_dim)
308
- k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
309
- k_pe = latent_cache[:, :, self.kv_lora_rank:]
310
-
311
- if self.use_normal_rope:
312
- seq_len = positions.size(0)
313
- ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
314
- q_pe = q_pe.reshape(seq_len, -1)
315
- k_pe = k_pe.reshape(seq_len, -1)
316
-
317
- q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
318
-
319
- if self.use_normal_rope:
320
- q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
321
-
322
- q[..., self.qk_nope_head_dim:] = q_pe
323
- k = torch.empty_like(q)
324
- k[..., :self.qk_nope_head_dim] = k_nope
325
- k[..., self.qk_nope_head_dim:] = k_pe
326
- # padding value to qk_head_dim for alignment
327
- v = torch.nn.functional.pad(
328
- v, [0, self.qk_head_dim - self.v_head_dim],
329
- value=0).view(-1, self.num_local_heads * self.qk_head_dim)
330
- attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
331
- attn_output = attn_output.view(
332
- -1, self.num_local_heads,
333
- self.qk_head_dim)[..., :self.v_head_dim].reshape(
334
- -1, self.num_local_heads * self.v_head_dim)
335
- output, _ = self.o_proj(attn_output)
336
- return output
337
-
338
-
339
-class DeepseekV3MLAAttention(nn.Module):
340
- """
341
- Main reference: DeepseekV2 paper, and FlashInfer Implementation
342
- (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
343
-
344
- For more info see MLACommonImpl in: vllm/attention/backends/mla/utils.py
345
- """
346
-
347
- def __init__(
348
- self,
349
- config: PretrainedConfig,
350
- hidden_size: int,
351
- num_heads: int,
352
- qk_nope_head_dim: int,
353
- qk_rope_head_dim: int,
354
- v_head_dim: int,
355
- q_lora_rank: Optional[int],
356
- kv_lora_rank: int,
357
- rope_theta: float = 10000,
358
- rope_scaling: Optional[Dict[str, Any]] = None,
359
- max_position_embeddings: int = 8192,
360
- cache_config: Optional[CacheConfig] = None,
361
- quant_config: Optional[QuantizationConfig] = None,
362
- prefix: str = "",
363
- ) -> None:
364
- super().__init__()
365
- self.hidden_size = hidden_size
366
- self.qk_nope_head_dim = qk_nope_head_dim
367
- self.qk_rope_head_dim = qk_rope_head_dim
368
- self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
369
- self.v_head_dim = v_head_dim
370
-
371
- self.q_lora_rank = q_lora_rank
372
- self.kv_lora_rank = kv_lora_rank
373
-
374
- self.num_heads = num_heads
375
- tp_size = get_tensor_model_parallel_world_size()
376
- assert num_heads % tp_size == 0
377
- self.num_local_heads = num_heads // tp_size
378
-
379
- self.scaling = self.qk_head_dim**-0.5
380
- self.rope_theta = rope_theta
381
- self.max_position_embeddings = max_position_embeddings
382
-
383
- if self.q_lora_rank is not None:
384
- self.q_a_proj = ReplicatedLinear(self.hidden_size,
385
- self.q_lora_rank,
386
- bias=False,
387
- quant_config=quant_config,
388
- prefix=f"{prefix}.q_a_proj")
389
- self.q_a_layernorm = RMSNorm(self.q_lora_rank,
390
- eps=config.rms_norm_eps)
391
- self.q_b_proj = ColumnParallelLinear(q_lora_rank,
392
- self.num_heads *
393
- self.qk_head_dim,
394
- bias=False,
395
- quant_config=quant_config,
396
- prefix=f"{prefix}.q_b_proj")
397
- else:
398
- self.q_proj = ColumnParallelLinear(self.hidden_size,
399
- self.num_heads *
400
- self.qk_head_dim,
401
- bias=False,
402
- quant_config=quant_config,
403
- prefix=f"{prefix}.q_proj")
404
-
405
- self.kv_a_proj_with_mqa = ReplicatedLinear(
406
- self.hidden_size,
407
- self.kv_lora_rank + self.qk_rope_head_dim,
408
- bias=False,
409
- quant_config=quant_config,
410
- prefix=f"{prefix}.kv_a_proj_with_mqa")
411
- self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
412
- eps=config.rms_norm_eps)
413
- self.kv_b_proj = ColumnParallelLinear(
414
- self.kv_lora_rank,
415
- self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
416
- bias=False,
417
- quant_config=quant_config,
418
- prefix=f"{prefix}.kv_b_proj")
419
- self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
420
- self.hidden_size,
421
- bias=False,
422
- quant_config=quant_config,
423
- prefix=f"{prefix}.o_proj")
424
-
425
- if rope_scaling:
426
- rope_scaling["rope_type"] = 'deepseek_yarn'
427
- self.rotary_emb = get_rope(qk_rope_head_dim,
428
- rotary_dim=qk_rope_head_dim,
429
- max_position=max_position_embeddings,
430
- base=rope_theta,
431
- rope_scaling=rope_scaling,
432
- is_neox_style=False)
433
- if rope_scaling:
434
- mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
435
- scaling_factor = rope_scaling["factor"]
436
- mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
437
- self.scaling = self.scaling * mscale * mscale
438
-
439
- self.mla_attn = Attention(
440
- num_heads=self.num_local_heads,
441
- head_size=self.kv_lora_rank,
442
- scale=self.scaling,
443
- num_kv_heads=1,
444
- cache_config=cache_config,
445
- quant_config=quant_config,
446
- prefix=f"{prefix}.attn",
447
- use_mla=True,
448
- # MLA Args
449
- q_lora_rank=self.q_lora_rank,
450
- kv_lora_rank=self.kv_lora_rank,
451
- qk_nope_head_dim=self.qk_nope_head_dim,
452
- qk_rope_head_dim=self.qk_rope_head_dim,
453
- qk_head_dim=self.qk_head_dim,
454
- v_head_dim=self.v_head_dim,
455
- rotary_emb=self.rotary_emb,
456
- q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj,
457
- kv_b_proj=self.kv_b_proj,
458
- o_proj=self.o_proj,
459
- )
460
-
461
- self.prefix = prefix
462
- self.debug_layer_idx = int(self.prefix.split(".")[-2])
463
-
464
- def forward(
465
- self,
466
- positions: torch.Tensor,
467
- hidden_states: torch.Tensor,
468
- kv_cache: torch.Tensor,
469
- attn_metadata: AttentionMetadata,
470
- ) -> torch.Tensor:
471
- if self.q_lora_rank is not None:
472
- ckq = self.q_a_proj(hidden_states)[0]
473
- hidden_states_or_q_c = self.q_a_layernorm(ckq)
474
- else:
475
- hidden_states_or_q_c = hidden_states
476
- kv_c, k_pe = self.kv_a_proj_with_mqa(hidden_states)[0].split(
477
- [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
478
- kv_c_normed = self.kv_a_layernorm(kv_c.contiguous())
479
- return self.mla_attn(hidden_states_or_q_c, kv_c_normed, k_pe, kv_cache,
480
- attn_metadata)
481
-
482
-
483
-class DeepseekV3DecoderLayer(nn.Module):
484
-
485
- def __init__(
486
- self,
487
- config: PretrainedConfig,
488
- prefix: str,
489
- model_config: ModelConfig,
490
- cache_config: Optional[CacheConfig] = None,
491
- quant_config: Optional[QuantizationConfig] = None,
492
- ) -> None:
493
- super().__init__()
494
- self.hidden_size = config.hidden_size
495
- rope_theta = getattr(config, "rope_theta", 10000)
496
- rope_scaling = getattr(config, "rope_scaling", None)
497
- max_position_embeddings = getattr(config, "max_position_embeddings",
498
- 8192)
499
- # DecoderLayers are created with `make_layers` which passes the prefix
500
- # with the layer's index.
501
- layer_idx = int(prefix.split(sep='.')[-1])
502
- if model_config.use_mla:
503
- attn_cls = DeepseekV3MLAAttention
504
- else:
505
- attn_cls = DeepseekV3Attention
506
- self.self_attn = attn_cls(
507
- config=config,
508
- hidden_size=self.hidden_size,
509
- num_heads=config.num_attention_heads,
510
- qk_nope_head_dim=config.qk_nope_head_dim,
511
- qk_rope_head_dim=config.qk_rope_head_dim,
512
- v_head_dim=config.v_head_dim,
513
- q_lora_rank=config.q_lora_rank
514
- if hasattr(config, "q_lora_rank") else None,
515
- kv_lora_rank=config.kv_lora_rank,
516
- rope_theta=rope_theta,
517
- rope_scaling=rope_scaling,
518
- max_position_embeddings=max_position_embeddings,
519
- cache_config=cache_config,
520
- quant_config=quant_config,
521
- prefix=f"{prefix}.self_attn",
522
- )
523
- if (config.n_routed_experts is not None
524
- and layer_idx >= config.first_k_dense_replace
525
- and layer_idx % config.moe_layer_freq == 0):
526
- self.mlp = DeepseekV3MoE(
527
- config=config,
528
- quant_config=quant_config,
529
- prefix=f"{prefix}.mlp",
530
- )
531
- else:
532
- self.mlp = DeepseekV3MLP(
533
- hidden_size=config.hidden_size,
534
- intermediate_size=config.intermediate_size,
535
- hidden_act=config.hidden_act,
536
- quant_config=quant_config,
537
- prefix=f"{prefix}.mlp",
538
- )
539
- self.input_layernorm = RMSNorm(config.hidden_size,
540
- eps=config.rms_norm_eps)
541
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
542
- eps=config.rms_norm_eps)
543
-
544
- def forward(
545
- self,
546
- positions: torch.Tensor,
547
- hidden_states: torch.Tensor,
548
- kv_cache: torch.Tensor,
549
- attn_metadata: AttentionMetadata,
550
- residual: Optional[torch.Tensor],
551
- ) -> torch.Tensor:
552
- # Self Attention
553
- if residual is None:
554
- residual = hidden_states
555
- hidden_states = self.input_layernorm(hidden_states)
556
- else:
557
- hidden_states, residual = self.input_layernorm(
558
- hidden_states, residual)
559
- hidden_states = self.self_attn(
560
- positions=positions,
561
- hidden_states=hidden_states,
562
- kv_cache=kv_cache,
563
- attn_metadata=attn_metadata,
564
- )
565
-
566
- # Fully Connected
567
- hidden_states, residual = self.post_attention_layernorm(
568
- hidden_states, residual)
569
- hidden_states = self.mlp(hidden_states)
570
- return hidden_states, residual
571
-
572
-
573
-@support_torch_compile
574
-class DeepseekV3Model(nn.Module):
575
-
576
- fall_back_to_pt_during_load = False
577
-
578
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
579
- super().__init__()
580
-
581
- config = vllm_config.model_config.hf_config
582
- model_config = vllm_config.model_config
583
- cache_config = vllm_config.cache_config
584
- quant_config = vllm_config.quant_config
585
-
586
- self.padding_idx = config.pad_token_id
587
- self.vocab_size = config.vocab_size
588
-
589
- if get_pp_group().is_first_rank:
590
- self.embed_tokens = VocabParallelEmbedding(
591
- config.vocab_size,
592
- config.hidden_size,
593
- )
594
- else:
595
- self.embed_tokens = PPMissingLayer()
596
-
597
- self.start_layer, self.end_layer, self.layers = make_layers(
598
- config.num_hidden_layers,
599
- lambda prefix: DeepseekV3DecoderLayer(
600
- config,
601
- prefix,
602
- model_config=model_config,
603
- cache_config=cache_config,
604
- quant_config=quant_config,
605
- ),
606
- prefix=f"{prefix}.layers")
607
-
608
- if get_pp_group().is_last_rank:
609
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
610
- else:
611
- self.norm = PPMissingLayer()
612
- self.make_empty_intermediate_tensors = (
613
- make_empty_intermediate_tensors_factory(
614
- ["hidden_states", "residual"], config.hidden_size))
615
-
616
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
617
- return self.embed_tokens(input_ids)
618
-
619
- def forward(
620
- self,
621
- input_ids: torch.Tensor,
622
- positions: torch.Tensor,
623
- kv_caches: List[torch.Tensor],
624
- attn_metadata: AttentionMetadata,
625
- intermediate_tensors: Optional[IntermediateTensors],
626
- inputs_embeds: Optional[torch.Tensor] = None,
627
- ) -> Union[torch.Tensor, IntermediateTensors]:
628
- if get_pp_group().is_first_rank:
629
- if inputs_embeds is not None:
630
- hidden_states = inputs_embeds
631
- else:
632
- hidden_states = self.get_input_embeddings(input_ids)
633
- residual = None
634
- else:
635
- assert intermediate_tensors is not None
636
- hidden_states = intermediate_tensors["hidden_states"]
637
- residual = intermediate_tensors["residual"]
638
-
639
- for i in range(self.start_layer, self.end_layer):
640
- layer = self.layers[i]
641
- hidden_states, residual = layer(positions, hidden_states,
642
- kv_caches[i - self.start_layer],
643
- attn_metadata, residual)
644
-
645
- if not get_pp_group().is_last_rank:
646
- return IntermediateTensors({
647
- "hidden_states": hidden_states,
648
- "residual": residual
649
- })
650
-
651
- hidden_states, _ = self.norm(hidden_states, residual)
652
- return hidden_states
653
-
654
-
655
-class DeepseekV3ForCausalLM(nn.Module, SupportsPP):
656
-
657
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
658
- super().__init__()
659
- config = vllm_config.model_config.hf_config
660
- quant_config = vllm_config.quant_config
661
- self.config = config
662
- self.quant_config = quant_config
663
- self.model = DeepseekV3Model(vllm_config=vllm_config,
664
- prefix=maybe_prefix(prefix, "model"))
665
- self.lm_head = ParallelLMHead(config.vocab_size,
666
- config.hidden_size,
667
- quant_config=quant_config)
668
- self.logits_processor = LogitsProcessor(config.vocab_size)
669
- self.sampler = get_sampler()
670
- self.make_empty_intermediate_tensors = (
671
- self.model.make_empty_intermediate_tensors)
672
-
673
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
674
- return self.model.get_input_embeddings(input_ids)
675
-
676
- def forward(
677
- self,
678
- input_ids: torch.Tensor,
679
- positions: torch.Tensor,
680
- kv_caches: List[torch.Tensor],
681
- attn_metadata: AttentionMetadata,
682
- intermediate_tensors: Optional[IntermediateTensors] = None,
683
- inputs_embeds: Optional[torch.Tensor] = None,
684
- ) -> Union[torch.Tensor, IntermediateTensors]:
685
- hidden_states = self.model(input_ids, positions, kv_caches,
686
- attn_metadata, intermediate_tensors,
687
- inputs_embeds)
688
- return hidden_states
689
-
690
- def compute_logits(
691
- self,
692
- hidden_states: torch.Tensor,
693
- sampling_metadata: SamplingMetadata,
694
- ) -> Optional[torch.Tensor]:
695
- logits = self.logits_processor(self.lm_head, hidden_states,
696
- sampling_metadata)
697
- return logits
698
-
699
- def sample(
700
- self,
701
- logits: Optional[torch.Tensor],
702
- sampling_metadata: SamplingMetadata,
703
- ) -> Optional[SamplerOutput]:
704
- next_tokens = self.sampler(logits, sampling_metadata)
705
- return next_tokens
706
-
707
- def make_empty_intermediate_tensors(
708
- self, batch_size: int, dtype: torch.dtype,
709
- device: torch.device) -> IntermediateTensors:
710
- return IntermediateTensors({
711
- "hidden_states":
712
- torch.zeros((batch_size, self.config.hidden_size),
713
- dtype=dtype,
714
- device=device),
715
- "residual":
716
- torch.zeros((batch_size, self.config.hidden_size),
717
- dtype=dtype,
718
- device=device),
719
- })
720
-
721
- def load_weights(self, weights: Iterable[Tuple[str,
722
- torch.Tensor]]) -> Set[str]:
723
- stacked_params_mapping = [
724
- # (param_name, shard_name, shard_id)
725
- ("gate_up_proj", "gate_proj", 0),
726
- ("gate_up_proj", "up_proj", 1),
727
- ]
728
-
729
- # Params for weights, fp8 weight scales, fp8 activation scales
730
- # (param_name, weight_name, expert_id, shard_id)
731
- expert_params_mapping = FusedMoE.make_expert_params_mapping(
732
- ckpt_gate_proj_name="gate_proj",
733
- ckpt_down_proj_name="down_proj",
734
- ckpt_up_proj_name="up_proj",
735
- num_experts=self.config.n_routed_experts)
736
-
737
- params_dict = dict(self.named_parameters())
738
- loaded_params: Set[str] = set()
739
- for name, loaded_weight in weights:
740
- if "rotary_emb.inv_freq" in name:
741
- continue
742
-
743
- # TODO(simon): support nextn predict layers
744
- if hasattr(self.config, "num_nextn_predict_layers"
745
- ) and self.config.num_nextn_predict_layers > 0:
746
- assert self.config.num_nextn_predict_layers == 1
747
- layer_idx = self.config.num_hidden_layers
748
- if name.startswith(f"model.layers.{layer_idx}"):
749
- continue
750
-
751
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
752
- # Skip non-stacked layers and experts (experts handled below).
753
- if weight_name not in name:
754
- continue
755
- # We have mlp.experts[0].gate_proj in the checkpoint.
756
- # Since we handle the experts below in expert_params_mapping,
757
- # we need to skip here BEFORE we update the name, otherwise
758
- # name will be updated to mlp.experts[0].gate_up_proj, which
759
- # will then be updated below in expert_params_mapping
760
- # for mlp.experts[0].gate_gate_up_proj, which breaks load.
761
- if (("mlp.experts." in name) and name not in params_dict):
762
- continue
763
- name = name.replace(weight_name, param_name)
764
- # Skip loading extra bias for GPTQ models.
765
- if name.endswith(".bias") and name not in params_dict:
766
- continue
767
-
768
- if is_pp_missing_parameter(name, self):
769
- continue
770
-
771
- param = params_dict[name]
772
- weight_loader = param.weight_loader
773
- weight_loader(param, loaded_weight, shard_id)
774
- break
775
- else:
776
- for mapping in expert_params_mapping:
777
- param_name, weight_name, expert_id, shard_id = mapping
778
- if weight_name not in name:
779
- continue
780
- name = name.replace(weight_name, param_name)
781
-
782
- if is_pp_missing_parameter(name, self):
783
- continue
784
-
785
- param = params_dict[name]
786
- weight_loader = param.weight_loader
787
- weight_loader(param,
788
- loaded_weight,
789
- name,
790
- shard_id=shard_id,
791
- expert_id=expert_id)
792
- break
793
- else:
794
- # Skip loading extra bias for GPTQ models.
795
- if name.endswith(".bias") and name not in params_dict:
796
- continue
797
-
798
- if is_pp_missing_parameter(name, self):
799
- continue
800
-
801
- param = params_dict[name]
802
- weight_loader = getattr(param, "weight_loader",
803
- default_weight_loader)
804
- weight_loader(param, loaded_weight)
805
- loaded_params.add(name)
806
- return loaded_params
vllm/model_executor/models/registry.py
+1
-1
@@ -45,7 +45,7 @@ _TEXT_GENERATION_MODELS = {
45
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
46
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
47
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
48
- "DeepseekV3ForCausalLM": ("deepseek_v3", "DeepseekV3ForCausalLM"),
48
+ "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
49
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
50
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
51
"Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),