A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
Deepseek v3 (#11502)
Signed-off-by: mgoin <michael@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: robertgshaw2-neuralmagic <rshaw@neuralmagic.com>
Simon Mo committed
Dec 26, 2024 at 16:09 UTC
f49777ba62b4926d0f8c100ab06edb03c5c10098
7 files changed
+887
-61
csrc/moe/moe_align_sum_kernels.cu
+141
-19
@@ -113,6 +113,92 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
113
}
114
}
115
116
+// TODO(simon): this is temporarily adapted from
117
+// https://github.com/sgl-project/sglang/commit/31548116a8dc8c6df7e146e0587335a59fc5b9d7
118
+// we did this to unblock Deepseek V3 but there should be a better
119
+// implementation to manage shared memory.
120
+template <typename scalar_t>
121
+__global__ void moe_align_block_size_global_mem_kernel(
122
+ scalar_t* __restrict__ topk_ids, int32_t* sorted_token_ids,
123
+ int32_t* expert_ids, int32_t* total_tokens_post_pad, int32_t num_experts,
124
+ int32_t block_size, size_t numel, int32_t* tokens_cnts, int32_t* cumsum) {
125
+ const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
126
+ const size_t start_idx = threadIdx.x * tokens_per_thread;
127
+
128
+ for (int i = 0; i < num_experts; ++i) {
129
+ tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
130
+ }
131
+
132
+ /**
133
+ * In the first step we compute token_cnts[thread_index + 1][expert_index],
134
+ * which counts how many tokens in the token shard of thread_index are
135
+ * assigned to expert expert_index.
136
+ */
137
+ for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
138
+ ++tokens_cnts[index(num_experts, threadIdx.x + 1, topk_ids[i])];
139
+ }
140
+
141
+ __syncthreads();
142
+
143
+ // For each expert we accumulate the token counts from the different threads.
144
+ if (threadIdx.x < num_experts) {
145
+ tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
146
+ for (int i = 1; i <= blockDim.x; ++i) {
147
+ tokens_cnts[index(num_experts, i, threadIdx.x)] +=
148
+ tokens_cnts[index(num_experts, i - 1, threadIdx.x)];
149
+ }
150
+ }
151
+
152
+ __syncthreads();
153
+
154
+ // We accumulate the token counts of all experts in thread 0.
155
+ if (threadIdx.x == 0) {
156
+ cumsum[0] = 0;
157
+ for (int i = 1; i <= num_experts; ++i) {
158
+ cumsum[i] = cumsum[i - 1] +
159
+ CEILDIV(tokens_cnts[index(num_experts, blockDim.x, i - 1)],
160
+ block_size) *
161
+ block_size;
162
+ }
163
+ *total_tokens_post_pad = cumsum[num_experts];
164
+ }
165
+
166
+ __syncthreads();
167
+
168
+ /**
169
+ * For each expert, each thread processes the tokens of the corresponding
170
+ * blocks and stores the corresponding expert_id for each block.
171
+ */
172
+ if (threadIdx.x < num_experts) {
173
+ for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
174
+ i += block_size) {
175
+ expert_ids[i / block_size] = threadIdx.x;
176
+ }
177
+ }
178
+
179
+ /**
180
+ * Each thread processes a token shard, calculating the index of each token
181
+ * after sorting by expert number. Given the example topk_ids =
182
+ * [0,1,2,1,2,3,0,3,4] and block_size = 4, then the output would be [0, 6, *,
183
+ * *, 1, 3, *, *, 2, 4, *, *, 5, 7, *, *, 8, *, *, *], where * represents a
184
+ * padding value(preset in python).
185
+ */
186
+ for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
187
+ int32_t expert_id = topk_ids[i];
188
+ /** The cumsum[expert_id] stores the starting index of the tokens that the
189
+ * expert with expert_id needs to process, and
190
+ * tokens_cnts[threadIdx.x][expert_id] stores the indices of the tokens
191
+ * processed by the expert with expert_id within the current thread's token
192
+ * shard.
193
+ */
194
+ int32_t rank_post_pad =
195
+ tokens_cnts[index(num_experts, threadIdx.x, expert_id)] +
196
+ cumsum[expert_id];
197
+ sorted_token_ids[rank_post_pad] = i;
198
+ ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
199
+ }
200
+}
201
+
202
template <typename scalar_t, int TOPK>
203
__global__ void moe_sum_kernel(
204
scalar_t* __restrict__ out, // [..., d]
@@ -137,25 +223,61 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
223
torch::Tensor experts_ids,
224
torch::Tensor num_tokens_post_pad) {
225
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
140
- VLLM_DISPATCH_INTEGRAL_TYPES(
141
- topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
142
- // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
143
- // tensors
144
- const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
145
- const int32_t shared_mem =
146
- ((num_thread + 1) * num_experts + (num_experts + 1)) *
147
- sizeof(int32_t);
148
-
149
- // set dynamic shared mem
150
- auto kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
151
- AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
152
- (void*)kernel, shared_mem));
153
- kernel<<<1, num_thread, shared_mem, stream>>>(
154
- topk_ids.data_ptr<scalar_t>(), sorted_token_ids.data_ptr<int32_t>(),
155
- experts_ids.data_ptr<int32_t>(),
156
- num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
157
- topk_ids.numel());
158
- });
226
+
227
+ // If we have very large number of experts, we can no longer use shared
228
+ // memory.
229
+ // TODO(simon): the right solution should be calculating the exact right
230
+ // amount of shared memory and use that. The num_experts >= 256 is just a
231
+ // temporary solution to unblock Deepseek V3.
232
+ if (num_experts >= 256) {
233
+ VLLM_DISPATCH_INTEGRAL_TYPES(
234
+ topk_ids.scalar_type(), "moe_align_block_size_global_mem_kernel", [&] {
235
+ // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
236
+ // tensors
237
+ const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
238
+
239
+ const int32_t mem_tokens_cnts =
240
+ ((num_experts + 1) * num_experts) * sizeof(int32_t);
241
+ const int32_t mem_cumsum = (num_experts + 1) * sizeof(int32_t);
242
+ // allocate global memory
243
+ int32_t* tokens_cnts;
244
+ int32_t* cumsum;
245
+ cudaMalloc(&tokens_cnts, mem_tokens_cnts);
246
+ cudaMalloc(&cumsum, mem_cumsum);
247
+
248
+ auto kernel =
249
+ vllm::moe::moe_align_block_size_global_mem_kernel<scalar_t>;
250
+ kernel<<<1, num_thread, 0, stream>>>(
251
+ topk_ids.data_ptr<scalar_t>(),
252
+ sorted_token_ids.data_ptr<int32_t>(),
253
+ experts_ids.data_ptr<int32_t>(),
254
+ num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
255
+ topk_ids.numel(), tokens_cnts, cumsum);
256
+ cudaFree(tokens_cnts);
257
+ cudaFree(cumsum);
258
+ });
259
+ } else {
260
+ VLLM_DISPATCH_INTEGRAL_TYPES(
261
+ topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
262
+ // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
263
+ // tensors
264
+ const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
265
+ const int32_t shared_mem =
266
+ ((num_thread + 1) * num_experts + (num_experts + 1)) *
267
+ sizeof(int32_t);
268
+
269
+ // set dynamic shared mem
270
+ auto kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
271
+ AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
272
+ (void*)kernel, shared_mem));
273
+ kernel<<<1, num_thread, shared_mem, stream>>>(
274
+ topk_ids.data_ptr<scalar_t>(),
275
+ sorted_token_ids.data_ptr<int32_t>(),
276
+ experts_ids.data_ptr<int32_t>(),
277
+ num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
278
+ topk_ids.numel());
279
+ });
280
+ }
281
}
282
283
void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
vllm/config.py
+9
-2
@@ -596,6 +596,12 @@ class ModelConfig:
596
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
597
self.max_model_len)
598
599
+ if (self.hf_config.model_type == 'deepseek_v3'
600
+ and not self.enforce_eager):
601
+ logger.warning("CUDA graph is not supported for Deepseek V3 yet, "
602
+ "fallback to the eager mode.")
603
+ self.enforce_eager = True
604
+
605
def _verify_bnb_config(self) -> None:
606
"""
607
The current version of bitsandbytes (0.44.0) with 8-bit models does not
@@ -712,8 +718,9 @@ class ModelConfig:
718
719
def get_head_size(self) -> int:
720
# TODO remove hard code
715
- if hasattr(self.hf_text_config, "model_type"
716
- ) and self.hf_text_config.model_type == 'deepseek_v2':
721
+ if hasattr(self.hf_text_config,
722
+ "model_type") and (self.hf_text_config.model_type
723
+ in ('deepseek_v2', 'deepseek_v3')):
724
# FlashAttention supports only head_size 32, 64, 128, 256,
725
# we need to pad head_size 192 to 256
726
return 256
vllm/model_executor/layers/fused_moe/fused_moe.py
+14
-3
@@ -476,18 +476,29 @@ def fused_topk(
476
return topk_weights, topk_ids
477
478
479
-# This is used by the Deepseek-V2 model
479
+# This is used by the Deepseek-V2 and Deepseek-V3 model
480
def grouped_topk(hidden_states: torch.Tensor,
481
gating_output: torch.Tensor,
482
topk: int,
483
renormalize: bool,
484
num_expert_group: int = 0,
485
- topk_group: int = 0):
485
+ topk_group: int = 0,
486
+ scoring_func: str = "softmax",
487
+ e_score_correction_bias: Optional[torch.Tensor] = None):
488
489
assert hidden_states.shape[0] == gating_output.shape[0], (
490
"Number of tokens mismatch")
491
490
- scores = torch.softmax(gating_output, dim=-1)
492
+ if scoring_func == "softmax":
493
+ scores = torch.softmax(gating_output, dim=-1)
494
+ elif scoring_func == "sigmoid":
495
+ scores = gating_output.sigmoid()
496
+ else:
497
+ raise ValueError(f"Unsupported scoring function: {scoring_func}")
498
+
499
+ if e_score_correction_bias is not None:
500
+ scores.add_(e_score_correction_bias.unsqueeze(0))
501
+
502
num_token = scores.shape[0]
503
group_scores = scores.view(num_token, num_expert_group,
504
-1).max(dim=-1).values # [n, n_group]
vllm/model_executor/layers/fused_moe/layer.py
+66
-36
@@ -73,16 +73,18 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
73
set_weight_attrs(w2_weight, extra_weight_attrs)
74
75
def apply(
76
- self,
77
- layer: torch.nn.Module,
78
- x: torch.Tensor,
79
- router_logits: torch.Tensor,
80
- top_k: int,
81
- renormalize: bool,
82
- use_grouped_topk: bool,
83
- topk_group: Optional[int] = None,
84
- num_expert_group: Optional[int] = None,
85
- custom_routing_function: Optional[Callable] = None
76
+ self,
77
+ layer: torch.nn.Module,
78
+ x: torch.Tensor,
79
+ router_logits: torch.Tensor,
80
+ top_k: int,
81
+ renormalize: bool,
82
+ use_grouped_topk: bool,
83
+ topk_group: Optional[int] = None,
84
+ num_expert_group: Optional[int] = None,
85
+ custom_routing_function: Optional[Callable] = None,
86
+ scoring_func: str = "softmax",
87
+ e_score_correction_bias: Optional[torch.Tensor] = None
88
) -> torch.Tensor:
89
return self.forward(x=x,
90
layer=layer,
@@ -92,19 +94,23 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
94
use_grouped_topk=use_grouped_topk,
95
topk_group=topk_group,
96
num_expert_group=num_expert_group,
95
- custom_routing_function=custom_routing_function)
97
+ custom_routing_function=custom_routing_function,
98
+ scoring_func=scoring_func,
99
+ e_score_correction_bias=e_score_correction_bias)
100
101
def forward_cuda(
98
- self,
99
- layer: torch.nn.Module,
100
- x: torch.Tensor,
101
- use_grouped_topk: bool,
102
- top_k: int,
103
- router_logits: torch.Tensor,
104
- renormalize: bool,
105
- topk_group: Optional[int] = None,
106
- num_expert_group: Optional[int] = None,
107
- custom_routing_function: Optional[Callable] = None
102
+ self,
103
+ layer: torch.nn.Module,
104
+ x: torch.Tensor,
105
+ use_grouped_topk: bool,
106
+ top_k: int,
107
+ router_logits: torch.Tensor,
108
+ renormalize: bool,
109
+ topk_group: Optional[int] = None,
110
+ num_expert_group: Optional[int] = None,
111
+ custom_routing_function: Optional[Callable] = None,
112
+ scoring_func: str = "softmax",
113
+ e_score_correction_bias: Optional[torch.Tensor] = None
114
) -> torch.Tensor:
115
topk_weights, topk_ids = FusedMoE.select_experts(
116
hidden_states=x,
@@ -114,7 +120,9 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
120
renormalize=renormalize,
121
topk_group=topk_group,
122
num_expert_group=num_expert_group,
117
- custom_routing_function=custom_routing_function)
123
+ custom_routing_function=custom_routing_function,
124
+ scoring_func=scoring_func,
125
+ e_score_correction_bias=e_score_correction_bias)
126
127
return fused_experts(hidden_states=x,
128
w1=layer.w13_weight,
@@ -128,21 +136,29 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
136
"The CPU backend currently does not support MoE.")
137
138
def forward_tpu(
131
- self,
132
- layer: torch.nn.Module,
133
- x: torch.Tensor,
134
- use_grouped_topk: bool,
135
- top_k: int,
136
- router_logits: torch.Tensor,
137
- renormalize: bool,
138
- topk_group: Optional[int] = None,
139
- num_expert_group: Optional[int] = None,
140
- custom_routing_function: Optional[Callable] = None
139
+ self,
140
+ layer: torch.nn.Module,
141
+ x: torch.Tensor,
142
+ use_grouped_topk: bool,
143
+ top_k: int,
144
+ router_logits: torch.Tensor,
145
+ renormalize: bool,
146
+ topk_group: Optional[int] = None,
147
+ num_expert_group: Optional[int] = None,
148
+ custom_routing_function: Optional[Callable] = None,
149
+ scoring_func: str = "softmax",
150
+ e_score_correction_bias: Optional[torch.Tensor] = None
151
) -> torch.Tensor:
152
assert not use_grouped_topk
153
assert num_expert_group is None
154
assert topk_group is None
155
assert custom_routing_function is None
156
+ if scoring_func != "softmax":
157
+ raise NotImplementedError(
158
+ "Only softmax scoring function is supported for TPU.")
159
+ if e_score_correction_bias is not None:
160
+ raise NotImplementedError(
161
+ "Expert score correction bias is not supported for TPU.")
162
return fused_moe_pallas(hidden_states=x,
163
w1=layer.w13_weight,
164
w2=layer.w2_weight,
@@ -156,7 +172,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
172
class FusedMoE(torch.nn.Module):
173
"""FusedMoE layer for MoE models.
174
159
- This layer contains both MergedColumnParallel weights (gate_up_proj /
175
+ This layer contains both MergedColumnParallel weights (gate_up_proj /
176
w13) and RowParallelLinear weights (down_proj/ w2).
177
178
Note: Mixtral uses w1, w2, and w3 for gate, up, and down_proj. We
@@ -190,6 +206,8 @@ class FusedMoE(torch.nn.Module):
206
tp_size: Optional[int] = None,
207
prefix: str = "",
208
custom_routing_function: Optional[Callable] = None,
209
+ scoring_func: str = "softmax",
210
+ e_score_correction_bias: Optional[torch.Tensor] = None,
211
):
212
super().__init__()
213
@@ -210,6 +228,12 @@ class FusedMoE(torch.nn.Module):
228
self.num_expert_group = num_expert_group
229
self.topk_group = topk_group
230
self.custom_routing_function = custom_routing_function
231
+ self.scoring_func = scoring_func
232
+ self.e_score_correction_bias = e_score_correction_bias
233
+
234
+ if self.scoring_func != "softmax" and not self.use_grouped_topk:
235
+ raise ValueError("Only softmax scoring function is supported for "
236
+ "non-grouped topk.")
237
238
if quant_config is None:
239
self.quant_method: Optional[QuantizeMethodBase] = (
@@ -446,7 +470,9 @@ class FusedMoE(torch.nn.Module):
470
renormalize: bool,
471
topk_group: Optional[int] = None,
472
num_expert_group: Optional[int] = None,
449
- custom_routing_function: Optional[Callable] = None):
473
+ custom_routing_function: Optional[Callable] = None,
474
+ scoring_func: str = "softmax",
475
+ e_score_correction_bias: Optional[torch.Tensor] = None):
476
from vllm.model_executor.layers.fused_moe.fused_moe import (
477
fused_topk, grouped_topk)
478
@@ -460,7 +486,9 @@ class FusedMoE(torch.nn.Module):
486
topk=top_k,
487
renormalize=renormalize,
488
num_expert_group=num_expert_group,
463
- topk_group=topk_group)
489
+ topk_group=topk_group,
490
+ scoring_func=scoring_func,
491
+ e_score_correction_bias=e_score_correction_bias)
492
elif custom_routing_function is None:
493
topk_weights, topk_ids = fused_topk(hidden_states=hidden_states,
494
gating_output=router_logits,
@@ -489,7 +517,9 @@ class FusedMoE(torch.nn.Module):
517
use_grouped_topk=self.use_grouped_topk,
518
topk_group=self.topk_group,
519
num_expert_group=self.num_expert_group,
492
- custom_routing_function=self.custom_routing_function)
520
+ custom_routing_function=self.custom_routing_function,
521
+ scoring_func=self.scoring_func,
522
+ e_score_correction_bias=self.e_score_correction_bias)
523
524
if self.reduce_results and self.tp_size > 1:
525
final_hidden_states = tensor_model_parallel_all_reduce(
vllm/model_executor/layers/quantization/fp8.py
+6
-1
@@ -605,6 +605,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
605
topk_group: Optional[int] = None,
606
num_expert_group: Optional[int] = None,
607
custom_routing_function: Optional[Callable] = None,
608
+ scoring_func: str = "softmax",
609
+ e_score_correction_bias: Optional[torch.Tensor] = None,
610
) -> torch.Tensor:
611
612
from vllm.model_executor.layers.fused_moe import fused_experts
@@ -617,7 +619,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
619
renormalize=renormalize,
620
topk_group=topk_group,
621
num_expert_group=num_expert_group,
620
- custom_routing_function=custom_routing_function)
622
+ custom_routing_function=custom_routing_function,
623
+ scoring_func=scoring_func,
624
+ e_score_correction_bias=e_score_correction_bias,
625
+ )
626
627
return fused_experts(
628
x,
vllm/model_executor/models/deepseek_v3.py
new
+650
@@ -0,0 +1,650 @@
1
+# Adapted from
2
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
+# Copyright 2023 The vLLM team.
4
+# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
5
+#
6
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7
+# and OPT implementations in this library. It has been modified from its
8
+# original forms to accommodate minor architectural differences compared
9
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10
+#
11
+# Licensed under the Apache License, Version 2.0 (the "License");
12
+# you may not use this file except in compliance with the License.
13
+# You may obtain a copy of the License at
14
+#
15
+# http://www.apache.org/licenses/LICENSE-2.0
16
+#
17
+# Unless required by applicable law or agreed to in writing, software
18
+# distributed under the License is distributed on an "AS IS" BASIS,
19
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
+# See the License for the specific language governing permissions and
21
+# limitations under the License.
22
+"""Inference-only DeepseekV3 model."""
23
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24
+
25
+import torch
26
+from torch import nn
27
+from transformers import PretrainedConfig
28
+
29
+from vllm.attention import Attention, AttentionMetadata
30
+from vllm.config import CacheConfig, VllmConfig
31
+from vllm.distributed import (get_pp_group,
32
+ get_tensor_model_parallel_world_size,
33
+ tensor_model_parallel_all_reduce)
34
+from vllm.model_executor.layers.activation import SiluAndMul
35
+from vllm.model_executor.layers.fused_moe import FusedMoE
36
+from vllm.model_executor.layers.layernorm import RMSNorm
37
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
38
+ MergedColumnParallelLinear,
39
+ ReplicatedLinear,
40
+ RowParallelLinear)
41
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
+from vllm.model_executor.layers.quantization import QuantizationConfig
43
+from vllm.model_executor.layers.rotary_embedding import get_rope
44
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
45
+from vllm.model_executor.layers.vocab_parallel_embedding import (
46
+ ParallelLMHead, VocabParallelEmbedding)
47
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
48
+from vllm.model_executor.sampling_metadata import SamplingMetadata
49
+from vllm.sequence import IntermediateTensors
50
+
51
+from .interfaces import SupportsPP
52
+from .utils import (PPMissingLayer, is_pp_missing_parameter,
53
+ make_empty_intermediate_tensors_factory, make_layers,
54
+ maybe_prefix)
55
+
56
+
57
+class DeepseekV3MLP(nn.Module):
58
+
59
+ def __init__(
60
+ self,
61
+ hidden_size: int,
62
+ intermediate_size: int,
63
+ hidden_act: str,
64
+ quant_config: Optional[QuantizationConfig] = None,
65
+ reduce_results: bool = True,
66
+ prefix: str = "",
67
+ ) -> None:
68
+ super().__init__()
69
+ self.gate_up_proj = MergedColumnParallelLinear(
70
+ hidden_size, [intermediate_size] * 2,
71
+ bias=False,
72
+ quant_config=quant_config,
73
+ prefix=f"{prefix}.gate_up_proj")
74
+ self.down_proj = RowParallelLinear(intermediate_size,
75
+ hidden_size,
76
+ bias=False,
77
+ quant_config=quant_config,
78
+ reduce_results=reduce_results,
79
+ prefix=f"{prefix}.down_proj")
80
+ if hidden_act != "silu":
81
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
82
+ "Only silu is supported for now.")
83
+ self.act_fn = SiluAndMul()
84
+
85
+ def forward(self, x):
86
+ gate_up, _ = self.gate_up_proj(x)
87
+ x = self.act_fn(gate_up)
88
+ x, _ = self.down_proj(x)
89
+ return x
90
+
91
+
92
+class DeepseekV3MoE(nn.Module):
93
+
94
+ def __init__(
95
+ self,
96
+ config: PretrainedConfig,
97
+ quant_config: Optional[QuantizationConfig] = None,
98
+ prefix: str = "",
99
+ ):
100
+ super().__init__()
101
+ self.tp_size = get_tensor_model_parallel_world_size()
102
+ self.routed_scaling_factor = config.routed_scaling_factor
103
+ self.n_shared_experts = config.n_shared_experts
104
+ self.routed_scaling_factor = config.routed_scaling_factor
105
+ if self.tp_size > config.n_routed_experts:
106
+ raise ValueError(
107
+ f"Tensor parallel size {self.tp_size} is greater than "
108
+ f"the number of experts {config.n_routed_experts}.")
109
+
110
+ if config.hidden_act != "silu":
111
+ raise ValueError(f"Unsupported activation: {config.hidden_act}. "
112
+ "Only silu is supported for now.")
113
+
114
+ self.gate = ReplicatedLinear(config.hidden_size,
115
+ config.n_routed_experts,
116
+ bias=False,
117
+ quant_config=None,
118
+ prefix=f"{prefix}.gate")
119
+ if config.topk_method == "noaux_tc":
120
+ self.gate.e_score_correction_bias = nn.Parameter(
121
+ torch.empty(config.n_routed_experts))
122
+ else:
123
+ self.gate.e_score_correction_bias = None
124
+
125
+ self.experts = FusedMoE(
126
+ num_experts=config.n_routed_experts,
127
+ top_k=config.num_experts_per_tok,
128
+ hidden_size=config.hidden_size,
129
+ intermediate_size=config.moe_intermediate_size,
130
+ reduce_results=False,
131
+ renormalize=config.norm_topk_prob,
132
+ quant_config=quant_config,
133
+ use_grouped_topk=True,
134
+ num_expert_group=config.n_group,
135
+ topk_group=config.topk_group,
136
+ prefix=f"{prefix}.experts",
137
+ scoring_func=config.scoring_func,
138
+ e_score_correction_bias=self.gate.e_score_correction_bias)
139
+
140
+ if config.n_shared_experts is not None:
141
+ intermediate_size = (config.moe_intermediate_size *
142
+ config.n_shared_experts)
143
+ self.shared_experts = DeepseekV3MLP(
144
+ hidden_size=config.hidden_size,
145
+ intermediate_size=intermediate_size,
146
+ hidden_act=config.hidden_act,
147
+ quant_config=quant_config,
148
+ reduce_results=False,
149
+ )
150
+
151
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
152
+ num_tokens, hidden_dim = hidden_states.shape
153
+ hidden_states = hidden_states.view(-1, hidden_dim)
154
+ if self.n_shared_experts is not None:
155
+ shared_output = self.shared_experts(hidden_states)
156
+ # router_logits: (num_tokens, n_experts)
157
+ router_logits, _ = self.gate(hidden_states)
158
+ final_hidden_states = self.experts(
159
+ hidden_states=hidden_states,
160
+ router_logits=router_logits) * self.routed_scaling_factor
161
+ if shared_output is not None:
162
+ final_hidden_states = final_hidden_states + shared_output
163
+ if self.tp_size > 1:
164
+ final_hidden_states = tensor_model_parallel_all_reduce(
165
+ final_hidden_states)
166
+
167
+ return final_hidden_states.view(num_tokens, hidden_dim)
168
+
169
+
170
+def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
171
+ import math
172
+ if scale <= 1:
173
+ return 1.0
174
+ return 0.1 * mscale * math.log(scale) + 1.0
175
+
176
+
177
+class DeepseekV3Attention(nn.Module):
178
+
179
+ def __init__(
180
+ self,
181
+ config: PretrainedConfig,
182
+ hidden_size: int,
183
+ num_heads: int,
184
+ qk_nope_head_dim: int,
185
+ qk_rope_head_dim: int,
186
+ v_head_dim: int,
187
+ q_lora_rank: int,
188
+ kv_lora_rank: int,
189
+ rope_theta: float = 10000,
190
+ rope_scaling: Optional[Dict[str, Any]] = None,
191
+ max_position_embeddings: int = 8192,
192
+ cache_config: Optional[CacheConfig] = None,
193
+ quant_config: Optional[QuantizationConfig] = None,
194
+ prefix: str = "",
195
+ ) -> None:
196
+ super().__init__()
197
+ self.hidden_size = hidden_size
198
+ self.qk_nope_head_dim = qk_nope_head_dim
199
+ self.qk_rope_head_dim = qk_rope_head_dim
200
+ self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
201
+ self.v_head_dim = v_head_dim
202
+ self.q_lora_rank = q_lora_rank
203
+ self.kv_lora_rank = kv_lora_rank
204
+ self.num_heads = num_heads
205
+ tp_size = get_tensor_model_parallel_world_size()
206
+ assert num_heads % tp_size == 0
207
+ self.num_local_heads = num_heads // tp_size
208
+ self.scaling = self.qk_head_dim**-0.5
209
+ self.rope_theta = rope_theta
210
+ self.max_position_embeddings = max_position_embeddings
211
+
212
+ if self.q_lora_rank is not None:
213
+ self.q_a_proj = ReplicatedLinear(self.hidden_size,
214
+ self.q_lora_rank,
215
+ bias=False,
216
+ quant_config=quant_config,
217
+ prefix=f"{prefix}.q_a_proj")
218
+ self.q_a_layernorm = RMSNorm(self.q_lora_rank,
219
+ eps=config.rms_norm_eps)
220
+ self.q_b_proj = ColumnParallelLinear(q_lora_rank,
221
+ self.num_heads *
222
+ self.qk_head_dim,
223
+ bias=False,
224
+ quant_config=quant_config,
225
+ prefix=f"{prefix}.q_b_proj")
226
+ else:
227
+ self.q_proj = ColumnParallelLinear(self.hidden_size,
228
+ self.num_heads *
229
+ self.qk_head_dim,
230
+ bias=False,
231
+ quant_config=quant_config,
232
+ prefix=f"{prefix}.q_proj")
233
+
234
+ self.kv_a_proj_with_mqa = ReplicatedLinear(
235
+ self.hidden_size,
236
+ self.kv_lora_rank + self.qk_rope_head_dim,
237
+ bias=False,
238
+ quant_config=quant_config,
239
+ prefix=f"{prefix}.kv_a_proj_with_mqa")
240
+ self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
241
+ eps=config.rms_norm_eps)
242
+ self.kv_b_proj = ColumnParallelLinear(
243
+ self.kv_lora_rank,
244
+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
245
+ bias=False,
246
+ quant_config=quant_config,
247
+ prefix=f"{prefix}.kv_b_proj")
248
+ # O projection.
249
+ self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
250
+ self.hidden_size,
251
+ bias=False,
252
+ quant_config=quant_config,
253
+ prefix=f"{prefix}.o_proj")
254
+ rope_scaling["rope_type"] = 'deepseek_yarn'
255
+ self.rotary_emb = get_rope(qk_rope_head_dim,
256
+ rotary_dim=qk_rope_head_dim,
257
+ max_position=max_position_embeddings,
258
+ base=rope_theta,
259
+ rope_scaling=rope_scaling,
260
+ is_neox_style=False)
261
+
262
+ if rope_scaling:
263
+ mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
264
+ scaling_factor = rope_scaling["factor"]
265
+ mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
266
+ self.scaling = self.scaling * mscale * mscale
267
+
268
+ # self.attn = Attention(self.num_heads,
269
+ # self.qk_head_dim,
270
+ # self.scaling,
271
+ # num_kv_heads=self.num_heads)
272
+
273
+ # TODO, support head_size 192
274
+ self.attn = Attention(self.num_local_heads,
275
+ 256,
276
+ self.scaling,
277
+ num_kv_heads=self.num_local_heads,
278
+ cache_config=cache_config,
279
+ quant_config=quant_config,
280
+ prefix=f"{prefix}.attn")
281
+
282
+ def forward(
283
+ self,
284
+ positions: torch.Tensor,
285
+ hidden_states: torch.Tensor,
286
+ kv_cache: torch.Tensor,
287
+ attn_metadata: AttentionMetadata,
288
+ ) -> torch.Tensor:
289
+ if self.q_lora_rank is not None:
290
+ q = self.q_a_proj(hidden_states)[0]
291
+ q = self.q_a_layernorm(q)
292
+ q = self.q_b_proj(q)[0].view(-1, self.num_local_heads,
293
+ self.qk_head_dim)
294
+ else:
295
+ q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads,
296
+ self.qk_head_dim)
297
+ q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim],
298
+ dim=-1)
299
+ latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0]
300
+ kv_a, _ = latent_cache.split(
301
+ [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
302
+ latent_cache = latent_cache.unsqueeze(1)
303
+ kv_a = self.kv_a_layernorm(kv_a.contiguous())
304
+ kv = self.kv_b_proj(kv_a)[0]
305
+ kv = kv.view(-1, self.num_local_heads,
306
+ self.qk_nope_head_dim + self.v_head_dim)
307
+ k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
308
+ k_pe = latent_cache[:, :, self.kv_lora_rank:]
309
+ q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
310
+ q[..., self.qk_nope_head_dim:] = q_pe
311
+ k = torch.empty_like(q)
312
+ k[..., :self.qk_nope_head_dim] = k_nope
313
+ k[..., self.qk_nope_head_dim:] = k_pe
314
+ q = torch.nn.functional.pad(q, [0, 256 - self.qk_head_dim],
315
+ value=0).view(-1,
316
+ self.num_local_heads * 256)
317
+ k = torch.nn.functional.pad(k, [0, 256 - self.qk_head_dim],
318
+ value=0).view(-1,
319
+ self.num_local_heads * 256)
320
+ v = torch.nn.functional.pad(v, [0, 256 - self.v_head_dim],
321
+ value=0).view(-1,
322
+ self.num_local_heads * 256)
323
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
324
+ attn_output = attn_output.view(
325
+ -1, self.num_local_heads, 256)[..., :self.v_head_dim].reshape(
326
+ -1, self.num_local_heads * self.v_head_dim)
327
+ output, _ = self.o_proj(attn_output)
328
+ return output
329
+
330
+
331
+class DeepseekV3DecoderLayer(nn.Module):
332
+
333
+ def __init__(
334
+ self,
335
+ config: PretrainedConfig,
336
+ prefix: str,
337
+ cache_config: Optional[CacheConfig] = None,
338
+ quant_config: Optional[QuantizationConfig] = None,
339
+ ) -> None:
340
+ super().__init__()
341
+ self.hidden_size = config.hidden_size
342
+ rope_theta = getattr(config, "rope_theta", 10000)
343
+ rope_scaling = getattr(config, "rope_scaling", None)
344
+ max_position_embeddings = getattr(config, "max_position_embeddings",
345
+ 8192)
346
+ # DecoderLayers are created with `make_layers` which passes the prefix
347
+ # with the layer's index.
348
+ layer_idx = int(prefix.split(sep='.')[-1])
349
+ self.self_attn = DeepseekV3Attention(
350
+ config=config,
351
+ hidden_size=self.hidden_size,
352
+ num_heads=config.num_attention_heads,
353
+ qk_nope_head_dim=config.qk_nope_head_dim,
354
+ qk_rope_head_dim=config.qk_rope_head_dim,
355
+ v_head_dim=config.v_head_dim,
356
+ q_lora_rank=config.q_lora_rank
357
+ if hasattr(config, "q_lora_rank") else None,
358
+ kv_lora_rank=config.kv_lora_rank,
359
+ rope_theta=rope_theta,
360
+ rope_scaling=rope_scaling,
361
+ max_position_embeddings=max_position_embeddings,
362
+ cache_config=cache_config,
363
+ quant_config=quant_config,
364
+ prefix=f"{prefix}.self_attn",
365
+ )
366
+ if (config.n_routed_experts is not None
367
+ and layer_idx >= config.first_k_dense_replace
368
+ and layer_idx % config.moe_layer_freq == 0):
369
+ self.mlp = DeepseekV3MoE(
370
+ config=config,
371
+ quant_config=quant_config,
372
+ prefix=f"{prefix}.mlp",
373
+ )
374
+ else:
375
+ self.mlp = DeepseekV3MLP(
376
+ hidden_size=config.hidden_size,
377
+ intermediate_size=config.intermediate_size,
378
+ hidden_act=config.hidden_act,
379
+ quant_config=quant_config,
380
+ prefix=f"{prefix}.mlp",
381
+ )
382
+ self.input_layernorm = RMSNorm(config.hidden_size,
383
+ eps=config.rms_norm_eps)
384
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
385
+ eps=config.rms_norm_eps)
386
+
387
+ def forward(
388
+ self,
389
+ positions: torch.Tensor,
390
+ hidden_states: torch.Tensor,
391
+ kv_cache: torch.Tensor,
392
+ attn_metadata: AttentionMetadata,
393
+ residual: Optional[torch.Tensor],
394
+ ) -> torch.Tensor:
395
+ # Self Attention
396
+ if residual is None:
397
+ residual = hidden_states
398
+ hidden_states = self.input_layernorm(hidden_states)
399
+ else:
400
+ hidden_states, residual = self.input_layernorm(
401
+ hidden_states, residual)
402
+ hidden_states = self.self_attn(
403
+ positions=positions,
404
+ hidden_states=hidden_states,
405
+ kv_cache=kv_cache,
406
+ attn_metadata=attn_metadata,
407
+ )
408
+
409
+ # Fully Connected
410
+ hidden_states, residual = self.post_attention_layernorm(
411
+ hidden_states, residual)
412
+ hidden_states = self.mlp(hidden_states)
413
+ return hidden_states, residual
414
+
415
+
416
+# TODO(simon): check whether we support torch compile for Deepseek V3
417
+# @support_torch_compile
418
+class DeepseekV3Model(nn.Module):
419
+
420
+ fall_back_to_pt_during_load = False
421
+
422
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
423
+ super().__init__()
424
+
425
+ config = vllm_config.model_config.hf_config
426
+ cache_config = vllm_config.cache_config
427
+ quant_config = vllm_config.quant_config
428
+
429
+ self.padding_idx = config.pad_token_id
430
+ self.vocab_size = config.vocab_size
431
+
432
+ if get_pp_group().is_first_rank:
433
+ self.embed_tokens = VocabParallelEmbedding(
434
+ config.vocab_size,
435
+ config.hidden_size,
436
+ )
437
+ else:
438
+ self.embed_tokens = PPMissingLayer()
439
+
440
+ self.start_layer, self.end_layer, self.layers = make_layers(
441
+ config.num_hidden_layers,
442
+ lambda prefix: DeepseekV3DecoderLayer(
443
+ config,
444
+ prefix,
445
+ cache_config=cache_config,
446
+ quant_config=quant_config,
447
+ ),
448
+ prefix=f"{prefix}.layers")
449
+
450
+ if get_pp_group().is_last_rank:
451
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
452
+ else:
453
+ self.norm = PPMissingLayer()
454
+ self.make_empty_intermediate_tensors = (
455
+ make_empty_intermediate_tensors_factory(
456
+ ["hidden_states", "residual"], config.hidden_size))
457
+
458
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
459
+ return self.embed_tokens(input_ids)
460
+
461
+ def forward(
462
+ self,
463
+ input_ids: torch.Tensor,
464
+ positions: torch.Tensor,
465
+ kv_caches: List[torch.Tensor],
466
+ attn_metadata: AttentionMetadata,
467
+ intermediate_tensors: Optional[IntermediateTensors],
468
+ inputs_embeds: Optional[torch.Tensor] = None,
469
+ ) -> Union[torch.Tensor, IntermediateTensors]:
470
+ if get_pp_group().is_first_rank:
471
+ if inputs_embeds is not None:
472
+ hidden_states = inputs_embeds
473
+ else:
474
+ hidden_states = self.get_input_embeddings(input_ids)
475
+ residual = None
476
+ else:
477
+ assert intermediate_tensors is not None
478
+ hidden_states = intermediate_tensors["hidden_states"]
479
+ residual = intermediate_tensors["residual"]
480
+
481
+ for i in range(self.start_layer, self.end_layer):
482
+ layer = self.layers[i]
483
+ hidden_states, residual = layer(positions, hidden_states,
484
+ kv_caches[i - self.start_layer],
485
+ attn_metadata, residual)
486
+
487
+ if not get_pp_group().is_last_rank:
488
+ return IntermediateTensors({
489
+ "hidden_states": hidden_states,
490
+ "residual": residual
491
+ })
492
+
493
+ hidden_states, _ = self.norm(hidden_states, residual)
494
+ return hidden_states
495
+
496
+
497
+class DeepseekV3ForCausalLM(nn.Module, SupportsPP):
498
+
499
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
500
+ super().__init__()
501
+ config = vllm_config.model_config.hf_config
502
+ quant_config = vllm_config.quant_config
503
+ self.config = config
504
+ self.quant_config = quant_config
505
+ self.model = DeepseekV3Model(vllm_config=vllm_config,
506
+ prefix=maybe_prefix(prefix, "model"))
507
+ self.lm_head = ParallelLMHead(config.vocab_size,
508
+ config.hidden_size,
509
+ quant_config=quant_config)
510
+ self.logits_processor = LogitsProcessor(config.vocab_size)
511
+ self.sampler = get_sampler()
512
+ self.make_empty_intermediate_tensors = (
513
+ self.model.make_empty_intermediate_tensors)
514
+
515
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
516
+ return self.model.get_input_embeddings(input_ids)
517
+
518
+ def forward(
519
+ self,
520
+ input_ids: torch.Tensor,
521
+ positions: torch.Tensor,
522
+ kv_caches: List[torch.Tensor],
523
+ attn_metadata: AttentionMetadata,
524
+ intermediate_tensors: Optional[IntermediateTensors] = None,
525
+ inputs_embeds: Optional[torch.Tensor] = None,
526
+ ) -> Union[torch.Tensor, IntermediateTensors]:
527
+ hidden_states = self.model(input_ids, positions, kv_caches,
528
+ attn_metadata, intermediate_tensors,
529
+ inputs_embeds)
530
+ return hidden_states
531
+
532
+ def compute_logits(
533
+ self,
534
+ hidden_states: torch.Tensor,
535
+ sampling_metadata: SamplingMetadata,
536
+ ) -> Optional[torch.Tensor]:
537
+ logits = self.logits_processor(self.lm_head, hidden_states,
538
+ sampling_metadata)
539
+ return logits
540
+
541
+ def sample(
542
+ self,
543
+ logits: Optional[torch.Tensor],
544
+ sampling_metadata: SamplingMetadata,
545
+ ) -> Optional[SamplerOutput]:
546
+ next_tokens = self.sampler(logits, sampling_metadata)
547
+ return next_tokens
548
+
549
+ def make_empty_intermediate_tensors(
550
+ self, batch_size: int, dtype: torch.dtype,
551
+ device: torch.device) -> IntermediateTensors:
552
+ return IntermediateTensors({
553
+ "hidden_states":
554
+ torch.zeros((batch_size, self.config.hidden_size),
555
+ dtype=dtype,
556
+ device=device),
557
+ "residual":
558
+ torch.zeros((batch_size, self.config.hidden_size),
559
+ dtype=dtype,
560
+ device=device),
561
+ })
562
+
563
+ def load_weights(self, weights: Iterable[Tuple[str,
564
+ torch.Tensor]]) -> Set[str]:
565
+ stacked_params_mapping = [
566
+ # (param_name, shard_name, shard_id)
567
+ ("gate_up_proj", "gate_proj", 0),
568
+ ("gate_up_proj", "up_proj", 1),
569
+ ]
570
+
571
+ # Params for weights, fp8 weight scales, fp8 activation scales
572
+ # (param_name, weight_name, expert_id, shard_id)
573
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
574
+ ckpt_gate_proj_name="gate_proj",
575
+ ckpt_down_proj_name="down_proj",
576
+ ckpt_up_proj_name="up_proj",
577
+ num_experts=self.config.n_routed_experts)
578
+
579
+ params_dict = dict(self.named_parameters())
580
+ loaded_params: Set[str] = set()
581
+ for name, loaded_weight in weights:
582
+ if "rotary_emb.inv_freq" in name:
583
+ continue
584
+
585
+ # TODO(simon): support nextn predict layers
586
+ if self.config.num_nextn_predict_layers > 0:
587
+ assert self.config.num_nextn_predict_layers == 1
588
+ layer_idx = self.config.num_hidden_layers
589
+ if name.startswith(f"model.layers.{layer_idx}"):
590
+ continue
591
+
592
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
593
+ # Skip non-stacked layers and experts (experts handled below).
594
+ if weight_name not in name:
595
+ continue
596
+ # We have mlp.experts[0].gate_proj in the checkpoint.
597
+ # Since we handle the experts below in expert_params_mapping,
598
+ # we need to skip here BEFORE we update the name, otherwise
599
+ # name will be updated to mlp.experts[0].gate_up_proj, which
600
+ # will then be updated below in expert_params_mapping
601
+ # for mlp.experts[0].gate_gate_up_proj, which breaks load.
602
+ if (("mlp.experts." in name) and name not in params_dict):
603
+ continue
604
+ name = name.replace(weight_name, param_name)
605
+ # Skip loading extra bias for GPTQ models.
606
+ if name.endswith(".bias") and name not in params_dict:
607
+ continue
608
+
609
+ if is_pp_missing_parameter(name, self):
610
+ continue
611
+
612
+ param = params_dict[name]
613
+ weight_loader = param.weight_loader
614
+ weight_loader(param, loaded_weight, shard_id)
615
+ break
616
+ else:
617
+ for mapping in expert_params_mapping:
618
+ param_name, weight_name, expert_id, shard_id = mapping
619
+ if weight_name not in name:
620
+ continue
621
+ name = name.replace(weight_name, param_name)
622
+
623
+ if is_pp_missing_parameter(name, self):
624
+ continue
625
+
626
+ param = params_dict[name]
627
+ weight_loader = param.weight_loader
628
+ weight_loader(param,
629
+ loaded_weight,
630
+ name,
631
+ shard_id=shard_id,
632
+ expert_id=expert_id)
633
+ break
634
+ else:
635
+ # Skip loading extra bias for GPTQ models.
636
+ if name.endswith(".bias") and name not in params_dict:
637
+ continue
638
+
639
+ if is_pp_missing_parameter(name, self):
640
+ continue
641
+
642
+ if name not in params_dict:
643
+ for key in params_dict:
644
+ print(key)
645
+ param = params_dict[name]
646
+ weight_loader = getattr(param, "weight_loader",
647
+ default_weight_loader)
648
+ weight_loader(param, loaded_weight)
649
+ loaded_params.add(name)
650
+ return loaded_params
vllm/model_executor/models/registry.py
+1
@@ -45,6 +45,7 @@ _TEXT_GENERATION_MODELS = {
45
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
46
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
47
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
48
+ "DeepseekV3ForCausalLM": ("deepseek_v3", "DeepseekV3ForCausalLM"),
49
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
50
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
51
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),