@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f49777ba62
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

Deepseek v3 (#11502)

Signed-off-by: mgoin <michael@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com> Co-authored-by: robertgshaw2-neuralmagic <rshaw@neuralmagic.com>

Simon Mo committed Dec 26, 2024 at 16:09 UTC f49777ba62b4926d0f8c100ab06edb03c5c10098
7 files changed +887 -61
csrc/moe/moe_align_sum_kernels.cu
+141 -19
@@ -113,6 +113,92 @@ __global__ void moe_align_block_size_kernel(scalar_t* __restrict__ topk_ids,
113 }
114 }
115
116 +// TODO(simon): this is temporarily adapted from
117 +// https://github.com/sgl-project/sglang/commit/31548116a8dc8c6df7e146e0587335a59fc5b9d7
118 +// we did this to unblock Deepseek V3 but there should be a better
119 +// implementation to manage shared memory.
120 +template <typename scalar_t>
121 +__global__ void moe_align_block_size_global_mem_kernel(
122 + scalar_t* __restrict__ topk_ids, int32_t* sorted_token_ids,
123 + int32_t* expert_ids, int32_t* total_tokens_post_pad, int32_t num_experts,
124 + int32_t block_size, size_t numel, int32_t* tokens_cnts, int32_t* cumsum) {
125 + const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
126 + const size_t start_idx = threadIdx.x * tokens_per_thread;
127 +
128 + for (int i = 0; i < num_experts; ++i) {
129 + tokens_cnts[index(num_experts, threadIdx.x + 1, i)] = 0;
130 + }
131 +
132 + /**
133 + * In the first step we compute token_cnts[thread_index + 1][expert_index],
134 + * which counts how many tokens in the token shard of thread_index are
135 + * assigned to expert expert_index.
136 + */
137 + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
138 + ++tokens_cnts[index(num_experts, threadIdx.x + 1, topk_ids[i])];
139 + }
140 +
141 + __syncthreads();
142 +
143 + // For each expert we accumulate the token counts from the different threads.
144 + if (threadIdx.x < num_experts) {
145 + tokens_cnts[index(num_experts, 0, threadIdx.x)] = 0;
146 + for (int i = 1; i <= blockDim.x; ++i) {
147 + tokens_cnts[index(num_experts, i, threadIdx.x)] +=
148 + tokens_cnts[index(num_experts, i - 1, threadIdx.x)];
149 + }
150 + }
151 +
152 + __syncthreads();
153 +
154 + // We accumulate the token counts of all experts in thread 0.
155 + if (threadIdx.x == 0) {
156 + cumsum[0] = 0;
157 + for (int i = 1; i <= num_experts; ++i) {
158 + cumsum[i] = cumsum[i - 1] +
159 + CEILDIV(tokens_cnts[index(num_experts, blockDim.x, i - 1)],
160 + block_size) *
161 + block_size;
162 + }
163 + *total_tokens_post_pad = cumsum[num_experts];
164 + }
165 +
166 + __syncthreads();
167 +
168 + /**
169 + * For each expert, each thread processes the tokens of the corresponding
170 + * blocks and stores the corresponding expert_id for each block.
171 + */
172 + if (threadIdx.x < num_experts) {
173 + for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1];
174 + i += block_size) {
175 + expert_ids[i / block_size] = threadIdx.x;
176 + }
177 + }
178 +
179 + /**
180 + * Each thread processes a token shard, calculating the index of each token
181 + * after sorting by expert number. Given the example topk_ids =
182 + * [0,1,2,1,2,3,0,3,4] and block_size = 4, then the output would be [0, 6, *,
183 + * *, 1, 3, *, *, 2, 4, *, *, 5, 7, *, *, 8, *, *, *], where * represents a
184 + * padding value(preset in python).
185 + */
186 + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
187 + int32_t expert_id = topk_ids[i];
188 + /** The cumsum[expert_id] stores the starting index of the tokens that the
189 + * expert with expert_id needs to process, and
190 + * tokens_cnts[threadIdx.x][expert_id] stores the indices of the tokens
191 + * processed by the expert with expert_id within the current thread's token
192 + * shard.
193 + */
194 + int32_t rank_post_pad =
195 + tokens_cnts[index(num_experts, threadIdx.x, expert_id)] +
196 + cumsum[expert_id];
197 + sorted_token_ids[rank_post_pad] = i;
198 + ++tokens_cnts[index(num_experts, threadIdx.x, expert_id)];
199 + }
200 +}
201 +
202 template <typename scalar_t, int TOPK>
203 __global__ void moe_sum_kernel(
204 scalar_t* __restrict__ out, // [..., d]
@@ -137,25 +223,61 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
223 torch::Tensor experts_ids,
224 torch::Tensor num_tokens_post_pad) {
225 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
140 - VLLM_DISPATCH_INTEGRAL_TYPES(
141 - topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
142 - // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
143 - // tensors
144 - const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
145 - const int32_t shared_mem =
146 - ((num_thread + 1) * num_experts + (num_experts + 1)) *
147 - sizeof(int32_t);
148 -
149 - // set dynamic shared mem
150 - auto kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
151 - AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
152 - (void*)kernel, shared_mem));
153 - kernel<<<1, num_thread, shared_mem, stream>>>(
154 - topk_ids.data_ptr<scalar_t>(), sorted_token_ids.data_ptr<int32_t>(),
155 - experts_ids.data_ptr<int32_t>(),
156 - num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
157 - topk_ids.numel());
158 - });
226 +
227 + // If we have very large number of experts, we can no longer use shared
228 + // memory.
229 + // TODO(simon): the right solution should be calculating the exact right
230 + // amount of shared memory and use that. The num_experts >= 256 is just a
231 + // temporary solution to unblock Deepseek V3.
232 + if (num_experts >= 256) {
233 + VLLM_DISPATCH_INTEGRAL_TYPES(
234 + topk_ids.scalar_type(), "moe_align_block_size_global_mem_kernel", [&] {
235 + // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
236 + // tensors
237 + const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
238 +
239 + const int32_t mem_tokens_cnts =
240 + ((num_experts + 1) * num_experts) * sizeof(int32_t);
241 + const int32_t mem_cumsum = (num_experts + 1) * sizeof(int32_t);
242 + // allocate global memory
243 + int32_t* tokens_cnts;
244 + int32_t* cumsum;
245 + cudaMalloc(&tokens_cnts, mem_tokens_cnts);
246 + cudaMalloc(&cumsum, mem_cumsum);
247 +
248 + auto kernel =
249 + vllm::moe::moe_align_block_size_global_mem_kernel<scalar_t>;
250 + kernel<<<1, num_thread, 0, stream>>>(
251 + topk_ids.data_ptr<scalar_t>(),
252 + sorted_token_ids.data_ptr<int32_t>(),
253 + experts_ids.data_ptr<int32_t>(),
254 + num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
255 + topk_ids.numel(), tokens_cnts, cumsum);
256 + cudaFree(tokens_cnts);
257 + cudaFree(cumsum);
258 + });
259 + } else {
260 + VLLM_DISPATCH_INTEGRAL_TYPES(
261 + topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] {
262 + // calc needed amount of shared mem for `tokens_cnts` and `cumsum`
263 + // tensors
264 + const int32_t num_thread = max((int32_t)num_experts, WARP_SIZE);
265 + const int32_t shared_mem =
266 + ((num_thread + 1) * num_experts + (num_experts + 1)) *
267 + sizeof(int32_t);
268 +
269 + // set dynamic shared mem
270 + auto kernel = vllm::moe::moe_align_block_size_kernel<scalar_t>;
271 + AT_CUDA_CHECK(VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(
272 + (void*)kernel, shared_mem));
273 + kernel<<<1, num_thread, shared_mem, stream>>>(
274 + topk_ids.data_ptr<scalar_t>(),
275 + sorted_token_ids.data_ptr<int32_t>(),
276 + experts_ids.data_ptr<int32_t>(),
277 + num_tokens_post_pad.data_ptr<int32_t>(), num_experts, block_size,
278 + topk_ids.numel());
279 + });
280 + }
281 }
282
283 void moe_sum(torch::Tensor& input, // [num_tokens, topk, hidden_size]
vllm/config.py
+9 -2
@@ -596,6 +596,12 @@ class ModelConfig:
596 self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
597 self.max_model_len)
598
599 + if (self.hf_config.model_type == 'deepseek_v3'
600 + and not self.enforce_eager):
601 + logger.warning("CUDA graph is not supported for Deepseek V3 yet, "
602 + "fallback to the eager mode.")
603 + self.enforce_eager = True
604 +
605 def _verify_bnb_config(self) -> None:
606 """
607 The current version of bitsandbytes (0.44.0) with 8-bit models does not
@@ -712,8 +718,9 @@ class ModelConfig:
718
719 def get_head_size(self) -> int:
720 # TODO remove hard code
715 - if hasattr(self.hf_text_config, "model_type"
716 - ) and self.hf_text_config.model_type == 'deepseek_v2':
721 + if hasattr(self.hf_text_config,
722 + "model_type") and (self.hf_text_config.model_type
723 + in ('deepseek_v2', 'deepseek_v3')):
724 # FlashAttention supports only head_size 32, 64, 128, 256,
725 # we need to pad head_size 192 to 256
726 return 256
vllm/model_executor/layers/fused_moe/fused_moe.py
+14 -3
@@ -476,18 +476,29 @@ def fused_topk(
476 return topk_weights, topk_ids
477
478
479 -# This is used by the Deepseek-V2 model
479 +# This is used by the Deepseek-V2 and Deepseek-V3 model
480 def grouped_topk(hidden_states: torch.Tensor,
481 gating_output: torch.Tensor,
482 topk: int,
483 renormalize: bool,
484 num_expert_group: int = 0,
485 - topk_group: int = 0):
485 + topk_group: int = 0,
486 + scoring_func: str = "softmax",
487 + e_score_correction_bias: Optional[torch.Tensor] = None):
488
489 assert hidden_states.shape[0] == gating_output.shape[0], (
490 "Number of tokens mismatch")
491
490 - scores = torch.softmax(gating_output, dim=-1)
492 + if scoring_func == "softmax":
493 + scores = torch.softmax(gating_output, dim=-1)
494 + elif scoring_func == "sigmoid":
495 + scores = gating_output.sigmoid()
496 + else:
497 + raise ValueError(f"Unsupported scoring function: {scoring_func}")
498 +
499 + if e_score_correction_bias is not None:
500 + scores.add_(e_score_correction_bias.unsqueeze(0))
501 +
502 num_token = scores.shape[0]
503 group_scores = scores.view(num_token, num_expert_group,
504 -1).max(dim=-1).values # [n, n_group]
vllm/model_executor/layers/fused_moe/layer.py
+66 -36
@@ -73,16 +73,18 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
73 set_weight_attrs(w2_weight, extra_weight_attrs)
74
75 def apply(
76 - self,
77 - layer: torch.nn.Module,
78 - x: torch.Tensor,
79 - router_logits: torch.Tensor,
80 - top_k: int,
81 - renormalize: bool,
82 - use_grouped_topk: bool,
83 - topk_group: Optional[int] = None,
84 - num_expert_group: Optional[int] = None,
85 - custom_routing_function: Optional[Callable] = None
76 + self,
77 + layer: torch.nn.Module,
78 + x: torch.Tensor,
79 + router_logits: torch.Tensor,
80 + top_k: int,
81 + renormalize: bool,
82 + use_grouped_topk: bool,
83 + topk_group: Optional[int] = None,
84 + num_expert_group: Optional[int] = None,
85 + custom_routing_function: Optional[Callable] = None,
86 + scoring_func: str = "softmax",
87 + e_score_correction_bias: Optional[torch.Tensor] = None
88 ) -> torch.Tensor:
89 return self.forward(x=x,
90 layer=layer,
@@ -92,19 +94,23 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
94 use_grouped_topk=use_grouped_topk,
95 topk_group=topk_group,
96 num_expert_group=num_expert_group,
95 - custom_routing_function=custom_routing_function)
97 + custom_routing_function=custom_routing_function,
98 + scoring_func=scoring_func,
99 + e_score_correction_bias=e_score_correction_bias)
100
101 def forward_cuda(
98 - self,
99 - layer: torch.nn.Module,
100 - x: torch.Tensor,
101 - use_grouped_topk: bool,
102 - top_k: int,
103 - router_logits: torch.Tensor,
104 - renormalize: bool,
105 - topk_group: Optional[int] = None,
106 - num_expert_group: Optional[int] = None,
107 - custom_routing_function: Optional[Callable] = None
102 + self,
103 + layer: torch.nn.Module,
104 + x: torch.Tensor,
105 + use_grouped_topk: bool,
106 + top_k: int,
107 + router_logits: torch.Tensor,
108 + renormalize: bool,
109 + topk_group: Optional[int] = None,
110 + num_expert_group: Optional[int] = None,
111 + custom_routing_function: Optional[Callable] = None,
112 + scoring_func: str = "softmax",
113 + e_score_correction_bias: Optional[torch.Tensor] = None
114 ) -> torch.Tensor:
115 topk_weights, topk_ids = FusedMoE.select_experts(
116 hidden_states=x,
@@ -114,7 +120,9 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
120 renormalize=renormalize,
121 topk_group=topk_group,
122 num_expert_group=num_expert_group,
117 - custom_routing_function=custom_routing_function)
123 + custom_routing_function=custom_routing_function,
124 + scoring_func=scoring_func,
125 + e_score_correction_bias=e_score_correction_bias)
126
127 return fused_experts(hidden_states=x,
128 w1=layer.w13_weight,
@@ -128,21 +136,29 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
136 "The CPU backend currently does not support MoE.")
137
138 def forward_tpu(
131 - self,
132 - layer: torch.nn.Module,
133 - x: torch.Tensor,
134 - use_grouped_topk: bool,
135 - top_k: int,
136 - router_logits: torch.Tensor,
137 - renormalize: bool,
138 - topk_group: Optional[int] = None,
139 - num_expert_group: Optional[int] = None,
140 - custom_routing_function: Optional[Callable] = None
139 + self,
140 + layer: torch.nn.Module,
141 + x: torch.Tensor,
142 + use_grouped_topk: bool,
143 + top_k: int,
144 + router_logits: torch.Tensor,
145 + renormalize: bool,
146 + topk_group: Optional[int] = None,
147 + num_expert_group: Optional[int] = None,
148 + custom_routing_function: Optional[Callable] = None,
149 + scoring_func: str = "softmax",
150 + e_score_correction_bias: Optional[torch.Tensor] = None
151 ) -> torch.Tensor:
152 assert not use_grouped_topk
153 assert num_expert_group is None
154 assert topk_group is None
155 assert custom_routing_function is None
156 + if scoring_func != "softmax":
157 + raise NotImplementedError(
158 + "Only softmax scoring function is supported for TPU.")
159 + if e_score_correction_bias is not None:
160 + raise NotImplementedError(
161 + "Expert score correction bias is not supported for TPU.")
162 return fused_moe_pallas(hidden_states=x,
163 w1=layer.w13_weight,
164 w2=layer.w2_weight,
@@ -156,7 +172,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
172 class FusedMoE(torch.nn.Module):
173 """FusedMoE layer for MoE models.
174
159 - This layer contains both MergedColumnParallel weights (gate_up_proj /
175 + This layer contains both MergedColumnParallel weights (gate_up_proj /
176 w13) and RowParallelLinear weights (down_proj/ w2).
177
178 Note: Mixtral uses w1, w2, and w3 for gate, up, and down_proj. We
@@ -190,6 +206,8 @@ class FusedMoE(torch.nn.Module):
206 tp_size: Optional[int] = None,
207 prefix: str = "",
208 custom_routing_function: Optional[Callable] = None,
209 + scoring_func: str = "softmax",
210 + e_score_correction_bias: Optional[torch.Tensor] = None,
211 ):
212 super().__init__()
213
@@ -210,6 +228,12 @@ class FusedMoE(torch.nn.Module):
228 self.num_expert_group = num_expert_group
229 self.topk_group = topk_group
230 self.custom_routing_function = custom_routing_function
231 + self.scoring_func = scoring_func
232 + self.e_score_correction_bias = e_score_correction_bias
233 +
234 + if self.scoring_func != "softmax" and not self.use_grouped_topk:
235 + raise ValueError("Only softmax scoring function is supported for "
236 + "non-grouped topk.")
237
238 if quant_config is None:
239 self.quant_method: Optional[QuantizeMethodBase] = (
@@ -446,7 +470,9 @@ class FusedMoE(torch.nn.Module):
470 renormalize: bool,
471 topk_group: Optional[int] = None,
472 num_expert_group: Optional[int] = None,
449 - custom_routing_function: Optional[Callable] = None):
473 + custom_routing_function: Optional[Callable] = None,
474 + scoring_func: str = "softmax",
475 + e_score_correction_bias: Optional[torch.Tensor] = None):
476 from vllm.model_executor.layers.fused_moe.fused_moe import (
477 fused_topk, grouped_topk)
478
@@ -460,7 +486,9 @@ class FusedMoE(torch.nn.Module):
486 topk=top_k,
487 renormalize=renormalize,
488 num_expert_group=num_expert_group,
463 - topk_group=topk_group)
489 + topk_group=topk_group,
490 + scoring_func=scoring_func,
491 + e_score_correction_bias=e_score_correction_bias)
492 elif custom_routing_function is None:
493 topk_weights, topk_ids = fused_topk(hidden_states=hidden_states,
494 gating_output=router_logits,
@@ -489,7 +517,9 @@ class FusedMoE(torch.nn.Module):
517 use_grouped_topk=self.use_grouped_topk,
518 topk_group=self.topk_group,
519 num_expert_group=self.num_expert_group,
492 - custom_routing_function=self.custom_routing_function)
520 + custom_routing_function=self.custom_routing_function,
521 + scoring_func=self.scoring_func,
522 + e_score_correction_bias=self.e_score_correction_bias)
523
524 if self.reduce_results and self.tp_size > 1:
525 final_hidden_states = tensor_model_parallel_all_reduce(
vllm/model_executor/layers/quantization/fp8.py
+6 -1
@@ -605,6 +605,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
605 topk_group: Optional[int] = None,
606 num_expert_group: Optional[int] = None,
607 custom_routing_function: Optional[Callable] = None,
608 + scoring_func: str = "softmax",
609 + e_score_correction_bias: Optional[torch.Tensor] = None,
610 ) -> torch.Tensor:
611
612 from vllm.model_executor.layers.fused_moe import fused_experts
@@ -617,7 +619,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
619 renormalize=renormalize,
620 topk_group=topk_group,
621 num_expert_group=num_expert_group,
620 - custom_routing_function=custom_routing_function)
622 + custom_routing_function=custom_routing_function,
623 + scoring_func=scoring_func,
624 + e_score_correction_bias=e_score_correction_bias,
625 + )
626
627 return fused_experts(
628 x,
vllm/model_executor/models/deepseek_v3.py new
+650
@@ -0,0 +1,650 @@
1 +# Adapted from
2 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 +# Copyright 2023 The vLLM team.
4 +# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
5 +#
6 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7 +# and OPT implementations in this library. It has been modified from its
8 +# original forms to accommodate minor architectural differences compared
9 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10 +#
11 +# Licensed under the Apache License, Version 2.0 (the "License");
12 +# you may not use this file except in compliance with the License.
13 +# You may obtain a copy of the License at
14 +#
15 +# http://www.apache.org/licenses/LICENSE-2.0
16 +#
17 +# Unless required by applicable law or agreed to in writing, software
18 +# distributed under the License is distributed on an "AS IS" BASIS,
19 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 +# See the License for the specific language governing permissions and
21 +# limitations under the License.
22 +"""Inference-only DeepseekV3 model."""
23 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
24 +
25 +import torch
26 +from torch import nn
27 +from transformers import PretrainedConfig
28 +
29 +from vllm.attention import Attention, AttentionMetadata
30 +from vllm.config import CacheConfig, VllmConfig
31 +from vllm.distributed import (get_pp_group,
32 + get_tensor_model_parallel_world_size,
33 + tensor_model_parallel_all_reduce)
34 +from vllm.model_executor.layers.activation import SiluAndMul
35 +from vllm.model_executor.layers.fused_moe import FusedMoE
36 +from vllm.model_executor.layers.layernorm import RMSNorm
37 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
38 + MergedColumnParallelLinear,
39 + ReplicatedLinear,
40 + RowParallelLinear)
41 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 +from vllm.model_executor.layers.quantization import QuantizationConfig
43 +from vllm.model_executor.layers.rotary_embedding import get_rope
44 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
45 +from vllm.model_executor.layers.vocab_parallel_embedding import (
46 + ParallelLMHead, VocabParallelEmbedding)
47 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
48 +from vllm.model_executor.sampling_metadata import SamplingMetadata
49 +from vllm.sequence import IntermediateTensors
50 +
51 +from .interfaces import SupportsPP
52 +from .utils import (PPMissingLayer, is_pp_missing_parameter,
53 + make_empty_intermediate_tensors_factory, make_layers,
54 + maybe_prefix)
55 +
56 +
57 +class DeepseekV3MLP(nn.Module):
58 +
59 + def __init__(
60 + self,
61 + hidden_size: int,
62 + intermediate_size: int,
63 + hidden_act: str,
64 + quant_config: Optional[QuantizationConfig] = None,
65 + reduce_results: bool = True,
66 + prefix: str = "",
67 + ) -> None:
68 + super().__init__()
69 + self.gate_up_proj = MergedColumnParallelLinear(
70 + hidden_size, [intermediate_size] * 2,
71 + bias=False,
72 + quant_config=quant_config,
73 + prefix=f"{prefix}.gate_up_proj")
74 + self.down_proj = RowParallelLinear(intermediate_size,
75 + hidden_size,
76 + bias=False,
77 + quant_config=quant_config,
78 + reduce_results=reduce_results,
79 + prefix=f"{prefix}.down_proj")
80 + if hidden_act != "silu":
81 + raise ValueError(f"Unsupported activation: {hidden_act}. "
82 + "Only silu is supported for now.")
83 + self.act_fn = SiluAndMul()
84 +
85 + def forward(self, x):
86 + gate_up, _ = self.gate_up_proj(x)
87 + x = self.act_fn(gate_up)
88 + x, _ = self.down_proj(x)
89 + return x
90 +
91 +
92 +class DeepseekV3MoE(nn.Module):
93 +
94 + def __init__(
95 + self,
96 + config: PretrainedConfig,
97 + quant_config: Optional[QuantizationConfig] = None,
98 + prefix: str = "",
99 + ):
100 + super().__init__()
101 + self.tp_size = get_tensor_model_parallel_world_size()
102 + self.routed_scaling_factor = config.routed_scaling_factor
103 + self.n_shared_experts = config.n_shared_experts
104 + self.routed_scaling_factor = config.routed_scaling_factor
105 + if self.tp_size > config.n_routed_experts:
106 + raise ValueError(
107 + f"Tensor parallel size {self.tp_size} is greater than "
108 + f"the number of experts {config.n_routed_experts}.")
109 +
110 + if config.hidden_act != "silu":
111 + raise ValueError(f"Unsupported activation: {config.hidden_act}. "
112 + "Only silu is supported for now.")
113 +
114 + self.gate = ReplicatedLinear(config.hidden_size,
115 + config.n_routed_experts,
116 + bias=False,
117 + quant_config=None,
118 + prefix=f"{prefix}.gate")
119 + if config.topk_method == "noaux_tc":
120 + self.gate.e_score_correction_bias = nn.Parameter(
121 + torch.empty(config.n_routed_experts))
122 + else:
123 + self.gate.e_score_correction_bias = None
124 +
125 + self.experts = FusedMoE(
126 + num_experts=config.n_routed_experts,
127 + top_k=config.num_experts_per_tok,
128 + hidden_size=config.hidden_size,
129 + intermediate_size=config.moe_intermediate_size,
130 + reduce_results=False,
131 + renormalize=config.norm_topk_prob,
132 + quant_config=quant_config,
133 + use_grouped_topk=True,
134 + num_expert_group=config.n_group,
135 + topk_group=config.topk_group,
136 + prefix=f"{prefix}.experts",
137 + scoring_func=config.scoring_func,
138 + e_score_correction_bias=self.gate.e_score_correction_bias)
139 +
140 + if config.n_shared_experts is not None:
141 + intermediate_size = (config.moe_intermediate_size *
142 + config.n_shared_experts)
143 + self.shared_experts = DeepseekV3MLP(
144 + hidden_size=config.hidden_size,
145 + intermediate_size=intermediate_size,
146 + hidden_act=config.hidden_act,
147 + quant_config=quant_config,
148 + reduce_results=False,
149 + )
150 +
151 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
152 + num_tokens, hidden_dim = hidden_states.shape
153 + hidden_states = hidden_states.view(-1, hidden_dim)
154 + if self.n_shared_experts is not None:
155 + shared_output = self.shared_experts(hidden_states)
156 + # router_logits: (num_tokens, n_experts)
157 + router_logits, _ = self.gate(hidden_states)
158 + final_hidden_states = self.experts(
159 + hidden_states=hidden_states,
160 + router_logits=router_logits) * self.routed_scaling_factor
161 + if shared_output is not None:
162 + final_hidden_states = final_hidden_states + shared_output
163 + if self.tp_size > 1:
164 + final_hidden_states = tensor_model_parallel_all_reduce(
165 + final_hidden_states)
166 +
167 + return final_hidden_states.view(num_tokens, hidden_dim)
168 +
169 +
170 +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
171 + import math
172 + if scale <= 1:
173 + return 1.0
174 + return 0.1 * mscale * math.log(scale) + 1.0
175 +
176 +
177 +class DeepseekV3Attention(nn.Module):
178 +
179 + def __init__(
180 + self,
181 + config: PretrainedConfig,
182 + hidden_size: int,
183 + num_heads: int,
184 + qk_nope_head_dim: int,
185 + qk_rope_head_dim: int,
186 + v_head_dim: int,
187 + q_lora_rank: int,
188 + kv_lora_rank: int,
189 + rope_theta: float = 10000,
190 + rope_scaling: Optional[Dict[str, Any]] = None,
191 + max_position_embeddings: int = 8192,
192 + cache_config: Optional[CacheConfig] = None,
193 + quant_config: Optional[QuantizationConfig] = None,
194 + prefix: str = "",
195 + ) -> None:
196 + super().__init__()
197 + self.hidden_size = hidden_size
198 + self.qk_nope_head_dim = qk_nope_head_dim
199 + self.qk_rope_head_dim = qk_rope_head_dim
200 + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
201 + self.v_head_dim = v_head_dim
202 + self.q_lora_rank = q_lora_rank
203 + self.kv_lora_rank = kv_lora_rank
204 + self.num_heads = num_heads
205 + tp_size = get_tensor_model_parallel_world_size()
206 + assert num_heads % tp_size == 0
207 + self.num_local_heads = num_heads // tp_size
208 + self.scaling = self.qk_head_dim**-0.5
209 + self.rope_theta = rope_theta
210 + self.max_position_embeddings = max_position_embeddings
211 +
212 + if self.q_lora_rank is not None:
213 + self.q_a_proj = ReplicatedLinear(self.hidden_size,
214 + self.q_lora_rank,
215 + bias=False,
216 + quant_config=quant_config,
217 + prefix=f"{prefix}.q_a_proj")
218 + self.q_a_layernorm = RMSNorm(self.q_lora_rank,
219 + eps=config.rms_norm_eps)
220 + self.q_b_proj = ColumnParallelLinear(q_lora_rank,
221 + self.num_heads *
222 + self.qk_head_dim,
223 + bias=False,
224 + quant_config=quant_config,
225 + prefix=f"{prefix}.q_b_proj")
226 + else:
227 + self.q_proj = ColumnParallelLinear(self.hidden_size,
228 + self.num_heads *
229 + self.qk_head_dim,
230 + bias=False,
231 + quant_config=quant_config,
232 + prefix=f"{prefix}.q_proj")
233 +
234 + self.kv_a_proj_with_mqa = ReplicatedLinear(
235 + self.hidden_size,
236 + self.kv_lora_rank + self.qk_rope_head_dim,
237 + bias=False,
238 + quant_config=quant_config,
239 + prefix=f"{prefix}.kv_a_proj_with_mqa")
240 + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
241 + eps=config.rms_norm_eps)
242 + self.kv_b_proj = ColumnParallelLinear(
243 + self.kv_lora_rank,
244 + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
245 + bias=False,
246 + quant_config=quant_config,
247 + prefix=f"{prefix}.kv_b_proj")
248 + # O projection.
249 + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
250 + self.hidden_size,
251 + bias=False,
252 + quant_config=quant_config,
253 + prefix=f"{prefix}.o_proj")
254 + rope_scaling["rope_type"] = 'deepseek_yarn'
255 + self.rotary_emb = get_rope(qk_rope_head_dim,
256 + rotary_dim=qk_rope_head_dim,
257 + max_position=max_position_embeddings,
258 + base=rope_theta,
259 + rope_scaling=rope_scaling,
260 + is_neox_style=False)
261 +
262 + if rope_scaling:
263 + mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
264 + scaling_factor = rope_scaling["factor"]
265 + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
266 + self.scaling = self.scaling * mscale * mscale
267 +
268 + # self.attn = Attention(self.num_heads,
269 + # self.qk_head_dim,
270 + # self.scaling,
271 + # num_kv_heads=self.num_heads)
272 +
273 + # TODO, support head_size 192
274 + self.attn = Attention(self.num_local_heads,
275 + 256,
276 + self.scaling,
277 + num_kv_heads=self.num_local_heads,
278 + cache_config=cache_config,
279 + quant_config=quant_config,
280 + prefix=f"{prefix}.attn")
281 +
282 + def forward(
283 + self,
284 + positions: torch.Tensor,
285 + hidden_states: torch.Tensor,
286 + kv_cache: torch.Tensor,
287 + attn_metadata: AttentionMetadata,
288 + ) -> torch.Tensor:
289 + if self.q_lora_rank is not None:
290 + q = self.q_a_proj(hidden_states)[0]
291 + q = self.q_a_layernorm(q)
292 + q = self.q_b_proj(q)[0].view(-1, self.num_local_heads,
293 + self.qk_head_dim)
294 + else:
295 + q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads,
296 + self.qk_head_dim)
297 + q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim],
298 + dim=-1)
299 + latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0]
300 + kv_a, _ = latent_cache.split(
301 + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
302 + latent_cache = latent_cache.unsqueeze(1)
303 + kv_a = self.kv_a_layernorm(kv_a.contiguous())
304 + kv = self.kv_b_proj(kv_a)[0]
305 + kv = kv.view(-1, self.num_local_heads,
306 + self.qk_nope_head_dim + self.v_head_dim)
307 + k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
308 + k_pe = latent_cache[:, :, self.kv_lora_rank:]
309 + q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
310 + q[..., self.qk_nope_head_dim:] = q_pe
311 + k = torch.empty_like(q)
312 + k[..., :self.qk_nope_head_dim] = k_nope
313 + k[..., self.qk_nope_head_dim:] = k_pe
314 + q = torch.nn.functional.pad(q, [0, 256 - self.qk_head_dim],
315 + value=0).view(-1,
316 + self.num_local_heads * 256)
317 + k = torch.nn.functional.pad(k, [0, 256 - self.qk_head_dim],
318 + value=0).view(-1,
319 + self.num_local_heads * 256)
320 + v = torch.nn.functional.pad(v, [0, 256 - self.v_head_dim],
321 + value=0).view(-1,
322 + self.num_local_heads * 256)
323 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
324 + attn_output = attn_output.view(
325 + -1, self.num_local_heads, 256)[..., :self.v_head_dim].reshape(
326 + -1, self.num_local_heads * self.v_head_dim)
327 + output, _ = self.o_proj(attn_output)
328 + return output
329 +
330 +
331 +class DeepseekV3DecoderLayer(nn.Module):
332 +
333 + def __init__(
334 + self,
335 + config: PretrainedConfig,
336 + prefix: str,
337 + cache_config: Optional[CacheConfig] = None,
338 + quant_config: Optional[QuantizationConfig] = None,
339 + ) -> None:
340 + super().__init__()
341 + self.hidden_size = config.hidden_size
342 + rope_theta = getattr(config, "rope_theta", 10000)
343 + rope_scaling = getattr(config, "rope_scaling", None)
344 + max_position_embeddings = getattr(config, "max_position_embeddings",
345 + 8192)
346 + # DecoderLayers are created with `make_layers` which passes the prefix
347 + # with the layer's index.
348 + layer_idx = int(prefix.split(sep='.')[-1])
349 + self.self_attn = DeepseekV3Attention(
350 + config=config,
351 + hidden_size=self.hidden_size,
352 + num_heads=config.num_attention_heads,
353 + qk_nope_head_dim=config.qk_nope_head_dim,
354 + qk_rope_head_dim=config.qk_rope_head_dim,
355 + v_head_dim=config.v_head_dim,
356 + q_lora_rank=config.q_lora_rank
357 + if hasattr(config, "q_lora_rank") else None,
358 + kv_lora_rank=config.kv_lora_rank,
359 + rope_theta=rope_theta,
360 + rope_scaling=rope_scaling,
361 + max_position_embeddings=max_position_embeddings,
362 + cache_config=cache_config,
363 + quant_config=quant_config,
364 + prefix=f"{prefix}.self_attn",
365 + )
366 + if (config.n_routed_experts is not None
367 + and layer_idx >= config.first_k_dense_replace
368 + and layer_idx % config.moe_layer_freq == 0):
369 + self.mlp = DeepseekV3MoE(
370 + config=config,
371 + quant_config=quant_config,
372 + prefix=f"{prefix}.mlp",
373 + )
374 + else:
375 + self.mlp = DeepseekV3MLP(
376 + hidden_size=config.hidden_size,
377 + intermediate_size=config.intermediate_size,
378 + hidden_act=config.hidden_act,
379 + quant_config=quant_config,
380 + prefix=f"{prefix}.mlp",
381 + )
382 + self.input_layernorm = RMSNorm(config.hidden_size,
383 + eps=config.rms_norm_eps)
384 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
385 + eps=config.rms_norm_eps)
386 +
387 + def forward(
388 + self,
389 + positions: torch.Tensor,
390 + hidden_states: torch.Tensor,
391 + kv_cache: torch.Tensor,
392 + attn_metadata: AttentionMetadata,
393 + residual: Optional[torch.Tensor],
394 + ) -> torch.Tensor:
395 + # Self Attention
396 + if residual is None:
397 + residual = hidden_states
398 + hidden_states = self.input_layernorm(hidden_states)
399 + else:
400 + hidden_states, residual = self.input_layernorm(
401 + hidden_states, residual)
402 + hidden_states = self.self_attn(
403 + positions=positions,
404 + hidden_states=hidden_states,
405 + kv_cache=kv_cache,
406 + attn_metadata=attn_metadata,
407 + )
408 +
409 + # Fully Connected
410 + hidden_states, residual = self.post_attention_layernorm(
411 + hidden_states, residual)
412 + hidden_states = self.mlp(hidden_states)
413 + return hidden_states, residual
414 +
415 +
416 +# TODO(simon): check whether we support torch compile for Deepseek V3
417 +# @support_torch_compile
418 +class DeepseekV3Model(nn.Module):
419 +
420 + fall_back_to_pt_during_load = False
421 +
422 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
423 + super().__init__()
424 +
425 + config = vllm_config.model_config.hf_config
426 + cache_config = vllm_config.cache_config
427 + quant_config = vllm_config.quant_config
428 +
429 + self.padding_idx = config.pad_token_id
430 + self.vocab_size = config.vocab_size
431 +
432 + if get_pp_group().is_first_rank:
433 + self.embed_tokens = VocabParallelEmbedding(
434 + config.vocab_size,
435 + config.hidden_size,
436 + )
437 + else:
438 + self.embed_tokens = PPMissingLayer()
439 +
440 + self.start_layer, self.end_layer, self.layers = make_layers(
441 + config.num_hidden_layers,
442 + lambda prefix: DeepseekV3DecoderLayer(
443 + config,
444 + prefix,
445 + cache_config=cache_config,
446 + quant_config=quant_config,
447 + ),
448 + prefix=f"{prefix}.layers")
449 +
450 + if get_pp_group().is_last_rank:
451 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
452 + else:
453 + self.norm = PPMissingLayer()
454 + self.make_empty_intermediate_tensors = (
455 + make_empty_intermediate_tensors_factory(
456 + ["hidden_states", "residual"], config.hidden_size))
457 +
458 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
459 + return self.embed_tokens(input_ids)
460 +
461 + def forward(
462 + self,
463 + input_ids: torch.Tensor,
464 + positions: torch.Tensor,
465 + kv_caches: List[torch.Tensor],
466 + attn_metadata: AttentionMetadata,
467 + intermediate_tensors: Optional[IntermediateTensors],
468 + inputs_embeds: Optional[torch.Tensor] = None,
469 + ) -> Union[torch.Tensor, IntermediateTensors]:
470 + if get_pp_group().is_first_rank:
471 + if inputs_embeds is not None:
472 + hidden_states = inputs_embeds
473 + else:
474 + hidden_states = self.get_input_embeddings(input_ids)
475 + residual = None
476 + else:
477 + assert intermediate_tensors is not None
478 + hidden_states = intermediate_tensors["hidden_states"]
479 + residual = intermediate_tensors["residual"]
480 +
481 + for i in range(self.start_layer, self.end_layer):
482 + layer = self.layers[i]
483 + hidden_states, residual = layer(positions, hidden_states,
484 + kv_caches[i - self.start_layer],
485 + attn_metadata, residual)
486 +
487 + if not get_pp_group().is_last_rank:
488 + return IntermediateTensors({
489 + "hidden_states": hidden_states,
490 + "residual": residual
491 + })
492 +
493 + hidden_states, _ = self.norm(hidden_states, residual)
494 + return hidden_states
495 +
496 +
497 +class DeepseekV3ForCausalLM(nn.Module, SupportsPP):
498 +
499 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
500 + super().__init__()
501 + config = vllm_config.model_config.hf_config
502 + quant_config = vllm_config.quant_config
503 + self.config = config
504 + self.quant_config = quant_config
505 + self.model = DeepseekV3Model(vllm_config=vllm_config,
506 + prefix=maybe_prefix(prefix, "model"))
507 + self.lm_head = ParallelLMHead(config.vocab_size,
508 + config.hidden_size,
509 + quant_config=quant_config)
510 + self.logits_processor = LogitsProcessor(config.vocab_size)
511 + self.sampler = get_sampler()
512 + self.make_empty_intermediate_tensors = (
513 + self.model.make_empty_intermediate_tensors)
514 +
515 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
516 + return self.model.get_input_embeddings(input_ids)
517 +
518 + def forward(
519 + self,
520 + input_ids: torch.Tensor,
521 + positions: torch.Tensor,
522 + kv_caches: List[torch.Tensor],
523 + attn_metadata: AttentionMetadata,
524 + intermediate_tensors: Optional[IntermediateTensors] = None,
525 + inputs_embeds: Optional[torch.Tensor] = None,
526 + ) -> Union[torch.Tensor, IntermediateTensors]:
527 + hidden_states = self.model(input_ids, positions, kv_caches,
528 + attn_metadata, intermediate_tensors,
529 + inputs_embeds)
530 + return hidden_states
531 +
532 + def compute_logits(
533 + self,
534 + hidden_states: torch.Tensor,
535 + sampling_metadata: SamplingMetadata,
536 + ) -> Optional[torch.Tensor]:
537 + logits = self.logits_processor(self.lm_head, hidden_states,
538 + sampling_metadata)
539 + return logits
540 +
541 + def sample(
542 + self,
543 + logits: Optional[torch.Tensor],
544 + sampling_metadata: SamplingMetadata,
545 + ) -> Optional[SamplerOutput]:
546 + next_tokens = self.sampler(logits, sampling_metadata)
547 + return next_tokens
548 +
549 + def make_empty_intermediate_tensors(
550 + self, batch_size: int, dtype: torch.dtype,
551 + device: torch.device) -> IntermediateTensors:
552 + return IntermediateTensors({
553 + "hidden_states":
554 + torch.zeros((batch_size, self.config.hidden_size),
555 + dtype=dtype,
556 + device=device),
557 + "residual":
558 + torch.zeros((batch_size, self.config.hidden_size),
559 + dtype=dtype,
560 + device=device),
561 + })
562 +
563 + def load_weights(self, weights: Iterable[Tuple[str,
564 + torch.Tensor]]) -> Set[str]:
565 + stacked_params_mapping = [
566 + # (param_name, shard_name, shard_id)
567 + ("gate_up_proj", "gate_proj", 0),
568 + ("gate_up_proj", "up_proj", 1),
569 + ]
570 +
571 + # Params for weights, fp8 weight scales, fp8 activation scales
572 + # (param_name, weight_name, expert_id, shard_id)
573 + expert_params_mapping = FusedMoE.make_expert_params_mapping(
574 + ckpt_gate_proj_name="gate_proj",
575 + ckpt_down_proj_name="down_proj",
576 + ckpt_up_proj_name="up_proj",
577 + num_experts=self.config.n_routed_experts)
578 +
579 + params_dict = dict(self.named_parameters())
580 + loaded_params: Set[str] = set()
581 + for name, loaded_weight in weights:
582 + if "rotary_emb.inv_freq" in name:
583 + continue
584 +
585 + # TODO(simon): support nextn predict layers
586 + if self.config.num_nextn_predict_layers > 0:
587 + assert self.config.num_nextn_predict_layers == 1
588 + layer_idx = self.config.num_hidden_layers
589 + if name.startswith(f"model.layers.{layer_idx}"):
590 + continue
591 +
592 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
593 + # Skip non-stacked layers and experts (experts handled below).
594 + if weight_name not in name:
595 + continue
596 + # We have mlp.experts[0].gate_proj in the checkpoint.
597 + # Since we handle the experts below in expert_params_mapping,
598 + # we need to skip here BEFORE we update the name, otherwise
599 + # name will be updated to mlp.experts[0].gate_up_proj, which
600 + # will then be updated below in expert_params_mapping
601 + # for mlp.experts[0].gate_gate_up_proj, which breaks load.
602 + if (("mlp.experts." in name) and name not in params_dict):
603 + continue
604 + name = name.replace(weight_name, param_name)
605 + # Skip loading extra bias for GPTQ models.
606 + if name.endswith(".bias") and name not in params_dict:
607 + continue
608 +
609 + if is_pp_missing_parameter(name, self):
610 + continue
611 +
612 + param = params_dict[name]
613 + weight_loader = param.weight_loader
614 + weight_loader(param, loaded_weight, shard_id)
615 + break
616 + else:
617 + for mapping in expert_params_mapping:
618 + param_name, weight_name, expert_id, shard_id = mapping
619 + if weight_name not in name:
620 + continue
621 + name = name.replace(weight_name, param_name)
622 +
623 + if is_pp_missing_parameter(name, self):
624 + continue
625 +
626 + param = params_dict[name]
627 + weight_loader = param.weight_loader
628 + weight_loader(param,
629 + loaded_weight,
630 + name,
631 + shard_id=shard_id,
632 + expert_id=expert_id)
633 + break
634 + else:
635 + # Skip loading extra bias for GPTQ models.
636 + if name.endswith(".bias") and name not in params_dict:
637 + continue
638 +
639 + if is_pp_missing_parameter(name, self):
640 + continue
641 +
642 + if name not in params_dict:
643 + for key in params_dict:
644 + print(key)
645 + param = params_dict[name]
646 + weight_loader = getattr(param, "weight_loader",
647 + default_weight_loader)
648 + weight_loader(param, loaded_weight)
649 + loaded_params.add(name)
650 + return loaded_params
vllm/model_executor/models/registry.py
+1
@@ -45,6 +45,7 @@ _TEXT_GENERATION_MODELS = {
45 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
46 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
47 "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
48 + "DeepseekV3ForCausalLM": ("deepseek_v3", "DeepseekV3ForCausalLM"),
49 "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
50 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
51 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),