A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix deepseekv3 gate bias error (#12002)
Signed-off-by: mgoin <michael@neuralmagic.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Steve Luo committed
Jan 14, 2025 at 04:43 UTC
f35ec461fc655a50abc5146fa27a79fdf42f55a1
1 file changed
+14
-5
vllm/model_executor/layers/fused_moe/fused_moe.py
+14
-5
@@ -497,7 +497,10 @@ def grouped_topk(hidden_states: torch.Tensor,
497
raise ValueError(f"Unsupported scoring function: {scoring_func}")
498
499
if e_score_correction_bias is not None:
500
- scores.add_(e_score_correction_bias.unsqueeze(0))
500
+ # Store original scores before applying correction bias. We use biased
501
+ # scores for expert selection but original scores for routing weights
502
+ original_scores = scores
503
+ scores = scores + e_score_correction_bias.unsqueeze(0)
504
505
num_token = scores.shape[0]
506
group_scores = scores.view(num_token, num_expert_group,
@@ -510,10 +513,16 @@ def grouped_topk(hidden_states: torch.Tensor,
513
num_token, num_expert_group,
514
scores.shape[-1] // num_expert_group).reshape(num_token, -1) # [n, e]
515
tmp_scores = scores.masked_fill(~score_mask.bool(), 0.0) # [n, e]
513
- topk_weights, topk_ids = torch.topk(tmp_scores,
514
- k=topk,
515
- dim=-1,
516
- sorted=False)
516
+
517
+ if e_score_correction_bias is not None:
518
+ topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
519
+ # Use original unbiased scores for the routing weights
520
+ topk_weights = original_scores.gather(1, topk_ids)
521
+ else:
522
+ topk_weights, topk_ids = torch.topk(tmp_scores,
523
+ k=topk,
524
+ dim=-1,
525
+ sorted=False)
526
527
if renormalize:
528
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)