A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 09:45 UTC
[Bug] Fix routing bias dtype for trtllm per-block fp8 moe (#38989)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Wei Zhao committed
Apr 8, 2026 at 22:42 UTC
92fbec391b0ae9dff61ddd360ba486f8de5bc53d
1 file changed
+5
vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py
+5
@@ -358,6 +358,11 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
358
if self.routing_method_type == RoutingMethodType.DeepSeekV3:
359
router_logits = router_logits.to(torch.float32)
360
361
+ # Currently FI requires bfloat16 routing bias.
362
+ # https://github.com/flashinfer-ai/flashinfer/issues/2909
363
+ if e_score_correction_bias is not None:
364
+ e_score_correction_bias = e_score_correction_bias.to(torch.bfloat16)
365
+
366
is_mxfp8 = self.quant_config.block_shape == [1, 32]
367
if is_mxfp8:
368
fp8_quant_type = Fp8QuantizationType.MxFp8