A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix routing_bias dtype (#25711)
Signed-off-by: Shu Wang. <shuw@nvidia.com>
Shu Wang committed
Sep 25, 2025 at 18:35 UTC
081b5594a2b1a37ea793659bb6767c497beef45d
1 file changed
+4
-1
vllm/model_executor/layers/quantization/modelopt.py
+4
-1
@@ -1454,10 +1454,13 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
1454
routing_method_type = flashinfer.RoutingMethodType.DeepSeekV3
1455
if use_llama4_routing:
1456
routing_method_type = flashinfer.RoutingMethodType.Llama4
1457
+ routing_bias = e_score_correction_bias
1458
+ if routing_bias is not None:
1459
+ routing_bias = routing_bias.to(torch.bfloat16)
1460
out = flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
1461
routing_logits=router_logits
1462
if use_llama4_routing else router_logits.to(torch.float32),
1460
- routing_bias=e_score_correction_bias,
1463
+ routing_bias=routing_bias,
1464
hidden_states=hidden_states_fp4,
1465
hidden_states_scale=hidden_states_scale_linear_fp4.view(
1466
torch.float8_e4m3fn).flatten(),