A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 20:15 UTC
Fix fallback to default tactic (flashinfer autotuner) with trtllm_fp4_block_scale_moe (#35088)
Signed-off-by: Daniel Serebrenik <daserebrenik@nvidia.com>
danisereb committed
Feb 24, 2026 at 17:25 UTC
a0c70816956298f7dd1d0cf47cfa1a169a413692
1 file changed
+2
-2
vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py
+2
-2
@@ -348,7 +348,7 @@ def flashinfer_trtllm_fp4_moe(
348
hidden_states=hidden_states_fp4,
349
hidden_states_scale=hidden_states_scale_linear_fp4.view(
350
torch.float8_e4m3fn
351
- ).flatten(),
351
+ ).reshape(*hidden_states_fp4.shape[:-1], -1),
352
gemm1_weights=layer.w13_weight.data,
353
gemm1_weights_scale=layer.w13_weight_scale.data.view(torch.float8_e4m3fn),
354
gemm1_bias=None,
@@ -432,7 +432,7 @@ def flashinfer_trtllm_fp4_routed_moe(
432
hidden_states=hidden_states_fp4,
433
hidden_states_scale=hidden_states_scale_linear_fp4.view(
434
torch.float8_e4m3fn
435
- ).flatten(),
435
+ ).reshape(*hidden_states_fp4.shape[:-1], -1),
436
gemm1_weights=layer.w13_weight.data,
437
gemm1_weights_scale=layer.w13_weight_scale.data.view(torch.float8_e4m3fn),
438
gemm1_bias=None,