A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 01:15 UTC
[Bugfix] Fix cutlass fp8 kernel on hopper for Qwen3.5 (#34914)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Feb 19, 2026 at 13:34 UTC
4fb8beefaa8b2c4bd2cd3b336b01ff006dc98bdc
1 file changed
+11
vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
+11
@@ -455,4 +455,15 @@ def prepare_fp8_moe_layer_for_fi(
455
w2_input_scale=w2_input_scale,
456
)
457
458
+ # Clamp block scales to avoid NaN from the FlashInfer CUTLASS kernel.
459
+ # Some FP8 models have near-zero block scales (~1e-23) for dead/unused
460
+ # experts. The CUTLASS kernel doesn't handle these correctly on Hopper
461
+ # (SM 9.0), producing NaN instead of near-zero output. Clamping to a
462
+ # small minimum prevents this without affecting model accuracy since
463
+ # these experts' effective weights are already zero.
464
+ if block_quant:
465
+ _FI_CUTLASS_MIN_BLOCK_SCALE = 1e-10
466
+ w13_scale.clamp_(min=_FI_CUTLASS_MIN_BLOCK_SCALE)
467
+ w2_scale.clamp_(min=_FI_CUTLASS_MIN_BLOCK_SCALE)
468
+
469
return w13, w2, w13_scale