A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[ModelOpt] Remove NVFP4 MoE K%16==0 constraint (#26891)
Signed-off-by: XiaobingSuper <xiaobingzhangupc@gmail.com>
XiaobingZhang committed
Oct 16, 2025 at 01:06 UTC
d7963752589f53b061da362b80663fab0aeee081
1 file changed
-12
vllm/model_executor/layers/quantization/modelopt.py
-12
@@ -1542,23 +1542,11 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
1542
del layer.w2_input_scale_quant
1543
else:
1544
# Non-TRT-LLM processing (Cutlass or non-flashinfer)
1545
- assert layer.w13_weight_scale.shape[2] % 16 == 0, (
1546
- "Expected weight_scale.dim(1) to be divisible by 16"
1547
- )
1548
- assert layer.w13_weight_scale.dtype == torch.float8_e4m3fn, (
1549
- "Weight Blockscale must be represented as FP8-E4M3"
1550
- )
1545
w13_blockscale_swizzled = swizzle_blockscale(layer.w13_weight_scale)
1546
layer.w13_weight_scale = Parameter(
1547
w13_blockscale_swizzled, requires_grad=False
1548
)
1549
1556
- assert layer.w2_weight_scale.shape[2] % 16 == 0, (
1557
- "Expected weight_scale.dim(1) to be divisible by 16"
1558
- )
1559
- assert layer.w2_weight_scale.dtype == torch.float8_e4m3fn, (
1560
- "Weight Blockscale must be represented as FP8-E4M3"
1561
- )
1550
w2_blockscale_swizzled = swizzle_blockscale(layer.w2_weight_scale)
1551
layer.w2_weight_scale = Parameter(
1552
w2_blockscale_swizzled, requires_grad=False