| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for Humming MXFP4-weight GEMM paired with a block-FP8 activation. |
| 4 | |
| 5 | The Humming kernel already implements MXFP4 weight (group-32 e8m0) with a |
| 6 | grouped FP8 activation via WGMMA software dequant -- see the grouped-fp8 |
| 7 | `test_mxfp4.py` cases in the humming repo, which run on SM90 (H200). Enabling |
| 8 | it in vLLM comes down to two things, both exercised here: |
| 9 | |
| 10 | 1. A grouped FP8 activation must classify to a float32-scaled key |
| 11 | (kFp8Dynamic128Sym), not a uint8 microscale key. MX microscale |
| 12 | activations (group size 32) keep their e8m0 (uint8) scale. |
| 13 | 2. (kMxfp4Static, kFp8Dynamic128Sym) must be an allowed MoE |
| 14 | (weight, activation) pair. |
| 15 | """ |
| 16 | |
| 17 | import pytest |
| 18 | import torch |
| 19 | |
| 20 | from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( |
| 21 | HummingExpertsBase, |
| 22 | ) |
| 23 | from vllm.model_executor.layers.quantization.utils.quant_utils import ( |
| 24 | kFp8Dynamic128Sym, |
| 25 | kFp8DynamicTokenSym, |
| 26 | kMxfp4Static, |
| 27 | kMxfp8Dynamic, |
| 28 | ) |
| 29 | from vllm.utils.import_utils import has_humming |
| 30 | |
| 31 | # make_humming_moe_quant_config stores per-sublayer humming LayerConfigs but the |
| 32 | # config-plumbing asserts below never read them, so a placeholder satisfies the |
| 33 | # required arg without pulling in the humming package. |
| 34 | _PLACEHOLDER_HUMMING_CONFIGS = {"w13": None, "w2": None} |
| 35 | |
| 36 | |
| 37 | def test_mxfp4_weight_with_block_fp8_activation_is_supported(): |
| 38 | """MXFP4 weight + block-FP8 (group-128) activation is an allowed pair.""" |
| 39 | assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8Dynamic128Sym) |
| 40 | |
| 41 | |
| 42 | def test_mxfp4_weight_with_per_token_fp8_activation_still_supported(): |
| 43 | """The pre-existing per-token FP8 pairing must keep working.""" |
| 44 | assert HummingExpertsBase._supports_quant_scheme(kMxfp4Static, kFp8DynamicTokenSym) |
| 45 | |
| 46 | |
| 47 | @pytest.mark.skipif(not has_humming(), reason="humming is not installed") |
| 48 | @pytest.mark.parametrize( |
| 49 | "group_size,expected_key", |
| 50 | [ |
| 51 | # per-token FP8 -> float32 per-token scale |
| 52 | (0, kFp8DynamicTokenSym), |
| 53 | # block FP8 -> float32 group-128 scale (the newly-fixed case) |
| 54 | (128, kFp8Dynamic128Sym), |
| 55 | # MXFP8 microscale -> e8m0 (uint8) group-32 scale (must be preserved) |
| 56 | (32, kMxfp8Dynamic), |
| 57 | ], |
| 58 | ) |
| 59 | def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key): |
| 60 | from vllm.model_executor.layers.quantization.utils.humming import ( |
| 61 | input_schema_to_quant_key, |
| 62 | ) |
| 63 | from vllm.utils.humming import HummingInputSchema |
| 64 | from vllm.utils.humming import dtypes as humming_dtypes |
| 65 | |
| 66 | schema = HummingInputSchema( |
| 67 | a_dtype=humming_dtypes.float8e4m3, |
| 68 | input_scale_group_size=group_size, |
| 69 | ) |
| 70 | assert input_schema_to_quant_key(schema, param_dtype=torch.bfloat16) == expected_key |
| 71 | |
| 72 | |
| 73 | @pytest.mark.skipif(not has_humming(), reason="humming is not installed") |
| 74 | def test_humming_bf16_input_schema_is_unquantized(): |
| 75 | from vllm.model_executor.layers.quantization.utils.humming import ( |
| 76 | input_schema_to_quant_key, |
| 77 | ) |
| 78 | from vllm.utils.humming import HummingInputSchema |
| 79 | |
| 80 | # No a_dtype -> unquantized (bf16/fp16) inputs -> None. |
| 81 | assert ( |
| 82 | input_schema_to_quant_key(HummingInputSchema(), param_dtype=torch.bfloat16) |
| 83 | is None |
| 84 | ) |
| 85 | |
| 86 | |
| 87 | def test_block_fp8_activation_quant_config_is_block_quantized(): |
| 88 | """A block-FP8 activation group shape yields a block-quantized MoE config |
| 89 | (FP8 dtype, [1, 128] block, not per-act-token). This is what makes the |
| 90 | DeepEP prepare/finalize step quantize activations to block FP8 *before* the |
| 91 | all-to-all dispatch instead of deferring to Humming -- and what makes |
| 92 | HummingExpertsBase.expects_unquantized_inputs return False.""" |
| 93 | from vllm.model_executor.layers.quantization.utils.humming import ( |
| 94 | make_humming_moe_quant_config, |
| 95 | ) |
| 96 | from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape |
| 97 | from vllm.platforms import current_platform |
| 98 | |
| 99 | qc = make_humming_moe_quant_config( |
| 100 | quant_dtype=current_platform.fp8_dtype(), |
| 101 | weight_dtype="float4e2m1", |
| 102 | activation_group_shape=GroupShape(row=1, col=128), |
| 103 | humming_configs=_PLACEHOLDER_HUMMING_CONFIGS, |
| 104 | ) |
| 105 | assert qc.is_block_quantized |
| 106 | assert qc.block_shape == [1, 128] |
| 107 | assert not qc.per_act_token_quant |
| 108 | assert qc.quant_dtype == current_platform.fp8_dtype() |
| 109 | |
| 110 | |
| 111 | def test_default_activation_quant_config_defers_to_humming(): |
| 112 | """Without a block activation shape the config stays per-token (the deferred |
| 113 | path): Humming quantizes internally, preserving pre-existing behavior for |
| 114 | every non-block-FP8 scheme.""" |
| 115 | from vllm.model_executor.layers.quantization.utils.humming import ( |
| 116 | make_humming_moe_quant_config, |
| 117 | ) |
| 118 | from vllm.platforms import current_platform |
| 119 | |
| 120 | qc = make_humming_moe_quant_config( |
| 121 | quant_dtype=current_platform.fp8_dtype(), |
| 122 | weight_dtype="float4e2m1", |
| 123 | humming_configs=_PLACEHOLDER_HUMMING_CONFIGS, |
| 124 | ) |
| 125 | assert not qc.is_block_quantized |
| 126 | assert qc.per_act_token_quant |