@tduncan / Nvidia-Optimization-Stack-VLLM / commits / fcde8e1460

[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: OpenAI Codex <noreply@openai.com>

Jinzhen Lin committed Aug 8, 2026 at 01:03 UTC fcde8e1460589849c8142d617420ccdc406d5bc9

37 files changed +448 -341

requirements/cuda.txt

+1 -1

@@ -32,4 +32,4 @@ quack-kernels>=0.6.1 # Required for CUTLASS DSL 4.6 by MSA

32 tokenspeed-mla==0.1.8; platform_system == "Linux"

34 # Humming kernels for quantization gemm

35 -humming-kernels[cu13]==0.1.10

35 +humming-kernels[cu13]==0.1.12

tests/evals/gsm8k/configs/humming/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml new

@@ -0,0 +1,9 @@

1 +model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"

2 +accuracy_threshold: 0.29

3 +num_questions: 1319

4 +num_fewshot: 5

5 +server_args: >-

6 + --enforce-eager

7 + --max-model-len 8192

8 + --moe-backend humming

9 + --linear-backend humming

tests/evals/gsm8k/configs/humming/Qwen3-4B-mixed-quant-RTN-humming.yaml

+1 -1

@@ -1,5 +1,5 @@

1 model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"

2 -accuracy_threshold: 0.85

2 +accuracy_threshold: 0.78

3 num_questions: 1319

4 num_fewshot: 5

5 server_args: >-

tests/evals/gsm8k/configs/humming/config-act-int8.txt

@@ -1,3 +1,4 @@

1 +gpt-oss-20b-humming.yaml

2 Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml

3 Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml

4 Qwen3-30B-A3B-AWQ-humming.yaml

tests/evals/gsm8k/configs/humming/config.txt

@@ -11,3 +11,4 @@ Qwen3-30B-A3B-NVFP4-humming.yaml

11 Qwen3-30B-A3B-AWQ-humming.yaml

12 Qwen3.5-35B-A3B-FP8-humming.yaml

13 Qwen3.5-35B-A3B-experts-int8-humming.yaml

14 +NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml

tests/kernels/moe/test_moe.py

+9 -13

@@ -1265,11 +1265,10 @@ def _make_humming_indexed_experts(activation: MoEActivation):

1265 layer.intermediate_size_per_partition = intermediate_size

1266 quant_config = humming_utils.get_humming_moe_quant_config(layer)

1267 experts = HummingIndexedExperts(

1268 - layer,

1269 - moe_config,

1270 - quant_config,

1268 + moe_config=moe_config,

1269 + quant_config=quant_config,

1270 )

1272 - return experts

1271 + return experts, layer

1272

1273

1274 @pytest.mark.parametrize("activation", list(MoEActivation))

@@ -1328,8 +1327,7 @@ def test_humming_delegates_to_instance_activation():

1327 def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):

1328 from vllm.utils import humming

1329

1331 - experts = _make_humming_indexed_experts(activation)

1332 - layer = experts.layer

1330 + experts, layer = _make_humming_indexed_experts(activation)

1331 moe_config = experts.moe_config

1332 top_k = moe_config.experts_per_token

1333 num_experts = moe_config.num_experts

@@ -1337,7 +1335,7 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):

1335 intermediate_size = moe_config.intermediate_size

1336 gate_up_size = intermediate_size * 2 if activation.is_gated else intermediate_size

1337

1340 - w13_meta, w2_meta = (layer.humming_metas[name] for name in ("w13", "w2"))

1338 + w13_meta, w2_meta = (experts.humming_configs[name] for name in ("w13", "w2"))

1339 for meta in (w13_meta, w2_meta):

1340 assert meta.a_dtype == humming.dtypes.bfloat16

1341 assert meta.b_dtype == humming.dtypes.float4e2m1

@@ -1364,7 +1362,7 @@ def test_humming_indexed_writes_supplied_output_buffer():

1362 from vllm.forward_context import set_forward_context

1363

1364 activation = MoEActivation.SILU

1367 - experts = _make_humming_indexed_experts(activation)

1365 + experts, layer = _make_humming_indexed_experts(activation)

1366 moe_config = experts.moe_config

1367 num_tokens = 1

1368 top_k = moe_config.experts_per_token

@@ -1382,7 +1380,7 @@ def test_humming_indexed_writes_supplied_output_buffer():

1380 )

1381

1382 device = torch.device("cuda")

1385 - dtype = experts.layer.params_dtype

1383 + dtype = layer.params_dtype

1384 workspace13 = torch.empty(workspace13_shape, dtype=dtype, device=device)

1385 workspace2 = torch.empty(workspace2_shape, dtype=dtype, device=device)

1386 hidden_states = torch.ones((num_tokens, hidden_size), dtype=dtype, device=device)

@@ -1394,14 +1392,12 @@ def test_humming_indexed_writes_supplied_output_buffer():

1392 device=device,

1393 )

1394 topk_ids = torch.arange(top_k, dtype=torch.int32, device=device).unsqueeze(0)

1397 - unused = torch.empty((num_experts, 0), device=device)

1398 -

1395 with set_forward_context(None, vllm_config, num_tokens=num_tokens):

1396 experts.apply(

1397 output=output,

1398 hidden_states=hidden_states,

1403 - w1=unused,

1404 - w2=unused,

1399 + w1=layer.w13_weight,

1400 + w2=layer.w2_weight,

1401 topk_weights=topk_weights,

1402 topk_ids=topk_ids,

1403 activation=activation,

tests/kernels/moe/test_ocp_mx_moe.py

-1

@@ -1489,7 +1489,6 @@ def test_rocm_mxfp4_moe_oracle(

1489 mxfp4_backend=backend,

1490 experts_cls=experts_cls,

1491 routing_tables=None,

1492 - layer=None,

1492 )

1493

1494 # Create inputs

tests/quantization/test_moe_wna16.py

+3 -2

@@ -161,7 +161,6 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):

161

162 assert method.moe_kernel is kernel

163 assert captured["backend"] == WNA16MoEBackend.HUMMING

164 - assert captured["layer"] is layer

164

165

166 def test_moe_wna16_humming_adapter_repacks_uint8_tensors():

@@ -198,7 +197,9 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):

197 monkeypatch.setattr(

198 humming_utils,

199 "get_humming_moe_quant_config",

201 - lambda actual_layer: quant_config if actual_layer is layer else None,

200 + lambda actual_layer, *args, **kwargs: (

201 + quant_config if actual_layer is layer else None

202 + ),

203 )

204

205 assert method.get_fused_moe_quant_config(layer) is quant_config

vllm/model_executor/kernels/linear/mixed_precision/humming.py

+14 -9

@@ -28,7 +28,8 @@ class HummingLinearKernel(MPLinearKernel):

28 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:

29 from vllm.model_executor.layers.quantization.utils.humming_utils import (

30 convert_linear_layer_to_humming_standard,

31 - prepare_humming_layer,

31 + get_humming_linear_compute_config,

32 + prepare_humming_linear_layer_config,

33 )

35 name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}

@@ -46,9 +47,11 @@ class HummingLinearKernel(MPLinearKernel):

48 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)

49 input_quant_config = getattr(layer, "_humming_input_quant_config", None)

49 - prepare_humming_layer(

50 + self.layer_config = prepare_humming_linear_layer_config(

51 layer, quant_config, input_quant_config=input_quant_config

52 )

53 + self.compute_config = get_humming_linear_compute_config()

54 + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)

56 def apply_weights(

57 self,

@@ -56,12 +59,14 @@ class HummingLinearKernel(MPLinearKernel):

59 x: torch.Tensor,

60 bias: torch.Tensor | None = None,

61 ) -> torch.Tensor:

59 - from vllm.utils.humming import HummingMethod

62 + from vllm.model_executor.layers.quantization.utils.humming_utils import (

63 + apply_humming_linear,

64 + )

61 - flatten_inputs = x.view(-1, x.size(-1))

62 - output = HummingMethod.forward_layer(

63 - layer=layer,

64 - inputs=flatten_inputs,

65 - compute_config=layer.compute_config,

66 + return apply_humming_linear(

67 + layer,

68 + x,

69 + layer_config=self.layer_config,

70 + compute_config=self.compute_config,

71 + locks=self.locks,

72 )

67 - return output.view(*x.shape[:-1], output.size(-1))

vllm/model_executor/kernels/linear/mxfp4/humming.py

+16 -10

@@ -5,11 +5,14 @@ import torch

6 from vllm.logger import init_logger

7 from vllm.model_executor.layers.quantization.utils.humming_utils import (

8 + apply_humming_linear,

9 convert_linear_layer_to_humming_standard,

9 - prepare_humming_layer,

10 + get_humming_linear_compute_config,

11 + prepare_humming_linear_layer_config,

12 )

13 from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic

14 from vllm.platforms import current_platform

15 +from vllm.utils.import_utils import has_humming

17 from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig

@@ -26,6 +29,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):

29 if not current_platform.is_cuda():

30 return False, "Humming only supported on CUDA"

32 + if not has_humming():

33 + return False, "Humming is not installed"

34 +

35 if not current_platform.has_device_capability(75):

36 return False, "Humming only supported on SM75+"

@@ -56,7 +62,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):

62 }

64 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)

59 - prepare_humming_layer(layer, quant_config)

65 + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)

66 + self.compute_config = get_humming_linear_compute_config()

67 + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)

69 def apply_weights(

70 self,

@@ -64,12 +72,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):

72 x: torch.Tensor,

73 bias: torch.Tensor | None = None,

74 ) -> torch.Tensor:

67 - from vllm.utils.humming import HummingMethod

68 -

69 - flatten_inputs = x.view(-1, x.size(-1))

70 - output = HummingMethod.forward_layer(

71 - layer=layer,

72 - inputs=flatten_inputs,

73 - compute_config=layer.compute_config,

75 + return apply_humming_linear(

76 + layer,

77 + x,

78 + layer_config=self.layer_config,

79 + compute_config=self.compute_config,

80 + locks=self.locks,

81 )

75 - return output.view(*x.shape[:-1], output.size(-1))

vllm/model_executor/kernels/linear/mxfp8/humming.py

+16 -10

@@ -4,10 +4,13 @@

4 import torch

6 from vllm.model_executor.layers.quantization.utils.humming_utils import (

7 + apply_humming_linear,

8 convert_linear_layer_to_humming_standard,

8 - prepare_humming_layer,

9 + get_humming_linear_compute_config,

10 + prepare_humming_linear_layer_config,

11 )

12 from vllm.platforms import current_platform

13 +from vllm.utils.import_utils import has_humming

15 from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig

@@ -22,6 +25,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):

25 if not current_platform.is_cuda():

26 return False, "Humming only supported on CUDA"

28 + if not has_humming():

29 + return False, "Humming is not installed"

30 +

31 if not current_platform.has_device_capability(75):

32 return False, "Humming only supported on SM75+"

@@ -44,7 +50,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):

50 }

52 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)

47 - prepare_humming_layer(layer, quant_config)

53 + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)

54 + self.compute_config = get_humming_linear_compute_config()

55 + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)

57 def apply_weights(

58 self,

@@ -52,12 +60,10 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):

60 x: torch.Tensor,

61 bias: torch.Tensor | None = None,

62 ) -> torch.Tensor:

55 - from vllm.utils.humming import HummingMethod

56 -

57 - flatten_inputs = x.view(-1, x.size(-1))

58 - output = HummingMethod.forward_layer(

59 - layer=layer,

60 - inputs=flatten_inputs,

61 - compute_config=layer.compute_config,

63 + return apply_humming_linear(

64 + layer,

65 + x,

66 + layer_config=self.layer_config,

67 + compute_config=self.compute_config,

68 + locks=self.locks,

69 )

63 - return output.view(*x.shape[:-1], output.size(-1))

vllm/model_executor/kernels/linear/nvfp4/humming.py

+16 -10

@@ -5,9 +5,12 @@ import torch

6 from vllm.logger import init_logger

7 from vllm.model_executor.layers.quantization.utils.humming_utils import (

8 - prepare_humming_layer,

8 + apply_humming_linear,

9 + get_humming_linear_compute_config,

10 + prepare_humming_linear_layer_config,

11 )

12 from vllm.platforms import current_platform

13 +from vllm.utils.import_utils import has_humming

15 from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig

@@ -24,6 +27,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):

27 if not current_platform.is_cuda():

28 return False, "Humming only supported on CUDA"

30 + if not has_humming():

31 + return False, "Humming is not installed"

32 +

33 if not current_platform.has_device_capability(75):

34 return False, "Humming only supported on SM75+"

@@ -54,7 +60,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):

60 layer.weight_global_scale = torch.nn.Parameter(

61 1.0 / layer.weight_global_scale, requires_grad=False

62 )

57 - prepare_humming_layer(layer, quant_config)

63 + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)

64 + self.compute_config = get_humming_linear_compute_config()

65 + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)

67 def apply_weights(

68 self,

@@ -62,12 +70,10 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):

70 x: torch.Tensor,

71 bias: torch.Tensor | None = None,

72 ) -> torch.Tensor:

65 - from vllm.utils.humming import HummingMethod

66 -

67 - flatten_inputs = x.view(-1, x.size(-1))

68 - output = HummingMethod.forward_layer(

69 - layer=layer,

70 - inputs=flatten_inputs,

71 - compute_config=layer.compute_config,

73 + return apply_humming_linear(

74 + layer,

75 + x,

76 + layer_config=self.layer_config,

77 + compute_config=self.compute_config,

78 + locks=self.locks,

79 )

73 - return output.view(*x.shape[:-1], output.size(-1))

vllm/model_executor/kernels/linear/scaled_mm/humming.py

+31 -23

@@ -5,10 +5,13 @@ import torch

6 from vllm.logger import init_logger

7 from vllm.model_executor.layers.quantization.utils.humming_utils import (

8 + apply_humming_linear,

9 convert_linear_layer_to_humming_standard,

9 - prepare_humming_layer,

10 + get_humming_linear_compute_config,

11 + prepare_humming_linear_layer_config,

12 )

13 from vllm.platforms import current_platform

14 +from vllm.utils.import_utils import has_humming

16 from .ScaledMMLinearKernel import (

17 FP8ScaledMMLinearKernel,

@@ -30,6 +33,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):

33 if not current_platform.is_cuda():

34 return False, "Humming only supported on CUDA"

36 + if not has_humming():

37 + return False, "Humming is not installed"

38 +

39 if not current_platform.has_device_capability(75):

40 return False, "Humming only supported on SM75+"

@@ -58,9 +64,6 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):

64 scale_group_shape = self.config.weight_quant_key.scale.group_shape

65 if scale_group_shape.is_per_tensor():

66 quant_config["weight_scale_type"] = "tensor"

61 - if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):

62 - del name_map["weight_scale"]

63 - name_map["global_scale"] = "weight_scale"

67 elif scale_group_shape.is_per_channel():

68 quant_config["weight_scale_type"] = "channel"

69 elif scale_group_shape.is_per_group():

@@ -76,7 +79,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):

79 name_map["weight_scale"] = "weight_scale_inv"

81 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)

79 - prepare_humming_layer(layer, quant_config)

82 + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)

83 + self.compute_config = get_humming_linear_compute_config()

84 + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)

86 def apply_weights(

87 self,

@@ -84,15 +89,13 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):

89 x: torch.Tensor,

90 bias: torch.Tensor | None = None,

91 ) -> torch.Tensor:

87 - from vllm.utils.humming import HummingMethod

88 -

89 - flatten_inputs = x.view(-1, x.size(-1))

90 - output = HummingMethod.forward_layer(

91 - layer=layer,

92 - inputs=flatten_inputs,

93 - compute_config=layer.compute_config,

92 + return apply_humming_linear(

93 + layer,

94 + x,

95 + layer_config=self.layer_config,

96 + compute_config=self.compute_config,

97 + locks=self.locks,

98 )

95 - return output.view(*x.shape[:-1], output.size(-1))

100 def apply_scaled_mm(

101 self,

@@ -105,7 +108,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):

108 bias: torch.Tensor | None,

109 output_shape: list,

110 ) -> torch.Tensor:

108 - pass

111 + raise NotImplementedError(

112 + "HummingFP8ScaledMMLinearKernel uses apply_weights directly"

113 + )

114

115

116 class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):

@@ -118,6 +123,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):

123 if not current_platform.is_cuda():

124 return False, "Humming only supported on CUDA"

125

126 + if not has_humming():

127 + return False, "Humming is not installed"

128 +

129 if not current_platform.has_device_capability(75):

130 return False, "Humming only supported on SM75+"

131

@@ -137,7 +145,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):

145 weight.data = weight.data + 128

146

147 convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)

140 - prepare_humming_layer(layer, quant_config)

148 + self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)

149 + self.compute_config = get_humming_linear_compute_config()

150 + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)

151

152 def apply_weights(

153 self,

@@ -145,12 +155,10 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):

155 x: torch.Tensor,

156 bias: torch.Tensor | None = None,

157 ) -> torch.Tensor:

148 - from vllm.utils.humming import HummingMethod

149 -

150 - flatten_inputs = x.view(-1, x.size(-1))

151 - output = HummingMethod.forward_layer(

152 - layer=layer,

153 - inputs=flatten_inputs,

154 - compute_config=layer.compute_config,

158 + return apply_humming_linear(

159 + layer,

160 + x,

161 + layer_config=self.layer_config,

162 + compute_config=self.compute_config,

163 + locks=self.locks,

164 )

156 - return output.view(*x.shape[:-1], output.size(-1))

vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py

+140 -92

@@ -4,7 +4,7 @@

5 import json

6 import math

7 -from typing import TYPE_CHECKING, Any

7 +from typing import TYPE_CHECKING, Any, cast

9 import torch

@@ -62,8 +62,15 @@ from vllm.utils.import_utils import has_humming

62 from vllm.v1.worker.workspace import current_workspace_manager

64 if TYPE_CHECKING:

65 - from vllm.model_executor.layers.fused_moe import RoutedExperts

66 - from vllm.utils.humming import GemmType as HummingGemmType

65 + from vllm.model_executor.layers.quantization.utils.humming_utils import (

66 + HummingMoEQuantConfig,

67 + )

68 + from vllm.utils.humming import (

69 + GemmType as HummingGemmType,

70 + )

71 + from vllm.utils.humming import (

72 + LayerConfig as HummingLayerConfig,

73 + )

76 logger = init_logger(__name__)

@@ -88,15 +95,20 @@ def get_humming_moe_gemm_type() -> str:

95 class HummingExpertsBase(mk.FusedMoEExpertsModular):

96 def __init__(

97 self,

91 - layer: "RoutedExperts",

98 moe_config: FusedMoEConfig,

99 quant_config: FusedMoEQuantConfig,

100 max_num_tokens: int | None = None,

101 num_dispatchers: int | None = None,

102 ):

97 - self.layer = layer

98 - self.num_experts = self.layer.local_num_experts

99 - self.global_num_experts = self.layer.global_num_experts

103 + humming_quant_config = cast("HummingMoEQuantConfig", quant_config)

104 + self.humming_configs: dict[str, HummingLayerConfig] = {

105 + "w13": humming_quant_config.w1_humming_config,

106 + "w2": humming_quant_config.w2_humming_config,

107 + }

108 + self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device)

109 + self.num_experts = moe_config.num_local_experts

110 + self.global_num_experts = moe_config.num_experts

111 + self.quant_config = quant_config

112 self.init_humming_moe()

113

114 if self.is_batched():

@@ -111,31 +123,73 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

123 self._permute_scratch: MoEPermuteScratch | None = None

124

125 def init_humming_moe(self):

114 - from vllm.utils.humming import HummingMethod

126 + from vllm.utils.humming import get_heuristics_config

127

128 self.compute_config = {

129 "use_batch_invariant": envs.VLLM_BATCH_INVARIANT,

130 "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,

131 "gemm_type": self.humming_gemm_type().value,

132 }

121 - self.w13_tuning_config = HummingMethod.get_default_tuning_configs(

122 - layer=self.layer,

133 + self.w13_tuning_config = get_heuristics_config(

134 + layer_config=self.humming_configs["w13"],

135 use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,

136 use_batch_invariant=envs.VLLM_BATCH_INVARIANT,

137 gemm_type=self.humming_gemm_type(),

126 - sublayer_name="w13",

138 )

128 - self.w2_tuning_config = HummingMethod.get_default_tuning_configs(

129 - layer=self.layer,

139 + self.w2_tuning_config = get_heuristics_config(

140 + layer_config=self.humming_configs["w2"],

141 use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,

142 use_batch_invariant=envs.VLLM_BATCH_INVARIANT,

143 gemm_type=self.humming_gemm_type(),

133 - sublayer_name="w2",

144 )

145 self.compute_config_str = json.dumps(self.compute_config)

146 self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)

147 self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)

148

149 + def quantize_input(

150 + self,

151 + sublayer_name: str,

152 + inputs: torch.Tensor,

153 + quanted_input: torch.Tensor | None,

154 + ) -> tuple[torch.Tensor, torch.Tensor | None]:

155 + from vllm.utils.humming import may_quant_input

156 +

157 + return may_quant_input(

158 + self.humming_configs[sublayer_name],

159 + inputs=inputs,

160 + quanted_input=quanted_input,

161 + )

162 +

163 + def humming_forward(

164 + self,

165 + sublayer_name: str,

166 + inputs: torch.Tensor,

167 + weight: torch.Tensor,

168 + input_scale: torch.Tensor | None,

169 + outputs: torch.Tensor,

170 + **kwargs: Any,

171 + ) -> torch.Tensor:

172 + from vllm.utils.humming import humming_forward

173 +

174 + is_w13 = sublayer_name == "w13"

175 + return humming_forward(

176 + self.humming_configs[sublayer_name],

177 + inputs=inputs,

178 + weight=weight,

179 + weight_scale=(

180 + self.quant_config.w1_scale if is_w13 else self.quant_config.w2_scale

181 + ),

182 + zero_point=(self.quant_config.w1_zp if is_w13 else self.quant_config.w2_zp),

183 + bias=(self.quant_config.w1_bias if is_w13 else self.quant_config.w2_bias),

184 + weight_scale_2=(

185 + self.quant_config.g1_alphas if is_w13 else self.quant_config.g2_alphas

186 + ),

187 + input_scale=input_scale,

188 + outputs=outputs,

189 + locks=self.locks,

190 + **kwargs,

191 + )

192 +

193 def _get_permute_scratch(self) -> MoEPermuteScratch | None:

194 if self._permute_scratch is None and moe_permute_unpermute_supported():

195 self._permute_scratch = MoEPermuteScratch(

@@ -218,8 +272,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

272 @property

273 def expects_unquantized_inputs(self) -> bool:

274 """

221 - Humming kernels handle input quantization internally via

222 - HummingMethod.may_quant_input() in the apply() method.

275 + Humming kernels handle input quantization internally in apply().

276

277 This property tells the prepare/finalize step to skip input

278 quantization (by setting defer_input_quant=True) and pass

@@ -255,6 +308,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

308 def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:

309 return True

310

311 + @staticmethod

312 + def _supports_batch_invariance() -> bool:

313 + return True

314 +

315 def moe_problem_size(

316 self,

317 a1: torch.Tensor,

@@ -262,10 +319,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

319 w2: torch.Tensor,

320 topk_ids: torch.Tensor,

321 ) -> tuple[int, int, int, int, int]:

265 - from vllm.utils.humming import HummingLayerMeta

266 -

267 - meta1: HummingLayerMeta = self.layer.humming_metas["w13"]

268 - meta2: HummingLayerMeta = self.layer.humming_metas["w2"]

322 + meta1 = self.humming_configs["w13"]

323 + meta2 = self.humming_configs["w2"]

324

325 assert meta1.num_experts == meta2.num_experts

326

@@ -282,25 +337,25 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

337 assert a1.size(0) == num_experts

338 num_tokens = a1.size(1)

339

285 - return (

286 - meta1.num_experts,

287 - num_tokens,

288 - # Logical intermediate width for both gated and non-gated activations

289 - self.layer.intermediate_size_per_partition,

290 - meta1.shape_k,

291 - top_k,

292 - )

340 + intermediate_dim = meta2.shape_k - meta2.pad_shape_k

341 + hidden_dim = meta1.shape_k - meta1.pad_shape_k

342 + return meta1.num_experts, num_tokens, intermediate_dim, hidden_dim, top_k

343

344 def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation):

345 from vllm.utils.humming import GemmType as HummingGemmType

346 from vllm.utils.humming import dtypes

347

348 num_experts = self.num_experts

299 - N = self.layer.intermediate_size_per_partition

300 - K = self.layer.hidden_size

349 + w13_config = self.humming_configs["w13"]

350 + w2_config = self.humming_configs["w2"]

351 + gate_up_dim = w13_config.shape_n - w13_config.pad_shape_n

352 + intermediate_dim = w2_config.shape_k - w2_config.pad_shape_k

353 + K = w13_config.shape_k - w13_config.pad_shape_k

354 assert isinstance(num_experts, int)

302 - assert isinstance(N, int)

355 + assert isinstance(gate_up_dim, int)

356 + assert isinstance(intermediate_dim, int)

357 assert isinstance(K, int)

358 + assert intermediate_dim == self.adjust_N_for_activation(gate_up_dim, activation)

359

360 # hidden_states

361 # (-> quanted_gate_up_input) (if not BF16/FP16 activation)

@@ -310,7 +365,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

365 # -> down_output

366 # (-> output) (if not is_batched)

367 # Neighboring nodes are required to utilize distinct workspaces.

313 - # The output must be derived from workspace1.

368 + # The final output buffer is supplied by the modular kernel and may

369 + # alias workspace1.

370

371 output_shape: tuple[int, ...]

372 if self.is_batched():

@@ -327,10 +383,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

383 real_shape_m = M * topk

384 output_shape = (M, K)

385

330 - gate_up_size = N * (2 if activation.is_gated else 1)

331 - down_input_size = N

332 - a_dtype = self.layer.humming_metas["w13"].a_dtype

333 - c_dtype = self.layer.humming_metas["w13"].c_dtype

386 + a_dtype = self.humming_configs["w13"].a_dtype

387 + c_dtype = self.humming_configs["w13"].c_dtype

388 num_bits = a_dtype.num_bits

389 torch_dtype_map = {

390 dtypes.float16: torch.float16,

@@ -348,15 +402,15 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

402 "dtype": torch_dtype_map[a_dtype],

403 },

404 "gate_up_output": {

351 - "shape": (real_shape_m, gate_up_size),

405 + "shape": (real_shape_m, gate_up_dim),

406 "dtype": torch_dtype_map[c_dtype],

407 },

408 "activation_output": {

355 - "shape": (real_shape_m, down_input_size),

409 + "shape": (real_shape_m, intermediate_dim),

410 "dtype": torch_dtype_map[c_dtype],

411 },

412 "quanted_down_input": {

359 - "shape": (real_shape_m, down_input_size),

413 + "shape": (real_shape_m, intermediate_dim),

414 "dtype": torch_dtype_map[a_dtype],

415 },

416 "down_output": {

@@ -414,7 +468,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

468

469 output_key = "down_output" if self.is_batched() else "output"

470 output_shape = buffer_metas[output_key]["shape"]

417 - elem_size = self.layer.params_dtype.itemsize

471 + elem_size = self.moe_config.in_dtype.itemsize

472

473 return (

474 (workspace1_nbytes // elem_size,),

@@ -438,7 +492,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):

492 def make_workspaces(self, M: int, topk: int, activation: MoEActivation):

493 shapes = self._workspace_shapes(M, topk, activation)

494 workspace1_shape, workspace2_shape, output_shape = shapes

441 - torch_dtype = self.layer.params_dtype

495 + torch_dtype = self.moe_config.in_dtype

496 workspace1, workspace2 = current_workspace_manager().get_simultaneous(

497 (workspace1_shape, torch_dtype),

498 (workspace2_shape, torch_dtype),

@@ -600,12 +654,11 @@ class HummingIndexedExperts(HummingExpertsBase):

654 """

655 Standard apply implementation for Humming indexed experts.

656

603 - Note: Humming kernels handle weights and quantization internally through

604 - the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.

605 - The output is written into workspace13 via the buffer management.

657 + Humming performs activation quantization internally and consumes the

658 + weights supplied by the modular kernel interface.

659 + Results are written to the output buffer supplied by the modular

660 + kernel, which may alias workspace13.

661 """

607 - from vllm.utils.humming import HummingMethod

608 -

662 assert not apply_router_weight_on_input

663

664 hidden_states = hidden_states.view(-1, hidden_states.size(-1))

@@ -616,6 +669,7 @@ class HummingIndexedExperts(HummingExpertsBase):

669 topk_ids.size(1),

670 activation,

671 )

672 + buffers["output"] = output

673

674 moe_kwargs1, moe_kwargs2 = self.prepare_humming_moe_kwargs(

675 topk_ids=topk_ids,

@@ -623,19 +677,18 @@ class HummingIndexedExperts(HummingExpertsBase):

677 expert_tokens_meta=expert_tokens_meta,

678 )

679

626 - inputs, input_scale = HummingMethod.may_quant_input(

627 - layer=self.layer,

680 + inputs, input_scale = self.quantize_input(

681 + "w13",

682 inputs=hidden_states,

683 quanted_input=buffers.get("quanted_gate_up_input", None),

630 - sublayer_name="w13",

684 )

685

633 - HummingMethod.forward_layer(

634 - layer=self.layer,

686 + self.humming_forward(

687 + "w13",

688 inputs=inputs,

689 + weight=w1,

690 input_scale=input_scale,

691 outputs=buffers["gate_up_output"],

638 - sublayer_name="w13",

692 **moe_kwargs1,

693 )

694

@@ -645,19 +698,18 @@ class HummingIndexedExperts(HummingExpertsBase):

698 output=buffers["activation_output"],

699 )

700

648 - inputs, input_scale = HummingMethod.may_quant_input(

649 - layer=self.layer,

701 + inputs, input_scale = self.quantize_input(

702 + "w2",

703 inputs=buffers["activation_output"],

704 quanted_input=buffers.get("quanted_down_input", None),

652 - sublayer_name="w2",

705 )

706

655 - HummingMethod.forward_layer(

656 - layer=self.layer,

707 + self.humming_forward(

708 + "w2",

709 inputs=inputs,

710 + weight=w2,

711 input_scale=input_scale,

712 outputs=buffers["down_output"].view(-1, hidden_states.size(-1)),

660 - sublayer_name="w2",

713 **moe_kwargs2,

714 )

715

@@ -705,12 +757,11 @@ class HummingGroupedExperts(HummingExpertsBase):

757 """

758 Standard apply implementation for Humming grouped experts.

759

708 - Note: Humming kernels handle weights and quantization internally through

709 - the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.

710 - The output is written into workspace13 via the buffer management.

760 + Humming performs activation quantization internally and consumes the

761 + weights supplied by the modular kernel interface.

762 + Results are written to the output buffer supplied by the modular

763 + kernel, which may alias workspace13.

764 """

712 - from vllm.utils.humming import HummingMethod

713 -

765 assert not apply_router_weight_on_input

766

767 valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)

@@ -722,6 +773,7 @@ class HummingGroupedExperts(HummingExpertsBase):

773 topk_ids.size(1),

774 activation,

775 )

776 + buffers["output"] = output

777

778 hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute(

779 hidden_states=hidden_states,

@@ -733,23 +785,22 @@ class HummingGroupedExperts(HummingExpertsBase):

785 scratch=self._get_permute_scratch(),

786 )

787

736 - inputs, input_scale = HummingMethod.may_quant_input(

737 - layer=self.layer,

788 + inputs, input_scale = self.quantize_input(

789 + "w13",

790 inputs=hidden_states,

791 quanted_input=buffers.get("quanted_gate_up_input", None),

740 - sublayer_name="w13",

792 )

793

743 - HummingMethod.forward_layer(

744 - layer=self.layer,

794 + self.humming_forward(

795 + "w13",

796 inputs=inputs,

797 + weight=w1,

798 input_scale=input_scale,

799 outputs=buffers["gate_up_output"],

800 valid_shape_m=valid_shape_m,

801 expert_layout=expert_first_token_offset,

802 compute_config=self.compute_config_str,

803 tuning_config=self.w13_tuning_config_str,

752 - sublayer_name="w13",

804 )

805

806 self.apply_activation(

@@ -758,23 +809,22 @@ class HummingGroupedExperts(HummingExpertsBase):

809 output=buffers["activation_output"],

810 )

811

761 - inputs, input_scale = HummingMethod.may_quant_input(

762 - layer=self.layer,

812 + inputs, input_scale = self.quantize_input(

813 + "w2",

814 inputs=buffers["activation_output"],

815 quanted_input=buffers.get("quanted_down_input", None),

765 - sublayer_name="w2",

816 )

817

768 - HummingMethod.forward_layer(

769 - layer=self.layer,

818 + self.humming_forward(

819 + "w2",

820 inputs=inputs,

821 + weight=w2,

822 input_scale=input_scale,

823 outputs=buffers["down_output"],

824 valid_shape_m=valid_shape_m,

825 expert_layout=expert_first_token_offset,

826 compute_config=self.compute_config_str,

827 tuning_config=self.w2_tuning_config_str,

777 - sublayer_name="w2",

828 )

829

830 moe_unpermute(

@@ -821,12 +871,11 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):

871 """

872 Standard apply implementation for Humming batched grouped experts.

873

824 - Note: Humming kernels handle weights and quantization internally through

825 - the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.

826 - The output is written into workspace13 via the buffer management.

874 + Humming performs activation quantization internally and consumes the

875 + weights supplied by the modular kernel interface.

876 + Results are written to the output buffer supplied by the modular

877 + kernel, which may alias workspace13.

878 """

828 - from vllm.utils.humming import HummingMethod

829 -

879 assert not apply_router_weight_on_input

880 assert expert_tokens_meta is not None

881

@@ -841,24 +890,24 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):

890 topk_ids.size(1),

891 activation,

892 )

893 + buffers["down_output"] = output

894

845 - inputs, input_scale = HummingMethod.may_quant_input(

846 - layer=self.layer,

895 + inputs, input_scale = self.quantize_input(

896 + "w13",

897 inputs=hidden_states,

898 quanted_input=buffers.get("quanted_gate_up_input", None),

849 - sublayer_name="w13",

899 )

900

852 - HummingMethod.forward_layer(

853 - layer=self.layer,

901 + self.humming_forward(

902 + "w13",

903 inputs=inputs,

904 + weight=w1,

905 input_scale=input_scale,

906 outputs=buffers["gate_up_output"],

907 valid_shape_m=valid_shape_m,

908 expert_layout=expert_num_tokens,

909 compute_config=self.compute_config_str,

910 tuning_config=self.w13_tuning_config_str,

861 - sublayer_name="w13",

911 )

912

913 self.apply_activation(

@@ -867,21 +916,20 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):

916 output=buffers["activation_output"],

917 )

918

870 - inputs, input_scale = HummingMethod.may_quant_input(

871 - layer=self.layer,

919 + inputs, input_scale = self.quantize_input(

920 + "w2",

921 inputs=buffers["activation_output"],

922 quanted_input=buffers.get("quanted_down_input", None),

874 - sublayer_name="w2",

923 )

924

877 - HummingMethod.forward_layer(

878 - layer=self.layer,

925 + self.humming_forward(

926 + "w2",

927 inputs=inputs,

928 + weight=w2,

929 input_scale=input_scale,

930 outputs=output.view(-1, hidden_states.size(-1)),

931 valid_shape_m=valid_shape_m,

932 expert_layout=expert_num_tokens,

933 compute_config=self.compute_config_str,

934 tuning_config=self.w2_tuning_config_str,

886 - sublayer_name="w2",

935 )

vllm/model_executor/layers/fused_moe/oracle/fp8.py

+6 -9

@@ -619,7 +619,12 @@ def make_fp8_moe_quant_config(

619 )

620

621 assert isinstance(layer, RoutedExperts)

622 - return get_humming_moe_quant_config(layer)

622 + return get_humming_moe_quant_config(

623 + layer,

624 + gemm1_alpha=gemm1_alpha,

625 + gemm1_beta=gemm1_beta,

626 + gemm1_clamp_limit=swiglu_limit,

627 + )

628

629 # Flashinfer CUTLASS or HPC per-tensor uses single dq scale

630 # (alpha = w_scale * a_scale) and inverse a2 scale.

@@ -694,7 +699,6 @@ def make_fp8_moe_kernel(

699 experts_cls: type[mk.FusedMoEExperts],

700 fp8_backend: Fp8MoeBackend,

701 routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,

697 - layer: torch.nn.Module | None = None,

702 ) -> mk.FusedMoEKernel:

703 # Create Prepare/Finalize.

704 prepare_finalize = maybe_make_prepare_finalize(

@@ -708,11 +712,6 @@ def make_fp8_moe_kernel(

712

713 logger.info_once("Using %s", prepare_finalize.__class__.__name__)

714

711 - extra_kwargs = {}

712 - if fp8_backend == Fp8MoeBackend.HUMMING:

713 - assert layer is not None

714 - extra_kwargs = {"layer": layer}

715 -

715 # Create Experts.

716 if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:

717 max_num_tokens = prepare_finalize.max_num_tokens_per_rank()

@@ -722,13 +721,11 @@ def make_fp8_moe_kernel(

721 quant_config=moe_quant_config,

722 max_num_tokens=max_num_tokens,

723 num_dispatchers=prepare_finalize.num_dispatchers(),

725 - **extra_kwargs,

724 )

725 else:

726 experts = experts_cls(

727 moe_config=moe_config,

728 quant_config=moe_quant_config,

731 - **extra_kwargs,

729 )

730

731 kernel = mk.FusedMoEKernel(

vllm/model_executor/layers/fused_moe/oracle/int8.py

+9 -12

@@ -195,7 +195,12 @@ def make_int8_moe_quant_config(

195 )

196

197 assert isinstance(layer, RoutedExperts)

198 - return get_humming_moe_quant_config(layer)

198 + return get_humming_moe_quant_config(

199 + layer,

200 + gemm1_alpha=getattr(layer, "swiglu_alpha", None),

201 + gemm1_beta=getattr(layer, "swiglu_beta", None),

202 + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),

203 + )

204

205 if scales_absent and not per_act_token_quant:

206 return int8_w8a16_moe_quant_config(

@@ -251,9 +256,9 @@ def convert_to_int8_moe_kernel_format(

256 )

257

258 assert layer is not None

254 - # Humming reads canonical CT scales (w*_weight_scale) from the layer.

255 - # Online int8 produces per-channel (E, N) w*_scale; expose them as the

256 - # (E, N, 1) w*_weight_scale humming's loader expects.

259 + # The Humming conversion expects canonical CT scale parameter names.

260 + # Online int8 produces per-channel (E, N) w*_scale; expose them as

261 + # (E, N, 1) w*_weight_scale tensors before conversion.

262 for sub in ("w13", "w2"):

263 if hasattr(layer, f"{sub}_weight_scale"):

264 continue

@@ -279,7 +284,6 @@ def make_int8_moe_kernel(

284 moe_config: FusedMoEConfig,

285 experts_cls: type[mk.FusedMoEExperts],

286 routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,

282 - layer: torch.nn.Module | None = None,

287 ) -> mk.FusedMoEKernel:

288 # Create Prepare/Finalize.

289 prepare_finalize = maybe_make_prepare_finalize(

@@ -293,11 +297,6 @@ def make_int8_moe_kernel(

297

298 logger.info_once("Using %s", prepare_finalize.__class__.__name__)

299

296 - extra_kwargs = {}

297 - if int8_backend == Int8MoeBackend.HUMMING:

298 - assert layer is not None

299 - extra_kwargs = {"layer": layer}

300 -

300 # Create Experts.

301 if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:

302 max_num_tokens = prepare_finalize.max_num_tokens_per_rank()

@@ -307,13 +306,11 @@ def make_int8_moe_kernel(

306 quant_config=moe_quant_config,

307 max_num_tokens=max_num_tokens,

308 num_dispatchers=prepare_finalize.num_dispatchers(),

310 - **extra_kwargs,

309 )

310 else:

311 experts = experts_cls(

312 moe_config=moe_config,

313 quant_config=moe_quant_config,

316 - **extra_kwargs,

314 )

315

316 kernel = mk.FusedMoEKernel(

vllm/model_executor/layers/fused_moe/oracle/int_wna16.py

+1 -5

@@ -358,7 +358,6 @@ def make_wna16_moe_kernel(

358 moe_config: FusedMoEConfig,

359 experts_cls: type[mk.FusedMoEExperts],

360 backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN,

361 - layer: torch.nn.Module | None = None,

361 is_k_full: bool = False,

362 w13_g_idx: torch.Tensor | None = None,

363 w2_g_idx: torch.Tensor | None = None,

@@ -410,10 +409,7 @@ def make_wna16_moe_kernel(

409 logger.info_once("Using %s", experts_cls.__name__, scope="local")

410

411 extra_args: dict[str, Any] = {}

413 - if backend == WNA16MoEBackend.HUMMING:

414 - assert layer is not None

415 - extra_args = {"layer": layer}

416 - elif issubclass(experts_cls, MarlinExpertsBase):

412 + if issubclass(experts_cls, MarlinExpertsBase):

413 extra_args = {

414 "w13_g_idx": w13_g_idx,

415 "w2_g_idx": w2_g_idx,

vllm/model_executor/layers/fused_moe/oracle/mxfp4.py

-8

@@ -1788,7 +1788,6 @@ def make_mxfp4_moe_kernel(

1788 experts_cls: type[mk.FusedMoEExperts],

1789 mxfp4_backend: Mxfp4MoeBackend,

1790 routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,

1791 - layer: RoutedExperts | None = None,

1791 ) -> mk.FusedMoEKernel:

1792 """Create a FusedMoEKernel for the given MXFP4 backend."""

1793 is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)

@@ -1805,11 +1804,6 @@ def make_mxfp4_moe_kernel(

1804 logger.info_once("Using %s", prepare_finalize.__class__.__name__)

1805 logger.info_once("Using %s", experts_cls.__name__)

1806

1808 - extra_kwargs = {}

1809 - if mxfp4_backend == Mxfp4MoeBackend.HUMMING:

1810 - assert layer is not None

1811 - extra_kwargs["layer"] = layer

1812 -

1807 # Create Experts.

1808 if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:

1809 max_num_tokens = prepare_finalize.max_num_tokens_per_rank()

@@ -1819,13 +1813,11 @@ def make_mxfp4_moe_kernel(

1813 quant_config=moe_quant_config,

1814 max_num_tokens=max_num_tokens,

1815 num_dispatchers=prepare_finalize.num_dispatchers(),

1822 - **extra_kwargs,

1816 )

1817 else:

1818 experts = experts_cls(

1819 moe_config=moe_config,

1820 quant_config=moe_quant_config,

1828 - **extra_kwargs,

1821 )

1822

1823 kernel = mk.FusedMoEKernel(

vllm/model_executor/layers/fused_moe/oracle/nvfp4.py

+8 -7

@@ -398,10 +398,10 @@ def convert_to_nvfp4_moe_kernel_format(

398 a2_scale = None

399 w13 = layer.w13_weight

400 w13_scale = layer.w13_weight_scale

401 - w13_scale_2 = getattr(layer, "w13_global_scale", None)

401 + w13_scale_2 = getattr(layer, "w13_weight_scale_2", None)

402 w2 = layer.w2_weight

403 w2_scale = layer.w2_weight_scale

404 - w2_scale_2 = getattr(layer, "w2_global_scale", None)

404 + w2_scale_2 = getattr(layer, "w2_weight_scale_2", None)

405 elif nvfp4_backend == NvFp4MoeBackend.MARLIN:

406 a13_scale = None

407 a2_scale = None

@@ -485,7 +485,12 @@ def make_nvfp4_moe_quant_config(

485 )

486

487 assert isinstance(layer, RoutedExperts)

488 - return get_humming_moe_quant_config(layer)

488 + return get_humming_moe_quant_config(

489 + layer,

490 + gemm1_alpha=getattr(layer, "swiglu_alpha", None),

491 + gemm1_beta=getattr(layer, "swiglu_beta", None),

492 + gemm1_clamp_limit=swiglu_limit,

493 + )

494 elif backend == NvFp4MoeBackend.MARLIN:

495 return nvfp4_w4a16_moe_quant_config(

496 g1_alphas=w13_scale_2,

@@ -544,7 +549,6 @@ def make_nvfp4_moe_kernel(

549 experts_cls: type[mk.FusedMoEExperts],

550 backend: NvFp4MoeBackend,

551 routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,

547 - layer: torch.nn.Module | None = None,

552 per_token_activation: bool = False,

553 ) -> mk.FusedMoEKernel:

554 # Create Prepare/Finalize.

@@ -560,9 +564,6 @@ def make_nvfp4_moe_kernel(

564 logger.info_once("Using %s", prepare_finalize.__class__.__name__)

565

566 extra_kwargs = {}

563 - if backend == NvFp4MoeBackend.HUMMING:

564 - assert layer is not None

565 - extra_kwargs = {"layer": layer}

567 if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM and per_token_activation:

568 extra_kwargs["per_token_activation"] = True

569

vllm/model_executor/layers/quantization/auto_awq.py

+6 -2

@@ -751,7 +751,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):

751 moe_config=self.moe,

752 experts_cls=self.experts_cls,

753 backend=self.wna16_moe_backend,

754 - layer=layer,

754 is_k_full=self.is_k_full,

755 w13_g_idx=getattr(layer, "w13_g_idx", None),

756 w2_g_idx=getattr(layer, "w2_g_idx", None),

@@ -766,7 +765,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):

765 get_humming_moe_quant_config,

766 )

767

769 - return get_humming_moe_quant_config(layer)

768 + return get_humming_moe_quant_config(

769 + layer,

770 + gemm1_alpha=getattr(layer, "swiglu_alpha", None),

771 + gemm1_beta=getattr(layer, "swiglu_beta", None),

772 + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),

773 + )

774 return make_wna16_moe_quant_config(

775 w1_scale=layer.w13_scales,

776 w2_scale=layer.w2_scales,

vllm/model_executor/layers/quantization/auto_gptq.py

+6 -2

@@ -772,7 +772,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):

772 moe_config=self.moe,

773 experts_cls=self.experts_cls,

774 backend=self.wna16_moe_backend,

775 - layer=layer,

775 is_k_full=self.is_k_full,

776 w13_g_idx=getattr(layer, "w13_g_idx", None),

777 w2_g_idx=getattr(layer, "w2_g_idx", None),

@@ -787,7 +786,12 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):

786 get_humming_moe_quant_config,

787 )

788

790 - return get_humming_moe_quant_config(layer)

789 + return get_humming_moe_quant_config(

790 + layer,

791 + gemm1_alpha=getattr(layer, "swiglu_alpha", None),

792 + gemm1_beta=getattr(layer, "swiglu_beta", None),

793 + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),

794 + )

795

796 from vllm.model_executor.layers.fused_moe.config import (

797 gptq_marlin_moe_quant_config,

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py

-1

@@ -251,7 +251,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):

251 experts_cls=self.experts_cls,

252 backend=self.nvfp4_backend,

253 routing_tables=layer._expert_routing_tables(),

254 - layer=layer,

254 )

255 self.moe_kernel.fused_experts.process_weights_after_loading(layer)

256

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py

-1

@@ -339,7 +339,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):

339 fp8_backend=self.fp8_backend,

340 experts_cls=self.experts_cls,

341 routing_tables=layer._expert_routing_tables(),

342 - layer=layer,

342 )

343

344 def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py

-1

@@ -159,7 +159,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):

159 moe_config=self.moe,

160 experts_cls=self.experts_cls,

161 routing_tables=layer._expert_routing_tables(),

162 - layer=layer,

162 )

163 self.moe_kernel.fused_experts.process_weights_after_loading(layer)

164

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py

-1

@@ -139,7 +139,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):

139 fp8_backend=self.fp8_backend,

140 experts_cls=self.experts_cls,

141 routing_tables=layer._expert_routing_tables(),

142 - layer=layer,

142 )

143

144 def get_fused_moe_quant_config(

vllm/model_executor/layers/quantization/fp8.py

-1

@@ -679,7 +679,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):

679 fp8_backend=self.fp8_backend,

680 experts_cls=self.experts_cls,

681 routing_tables=layer._expert_routing_tables(),

682 - layer=layer,

682 )

683

684 def process_weights_after_loading(self, layer: RoutedExperts) -> None:

vllm/model_executor/layers/quantization/humming.py

+36 -37

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -import json

3 import math

4 from collections.abc import Callable

5 from typing import TYPE_CHECKING, Any

@@ -32,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (

31 )

32 from vllm.model_executor.layers.quantization.utils.humming_utils import (

33 convert_to_humming_moe_kernel_format,

34 + get_humming_linear_compute_config,

35 get_humming_moe_quant_config,

36 input_schema_to_quant_key,

37 make_humming_moe_kernel,

@@ -425,7 +425,6 @@ class HummingLinearMethod(LinearMethodBase):

425 for name, _ in list(layer.named_parameters()):

426 if name != "bias":

427 delattr(layer, name)

428 - delattr(layer, "locks")

428 self.__class__ = UnquantizedLinearMethod # type: ignore

429 tensor = torch.empty(

430 (

@@ -513,9 +512,6 @@ class HummingLinearMethod(LinearMethodBase):

512 param = prepare_param(tensor, name, extra_attrs)

513 setattr(layer, name, param)

514

516 - locks = torch.zeros(1024, dtype=torch.int32)

517 - layer.register_buffer("locks", locks)

518 -

515 if self.force_input_schema is not None:

516 self.input_schema = self.force_input_schema

517

@@ -574,9 +570,7 @@ class HummingLinearMethod(LinearMethodBase):

570

571 del tensors

572

577 - # prepare layer config from humming kernel

578 - _hm.HummingMethod.prepare_layer_meta(

579 - layer=layer,

573 + self.layer_config = _hm.prepare_layer_config(

574 shape_n=layer.output_partition_sizes_sum,

575 shape_k=layer.input_size_per_partition,

576 weight_schema=self.weight_schema,

@@ -586,19 +580,17 @@ class HummingLinearMethod(LinearMethodBase):

580 has_bias=layer.has_bias,

581 torch_dtype=layer.param_dtype,

582 )

583 + tensors = _hm.transform_humming_tensors(

584 + self.layer_config, dict(layer.named_parameters())

585 + )

586 + for name, _ in list(layer.named_parameters()):

587 + delattr(layer, name)

588 + for name, tensor in tensors.items():

589 + param = torch.nn.Parameter(tensor, requires_grad=False)

590 + setattr(layer, name, param)

591

590 - # preprocess weight for inference

591 - _hm.HummingMethod.transform_humming_layer(layer)

592 -

593 - # compute_config: kernel configs that do not directly affect weights

594 - # but significantly impact kernel behavior or computation precision.

595 - # see https://github.com/inclusionAI/humming/blob/main/docs/config.md

596 - compute_config = {

597 - "use_batch_invariant": envs.VLLM_BATCH_INVARIANT,

598 - "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,

599 - "gemm_type": "dense",

600 - }

601 - self.compute_config = json.dumps(compute_config)

592 + self.compute_config = get_humming_linear_compute_config()

593 + self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)

594

595 def apply(

596 self,

@@ -606,10 +598,16 @@ class HummingLinearMethod(LinearMethodBase):

598 x: torch.Tensor,

599 bias: torch.Tensor | None = None,

600 ) -> torch.Tensor:

609 - flatten_inputs = x.view(-1, x.size(-1))

610 - output = _hm.HummingMethod.forward_layer(

611 - layer=layer,

601 + flatten_inputs = x.reshape(-1, x.size(-1))

602 + output = _hm.humming_forward(

603 + self.layer_config,

604 inputs=flatten_inputs,

605 + weight=layer.weight,

606 + weight_scale=getattr(layer, "weight_scale", None),

607 + zero_point=getattr(layer, "zero_point", None),

608 + bias=getattr(layer, "bias", None),

609 + weight_scale_2=getattr(layer, "weight_scale_2", None),

610 + locks=self.locks,

611 compute_config=self.compute_config,

612 )

613 output = output.view(*x.shape[:-1], output.size(-1))

@@ -726,12 +724,13 @@ class HummingMoEMethod(FusedMoEMethodBase):

724 # sublayer: a layer contains multiple sets of weights for quantized GEMM

725 # (e.g., weight, weight_scale, etc.).

726 # The weight names of sublayer start with the prefix "{sublayer_name}_"

727 + w13_output_size = intermediate_size_per_partition * self.moe.w13_num_shards

728 layer.sublayer_configs = {

729 "w13": {

731 - "shape_n": intermediate_size_per_partition * self.moe.w13_num_shards,

730 + "shape_n": w13_output_size,

731 "shape_k": hidden_size,

732 "tensors_attrs": self.weight_schema.get_padded_tensors_attrs(

734 - shape_n=intermediate_size_per_partition * self.moe.w13_num_shards,

733 + shape_n=w13_output_size,

734 shape_k=hidden_size,

735 num_experts=num_experts,

736 param_dtype=params_dtype,

@@ -763,11 +762,14 @@ class HummingMoEMethod(FusedMoEMethodBase):

762 if self.force_input_schema is not None:

763 self.input_schema = self.force_input_schema

764

766 - locks = torch.zeros(1024, dtype=torch.int32)

767 - layer.register_buffer("locks", locks)

768 -

765 def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:

770 - return get_humming_moe_quant_config(layer)

766 + return get_humming_moe_quant_config(

767 + layer,

768 + self.humming_configs,

769 + gemm1_alpha=getattr(layer, "swiglu_alpha", None),

770 + gemm1_beta=getattr(layer, "swiglu_beta", None),

771 + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),

772 + )

773

774 def process_weights_after_loading(self, layer: RoutedExperts) -> None:

775 if getattr(self, "processed", False):

@@ -775,7 +777,7 @@ class HummingMoEMethod(FusedMoEMethodBase):

777 self.processed = True

778

779 # Convert weights to Humming kernel format

778 - convert_to_humming_moe_kernel_format(

780 + self.humming_configs = convert_to_humming_moe_kernel_format(

781 layer=layer,

782 sublayer_configs=layer.sublayer_configs,

783 weight_schema=self.weight_schema,

@@ -791,7 +793,6 @@ class HummingMoEMethod(FusedMoEMethodBase):

793 self.moe_quant_config,

794 self.moe,

795 self.experts_cls,

794 - layer=layer,

796 routing_tables=layer._expert_routing_tables(),

797 )

798

@@ -814,13 +815,11 @@ class HummingMoEMethod(FusedMoEMethodBase):

815 3. Finalization (weight application & reduction - no-op for Humming

816 since it's already done internally)

817

817 - Humming handles all quantization, weight application, and reduction

818 - internally in the experts.apply() method via HummingMethod calls.

818 + Humming handles quantization, weight application, and reduction

819 + internally in the experts implementation.

820

820 - Note: Although w1/w2 weights are passed to the kernel for interface

821 - consistency, Humming's experts.apply() reads weights directly from

822 - the layer object via HummingMethod.forward_layer() and ignores the

823 - w1/w2 parameters.

821 + Humming experts consume the weights and quantization tensors passed

822 + through the standard modular kernel interface.

823 """

824 assert self.moe_kernel is not None

825 return self.moe_kernel.apply(

vllm/model_executor/layers/quantization/modelopt.py

-3

@@ -886,7 +886,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):

886 fp8_backend=self.fp8_backend,

887 experts_cls=self.experts_cls,

888 routing_tables=layer._expert_routing_tables(),

889 - layer=layer,

889 )

890

891 def process_weights_after_loading(self, layer: RoutedExperts) -> None:

@@ -1579,7 +1578,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):

1578 experts_cls=self.experts_cls,

1579 backend=self.nvfp4_backend,

1580 routing_tables=layer._expert_routing_tables(),

1582 - layer=layer,

1581 )

1582 self.moe_kernel.fused_experts.process_weights_after_loading(layer)

1583

@@ -2058,7 +2056,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):

2056 fp8_backend=self.mxfp8_backend,

2057 experts_cls=self.experts_cls,

2058 routing_tables=layer._expert_routing_tables(),

2061 - layer=layer,

2059 )

2060

2061 # No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation

vllm/model_executor/layers/quantization/moe_wna16.py

+6 -2

@@ -370,7 +370,12 @@ class MoeWNA16Method(FusedMoEMethodBase):

370 get_humming_moe_quant_config,

371 )

372

373 - return get_humming_moe_quant_config(layer)

373 + return get_humming_moe_quant_config(

374 + layer,

375 + gemm1_alpha=getattr(layer, "swiglu_alpha", None),

376 + gemm1_beta=getattr(layer, "swiglu_beta", None),

377 + gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),

378 + )

379

380 has_zp = self.quant_config.has_zp

381 return make_wna16_moe_quant_config(

@@ -391,7 +396,6 @@ class MoeWNA16Method(FusedMoEMethodBase):

396 moe_config=self.moe,

397 experts_cls=self.experts_cls,

398 backend=self.wna16_backend,

394 - layer=layer,

399 routing_tables=layer._expert_routing_tables(),

400 )

401

vllm/model_executor/layers/quantization/mxfp4.py

-2

@@ -382,7 +382,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):

382 mxfp4_backend=self.mxfp4_backend,

383 experts_cls=self.experts_cls,

384 routing_tables=layer._expert_routing_tables(),

385 - layer=layer,

385 )

386

387 def process_weights_after_loading(self, layer: RoutedExperts) -> None:

@@ -773,7 +772,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):

772 mxfp4_backend=self.mxfp4_backend,

773 experts_cls=self.experts_cls,

774 routing_tables=layer._expert_routing_tables(),

776 - layer=layer,

775 )

776

777 def _convert_k3_situ_weight_to_kernel_format(

vllm/model_executor/layers/quantization/online/fp8.py

-1

@@ -516,7 +516,6 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):

516 fp8_backend=self.fp8_backend,

517 experts_cls=self.experts_cls,

518 routing_tables=layer._expert_routing_tables(),

519 - layer=layer,

519 )

520

521 def get_fused_moe_quant_config(

vllm/model_executor/layers/quantization/online/int8.py

-1

@@ -117,7 +117,6 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):

117 moe_config=self.moe,

118 experts_cls=self.experts_cls,

119 routing_tables=layer._expert_routing_tables(),

120 - layer=layer,

120 )

121 self.moe_kernel.fused_experts.process_weights_after_loading(layer)

122

vllm/model_executor/layers/quantization/online/mxfp8.py

-1

@@ -200,7 +200,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):

200 fp8_backend=self.fp8_backend,

201 experts_cls=self.experts_cls,

202 routing_tables=layer._expert_routing_tables(),

203 - layer=layer,

203 )

204

205 def get_fused_moe_quant_config(

vllm/model_executor/layers/quantization/online/nvfp4.py

-1

@@ -172,7 +172,6 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):

172 experts_cls=self.experts_cls,

173 backend=self.nvfp4_backend,

174 routing_tables=layer._expert_routing_tables(),

175 - layer=layer,

175 per_token_activation=True,

176 )

177 self.moe_kernel.fused_experts.process_weights_after_loading(layer)

vllm/model_executor/layers/quantization/quark/quark_moe.py

-2

@@ -804,7 +804,6 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):

804 moe_config=self.moe,

805 experts_cls=self.experts_cls,

806 routing_tables=layer._expert_routing_tables(),

807 - layer=layer,

807 )

808

809 def get_fused_moe_quant_config(

@@ -1646,7 +1645,6 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):

1645 experts_cls=self.experts_cls,

1646 backend=self.nvfp4_backend,

1647 routing_tables=layer._expert_routing_tables(),

1649 - layer=layer,

1648 )

1649

1650 def get_fused_moe_quant_config(

vllm/model_executor/layers/quantization/utils/humming_utils.py

+106 -67

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import json

4 +from dataclasses import dataclass

5 from typing import TYPE_CHECKING, Any

7 import regex as re

@@ -47,11 +48,19 @@ if TYPE_CHECKING:

48 GPTQWeightSchema,

49 HummingInputSchema,

50 HummingWeightSchema,

51 + LayerConfig,

52 )

53 from vllm.utils.humming import dtypes as humming_dtypes

55 logger = init_logger(__name__)

57 +

58 +@dataclass(kw_only=True)

59 +class HummingMoEQuantConfig(FusedMoEQuantConfig):

60 + w1_humming_config: "LayerConfig"

61 + w2_humming_config: "LayerConfig"

62 +

63 +

64 if has_humming():

65 from vllm.utils.humming import dtypes as humming_dtypes

@@ -435,16 +444,17 @@ def convert_linear_layer_to_humming_standard(

444 setattr(layer, name, param)

445

446

438 -def prepare_humming_layer(

447 +def prepare_humming_linear_layer_config(

448 layer: LinearBase,

449 quant_config: dict,

450 input_quant_config: dict | None = None,

442 -):

451 +) -> "LayerConfig":

452 from vllm.utils.humming import (

453 BaseInputSchema,

454 BaseWeightSchema,

455 HummingInputSchema,

447 - HummingMethod,

456 + prepare_layer_config,

457 + transform_humming_tensors,

458 )

459

460 weight_schema = BaseWeightSchema.from_config(quant_config)

@@ -480,18 +490,8 @@ def prepare_humming_layer(

490

491 layer.weight_schema = weight_schema

492

483 - for name, _ in list(layer.named_parameters()):

484 - delattr(layer, name)

485 -

486 - for name, tensor in tensors.items():

487 - if isinstance(tensor, torch.nn.Parameter):

488 - tensor = tensor.data

489 - param = torch.nn.Parameter(tensor, requires_grad=False)

490 - setattr(layer, name, param)

491 -

492 - # Step 2: transform weight (humming standard format) for forwarding

493 - HummingMethod.prepare_layer_meta(

494 - layer=layer,

493 + # Step 2: transform weight (humming standard format) for forwarding.

494 + config = prepare_layer_config(

495 shape_n=sum(layer.output_partition_sizes),

496 shape_k=input_size_per_partition,

497 weight_schema=weight_schema,

@@ -501,20 +501,49 @@ def prepare_humming_layer(

501 has_bias=layer.has_bias,

502 torch_dtype=layer.params_dtype,

503 )

504 + tensors = transform_humming_tensors(config, tensors)

505 + for name, _ in list(layer.named_parameters()):

506 + delattr(layer, name)

507 + for name, tensor in tensors.items():

508 + param = torch.nn.Parameter(tensor, requires_grad=False)

509 + setattr(layer, name, param)

510

505 - HummingMethod.transform_humming_layer(layer)

506 - if not hasattr(layer, "locks"):

507 - device = layer.weight.device

508 - locks = torch.zeros(1024, dtype=torch.int32, device=device)

509 - layer.register_buffer("locks", locks)

511 + return config

512

511 - compute_config = {

512 - "use_batch_invariant": envs.VLLM_BATCH_INVARIANT,

513 - "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,

514 - "gemm_type": "dense",

515 - }

513

517 - layer.compute_config = json.dumps(compute_config)

514 +def get_humming_linear_compute_config() -> str:

515 + return json.dumps(

516 + {

517 + "use_batch_invariant": envs.VLLM_BATCH_INVARIANT,

518 + "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,

519 + "gemm_type": "dense",

520 + }

521 + )

522 +

523 +

524 +def apply_humming_linear(

525 + layer: LinearBase,

526 + x: torch.Tensor,

527 + *,

528 + layer_config: "LayerConfig",

529 + compute_config: str,

530 + locks: torch.Tensor,

531 +) -> torch.Tensor:

532 + from vllm.utils.humming import humming_forward

533 +

534 + flatten_inputs = x.reshape(-1, x.size(-1))

535 + output = humming_forward(

536 + layer_config,

537 + inputs=flatten_inputs,

538 + weight=layer.weight,

539 + weight_scale=getattr(layer, "weight_scale", None),

540 + zero_point=getattr(layer, "zero_point", None),

541 + bias=getattr(layer, "bias", None),

542 + weight_scale_2=getattr(layer, "weight_scale_2", None),

543 + locks=locks,

544 + compute_config=compute_config,

545 + )

546 + return output.view(*x.shape[:-1], output.size(-1))

547

548

549 def make_humming_moe_quant_config(

@@ -532,7 +561,9 @@ def make_humming_moe_quant_config(

561 gemm1_alpha: float | None = None,

562 gemm1_beta: float | None = None,

563 gemm1_clamp_limit: float | None = None,

535 -) -> FusedMoEQuantConfig:

564 + humming_configs: dict[str, "LayerConfig"] | None = None,

565 +) -> HummingMoEQuantConfig:

566 + assert humming_configs is not None

567 if quant_dtype is None:

568 a_quant_desc = FusedMoEQuantDesc(dtype=None)

569 else:

@@ -557,7 +588,7 @@ def make_humming_moe_quant_config(

588 bias=w2_bias,

589 )

590

560 - return FusedMoEQuantConfig(

591 + return HummingMoEQuantConfig(

592 _a1=a_quant_desc,

593 _a2=a_quant_desc,

594 _w1=w1_quant_desc,

@@ -565,15 +596,20 @@ def make_humming_moe_quant_config(

596 gemm1_alpha=gemm1_alpha,

597 gemm1_beta=gemm1_beta,

598 gemm1_clamp_limit=gemm1_clamp_limit,

599 + w1_humming_config=humming_configs["w13"],

600 + w2_humming_config=humming_configs["w2"],

601 )

602

603

604 def get_humming_moe_quant_config(

605 layer: "RoutedExperts",

606 + humming_configs: dict[str, "LayerConfig"] | None = None,

607 gemm1_alpha: float | None = None,

608 gemm1_beta: float | None = None,

609 gemm1_clamp_limit: float | None = None,

610 ):

611 + if humming_configs is None:

612 + humming_configs = layer.humming_configs

613 input_schema = layer.input_schemas["w13"]

614 weight_schema = layer.weight_schemas["w13"]

615

@@ -600,13 +636,17 @@ def get_humming_moe_quant_config(

636 weight_dtype=str(weight_schema.b_dtype),

637 weight_group_shape=weight_group_shape,

638 w1_scale=getattr(layer, "w13_weight_scale", None),

603 - w1_gscale=getattr(layer, "w13_global_scale", None),

639 + w1_gscale=getattr(layer, "w13_weight_scale_2", None),

640 w1_zp=getattr(layer, "w13_zero_point", None),

641 w1_bias=getattr(layer, "w13_bias", None),

642 w2_scale=getattr(layer, "w2_weight_scale", None),

607 - w2_gscale=getattr(layer, "w2_global_scale", None),

643 + w2_gscale=getattr(layer, "w2_weight_scale_2", None),

644 w2_zp=getattr(layer, "w2_zero_point", None),

645 w2_bias=getattr(layer, "w2_bias", None),

646 + gemm1_alpha=gemm1_alpha,

647 + gemm1_beta=gemm1_beta,

648 + gemm1_clamp_limit=gemm1_clamp_limit,

649 + humming_configs=humming_configs,

650 )

651

652

@@ -677,7 +717,6 @@ def make_humming_moe_kernel(

717 moe_quant_config: FusedMoEQuantConfig,

718 moe_config: FusedMoEConfig,

719 experts_cls: type[mk.FusedMoEExperts],

680 - layer: "RoutedExperts",

720 routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,

721 ) -> mk.FusedMoEKernel:

722 # Create Prepare/Finalize.

@@ -692,8 +731,6 @@ def make_humming_moe_kernel(

731

732 logger.info_once("Using %s", prepare_finalize.__class__.__name__)

733

695 - extra_args: dict[str, Any] = {"layer": layer}

696 -

734 # Create Experts.

735 if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:

736 max_num_tokens = prepare_finalize.max_num_tokens_per_rank()

@@ -703,13 +740,11 @@ def make_humming_moe_kernel(

740 quant_config=moe_quant_config,

741 max_num_tokens=max_num_tokens,

742 num_dispatchers=prepare_finalize.num_dispatchers(),

706 - **extra_args,

743 )

744 else:

745 experts = experts_cls(

746 moe_config=moe_config,

747 quant_config=moe_quant_config,

712 - **extra_args,

748 )

749

750 kernel = mk.FusedMoEKernel(

@@ -822,16 +857,14 @@ def _prepare_and_transform_sublayer(

857 has_bias: bool,

858 num_experts: int,

859 param_dtype: torch.dtype,

825 -) -> None:

826 - """

827 - Prepare layer metadata and transform weights for a sublayer.

828 -

829 - This calls Humming's prepare_layer_meta and transform_humming_layer.

830 - """

831 - from vllm.utils.humming import HummingMethod

860 +) -> "LayerConfig":

861 + """Prepare Humming configuration and transform one sublayer's tensors."""

862 + from vllm.utils.humming import (

863 + prepare_layer_config,

864 + transform_humming_tensors,

865 + )

866

833 - HummingMethod.prepare_layer_meta(

834 - layer=layer,

867 + config = prepare_layer_config(

868 shape_n=shape_n,

869 shape_k=shape_k,

870 pad_n_to_multiple=256,

@@ -841,10 +874,13 @@ def _prepare_and_transform_sublayer(

874 has_bias=has_bias,

875 num_experts=num_experts,

876 torch_dtype=param_dtype,

844 - sublayer_name=sublayer_name,

877 )

846 -

847 - HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)

878 + tensors = transform_humming_tensors(

879 + config,

880 + _extract_sublayer_tensors(layer, sublayer_name),

881 + )

882 + _replace_layer_parameters(layer, sublayer_name, tensors)

883 + return config

884

885

886 def _process_single_sublayer(

@@ -858,7 +894,7 @@ def _process_single_sublayer(

894 num_experts: int,

895 param_dtype: torch.dtype,

896 force_weight_schema: Any | None = None,

861 -) -> tuple[Any, Any]:

897 +) -> tuple[Any, Any, "LayerConfig"]:

898 """

899 Process a single sublayer: convert, optionally requant, prepare, and transform.

900

@@ -878,7 +914,7 @@ def _process_single_sublayer(

914 force_weight_schema: Optional schema to force requantization to

915

916 Returns:

881 - Tuple of (final_weight_schema, final_input_schema)

917 + Tuple of the final weight schema, input schema, and Humming layer config.

918 """

919 from vllm.utils.humming import HummingWeightSchema

920

@@ -910,7 +946,7 @@ def _process_single_sublayer(

946 del tensors

947

948 # Step 3: Prepare layer metadata and transform weights

913 - _prepare_and_transform_sublayer(

949 + config = _prepare_and_transform_sublayer(

950 layer=layer,

951 sublayer_name=sublayer_name,

952 shape_n=shape_n,

@@ -922,7 +958,7 @@ def _process_single_sublayer(

958 param_dtype=param_dtype,

959 )

960

925 - return current_weight_schema, current_input_schema

961 + return current_weight_schema, current_input_schema, config

962

963

964 def convert_to_humming_moe_kernel_format(

@@ -932,7 +968,7 @@ def convert_to_humming_moe_kernel_format(

968 weight_schema: Any | None = None,

969 input_schema: Any | None = None,

970 force_weight_schema: Any | None = None,

935 -) -> None:

971 +) -> dict[str, "LayerConfig"]:

972 """

973 Convert MoE weights from checkpoint format to Humming kernel format.

974

@@ -959,6 +995,7 @@ def convert_to_humming_moe_kernel_format(

995 Side effects:

996 - Modifies layer parameters in place

997 - Sets layer.weight_schemas and layer.input_schemas

998 + - Sets layer.humming_configs for quant config construction

999 """

1000

1001 # Build schemas from quant_config if not provided

@@ -1005,25 +1042,27 @@ def convert_to_humming_moe_kernel_format(

1042

1043 layer.weight_schemas = {}

1044 layer.input_schemas = {}

1045 + humming_configs = {}

1046

1047 for sublayer_name, configs in sublayer_configs.items():

1010 - final_weight_schema, final_input_schema = _process_single_sublayer(

1011 - layer=layer,

1012 - sublayer_name=sublayer_name,

1013 - shape_n=configs["shape_n"],

1014 - shape_k=configs["shape_k"],

1015 - weight_schema=weight_schema,

1016 - input_schema=input_schema,

1017 - has_bias=has_bias,

1018 - num_experts=num_experts,

1019 - param_dtype=param_dtype,

1020 - force_weight_schema=force_weight_schema,

1048 + final_weight_schema, final_input_schema, humming_config = (

1049 + _process_single_sublayer(

1050 + layer=layer,

1051 + sublayer_name=sublayer_name,

1052 + shape_n=configs["shape_n"],

1053 + shape_k=configs["shape_k"],

1054 + weight_schema=weight_schema,

1055 + input_schema=input_schema,

1056 + has_bias=has_bias,

1057 + num_experts=num_experts,

1058 + param_dtype=param_dtype,

1059 + force_weight_schema=force_weight_schema,

1060 + )

1061 )

1062

1063 layer.weight_schemas[sublayer_name] = final_weight_schema

1064 layer.input_schemas[sublayer_name] = final_input_schema

1065 + humming_configs[sublayer_name] = humming_config

1066

1026 - if not hasattr(layer, "locks"):

1027 - device = layer.w13_weight.device

1028 - locks = torch.zeros(1024, dtype=torch.int32, device=device)

1029 - layer.register_buffer("locks", locks)

1067 + layer.humming_configs = humming_configs

1068 + return humming_configs

vllm/utils/humming.py

+6 -2

@@ -15,9 +15,13 @@ _EXPORTS: dict[str, str] = {

15 "dtypes": "humming.dtypes",

16 "DataType": "humming.dtypes:DataType",

17 "GemmType": "humming.config:GemmType",

18 + "LayerConfig": "humming.config:LayerConfig",

19 "WeightScaleType": "humming.config:WeightScaleType",

19 - "HummingMethod": "humming.layer:HummingMethod",

20 - "HummingLayerMeta": "humming.layer:HummingLayerMeta",

20 + "humming_forward": "humming.forward:humming_forward",

21 + "may_quant_input": "humming.forward:may_quant_input",

22 + "prepare_layer_config": "humming.transform:prepare_layer_config",

23 + "transform_humming_tensors": "humming.transform:transform_humming_tensors",

24 + "get_heuristics_config": "humming.tune:get_heuristics_config",

25 "BaseInputSchema": "humming.schema:BaseInputSchema",

26 "BaseWeightSchema": "humming.schema:BaseWeightSchema",

27 "HummingInputSchema": "humming.schema:HummingInputSchema",