A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix] Lazily import the humming quantization backend (#44921)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
Michael Goin committed
Jun 10, 2026 at 09:06 UTC
87c15d46e3a3807258eb5f859875714c78ba173d
5 files changed
+86
-52
vllm/model_executor/kernels/linear/mixed_precision/humming.py
+3
-3
@@ -5,7 +5,7 @@
5
import torch
6
7
from vllm.platforms import current_platform
8
-from vllm.utils.import_utils import _has_module
8
+from vllm.utils.import_utils import has_humming
9
10
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
11
@@ -19,7 +19,7 @@ class HummingLinearKernel(MPLinearKernel):
19
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
20
if not current_platform.is_cuda():
21
return False, "Humming is only supported on CUDA"
22
- if not _has_module("humming"):
22
+ if not has_humming():
23
return False, "Humming is not installed"
24
if c.has_g_idx:
25
return False, "Humming does not support act-order (g_idx)"
@@ -50,7 +50,7 @@ class HummingLinearKernel(MPLinearKernel):
50
x: torch.Tensor,
51
bias: torch.Tensor | None = None,
52
) -> torch.Tensor:
53
- from humming.layer import HummingMethod
53
+ from vllm.utils.humming import HummingMethod
54
55
flatten_inputs = x.view(-1, x.size(-1))
56
output = HummingMethod.forward_layer(
vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
+2
-3
@@ -7,9 +7,6 @@ import math
7
from typing import TYPE_CHECKING, Any
8
9
import torch
10
-from humming import dtypes
11
-from humming.config import GemmType as HummingGemmType
12
-from humming.layer import HummingLayerMeta, HummingMethod
10
11
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
12
from vllm import envs
@@ -41,6 +38,8 @@ from vllm.model_executor.layers.fused_moe.utils import (
38
)
39
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
40
from vllm.platforms import current_platform
41
+from vllm.utils.humming import GemmType as HummingGemmType
42
+from vllm.utils.humming import HummingLayerMeta, HummingMethod, dtypes
43
from vllm.v1.worker.workspace import current_workspace_manager
44
45
if TYPE_CHECKING:
vllm/model_executor/layers/quantization/humming.py
+38
-44
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
8
import regex as re
9
import torch
10
11
+import vllm.utils.humming as _hm
12
from vllm import envs
13
from vllm.model_executor.layers.fused_moe import (
14
FusedMoEConfig,
@@ -41,37 +42,16 @@ from vllm.model_executor.parameter import (
42
RowvLLMParameter,
43
)
44
from vllm.model_executor.utils import set_weight_attrs
44
-from vllm.platforms import current_platform
45
-from vllm.utils.import_utils import has_humming
46
-
47
-if has_humming() and current_platform.is_cuda():
48
- from humming.dtypes import DataType
49
- from humming.layer import HummingMethod
50
- from humming.schema import (
51
- BaseInputSchema,
52
- BaseWeightSchema,
53
- HummingInputSchema,
54
- HummingWeightSchema,
55
- )
56
- from humming.utils.weight import quantize_weight
57
-
58
- from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
59
- BatchedHummingGroupedExperts,
60
- HummingGroupedExperts,
61
- HummingIndexedExperts,
62
- get_humming_moe_gemm_type,
63
- )
45
46
if TYPE_CHECKING:
66
- from humming.schema import (
47
+ from vllm.model_executor.models.utils import WeightsMapper
48
+ from vllm.utils.humming import (
49
BaseInputSchema,
50
BaseWeightSchema,
51
HummingInputSchema,
52
HummingWeightSchema,
53
)
54
73
- from vllm.model_executor.models.utils import WeightsMapper
74
-
55
56
def prepare_padded_shape(shape, x):
57
padded_shape = math.ceil(shape / x) * x
@@ -265,7 +245,7 @@ class HummingConfig(QuantizationConfig):
245
break
246
247
if "quant_method" in layer_config:
268
- return BaseWeightSchema.from_config(layer_config)
248
+ return _hm.BaseWeightSchema.from_config(layer_config)
249
return None
250
251
def get_layer_input_schema(self, config: dict[str, Any], prefix: str):
@@ -277,8 +257,8 @@ class HummingConfig(QuantizationConfig):
257
return None
258
config = group_config
259
280
- if config.get("quant_method", None) in BaseInputSchema.INPUT_SCHEMA_MAP:
281
- return BaseInputSchema.from_config(config)
260
+ if config.get("quant_method", None) in _hm.BaseInputSchema.INPUT_SCHEMA_MAP:
261
+ return _hm.BaseInputSchema.from_config(config)
262
return None
263
264
def get_quant_config_for_layer(
@@ -316,7 +296,7 @@ class HummingConfig(QuantizationConfig):
296
input_schema = force_input_schema
297
298
if force_weight_schema is not None and force_input_schema is None:
319
- force_input_schema = HummingInputSchema()
299
+ force_input_schema = _hm.HummingInputSchema()
300
301
return HummingLayerQuantizationConfig(
302
weight_schema=weight_schema,
@@ -360,7 +340,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
340
):
341
self.weight_schema = weight_schema
342
if input_schema is None:
363
- input_schema = HummingInputSchema()
343
+ input_schema = _hm.HummingInputSchema()
344
self.input_schema = input_schema
345
self.force_weight_schema = force_weight_schema
346
self.force_input_schema = force_input_schema
@@ -368,7 +348,7 @@ class HummingLayerQuantizationConfig(HummingConfig):
348
349
@classmethod
350
def from_config(cls, config):
371
- weight_schema = BaseWeightSchema.from_config(config)
351
+ weight_schema = _hm.BaseWeightSchema.from_config(config)
352
return cls(weight_schema)
353
354
def get_quant_method(
@@ -397,10 +377,10 @@ class HummingLinearMethod(LinearMethodBase):
377
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
378
if is_unquantized and self.is_online_quant:
379
# online quant (fp16/bf16 -> quant_type)
400
- assert isinstance(self.weight_schema, HummingWeightSchema)
401
- f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
380
+ assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
381
+ f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
382
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
403
- tensor_list = quantize_weight(
383
+ tensor_list = _hm.quantize_weight(
384
weight=loaded_weight,
385
dtype=self.weight_schema.b_dtype,
386
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
@@ -531,7 +511,7 @@ class HummingLinearMethod(LinearMethodBase):
511
return None
512
513
# convert from checkpoint format to humming format
534
- if not isinstance(self.weight_schema, HummingWeightSchema):
514
+ if not isinstance(self.weight_schema, _hm.HummingWeightSchema):
515
self.weight_schema, tensors = self.weight_schema.convert_humming(
516
tensors=layer.state_dict(),
517
shape_n_stacks=layer.output_partition_sizes,
@@ -556,7 +536,7 @@ class HummingLinearMethod(LinearMethodBase):
536
del tensors
537
538
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
559
- assert isinstance(self.weight_schema, HummingWeightSchema)
539
+ assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
540
force_requant = self.force_weight_schema is not None
541
if force_requant and self.weight_schema != self.force_weight_schema:
542
tensors = self.weight_schema.requant_tensors(
@@ -578,7 +558,7 @@ class HummingLinearMethod(LinearMethodBase):
558
del tensors
559
560
# prepare layer config from humming kernel
581
- HummingMethod.prepare_layer_meta(
561
+ _hm.HummingMethod.prepare_layer_meta(
562
layer=layer,
563
shape_n=layer.output_partition_sizes_sum,
564
shape_k=layer.input_size_per_partition,
@@ -591,7 +571,7 @@ class HummingLinearMethod(LinearMethodBase):
571
)
572
573
# preprocess weight for inference
594
- HummingMethod.transform_humming_layer(layer)
574
+ _hm.HummingMethod.transform_humming_layer(layer)
575
576
# compute_config: kernel configs that do not directly affect weights
577
# but significantly impact kernel behavior or computation precision.
@@ -610,7 +590,7 @@ class HummingLinearMethod(LinearMethodBase):
590
bias: torch.Tensor | None = None,
591
) -> torch.Tensor:
592
flatten_inputs = x.view(-1, x.size(-1))
613
- output = HummingMethod.forward_layer(
593
+ output = _hm.HummingMethod.forward_layer(
594
layer=layer,
595
inputs=flatten_inputs,
596
compute_config=self.compute_config,
@@ -645,10 +625,10 @@ class HummingMoEMethod(FusedMoEMethodBase):
625
is_unquantized = name == "weight" and loaded_weight.dtype in float_dtypes
626
# online quant (fp16/bf16 -> quant_type)
627
if is_unquantized:
648
- assert isinstance(self.weight_schema, HummingWeightSchema)
649
- f16_dtype = DataType.from_torch_dtype(layer.param_dtype)
628
+ assert isinstance(self.weight_schema, _hm.HummingWeightSchema)
629
+ f16_dtype = _hm.DataType.from_torch_dtype(layer.param_dtype)
630
has_global_scale = "TENSOR" in str(self.weight_schema.weight_scale_type)
651
- tensor_list = quantize_weight(
631
+ tensor_list = _hm.quantize_weight(
632
weight=loaded_weight,
633
dtype=self.weight_schema.b_dtype,
634
scale_dtype=self.weight_schema.bs_dtype or f16_dtype,
@@ -771,7 +751,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
751
input_schema = self.input_schema
752
weight_schema = self.weight_schema
753
# convert from checkpoint format to humming format
774
- if not isinstance(weight_schema, HummingWeightSchema):
754
+ if not isinstance(weight_schema, _hm.HummingWeightSchema):
755
tensors: dict[str, torch.Tensor] = dict(
756
(key.removeprefix(sublayer_name + "_"), value)
757
for key, value in layer.state_dict().items()
@@ -813,7 +793,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
793
layer.input_schemas[sublayer_name] = input_schema
794
795
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
816
- assert isinstance(weight_schema, HummingWeightSchema)
796
+ assert isinstance(weight_schema, _hm.HummingWeightSchema)
797
force_requant = self.force_weight_schema is not None
798
if force_requant and weight_schema != self.force_weight_schema:
799
tensors = dict(
@@ -845,7 +825,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
825
del tensors
826
827
# prepare layer config from humming kernel
848
- HummingMethod.prepare_layer_meta(
828
+ _hm.HummingMethod.prepare_layer_meta(
829
layer=layer,
830
shape_n=configs["shape_n"],
831
shape_k=configs["shape_k"],
@@ -860,7 +840,15 @@ class HummingMoEMethod(FusedMoEMethodBase):
840
)
841
842
# preprocess weight for inference
863
- HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
843
+ _hm.HummingMethod.transform_humming_layer(
844
+ layer, sublayer_name=sublayer_name
845
+ )
846
+
847
+ from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
848
+ HummingGroupedExperts,
849
+ HummingIndexedExperts,
850
+ get_humming_moe_gemm_type,
851
+ )
852
853
# use moe modular
854
experts: HummingIndexedExperts | HummingGroupedExperts
@@ -878,6 +866,12 @@ class HummingMoEMethod(FusedMoEMethodBase):
866
layer: torch.nn.Module,
867
):
868
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
869
+ from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
870
+ BatchedHummingGroupedExperts,
871
+ HummingGroupedExperts,
872
+ HummingIndexedExperts,
873
+ get_humming_moe_gemm_type,
874
+ )
875
876
activation_format = prepare_finalize.activation_format
877
assert self.moe_quant_config is not None
vllm/model_executor/layers/quantization/utils/humming_utils.py
+1
-2
@@ -5,8 +5,6 @@ from typing import Any
5
6
import regex as re
7
import torch
8
-from humming.layer import HummingInputSchema, HummingMethod
9
-from humming.schema import BaseWeightSchema
8
9
from vllm import envs
10
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -16,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
14
)
15
from vllm.model_executor.layers.linear import LinearBase
16
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
17
+from vllm.utils.humming import BaseWeightSchema, HummingInputSchema, HummingMethod
18
19
20
def humming_is_layer_skipped(config: dict[str, Any], prefix: str):
vllm/utils/humming.py
new
+42
@@ -0,0 +1,42 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Lazy facade for the optional ``humming`` package.
4
+
5
+vLLM code should import humming symbols from here so that ``import humming``
6
+(which has import-time side effects) is deferred until first use. Add new
7
+symbols by appending one entry to ``_EXPORTS`` as ``"module.path:attr"``,
8
+or ``"module.path"`` for a whole-module re-export.
9
+"""
10
+
11
+import importlib
12
+from typing import Any
13
+
14
+_EXPORTS: dict[str, str] = {
15
+ "dtypes": "humming.dtypes",
16
+ "DataType": "humming.dtypes:DataType",
17
+ "GemmType": "humming.config:GemmType",
18
+ "HummingMethod": "humming.layer:HummingMethod",
19
+ "HummingLayerMeta": "humming.layer:HummingLayerMeta",
20
+ "BaseInputSchema": "humming.schema:BaseInputSchema",
21
+ "BaseWeightSchema": "humming.schema:BaseWeightSchema",
22
+ "HummingInputSchema": "humming.schema:HummingInputSchema",
23
+ "HummingWeightSchema": "humming.schema:HummingWeightSchema",
24
+ "quantize_weight": "humming.utils.weight:quantize_weight",
25
+}
26
+
27
+
28
+def __getattr__(name: str) -> Any:
29
+ spec = _EXPORTS.get(name)
30
+ if spec is None:
31
+ raise AttributeError(f"module 'vllm.utils.humming' has no attribute {name!r}")
32
+ if ":" in spec:
33
+ mod_path, attr = spec.split(":", 1)
34
+ obj = getattr(importlib.import_module(mod_path), attr)
35
+ else:
36
+ obj = importlib.import_module(spec)
37
+ globals()[name] = obj
38
+ return obj
39
+
40
+
41
+def __dir__() -> list[str]:
42
+ return sorted({*globals(), *_EXPORTS})