A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 04:45 UTC
[BugFix] Fix Humming MoE deploy error (#43100)
Signed-off-by: Alireza Dadgarnia <dadgarnia@Alirezas-MacBook-Pro-2.local> Signed-off-by: Alireza Dadgarnia <49554709+adotdad@users.noreply.github.com> Co-authored-by: Alireza Dadgarnia <dadgarnia@Alirezas-MacBook-Pro-2.local> Co-authored-by: Jinzhen Lin <linjinzhen@hotmail.com>
Alireza Dadgarnia committed
Jun 2, 2026 at 20:02 UTC
afcb5807159fdc0de6866a8a6c63c1d7597aa73a
1 file changed
+3
-2
vllm/model_executor/layers/quantization/humming.py
+3
-2
@@ -810,8 +810,8 @@ class HummingMoEMethod(FusedMoEMethodBase):
810
param = torch.nn.Parameter(tensor, requires_grad=False)
811
setattr(layer, name, param)
812
813
- layer.weight_schemas[sublayer_name] = weight_schema
814
- layer.input_schemas[sublayer_name] = input_schema
813
+ layer.weight_schemas[sublayer_name] = weight_schema
814
+ layer.input_schemas[sublayer_name] = input_schema
815
816
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
817
assert isinstance(weight_schema, HummingWeightSchema)
@@ -865,6 +865,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
865
866
# use moe modular
867
experts: HummingIndexedExperts | HummingGroupedExperts
868
+ layer.ensure_moe_quant_config_init()
869
assert self.moe_quant_config is not None
870
if get_humming_moe_gemm_type() == "indexed":
871
experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config)