A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Enable 4bit bnb prequant MOE (#21548)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Andy Chen committed
Aug 11, 2025 at 19:02 UTC
9b94d6ec8f5f2e8ec2d3897ed05fb2b13cc012da
2 files changed
+4
-8
vllm/model_executor/model_loader/bitsandbytes_loader.py
+3
-7
@@ -427,14 +427,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
427
elif isinstance(module, FusedMoE) and hasattr(
428
module.quant_method, "quant_config"):
429
# TODO: support FusedMoE with prequant and 8bit.
430
- if self.pre_quant:
430
+ if self.pre_quant and self.load_8bit:
431
raise ValueError(
432
- "Prequant BitsAndBytes models with FusedMoE is not "
433
- "supported yet.")
434
- if self.load_8bit:
435
- raise ValueError(
436
- "BitsAndBytes 8bit quantization with FusedMoE is not "
437
- "supported yet.")
432
+ "Prequant BitsAndBytes 8bit models with FusedMoE "
433
+ "is not supported yet.")
434
# Get the corresponding weight name using module name and
435
# expert_params_mapping.
436
vllm/model_executor/models/qwen3_moe.py
+1
-1
@@ -684,4 +684,4 @@ class Qwen3MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA,
684
return loader.load_weights(weights)
685
686
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
687
- return self.model.get_expert_mapping()
687
+ return self.model.get_expert_mapping()
\ No newline at end of file