@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0310029a2f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix awq_marlin and gptq_marlin flags (#6745)

Alexander Matveev committed Jul 25, 2024 at 01:34 UTC 0310029a2fc62171fae87155150326125e082a5a
2 files changed +5 -3
vllm/model_executor/layers/quantization/awq_marlin.py
+3 -2
@@ -25,7 +25,7 @@ class AWQMarlinConfig(QuantizationConfig):
25 def __init__(self, weight_bits: int, group_size: int, has_zp: bool,
26 lm_head_quantized: bool) -> None:
27 self.weight_bits = weight_bits
28 - self.pack_factor = 32 // self.weight_bits # packed into int32
28 + self.pack_factor = 32 // self.weight_bits # packed into 32bits
29 self.group_size = group_size
30 self.has_zp = has_zp
31 self.lm_head_quantized = lm_head_quantized
@@ -69,7 +69,8 @@ class AWQMarlinConfig(QuantizationConfig):
69 def override_quantization_method(cls, hf_quant_cfg,
70 user_quant) -> Optional[str]:
71 can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
72 - is_valid_user_quant = (user_quant is None or user_quant == "marlin")
72 + is_valid_user_quant = (user_quant is None or user_quant == "marlin"
73 + or user_quant == "awq_marlin")
74
75 if can_convert and is_valid_user_quant:
76 msg = ("The model is convertible to {} during runtime."
vllm/model_executor/layers/quantization/gptq_marlin.py
+2 -1
@@ -79,7 +79,8 @@ class GPTQMarlinConfig(QuantizationConfig):
79 user_quant) -> Optional[str]:
80 can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
81
82 - is_valid_user_quant = (user_quant is None or user_quant == "marlin")
82 + is_valid_user_quant = (user_quant is None or user_quant == "marlin"
83 + or user_quant == "gptq_marlin")
84
85 if can_convert and is_valid_user_quant:
86 msg = ("The model is convertible to {} during runtime."