A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix awq_marlin and gptq_marlin flags (#6745)
Alexander Matveev committed
Jul 25, 2024 at 01:34 UTC
0310029a2fc62171fae87155150326125e082a5a
2 files changed
+5
-3
vllm/model_executor/layers/quantization/awq_marlin.py
+3
-2
@@ -25,7 +25,7 @@ class AWQMarlinConfig(QuantizationConfig):
25
def __init__(self, weight_bits: int, group_size: int, has_zp: bool,
26
lm_head_quantized: bool) -> None:
27
self.weight_bits = weight_bits
28
- self.pack_factor = 32 // self.weight_bits # packed into int32
28
+ self.pack_factor = 32 // self.weight_bits # packed into 32bits
29
self.group_size = group_size
30
self.has_zp = has_zp
31
self.lm_head_quantized = lm_head_quantized
@@ -69,7 +69,8 @@ class AWQMarlinConfig(QuantizationConfig):
69
def override_quantization_method(cls, hf_quant_cfg,
70
user_quant) -> Optional[str]:
71
can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
72
- is_valid_user_quant = (user_quant is None or user_quant == "marlin")
72
+ is_valid_user_quant = (user_quant is None or user_quant == "marlin"
73
+ or user_quant == "awq_marlin")
74
75
if can_convert and is_valid_user_quant:
76
msg = ("The model is convertible to {} during runtime."
vllm/model_executor/layers/quantization/gptq_marlin.py
+2
-1
@@ -79,7 +79,8 @@ class GPTQMarlinConfig(QuantizationConfig):
79
user_quant) -> Optional[str]:
80
can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
81
82
- is_valid_user_quant = (user_quant is None or user_quant == "marlin")
82
+ is_valid_user_quant = (user_quant is None or user_quant == "marlin"
83
+ or user_quant == "gptq_marlin")
84
85
if can_convert and is_valid_user_quant:
86
msg = ("The model is convertible to {} during runtime."