A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
Move verify_marlin_supported to GPTQMarlinLinearMethod (#8165)
Michael Goin committed
Sep 5, 2024 at 11:09 UTC
2ee45281a5012072f41573eb09e1f82985adc761
1 file changed
+4
-4
vllm/model_executor/layers/quantization/gptq_marlin.py
+4
-4
@@ -51,10 +51,6 @@ class GPTQMarlinConfig(QuantizationConfig):
51
52
self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)]
53
54
- # Verify supported on platform.
55
- verify_marlin_supported(quant_type=self.quant_type,
56
- group_size=self.group_size)
57
-
54
def __repr__(self) -> str:
55
return (f"GPTQMarlinConfig(quant_type={self.quant_type}, "
56
f"group_size={self.group_size}, "
@@ -153,6 +149,10 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
149
def __init__(self, quant_config: GPTQMarlinConfig) -> None:
150
self.quant_config = quant_config
151
152
+ # Verify supported on platform.
153
+ verify_marlin_supported(quant_type=self.quant_config.quant_type,
154
+ group_size=self.quant_config.group_size)
155
+
156
def create_weights(
157
self,
158
layer: torch.nn.Module,