@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ee45281a5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

Move verify_marlin_supported to GPTQMarlinLinearMethod (#8165)

Michael Goin committed Sep 5, 2024 at 11:09 UTC 2ee45281a5012072f41573eb09e1f82985adc761
1 file changed +4 -4
vllm/model_executor/layers/quantization/gptq_marlin.py
+4 -4
@@ -51,10 +51,6 @@ class GPTQMarlinConfig(QuantizationConfig):
51
52 self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)]
53
54 - # Verify supported on platform.
55 - verify_marlin_supported(quant_type=self.quant_type,
56 - group_size=self.group_size)
57 -
54 def __repr__(self) -> str:
55 return (f"GPTQMarlinConfig(quant_type={self.quant_type}, "
56 f"group_size={self.group_size}, "
@@ -153,6 +149,10 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
149 def __init__(self, quant_config: GPTQMarlinConfig) -> None:
150 self.quant_config = quant_config
151
152 + # Verify supported on platform.
153 + verify_marlin_supported(quant_type=self.quant_config.quant_type,
154 + group_size=self.quant_config.group_size)
155 +
156 def create_weights(
157 self,
158 layer: torch.nn.Module,