A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] Automatically Detect SparseML models (#5119)
Robert Shaw committed
May 30, 2024 at 05:58 UTC
d910816c7356f4decd56eefb80e963b476cdf3e5
1 file changed
+14
-2
vllm/config.py
+14
-2
@@ -156,6 +156,17 @@ class ModelConfig:
156
self.embedding_mode = any(
157
ModelRegistry.is_embedding_model(arch) for arch in architectures)
158
159
+ def _parse_quant_hf_config(self):
160
+ quant_cfg = getattr(self.hf_config, "quantization_config", None)
161
+ if quant_cfg is None:
162
+ # SparseML uses a "compression_config" with a "quantization_config".
163
+ compression_cfg = getattr(self.hf_config, "compression_config",
164
+ None)
165
+ if compression_cfg is not None:
166
+ quant_cfg = compression_cfg.get("quantization_config", None)
167
+
168
+ return quant_cfg
169
+
170
def _verify_quantization(self) -> None:
171
supported_quantization = [*QUANTIZATION_METHODS]
172
rocm_supported_quantization = ["gptq", "squeezellm"]
@@ -163,12 +174,13 @@ class ModelConfig:
174
self.quantization = self.quantization.lower()
175
176
# Parse quantization method from the HF model config, if available.
166
- quant_cfg = getattr(self.hf_config, "quantization_config", None)
177
+ quant_cfg = self._parse_quant_hf_config()
178
+
179
if quant_cfg is not None:
180
quant_method = quant_cfg.get("quant_method", "").lower()
181
182
# Detect which checkpoint is it
171
- for name, method in QUANTIZATION_METHODS.items():
183
+ for _, method in QUANTIZATION_METHODS.items():
184
quantization_override = method.override_quantization_method(
185
quant_cfg, self.quantization)
186
if quantization_override: