@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d910816c73
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Automatically Detect SparseML models (#5119)

Robert Shaw committed May 30, 2024 at 05:58 UTC d910816c7356f4decd56eefb80e963b476cdf3e5
1 file changed +14 -2
vllm/config.py
+14 -2
@@ -156,6 +156,17 @@ class ModelConfig:
156 self.embedding_mode = any(
157 ModelRegistry.is_embedding_model(arch) for arch in architectures)
158
159 + def _parse_quant_hf_config(self):
160 + quant_cfg = getattr(self.hf_config, "quantization_config", None)
161 + if quant_cfg is None:
162 + # SparseML uses a "compression_config" with a "quantization_config".
163 + compression_cfg = getattr(self.hf_config, "compression_config",
164 + None)
165 + if compression_cfg is not None:
166 + quant_cfg = compression_cfg.get("quantization_config", None)
167 +
168 + return quant_cfg
169 +
170 def _verify_quantization(self) -> None:
171 supported_quantization = [*QUANTIZATION_METHODS]
172 rocm_supported_quantization = ["gptq", "squeezellm"]
@@ -163,12 +174,13 @@ class ModelConfig:
174 self.quantization = self.quantization.lower()
175
176 # Parse quantization method from the HF model config, if available.
166 - quant_cfg = getattr(self.hf_config, "quantization_config", None)
177 + quant_cfg = self._parse_quant_hf_config()
178 +
179 if quant_cfg is not None:
180 quant_method = quant_cfg.get("quant_method", "").lower()
181
182 # Detect which checkpoint is it
171 - for name, method in QUANTIZATION_METHODS.items():
183 + for _, method in QUANTIZATION_METHODS.items():
184 quantization_override = method.override_quantization_method(
185 quant_cfg, self.quantization)
186 if quantization_override: