A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Disable CUDA graph for SqueezeLLM (#2161)
Woosuk Kwon committed
Dec 17, 2023 at 10:24 UTC
6f41f0e377708f223871c888ce84bb575bae732f
1 file changed
+4
-3
vllm/config.py
+4
-3
@@ -185,10 +185,11 @@ class ModelConfig:
185
self.max_context_len_to_capture = self.max_model_len
186
self.max_context_len_to_capture = min(self.max_context_len_to_capture,
187
self.max_model_len)
188
- if self.quantization == "gptq" and not self.enforce_eager:
188
+ if (self.quantization in ["gptq", "squeezellm"]
189
+ and not self.enforce_eager):
190
# Related issue: https://github.com/vllm-project/vllm/issues/2147
190
- logger.warning("GPTQ does not support CUDA graph yet. Disabling "
191
- "CUDA graph.")
191
+ logger.warning(f"{self.quantization} does not support CUDA graph "
192
+ "yet. Disabling CUDA graph.")
193
self.enforce_eager = True
194
195
def verify_with_parallel_config(