A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Temporarily enforce eager mode for GPTQ models (#2154)
Woosuk Kwon committed
Dec 17, 2023 at 01:51 UTC
3a765bd5e1891b8c6454e60b56c2405fbe35bb9e
1 file changed
+5
vllm/config.py
+5
@@ -185,6 +185,11 @@ class ModelConfig:
185
self.max_context_len_to_capture = self.max_model_len
186
self.max_context_len_to_capture = min(self.max_context_len_to_capture,
187
self.max_model_len)
188
+ if self.quantization == "gptq" and not self.enforce_eager:
189
+ # Related issue: https://github.com/vllm-project/vllm/issues/2147
190
+ logger.warning("GPTQ does not support CUDA graph yet. Disabling "
191
+ "CUDA graph.")
192
+ self.enforce_eager = True
193
194
def verify_with_parallel_config(
195
self,