@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6f41f0e377
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Disable CUDA graph for SqueezeLLM (#2161)

Woosuk Kwon committed Dec 17, 2023 at 10:24 UTC 6f41f0e377708f223871c888ce84bb575bae732f
1 file changed +4 -3
vllm/config.py
+4 -3
@@ -185,10 +185,11 @@ class ModelConfig:
185 self.max_context_len_to_capture = self.max_model_len
186 self.max_context_len_to_capture = min(self.max_context_len_to_capture,
187 self.max_model_len)
188 - if self.quantization == "gptq" and not self.enforce_eager:
188 + if (self.quantization in ["gptq", "squeezellm"]
189 + and not self.enforce_eager):
190 # Related issue: https://github.com/vllm-project/vllm/issues/2147
190 - logger.warning("GPTQ does not support CUDA graph yet. Disabling "
191 - "CUDA graph.")
191 + logger.warning(f"{self.quantization} does not support CUDA graph "
192 + "yet. Disabling CUDA graph.")
193 self.enforce_eager = True
194
195 def verify_with_parallel_config(