A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Temporarily resolve the error of BitAndBytes (#7308)
Jee Jee Li committed
Aug 9, 2024 at 04:42 UTC
a049b107e207db796817fb83c4536e0625531d54
1 file changed
+3
-2
vllm/config.py
+3
-2
@@ -322,8 +322,9 @@ class ModelConfig:
322
"BitAndBytes quantization with TP or PP is not supported yet.")
323
324
if self.quantization == "bitsandbytes" and self.enforce_eager is False:
325
- raise ValueError(
326
- "BitAndBytes with enforce_eager = False is not supported yet.")
325
+ logger.warning("CUDA graph is not supported on BitAndBytes yet, "
326
+ "fallback to the eager mode.")
327
+ self.enforce_eager = True
328
329
def get_hf_config_sliding_window(self) -> Optional[int]:
330
"""Get the sliding window size, or None if disabled."""