@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a049b107e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Temporarily resolve the error of BitAndBytes (#7308)

Jee Jee Li committed Aug 9, 2024 at 04:42 UTC a049b107e207db796817fb83c4536e0625531d54
1 file changed +3 -2
vllm/config.py
+3 -2
@@ -322,8 +322,9 @@ class ModelConfig:
322 "BitAndBytes quantization with TP or PP is not supported yet.")
323
324 if self.quantization == "bitsandbytes" and self.enforce_eager is False:
325 - raise ValueError(
326 - "BitAndBytes with enforce_eager = False is not supported yet.")
325 + logger.warning("CUDA graph is not supported on BitAndBytes yet, "
326 + "fallback to the eager mode.")
327 + self.enforce_eager = True
328
329 def get_hf_config_sliding_window(self) -> Optional[int]:
330 """Get the sliding window size, or None if disabled."""