@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb5494676f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

enforce eager mode with bnb quantization temporarily (#6846)

chenqianfzh committed Jul 26, 2024 at 18:32 UTC bb5494676f5f57f1cf7cf72598de5434a2a22865
1 file changed +4
vllm/config.py
+4
@@ -282,6 +282,10 @@ class ModelConfig:
282 raise ValueError(
283 "BitAndBytes quantization with TP or PP is not supported yet.")
284
285 + if self.quantization == "bitsandbytes" and self.enforce_eager is False:
286 + raise ValueError(
287 + "BitAndBytes with enforce_eager = False is not supported yet.")
288 +
289 def get_hf_config_sliding_window(self) -> Optional[int]:
290 """Get the sliding window size, or None if disabled."""
291