A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
enforce eager mode with bnb quantization temporarily (#6846)
chenqianfzh committed
Jul 26, 2024 at 18:32 UTC
bb5494676f5f57f1cf7cf72598de5434a2a22865
1 file changed
+4
vllm/config.py
+4
@@ -282,6 +282,10 @@ class ModelConfig:
282
raise ValueError(
283
"BitAndBytes quantization with TP or PP is not supported yet.")
284
285
+ if self.quantization == "bitsandbytes" and self.enforce_eager is False:
286
+ raise ValueError(
287
+ "BitAndBytes with enforce_eager = False is not supported yet.")
288
+
289
def get_hf_config_sliding_window(self) -> Optional[int]:
290
"""Get the sliding window size, or None if disabled."""
291