A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Misc] Improve the readability of BNB error messages (#12320)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jan 23, 2025 at 00:56 UTC
84bee4bd5c41896d626186c9265f30824b928f7a
1 file changed
+2
-2
vllm/model_executor/model_loader/loader.py
+2
-2
@@ -1076,8 +1076,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1076
# weight tensor. So TP does not work with pre_quantized bnb models.
1077
if pre_quant and get_tensor_model_parallel_world_size() > 1:
1078
raise ValueError(
1079
- "Prequant BitsAndBytes models with TP is not supported."
1080
- "Please try with PP.")
1079
+ "Prequant BitsAndBytes models with tensor parallelism is not "
1080
+ "supported. Please try with pipeline parallelism.")
1081
1082
load_8bit = False
1083
if pre_quant: