A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Misc] Clean up the BitsAndBytes arguments (#15140)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 21, 2025 at 10:17 UTC
10f55fe6c5a7836f173ab93446f8a298d9fb00f6
3 files changed
+7
-15
docs/source/features/quantization/bnb.md
+3
-3
@@ -25,7 +25,7 @@ import torch
25
# unsloth/tinyllama-bnb-4bit is a pre-quantized checkpoint.
26
model_id = "unsloth/tinyllama-bnb-4bit"
27
llm = LLM(model=model_id, dtype=torch.bfloat16, trust_remote_code=True, \
28
-quantization="bitsandbytes", load_format="bitsandbytes")
28
+quantization="bitsandbytes")
29
```
30
31
## Inflight quantization: load as 4bit quantization
@@ -35,7 +35,7 @@ from vllm import LLM
35
import torch
36
model_id = "huggyllama/llama-7b"
37
llm = LLM(model=model_id, dtype=torch.bfloat16, trust_remote_code=True, \
38
-quantization="bitsandbytes", load_format="bitsandbytes")
38
+quantization="bitsandbytes")
39
```
40
41
## OpenAI Compatible Server
@@ -43,5 +43,5 @@ quantization="bitsandbytes", load_format="bitsandbytes")
43
Append the following to your 4bit model arguments:
44
45
```console
46
---quantization bitsandbytes --load-format bitsandbytes
46
+--quantization bitsandbytes
47
```
examples/offline_inference/lora_with_quantization_inference.py
-1
@@ -83,7 +83,6 @@ def initialize_engine(model: str, quantization: str,
83
engine_args = EngineArgs(model=model,
84
quantization=quantization,
85
qlora_adapter_name_or_path=lora_repo,
86
- load_format="bitsandbytes",
86
enable_lora=True,
87
max_lora_rank=64)
88
else:
vllm/engine/arg_utils.py
+4
-11
@@ -1170,22 +1170,15 @@ class EngineArgs:
1170
)
1171
1172
def create_load_config(self) -> LoadConfig:
1173
- # bitsandbytes quantization needs a specific model loader
1174
- # so we make sure the quant method and the load format are consistent
1175
- if (self.quantization == "bitsandbytes" or
1176
- self.qlora_adapter_name_or_path is not None) and \
1177
- self.load_format != "bitsandbytes":
1178
- raise ValueError(
1179
- "BitsAndBytes quantization and QLoRA adapter only support "
1180
- f"'bitsandbytes' load format, but got {self.load_format}")
1173
1182
- if (self.load_format == "bitsandbytes" or
1183
- self.qlora_adapter_name_or_path is not None) and \
1174
+ if(self.qlora_adapter_name_or_path is not None) and \
1175
self.quantization != "bitsandbytes":
1176
raise ValueError(
1186
- "BitsAndBytes load format and QLoRA adapter only support "
1177
+ "QLoRA adapter only support "
1178
f"'bitsandbytes' quantization, but got {self.quantization}")
1179
1180
+ if self.quantization == "bitsandbytes":
1181
+ self.load_format = "bitsandbytes"
1182
return LoadConfig(
1183
load_format=self.load_format,
1184
download_dir=self.download_dir,