A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Quantization] Bump to use latest bitsandbytes (#20424)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jul 3, 2025 at 21:58 UTC
1819fbda638d4e10512570ebad0e6b16661238d8
8 files changed
+14
-14
docker/Dockerfile
+1
-1
@@ -498,7 +498,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
498
if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
499
uv pip install --system accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
500
else \
501
- uv pip install --system accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.3' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
501
+ uv pip install --system accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.46.1' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
502
fi
503
504
ENV VLLM_USAGE_SOURCE production-docker-image
docs/features/quantization/bnb.md
+1
-1
@@ -10,7 +10,7 @@ Compared to other quantization methods, BitsAndBytes eliminates the need for cal
10
Below are the steps to utilize BitsAndBytes with vLLM.
11
12
```bash
13
-pip install bitsandbytes>=0.45.3
13
+pip install bitsandbytes>=0.46.1
14
```
15
16
vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.
requirements/nightly_torch_test.txt
+1
-1
@@ -34,7 +34,7 @@ tokenizers==0.21.1
34
huggingface-hub[hf_xet]>=0.30.0 # Required for Xet downloads.
35
schemathesis>=3.39.15 # Required for openai schema test.
36
# quantization
37
-bitsandbytes>=0.45.3
37
+bitsandbytes>=0.46.1
38
buildkite-test-collector==0.1.9
39
40
requirements/test.in
+1
-1
@@ -39,7 +39,7 @@ tokenizers==0.21.1
39
huggingface-hub[hf_xet]>=0.33.0 # Required for Xet downloads.
40
schemathesis>=3.39.15 # Required for openai schema test.
41
# quantization
42
-bitsandbytes>=0.45.3
42
+bitsandbytes==0.46.1
43
buildkite-test-collector==0.1.9
44
45
requirements/test.txt
+1
-1
@@ -45,7 +45,7 @@ backoff==2.2.1
45
# via
46
# -r requirements/test.in
47
# schemathesis
48
-bitsandbytes==0.45.3
48
+bitsandbytes==0.46.1
49
# via -r requirements/test.in
50
black==24.10.0
51
# via datamodel-code-generator
vllm/config.py
+1
-1
@@ -969,7 +969,7 @@ class ModelConfig:
969
970
def _verify_bnb_config(self) -> None:
971
"""
972
- The current version of bitsandbytes (0.45.3) with 8-bit models does not
972
+ The current version of bitsandbytes (0.46.1) with 8-bit models does not
973
yet support CUDA graph.
974
# TODO Remove this when bitsandbytes supports.
975
"""
vllm/model_executor/layers/quantization/bitsandbytes.py
+4
-4
@@ -156,12 +156,12 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
156
def __init__(self, quant_config: BitsAndBytesConfig):
157
try:
158
import bitsandbytes
159
- if bitsandbytes.__version__ < "0.45.3":
159
+ if bitsandbytes.__version__ < "0.46.1":
160
raise ImportError("bitsandbytes version is wrong. Please "
161
- "install bitsandbytes>=0.45.3.")
161
+ "install bitsandbytes>=0.46.1.")
162
except ImportError as err:
163
- raise ImportError("Please install bitsandbytes>=0.45.3 via "
164
- "`pip install bitsandbytes>=0.45.3` to use "
163
+ raise ImportError("Please install bitsandbytes>=0.46.1 via "
164
+ "`pip install bitsandbytes>=0.46.1` to use "
165
"bitsandbytes quantizer.") from err
166
167
self.quant_config = quant_config
vllm/model_executor/model_loader/bitsandbytes_loader.py
+4
-4
@@ -183,12 +183,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
183
try:
184
import bitsandbytes
185
186
- if bitsandbytes.__version__ < "0.45.3":
186
+ if bitsandbytes.__version__ < "0.46.1":
187
raise ImportError("bitsandbytes version is wrong. Please "
188
- "install bitsandbytes>=0.45.3.")
188
+ "install bitsandbytes>=0.46.1.")
189
except ImportError as err:
190
- raise ImportError("Please install bitsandbytes>=0.45.3 via "
191
- "`pip install bitsandbytes>=0.45.3` to use "
190
+ raise ImportError("Please install bitsandbytes>=0.46.1 via "
191
+ "`pip install bitsandbytes>=0.46.1` to use "
192
"bitsandbytes quantizer.") from err
193
194
hf_weights_files, use_safetensors = self._prepare_weights(