A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Misc] Upgrade BNB version (#15183)
Jee Jee Li committed
Mar 24, 2025 at 13:51 UTC
3892e58ad7c0f325c9257cb3468aa1cde9e1ed6f
4 files changed
+10
-10
Dockerfile
+1
-1
@@ -286,7 +286,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
286
if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
287
uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.42.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
288
else \
289
- uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
289
+ uv pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.3' 'timm==0.9.10' boto3 runai-model-streamer runai-model-streamer[s3]; \
290
fi
291
292
ENV VLLM_USAGE_SOURCE production-docker-image
docs/source/features/quantization/bnb.md
+1
-1
@@ -9,7 +9,7 @@ Compared to other quantization methods, BitsAndBytes eliminates the need for cal
9
Below are the steps to utilize BitsAndBytes with vLLM.
10
11
```console
12
-pip install bitsandbytes>=0.45.0
12
+pip install bitsandbytes>=0.45.3
13
```
14
15
vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.
vllm/model_executor/layers/quantization/bitsandbytes.py
+4
-4
@@ -155,12 +155,12 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
155
def __init__(self, quant_config: BitsAndBytesConfig):
156
try:
157
import bitsandbytes
158
- if bitsandbytes.__version__ < "0.45.0":
158
+ if bitsandbytes.__version__ < "0.45.3":
159
raise ImportError("bitsandbytes version is wrong. Please "
160
- "install bitsandbytes>=0.45.0.")
160
+ "install bitsandbytes>=0.45.3.")
161
except ImportError as err:
162
- raise ImportError("Please install bitsandbytes>=0.45.0 via "
163
- "`pip install bitsandbytes>=0.45.0` to use "
162
+ raise ImportError("Please install bitsandbytes>=0.45.3 via "
163
+ "`pip install bitsandbytes>=0.45.3` to use "
164
"bitsandbytes quantizer.") from err
165
166
self.quant_config = quant_config
vllm/model_executor/model_loader/loader.py
+4
-4
@@ -862,12 +862,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
862
try:
863
import bitsandbytes
864
865
- if bitsandbytes.__version__ < "0.45.0":
865
+ if bitsandbytes.__version__ < "0.45.3":
866
raise ImportError("bitsandbytes version is wrong. Please "
867
- "install bitsandbytes>=0.45.0.")
867
+ "install bitsandbytes>=0.45.3.")
868
except ImportError as err:
869
- raise ImportError("Please install bitsandbytes>=0.45.0 via "
870
- "`pip install bitsandbytes>=0.45.0` to use "
869
+ raise ImportError("Please install bitsandbytes>=0.45.3 via "
870
+ "`pip install bitsandbytes>=0.45.3` to use "
871
"bitsandbytes quantizer.") from err
872
873
hf_weights_files, use_safetensors = self._prepare_weights(