A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[[Misc]Upgrade bitsandbytes to the latest version 0.45.0 (#11201)
Jee Jee Li committed
Dec 15, 2024 at 11:03 UTC
15859f2357059ef488405e5336d2c6e5d246687b
5 files changed
+8
-8
Dockerfile
+1
-1
@@ -218,7 +218,7 @@ FROM vllm-base AS vllm-openai
218
219
# install additional dependencies for openai api server
220
RUN --mount=type=cache,target=/root/.cache/pip \
221
- pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.44.0' timm==0.9.10
221
+ pip install accelerate hf_transfer 'modelscope!=1.15.0' 'bitsandbytes>=0.45.0' timm==0.9.10
222
223
ENV VLLM_USAGE_SOURCE production-docker-image
224
docs/source/quantization/bnb.rst
+1
-1
@@ -11,7 +11,7 @@ Below are the steps to utilize BitsAndBytes with vLLM.
11
12
.. code-block:: console
13
14
- $ pip install bitsandbytes>=0.44.0
14
+ $ pip install bitsandbytes>=0.45.0
15
16
vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.
17
requirements-test.in
+1
-1
@@ -25,7 +25,7 @@ datamodel_code_generator # required for minicpm3 test
25
lm-eval[api]==0.4.4 # required for model evaluation test
26
27
# quantization
28
-bitsandbytes>=0.44.0
28
+bitsandbytes>=0.45.0
29
buildkite-test-collector==0.1.9
30
31
numpy < 2.0.0
requirements-test.txt
+1
-1
@@ -37,7 +37,7 @@ audioread==3.0.1
37
# via librosa
38
awscli==1.35.23
39
# via -r requirements-test.in
40
-bitsandbytes==0.44.1
40
+bitsandbytes>=0.45.0
41
# via -r requirements-test.in
42
black==24.10.0
43
# via datamodel-code-generator
vllm/model_executor/layers/quantization/bitsandbytes.py
+4
-4
@@ -145,12 +145,12 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
145
def __init__(self, quant_config: BitsAndBytesConfig):
146
try:
147
import bitsandbytes
148
- if bitsandbytes.__version__ < "0.44.0":
148
+ if bitsandbytes.__version__ < "0.45.0":
149
raise ImportError("bitsandbytes version is wrong. Please "
150
- "install bitsandbytes>=0.44.0.")
150
+ "install bitsandbytes>=0.45.0.")
151
except ImportError as err:
152
- raise ImportError("Please install bitsandbytes>=0.44.0 via "
153
- "`pip install bitsandbytes>=0.44.0` to use "
152
+ raise ImportError("Please install bitsandbytes>=0.45.0 via "
153
+ "`pip install bitsandbytes>=0.45.0` to use "
154
"bitsandbytes quantizer.") from err
155
156
self.quant_config = quant_config