A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 22:45 UTC
[ROCm] Enable bitsandbytes quantization support on ROCm (#34688)
Signed-off-by: badaoui <abdennacerbadaoui0@gmail.com>
BADAOUI Abdennacer committed
Feb 21, 2026 at 09:34 UTC
8dc8a99b56e1a8427c83217a37595d0cd12b1ff2
8 files changed
+29
-44
docs/features/quantization/bnb.md
+1
-1
@@ -7,7 +7,7 @@ Compared to other quantization methods, BitsAndBytes eliminates the need for cal
7
Below are the steps to utilize BitsAndBytes with vLLM.
8
9
```bash
10
-pip install bitsandbytes>=0.46.1
10
+pip install bitsandbytes>=0.49.2
11
```
12
13
vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.
requirements/nightly_torch_test.txt
+1
-1
@@ -33,7 +33,7 @@ transformers==4.57.5
33
tokenizers==0.22.0
34
schemathesis>=3.39.15 # Required for openai schema test.
35
# quantization
36
-bitsandbytes>=0.46.1
36
+bitsandbytes>=0.49.2
37
buildkite-test-collector==0.1.9
38
39
requirements/rocm-test.txt
+2
@@ -102,3 +102,5 @@ terratorch==1.2.2
102
segmentation-models-pytorch==0.5.0
103
# Required for Prithvi tests
104
imagehash==4.3.2
105
+# Required for bitsandbytes quantization test
106
+bitsandbytes==0.49.2
requirements/test.in
+1
-1
@@ -41,7 +41,7 @@ transformers==4.57.5
41
tokenizers==0.22.0
42
schemathesis>=3.39.15 # Required for openai schema test.
43
# quantization
44
-bitsandbytes==0.46.1
44
+bitsandbytes==0.49.2
45
buildkite-test-collector==0.1.9
46
47
requirements/test.txt
+2
-1
@@ -66,7 +66,7 @@ backoff==2.2.1
66
# via
67
# -r requirements/test.in
68
# schemathesis
69
-bitsandbytes==0.46.1
69
+bitsandbytes==0.49.2
70
# via
71
# -r requirements/test.in
72
# lightning
@@ -653,6 +653,7 @@ orjson==3.11.5
653
packaging==24.2
654
# via
655
# accelerate
656
+ # bitsandbytes
657
# black
658
# datamodel-code-generator
659
# datasets
tests/models/test_transformers.py
+1
-8
@@ -6,8 +6,6 @@ from typing import Any
6
7
import pytest
8
9
-from vllm.platforms import current_platform
10
-
9
from ..conftest import HfRunner, VllmRunner
10
from ..utils import multi_gpu_test, prep_prompts
11
from .registry import HF_EXAMPLE_MODELS
@@ -131,6 +129,7 @@ def test_distributed(
129
"quantization": "bitsandbytes",
130
},
131
),
132
+ ("unsloth/tinyllama-bnb-4bit", {}),
133
],
134
)
135
@pytest.mark.parametrize("max_tokens", [32])
@@ -143,12 +142,6 @@ def test_quantization(
142
max_tokens: int,
143
num_logprobs: int,
144
) -> None:
146
- if (
147
- current_platform.is_rocm()
148
- and quantization_kwargs.get("quantization", "") == "bitsandbytes"
149
- ):
150
- pytest.skip("bitsandbytes quantization is currently not supported in rocm.")
151
-
145
with vllm_runner(
146
model,
147
model_impl="auto",
vllm/model_executor/layers/quantization/bitsandbytes.py
+20
-29
@@ -28,6 +28,24 @@ from vllm.platforms import current_platform
28
from vllm.utils.torch_utils import direct_register_custom_op
29
30
31
+def _check_bitsandbytes_version():
32
+ min_version = "0.49.2" if current_platform.is_rocm() else "0.48.1"
33
+ try:
34
+ import bitsandbytes
35
+
36
+ if version.parse(bitsandbytes.__version__) < version.parse(min_version):
37
+ raise ImportError(
38
+ "bitsandbytes version is wrong. Please "
39
+ f"install bitsandbytes>={min_version}."
40
+ )
41
+ except ImportError as err:
42
+ raise ImportError(
43
+ f"Please install bitsandbytes>={min_version} via "
44
+ f"`pip install bitsandbytes>={min_version}` to use "
45
+ "bitsandbytes quantizer."
46
+ ) from err
47
+
48
+
49
class BitsAndBytesConfig(QuantizationConfig):
50
"""Config class for BitsAndBytes Quantization.
51
@@ -183,21 +201,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
201
"""
202
203
def __init__(self, quant_config: BitsAndBytesConfig):
186
- try:
187
- import bitsandbytes
188
-
189
- if version.parse(bitsandbytes.__version__) < version.parse("0.46.1"):
190
- raise ImportError(
191
- "bitsandbytes version is wrong. Please "
192
- "install bitsandbytes>=0.46.1."
193
- )
194
- except ImportError as err:
195
- raise ImportError(
196
- "Please install bitsandbytes>=0.46.1 via "
197
- "`pip install bitsandbytes>=0.46.1` to use "
198
- "bitsandbytes quantizer."
199
- ) from err
200
-
204
+ _check_bitsandbytes_version()
205
self.quant_config = quant_config
206
207
def create_weights(
@@ -442,20 +446,7 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase):
446
moe: FusedMoEConfig,
447
):
448
super().__init__(moe)
445
- try:
446
- import bitsandbytes
447
-
448
- if version.parse(bitsandbytes.__version__) < version.parse("0.46.1"):
449
- raise ImportError(
450
- "bitsandbytes version is wrong. Please "
451
- "install bitsandbytes>=0.46.1."
452
- )
453
- except ImportError as err:
454
- raise ImportError(
455
- "Please install bitsandbytes>=0.46.1 via "
456
- "`pip install bitsandbytes>=0.46.1` to use "
457
- "bitsandbytes quantizer."
458
- ) from err
449
+ _check_bitsandbytes_version()
450
self.quant_config = quant_config
451
452
def create_weights(
vllm/platforms/rocm.py
+1
-3
@@ -244,10 +244,8 @@ class RocmPlatform(Platform):
244
"mxfp4",
245
"petit_nvfp4",
246
"torchao",
247
+ "bitsandbytes",
248
]
248
- # bitsandbytes not supported on gfx9 (warp size 64 limitation)
249
- if not on_gfx9():
250
- supported_quantization += ["bitsandbytes"]
249
250
@classmethod
251
def import_kernels(cls) -> None: