A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 00:45 UTC
Enable bitsandbytes quantization on AMD GPUs that use warp size 32 (#27307)
Signed-off-by: sstamenk <strahinja.stamenkovic@amd.com>
Strahinja Stamenkovic committed
Nov 19, 2025 at 04:12 UTC
814843e021a3618f7f8e494d4f0d4fd561cf3225
2 files changed
+10
-4
tests/models/quantization/test_bitsandbytes.py
+7
-4
@@ -14,10 +14,13 @@ from vllm.platforms import current_platform
14
from ...utils import compare_two_settings, multi_gpu_test
15
from ..utils import check_embeddings_close, check_logprobs_close
16
17
-pytestmark = pytest.mark.skipif(
18
- current_platform.is_rocm(),
19
- reason="bitsandbytes quantization not supported on ROCm (CUDA-only kernels)",
20
-)
17
+if current_platform.is_rocm():
18
+ from vllm.platforms.rocm import on_gfx9
19
+
20
+ pytestmark = pytest.mark.skipif(
21
+ on_gfx9(),
22
+ reason="bitsandbytes not supported on gfx9 (warp size 64 limitation)",
23
+ )
24
25
models_4bit_to_test = [
26
("facebook/opt-125m", "quantize opt model inflight"),
vllm/platforms/rocm.py
+3
@@ -185,6 +185,9 @@ class RocmPlatform(Platform):
185
"petit_nvfp4",
186
"torchao",
187
]
188
+ # bitsandbytes not supported on gfx9 (warp size 64 limitation)
189
+ if not on_gfx9():
190
+ supported_quantization += ["bitsandbytes"]
191
192
@classmethod
193
def get_vit_attn_backend(