@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 073c510c91

[Migration] Migrate bitsandbytes support to OOT plugin (#43529)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com>

Isotr0py committed Aug 9, 2026 at 10:27 UTC 073c510c916f385315a5366173c883781762bb9e

31 files changed +187 -1895

.buildkite/test_areas/plugins.yaml

+28

@@ -63,3 +63,31 @@ steps:

63 commands:

64 - pip install "vllm-gguf-plugin >= 0.0.2"

65 - pytest -v -s plugins_tests/gguf

66 +

67 +- label: BitsAndBytes Plugin

68 + key: bitsandbytes-plugin

69 + device: h200_18gb

70 + timeout_in_minutes: 30

71 + working_dir: "/vllm-workspace/tests"

72 + soft_fail: true

73 + optional: true

74 + source_file_dependencies:

75 + - vllm/model_executor/layers/quantization

76 + - tests/plugins_tests/bitsandbytes

77 + commands:

78 + - pip install "vllm-bnb-plugin >= 0.0.1"

79 + - pytest -v -s plugins_tests/bitsandbytes -m 'not distributed'

80 +

81 +- label: BitsAndBytes Plugin (2 GPUs)

82 + key: bitsandbytes-plugin-2-gpus

83 + timeout_in_minutes: 15

84 + working_dir: "/vllm-workspace/tests"

85 + num_devices: 2

86 + soft_fail: true

87 + optional: true

88 + source_file_dependencies:

89 + - vllm/model_executor/layers/quantization

90 + - tests/plugins_tests/bitsandbytes

91 + commands:

92 + - pip install "vllm-bnb-plugin >= 0.0.1"

93 + - pytest -v -s plugins_tests/bitsandbytes -m 'distributed(num_gpus=2)'

docker/Dockerfile

+1 -9

@@ -779,20 +779,12 @@ RUN set -eux; \

779

780 # Install vllm-openai dependencies (saves ~2.6s per build)

781 # These are stable packages that don't depend on vLLM itself

782 -# From versions.json: .bitsandbytes.x86_64, .bitsandbytes.arm64

782 # From versions.json: .openai_server_extras.timm, .openai_server_extras.runai_model_streamer

784 -ARG BITSANDBYTES_VERSION_X86=0.46.1

785 -ARG BITSANDBYTES_VERSION_ARM64=0.42.0

783 ARG TIMM_VERSION=">=1.0.17"

784 ARG RUNAI_MODEL_STREAMER_VERSION=">=0.15.7"

785 RUN --mount=type=cache,target=/opt/uv/cache \

789 - if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \

790 - BITSANDBYTES_VERSION="${BITSANDBYTES_VERSION_ARM64}"; \

791 - else \

792 - BITSANDBYTES_VERSION="${BITSANDBYTES_VERSION_X86}"; \

793 - fi; \

786 uv pip install --system accelerate 'modelscope<1.38' \

795 - "bitsandbytes>=${BITSANDBYTES_VERSION}" "timm${TIMM_VERSION}" "runai-model-streamer[s3,gcs,azure]${RUNAI_MODEL_STREAMER_VERSION}"

787 + "timm${TIMM_VERSION}" "runai-model-streamer[s3,gcs,azure]${RUNAI_MODEL_STREAMER_VERSION}"

788

789 # ============================================================

790 # VLLM INSTALLATION (depends on build stage)

docker/versions.json

-6

@@ -76,12 +76,6 @@

76 "GDRCOPY_OS_VERSION": {

77 "default": "Ubuntu24_04"

78 },

79 - "BITSANDBYTES_VERSION_X86": {

80 - "default": "0.46.1"

81 - },

82 - "BITSANDBYTES_VERSION_ARM64": {

83 - "default": "0.42.0"

84 - },

79 "TIMM_VERSION": {

80 "default": ">=1.0.17"

81 },

docs/features/quantization/bnb.md

+4 -2

@@ -4,10 +4,12 @@ vLLM now supports [BitsAndBytes](https://github.com/TimDettmers/bitsandbytes) fo

4 BitsAndBytes quantizes models to reduce memory usage and enhance performance without significantly sacrificing accuracy.

5 Compared to other quantization methods, BitsAndBytes eliminates the need for calibrating the quantized model with input data.

7 -Below are the steps to utilize BitsAndBytes with vLLM.

7 +BitsAndBytes support is provided by the out-of-tree

8 +[`vllm-bnb-plugin`](https://github.com/vllm-project/vllm-bnb-plugin).

9 +Install the plugin before using BitsAndBytes with vLLM:

11 ```bash

10 -pip install bitsandbytes>=0.49.2

12 +uv pip install vllm-bnb-plugin

13 ```

15 vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.

examples/features/lora/lora_with_quantization_offline.py

@@ -5,6 +5,7 @@ This example shows how to use LoRA with different quantization techniques

5 for offline inference.

7 Requires HuggingFace credentials for access.

8 +The bitsandbytes example also requires installing vllm-bnb-plugin.

9 """

11 import gc

requirements/test/cpu.txt

-5

@@ -70,8 +70,6 @@ azure-storage-blob==12.28.0

70 # via runai-model-streamer-azure

71 backoff==2.2.1

72 # via -r requirements/test/cuda.in

73 -bitsandbytes==0.49.2

74 - # via -r requirements/test/cuda.in

73 black==24.10.0

74 # via datamodel-code-generator

75 blake3==1.0.9

@@ -535,7 +533,6 @@ numpy==2.2.6

533 # -r requirements/test/../common.txt

534 # accelerate

535 # albumentations

538 - # bitsandbytes

536 # bm25s

537 # contourpy

538 # cupy-cuda12x

@@ -642,7 +639,6 @@ outlines-core==0.2.14

639 packaging==24.2

640 # via

641 # accelerate

645 - # bitsandbytes

642 # black

643 # datamodel-code-generator

644 # datasets

@@ -1126,7 +1122,6 @@ torch==2.13.0+cpu

1122 # via

1123 # -r requirements/test/cuda.in

1124 # accelerate

1129 - # bitsandbytes

1125 # compressed-tensors

1126 # encodec

1127 # instanttensor

requirements/test/cuda.in

-1

@@ -43,7 +43,6 @@ transformers==5.14.1

43 tokenizers==0.22.2

44 schemathesis>=4.0.0 # Required for openai schema test.

45 # quantization

46 -bitsandbytes==0.49.2

46 buildkite-test-collector==0.1.9

47 torch-abi-audit # CI check for PyTorch stable ABI compliance

requirements/test/cuda.txt

-5

@@ -75,8 +75,6 @@ azure-storage-blob==12.28.0

75 # via runai-model-streamer-azure

76 backoff==2.2.1

77 # via -r requirements/test/cuda.in

78 -bitsandbytes==0.49.2

79 - # via -r requirements/test/cuda.in

78 black==24.10.0

79 # via datamodel-code-generator

80 blake3==1.0.9

@@ -566,7 +564,6 @@ numpy==2.2.6

564 # -r requirements/test/../common.txt

565 # accelerate

566 # albumentations

569 - # bitsandbytes

567 # bm25s

568 # contourpy

569 # cupy-cuda12x

@@ -726,7 +723,6 @@ outlines-core==0.2.14

723 packaging==24.2

724 # via

725 # accelerate

729 - # bitsandbytes

726 # black

727 # datamodel-code-generator

728 # datasets

@@ -1225,7 +1221,6 @@ torch==2.13.0+cu130

1221 # -c requirements/cuda.txt

1222 # -r requirements/test/cuda.in

1223 # accelerate

1228 - # bitsandbytes

1224 # compressed-tensors

1225 # encodec

1226 # instanttensor

requirements/test/nightly-torch.txt

-1

@@ -33,7 +33,6 @@ transformers==5.14.1

33 tokenizers==0.22.2

34 schemathesis>=4.0.0 # Required for openai schema test.

35 # quantization

36 -bitsandbytes>=0.49.2

36 buildkite-test-collector==0.1.9

requirements/test/rocm.in

-1

@@ -39,7 +39,6 @@ transformers==5.14.1

39 tokenizers==0.22.2

40 schemathesis>=4.0.0 # Required for openai schema test

41 # quantization

42 -bitsandbytes==0.49.2

42 buildkite-test-collector==0.1.9

43 tilelang==0.1.10

requirements/test/rocm.txt

-4

@@ -72,8 +72,6 @@ azure-storage-blob==12.28.0

72 # via runai-model-streamer-azure

73 backoff==2.2.1

74 # via -r requirements/test/rocm.in

75 -bitsandbytes==0.49.2

76 - # via -r requirements/test/rocm.in

75 black==26.3.1

76 # via datamodel-code-generator

77 blake3==1.0.8

@@ -568,7 +566,6 @@ numpy==2.2.6

566 # -r requirements/test/../common.txt

567 # accelerate

568 # albumentations

571 - # bitsandbytes

569 # bm25s

570 # contourpy

571 # cupy-cuda12x

@@ -698,7 +695,6 @@ packaging==26.0

695 # via

696 # -c requirements/rocm.txt

697 # accelerate

701 - # bitsandbytes

698 # black

699 # datasets

700 # evaluate

tests/conftest.py

+1 -4

@@ -521,10 +521,7 @@ class HfRunner:

521 ):

522 model = model.to(dtype=self.dtype)

523

524 - if (

525 - getattr(model, "quantization_method", None) != "bitsandbytes"

526 - and len({p.device for p in model.parameters()}) < 2

527 - ):

524 + if len({p.device for p in model.parameters()}) < 2:

525 model = model.to(device=self.device)

526

527 self.model = model

tests/kernels/moe/test_moe_weight_loading_padded.py

-34

@@ -343,40 +343,6 @@ class TestWeightLoadingWithPaddedHiddenSize:

343 torch.zeros(original_hidden, padded_intermediate - original_intermediate),

344 )

345

346 - def test_bnb_shape_mismatch_raises(self):

347 - """BnB + padded hidden_size should raise via weight_loader."""

348 - from unittest.mock import MagicMock

349 -

350 - num_experts = 1

351 - padded_packed = 3072 # padded packed size

352 - original_packed = 2688 # original packed size

353 -

354 - # Build a param that looks like a BnB 4-bit MoE weight.

355 - param_data = torch.zeros(num_experts, padded_packed, 1, dtype=torch.uint8)

356 - param = torch.nn.Parameter(param_data, requires_grad=False)

357 - param.use_bitsandbytes_4bit = True

358 -

359 - loaded_weight = torch.randint(0, 255, (original_packed, 1), dtype=torch.uint8)

360 -

361 - # Minimal RoutedExperts mock so weight_loader reaches the BnB path.

362 - moe = MagicMock(spec=RoutedExperts)

363 - moe.quant_config = None

364 - moe.quant_method = MagicMock()

365 - moe.quant_method.__class__.__name__ = "BitsAndBytesMethod"

366 - moe._expert_map = None

367 - moe.tp_rank = 0

368 -

369 - # Call the real weight_loader (unbound) with our mock as self.

370 - with pytest.raises(ValueError, match="BitsAndBytes"):

371 - RoutedExperts.weight_loader(

372 - moe,

373 - param,

374 - loaded_weight,

375 - weight_name="w2",

376 - shard_id="w2",

377 - expert_id=0,

378 - )

379 -

346

347 class TestLoadWeightsExpertBias:

348 """Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize

tests/models/transformers/test_backend.py

-7

@@ -189,13 +189,6 @@ def test_distributed(

189 [

190 ("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {}),

191 ("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {}),

192 - (

193 - "meta-llama/Llama-3.2-1B-Instruct",

194 - {

195 - "quantization": "bitsandbytes",

196 - },

197 - ),

198 - ("unsloth/tinyllama-bnb-4bit", {}),

192 ],

193 )

194 @pytest.mark.parametrize("max_tokens", [32])

tests/plugins_tests/bitsandbytes/__init__.py new

@@ -0,0 +1,2 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

tests/plugins_tests/bitsandbytes/test_bitsandbytes.py renamed

+78 -130

@@ -1,9 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""Tests whether bitsandbytes computation is enabled correctly.

4 -

5 -Run `pytest tests/quantization/test_bitsandbytes.py`.

6 -"""

3 +"""End-to-end tests for the out-of-tree bitsandbytes plugin."""

5 import types

6 from unittest.mock import MagicMock, patch

@@ -12,13 +9,12 @@ import pytest

9 from packaging.version import Version

10 from transformers import BitsAndBytesConfig

11 from transformers import __version__ as TRANSFORMERS_VERSION

12 +from vllm_bnb_plugin import bitsandbytes_loader as bnb

16 -from tests.quantization.utils import is_quant_method_supported

17 -from vllm.model_executor.model_loader import bitsandbytes_loader as bnb

14 from vllm.platforms import current_platform

20 -from ...utils import compare_two_settings, multi_gpu_test

21 -from ..utils import check_embeddings_close, check_logprobs_close

16 +from ...models.utils import check_embeddings_close, check_logprobs_close

17 +from ...utils import multi_gpu_test

19 if current_platform.is_rocm():

20 from vllm.platforms.rocm import on_cdna

@@ -44,7 +40,7 @@ models_4bit_to_moe_test = [

40 ("allenai/OLMoE-1B-7B-0125-Instruct", "quantize moe model inflight"),

41 ]

47 -models_pre_qaunt_4bit_to_test = [

43 +models_pre_quant_4bit_to_test = [

44 (

45 "PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed",

46 "read pre-quantized 4-bit FP4 model",

@@ -58,10 +54,64 @@ models_pre_quant_8bit_to_test = [

54 ]

61 -@pytest.mark.skipif(

62 - not is_quant_method_supported("bitsandbytes"),

63 - reason="bitsandbytes is not supported on this GPU type.",

64 -)

57 +def log_generated_texts(prompts, outputs, runner_name):

58 + logged_texts = []

59 + for i, (_, generated_text) in enumerate(outputs):

60 + logged_texts.append(

61 + {

62 + "prompt": prompts[i],

63 + "runner_name": runner_name,

64 + "generated_text": generated_text,

65 + }

66 + )

67 + return logged_texts

68 +

69 +

70 +def validate_generated_texts(

71 + hf_runner,

72 + vllm_runner,

73 + prompts,

74 + model_name,

75 + pre_quant=False,

76 + hf_model_kwargs=None,

77 + vllm_tp_size=1,

78 + max_tokens=8,

79 +):

80 + with vllm_runner(

81 + model_name,

82 + quantization=None if pre_quant else "bitsandbytes",

83 + tensor_parallel_size=vllm_tp_size,

84 + enforce_eager=False,

85 + default_torch_num_threads=1,

86 + tokenizer_mode="hf",

87 + load_format="hf",

88 + config_format="hf",

89 + ) as llm:

90 + vllm_outputs = llm.generate_greedy(prompts, max_tokens)

91 + vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")

92 +

93 + if hf_model_kwargs is None:

94 + hf_model_kwargs = {}

95 +

96 + with hf_runner(

97 + model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1

98 + ) as llm:

99 + hf_outputs = llm.generate_greedy(prompts, max_tokens)

100 + hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")

101 +

102 + for hf_log, vllm_log in zip(hf_logs, vllm_logs):

103 + hf_str = hf_log["generated_text"]

104 + vllm_str = vllm_log["generated_text"]

105 + prompt = hf_log["prompt"]

106 + assert hf_str == vllm_str, (

107 + f"Model: {model_name}"

108 + f"Mismatch between HF and vLLM outputs:\n"

109 + f"Prompt: {prompt}\n"

110 + f"HF Output: '{hf_str}'\n"

111 + f"vLLM Output: '{vllm_str}'"

112 + )

113 +

114 +

115 @pytest.mark.parametrize("model_name, description", models_4bit_to_test)

116 def test_load_4bit_bnb_model(

117 hf_runner, vllm_runner, example_prompts, model_name, description

@@ -72,11 +122,7 @@ def test_load_4bit_bnb_model(

122 )

123

124

75 -@pytest.mark.skipif(

76 - not is_quant_method_supported("bitsandbytes"),

77 - reason="bitsandbytes is not supported on this GPU type.",

78 -)

79 -@pytest.mark.parametrize("model_name, description", models_pre_qaunt_4bit_to_test)

125 +@pytest.mark.parametrize("model_name, description", models_pre_quant_4bit_to_test)

126 def test_load_pre_quant_4bit_bnb_model(

127 hf_runner, vllm_runner, example_prompts, model_name, description

128 ) -> None:

@@ -85,10 +131,6 @@ def test_load_pre_quant_4bit_bnb_model(

131 )

132

133

88 -@pytest.mark.skipif(

89 - not is_quant_method_supported("bitsandbytes"),

90 - reason="bitsandbytes is not supported on this GPU type.",

91 -)

134 @pytest.mark.parametrize("model_name, description", models_pre_quant_8bit_to_test)

135 def test_load_8bit_bnb_model(

136 hf_runner, vllm_runner, example_prompts, model_name, description

@@ -98,10 +140,6 @@ def test_load_8bit_bnb_model(

140 )

141

142

101 -@pytest.mark.skipif(

102 - not is_quant_method_supported("bitsandbytes"),

103 - reason="bitsandbytes is not supported on this GPU type.",

104 -)

143 @pytest.mark.parametrize("model_name, description", models_4bit_to_test)

144 @multi_gpu_test(num_gpus=2)

145 def test_load_tp_4bit_bnb_model(

@@ -119,44 +157,28 @@ def test_load_tp_4bit_bnb_model(

157 )

158

159

122 -@pytest.mark.skipif(

123 - not is_quant_method_supported("bitsandbytes"),

124 - reason="bitsandbytes is not supported on this GPU type.",

125 -)

160 @pytest.mark.parametrize("model_name, description", models_4bit_to_test)

161 @multi_gpu_test(num_gpus=2)

128 -def test_load_pp_4bit_bnb_model(model_name, description) -> None:

129 - common_args = [

130 - "--disable-log-stats",

131 - "--dtype",

132 - "bfloat16",

133 - "--enable-prefix-caching",

134 - "--quantization",

135 - "bitsandbytes",

136 - "--gpu-memory-utilization",

137 - "0.7",

138 - ]

139 - pp_args = [

140 - *common_args,

141 - "--pipeline-parallel-size",

142 - "2",

143 - ]

144 - compare_two_settings(

162 +def test_load_pp_4bit_bnb_model(

163 + hf_runner, vllm_runner, example_prompts, model_name, description

164 +) -> None:

165 + hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))

166 + validate_generated_texts(

167 + hf_runner,

168 + vllm_runner,

169 + example_prompts[:1],

170 model_name,

146 - common_args,

147 - pp_args,

171 + False,

172 + hf_model_kwargs,

173 + vllm_tp_size=2,

174 )

175

176

177 @pytest.mark.skipif(

178 Version(TRANSFORMERS_VERSION) >= Version("5.0.0"),

153 - reason="Need to add support for quantizing MoE experts with bnb"

154 - " in transformers v5. See"

155 - " https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1849",

156 -)

157 -@pytest.mark.skipif(

158 - not is_quant_method_supported("bitsandbytes"),

159 - reason="bitsandbytes is not supported on this GPU type.",

179 + reason="Need to add support for quantizing MoE experts with bnb in "

180 + "transformers v5. See https://github.com/bitsandbytes-foundation/"

181 + "bitsandbytes/issues/1849",

182 )

183 @pytest.mark.parametrize("model_name, description", models_4bit_to_moe_test)

184 def test_4bit_bnb_moe_model(

@@ -193,10 +215,6 @@ def test_4bit_bnb_moe_model(

215 )

216

217

196 -@pytest.mark.skipif(

197 - not is_quant_method_supported("bitsandbytes"),

198 - reason="bitsandbytes is not supported on this GPU type.",

199 -)

218 @pytest.mark.parametrize("model_name, description", models_4bit_to_embedding_test)

219 @pytest.mark.parametrize("dtype", ["half"])

220 def test_4bit_bnb_embedding_model(

@@ -207,15 +225,8 @@ def test_4bit_bnb_embedding_model(

225 example_prompts,

226 dtype: str,

227 ) -> None:

210 - # The example_prompts has ending "\n", for example:

211 - # "Write a short story about a robot that dreams for the first time.\n"

212 - # sentence_transformers will strip the input texts, see:

213 - # https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159

214 - # This makes the input_ids different between hf_model and vllm_model.

215 - # So we need to strip the input texts to avoid test failing.

228 example_prompts = [str(s).strip() for s in example_prompts]

229

218 - # Inflight 4bit quantization

230 with vllm_runner(

231 model_name,

232 runner="pooling",

@@ -245,70 +256,7 @@ def test_4bit_bnb_embedding_model(

256 )

257

258

248 -def log_generated_texts(prompts, outputs, runner_name):

249 - logged_texts = []

250 - for i, (_, generated_text) in enumerate(outputs):

251 - log_entry = {

252 - "prompt": prompts[i],

253 - "runner_name": runner_name,

254 - "generated_text": generated_text,

255 - }

256 - logged_texts.append(log_entry)

257 - return logged_texts

258 -

259 -

260 -def validate_generated_texts(

261 - hf_runner,

262 - vllm_runner,

263 - prompts,

264 - model_name,

265 - pre_quant=False,

266 - hf_model_kwargs=None,

267 - vllm_tp_size=1,

268 - max_tokens=8,

269 -):

270 - # NOTE: run vLLM first, as it requires a clean process

271 - # when using distributed inference

272 - with vllm_runner(

273 - model_name,

274 - quantization=None if pre_quant else "bitsandbytes",

275 - tensor_parallel_size=vllm_tp_size,

276 - enforce_eager=False,

277 - default_torch_num_threads=1,

278 - tokenizer_mode="hf",

279 - load_format="hf",

280 - config_format="hf",

281 - ) as llm:

282 - vllm_outputs = llm.generate_greedy(prompts, max_tokens)

283 - vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")

284 -

285 - if hf_model_kwargs is None:

286 - hf_model_kwargs = {}

287 -

288 - # Run with HF runner

289 - with hf_runner(

290 - model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1

291 - ) as llm:

292 - hf_outputs = llm.generate_greedy(prompts, max_tokens)

293 - hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")

294 -

295 - # Compare the generated strings

296 - for hf_log, vllm_log in zip(hf_logs, vllm_logs):

297 - hf_str = hf_log["generated_text"]

298 - vllm_str = vllm_log["generated_text"]

299 - prompt = hf_log["prompt"]

300 - assert hf_str == vllm_str, (

301 - f"Model: {model_name}"

302 - f"Mismatch between HF and vLLM outputs:\n"

303 - f"Prompt: {prompt}\n"

304 - f"HF Output: '{hf_str}'\n"

305 - f"vLLM Output: '{vllm_str}'"

306 - )

307 -

308 -

259 def test_bitsandbytes_passes_revision_by_name():

310 - # revision must reach download_safetensors_index_file_from_hf as the

311 - # ``revision`` keyword, not a positional slot.

260 fake_self = types.SimpleNamespace(

261 load_config=types.SimpleNamespace(download_dir="/cache"),

262 _get_weight_files=MagicMock(

tests/plugins_tests/bitsandbytes/test_transformers.py new

+70

@@ -0,0 +1,70 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Transformers backend tests for bitsandbytes plugin models."""

4 +

5 +import pytest

6 +

7 +from vllm.platforms import current_platform

8 +

9 +from ...models.utils import check_logprobs_close

10 +

11 +if current_platform.is_rocm():

12 + from vllm.platforms.rocm import on_cdna

13 +

14 + pytestmark = pytest.mark.skipif(

15 + on_cdna(),

16 + reason="bitsandbytes not supported on CDNA (warp size 64 limitation)",

17 + )

18 +

19 +

20 +@pytest.mark.parametrize(

21 + "model, quantization_kwargs",

22 + [

23 + (

24 + "meta-llama/Llama-3.2-1B-Instruct",

25 + {

26 + "quantization": "bitsandbytes",

27 + },

28 + ),

29 + ("unsloth/tinyllama-bnb-4bit", {}),

30 + ],

31 +)

32 +@pytest.mark.parametrize("max_tokens", [32])

33 +@pytest.mark.parametrize("num_logprobs", [5])

34 +def test_transformers_backend_quantization(

35 + vllm_runner,

36 + example_prompts,

37 + model: str,

38 + quantization_kwargs: dict[str, str],

39 + max_tokens: int,

40 + num_logprobs: int,

41 +) -> None:

42 + with vllm_runner(

43 + model,

44 + model_impl="auto",

45 + enforce_eager=True,

46 + **quantization_kwargs, # type: ignore[arg-type]

47 + ) as vllm_model:

48 + vllm_outputs = vllm_model.generate_greedy_logprobs(

49 + example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs

50 + )

51 +

52 + with vllm_runner(

53 + model,

54 + model_impl="transformers",

55 + enforce_eager=True,

56 + **quantization_kwargs, # type: ignore[arg-type]

57 + ) as vllm_model:

58 + model_config = vllm_model.llm.llm_engine.model_config

59 + assert model_config.using_transformers_backend()

60 +

61 + transformers_outputs = vllm_model.generate_greedy_logprobs(

62 + example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs

63 + )

64 +

65 + check_logprobs_close(

66 + outputs_0_lst=transformers_outputs,

67 + outputs_1_lst=vllm_outputs,

68 + name_0="transformers",

69 + name_1="vllm",

70 + )

vllm/config/load.py

-1

@@ -49,7 +49,6 @@ class LoadConfig:

49 Streamer.

50 - "runai_streamer_sharded" will load weights from pre-sharded checkpoint

51 files using Run:ai Model Streamer.

52 - - "bitsandbytes" will load the weights using bitsandbytes quantization.

52 - "sharded_state" will load weights from pre-sharded checkpoint files,

53 supporting efficient loading of tensor-parallel models.

54 - "mistral" will load weights from consolidated safetensors files used by

vllm/config/model.py

-29

@@ -849,7 +849,6 @@ class ModelConfig:

849 self._try_verify_and_update_model_config()

850 self._verify_quantization()

851 self._verify_cuda_graph()

852 - self._verify_bnb_config()

852

853 def _supports_multimodal_for_mm_prefix(self) -> bool:

854 """Whether multimodal inputs can still appear for this deployment.

@@ -1317,34 +1316,6 @@ class ModelConfig:

1316 )

1317 self.enforce_eager = True

1318

1320 - def _verify_bnb_config(self) -> None:

1321 - """

1322 - The current version of bitsandbytes (0.46.1) with 8-bit models does not

1323 - yet support CUDA graph.

1324 - # TODO Remove this when bitsandbytes supports.

1325 - """

1326 - is_bitsandbytes = self.quantization == "bitsandbytes"

1327 - has_quantization_config = self.model_arch_config.quantization_config is not None

1328 - is_8bit = (

1329 - self.model_arch_config.quantization_config.get("load_in_8bit", False) # type: ignore[union-attr]

1330 - if has_quantization_config

1331 - else False

1332 - )

1333 - if all(

1334 - [

1335 - is_bitsandbytes,

1336 - has_quantization_config,

1337 - is_8bit,

1338 - not self.enforce_eager,

1339 - ]

1340 - ):

1341 - logger.warning(

1342 - "CUDA graph is not supported on BitsAndBytes 8bit yet, "

1343 - "fallback to the eager mode."

1344 - )

1345 -

1346 - self.enforce_eager = True

1347 -

1319 def _verify_with_expert_parallelism(self) -> None:

1320 if not self.is_moe:

1321 raise ValueError(

vllm/engine/arg_utils.py

-7

@@ -1794,9 +1794,6 @@ class EngineArgs:

1794 )

1795

1796 def create_load_config(self) -> LoadConfig:

1797 - if self.quantization == "bitsandbytes":

1798 - self.load_format = "bitsandbytes"

1799 -

1797 if self.load_format == "tensorizer":

1798 if hasattr(self.model_loader_extra_config, "to_serializable"):

1799 self.model_loader_extra_config = (

@@ -2360,10 +2357,6 @@ class EngineArgs:

2357 "decreasing num_speculative_tokens"

2358 )

2359

2363 - # bitsandbytes pre-quantized model need a specific model loader

2364 - if model_config.quantization == "bitsandbytes":

2365 - self.quantization = self.load_format = "bitsandbytes"

2366 -

2360 # Attention config overrides

2361 attention_config = copy.deepcopy(self.attention_config)

2362 if self.attention_backend is not None:

vllm/model_executor/layers/fused_moe/routed_experts.py

-38

@@ -674,44 +674,6 @@ class RoutedExperts(PluggableLayer):

674 # based on the shard id. This will be whatever

675 # dimension intermediate_size_per_partition is used.

676 SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}

677 -

678 - # Case for BitsAndBytes

679 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)

680 - if use_bitsandbytes_4bit:

681 - shard_dim = 0

682 -

683 - expert_data = param.data[expert_id]

684 - if shard_id == "w2":

685 - # BnB params are stored as flat packed tensors (e.g.

686 - # (packed_size, 1)), not in the logical weight layout.

687 - # Narrowing packed data for hidden-dim padding is not

688 - # meaningful, so require an exact shape match.

689 - if expert_data.shape != loaded_weight.shape:

690 - raise ValueError(

691 - "BitsAndBytes quantization with padded hidden_size "

692 - "(e.g., from DeepEP) is not supported. "

693 - f"Parameter shape {tuple(expert_data.shape)} != "

694 - f"checkpoint shape {tuple(loaded_weight.shape)}"

695 - )

696 - expert_data.copy_(loaded_weight)

697 - elif shard_id in ("w1", "w3"):

698 - # BnB stores weights as flat packed tensors. _load_w13 is

699 - # still used to split the w1/w3 portions along shard_dim.

700 - # _narrow_expert_data_for_padding will be a no-op since

701 - # packed sizes should already match; if DeepEP padding

702 - # causes a mismatch the copy_() will fail with a clear

703 - # shape error.

704 - full_load = True

705 - self._load_w13(

706 - shard_id=shard_id,

707 - shard_dim=shard_dim,

708 - loaded_weight=loaded_weight,

709 - expert_data=expert_data,

710 - tp_rank=self.moe_config.tp_rank,

711 - load_full=full_load,

712 - )

713 - return True if return_success else None

714 -

677 shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]

678 if is_transposed:

679 shard_dim = int(not shard_dim)

vllm/model_executor/layers/linear.py

-113

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 -import itertools

4 from abc import abstractmethod

5 from collections.abc import Iterable

6 from typing import Any

@@ -96,23 +95,6 @@ def adjust_block_scale_shard(

95 return shard_size, shard_offset

99 -def adjust_bitsandbytes_4bit_shard(

100 - param: Parameter,

101 - shard_offsets: dict[str, tuple[int, int]],

102 - loaded_shard_id: str,

103 -) -> tuple[int, int]:

104 - """Adjust the quantization offsets and sizes for BitsAndBytes sharding."""

105 -

106 - total, _ = shard_offsets["total"]

107 - orig_offset, orig_size = shard_offsets[loaded_shard_id]

108 -

109 - quantized_total = param.data.shape[0]

110 - quantized_offset = orig_offset * quantized_total // total

111 - quantized_size = orig_size * quantized_total // total

112 -

113 - return quantized_size, quantized_offset

114 -

115 -

98 def adjust_scalar_to_fused_array(

99 param_data: torch.Tensor,

100 loaded_weight: torch.Tensor,

@@ -561,10 +543,6 @@ class ColumnParallelLinear(LinearBase):

543 output_dim = getattr(param, "output_dim", None)

544

545 is_sharded_weight = getattr(param, "is_sharded_weight", False)

564 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)

565 - # bitsandbytes loads the weights of the specific portion

566 - # no need to narrow

567 - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit

546

547 param_data = param.data

548 if output_dim is not None and not is_sharded_weight:

@@ -775,16 +753,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):

753 else self.output_sizes

754 )

755 current_shard_offset = 0

778 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)

779 - if (

780 - use_bitsandbytes_4bit

781 - and isinstance(loaded_shard_id, tuple)

782 - and self.tp_size > 1

783 - ):

784 - raise NotImplementedError(

785 - "Shard id with multiple indices is not supported "

786 - "for BNB quantization with TP yet."

787 - )

756 shard_offsets: list[tuple[int, int, int]] = []

757 for i, output_size in enumerate(output_sizes):

758 shard_offsets.append((i, current_shard_offset, output_size))

@@ -809,17 +777,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):

777 param, shard_size, shard_offset

778 )

779

812 - if use_bitsandbytes_4bit:

813 - index = list(itertools.accumulate([0] + self.output_sizes))

814 - orig_offsets = {

815 - str(i): (index[i], size)

816 - for i, size in enumerate(self.output_sizes)

817 - }

818 - orig_offsets["total"] = (self.output_size, 0)

819 - shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(

820 - param, orig_offsets, str(shard_id)

821 - )

822 -

780 loaded_weight_shard = loaded_weight.narrow(

781 output_dim, shard_offset, shard_size

782 )

@@ -851,21 +808,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):

808 param, shard_size, shard_offset

809 )

810

854 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)

811 is_sharded_weight = getattr(param, "is_sharded_weight", False)

856 - # bitsandbytes loads the weights of the specific portion

857 - # no need to narrow

858 - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit

859 -

860 - if use_bitsandbytes_4bit:

861 - index = list(itertools.accumulate([0] + self.output_sizes))

862 - orig_offsets = {

863 - str(i): (index[i], size) for i, size in enumerate(self.output_sizes)

864 - }

865 - orig_offsets["total"] = (self.output_size, 0)

866 - shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(

867 - param, orig_offsets, str(loaded_shard_id)

868 - )

812 param_data = param_data.narrow(output_dim, shard_offset, shard_size)

813 start_idx = self.tp_rank * shard_size

814 if not is_sharded_weight:

@@ -1259,8 +1202,6 @@ class QKVParallelLinear(ColumnParallelLinear):

1202 self.total_num_kv_heads * self.v_head_size,

1203 ),

1204 ]

1262 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)

1263 -

1205 packed_dim = getattr(param, "packed_dim", None)

1206 for shard_id, shard_offset, shard_size in shard_offsets:

1207 # Special case for Quantized Weights.

@@ -1282,30 +1223,6 @@ class QKVParallelLinear(ColumnParallelLinear):

1223 param, shard_size, shard_offset

1224 )

1225

1285 - if use_bitsandbytes_4bit:

1286 - orig_qkv_offsets = {

1287 - "q": (0, self.total_num_heads * self.head_size),

1288 - "k": (

1289 - self.total_num_heads * self.head_size,

1290 - self.total_num_kv_heads * self.head_size,

1291 - ),

1292 - "v": (

1293 - (self.total_num_heads + self.total_num_kv_heads)

1294 - * self.head_size,

1295 - self.total_num_kv_heads * self.v_head_size,

1296 - ),

1297 - "total": (

1298 - (self.total_num_heads + self.total_num_kv_heads)

1299 - * self.head_size

1300 - + self.total_num_kv_heads * self.v_head_size,

1301 - 0,

1302 - ),

1303 - }

1304 -

1305 - shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(

1306 - param, orig_qkv_offsets, shard_id

1307 - )

1308 -

1226 loaded_weight_shard = loaded_weight.narrow(

1227 output_dim, shard_offset, shard_size

1228 )

@@ -1345,33 +1262,7 @@ class QKVParallelLinear(ColumnParallelLinear):

1262 param, shard_size, shard_offset

1263 )

1264

1348 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)

1265 is_sharded_weight = getattr(param, "is_sharded_weight", False)

1350 - # bitsandbytes loads the weights of the specific portion

1351 - # no need to narrow

1352 - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit

1353 -

1354 - if use_bitsandbytes_4bit:

1355 - orig_qkv_offsets = {

1356 - "q": (0, self.num_heads * self.head_size),

1357 - "k": (

1358 - self.num_heads * self.head_size,

1359 - self.num_kv_heads * self.head_size,

1360 - ),

1361 - "v": (

1362 - (self.num_heads + self.num_kv_heads) * self.head_size,

1363 - self.num_kv_heads * self.v_head_size,

1364 - ),

1365 - "total": (

1366 - (self.num_heads + self.num_kv_heads) * self.head_size

1367 - + self.num_kv_heads * self.v_head_size,

1368 - 0,

1369 - ),

1370 - }

1371 - shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(

1372 - param, orig_qkv_offsets, loaded_shard_id

1373 - )

1374 -

1266 param_data = param_data.narrow(output_dim, shard_offset, shard_size)

1267 if loaded_shard_id == "q":

1268 shard_rank = self.tp_rank

@@ -1716,11 +1607,7 @@ class RowParallelLinear(LinearBase):

1607

1608 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):

1609 input_dim = getattr(param, "input_dim", None)

1719 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)

1610 is_sharded_weight = getattr(param, "is_sharded_weight", False)

1721 - # bitsandbytes loads the weights of the specific portion

1722 - # no need to narrow

1723 - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit

1611

1612 param_data = param.data

1613 if input_dim is not None and not is_sharded_weight:

vllm/model_executor/layers/quantization/__init__.py

-3

@@ -25,7 +25,6 @@ QuantizationMethods = Literal[

25 "awq_marlin",

26 "humming",

27 "compressed-tensors",

28 - "bitsandbytes",

28 "experts_int8",

29 "quark",

30 "moe_wna16",

@@ -117,7 +116,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:

116

117 from .auto_awq import AutoAWQConfig

118 from .auto_gptq import AutoGPTQConfig

120 - from .bitsandbytes import BitsAndBytesConfig

119 from .compressed_tensors.compressed_tensors import (

120 CompressedTensorsConfig,

121 )

@@ -153,7 +151,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:

151 "gptq": AutoGPTQConfig,

152 "gptq_marlin": AutoGPTQConfig,

153 "compressed-tensors": CompressedTensorsConfig,

156 - "bitsandbytes": BitsAndBytesConfig,

154 "experts_int8": ExpertsInt8Config,

155 "quark": QuarkConfig,

156 "moe_wna16": MoeWNA16Config,

vllm/model_executor/layers/quantization/bitsandbytes.py deleted

-614

@@ -1,614 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -from functools import cached_property

5 -from typing import Any, Union

6 -

7 -import torch

8 -from packaging import version

9 -

10 -from vllm.model_executor.layers.fused_moe import (

11 - FusedMoEConfig,

12 - FusedMoEMethodBase,

13 - FusedMoEQuantConfig,

14 - RoutedExperts,

15 - SharedExperts,

16 -)

17 -from vllm.model_executor.layers.linear import (

18 - LinearBase,

19 - LinearMethodBase,

20 - UnquantizedLinearMethod,

21 - set_weight_attrs,

22 -)

23 -from vllm.model_executor.layers.quantization import (

24 - QuantizationConfig,

25 - QuantizationMethods,

26 -)

27 -from vllm.platforms import current_platform

28 -from vllm.utils.torch_utils import direct_register_custom_op

29 -

30 -

31 -def _check_bitsandbytes_version():

32 - min_version = "0.49.2" if current_platform.is_rocm() else "0.48.1"

33 - try:

34 - import bitsandbytes

35 -

36 - if version.parse(bitsandbytes.__version__) < version.parse(min_version):

37 - raise ImportError(

38 - "bitsandbytes version is wrong. Please "

39 - f"install bitsandbytes>={min_version}."

40 - )

41 - except ImportError as err:

42 - raise ImportError(

43 - f"Please install bitsandbytes>={min_version} via "

44 - f"`pip install bitsandbytes>={min_version}` to use "

45 - "bitsandbytes quantizer."

46 - ) from err

47 -

48 -

49 -class BitsAndBytesConfig(QuantizationConfig):

50 - """Config class for BitsAndBytes Quantization.

51 -

52 - Reference: https://arxiv.org/abs/2305.14314

53 - """

54 -

55 - def __init__(

56 - self,

57 - load_in_8bit: bool = False,

58 - load_in_4bit: bool = True,

59 - bnb_4bit_compute_dtype: str = "float32",

60 - bnb_4bit_quant_storage: str = "uint8",

61 - bnb_4bit_quant_type: str = "fp4",

62 - bnb_4bit_use_double_quant: bool = False,

63 - llm_int8_enable_fp32_cpu_offload: bool = False,

64 - llm_int8_has_fp16_weight: bool = False,

65 - llm_int8_skip_modules: list[str] | None = None,

66 - llm_int8_threshold: float = 6.0,

67 - ) -> None:

68 - super().__init__()

69 - self.load_in_8bit = load_in_8bit

70 - self.load_in_4bit = load_in_4bit

71 - self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype

72 - self.bnb_4bit_quant_storage = bnb_4bit_quant_storage

73 - self.bnb_4bit_quant_type = bnb_4bit_quant_type

74 - self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant

75 - self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload

76 - self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight

77 - self.llm_int8_skip_modules = llm_int8_skip_modules or []

78 - self.llm_int8_threshold = llm_int8_threshold

79 -

80 - if self.bnb_4bit_quant_storage not in ["uint8"]:

81 - raise ValueError(

82 - f"Unsupported bnb_4bit_quant_storage: {self.bnb_4bit_quant_storage}"

83 - )

84 -

85 - def __repr__(self) -> str:

86 - return (

87 - f"BitsAndBytesConfig(load_in_8bit={self.load_in_8bit}, "

88 - f"load_in_4bit={self.load_in_4bit}, "

89 - f"bnb_4bit_compute_dtype={self.bnb_4bit_compute_dtype}, "

90 - f"bnb_4bit_quant_storage={self.bnb_4bit_quant_storage}, "

91 - f"bnb_4bit_quant_type={self.bnb_4bit_quant_type}, "

92 - f"llm_int8_skip_modules={self.llm_int8_skip_modules})"

93 - )

94 -

95 - @classmethod

96 - def get_name(self) -> QuantizationMethods:

97 - return "bitsandbytes"

98 -

99 - @classmethod

100 - def get_supported_act_dtypes(self) -> list[torch.dtype]:

101 - return [torch.float32, torch.float16, torch.bfloat16]

102 -

103 - @classmethod

104 - def get_min_capability(cls) -> int:

105 - return 70

106 -

107 - @staticmethod

108 - def get_config_filenames() -> list[str]:

109 - return []

110 -

111 - @classmethod

112 - def from_config(cls, config: dict[str, Any]) -> "BitsAndBytesConfig":

113 - def get_safe_value(config, keys, default_value=None):

114 - try:

115 - value = cls.get_from_keys(config, keys)

116 - return value if value is not None else default_value

117 - except ValueError:

118 - return default_value

119 -

120 - load_in_8bit = get_safe_value(config, ["load_in_8bit"], default_value=False)

121 - load_in_4bit = get_safe_value(config, ["load_in_4bit"], default_value=True)

122 - bnb_4bit_compute_dtype = get_safe_value(

123 - config, ["bnb_4bit_compute_dtype"], default_value="float32"

124 - )

125 - bnb_4bit_quant_storage = get_safe_value(

126 - config, ["bnb_4bit_quant_storage"], default_value="uint8"

127 - )

128 - bnb_4bit_quant_type = get_safe_value(

129 - config, ["bnb_4bit_quant_type"], default_value="fp4"

130 - )

131 - bnb_4bit_use_double_quant = get_safe_value(

132 - config, ["bnb_4bit_use_double_quant"], default_value=False

133 - )

134 - llm_int8_enable_fp32_cpu_offload = get_safe_value(

135 - config, ["llm_int8_enable_fp32_cpu_offload"], default_value=False

136 - )

137 - llm_int8_has_fp16_weight = get_safe_value(

138 - config, ["llm_int8_has_fp16_weight"], default_value=False

139 - )

140 - llm_int8_skip_modules = get_safe_value(

141 - config, ["llm_int8_skip_modules"], default_value=[]

142 - )

143 - llm_int8_threshold = get_safe_value(

144 - config, ["llm_int8_threshold"], default_value=6.0

145 - )

146 -

147 - return cls(

148 - load_in_8bit=load_in_8bit,

149 - load_in_4bit=load_in_4bit,

150 - bnb_4bit_compute_dtype=bnb_4bit_compute_dtype,

151 - bnb_4bit_quant_storage=bnb_4bit_quant_storage,

152 - bnb_4bit_quant_type=bnb_4bit_quant_type,

153 - bnb_4bit_use_double_quant=bnb_4bit_use_double_quant,

154 - llm_int8_enable_fp32_cpu_offload=llm_int8_enable_fp32_cpu_offload,

155 - llm_int8_has_fp16_weight=llm_int8_has_fp16_weight,

156 - llm_int8_skip_modules=llm_int8_skip_modules,

157 - llm_int8_threshold=llm_int8_threshold,

158 - )

159 -

160 - def get_quant_method(

161 - self, layer: torch.nn.Module, prefix: str

162 - ) -> Union["LinearMethodBase", "BitsAndBytesMoEMethod"] | None:

163 - if isinstance(layer, LinearBase):

164 - if is_layer_skipped_bnb(prefix, self.llm_int8_skip_modules):

165 - return UnquantizedLinearMethod()

166 - return BitsAndBytesLinearMethod(self)

167 - elif isinstance(layer, RoutedExperts):

168 - return BitsAndBytesMoEMethod(self, layer.moe_config)

169 - return None

170 -

171 -

172 -class BitsAndBytesWeightParameter(torch.nn.Parameter):

173 - @cached_property

174 - def dtype(self) -> torch.dtype:

175 - return torch.get_default_dtype()

176 -

177 -

178 -def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: list[str]):

179 - # Split the prefix into its dot-separated components

180 - components = prefix.split(".")

181 -

182 - # Check if any of the skip modules exactly matches any component

183 - substr_check = any(

184 - module_name in components for module_name in llm_int8_skip_modules

185 - )

186 -

187 - # Allow certain layers to not be quantized

188 - set_components = set(".".join(components[: i + 1]) for i in range(len(components)))

189 - set_llm_int8_skip_modules = set(llm_int8_skip_modules)

190 - prefix_check = len(set_llm_int8_skip_modules & set_components) != 0

191 -

192 - return substr_check or prefix_check

193 -

194 -

195 -def calculate_quant_ratio(dtype):

196 - if dtype.is_floating_point:

197 - return torch.finfo(dtype).bits // torch.iinfo(torch.uint8).bits

198 - else:

199 - return torch.iinfo(dtype).bits // torch.iinfo(torch.uint8).bits

200 -

201 -

202 -class BitsAndBytesLinearMethod(LinearMethodBase):

203 - """Linear method for BitsAndBytes.

204 -

205 - Args:

206 - quant_config: The BitsAndBytes quantization config.

207 - """

208 -

209 - def __init__(self, quant_config: BitsAndBytesConfig):

210 - _check_bitsandbytes_version()

211 - self.quant_config = quant_config

212 -

213 - def create_weights(

214 - self,

215 - layer: torch.nn.Module,

216 - input_size_per_partition: int,

217 - output_partition_sizes: list[int],

218 - input_size: int,

219 - output_size: int,

220 - params_dtype: torch.dtype,

221 - **extra_weight_attrs,

222 - ):

223 - from bitsandbytes.nn import Int8Params

224 -

225 - def create_qweight_for_8bit():

226 - qweight = Int8Params(

227 - data=torch.empty(

228 - sum(output_partition_sizes),

229 - input_size_per_partition,

230 - dtype=torch.int8,

231 - ),

232 - has_fp16_weights=self.quant_config.llm_int8_has_fp16_weight,

233 - requires_grad=False,

234 - )

235 - set_weight_attrs(

236 - qweight,

237 - {

238 - "input_dim": 0,

239 - "output_dim": 0,

240 - "pack_factor": 1,

241 - "use_bitsandbytes_8bit": True,

242 - "generation": 0,

243 - },

244 - )

245 - return qweight

246 -

247 - def create_qweight_for_4bit():

248 - quant_ratio = calculate_quant_ratio(params_dtype)

249 -

250 - total_size = input_size_per_partition * sum(output_partition_sizes)

251 - if total_size % quant_ratio != 0:

252 - raise ValueError(

253 - "The input size is not aligned with the quantized weight shape."

254 - )

255 -

256 - qweight = BitsAndBytesWeightParameter(

257 - torch.empty(total_size // quant_ratio, 1, dtype=torch.uint8),

258 - requires_grad=False,

259 - )

260 - set_weight_attrs(

261 - qweight,

262 - {

263 - "input_dim": 0,

264 - "output_dim": 0,

265 - "pack_factor": quant_ratio,

266 - "use_bitsandbytes_4bit": True,

267 - },

268 - )

269 - return qweight

270 -

271 - if self.quant_config.load_in_8bit:

272 - qweight = create_qweight_for_8bit()

273 - else:

274 - qweight = create_qweight_for_4bit()

275 - # Enable parameters to have the same name as in the BNB

276 - # checkpoint format.

277 - layer.register_parameter("weight", qweight)

278 - set_weight_attrs(qweight, extra_weight_attrs)

279 -

280 - def apply(

281 - self,

282 - layer: torch.nn.Module,

283 - x: torch.Tensor,

284 - bias: torch.Tensor | None = None,

285 - ) -> torch.Tensor:

286 - if self.quant_config.load_in_8bit:

287 - return self._apply_8bit_weight(layer, x, bias)

288 - else:

289 - return self._apply_4bit_weight(layer, x, bias)

290 -

291 - def _apply_8bit_weight(

292 - self,

293 - layer: torch.nn.Module,

294 - x: torch.Tensor,

295 - bias: torch.Tensor | None = None,

296 - ) -> torch.Tensor:

297 - # only load the bitsandbytes module when needed

298 - from bitsandbytes import MatmulLtState, matmul

299 -

300 - original_type = x.dtype

301 - original_shape = x.shape

302 - reshape_after_matmul = False

303 - if x.ndim > 2:

304 - x = x.reshape(-1, x.size(-1))

305 - reshape_after_matmul = True

306 - bf_x = x.to(torch.bfloat16)

307 -

308 - qweight = layer.weight

309 - offsets = qweight.bnb_shard_offsets

310 - quant_states = qweight.bnb_quant_state

311 - matmul_states = qweight.matmul_state

312 - generation = qweight.generation

313 -

314 - out_dim_0 = x.shape[0]

315 - out_dim_1 = sum(

316 - [quant_state[1].shape[0] for quant_state in quant_states.items()]

317 - )

318 - out = torch.empty(out_dim_0, out_dim_1, dtype=torch.float16, device=x.device)

319 -

320 - current_index = 0

321 - for i in range(len(quant_states)):

322 - output_size = quant_states[i].shape[0]

323 -

324 - # in profile_run or the first generation of inference,

325 - # create new matmul_states

326 - if generation == 0 or generation == 1:

327 - matmul_states[i] = MatmulLtState()

328 - matmul_states[i].CB = qweight[offsets[i] : offsets[i + 1]]

329 - matmul_states[i].SCB = quant_states[i].to(x.device)

330 - matmul_states[i].threshold = self.quant_config.llm_int8_threshold

331 - matmul_states[

332 - i

333 - ].has_fp16_weights = self.quant_config.llm_int8_has_fp16_weight

334 - matmul_states[i].is_training = False

335 - if (

336 - matmul_states[i].threshold > 0.0

337 - and not matmul_states[i].has_fp16_weights

338 - ):

339 - matmul_states[i].use_pool = True

340 -

341 - new_x = bf_x.unsqueeze(0)

342 -

343 - out[:, current_index : current_index + output_size] = matmul(

344 - new_x, qweight[offsets[i] : offsets[i + 1]], state=matmul_states[i]

345 - )

346 -

347 - current_index += output_size

348 -

349 - out = out.to(original_type)

350 -

351 - if reshape_after_matmul:

352 - out = out.view(*original_shape[:-1], out.size(-1))

353 -

354 - if bias is not None:

355 - out += bias

356 -

357 - qweight.generation += 1

358 -

359 - return out

360 -

361 - def _apply_4bit_weight(

362 - self,

363 - layer: torch.nn.Module,

364 - x: torch.Tensor,

365 - bias: torch.Tensor | None = None,

366 - ) -> torch.Tensor:

367 - original_type = x.dtype

368 - original_shape = x.shape

369 - reshape_after_matmul = False

370 - if x.ndim > 2:

371 - x = x.reshape(-1, x.size(-1))

372 - reshape_after_matmul = True

373 - bf_x = x.to(torch.bfloat16)

374 -

375 - qweight = layer.weight

376 - quant_states = qweight.bnb_quant_state

377 - offsets = qweight.bnb_shard_offsets

378 -

379 - out_dim_0 = x.shape[0]

380 - out_dim_1 = sum(

381 - [quant_state[1].shape[0] for quant_state in quant_states.items()]

382 - )

383 - out = torch.empty(out_dim_0, out_dim_1, dtype=torch.bfloat16, device=x.device)

384 - apply_bnb_4bit(bf_x, qweight, offsets, out)

385 - out = out.to(original_type)

386 -

387 - if reshape_after_matmul:

388 - out = out.view(*original_shape[:-1], out.size(-1))

389 -

390 - if bias is not None:

391 - out += bias

392 -

393 - return out

394 -

395 -

396 -def _apply_bnb_4bit(

397 - x: torch.Tensor,

398 - weight: torch.Tensor,

399 - offsets: torch.Tensor,

400 - out: torch.Tensor,

401 -) -> None:

402 - # only load the bitsandbytes module when needed

403 - from bitsandbytes import matmul_4bit

404 -

405 - quant_states = weight.bnb_quant_state

406 - current_index = 0

407 - for i in range(len(quant_states)):

408 - output_size = quant_states[i].shape[0]

409 - # It is more efficient to use out kwarg like

410 - # matmul_4bit(..., out = ...). Infeasible now due to the bug

411 - # https://github.com/TimDettmers/bitsandbytes/issues/1235.

412 - # Need to change after the bug is fixed.

413 - out[:, current_index : current_index + output_size] = matmul_4bit(

414 - x, weight[offsets[i] : offsets[i + 1]].t(), quant_states[i]

415 - )

416 - current_index += output_size

417 -

418 -

419 -def _apply_bnb_4bit_fake(

420 - x: torch.Tensor,

421 - weight: torch.Tensor,

422 - offsets: torch.Tensor,

423 - out: torch.Tensor,

424 -) -> None:

425 - return

426 -

427 -

428 -try:

429 - direct_register_custom_op(

430 - op_name="apply_bnb_4bit",

431 - op_func=_apply_bnb_4bit,

432 - mutates_args=["out"],

433 - fake_impl=_apply_bnb_4bit_fake,

434 - dispatch_key=current_platform.dispatch_key,

435 - )

436 - apply_bnb_4bit = torch.ops.vllm.apply_bnb_4bit

437 -

438 -except AttributeError as error:

439 - raise error

440 -

441 -

442 -class BitsAndBytesMoEMethod(FusedMoEMethodBase):

443 - """MoE method for BitsAndBytes.

444 -

445 - Args:

446 - quant_config: The BitsAndBytes quantization config.

447 - """

448 -

449 - def __init__(

450 - self,

451 - quant_config: BitsAndBytesConfig,

452 - moe: FusedMoEConfig,

453 - ):

454 - super().__init__(moe)

455 - _check_bitsandbytes_version()

456 - self.quant_config = quant_config

457 -

458 - def create_weights(

459 - self,

460 - layer: RoutedExperts,

461 - num_experts: int,

462 - hidden_size: int,

463 - intermediate_size_per_partition: int,

464 - params_dtype: torch.dtype,

465 - **extra_weight_attrs,

466 - ):

467 - if self.quant_config.load_in_8bit:

468 - call_fun = self._create_weights_8bit

469 - else:

470 - call_fun = self._create_weights_4bit

471 - call_fun(

472 - layer,

473 - num_experts,

474 - hidden_size,

475 - intermediate_size_per_partition,

476 - params_dtype,

477 - **extra_weight_attrs,

478 - )

479 -

480 - def get_fused_moe_quant_config(

481 - self, layer: RoutedExperts

482 - ) -> FusedMoEQuantConfig | None:

483 - return None

484 -

485 - def apply(

486 - self,

487 - layer: RoutedExperts,

488 - x: torch.Tensor,

489 - topk_weights: torch.Tensor,

490 - topk_ids: torch.Tensor,

491 - shared_experts: SharedExperts | None,

492 - shared_experts_input: torch.Tensor | None,

493 - ) -> torch.Tensor:

494 - from vllm.model_executor.layers.fused_moe import fused_experts

495 -

496 - # TODO(bnell): Do these need to be called on the hot path?

497 - if self.quant_config.load_in_8bit:

498 - w13, w2 = self._apply_8bit_dequant(layer)

499 - else:

500 - w13, w2 = self._apply_4bit_dequnt(layer)

501 - return fused_experts(

502 - hidden_states=x,

503 - w1=w13,

504 - w2=w2,

505 - topk_weights=topk_weights,

506 - topk_ids=topk_ids,

507 - activation=layer.activation,

508 - apply_router_weight_on_input=layer.apply_router_weight_on_input,

509 - global_num_experts=layer.global_num_experts,

510 - expert_map=layer.expert_map,

511 - quant_config=self.moe_quant_config,

512 - )

513 -

514 - def _create_weights_4bit(

515 - self,

516 - layer: torch.nn.Module,

517 - num_experts: int,

518 - hidden_size: int,

519 - intermediate_size_per_partition: int,

520 - params_dtype: torch.dtype,

521 - **extra_weight_attrs,

522 - ):

523 - quant_ratio = calculate_quant_ratio(params_dtype)

524 - # Fused gate_up_proj (column parallel)

525 - w13_total_size = (

526 - hidden_size * self.moe.w13_num_shards * intermediate_size_per_partition

527 - ) // quant_ratio

528 - w13_qweight = torch.nn.Parameter(

529 - torch.empty(

530 - num_experts,

531 - w13_total_size,

532 - 1,

533 - dtype=torch.uint8,

534 - ),

535 - requires_grad=False,

536 - )

537 - layer.register_parameter("w13_weight", w13_qweight)

538 - set_weight_attrs(w13_qweight, extra_weight_attrs)

539 - set_weight_attrs(

540 - w13_qweight,

541 - {

542 - "num_experts": num_experts,

543 - "input_dim": hidden_size,

544 - "output_dim": self.moe.w13_num_shards * intermediate_size_per_partition,

545 - "experts_shape": (

546 - num_experts,

547 - intermediate_size_per_partition * self.moe.w13_num_shards,

548 - hidden_size,

549 - ),

550 - "pack_factor": quant_ratio,

551 - "use_bitsandbytes_4bit": True,

552 - },

553 - )

554 - # down_proj (row parallel)

555 - w2_total_size = (hidden_size * intermediate_size_per_partition) // quant_ratio

556 - w2_qweight = torch.nn.Parameter(

557 - torch.empty(

558 - num_experts,

559 - w2_total_size,

560 - 1,

561 - dtype=torch.uint8,

562 - ),

563 - requires_grad=False,

564 - )

565 - set_weight_attrs(

566 - w2_qweight,

567 - {

568 - "num_experts": num_experts,

569 - "input_dim": intermediate_size_per_partition,

570 - "output_dim": hidden_size,

571 - "experts_shape": (

572 - num_experts,

573 - hidden_size,

574 - intermediate_size_per_partition,

575 - ),

576 - "pack_factor": quant_ratio,

577 - "use_bitsandbytes_4bit": True,

578 - },

579 - )

580 - layer.register_parameter("w2_weight", w2_qweight)

581 - set_weight_attrs(w2_qweight, extra_weight_attrs)

582 -

583 - def _create_weights_8bit(

584 - self,

585 - layer: torch.nn.Module,

586 - num_experts: int,

587 - hidden_size: int,

588 - intermediate_size_per_partition: int,

589 - params_dtype: torch.dtype,

590 - **extra_weight_attrs,

591 - ):

592 - raise NotImplementedError

593 -

594 - def _apply_4bit_dequnt(

595 - self, layer: torch.nn.Module

596 - ) -> tuple[torch.Tensor, torch.Tensor]:

597 - from bitsandbytes.functional import dequantize_4bit

598 -

599 - w13 = dequantize_4bit(

600 - layer.w13_weight.reshape(-1, 1),

601 - layer.w13_weight.bnb_quant_state,

602 - )

603 - w2 = dequantize_4bit(

604 - layer.w2_weight.reshape(-1, 1),

605 - layer.w2_weight.bnb_quant_state,

606 - )

607 - w13 = w13.reshape(layer.w13_weight.experts_shape)

608 - w2 = w2.reshape(layer.w2_weight.experts_shape)

609 - return w13, w2

610 -

611 - def _apply_8bit_dequant(

612 - self, layer: torch.nn.Module

613 - ) -> tuple[torch.Tensor, torch.Tensor]:

614 - raise NotImplementedError

vllm/model_executor/model_loader/__init__.py

-4

@@ -9,7 +9,6 @@ from vllm.config import ModelConfig, VllmConfig

9 from vllm.config.load import LoadConfig

10 from vllm.logger import init_logger

11 from vllm.model_executor.model_loader.base_loader import BaseModelLoader

12 -from vllm.model_executor.model_loader.bitsandbytes_loader import BitsAndBytesModelLoader

12 from vllm.model_executor.model_loader.default_loader import DefaultModelLoader

13 from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader

14 from vllm.model_executor.model_loader.modelexpress_loader import (

@@ -33,7 +32,6 @@ logger = init_logger(__name__)

32 LoadFormats = Literal[

33 "auto",

34 "hf",

36 - "bitsandbytes",

35 "dummy",

36 "fastsafetensors",

37 "instanttensor",

@@ -50,7 +48,6 @@ LoadFormats = Literal[

48 _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = {

49 "auto": DefaultModelLoader,

50 "hf": DefaultModelLoader,

53 - "bitsandbytes": BitsAndBytesModelLoader,

51 "dummy": DummyModelLoader,

52 "fastsafetensors": DefaultModelLoader,

53 "instanttensor": DefaultModelLoader,

@@ -150,7 +147,6 @@ __all__ = [

147 "get_model_cls",

148 "register_model_loader",

149 "BaseModelLoader",

153 - "BitsAndBytesModelLoader",

150 "ModelExpressModelLoader",

151 "DefaultModelLoader",

152 "DummyModelLoader",

vllm/model_executor/model_loader/bitsandbytes_loader.py deleted

-834

@@ -1,834 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -import fnmatch

4 -import glob

5 -import itertools

6 -import math

7 -import os

8 -from collections.abc import Callable, Generator

9 -from typing import Any

10 -

11 -import numpy as np

12 -import torch

13 -from packaging import version

14 -from torch import nn

15 -from transformers.utils import SAFE_WEIGHTS_INDEX_NAME

16 -

17 -from vllm.config import ModelConfig

18 -from vllm.config.load import LoadConfig

19 -from vllm.distributed import (

20 - get_tensor_model_parallel_rank,

21 - get_tensor_model_parallel_world_size,

22 -)

23 -from vllm.logger import init_logger

24 -from vllm.lora.utils import is_moe_model

25 -from vllm.model_executor.layers.fused_moe import RoutedExperts

26 -from vllm.model_executor.layers.linear import (

27 - LinearBase,

28 - MergedColumnParallelLinear,

29 - QKVParallelLinear,

30 - ReplicatedLinear,

31 - RowParallelLinear,

32 -)

33 -from vllm.model_executor.model_loader.base_loader import BaseModelLoader

34 -from vllm.model_executor.model_loader.utils import ParamMapping

35 -from vllm.model_executor.model_loader.weight_utils import (

36 - download_safetensors_index_file_from_hf,

37 - download_weights_from_hf,

38 - filter_duplicate_safetensors_files,

39 - filter_files_not_needed_for_inference,

40 - pt_weights_iterator,

41 - safetensors_weights_iterator,

42 -)

43 -from vllm.model_executor.models import is_pooling_model

44 -from vllm.model_executor.utils import (

45 - get_moe_expert_mapping,

46 - get_packed_modules_mapping,

47 - set_weight_attrs,

48 -)

49 -from vllm.platforms import current_platform

50 -from vllm.transformers_utils.repo_utils import hf_api

51 -from vllm.utils.torch_utils import set_default_torch_dtype

52 -

53 -logger = init_logger(__name__)

54 -

55 -

56 -class BitsAndBytesModelLoader(BaseModelLoader):

57 - """Model loader to load model weights with BitsAndBytes quantization."""

58 -

59 - possible_config_file_names = ["adapter_config.json"]

60 -

61 - def __init__(self, load_config: LoadConfig):

62 - super().__init__(load_config)

63 -

64 - # Save the module names without sharding.

65 - self.unsharded_weights_modules: list[str] = []

66 - # Save the module names that are sharded by column.

67 - self.column_sharded_weights_modules: list[str] = []

68 - # Modules whose weights might have fused on disk

69 - # we need their output_sizes to make shard in flight correctly with TP

70 - self.maybe_fused_weights_modules: dict[str, list[int]] = {}

71 - # Store all module names (from transformers) that support

72 - # BNB quantization.

73 - self.target_modules: list[str] = []

74 - self.tp_disabled_modules: list[str] = []

75 - # Store the mapping of expert parameters for MoE models.

76 - self.expert_params_mapping: list[tuple[str, str, int, str]] = []

77 - # mapping weight names from transformers to vllm.

78 - self.weight_mapper: Callable = lambda name: name

79 - self.pre_quant: bool = False

80 - self.load_8bit: bool = False

81 - self.is_pool_model: bool = False

82 -

83 - def _get_weight_files(

84 - self,

85 - model_name_or_path: str,

86 - allowed_patterns: list[str],

87 - revision: str | None = None,

88 - ) -> tuple[str, list[str], str]:

89 - """Retrieve weight files. Download the files if necessary.

90 -

91 - Return the weight files and the file pattern."""

92 - is_local = os.path.isdir(model_name_or_path)

93 -

94 - if is_local:

95 - for pattern in allowed_patterns:

96 - weight_files = glob.glob(os.path.join(model_name_or_path, pattern))

97 - if weight_files:

98 - return model_name_or_path, weight_files, pattern

99 - else:

100 - repo_files = hf_api().list_repo_files(repo_id=model_name_or_path)

101 - for pattern in allowed_patterns:

102 - matching_files = fnmatch.filter(repo_files, pattern)

103 - if matching_files:

104 - hf_folder = download_weights_from_hf(

105 - model_name_or_path,

106 - self.load_config.download_dir,

107 - [pattern],

108 - revision,

109 - ignore_patterns=self.load_config.ignore_patterns,

110 - )

111 - return (

112 - hf_folder,

113 - glob.glob(os.path.join(hf_folder, pattern)),

114 - pattern,

115 - )

116 -

117 - raise RuntimeError(f"No model weights found in: `{model_name_or_path}`")

118 -

119 - def _prepare_weights(

120 - self, model_name_or_path: str, revision: str | None

121 - ) -> tuple[list[str], bool]:

122 - """Prepare weight files for the model."""

123 -

124 - allowed_patterns = ["*.safetensors", "*.bin", "*.pt"]

125 -

126 - hf_folder, hf_weights_files, matched_pattern = self._get_weight_files(

127 - model_name_or_path, allowed_patterns, revision

128 - )

129 -

130 - use_safetensors = matched_pattern == "*.safetensors"

131 - is_local = os.path.isdir(model_name_or_path)

132 - index_file = SAFE_WEIGHTS_INDEX_NAME

133 - if use_safetensors:

134 - # For models like Mistral-7B-Instruct-v0.3

135 - # there are both sharded safetensors files and a consolidated

136 - # safetensors file. Using both breaks.

137 - # Here, we download the `model.safetensors.index.json` and filter

138 - # any files not found in the index.

139 - if not is_local:

140 - download_safetensors_index_file_from_hf(

141 - model_name_or_path,

142 - index_file,

143 - cache_dir=self.load_config.download_dir,

144 - revision=revision,

145 - )

146 - hf_weights_files = filter_duplicate_safetensors_files(

147 - hf_weights_files, hf_folder, index_file

148 - )

149 - else:

150 - hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files)

151 -

152 - if len(hf_weights_files) == 0:

153 - raise RuntimeError(

154 - f"Cannot find any model weights with `{model_name_or_path}`"

155 - )

156 -

157 - return hf_weights_files, use_safetensors

158 -

159 - def _hf_weight_iter(self, hf_weights_files, use_safetensors: bool):

160 - def _maybe_pool_model(module_name: str):

161 - # For pool model, we need to add the prefix `model.`

162 - # for the weight name if possible.

163 - if (

164 - self.is_pool_model

165 - and self.target_modules[0].startswith("model.")

166 - and not module_name.startswith("model.")

167 - ):

168 - return "model." + module_name

169 -

170 - return module_name

171 -

172 - if use_safetensors:

173 - iterator = safetensors_weights_iterator(

174 - hf_weights_files,

175 - self.load_config.use_tqdm_on_load,

176 - )

177 - else:

178 - iterator = pt_weights_iterator(

179 - hf_weights_files,

180 - self.load_config.use_tqdm_on_load,

181 - self.load_config.pt_load_map_location,

182 - )

183 - for org_name, param in iterator:

184 - # mapping weight names from transformers to vllm while preserving

185 - # original names.

186 - mapped_name = self.weight_mapper(org_name)

187 - mapped_name = _maybe_pool_model(mapped_name)

188 -

189 - yield org_name, mapped_name, param

190 -

191 - def _get_quantized_weights_iterator(

192 - self,

193 - model_name_or_path: str,

194 - revision: str | None,

195 - ) -> tuple[Generator[tuple[str, torch.Tensor], None, None], dict[str, Any]]:

196 - """Get an iterator to the model weights with bitsandbytes quantization,

197 - as well as the quantization state dictionary."""

198 -

199 - # only load the bitsandbytes module when needed

200 - try:

201 - import bitsandbytes

202 -

203 - if version.parse(bitsandbytes.__version__) < version.parse("0.46.1"):

204 - raise ImportError(

205 - "bitsandbytes version is wrong. Please "

206 - "install bitsandbytes>=0.46.1."

207 - )

208 - except ImportError as err:

209 - raise ImportError(

210 - "Please install bitsandbytes>=0.46.1 via "

211 - "`pip install bitsandbytes>=0.46.1` to use "

212 - "bitsandbytes quantizer."

213 - ) from err

214 -

215 - hf_weights_files, use_safetensors = self._prepare_weights(

216 - model_name_or_path, revision

217 - )

218 -

219 - quant_state_dict: dict[str, Any] = {}

220 -

221 - if self.pre_quant:

222 - if self.load_8bit:

223 - return self._quantized_8bit_generator(

224 - hf_weights_files, use_safetensors, quant_state_dict

225 - ), quant_state_dict

226 - else:

227 - return self._quantized_4bit_generator(

228 - hf_weights_files, use_safetensors, quant_state_dict

229 - ), quant_state_dict

230 -

231 - return self._unquantized_generator(

232 - hf_weights_files, use_safetensors, quant_state_dict

233 - ), quant_state_dict

234 -

235 - def _is_8bit_weight_name(self, weight_name: str):

236 - quantized_suffix = {".scb", ".weight_format"}

237 - return any(weight_name.lower().endswith(suffix) for suffix in quantized_suffix)

238 -

239 - def _is_4bit_weight_name(self, weight_name: str):

240 - quantized_suffix = {

241 - "absmax",

242 - "quant_map",

243 - "nested_absmax",

244 - "nested_quant_map",

245 - "bitsandbytes",

246 - }

247 - suffix = weight_name.split(".")[-1]

248 - return any(q_suffix in suffix for q_suffix in quantized_suffix)

249 -

250 - def _quantized_8bit_generator(

251 - self, hf_weights_files, use_safetensors, quant_state_dict

252 - ) -> Generator:

253 - for (

254 - org_weight_name,

255 - mapped_weight_name,

256 - weight_tensor,

257 - ) in self._hf_weight_iter(hf_weights_files, use_safetensors):

258 - if not mapped_weight_name.lower().endswith(".scb"):

259 - continue

260 -

261 - weight_key = mapped_weight_name.lower().replace(".scb", ".weight")

262 - quant_state_dict[weight_key] = weight_tensor

263 -

264 - for (

265 - org_weight_name,

266 - mapped_weight_name,

267 - weight_tensor,

268 - ) in self._hf_weight_iter(hf_weights_files, use_safetensors):

269 - if self._is_8bit_weight_name(mapped_weight_name):

270 - continue

271 -

272 - if mapped_weight_name in quant_state_dict:

273 - set_weight_attrs(weight_tensor, {"load_in_8bit": True})

274 - yield org_weight_name, weight_tensor

275 - else:

276 - yield org_weight_name, weight_tensor

277 -

278 - def _quantized_4bit_generator(

279 - self, hf_weights_files, use_safetensors, quant_state_dict

280 - ) -> Generator:

281 - from bitsandbytes.functional import QuantState

282 -

283 - # First iterate over all quant state weights

284 - weight_iterator = self._hf_weight_iter(hf_weights_files, use_safetensors)

285 - temp_state_dict = {}

286 - for (

287 - org_weight_name,

288 - mapped_weight_name,

289 - weight_tensor,

290 - ) in weight_iterator:

291 - if not self._is_4bit_weight_name(mapped_weight_name):

292 - continue

293 - # bitsandbytes library requires

294 - # weight.quant_state.bitsandbytes__* in CPU

295 - if "quant_state.bitsandbytes" in mapped_weight_name:

296 - temp_state_dict[mapped_weight_name] = weight_tensor.cpu().data

297 - else:

298 - temp_state_dict[mapped_weight_name] = weight_tensor

299 -

300 - # Closure to parse quant_state for each prequant weight

301 - def _parse_quant_state(param_name: str, temp_state_dict: dict) -> QuantState:

302 - quant_state = {}

303 - for k in temp_state_dict:

304 - if param_name + "." in k:

305 - quant_state[k] = temp_state_dict[k]

306 -

307 - return QuantState.from_dict(

308 - quant_state, device=current_platform.device_type

309 - )

310 -

311 - # Second iterate over all prequant and normal weights

312 - # pre quantized weights would have a quant_state

313 - for (

314 - org_weight_name,

315 - mapped_weight_name,

316 - weight_tensor,

317 - ) in self._hf_weight_iter(hf_weights_files, use_safetensors):

318 - if self._is_4bit_weight_name(mapped_weight_name):

319 - continue

320 -

321 - if (

322 - f"{mapped_weight_name}.quant_state.bitsandbytes__nf4" in temp_state_dict

323 - ) or (

324 - f"{mapped_weight_name}.quant_state.bitsandbytes__fp4" in temp_state_dict

325 - ):

326 - quant_state = _parse_quant_state(mapped_weight_name, temp_state_dict)

327 - quant_state_dict[mapped_weight_name] = quant_state

328 - yield org_weight_name, weight_tensor

329 - else:

330 - yield org_weight_name, weight_tensor

331 -

332 - def _unquantized_generator(

333 - self, hf_weights_files, use_safetensors, quant_state_dict

334 - ) -> Generator:

335 - from bitsandbytes.functional import quantize_4bit

336 -

337 - global_tp_size = get_tensor_model_parallel_world_size()

338 - global_tp_rank = get_tensor_model_parallel_rank()

339 - check_match = (

340 - lambda weight_name, module_name: weight_name.removesuffix(".weight")

341 - == module_name

342 - )

343 - for (

344 - org_weight_name,

345 - mapped_weight_name,

346 - weight_tensor,

347 - ) in self._hf_weight_iter(hf_weights_files, use_safetensors):

348 - # override tp_size and tp_rank if the module has disabled TP

349 - if any(

350 - tp_disabled_module in mapped_weight_name

351 - for tp_disabled_module in self.tp_disabled_modules

352 - ):

353 - tp_size = 1

354 - tp_rank = 0

355 - else:

356 - tp_size = global_tp_size

357 - tp_rank = global_tp_rank

358 -

359 - if any(

360 - target_module in mapped_weight_name

361 - for target_module in self.target_modules

362 - ) and mapped_weight_name.endswith(".weight"):

363 - # Without sharding

364 - if any(

365 - check_match(mapped_weight_name, module)

366 - for module in self.unsharded_weights_modules

367 - ):

368 - weight_sub_tensor = weight_tensor

369 - # Shard by column

370 - elif any(

371 - check_match(mapped_weight_name, module)

372 - for module in self.column_sharded_weights_modules

373 - ):

374 - total_size = weight_tensor.size(-1)

375 - start_index = total_size // tp_size * tp_rank

376 - end_index = total_size // tp_size * (tp_rank + 1)

377 - weight_sub_tensor = weight_tensor[..., start_index:end_index]

378 - # Weights have fused on disk. In this case, we assume that the

379 - # weight and module use same name.

380 - elif any(

381 - check_match(mapped_weight_name, module)

382 - for module in self.maybe_fused_weights_modules

383 - ):

384 - # special case for fused weights

385 - # get the size of each shard weight tensor

386 - total_shard_sizes = next(

387 - (

388 - sizes

389 - for module, sizes in self.maybe_fused_weights_modules.items() # noqa: E501

390 - if check_match(mapped_weight_name, module)

391 - )

392 - )

393 - total_size = weight_tensor.size(0)

394 - assert total_size == sum(total_shard_sizes)

395 - # get the start/end index of each shard weight tensor

396 - total_start_index = list(

397 - itertools.accumulate([0] + total_shard_sizes)

398 - )[:-1]

399 - shard_weights_index = [

400 - (

401 - idx + size // tp_size * tp_rank,

402 - idx + size // tp_size * (tp_rank + 1),

403 - )

404 - for idx, size in zip(total_start_index, total_shard_sizes)

405 - ]

406 - # slice and reorder the weight tensor

407 - weight_tensor = [

408 - weight_tensor[start_index:end_index, ...]

409 - for start_index, end_index in shard_weights_index

410 - ]

411 - weight_sub_tensor = torch.cat(weight_tensor, dim=0)

412 - # Shard by row

413 - else:

414 - total_size = weight_tensor.size(0)

415 - start_index = total_size // tp_size * tp_rank

416 - end_index = total_size // tp_size * (tp_rank + 1)

417 - weight_sub_tensor = weight_tensor[start_index:end_index, ...]

418 -

419 - # bitsandbytes requires data in GPU

420 - if weight_sub_tensor.is_cuda:

421 - loaded_weight = weight_sub_tensor

422 - else:

423 - loaded_weight = weight_sub_tensor.to(

424 - device=current_platform.device_type

425 - )

426 -

427 - # remove the following after the issue is fixed:

428 - # https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1342

429 - if loaded_weight.is_contiguous() is False:

430 - loaded_weight = loaded_weight.contiguous()

431 -

432 - with set_default_torch_dtype(torch.float32):

433 - processed_weight, quant_state = quantize_4bit(

434 - loaded_weight,

435 - compress_statistics=True,

436 - quant_type="nf4",

437 - )

438 -

439 - quant_state_dict[mapped_weight_name] = quant_state

440 - else:

441 - processed_weight = weight_tensor

442 - yield org_weight_name, processed_weight

443 -

444 - def _get_bnb_target_modules(self, model: nn.Module) -> None:

445 - """

446 - Identify and collect all modules that support BitsAndBytes

447 - quantization.

448 - """

449 - for name, module in model.named_modules():

450 - if isinstance(module, LinearBase) and hasattr(

451 - module.quant_method, "quant_config"

452 - ):

453 - if modules_info := self.modules_mapping.get_sub_modules(name):

454 - # Map vllm's names to transformers's names.

455 - rep_name, sub_modules = modules_info

456 - for sub_name in sub_modules:

457 - new_name = name.replace(rep_name, sub_name)

458 - self.target_modules.append(new_name)

459 - if module.disable_tp:

460 - self.tp_disabled_modules.append(new_name)

461 - # Add original module name even if the module has stacked map,

462 - # in case model has a mixture of disk-merged and disk-split

463 - # weights with same last name.

464 - self.target_modules.append(name)

465 - if module.disable_tp:

466 - self.tp_disabled_modules.append(name)

467 - elif isinstance(module, RoutedExperts) and hasattr(

468 - module.quant_method, "quant_config"

469 - ):

470 - # TODO: support RoutedExperts with prequant and 8bit.

471 - if self.pre_quant and self.load_8bit:

472 - raise ValueError(

473 - "Prequant BitsAndBytes 8bit models with RoutedExperts "

474 - "is not supported yet."

475 - )

476 - # Get the corresponding weight name using module name and

477 - # expert_params_mapping.

478 -

479 - for exp in self.expert_params_mapping:

480 - weight_name = exp[1]

481 - rep_name = name.replace("experts", "") + weight_name.removesuffix(

482 - "."

483 - )

484 - self.target_modules.append(rep_name)

485 -

486 - assert self.target_modules, (

487 - "vLLM currently does not support BNB quantization for"

488 - )

489 - f" {type(model).__name__}"

490 -

491 - def _classify_module_sharding(self, model: nn.Module):

492 - """

493 - Categorize modules based on their weight sharding requirements

494 - for tensor parallelism.

495 - """

496 - for name, module in model.named_modules():

497 - # Some modules like `ReplicatedLinear` should not have their weights

498 - # sharded. The reason for implementing it this way is to avoid new

499 - # static variable in the model implementation.

500 - if isinstance(module, (ReplicatedLinear,)):

501 - self.unsharded_weights_modules.append(name)

502 - # `QKVParallelLinear` and `MergedColumnParallelLinear` might have

503 - # fused weights on disk. We need to use the output sizes of these

504 - # modules to shard the weights correctly.

505 - elif isinstance(module, (QKVParallelLinear, MergedColumnParallelLinear)):

506 - self.maybe_fused_weights_modules[name] = module.output_sizes

507 - # In TP, these weights are partitioned along the column

508 - # dimension (dim=-1)

509 - elif isinstance(module, (RowParallelLinear,)):

510 - self.column_sharded_weights_modules.append(name)

511 - elif isinstance(module, RoutedExperts):

512 - expert_mapping = self.expert_params_mapping

513 - for exp in expert_mapping:

514 - if exp[-1] == "w2":

515 - weight_name = exp[1]

516 - rep_name = name.replace(

517 - "experts", ""

518 - ) + weight_name.removesuffix(".")

519 - self.column_sharded_weights_modules.append(rep_name)

520 -

521 - def _verify_model_compatibility(

522 - self, model: nn.Module, model_config: ModelConfig

523 - ) -> None:

524 - """

525 - Verify that the model is compatible with BitsAndBytes quantization.

526 - """

527 - if not hasattr(model, "load_weights"):

528 - raise AttributeError(

529 - "The required method 'load_weights' is not defined in class"

530 - f" {type(model).__name__}."

531 - )

532 -

533 - if not hasattr(model, "packed_modules_mapping"):

534 - raise AttributeError(

535 - f"Model {type(model).__name__} does not support BitsAndBytes "

536 - "quantization yet. No 'packed_modules_mapping' found."

537 - )

538 -

539 - quant_config = getattr(model_config.hf_config, "quantization_config", None)

540 - if quant_config and (quant_method := quant_config.get("quant_method")):

541 - if quant_method == "bitsandbytes":

542 - self.pre_quant = True

543 - else:

544 - raise ValueError(

545 - f"BitsAndBytes loader does not support {quant_method} quantization"

546 - )

547 -

548 - # The quant_states in pre_quantized models cannot work with a split

549 - # weight tensor. So TP does not work with pre_quantized bnb models.

550 - if self.pre_quant and get_tensor_model_parallel_world_size() > 1:

551 - raise ValueError(

552 - "Prequant BitsAndBytes models with tensor parallelism is not "

553 - "supported. Please try with pipeline parallelism."

554 - )

555 - if quant_config and self.pre_quant:

556 - self.load_8bit = quant_config.get("load_in_8bit", False)

557 -

558 - def _initialize_loader_state(

559 - self, model: nn.Module, model_config: ModelConfig

560 - ) -> None:

561 - """

562 - Initialize the loader's internal state based on the model and

563 - configuration.

564 - """

565 - self.is_pool_model = is_pooling_model(model)

566 - self.modules_mapping = ParamMapping(get_packed_modules_mapping(model))

567 -

568 - if is_moe_model(model):

569 - self.expert_params_mapping = get_moe_expert_mapping(model)

570 - # For some models like Molmo, we need to use hf_to_vllm_mapper

571 - # to ensure correct loading of weights.

572 - if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None):

573 - unstacked_mapper = hf_to_vllm_mapper.get_unstacked_mapper()

574 - self.weight_mapper = lambda name, m=unstacked_mapper: m._map_name(name)

575 -

576 - self._get_bnb_target_modules(model)

577 - self._classify_module_sharding(model)

578 -

579 - def _dequantize_dq(self, quant_states: Any):

580 - """

581 - When BNB employs Double Quantization, we perform the dequantization of

582 - these constants during weight loading rather than at inference time,

583 - thereby avoiding this computational overhead during inference. This

584 - comes at the cost of increased memory usage.

585 - """

586 - from bitsandbytes.functional import QuantState, dequantize_blockwise

587 -

588 - def _dequantize_single_state(quant_state):

589 - """Helper function to dequantize a single QuantState object."""

590 - if not (isinstance(quant_state, QuantState) and quant_state.nested):

591 - return

592 -

593 - # Copied from: https://github.com/bitsandbytes-foundation/bitsandbytes/blob/0.45.3/bitsandbytes/functional.py#L1352-#L1356

594 - absmax = dequantize_blockwise(quant_state.absmax, quant_state.state2)

595 - absmax += quant_state.offset

596 -

597 - # Ensure float32 dtype

598 - if absmax.dtype != torch.float32:

599 - absmax = absmax.float()

600 -

601 - quant_state.absmax = absmax

602 - quant_state.nested = False

603 - quant_state.offset = None

604 - quant_state.state2 = None

605 -

606 - if isinstance(quant_states, dict):

607 - for quant_state in quant_states.values():

608 - _dequantize_single_state(quant_state)

609 - else:

610 - _dequantize_single_state(quant_states)

611 - return quant_states

612 -

613 - def _fuse_moe_quant_states(self, model: nn.Module, quant_states_dict: dict) -> dict:

614 - """

615 -

616 - This function consolidates individual expert quantization states into

617 - fused representations for w13 and w2.

618 - """

619 - from bitsandbytes.functional import QuantState

620 -

621 - if not self.expert_params_mapping:

622 - return dict()

623 -

624 - expert_mapping = self.expert_params_mapping

625 - expert_qs_dict = {}

626 - for name, module in model.named_modules():

627 - if not isinstance(module, RoutedExperts):

628 - continue

629 - w1_states_lst = []

630 - w2_states_lst = []

631 - w3_states_lst = []

632 - for exp in expert_mapping:

633 - shard_id = exp[-1]

634 - if shard_id not in ("w1", "w2", "w3"):

635 - raise ValueError(

636 - f"shard_id must be ['w1','w2','w3'] but got {shard_id}."

637 - )

638 - layer_prefix = name.split("experts")[0]

639 - weight_qual_name = layer_prefix + exp[1] + "weight"

640 - quant_state = self._dequantize_dq(quant_states_dict[weight_qual_name])

641 - if shard_id == "w1":

642 - w1_states_lst.append(quant_state)

643 - elif shard_id == "w2":

644 - w2_states_lst.append(quant_state)

645 - else:

646 - w3_states_lst.append(quant_state)

647 - del quant_states_dict[weight_qual_name]

648 - assert len(w1_states_lst) == len(w2_states_lst) == len(w3_states_lst)

649 - w13_absmax_lst = []

650 - w2_absmax_lst = []

651 - w13_total_dim0 = 0

652 - w2_total_dim0 = 0

653 - for w1_qs, w2_qs, w3_qs in zip(w1_states_lst, w2_states_lst, w3_states_lst):

654 - assert w1_qs.shape == w3_qs.shape

655 - assert w1_qs.blocksize == w2_qs.blocksize == w3_qs.blocksize

656 - assert w1_qs.dtype == w2_qs.dtype == w3_qs.dtype

657 - # w1 and w3 are interleaved in storage

658 - w13_absmax_lst.append(w1_qs.absmax)

659 - w13_absmax_lst.append(w3_qs.absmax)

660 - w2_absmax_lst.append(w2_qs.absmax)

661 - w13_total_dim0 += w1_qs.shape[0] + w3_qs.shape[0]

662 - w2_total_dim0 += w2_qs.shape[0]

663 -

664 - w13_absmax = torch.cat(w13_absmax_lst)

665 - w2_absmax = torch.cat(w2_absmax_lst)

666 - # Create fused quantization state for w13.

667 - w13_qs = QuantState(

668 - absmax=w13_absmax,

669 - shape=(w13_total_dim0, w1_states_lst[0].shape[1]),

670 - code=w1_states_lst[0].code,

671 - blocksize=w1_states_lst[0].blocksize,

672 - quant_type="nf4",

673 - dtype=w1_states_lst[0].dtype,

674 - )

675 - # Create fused quantization state for w2.

676 - w2_qs = QuantState(

677 - absmax=w2_absmax,

678 - shape=(w2_total_dim0, w2_states_lst[0].shape[1]),

679 - code=w2_states_lst[0].code,

680 - blocksize=w2_states_lst[0].blocksize,

681 - quant_type="nf4",

682 - dtype=w2_states_lst[0].dtype,

683 - )

684 - # The weight suffixes .w13_weight and .w2_weight are consistent

685 - # with the param in BitsAndBytesMoEMethod.

686 - w13_weight_name = name + ".w13_weight"

687 - w2_weight_name = name + ".w2_weight"

688 - expert_qs_dict[w13_weight_name] = w13_qs

689 - expert_qs_dict[w2_weight_name] = w2_qs

690 - return expert_qs_dict

691 -

692 - def _stack_quantization_states(

693 - self, model: nn.Module, quant_state_dict: dict

694 - ) -> dict[str, dict[int, Any]]:

695 - stacked_quant_state_dict: dict[str, dict[int, Any]] = {}

696 - # TODO: Change this lazy import to normal import

697 - # after the checks are updated to run on a new version

698 - from vllm.model_executor.models.utils import is_pp_missing_parameter

699 -

700 - param_dict = dict(model.named_parameters())

701 - for quant_param_name in quant_state_dict:

702 - if is_pp_missing_parameter(quant_param_name, model):

703 - continue

704 -

705 - non_stacked_param_name = quant_param_name

706 -

707 - shard_index = 0

708 - for shard_name, (

709 - weight_name,

710 - index,

711 - ) in self.modules_mapping.inverse_packed_mapping.items():

712 - # Some models, such as MiniCPM V2.5/2.6, contain both

713 - # module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'

714 - # from being incorrectly identified as being present in

715 - # 'vpm.encoder.layers.0.self_attn.qkv_proj.weight

716 - shard_pos = quant_param_name.find(shard_name)

717 - can_correct_rename = (shard_pos > 0) and (

718 - quant_param_name[shard_pos - 1] == "."

719 - )

720 - # If the quant_param_name is packed, it won't occur in the

721 - # param_dict before renaming.

722 - new_quant_param_name = quant_param_name.replace(shard_name, weight_name)

723 - need_rename = (quant_param_name not in param_dict) and (

724 - new_quant_param_name in param_dict

725 - )

726 - if can_correct_rename and need_rename:

727 - shard_index = index

728 - quant_param_name = new_quant_param_name

729 - break

730 -

731 - # Models like Clip/Siglip may skip some layers in initialization,

732 - # causing unused quant_param_name in state_dict.

733 - if quant_param_name not in param_dict:

734 - continue

735 -

736 - if quant_param_name not in stacked_quant_state_dict:

737 - stacked_quant_state_dict[quant_param_name] = {}

738 -

739 - stacked_quant_state_dict[quant_param_name][shard_index] = quant_state_dict[

740 - non_stacked_param_name

741 - ]

742 -

743 - # repeat k_proj for v_proj for k_eq_v models (e.g. Gemma4)

744 - config = getattr(model, "config", None)

745 - if config is not None:

746 - text_config = config.get_text_config()

747 - if getattr(text_config, "attention_k_eq_v", False):

748 - shard_packed = {

749 - name

750 - for name, subs in self.modules_mapping.packed_mapping.items()

751 - if len(subs) == 3

752 - }

753 - for param_name, shards in stacked_quant_state_dict.items():

754 - is_target = (

755 - isinstance(shards, dict)

756 - and len(shards) == 2

757 - and any(

758 - param_name.endswith(f"{p}.weight") for p in shard_packed

759 - )

760 - )

761 - if is_target:

762 - assert 1 in shards and 2 not in shards

763 - shards[2] = shards[1]

764 -

765 - return stacked_quant_state_dict

766 -

767 - def _bind_quant_states_to_params(

768 - self, model: nn.Module, stacked_quant_state_dict: dict

769 - ) -> None:

770 - # save quant_states and offsets as the attributes of the parameters

771 - param_dict = dict(model.named_parameters())

772 - for param_name, param in param_dict.items():

773 - if param_name in stacked_quant_state_dict:

774 - quant_states = stacked_quant_state_dict[param_name]

775 - # Dequantize double quantized values during weight loading.

776 - self._dequantize_dq(quant_states)

777 - set_weight_attrs(param, {"bnb_quant_state": quant_states})

778 - if not isinstance(quant_states, dict):

779 - continue

780 -

781 - pack_ratio = getattr(param, "pack_factor", -1)

782 - if pack_ratio == -1:

783 - raise ValueError(f"pack_factor not set for parameter {param_name}.")

784 -

785 - num_elements = [0] * len(quant_states)

786 - for seq, quant_state in quant_states.items():

787 - num_elements[seq] = math.prod(quant_state.shape) // pack_ratio

788 -

789 - offsets = np.concatenate(([0], np.cumsum(num_elements)))

790 - # Make torch infer_schema happy

791 - offsets = torch.tensor(offsets).cpu()

792 - set_weight_attrs(param, {"bnb_shard_offsets": offsets})

793 -

794 - if self.load_8bit:

795 - set_weight_attrs(

796 - param, {"matmul_state": [None] * len(quant_states)}

797 - )

798 -

799 - def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:

800 - self._verify_model_compatibility(model, model_config)

801 - self._initialize_loader_state(model, model_config)

802 -

803 - logger.info(

804 - "Loading weights with BitsAndBytes quantization. May take a while ..."

805 - )

806 - qweight_iterator, quant_state_dict = self._get_quantized_weights_iterator(

807 - model_config.model,

808 - model_config.revision,

809 - )

810 - weights_to_load = {name for name, _ in model.named_parameters()}

811 - loaded_weights = model.load_weights(qweight_iterator)

812 - # Some models may have weights loading tracker unimplemented.

813 - if loaded_weights is not None:

814 - weights_not_loaded = weights_to_load - loaded_weights

815 - if weights_not_loaded:

816 - raise ValueError(

817 - "Following weights were not initialized from "

818 - f"checkpoint: {weights_not_loaded}"

819 - )

820 - expert_quant_state_dict = self._fuse_moe_quant_states(model, quant_state_dict)

821 -

822 - stacked_quant_state_dict = self._stack_quantization_states(

823 - model, quant_state_dict

824 - )

825 -

826 - stacked_quant_state_dict = {

827 - **expert_quant_state_dict,

828 - **stacked_quant_state_dict,

829 - }

830 - self._bind_quant_states_to_params(model, stacked_quant_state_dict)

831 - torch.accelerator.empty_cache()

832 -

833 - def download_model(self, model_config: ModelConfig) -> None:

834 - self._prepare_weights(model_config.model, model_config.revision)

vllm/model_executor/model_loader/utils.py

-30

@@ -5,7 +5,6 @@

5 import inspect

6 import warnings

7 from contextlib import contextmanager

8 -from dataclasses import dataclass, field

8 from typing import Any

10 import torch

@@ -263,35 +262,6 @@ def get_architecture_class_name(model_config: ModelConfig) -> str:

262 return get_model_architecture(model_config)[1]

263

264

266 -@dataclass

267 -class ParamMapping:

268 - """

269 - A class to handle parameter mapping for model weight loading.

270 - It creates a bidirectional mapping between packed parameters and their

271 - constituent parts.

272 - """

273 -

274 - packed_mapping: dict[str, list[str]]

275 - inverse_packed_mapping: dict[str, tuple[str, int]] = field(default_factory=dict)

276 -

277 - def __post_init__(self):

278 - for packed_name, sub_params in self.packed_mapping.items():

279 - # Skip self-contained cases (e.g., {"W_pack": ["W_pack"]})

280 - if len(sub_params) == 1 and sub_params[0] == packed_name:

281 - continue

282 - for index, param_name in enumerate(sub_params):

283 - self.inverse_packed_mapping[param_name] = (

284 - packed_name,

285 - index,

286 - )

287 -

288 - def get_sub_modules(self, module_name: str) -> tuple[str, list[str]] | None:

289 - for key, value in self.packed_mapping.items():

290 - if module_name.endswith(key):

291 - return key, value

292 - return None

293 -

294 -

265 def configure_quant_config(

266 quant_config: QuantizationConfig, model_class: type[nn.Module]

267 ):

vllm/model_executor/model_loader/weight_utils.py

+1 -6

@@ -330,9 +330,6 @@ def get_quant_config(

330 assert isinstance(model_config.quantization_config, QuantizationConfigArgs)

331 return OnlineQuantizationConfig(args=model_config.quantization_config)

332

333 - # Inflight BNB quantization

334 - if model_config.quantization == "bitsandbytes":

335 - return quant_cls.from_config({})

333 model_name_or_path = (

334 maybe_download_from_modelscope(

335 model_config.model,

@@ -380,9 +377,7 @@ def get_quant_config(

377 with open(quant_config_file) as f:

378 config = json.load(f)

379

383 - if model_config.quantization == "bitsandbytes":

384 - config["adapter_name_or_path"] = model_config.model

385 - elif model_config.quantization in ("modelopt", "modelopt_mixed"):

380 + if model_config.quantization in ("modelopt", "modelopt_mixed"):

381 if config.get("producer", {}).get("name") == "modelopt":

382 return quant_cls.from_config(config)

383 else:

vllm/model_executor/models/openpangu.py

-4

@@ -699,10 +699,6 @@ class OpenPanguSinkAttention(nn.Module):

699 output_dim = getattr(param, "output_dim", None)

700

701 is_sharded_weight = getattr(param, "is_sharded_weight", False)

702 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)

703 - # bitsandbytes loads the weights of the specific portion

704 - # no need to narrow

705 - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit

702

703 param_data = param.data

704 if output_dim is not None and not is_sharded_weight:

vllm/platforms/rocm.py

-1

@@ -524,7 +524,6 @@ class RocmPlatform(Platform):

524 "mxfp4",

525 "mxfp8",

526 "torchao",

527 - "bitsandbytes",

527 "modelopt",

528 "modelopt_fp4",

529 "modelopt_mxfp8",

vllm/v1/metrics/perf.py

+1 -2

@@ -46,7 +46,7 @@ class InvalidComponent(Exception):

46 # FfnQuantizationConfigParser to determine the weight_byte_size for

47 # flops/memory estimation.

48 #

49 -# NOTE: Methods like GPTQ and BitsAndBytes support variable bit-widths

49 +# NOTE: Methods like GPTQ support variable bit-widths

50 # (e.g., 4-bit and 8-bit). We default to 4-bit (0.5 bytes) since this

51 # is by far the most common configuration.

52 _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = {

@@ -63,7 +63,6 @@ _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = {

63 "awq_marlin": 0.5,

64 "gptq": 0.5,

65 "gptq_marlin": 0.5,

66 - "bitsandbytes": 0.5,

66 "modelopt_fp4": 0.5,

67 "petit_nvfp4": 0.5,

68 "compressed-tensors": 0.5,