| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """End-to-end tests for the out-of-tree bitsandbytes plugin.""" |
| 4 | |
| 5 | import types |
| 6 | from unittest.mock import MagicMock, patch |
| 7 | |
| 8 | import pytest |
| 9 | from packaging.version import Version |
| 10 | from transformers import BitsAndBytesConfig |
| 11 | from transformers import __version__ as TRANSFORMERS_VERSION |
| 12 | from vllm_bnb_plugin import bitsandbytes_loader as bnb |
| 13 | |
| 14 | from vllm.platforms import current_platform |
| 15 | |
| 16 | from ...models.utils import check_embeddings_close, check_logprobs_close |
| 17 | from ...utils import multi_gpu_test |
| 18 | |
| 19 | if current_platform.is_rocm(): |
| 20 | from vllm.platforms.rocm import on_cdna |
| 21 | |
| 22 | pytestmark = pytest.mark.skipif( |
| 23 | on_cdna(), |
| 24 | reason="bitsandbytes not supported on CDNA (warp size 64 limitation)", |
| 25 | ) |
| 26 | |
| 27 | models_4bit_to_test = [ |
| 28 | ("facebook/opt-125m", "quantize opt model inflight"), |
| 29 | ( |
| 30 | "mistralai/Mistral-7B-Instruct-v0.3", |
| 31 | "quantize inflight model with both HF and Mistral format weights", |
| 32 | ), |
| 33 | ] |
| 34 | |
| 35 | models_4bit_to_embedding_test = [ |
| 36 | ("intfloat/e5-mistral-7b-instruct", "quantize embedding model inflight"), |
| 37 | ] |
| 38 | |
| 39 | models_4bit_to_moe_test = [ |
| 40 | ("allenai/OLMoE-1B-7B-0125-Instruct", "quantize moe model inflight"), |
| 41 | ] |
| 42 | |
| 43 | models_pre_quant_4bit_to_test = [ |
| 44 | ( |
| 45 | "PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed", |
| 46 | "read pre-quantized 4-bit FP4 model", |
| 47 | ), |
| 48 | ("poedator/opt-125m-bnb-4bit", "read pre-quantized 4-bit NF4 opt model"), |
| 49 | ] |
| 50 | |
| 51 | models_pre_quant_8bit_to_test = [ |
| 52 | ("meta-llama/Llama-Guard-3-8B-INT8", "read pre-quantized llama 8-bit model"), |
| 53 | ("yec019/fbopt-350m-8bit", "read pre-quantized 8-bit opt model"), |
| 54 | ] |
| 55 | |
| 56 | |
| 57 | def log_generated_texts(prompts, outputs, runner_name): |
| 58 | logged_texts = [] |
| 59 | for i, (_, generated_text) in enumerate(outputs): |
| 60 | logged_texts.append( |
| 61 | { |
| 62 | "prompt": prompts[i], |
| 63 | "runner_name": runner_name, |
| 64 | "generated_text": generated_text, |
| 65 | } |
| 66 | ) |
| 67 | return logged_texts |
| 68 | |
| 69 | |
| 70 | def validate_generated_texts( |
| 71 | hf_runner, |
| 72 | vllm_runner, |
| 73 | prompts, |
| 74 | model_name, |
| 75 | pre_quant=False, |
| 76 | hf_model_kwargs=None, |
| 77 | vllm_tp_size=1, |
| 78 | max_tokens=8, |
| 79 | ): |
| 80 | with vllm_runner( |
| 81 | model_name, |
| 82 | quantization=None if pre_quant else "bitsandbytes", |
| 83 | tensor_parallel_size=vllm_tp_size, |
| 84 | enforce_eager=False, |
| 85 | default_torch_num_threads=1, |
| 86 | tokenizer_mode="hf", |
| 87 | load_format="hf", |
| 88 | config_format="hf", |
| 89 | ) as llm: |
| 90 | vllm_outputs = llm.generate_greedy(prompts, max_tokens) |
| 91 | vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner") |
| 92 | |
| 93 | if hf_model_kwargs is None: |
| 94 | hf_model_kwargs = {} |
| 95 | |
| 96 | with hf_runner( |
| 97 | model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1 |
| 98 | ) as llm: |
| 99 | hf_outputs = llm.generate_greedy(prompts, max_tokens) |
| 100 | hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner") |
| 101 | |
| 102 | for hf_log, vllm_log in zip(hf_logs, vllm_logs): |
| 103 | hf_str = hf_log["generated_text"] |
| 104 | vllm_str = vllm_log["generated_text"] |
| 105 | prompt = hf_log["prompt"] |
| 106 | assert hf_str == vllm_str, ( |
| 107 | f"Model: {model_name}" |
| 108 | f"Mismatch between HF and vLLM outputs:\n" |
| 109 | f"Prompt: {prompt}\n" |
| 110 | f"HF Output: '{hf_str}'\n" |
| 111 | f"vLLM Output: '{vllm_str}'" |
| 112 | ) |
| 113 | |
| 114 | |
| 115 | @pytest.mark.parametrize("model_name, description", models_4bit_to_test) |
| 116 | def test_load_4bit_bnb_model( |
| 117 | hf_runner, vllm_runner, example_prompts, model_name, description |
| 118 | ) -> None: |
| 119 | hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True)) |
| 120 | validate_generated_texts( |
| 121 | hf_runner, vllm_runner, example_prompts[:1], model_name, False, hf_model_kwargs |
| 122 | ) |
| 123 | |
| 124 | |
| 125 | @pytest.mark.parametrize("model_name, description", models_pre_quant_4bit_to_test) |
| 126 | def test_load_pre_quant_4bit_bnb_model( |
| 127 | hf_runner, vllm_runner, example_prompts, model_name, description |
| 128 | ) -> None: |
| 129 | validate_generated_texts( |
| 130 | hf_runner, vllm_runner, example_prompts[:1], model_name, True |
| 131 | ) |
| 132 | |
| 133 | |
| 134 | @pytest.mark.parametrize("model_name, description", models_pre_quant_8bit_to_test) |
| 135 | def test_load_8bit_bnb_model( |
| 136 | hf_runner, vllm_runner, example_prompts, model_name, description |
| 137 | ) -> None: |
| 138 | validate_generated_texts( |
| 139 | hf_runner, vllm_runner, example_prompts[:1], model_name, True |
| 140 | ) |
| 141 | |
| 142 | |
| 143 | @pytest.mark.parametrize("model_name, description", models_4bit_to_test) |
| 144 | @multi_gpu_test(num_gpus=2) |
| 145 | def test_load_tp_4bit_bnb_model( |
| 146 | hf_runner, vllm_runner, example_prompts, model_name, description |
| 147 | ) -> None: |
| 148 | hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True)) |
| 149 | validate_generated_texts( |
| 150 | hf_runner, |
| 151 | vllm_runner, |
| 152 | example_prompts[:1], |
| 153 | model_name, |
| 154 | False, |
| 155 | hf_model_kwargs, |
| 156 | vllm_tp_size=2, |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | @pytest.mark.parametrize("model_name, description", models_4bit_to_test) |
| 161 | @multi_gpu_test(num_gpus=2) |
| 162 | def test_load_pp_4bit_bnb_model( |
| 163 | hf_runner, vllm_runner, example_prompts, model_name, description |
| 164 | ) -> None: |
| 165 | hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True)) |
| 166 | validate_generated_texts( |
| 167 | hf_runner, |
| 168 | vllm_runner, |
| 169 | example_prompts[:1], |
| 170 | model_name, |
| 171 | False, |
| 172 | hf_model_kwargs, |
| 173 | vllm_tp_size=2, |
| 174 | ) |
| 175 | |
| 176 | |
| 177 | @pytest.mark.skipif( |
| 178 | Version(TRANSFORMERS_VERSION) >= Version("5.0.0"), |
| 179 | reason="Need to add support for quantizing MoE experts with bnb in " |
| 180 | "transformers v5. See https://github.com/bitsandbytes-foundation/" |
| 181 | "bitsandbytes/issues/1849", |
| 182 | ) |
| 183 | @pytest.mark.parametrize("model_name, description", models_4bit_to_moe_test) |
| 184 | def test_4bit_bnb_moe_model( |
| 185 | hf_runner, vllm_runner, example_prompts, model_name, description |
| 186 | ) -> None: |
| 187 | hf_model_kwargs = dict( |
| 188 | quantization_config=BitsAndBytesConfig( |
| 189 | load_in_4bit=True, |
| 190 | bnb_4bit_quant_type="nf4", |
| 191 | bnb_4bit_use_double_quant=True, |
| 192 | ) |
| 193 | ) |
| 194 | with vllm_runner( |
| 195 | model_name, |
| 196 | quantization="bitsandbytes", |
| 197 | enforce_eager=False, |
| 198 | default_torch_num_threads=1, |
| 199 | ) as llm: |
| 200 | vllm_outputs = llm.generate_greedy_logprobs( |
| 201 | example_prompts, max_tokens=32, num_logprobs=5 |
| 202 | ) |
| 203 | |
| 204 | with hf_runner( |
| 205 | model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1 |
| 206 | ) as llm: |
| 207 | transformers_outputs = llm.generate_greedy_logprobs_limit( |
| 208 | example_prompts, max_tokens=32, num_logprobs=5 |
| 209 | ) |
| 210 | check_logprobs_close( |
| 211 | outputs_0_lst=transformers_outputs, |
| 212 | outputs_1_lst=vllm_outputs, |
| 213 | name_0="transformers", |
| 214 | name_1="vllm", |
| 215 | ) |
| 216 | |
| 217 | |
| 218 | @pytest.mark.parametrize("model_name, description", models_4bit_to_embedding_test) |
| 219 | @pytest.mark.parametrize("dtype", ["half"]) |
| 220 | def test_4bit_bnb_embedding_model( |
| 221 | model_name, |
| 222 | description, |
| 223 | hf_runner, |
| 224 | vllm_runner, |
| 225 | example_prompts, |
| 226 | dtype: str, |
| 227 | ) -> None: |
| 228 | example_prompts = [str(s).strip() for s in example_prompts] |
| 229 | |
| 230 | with vllm_runner( |
| 231 | model_name, |
| 232 | runner="pooling", |
| 233 | dtype=dtype, |
| 234 | gpu_memory_utilization=0.5, |
| 235 | quantization="bitsandbytes", |
| 236 | default_torch_num_threads=1, |
| 237 | ) as vllm_model: |
| 238 | vllm_outputs = vllm_model.embed(example_prompts) |
| 239 | |
| 240 | hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True)) |
| 241 | with hf_runner( |
| 242 | model_name, |
| 243 | dtype=dtype, |
| 244 | model_kwargs=hf_model_kwargs, |
| 245 | is_sentence_transformer=True, |
| 246 | default_torch_num_threads=1, |
| 247 | ) as hf_model: |
| 248 | hf_outputs = hf_model.encode(example_prompts) |
| 249 | |
| 250 | check_embeddings_close( |
| 251 | embeddings_0_lst=hf_outputs, |
| 252 | embeddings_1_lst=vllm_outputs, |
| 253 | name_0="hf", |
| 254 | name_1="vllm", |
| 255 | tol=5e-2, |
| 256 | ) |
| 257 | |
| 258 | |
| 259 | def test_bitsandbytes_passes_revision_by_name(): |
| 260 | fake_self = types.SimpleNamespace( |
| 261 | load_config=types.SimpleNamespace(download_dir="/cache"), |
| 262 | _get_weight_files=MagicMock( |
| 263 | return_value=("/folder", ["/folder/model.safetensors"], "*.safetensors") |
| 264 | ), |
| 265 | ) |
| 266 | with ( |
| 267 | patch.object(bnb, "download_safetensors_index_file_from_hf") as mock_idx, |
| 268 | patch.object( |
| 269 | bnb, |
| 270 | "filter_duplicate_safetensors_files", |
| 271 | return_value=["/folder/model.safetensors"], |
| 272 | ), |
| 273 | ): |
| 274 | bnb.BitsAndBytesModelLoader._prepare_weights(fake_self, "org/model", "myrev") |
| 275 | |
| 276 | mock_idx.assert_called_once() |
| 277 | assert mock_idx.call_args.kwargs.get("revision") == "myrev" |
| 278 | assert "myrev" not in mock_idx.call_args.args |