| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Transformers backend tests for bitsandbytes plugin models.""" |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | from vllm.platforms import current_platform |
| 8 | |
| 9 | from ...models.utils import check_logprobs_close |
| 10 | |
| 11 | if current_platform.is_rocm(): |
| 12 | from vllm.platforms.rocm import on_cdna |
| 13 | |
| 14 | pytestmark = pytest.mark.skipif( |
| 15 | on_cdna(), |
| 16 | reason="bitsandbytes not supported on CDNA (warp size 64 limitation)", |
| 17 | ) |
| 18 | |
| 19 | |
| 20 | @pytest.mark.parametrize( |
| 21 | "model, quantization_kwargs", |
| 22 | [ |
| 23 | ( |
| 24 | "meta-llama/Llama-3.2-1B-Instruct", |
| 25 | { |
| 26 | "quantization": "bitsandbytes", |
| 27 | }, |
| 28 | ), |
| 29 | ("unsloth/tinyllama-bnb-4bit", {}), |
| 30 | ], |
| 31 | ) |
| 32 | @pytest.mark.parametrize("max_tokens", [32]) |
| 33 | @pytest.mark.parametrize("num_logprobs", [5]) |
| 34 | def test_transformers_backend_quantization( |
| 35 | vllm_runner, |
| 36 | example_prompts, |
| 37 | model: str, |
| 38 | quantization_kwargs: dict[str, str], |
| 39 | max_tokens: int, |
| 40 | num_logprobs: int, |
| 41 | ) -> None: |
| 42 | with vllm_runner( |
| 43 | model, |
| 44 | model_impl="auto", |
| 45 | enforce_eager=True, |
| 46 | **quantization_kwargs, |
| 47 | ) as vllm_model: |
| 48 | vllm_outputs = vllm_model.generate_greedy_logprobs( |
| 49 | example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs |
| 50 | ) |
| 51 | |
| 52 | with vllm_runner( |
| 53 | model, |
| 54 | model_impl="transformers", |
| 55 | enforce_eager=True, |
| 56 | **quantization_kwargs, |
| 57 | ) as vllm_model: |
| 58 | model_config = vllm_model.llm.llm_engine.model_config |
| 59 | assert model_config.using_transformers_backend() |
| 60 | |
| 61 | transformers_outputs = vllm_model.generate_greedy_logprobs( |
| 62 | example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs |
| 63 | ) |
| 64 | |
| 65 | check_logprobs_close( |
| 66 | outputs_0_lst=transformers_outputs, |
| 67 | outputs_1_lst=vllm_outputs, |
| 68 | name_0="transformers", |
| 69 | name_1="vllm", |
| 70 | ) |