| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | import torch |
| 6 | from transformers import AutoModelForMaskedLM, AutoModelForTokenClassification |
| 7 | |
| 8 | from tests.models.registry import HF_EXAMPLE_MODELS |
| 9 | from tests.models.utils import softmax |
| 10 | from vllm.platforms import current_platform |
| 11 | from vllm.utils.torch_utils import set_random_seed |
| 12 | |
| 13 | |
| 14 | @pytest.fixture(autouse=True) |
| 15 | def seed_everything(): |
| 16 | """Seed all random number generators for reproducibility.""" |
| 17 | seed = 0 |
| 18 | set_random_seed(seed) |
| 19 | torch.backends.cudnn.deterministic = True |
| 20 | torch.backends.cudnn.benchmark = False |
| 21 | yield |
| 22 | |
| 23 | |
| 24 | @pytest.mark.parametrize( |
| 25 | "model", |
| 26 | [ |
| 27 | "boltuix/NeuroBERT-NER", |
| 28 | ], |
| 29 | ) |
| 30 | # The float32 is required for this tiny model to pass the test. |
| 31 | @pytest.mark.parametrize("dtype", ["float"]) |
| 32 | @pytest.mark.core_model |
| 33 | @torch.inference_mode |
| 34 | def test_bert_model_runner_v2( |
| 35 | hf_runner, |
| 36 | vllm_runner, |
| 37 | example_prompts, |
| 38 | monkeypatch, |
| 39 | model: str, |
| 40 | dtype: str, |
| 41 | ) -> None: |
| 42 | prompt_batches = [[example_prompts[0]], example_prompts] |
| 43 | |
| 44 | monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") |
| 45 | with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model: |
| 46 | assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner |
| 47 | vllm_output_batches = [ |
| 48 | vllm_model.token_classify(prompts) for prompts in prompt_batches |
| 49 | ] |
| 50 | |
| 51 | with hf_runner( |
| 52 | model, |
| 53 | dtype=dtype, |
| 54 | auto_cls=AutoModelForTokenClassification, |
| 55 | ) as hf_model: |
| 56 | tokenizer = hf_model.tokenizer |
| 57 | hf_outputs = [] |
| 58 | for prompt in example_prompts: |
| 59 | inputs = tokenizer([prompt], return_tensors="pt") |
| 60 | inputs = hf_model.wrap_device(inputs) |
| 61 | output = hf_model.model(**inputs) |
| 62 | hf_outputs.append(softmax(output.logits[0])) |
| 63 | hf_output_batches = [[hf_outputs[0]], hf_outputs] |
| 64 | |
| 65 | # check logits difference |
| 66 | for hf_outputs, vllm_outputs in zip(hf_output_batches, vllm_output_batches): |
| 67 | for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): |
| 68 | hf_output = hf_output.detach().clone().cpu().float() |
| 69 | vllm_output = vllm_output.detach().clone().cpu().float() |
| 70 | torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3) |
| 71 | |
| 72 | |
| 73 | @pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"]) |
| 74 | @pytest.mark.parametrize("dtype", ["float"]) |
| 75 | @pytest.mark.flaky(reruns=3) |
| 76 | @torch.inference_mode |
| 77 | def test_modernbert_models( |
| 78 | hf_runner, |
| 79 | vllm_runner, |
| 80 | example_prompts, |
| 81 | model: str, |
| 82 | dtype: str, |
| 83 | ) -> None: |
| 84 | # NOTE: https://github.com/vllm-project/vllm/pull/32403 |
| 85 | # `disham993/electrical-ner-ModernBERT-base` is a randomly initialized |
| 86 | # model, which can cause numerical precision variance and edge cases. |
| 87 | # We use @flaky(reruns=3) to mitigate intermittent failures. |
| 88 | print( |
| 89 | f"\n[NOTE] Testing {model} (randomly initialized weights) - " |
| 90 | "flaky tolerance enabled due to numerical precision variance." |
| 91 | ) |
| 92 | |
| 93 | with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model: |
| 94 | vllm_outputs = vllm_model.token_classify(example_prompts) |
| 95 | |
| 96 | with hf_runner( |
| 97 | model, |
| 98 | dtype=dtype, |
| 99 | auto_cls=AutoModelForTokenClassification, |
| 100 | ) as hf_model: |
| 101 | tokenizer = hf_model.tokenizer |
| 102 | hf_outputs = [] |
| 103 | for prompt in example_prompts: |
| 104 | inputs = tokenizer([prompt], return_tensors="pt") |
| 105 | inputs = hf_model.wrap_device(inputs) |
| 106 | output = hf_model.model(**inputs) |
| 107 | hf_outputs.append(softmax(output.logits[0])) |
| 108 | |
| 109 | # check logits difference |
| 110 | for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): |
| 111 | hf_output = hf_output.detach().clone().cpu().float() |
| 112 | vllm_output = vllm_output.detach().clone().cpu().float() |
| 113 | torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3) |
| 114 | |
| 115 | |
| 116 | @pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"]) |
| 117 | @pytest.mark.parametrize("dtype", ["float"]) |
| 118 | @pytest.mark.flaky(reruns=3) |
| 119 | @torch.inference_mode |
| 120 | def test_xlm_roberta_models( |
| 121 | hf_runner, |
| 122 | vllm_runner, |
| 123 | example_prompts, |
| 124 | model: str, |
| 125 | dtype: str, |
| 126 | ) -> None: |
| 127 | with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model: |
| 128 | vllm_outputs = vllm_model.token_classify(example_prompts) |
| 129 | |
| 130 | with hf_runner( |
| 131 | model, |
| 132 | dtype=dtype, |
| 133 | auto_cls=AutoModelForTokenClassification, |
| 134 | ) as hf_model: |
| 135 | tokenizer = hf_model.tokenizer |
| 136 | hf_outputs = [] |
| 137 | for prompt in example_prompts: |
| 138 | inputs = tokenizer([prompt], return_tensors="pt") |
| 139 | inputs = hf_model.wrap_device(inputs) |
| 140 | output = hf_model.model(**inputs) |
| 141 | hf_outputs.append(softmax(output.logits[0])) |
| 142 | |
| 143 | # check logits difference |
| 144 | for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): |
| 145 | hf_output = hf_output.detach().clone().cpu().float() |
| 146 | vllm_output = vllm_output.detach().clone().cpu().float() |
| 147 | torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3) |
| 148 | |
| 149 | |
| 150 | PRIVACY_FILTER_PROMPTS = [ |
| 151 | "My name is Harry Potter.", |
| 152 | "Email me at harry.potter@hogwarts.edu.", |
| 153 | "Call me on +44 20 7946 0958 tomorrow.", |
| 154 | "My account number is 12345678 and the API key is sk-live-abc123def456.", |
| 155 | "I live at 4 Privet Drive, Little Whinging.", |
| 156 | "Visit https://example.com/profile/harry for more info.", |
| 157 | "We met on 12 January 2024.", |
| 158 | ] |
| 159 | |
| 160 | |
| 161 | @pytest.mark.skipif( |
| 162 | current_platform.is_rocm(), |
| 163 | reason="Workspace growth allocation issue on ROCm. See https://github.com/vllm-project/vllm/issues/48510", |
| 164 | ) |
| 165 | @pytest.mark.parametrize("model", ["openai/privacy-filter"]) |
| 166 | @pytest.mark.parametrize("dtype", ["bfloat16"]) |
| 167 | @torch.inference_mode |
| 168 | def test_openai_privacy_filter( |
| 169 | hf_runner, |
| 170 | vllm_runner, |
| 171 | model: str, |
| 172 | dtype: str, |
| 173 | ) -> None: |
| 174 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model) |
| 175 | model_info.check_transformers_version(on_fail="skip") |
| 176 | |
| 177 | with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model: |
| 178 | vllm_outputs = vllm_model.token_classify(PRIVACY_FILTER_PROMPTS) |
| 179 | |
| 180 | with hf_runner( |
| 181 | model, |
| 182 | dtype=dtype, |
| 183 | auto_cls=AutoModelForTokenClassification, |
| 184 | ) as hf_model: |
| 185 | tokenizer = hf_model.tokenizer |
| 186 | hf_outputs = [] |
| 187 | for prompt in PRIVACY_FILTER_PROMPTS: |
| 188 | inputs = tokenizer([prompt], return_tensors="pt") |
| 189 | inputs = hf_model.wrap_device(inputs) |
| 190 | output = hf_model.model(**inputs) |
| 191 | hf_outputs.append(softmax(output.logits[0])) |
| 192 | |
| 193 | for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): |
| 194 | hf_output = hf_output.detach().clone().cpu().float() |
| 195 | vllm_output = vllm_output.detach().clone().cpu().float() |
| 196 | torch.testing.assert_close(hf_output, vllm_output, atol=0.1, rtol=1e-2) |
| 197 | |
| 198 | |
| 199 | @pytest.mark.parametrize("model", ["bd2lcco/Qwen3-0.6B-finetuned"]) |
| 200 | @pytest.mark.parametrize("dtype", ["float"]) |
| 201 | @pytest.mark.flaky(reruns=3) |
| 202 | @torch.inference_mode |
| 203 | def test_auto_conversion( |
| 204 | hf_runner, |
| 205 | vllm_runner, |
| 206 | example_prompts, |
| 207 | model: str, |
| 208 | dtype: str, |
| 209 | ) -> None: |
| 210 | with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model: |
| 211 | vllm_outputs = vllm_model.token_classify(example_prompts) |
| 212 | |
| 213 | with hf_runner( |
| 214 | model, dtype=dtype, auto_cls=AutoModelForTokenClassification |
| 215 | ) as hf_model: |
| 216 | tokenizer = hf_model.tokenizer |
| 217 | hf_outputs = [] |
| 218 | for prompt in example_prompts: |
| 219 | inputs = tokenizer([prompt], return_tensors="pt") |
| 220 | inputs = hf_model.wrap_device(inputs) |
| 221 | output = hf_model.model(**inputs) |
| 222 | hf_outputs.append(softmax(output.logits[0])) |
| 223 | |
| 224 | # check logits difference |
| 225 | for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): |
| 226 | hf_output = hf_output.detach().clone().cpu().float() |
| 227 | vllm_output = vllm_output.detach().clone().cpu().float() |
| 228 | assert torch.allclose(hf_output, vllm_output, atol=1e-2) |
| 229 | |
| 230 | |
| 231 | @pytest.mark.parametrize( |
| 232 | "model", |
| 233 | [ |
| 234 | # Original Google checkpoint: legacy `gamma`/`beta` LayerNorm names, an |
| 235 | # NSP head (`cls.seq_relationship.*`) and a decoder tied to the input |
| 236 | # embeddings (no explicit decoder weight in the checkpoint). |
| 237 | "google-bert/bert-base-uncased", |
| 238 | ], |
| 239 | ) |
| 240 | @pytest.mark.parametrize("dtype", ["float"]) |
| 241 | @torch.inference_mode |
| 242 | def test_bert_for_masked_lm( |
| 243 | hf_runner, |
| 244 | vllm_runner, |
| 245 | example_prompts, |
| 246 | model: str, |
| 247 | dtype: str, |
| 248 | ) -> None: |
| 249 | # BertForMaskedLM exposes its MLM head as a token-level pooling task; the |
| 250 | # head applies softmax over the vocabulary, so each output row is a |
| 251 | # distribution (matching HF's softmax(logits) below). |
| 252 | with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model: |
| 253 | vllm_outputs = vllm_model.token_classify(example_prompts) |
| 254 | |
| 255 | # Run hf_runner reference with "highest" fp32 precision to match |
| 256 | # default behvior of vLLM. This is needed on ROCm since the |
| 257 | # pooling tests set matmul precision to "high" in conftest.py |
| 258 | prev_matmul_precision = torch.get_float32_matmul_precision() |
| 259 | torch.set_float32_matmul_precision("highest") |
| 260 | with hf_runner( |
| 261 | model, |
| 262 | dtype=dtype, |
| 263 | auto_cls=AutoModelForMaskedLM, |
| 264 | ) as hf_model: |
| 265 | tokenizer = hf_model.tokenizer |
| 266 | hf_outputs = [] |
| 267 | for prompt in example_prompts: |
| 268 | inputs = tokenizer([prompt], return_tensors="pt") |
| 269 | inputs = hf_model.wrap_device(inputs) |
| 270 | output = hf_model.model(**inputs) |
| 271 | hf_outputs.append(softmax(output.logits[0])) |
| 272 | torch.set_float32_matmul_precision(prev_matmul_precision) |
| 273 | |
| 274 | # Compare the per-token vocabulary distributions position by position. |
| 275 | for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): |
| 276 | hf_output = hf_output.detach().clone().cpu().float() |
| 277 | vllm_output = vllm_output.detach().clone().cpu().float() |
| 278 | assert hf_output.shape == vllm_output.shape |
| 279 | assert torch.equal(hf_output.argmax(dim=-1), vllm_output.argmax(dim=-1)) |
| 280 | torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3) |