| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from dataclasses import dataclass |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | import vllm |
| 8 | from tests.conftest import VllmRunner |
| 9 | from vllm.assets.image import ImageAsset |
| 10 | from vllm.lora.request import LoRARequest |
| 11 | from vllm.platforms import current_platform |
| 12 | from vllm.sampling_params import BeamSearchParams |
| 13 | |
| 14 | |
| 15 | @dataclass |
| 16 | class TestConfig: |
| 17 | model_path: str |
| 18 | lora_path: str |
| 19 | max_num_seqs: int = 2 |
| 20 | max_loras: int = 2 |
| 21 | max_lora_rank: int = 32 |
| 22 | enable_tower_connector_lora: bool = False |
| 23 | max_model_len: int = 8192 |
| 24 | gpu_memory_utilization: float = 0.85 |
| 25 | mm_processor_kwargs: dict[str, object] | None = None |
| 26 | mm_processor_cache_gb: float = 4 |
| 27 | |
| 28 | def __post_init__(self): |
| 29 | if self.mm_processor_kwargs is None: |
| 30 | self.mm_processor_kwargs = { |
| 31 | "size": { |
| 32 | "shortest_edge": 28 * 28, |
| 33 | "longest_edge": 1280 * 28 * 28, |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | |
| 38 | class Qwen2VLTester: |
| 39 | """Test helper for Qwen2 VL models with LoRA.""" |
| 40 | |
| 41 | PROMPT_TEMPLATE = ( |
| 42 | "<|im_start|>system\nYou are a helpful assistant.<|im_end|>" |
| 43 | "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" |
| 44 | "What is in the image?<|im_end|>\n" |
| 45 | "<|im_start|>assistant\n" |
| 46 | ) |
| 47 | |
| 48 | def __init__(self, config: TestConfig): |
| 49 | self.config = config |
| 50 | self._runner = VllmRunner( |
| 51 | model_name=config.model_path, |
| 52 | max_num_seqs=config.max_num_seqs, |
| 53 | enable_lora=True, |
| 54 | max_loras=config.max_loras, |
| 55 | max_lora_rank=config.max_lora_rank, |
| 56 | enable_tower_connector_lora=config.enable_tower_connector_lora, |
| 57 | gpu_memory_utilization=config.gpu_memory_utilization, |
| 58 | mm_processor_kwargs=config.mm_processor_kwargs, |
| 59 | mm_processor_cache_gb=config.mm_processor_cache_gb, |
| 60 | max_model_len=config.max_model_len, |
| 61 | ) |
| 62 | |
| 63 | @property |
| 64 | def llm(self) -> vllm.LLM: |
| 65 | return self._runner.get_llm() |
| 66 | |
| 67 | def __enter__(self) -> "Qwen2VLTester": |
| 68 | return self |
| 69 | |
| 70 | def __exit__(self, exc_type, exc_value, traceback) -> None: |
| 71 | self._runner.__exit__(exc_type, exc_value, traceback) |
| 72 | |
| 73 | def run_test( |
| 74 | self, |
| 75 | images: list[ImageAsset], |
| 76 | expected_outputs: list[str], |
| 77 | lora_id: int | None = None, |
| 78 | lora_name: str | None = None, |
| 79 | temperature: float = 0, |
| 80 | max_tokens: int = 5, |
| 81 | ): |
| 82 | sampling_params = vllm.SamplingParams( |
| 83 | temperature=temperature, |
| 84 | max_tokens=max_tokens, |
| 85 | ) |
| 86 | inputs = [ |
| 87 | { |
| 88 | "prompt": self.PROMPT_TEMPLATE, |
| 89 | "multi_modal_data": {"image": asset.pil_image}, |
| 90 | } |
| 91 | for asset in images |
| 92 | ] |
| 93 | |
| 94 | lora_request = LoRARequest( |
| 95 | lora_name if lora_name else str(lora_id), lora_id, self.config.lora_path |
| 96 | ) |
| 97 | outputs = self.llm.generate(inputs, sampling_params, lora_request=lora_request) |
| 98 | generated_texts = [output.outputs[0].text.strip() for output in outputs] |
| 99 | # Validate outputs |
| 100 | for generated, expected in zip(generated_texts, expected_outputs): |
| 101 | assert expected.startswith(generated), ( |
| 102 | f"Generated text {generated} doesn't match expected pattern {expected}" |
| 103 | ) |
| 104 | |
| 105 | def run_beam_search_test( |
| 106 | self, |
| 107 | images: list[ImageAsset], |
| 108 | expected_outputs: list[list[str]], |
| 109 | lora_id: int | None = None, |
| 110 | temperature: float = 0, |
| 111 | beam_width: int = 2, |
| 112 | max_tokens: int = 5, |
| 113 | ): |
| 114 | beam_search_params = BeamSearchParams( |
| 115 | beam_width=beam_width, max_tokens=max_tokens, temperature=temperature |
| 116 | ) |
| 117 | |
| 118 | inputs = [ |
| 119 | { |
| 120 | "prompt": self.PROMPT_TEMPLATE, |
| 121 | "multi_modal_data": {"image": asset.pil_image}, |
| 122 | } |
| 123 | for asset in images |
| 124 | ] |
| 125 | |
| 126 | lora_request = LoRARequest(str(lora_id), lora_id, self.config.lora_path) |
| 127 | outputs = self.llm.beam_search( |
| 128 | inputs, beam_search_params, lora_request=lora_request |
| 129 | ) |
| 130 | |
| 131 | for output_obj, expected_texts in zip(outputs, expected_outputs): |
| 132 | output_texts = [seq.text for seq in output_obj.sequences] |
| 133 | |
| 134 | for output_text, expected_text in zip(output_texts, expected_texts): |
| 135 | # NOTE beam search .text contains the whole text including inputs |
| 136 | assert output_text.endswith(expected_text), ( |
| 137 | f"Generated {output_text} does not match expected {expected_text}" |
| 138 | ) |
| 139 | |
| 140 | |
| 141 | TEST_IMAGES = [ |
| 142 | ImageAsset("stop_sign"), |
| 143 | ImageAsset("cherry_blossom"), |
| 144 | ] |
| 145 | |
| 146 | EXPECTED_OUTPUTS = [ |
| 147 | "A red stop sign stands prominently in the foreground, with a traditional Chinese gate and a black SUV in the background, illustrating a blend of modern and cultural elements.", # noqa: E501 |
| 148 | "A majestic skyscraper stands tall, partially obscured by a vibrant canopy of cherry blossoms, against a clear blue sky.", # noqa: E501 |
| 149 | ] |
| 150 | |
| 151 | EXPECTED_OUTPUTS_LANGUAGE = [ |
| 152 | "A stop sign is shown in an Asian city, with buildings and a car in the " |
| 153 | "background.", |
| 154 | "The Tokyo Skytree can be seen behind the pink blossoms of the cherry trees.", |
| 155 | ] |
| 156 | |
| 157 | EXPECTED_OUTPUTS_VISION = [ |
| 158 | "A stop sign in front of oriental buildings.", |
| 159 | "A tree with pink flowers in front of it and a blue sky behind the flowers.", |
| 160 | ] |
| 161 | |
| 162 | EXPECTED_OUTPUTS_VISION_NO_CONNECTOR = [ |
| 163 | "A stop sign is located on the street of a Chinese neighborhood.", |
| 164 | "A closeup shot of the Tokyo Skytree with pink flowers in the foreground.", |
| 165 | ] |
| 166 | |
| 167 | EXPECTED_BEAM_SEARCH_OUTPUTS = [ |
| 168 | [ |
| 169 | "A majestic skyscraper stands", |
| 170 | "A majestic tower stands tall", |
| 171 | ], |
| 172 | ] |
| 173 | |
| 174 | QWEN2VL_MODEL_PATH = "Qwen/Qwen2-VL-2B-Instruct" |
| 175 | QWEN25VL_MODEL_PATH = "Qwen/Qwen2.5-VL-3B-Instruct" |
| 176 | QWEN3VL_MODEL_PATH = "Qwen/Qwen3-VL-4B-Instruct" |
| 177 | |
| 178 | |
| 179 | def _enable_deterministic_lora_shrink(monkeypatch: pytest.MonkeyPatch) -> None: |
| 180 | # These tests assert exact greedy outputs. Force the Triton LoRA shrink |
| 181 | # kernel to use SPLIT_K=1 so it stores the complete reduction directly |
| 182 | # instead of accumulating split-K partial results with atomic_add. This |
| 183 | # targets reduction determinism, not full batch invariance. |
| 184 | monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1") |
| 185 | # The kernel configuration reads VLLM_BATCH_INVARIANT at import time. |
| 186 | # Spawn the engine process so it observes this setting even if the LoRA |
| 187 | # Triton utilities were already imported during test collection. |
| 188 | monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") |
| 189 | |
| 190 | |
| 191 | def test_qwen2vl_lora(qwen2vl_lora_files): |
| 192 | """Test Qwen 2.0 VL model with LoRA.""" |
| 193 | config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files) |
| 194 | with Qwen2VLTester(config) as tester: |
| 195 | # Test with different LoRA IDs |
| 196 | for lora_id in [1, 2]: |
| 197 | tester.run_test( |
| 198 | TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id |
| 199 | ) |
| 200 | |
| 201 | |
| 202 | def test_qwen2vl_lora_beam_search(qwen2vl_lora_files): |
| 203 | """Test Qwen 2.0 VL model with LoRA through beam search.""" |
| 204 | config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files) |
| 205 | with Qwen2VLTester(config) as tester: |
| 206 | # Test with different LoRA IDs |
| 207 | for lora_id in [1, 2]: |
| 208 | # NOTE currently, we only test cherry blossom since stop sign |
| 209 | # output is slightly different for v1; - the root cause is likely |
| 210 | # independent of the intent of this test, which is to ensure beam |
| 211 | # search passes through lora through correctly. |
| 212 | tester.run_beam_search_test( |
| 213 | [ImageAsset("cherry_blossom")], |
| 214 | expected_outputs=EXPECTED_BEAM_SEARCH_OUTPUTS, |
| 215 | lora_id=lora_id, |
| 216 | ) |
| 217 | |
| 218 | |
| 219 | @pytest.mark.skipif( |
| 220 | current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests" |
| 221 | ) |
| 222 | def test_qwen25vl_lora(qwen25vl_lora_files): |
| 223 | """Test Qwen 2.5 VL model with LoRA.""" |
| 224 | config = TestConfig(model_path=QWEN25VL_MODEL_PATH, lora_path=qwen25vl_lora_files) |
| 225 | with Qwen2VLTester(config) as tester: |
| 226 | # Test with different LoRA IDs |
| 227 | for lora_id in [1, 2]: |
| 228 | tester.run_test( |
| 229 | TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id |
| 230 | ) |
| 231 | |
| 232 | |
| 233 | @pytest.mark.skipif( |
| 234 | current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests" |
| 235 | ) |
| 236 | def test_qwen25vl_vision_lora(qwen25vl_vision_lora_files): |
| 237 | config = TestConfig( |
| 238 | model_path=QWEN25VL_MODEL_PATH, |
| 239 | lora_path=qwen25vl_vision_lora_files, |
| 240 | # Currently, tower_connector_lora is incompatible with |
| 241 | # the multi-modal processor cache. |
| 242 | # TODO: Remove this restriction |
| 243 | mm_processor_cache_gb=0, |
| 244 | enable_tower_connector_lora=True, |
| 245 | ) |
| 246 | with Qwen2VLTester(config) as tester: |
| 247 | for lora_id in [1, 2]: |
| 248 | tester.run_test( |
| 249 | TEST_IMAGES, |
| 250 | expected_outputs=EXPECTED_OUTPUTS, |
| 251 | lora_id=lora_id, |
| 252 | ) |
| 253 | |
| 254 | |
| 255 | def test_qwen3vl_vision_lora( |
| 256 | qwen3vl_vision_lora_files, |
| 257 | monkeypatch: pytest.MonkeyPatch, |
| 258 | ): |
| 259 | _enable_deterministic_lora_shrink(monkeypatch) |
| 260 | |
| 261 | config = TestConfig( |
| 262 | model_path=QWEN3VL_MODEL_PATH, |
| 263 | lora_path=qwen3vl_vision_lora_files, |
| 264 | # Currently, tower_connector_lora is incompatible with |
| 265 | # the multi-modal processor cache. |
| 266 | # TODO: Remove this restriction |
| 267 | mm_processor_cache_gb=0, |
| 268 | enable_tower_connector_lora=True, |
| 269 | ) |
| 270 | with Qwen2VLTester(config) as tester: |
| 271 | for lora_id in [1, 2]: |
| 272 | tester.run_test( |
| 273 | TEST_IMAGES, |
| 274 | expected_outputs=EXPECTED_OUTPUTS, |
| 275 | lora_id=lora_id, |
| 276 | ) |
| 277 | |
| 278 | |
| 279 | def test_qwen2vl_multiple_lora_types( |
| 280 | qwen2vl_language_lora_files, |
| 281 | qwen2vl_vision_tower_connector_lora_files, |
| 282 | qwen2vl_vision_tower_lora_files, |
| 283 | monkeypatch: pytest.MonkeyPatch, |
| 284 | ): |
| 285 | """Test multiple LoRA adapter types (language, vision tower + connector, |
| 286 | vision tower only) using the same LLM instance to verify mm_encoder_cache |
| 287 | behavior with different LoRA requests. |
| 288 | |
| 289 | By reusing the same LLM instance across different LoRA requests, we ensure that |
| 290 | the multimodal encoder cache correctly manages state transitions between |
| 291 | language-only and vision-enabled LoRA adapters. |
| 292 | """ |
| 293 | _enable_deterministic_lora_shrink(monkeypatch) |
| 294 | |
| 295 | config = TestConfig( |
| 296 | model_path=QWEN2VL_MODEL_PATH, |
| 297 | # We'll override the lora_path for each specific test, but need to provide |
| 298 | # an initial path for initialization |
| 299 | lora_path=qwen2vl_language_lora_files, |
| 300 | # Currently, tower_connector_lora is incompatible with |
| 301 | # the multi-modal processor cache. |
| 302 | # TODO: Remove this restriction |
| 303 | mm_processor_cache_gb=0, |
| 304 | enable_tower_connector_lora=True, |
| 305 | ) |
| 306 | with Qwen2VLTester(config) as tester: |
| 307 | # Test 1: Language-only LoRA adapter |
| 308 | tester.config.lora_path = qwen2vl_language_lora_files |
| 309 | for lora_id in [1, 2]: |
| 310 | tester.run_test( |
| 311 | TEST_IMAGES, |
| 312 | expected_outputs=EXPECTED_OUTPUTS_LANGUAGE, |
| 313 | lora_id=lora_id, |
| 314 | lora_name="language_only", |
| 315 | ) |
| 316 | |
| 317 | # Test 2: Vision tower + connector LoRA adapter |
| 318 | tester.config.lora_path = qwen2vl_vision_tower_connector_lora_files |
| 319 | for lora_id in [3, 4]: |
| 320 | tester.run_test( |
| 321 | TEST_IMAGES, |
| 322 | expected_outputs=EXPECTED_OUTPUTS_VISION, |
| 323 | lora_id=lora_id, |
| 324 | lora_name="vision_tower_connector", |
| 325 | ) |
| 326 | |
| 327 | # Test 3: Vision tower only LoRA adapter (no connector) |
| 328 | tester.config.lora_path = qwen2vl_vision_tower_lora_files |
| 329 | for lora_id in [5, 6]: |
| 330 | tester.run_test( |
| 331 | TEST_IMAGES, |
| 332 | expected_outputs=EXPECTED_OUTPUTS_VISION_NO_CONNECTOR, |
| 333 | lora_id=lora_id, |
| 334 | lora_name="vision_tower", |
| 335 | ) |