| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import copy |
| 5 | from unittest.mock import MagicMock, patch |
| 6 | |
| 7 | import pytest |
| 8 | import torch |
| 9 | from transformers import PreTrainedConfig |
| 10 | |
| 11 | from tests.models.utils import build_model_context |
| 12 | from vllm.config.ec_transfer import ECRole, ECTransferConfig |
| 13 | from vllm.config.model import ModelConfig |
| 14 | from vllm.config.multimodal import MultiModalConfig |
| 15 | from vllm.config.vllm import VllmConfig |
| 16 | from vllm.transformers_utils.model_arch_config_convertor import ( |
| 17 | ModelArchConfigConvertorBase, |
| 18 | ) |
| 19 | from vllm.v1.attention.backends.registry import AttentionBackendEnum |
| 20 | |
| 21 | |
| 22 | @pytest.mark.parametrize( |
| 23 | "model_id,limit_mm_per_prompt,expected", |
| 24 | [ |
| 25 | ("Qwen/Qwen2-0.5B-Instruct", {}, False), |
| 26 | ("Qwen/Qwen2.5-VL-3B-Instruct", {}, True), |
| 27 | ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0, "video": 0}, False), |
| 28 | ("Qwen/Qwen2.5-VL-3B-Instruct", {"image": 0}, True), |
| 29 | ], |
| 30 | ) |
| 31 | @pytest.mark.core_model |
| 32 | def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected): |
| 33 | """Test supports_multimodal_inputs returns correct boolean for various |
| 34 | configs.""" |
| 35 | ctx = build_model_context( |
| 36 | model_id, |
| 37 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 38 | ) |
| 39 | assert ctx.model_config.supports_multimodal_inputs is expected |
| 40 | |
| 41 | |
| 42 | def test_mm_encoder_attn_backend_str_conversion(): |
| 43 | config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN") |
| 44 | assert config.mm_encoder_attn_backend == AttentionBackendEnum.FLASH_ATTN |
| 45 | |
| 46 | |
| 47 | def test_mm_encoder_attn_backend_invalid(): |
| 48 | with pytest.raises(ValueError): |
| 49 | MultiModalConfig(mm_encoder_attn_backend="not_a_backend") |
| 50 | |
| 51 | |
| 52 | def test_mm_hasher_algorithm_invalid(): |
| 53 | with pytest.raises(ValueError, match="mm_hasher_algorithm"): |
| 54 | MultiModalConfig(mm_hasher_algorithm="md5") |
| 55 | |
| 56 | |
| 57 | def test_mm_encoder_attn_backend_hash_updates(): |
| 58 | base_hash = MultiModalConfig().compute_hash() |
| 59 | overridden_hash = MultiModalConfig( |
| 60 | mm_encoder_attn_backend=AttentionBackendEnum.FLASH_ATTN |
| 61 | ).compute_hash() |
| 62 | assert base_hash != overridden_hash |
| 63 | |
| 64 | |
| 65 | def test_language_model_only_does_not_affect_mm_hash(): |
| 66 | """language_model_only does not affect the ViT computation graph, |
| 67 | so it should not change the multimodal config hash.""" |
| 68 | base_hash = MultiModalConfig().compute_hash() |
| 69 | lm_only_hash = MultiModalConfig(language_model_only=True).compute_hash() |
| 70 | assert base_hash == lm_only_hash |
| 71 | |
| 72 | |
| 73 | def test_language_model_only_affects_model_hash(): |
| 74 | """language_model_only affects the LM computation graph, |
| 75 | so it should change the model config hash.""" |
| 76 | model = "llava-hf/llava-1.5-7b-hf" |
| 77 | base_hash = ModelConfig(model).compute_hash() |
| 78 | lm_only_hash = ModelConfig(model, language_model_only=True).compute_hash() |
| 79 | assert base_hash != lm_only_hash |
| 80 | |
| 81 | |
| 82 | @pytest.mark.parametrize("backend_arg", ["video_backend", "backend"]) |
| 83 | def test_use_gpu_video_backend_from_media_io_kwargs(backend_arg: str): |
| 84 | config = MultiModalConfig( |
| 85 | media_io_kwargs={"video": {backend_arg: "pynvvideocodec"}} |
| 86 | ) |
| 87 | |
| 88 | assert config.use_gpu_video_backend() |
| 89 | |
| 90 | |
| 91 | def test_mm_encoder_fp8_scale_path_requires_fp8(): |
| 92 | with pytest.raises(ValueError, match="mm_encoder_attn_dtype"): |
| 93 | MultiModalConfig(mm_encoder_fp8_scale_path="/tmp/scales.json") |
| 94 | |
| 95 | |
| 96 | def test_mm_encoder_attn_dtype_hash_updates(tmp_path): |
| 97 | scale_file = tmp_path / "scales.json" |
| 98 | scale_file.write_text("{}") |
| 99 | base_hash = MultiModalConfig().compute_hash() |
| 100 | fp8_hash = MultiModalConfig(mm_encoder_attn_dtype="fp8").compute_hash() |
| 101 | fp8_static_hash = MultiModalConfig( |
| 102 | mm_encoder_attn_dtype="fp8", |
| 103 | mm_encoder_fp8_scale_path=str(scale_file), |
| 104 | ).compute_hash() |
| 105 | assert base_hash != fp8_hash |
| 106 | assert fp8_hash != fp8_static_hash |
| 107 | |
| 108 | |
| 109 | _MULTIMODAL_MODEL = "llava-hf/llava-1.5-7b-hf" |
| 110 | _TEXT_ONLY_MODEL = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" |
| 111 | |
| 112 | |
| 113 | def _make_mm_prefix_model_config( |
| 114 | model: str = _MULTIMODAL_MODEL, |
| 115 | *, |
| 116 | language_model_only: bool = False, |
| 117 | ) -> ModelConfig: |
| 118 | return ModelConfig(model, language_model_only=language_model_only) |
| 119 | |
| 120 | |
| 121 | def test_supports_multimodal_for_mm_prefix_before_multimodal_config(): |
| 122 | """A text-only model never builds a multimodal config, so the early |
| 123 | return must neither clear mm_prefix nor write the sticky cache.""" |
| 124 | model_config = _make_mm_prefix_model_config(_TEXT_ONLY_MODEL) |
| 125 | assert model_config.multimodal_config is None |
| 126 | |
| 127 | assert model_config._supports_multimodal_for_mm_prefix() is True |
| 128 | assert not getattr(model_config, "_supports_multimodal_inputs_cache", None) |
| 129 | |
| 130 | |
| 131 | def test_language_model_only_disables_via_supports_multimodal_inputs(): |
| 132 | """language_model_only zeros all limits, so registry reports text-only.""" |
| 133 | model_config = _make_mm_prefix_model_config(language_model_only=True) |
| 134 | |
| 135 | assert model_config._supports_multimodal_for_mm_prefix() is False |
| 136 | |
| 137 | |
| 138 | def test_convertor_clears_mm_prefix_when_multimodal_disabled(): |
| 139 | hf_config = PreTrainedConfig( |
| 140 | model_type="gemma3", |
| 141 | architectures=["Gemma3ForConditionalGeneration"], |
| 142 | ) |
| 143 | hf_config.is_mm_prefix_lm = True |
| 144 | convertor = ModelArchConfigConvertorBase(hf_config, hf_config) |
| 145 | |
| 146 | assert convertor.is_mm_prefix_lm(supports_multimodal=True) is True |
| 147 | assert convertor.is_mm_prefix_lm(supports_multimodal=False) is False |
| 148 | |
| 149 | enabled = convertor.convert(supports_multimodal=True) |
| 150 | disabled = convertor.convert(supports_multimodal=False) |
| 151 | assert enabled.is_mm_prefix_lm is True |
| 152 | assert disabled.is_mm_prefix_lm is False |
| 153 | |
| 154 | |
| 155 | def test_sticky_cache_survives_text_subconfig_regeneration(): |
| 156 | """with_hf_config deepcopies the cached decision onto text submodules.""" |
| 157 | model_config = _make_mm_prefix_model_config() |
| 158 | assert model_config._supports_multimodal_for_mm_prefix() is True |
| 159 | |
| 160 | # `with_hf_config` deep-copies this config and swaps `hf_config` for a |
| 161 | # text-only submodule (e.g. Gemma4ForCausalLM). |
| 162 | text_config = copy.deepcopy(model_config) |
| 163 | text_config.hf_config = model_config.hf_text_config |
| 164 | assert text_config._supports_multimodal_for_mm_prefix() is True |
| 165 | |
| 166 | # Without the copied cache the submodule architecture has no registered |
| 167 | # multimodal processor, so re-querying the registry wrongly settles on |
| 168 | # text-only and would clear mm_prefix. |
| 169 | del text_config._supports_multimodal_inputs_cache |
| 170 | assert text_config._supports_multimodal_for_mm_prefix() is False |
| 171 | |
| 172 | |
| 173 | @pytest.mark.parametrize( |
| 174 | ("device", "expected"), |
| 175 | [ |
| 176 | (None, None), |
| 177 | ("cpu", "cpu"), |
| 178 | ("cuda", "cuda"), |
| 179 | # Callers compare against a bare device type, so an indexed device and a |
| 180 | # torch.device must normalise to the same thing as "cuda". |
| 181 | ("cuda:1", "cuda"), |
| 182 | (torch.device("cuda", 1), "cuda"), |
| 183 | ], |
| 184 | ) |
| 185 | def test_mm_processor_device_type_normalizes(device: object, expected: str | None): |
| 186 | kwargs = {} if device is None else {"device": device} |
| 187 | config = MultiModalConfig(mm_processor_kwargs=kwargs) |
| 188 | assert config.get_mm_processor_device_type() == expected |
| 189 | |
| 190 | |
| 191 | def test_merge_mm_processor_kwargs_scoped_precedence(): |
| 192 | configured = { |
| 193 | "size": { |
| 194 | "shortest_edge": 64, |
| 195 | "longest_edge": 512, |
| 196 | }, |
| 197 | "do_resize": True, |
| 198 | "padding": False, |
| 199 | "images_kwargs": { |
| 200 | "size": {"longest_edge": 1024}, |
| 201 | "do_resize": False, |
| 202 | }, |
| 203 | "videos_kwargs": { |
| 204 | "size": {"longest_edge": 2048}, |
| 205 | }, |
| 206 | } |
| 207 | inference = { |
| 208 | "size": {"shortest_edge": 128}, |
| 209 | "do_resize": True, |
| 210 | "images_kwargs": { |
| 211 | "size": {"shortest_edge": 256}, |
| 212 | }, |
| 213 | } |
| 214 | |
| 215 | config = MultiModalConfig( |
| 216 | mm_processor_kwargs=configured, |
| 217 | mm_device_do_normalize=False, |
| 218 | ) |
| 219 | |
| 220 | assert config.merge_mm_processor_kwargs(inference) == { |
| 221 | "size": { |
| 222 | "shortest_edge": 128, |
| 223 | "longest_edge": 512, |
| 224 | }, |
| 225 | "do_resize": True, |
| 226 | "padding": False, |
| 227 | "images_kwargs": { |
| 228 | "size": { |
| 229 | "shortest_edge": 256, |
| 230 | "longest_edge": 1024, |
| 231 | }, |
| 232 | "do_resize": True, |
| 233 | }, |
| 234 | "videos_kwargs": { |
| 235 | "size": { |
| 236 | "shortest_edge": 128, |
| 237 | "longest_edge": 2048, |
| 238 | }, |
| 239 | }, |
| 240 | } |
| 241 | |
| 242 | |
| 243 | def test_merge_mm_processor_kwargs_preserves_scoped_siblings_and_precedence(): |
| 244 | config = MultiModalConfig( |
| 245 | mm_processor_kwargs={ |
| 246 | "size": { |
| 247 | "shortest_edge": 100, |
| 248 | "longest_edge": 1000, |
| 249 | }, |
| 250 | "videos_kwargs": { |
| 251 | "fps": 2, |
| 252 | "size": {"longest_edge": 1200}, |
| 253 | }, |
| 254 | }, |
| 255 | mm_device_do_normalize=False, |
| 256 | ) |
| 257 | |
| 258 | assert config.merge_mm_processor_kwargs( |
| 259 | { |
| 260 | "size": {"longest_edge": 1800}, |
| 261 | "videos_kwargs": { |
| 262 | "size": {"shortest_edge": 200}, |
| 263 | }, |
| 264 | } |
| 265 | ) == { |
| 266 | "size": { |
| 267 | "shortest_edge": 100, |
| 268 | "longest_edge": 1800, |
| 269 | }, |
| 270 | "videos_kwargs": { |
| 271 | "fps": 2, |
| 272 | "size": { |
| 273 | "shortest_edge": 200, |
| 274 | "longest_edge": 1800, |
| 275 | }, |
| 276 | }, |
| 277 | } |
| 278 | |
| 279 | |
| 280 | def test_merge_mm_processor_kwargs_mapping_vs_replacement(): |
| 281 | config = MultiModalConfig( |
| 282 | mm_processor_kwargs={ |
| 283 | "size": {"shortest_edge": 64}, |
| 284 | "images_kwargs": {"size": {"longest_edge": 1024}}, |
| 285 | }, |
| 286 | mm_device_do_normalize=False, |
| 287 | ) |
| 288 | |
| 289 | assert config.merge_mm_processor_kwargs({"size": 7}) == { |
| 290 | "size": 7, |
| 291 | "images_kwargs": {"size": 7}, |
| 292 | } |
| 293 | assert config.merge_mm_processor_kwargs({"size": {}}) == { |
| 294 | "size": {"shortest_edge": 64}, |
| 295 | "images_kwargs": { |
| 296 | "size": { |
| 297 | "shortest_edge": 64, |
| 298 | "longest_edge": 1024, |
| 299 | }, |
| 300 | }, |
| 301 | } |
| 302 | |
| 303 | |
| 304 | _EMPTY_IMAGE_SCOPE_KWARGS: tuple[dict[str, object], ...] = ( |
| 305 | {}, |
| 306 | {"images_kwargs": None}, |
| 307 | {"images_kwargs": {}}, |
| 308 | ) |
| 309 | |
| 310 | |
| 311 | @pytest.mark.parametrize("configured", _EMPTY_IMAGE_SCOPE_KWARGS) |
| 312 | @pytest.mark.parametrize("inference", _EMPTY_IMAGE_SCOPE_KWARGS) |
| 313 | def test_merge_mm_processor_kwargs_empty_scopes_are_absent( |
| 314 | configured: dict[str, object], |
| 315 | inference: dict[str, object], |
| 316 | ): |
| 317 | config = MultiModalConfig( |
| 318 | mm_processor_kwargs=configured, |
| 319 | mm_device_do_normalize=False, |
| 320 | ) |
| 321 | |
| 322 | assert config.merge_mm_processor_kwargs(inference) == {} |
| 323 | |
| 324 | |
| 325 | @pytest.mark.parametrize("empty_inference", _EMPTY_IMAGE_SCOPE_KWARGS) |
| 326 | def test_merge_mm_processor_kwargs_empty_inference_scope_preserves_config_merge( |
| 327 | empty_inference: dict[str, object], |
| 328 | ): |
| 329 | config = MultiModalConfig( |
| 330 | mm_processor_kwargs={ |
| 331 | "images_kwargs": { |
| 332 | "size": { |
| 333 | "shortest_edge": 64, |
| 334 | "longest_edge": 512, |
| 335 | }, |
| 336 | }, |
| 337 | }, |
| 338 | mm_device_do_normalize=False, |
| 339 | ) |
| 340 | inference = { |
| 341 | "size": {"shortest_edge": 128}, |
| 342 | **empty_inference, |
| 343 | } |
| 344 | |
| 345 | assert config.merge_mm_processor_kwargs(inference) == { |
| 346 | "size": {"shortest_edge": 128}, |
| 347 | "images_kwargs": { |
| 348 | "size": { |
| 349 | "shortest_edge": 128, |
| 350 | "longest_edge": 512, |
| 351 | }, |
| 352 | }, |
| 353 | } |
| 354 | |
| 355 | |
| 356 | @pytest.mark.parametrize("empty_config", _EMPTY_IMAGE_SCOPE_KWARGS) |
| 357 | def test_merge_mm_processor_kwargs_empty_config_scope_allows_inference_merge( |
| 358 | empty_config: dict[str, object], |
| 359 | ): |
| 360 | config = MultiModalConfig( |
| 361 | mm_processor_kwargs={ |
| 362 | "size": {"longest_edge": 512}, |
| 363 | **empty_config, |
| 364 | }, |
| 365 | mm_device_do_normalize=False, |
| 366 | ) |
| 367 | |
| 368 | assert config.merge_mm_processor_kwargs( |
| 369 | { |
| 370 | "size": {"shortest_edge": 128}, |
| 371 | "images_kwargs": {"size": {"longest_edge": 1024}}, |
| 372 | } |
| 373 | ) == { |
| 374 | "size": { |
| 375 | "shortest_edge": 128, |
| 376 | "longest_edge": 512, |
| 377 | }, |
| 378 | "images_kwargs": { |
| 379 | "size": { |
| 380 | "shortest_edge": 128, |
| 381 | "longest_edge": 1024, |
| 382 | }, |
| 383 | }, |
| 384 | } |
| 385 | |
| 386 | |
| 387 | @pytest.mark.parametrize("non_mapping_scope", [False, 0, "", [], 123]) |
| 388 | def test_merge_mm_processor_kwargs_falsey_non_mapping_scopes_are_not_empty( |
| 389 | non_mapping_scope: object, |
| 390 | ): |
| 391 | config = MultiModalConfig( |
| 392 | mm_processor_kwargs={"images_kwargs": non_mapping_scope}, |
| 393 | mm_device_do_normalize=False, |
| 394 | ) |
| 395 | |
| 396 | for empty_inference in _EMPTY_IMAGE_SCOPE_KWARGS: |
| 397 | assert config.merge_mm_processor_kwargs(empty_inference) == { |
| 398 | "images_kwargs": non_mapping_scope |
| 399 | } |
| 400 | |
| 401 | |
| 402 | @pytest.mark.parametrize( |
| 403 | ("configured_scope", "inference_scope", "expected_scope"), |
| 404 | [ |
| 405 | (123, {"size": {"shortest_edge": 128}}, {"size": {"shortest_edge": 128}}), |
| 406 | ({"size": {"shortest_edge": 64}}, 123, 123), |
| 407 | (123, 456, 456), |
| 408 | ], |
| 409 | ) |
| 410 | def test_merge_mm_processor_kwargs_non_mapping_scope_precedence( |
| 411 | configured_scope: object, |
| 412 | inference_scope: object, |
| 413 | expected_scope: object, |
| 414 | ): |
| 415 | config = MultiModalConfig( |
| 416 | mm_processor_kwargs={"images_kwargs": configured_scope}, |
| 417 | mm_device_do_normalize=False, |
| 418 | ) |
| 419 | |
| 420 | merged = config.merge_mm_processor_kwargs({"images_kwargs": inference_scope}) |
| 421 | |
| 422 | assert merged["images_kwargs"] == expected_scope |
| 423 | |
| 424 | |
| 425 | def test_merge_mm_processor_kwargs_none_leaf_is_explicit(): |
| 426 | config = MultiModalConfig(mm_device_do_normalize=False) |
| 427 | |
| 428 | assert config.merge_mm_processor_kwargs({"images_kwargs": {"do_resize": None}}) == { |
| 429 | "images_kwargs": {"do_resize": None} |
| 430 | } |
| 431 | |
| 432 | |
| 433 | def test_merge_mm_processor_kwargs_normalize_and_ownership(): |
| 434 | configured: dict[str, object] = { |
| 435 | "images_kwargs": { |
| 436 | "size": {"shortest_edge": 64}, |
| 437 | }, |
| 438 | } |
| 439 | inference: dict[str, object] = { |
| 440 | "do_normalize": True, |
| 441 | "do_rescale": True, |
| 442 | "images_kwargs": { |
| 443 | "size": {"longest_edge": 1024}, |
| 444 | }, |
| 445 | } |
| 446 | configured_before = copy.deepcopy(configured) |
| 447 | inference_before = copy.deepcopy(inference) |
| 448 | |
| 449 | config = MultiModalConfig( |
| 450 | mm_processor_kwargs=configured, |
| 451 | mm_device_do_normalize=True, |
| 452 | ) |
| 453 | merged = config.merge_mm_processor_kwargs(inference) |
| 454 | |
| 455 | assert merged["do_normalize"] is False |
| 456 | assert merged["do_rescale"] is False |
| 457 | assert configured == configured_before |
| 458 | assert inference == inference_before |
| 459 | assert config.mm_processor_kwargs == configured_before |
| 460 | |
| 461 | images_kwargs = merged["images_kwargs"] |
| 462 | assert isinstance(images_kwargs, dict) |
| 463 | size = images_kwargs["size"] |
| 464 | assert isinstance(size, dict) |
| 465 | size["shortest_edge"] = 999 |
| 466 | |
| 467 | assert configured == configured_before |
| 468 | assert inference == inference_before |
| 469 | assert config.mm_processor_kwargs == configured_before |
| 470 | |
| 471 | |
| 472 | def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None: |
| 473 | ec_config = ( |
| 474 | None |
| 475 | if ec_role is None |
| 476 | # `is_ec_producer`/`is_ec_consumer` are False without a connector, so an |
| 477 | # unset one would make every role look like "no EC role at all". |
| 478 | else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role) |
| 479 | ) |
| 480 | MultiModalConfig( |
| 481 | mm_processor_kwargs={"device": device} |
| 482 | ).validate_mm_processor_device(ec_config) |
| 483 | |
| 484 | |
| 485 | @pytest.mark.parametrize("ec_role", [None, "ec_producer"]) |
| 486 | def test_bad_mm_processor_device_rejected(ec_role: ECRole | None): |
| 487 | """A bad device must fail during startup, not mid-request. |
| 488 | |
| 489 | Rejected for every role, and on a CPU-only platform too, so a typo can never |
| 490 | silently fall through to running the processor somewhere unintended. |
| 491 | """ |
| 492 | with ( |
| 493 | patch("vllm.platforms.current_platform.device_type", "cpu"), |
| 494 | pytest.raises(ValueError, match='Invalid "device" in mm_processor_kwargs'), |
| 495 | ): |
| 496 | _validate_mm_processor_device(device="not-a-device", ec_role=ec_role) |
| 497 | |
| 498 | |
| 499 | @pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"]) |
| 500 | def test_accelerator_mm_processor_rejected_outside_encoder_instance( |
| 501 | ec_role: ECRole | None, |
| 502 | ): |
| 503 | """Only an encode-only EPD instance has the device to itself. |
| 504 | |
| 505 | Every other role runs the language model in the same process, so frontend |
| 506 | accelerator work would contend with the forward pass and allocate outside |
| 507 | the memory profiled for the KV cache. |
| 508 | """ |
| 509 | with ( |
| 510 | patch("vllm.platforms.current_platform.device_type", "cuda"), |
| 511 | pytest.raises(ValueError, match="also runs the language model"), |
| 512 | ): |
| 513 | _validate_mm_processor_device(device="cuda", ec_role=ec_role) |
| 514 | |
| 515 | |
| 516 | def test_accelerator_mm_processor_allowed_on_encoder_instance(): |
| 517 | with patch("vllm.platforms.current_platform.device_type", "cuda"): |
| 518 | _validate_mm_processor_device(device="cuda", ec_role="ec_producer") |
| 519 | |
| 520 | |
| 521 | def test_cpu_mm_processor_needs_no_ec_role(): |
| 522 | """The gate only applies to the accelerator, so CPU stays unrestricted.""" |
| 523 | with patch("vllm.platforms.current_platform.device_type", "cuda"): |
| 524 | _validate_mm_processor_device(device="cpu", ec_role=None) |
| 525 | |
| 526 | |
| 527 | @pytest.mark.parametrize( |
| 528 | ("flag", "kwargs", "expected"), |
| 529 | [ |
| 530 | # "auto" is only settled once the EC role is known, so it must leave no |
| 531 | # device behind for `VllmConfig` to mistake for an explicit request. |
| 532 | ("auto", None, None), |
| 533 | (None, None, None), |
| 534 | ("cpu", None, "cpu"), |
| 535 | ("cuda", None, "cuda"), |
| 536 | # Any explicit value other than "cpu" means "whatever this platform |
| 537 | # calls its accelerator". |
| 538 | ("gpu", None, "cuda"), |
| 539 | # An explicit device in the kwargs wins over the convenience flag. |
| 540 | ("cpu", {"device": "cuda"}, "cuda"), |
| 541 | ("cuda", {"device": "cpu"}, "cpu"), |
| 542 | ], |
| 543 | ) |
| 544 | def test_fold_mm_processor_device( |
| 545 | flag: str | None, kwargs: dict | None, expected: str | None |
| 546 | ): |
| 547 | with patch("vllm.platforms.current_platform.device_type", "cuda"): |
| 548 | folded = MultiModalConfig.fold_mm_processor_device(kwargs, flag) |
| 549 | |
| 550 | config = MultiModalConfig(mm_processor_kwargs=folded or {}) |
| 551 | assert config.get_mm_processor_device_type() == expected |
| 552 | |
| 553 | |
| 554 | def _resolve_mm_processor_device( |
| 555 | *, |
| 556 | ec_role: ECRole | None, |
| 557 | mm_tensor_ipc: str = "torch_shm", |
| 558 | device: str | None = None, |
| 559 | ) -> str | None: |
| 560 | """Run the `auto` resolution and report where the processor ended up.""" |
| 561 | mm_config = MultiModalConfig( |
| 562 | mm_processor_kwargs={} if device is None else {"device": device}, |
| 563 | mm_tensor_ipc=mm_tensor_ipc, |
| 564 | ) |
| 565 | model_config = MagicMock(spec=ModelConfig) |
| 566 | model_config.multimodal_config = mm_config |
| 567 | vllm_config = MagicMock(spec=VllmConfig) |
| 568 | vllm_config.model_config = model_config |
| 569 | vllm_config.ec_transfer_config = ( |
| 570 | None |
| 571 | if ec_role is None |
| 572 | else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role) |
| 573 | ) |
| 574 | |
| 575 | with patch("vllm.platforms.current_platform.device_type", "cuda"): |
| 576 | VllmConfig._resolve_mm_processor_device(vllm_config) |
| 577 | return mm_config.get_mm_processor_device_type() |
| 578 | |
| 579 | |
| 580 | def test_auto_mm_processor_device_uses_accelerator_on_encoder_instance(): |
| 581 | """The one deployment that has the device to itself and can hand it over.""" |
| 582 | assert _resolve_mm_processor_device(ec_role="ec_producer") == "cuda" |
| 583 | |
| 584 | |
| 585 | @pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"]) |
| 586 | def test_auto_mm_processor_device_stays_on_cpu_off_encoder_instance( |
| 587 | ec_role: ECRole | None, |
| 588 | ): |
| 589 | """Every other role runs the language model in the same process.""" |
| 590 | assert _resolve_mm_processor_device(ec_role=ec_role) is None |
| 591 | |
| 592 | |
| 593 | def test_auto_mm_processor_device_needs_a_device_capable_transport(): |
| 594 | """Other transports serialize host bytes, so the output is copied back.""" |
| 595 | assert ( |
| 596 | _resolve_mm_processor_device(ec_role="ec_producer", mm_tensor_ipc="direct_rpc") |
| 597 | is None |
| 598 | ) |
| 599 | |
| 600 | |
| 601 | def test_auto_mm_processor_device_leaves_an_explicit_request_alone(): |
| 602 | assert _resolve_mm_processor_device(ec_role="ec_producer", device="cpu") == "cpu" |
| 603 | |
| 604 | |
| 605 | @pytest.mark.parametrize( |
| 606 | ("ec_role", "expected"), |
| 607 | [ |
| 608 | (None, False), |
| 609 | ("ec_consumer", False), |
| 610 | ("ec_both", False), |
| 611 | ("ec_producer", True), |
| 612 | ], |
| 613 | ) |
| 614 | def test_ec_producer_only_enables_mm_encoder_only( |
| 615 | ec_role: ECRole | None, expected: bool |
| 616 | ): |
| 617 | mm_config = MultiModalConfig() |
| 618 | model_config = MagicMock(spec=ModelConfig) |
| 619 | model_config.multimodal_config = mm_config |
| 620 | vllm_config = MagicMock(spec=VllmConfig) |
| 621 | vllm_config.model_config = model_config |
| 622 | vllm_config.ec_transfer_config = ( |
| 623 | None |
| 624 | if ec_role is None |
| 625 | else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role) |
| 626 | ) |
| 627 | |
| 628 | VllmConfig._resolve_mm_encoder_only(vllm_config) |
| 629 | |
| 630 | assert mm_config.mm_encoder_only is expected |
| 631 | |
| 632 | |
| 633 | def test_vllm_config_runs_the_mm_processor_device_check(): |
| 634 | """Startup must reach the check; the rule itself is covered above. |
| 635 | |
| 636 | Guards the wiring, which no other test would notice going missing. |
| 637 | """ |
| 638 | model_config = MagicMock(spec=ModelConfig) |
| 639 | model_config.multimodal_config = MultiModalConfig( |
| 640 | mm_processor_kwargs={"device": "cuda"} |
| 641 | ) |
| 642 | vllm_config = MagicMock(spec=VllmConfig) |
| 643 | vllm_config.model_config = model_config |
| 644 | vllm_config.ec_transfer_config = None |
| 645 | |
| 646 | with ( |
| 647 | patch("vllm.platforms.current_platform.device_type", "cuda"), |
| 648 | pytest.raises(ValueError, match="also runs the language model"), |
| 649 | ): |
| 650 | VllmConfig._validate_mm_processor_device(vllm_config) |
| 651 | |
| 652 | |
| 653 | def _resolve_mm_video_decode_device( |
| 654 | *, |
| 655 | ec_role: ECRole | None, |
| 656 | mm_tensor_ipc: str = "torch_shm", |
| 657 | device: str | None = None, |
| 658 | video_kwargs: dict | None = None, |
| 659 | torchcodec_available: bool = True, |
| 660 | ) -> dict: |
| 661 | """Run processor-device then video-decode-device resolution and report |
| 662 | the resulting video media IO kwargs.""" |
| 663 | mm_config = MultiModalConfig( |
| 664 | mm_processor_kwargs={} if device is None else {"device": device}, |
| 665 | mm_tensor_ipc=mm_tensor_ipc, |
| 666 | media_io_kwargs={} if video_kwargs is None else {"video": dict(video_kwargs)}, |
| 667 | ) |
| 668 | model_config = MagicMock(spec=ModelConfig) |
| 669 | model_config.multimodal_config = mm_config |
| 670 | vllm_config = MagicMock(spec=VllmConfig) |
| 671 | vllm_config.model_config = model_config |
| 672 | vllm_config.ec_transfer_config = ( |
| 673 | None |
| 674 | if ec_role is None |
| 675 | else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role) |
| 676 | ) |
| 677 | |
| 678 | with ( |
| 679 | patch("vllm.platforms.current_platform.device_type", "cuda"), |
| 680 | patch( |
| 681 | "vllm.utils.import_utils.check_torchcodec_available", |
| 682 | side_effect=None if torchcodec_available else ImportError("torchcodec"), |
| 683 | ), |
| 684 | ): |
| 685 | VllmConfig._resolve_mm_processor_device(vllm_config) |
| 686 | VllmConfig._resolve_mm_video_decode_device(vllm_config) |
| 687 | return mm_config.media_io_kwargs.get("video", {}) |
| 688 | |
| 689 | |
| 690 | def test_auto_video_decode_uses_nvdec_on_encoder_instance(): |
| 691 | """The processor runs on the accelerator there, so decoded frames should |
| 692 | stay on-device too.""" |
| 693 | assert _resolve_mm_video_decode_device(ec_role="ec_producer") == { |
| 694 | "backend": "torchcodec", |
| 695 | "device": "cuda", |
| 696 | } |
| 697 | |
| 698 | |
| 699 | @pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"]) |
| 700 | def test_auto_video_decode_stays_on_cpu_off_encoder_instance( |
| 701 | ec_role: ECRole | None, |
| 702 | ): |
| 703 | """The processor stays on CPU off encode-only instances, so video |
| 704 | decoding should too.""" |
| 705 | assert _resolve_mm_video_decode_device(ec_role=ec_role) == {} |
| 706 | |
| 707 | |
| 708 | def test_auto_video_decode_follows_explicit_processor_device(): |
| 709 | assert _resolve_mm_video_decode_device(ec_role="ec_producer", device="cpu") == {} |
| 710 | assert _resolve_mm_video_decode_device(ec_role="ec_producer", device="cuda") == { |
| 711 | "backend": "torchcodec", |
| 712 | "device": "cuda", |
| 713 | } |
| 714 | |
| 715 | |
| 716 | def test_auto_video_decode_respects_explicit_media_io_kwargs(): |
| 717 | assert _resolve_mm_video_decode_device( |
| 718 | ec_role="ec_producer", video_kwargs={"backend": "opencv"} |
| 719 | ) == {"backend": "opencv"} |
| 720 | assert _resolve_mm_video_decode_device( |
| 721 | ec_role="ec_producer", video_kwargs={"device": "cpu"} |
| 722 | ) == {"device": "cpu"} |
| 723 | |
| 724 | |
| 725 | def test_auto_video_decode_skipped_without_torchcodec(): |
| 726 | assert ( |
| 727 | _resolve_mm_video_decode_device( |
| 728 | ec_role="ec_producer", |
| 729 | torchcodec_available=False, |
| 730 | ) |
| 731 | == {} |
| 732 | ) |