| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from http import HTTPStatus |
| 5 | |
| 6 | import pytest |
| 7 | |
| 8 | from vllm.assets.image import ImageAsset |
| 9 | from vllm.assets.video import VideoAsset |
| 10 | from vllm.config import CacheConfig, ModelConfig, VllmConfig |
| 11 | from vllm.entrypoints.serve import create_error_response |
| 12 | from vllm.multimodal.parse import parse_mm_uuids |
| 13 | from vllm.renderers.hf import HfRenderer |
| 14 | from vllm.tokenizers.registry import cached_tokenizer_from_config |
| 15 | |
| 16 | cherry_pil_image = ImageAsset("cherry_blossom").pil_image |
| 17 | stop_pil_image = ImageAsset("stop_sign").pil_image |
| 18 | baby_reading_video_asset = VideoAsset("baby_reading") |
| 19 | baby_reading_video = ( |
| 20 | baby_reading_video_asset.np_ndarrays, |
| 21 | baby_reading_video_asset.metadata, |
| 22 | ) |
| 23 | |
| 24 | |
| 25 | def _build_renderer( |
| 26 | *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True |
| 27 | ) -> HfRenderer: |
| 28 | model_config = ModelConfig( |
| 29 | model="Qwen/Qwen2.5-VL-3B-Instruct", |
| 30 | max_model_len=128, |
| 31 | mm_processor_cache_gb=mm_cache_gb, |
| 32 | ) |
| 33 | |
| 34 | vllm_config = VllmConfig( |
| 35 | model_config=model_config, |
| 36 | cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching), |
| 37 | ) |
| 38 | |
| 39 | return HfRenderer( |
| 40 | vllm_config, |
| 41 | cached_tokenizer_from_config(model_config), |
| 42 | ) |
| 43 | |
| 44 | |
| 45 | def _build_text_only_renderer() -> HfRenderer: |
| 46 | model_config = ModelConfig(model="openai-community/gpt2", max_model_len=128) |
| 47 | |
| 48 | return HfRenderer( |
| 49 | VllmConfig(model_config=model_config), |
| 50 | cached_tokenizer_from_config(model_config), |
| 51 | ) |
| 52 | |
| 53 | |
| 54 | def test_text_only_model_mm_data_maps_to_bad_request(): |
| 55 | """Sending multimodal data to a text-only model is a client mistake, so it |
| 56 | must surface as a ValueError and reach the client as HTTP 400, not 500.""" |
| 57 | renderer = _build_text_only_renderer() |
| 58 | |
| 59 | with pytest.raises(ValueError, match="text-only") as exc_info: |
| 60 | renderer._process_multimodal( |
| 61 | prompt=[1], |
| 62 | mm_data={"image": [cherry_pil_image]}, |
| 63 | mm_uuids=None, |
| 64 | mm_processor_kwargs=None, |
| 65 | ) |
| 66 | |
| 67 | error_response = create_error_response(exc_info.value) |
| 68 | assert error_response.error.code == HTTPStatus.BAD_REQUEST |
| 69 | |
| 70 | |
| 71 | def test_multi_modal_uuids_length_mismatch_raises(): |
| 72 | renderer = _build_renderer() |
| 73 | |
| 74 | mm_data = {"image": [cherry_pil_image, stop_pil_image]} |
| 75 | |
| 76 | # Mismatch: 2 items but only 0 uuids provided |
| 77 | mm_uuids = {"image": []} # type: ignore[var-annotated] |
| 78 | |
| 79 | mm_processor = renderer.get_mm_processor() |
| 80 | mm_data_items = mm_processor.info.parse_mm_data(mm_data) |
| 81 | mm_uuid_items = parse_mm_uuids(mm_uuids) |
| 82 | |
| 83 | with pytest.raises(ValueError, match="must have same length as"): |
| 84 | renderer._process_mm_uuids(mm_data, mm_data_items, mm_uuid_items, "req-1a") |
| 85 | |
| 86 | # Mismatch: 2 items but only 1 uuid provided |
| 87 | mm_uuids = {"image": ["hash_cherry"]} |
| 88 | |
| 89 | mm_processor = renderer.get_mm_processor() |
| 90 | mm_data_items = mm_processor.info.parse_mm_data(mm_data) |
| 91 | mm_uuid_items = parse_mm_uuids(mm_uuids) |
| 92 | |
| 93 | with pytest.raises(ValueError, match="must have same length as"): |
| 94 | renderer._process_mm_uuids(mm_data, mm_data_items, mm_uuid_items, "req-1b") |
| 95 | |
| 96 | |
| 97 | def test_multi_modal_uuids_missing_modality_raises(): |
| 98 | renderer = _build_renderer() |
| 99 | |
| 100 | mm_data = { |
| 101 | "image": [cherry_pil_image], |
| 102 | "video": None, |
| 103 | } |
| 104 | |
| 105 | # Only image uuids provided; video missing should raise |
| 106 | mm_uuids = {"image": ["hash_cherry"]} |
| 107 | |
| 108 | mm_processor = renderer.get_mm_processor() |
| 109 | mm_data_items = mm_processor.info.parse_mm_data(mm_data) |
| 110 | mm_uuid_items = parse_mm_uuids(mm_uuids) |
| 111 | |
| 112 | with pytest.raises(ValueError, match="is empty but .* is missing"): |
| 113 | renderer._process_mm_uuids(mm_data, mm_data_items, mm_uuid_items, "req-2") |
| 114 | |
| 115 | |
| 116 | @pytest.mark.parametrize( |
| 117 | "mm_cache_gb, enable_prefix_caching", |
| 118 | [ |
| 119 | (4.0, True), # default behavior |
| 120 | (4.0, False), # prefix caching disabled |
| 121 | (0.0, True), # processor cache disabled |
| 122 | ], |
| 123 | ) |
| 124 | def test_multi_modal_uuids_accepts_none_and_passes_through( |
| 125 | mm_cache_gb: float, enable_prefix_caching: bool |
| 126 | ): |
| 127 | renderer = _build_renderer( |
| 128 | mm_cache_gb=mm_cache_gb, |
| 129 | enable_prefix_caching=enable_prefix_caching, |
| 130 | ) |
| 131 | |
| 132 | mm_data = { |
| 133 | "image": [cherry_pil_image, stop_pil_image], |
| 134 | "video": baby_reading_video, |
| 135 | } |
| 136 | |
| 137 | # Use a consistent two-image scenario across all configurations |
| 138 | mm_uuids = {"image": [None, "hash_stop"], "video": None} |
| 139 | |
| 140 | mm_processor = renderer.get_mm_processor() |
| 141 | mm_data_items = mm_processor.info.parse_mm_data(mm_data) |
| 142 | mm_uuid_items = parse_mm_uuids(mm_uuids) |
| 143 | |
| 144 | processed_mm_uuids = renderer._process_mm_uuids( |
| 145 | mm_data, mm_data_items, mm_uuid_items, "req-3" |
| 146 | ) |
| 147 | |
| 148 | assert processed_mm_uuids == mm_uuids |
| 149 | |
| 150 | |
| 151 | @pytest.mark.parametrize( |
| 152 | "mm_cache_gb, enable_prefix_caching", |
| 153 | [ |
| 154 | (4.0, True), # default behavior |
| 155 | (4.0, False), # prefix caching disabled |
| 156 | (0.0, True), # processor cache disabled |
| 157 | ], |
| 158 | ) |
| 159 | def test_multi_modal_uuids_accepts_empty( |
| 160 | mm_cache_gb: float, enable_prefix_caching: bool |
| 161 | ): |
| 162 | renderer = _build_renderer( |
| 163 | mm_cache_gb=mm_cache_gb, |
| 164 | enable_prefix_caching=enable_prefix_caching, |
| 165 | ) |
| 166 | |
| 167 | # While None means cached multi-modal input requiring UUIDs |
| 168 | # an empty list means no multi-modal input |
| 169 | mm_data = {"image": [], "video": [], "audio": None} # type: ignore[var-annotated] |
| 170 | mm_uuids = {"image": [], "video": None, "audio": []} # type: ignore[var-annotated] |
| 171 | |
| 172 | mm_processor = renderer.get_mm_processor() |
| 173 | mm_data_items = mm_processor.info.parse_mm_data(mm_data) |
| 174 | mm_uuid_items = parse_mm_uuids(mm_uuids) |
| 175 | |
| 176 | processed_mm_uuids = renderer._process_mm_uuids( |
| 177 | mm_data, mm_data_items, mm_uuid_items, "req-4" |
| 178 | ) |
| 179 | |
| 180 | assert processed_mm_uuids == mm_uuids |
| 181 | |
| 182 | |
| 183 | @pytest.mark.parametrize( |
| 184 | "mm_uuids, expected", |
| 185 | [ |
| 186 | ( |
| 187 | {"image": ["hash_cherry", "hash_stop"], "video": ["hash_video"]}, |
| 188 | {"image": ["hash_cherry", "hash_stop"], "video": ["hash_video"]}, |
| 189 | ), |
| 190 | ( |
| 191 | {"image": [None, "hash_stop"], "video": None}, |
| 192 | {"image": ["req-42-image-0", "hash_stop"], "video": ["req-42-video-0"]}, |
| 193 | ), |
| 194 | ( |
| 195 | {"image": ["", None]}, |
| 196 | {"image": ["", "req-42-image-1"], "video": ["req-42-video-0"]}, |
| 197 | ), |
| 198 | ( |
| 199 | {}, |
| 200 | { |
| 201 | "image": ["req-42-image-0", "req-42-image-1"], |
| 202 | "video": ["req-42-video-0"], |
| 203 | }, |
| 204 | ), |
| 205 | ], |
| 206 | ) |
| 207 | def test_multi_modal_uuids_preserved_when_caching_disabled(mm_uuids, expected): |
| 208 | """Only missing UUIDs get request-local IDs when both caches are disabled.""" |
| 209 | renderer = _build_renderer(mm_cache_gb=0.0, enable_prefix_caching=False) |
| 210 | |
| 211 | request_id = "req-42" |
| 212 | mm_data = { |
| 213 | "image": [cherry_pil_image, stop_pil_image], |
| 214 | "video": baby_reading_video, |
| 215 | } |
| 216 | |
| 217 | mm_processor = renderer.get_mm_processor() |
| 218 | mm_data_items = mm_processor.info.parse_mm_data(mm_data) |
| 219 | mm_uuid_items = parse_mm_uuids(mm_uuids) |
| 220 | |
| 221 | processed_mm_uuids = renderer._process_mm_uuids( |
| 222 | mm_data, mm_data_items, mm_uuid_items, request_id |
| 223 | ) |
| 224 | |
| 225 | assert processed_mm_uuids == expected |