| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import os |
| 5 | from collections.abc import Sequence |
| 6 | |
| 7 | import pytest |
| 8 | import regex as re |
| 9 | from transformers import AutoTokenizer |
| 10 | |
| 11 | from vllm.assets.image import ImageAsset |
| 12 | from vllm.logprobs import SampleLogprobs |
| 13 | from vllm.lora.request import LoRARequest |
| 14 | from vllm.multimodal.image import convert_image_mode, rescale_image_size |
| 15 | from vllm.multimodal.media.audio import load_audio |
| 16 | from vllm.transformers_utils.repo_utils import hf_api |
| 17 | |
| 18 | from ....conftest import ( |
| 19 | IMAGE_ASSETS, |
| 20 | HfRunner, |
| 21 | PromptAudioInput, |
| 22 | PromptImageInput, |
| 23 | VllmRunner, |
| 24 | ) |
| 25 | from ....utils import large_gpu_test |
| 26 | from ...utils import check_logprobs_close |
| 27 | |
| 28 | HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts( |
| 29 | { |
| 30 | "stop_sign": "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501 |
| 31 | "cherry_blossom": "<|user|>\n<|image_1|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501 |
| 32 | } |
| 33 | ) |
| 34 | HF_MULTIIMAGE_IMAGE_PROMPT = ( |
| 35 | "<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501 |
| 36 | ) |
| 37 | |
| 38 | model_path = hf_api().snapshot_download("microsoft/Phi-4-multimodal-instruct") |
| 39 | # Since the vision-lora and speech-lora co-exist with the base model, |
| 40 | # we have to manually specify the path of the lora weights. |
| 41 | vision_lora_path = os.path.join(model_path, "vision-lora") |
| 42 | speech_question = os.path.join( |
| 43 | model_path, "examples", "what_is_shown_in_this_image.wav" |
| 44 | ) |
| 45 | models = [model_path] |
| 46 | |
| 47 | |
| 48 | def vllm_to_hf_output( |
| 49 | vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str |
| 50 | ): |
| 51 | """Sanitize vllm output to be comparable with hf output.""" |
| 52 | _, output_str, out_logprobs = vllm_output |
| 53 | |
| 54 | output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str) |
| 55 | assert output_str_without_image[0] == " " |
| 56 | output_str_without_image = output_str_without_image[1:] |
| 57 | |
| 58 | hf_output_str = output_str_without_image + "<|end|><|endoftext|>" |
| 59 | |
| 60 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 61 | hf_output_ids = tokenizer.encode(output_str_without_image) |
| 62 | assert hf_output_ids[0] == 1 |
| 63 | hf_output_ids = hf_output_ids[1:] |
| 64 | |
| 65 | return hf_output_ids, hf_output_str, out_logprobs |
| 66 | |
| 67 | |
| 68 | target_dtype = "half" |
| 69 | IMAGE_SIZE_FACTOR_GROUPS = ( |
| 70 | (1.0,), |
| 71 | (1.0, 1.0, 1.0), |
| 72 | (0.25, 0.5, 1.0), |
| 73 | ) |
| 74 | |
| 75 | |
| 76 | def run_test( |
| 77 | hf_runner: type[HfRunner], |
| 78 | vllm_runner: type[VllmRunner], |
| 79 | inputs: Sequence[tuple[list[str], PromptImageInput, PromptAudioInput | None]], |
| 80 | model: str, |
| 81 | *, |
| 82 | max_model_len: int, |
| 83 | dtype: str, |
| 84 | max_tokens: int, |
| 85 | num_logprobs: int, |
| 86 | mm_limit: int, |
| 87 | tensor_parallel_size: int, |
| 88 | distributed_executor_backend: str | None = None, |
| 89 | ): |
| 90 | """Inference result should be the same between hf and vllm. |
| 91 | |
| 92 | All the image fixtures for the test are from IMAGE_ASSETS. |
| 93 | For huggingface runner, we provide the PIL images as input. |
| 94 | For vllm runner, we provide MultiModalDataDict objects |
| 95 | and corresponding MultiModalConfig as input. |
| 96 | Note, the text input is also adjusted to abide by vllm contract. |
| 97 | The text output is sanitized to be able to compare with hf. |
| 98 | """ |
| 99 | # NOTE: take care of the order. run vLLM first, and then run HF. |
| 100 | # vLLM needs a fresh new process without cuda initialization. |
| 101 | # if we run HF first, the cuda initialization will be done and it |
| 102 | # will hurt multiprocessing backend with fork method (the default method). |
| 103 | # max_model_len should be greater than image_feature_size |
| 104 | with vllm_runner( |
| 105 | model, |
| 106 | runner="generate", |
| 107 | max_model_len=max_model_len, |
| 108 | max_num_seqs=2, |
| 109 | dtype=dtype, |
| 110 | limit_mm_per_prompt={"image": mm_limit}, |
| 111 | tensor_parallel_size=tensor_parallel_size, |
| 112 | distributed_executor_backend=distributed_executor_backend, |
| 113 | enable_lora=True, |
| 114 | max_lora_rank=320, |
| 115 | gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI |
| 116 | enforce_eager=True, |
| 117 | ) as vllm_model: |
| 118 | lora_request = LoRARequest("vision", 1, vision_lora_path) |
| 119 | vllm_outputs_per_case = [ |
| 120 | vllm_model.generate_greedy_logprobs( |
| 121 | prompts, |
| 122 | max_tokens, |
| 123 | num_logprobs=num_logprobs, |
| 124 | images=images, |
| 125 | audios=audios, |
| 126 | lora_request=lora_request, |
| 127 | ) |
| 128 | for prompts, images, audios in inputs |
| 129 | ] |
| 130 | |
| 131 | # This error occurs inside `get_peft_model` |
| 132 | # FIXME: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/75 |
| 133 | pytest.skip("HF impl is not compatible with current transformers") |
| 134 | |
| 135 | hf_model_kwargs = {"_attn_implementation": "sdpa"} |
| 136 | with hf_runner(model, dtype=dtype, model_kwargs=hf_model_kwargs) as hf_model: |
| 137 | hf_processor = hf_model.processor |
| 138 | eos_token_id = hf_processor.tokenizer.eos_token_id |
| 139 | |
| 140 | def patch_hf_processor( |
| 141 | *args, text="", images=None, audio=None, sampling_rate=None, **kwargs |
| 142 | ): |
| 143 | audios = None |
| 144 | if audio is not None and sampling_rate is not None: |
| 145 | audios = [(audio, sampling_rate)] |
| 146 | return hf_processor( |
| 147 | *args, text=text, images=images, audios=audios, **kwargs |
| 148 | ) |
| 149 | |
| 150 | hf_model.processor = patch_hf_processor |
| 151 | |
| 152 | hf_outputs_per_case = [ |
| 153 | hf_model.generate_greedy_logprobs_limit( |
| 154 | prompts, |
| 155 | max_tokens, |
| 156 | num_logprobs=num_logprobs, |
| 157 | images=images, |
| 158 | audios=audios, |
| 159 | eos_token_id=eos_token_id, |
| 160 | num_logits_to_keep=0, |
| 161 | ) |
| 162 | for prompts, images, audios in inputs |
| 163 | ] |
| 164 | |
| 165 | for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case): |
| 166 | check_logprobs_close( |
| 167 | outputs_0_lst=hf_outputs, |
| 168 | outputs_1_lst=vllm_outputs, |
| 169 | name_0="hf", |
| 170 | name_1="vllm", |
| 171 | ) |
| 172 | |
| 173 | |
| 174 | @pytest.mark.parametrize("model", models) |
| 175 | @pytest.mark.parametrize("dtype", [target_dtype]) |
| 176 | @pytest.mark.parametrize("max_model_len", [12800]) |
| 177 | @pytest.mark.parametrize("max_tokens", [128]) |
| 178 | @pytest.mark.parametrize("num_logprobs", [10]) |
| 179 | def test_models( |
| 180 | hf_runner, |
| 181 | vllm_runner, |
| 182 | image_assets, |
| 183 | model, |
| 184 | dtype: str, |
| 185 | max_model_len: int, |
| 186 | max_tokens: int, |
| 187 | num_logprobs: int, |
| 188 | ) -> None: |
| 189 | images = [asset.pil_image for asset in image_assets] |
| 190 | |
| 191 | inputs_per_image = [ |
| 192 | ( |
| 193 | [prompt for _ in size_factors], |
| 194 | [rescale_image_size(image, factor) for factor in size_factors], |
| 195 | None, |
| 196 | ) |
| 197 | for size_factors in IMAGE_SIZE_FACTOR_GROUPS |
| 198 | for image, prompt in zip(images, HF_IMAGE_PROMPTS) |
| 199 | ] |
| 200 | |
| 201 | run_test( |
| 202 | hf_runner, |
| 203 | vllm_runner, |
| 204 | inputs_per_image, |
| 205 | model, |
| 206 | dtype=dtype, |
| 207 | max_model_len=max_model_len, |
| 208 | max_tokens=max_tokens, |
| 209 | num_logprobs=num_logprobs, |
| 210 | mm_limit=1, |
| 211 | tensor_parallel_size=1, |
| 212 | ) |
| 213 | |
| 214 | |
| 215 | @large_gpu_test(min_gb=48) |
| 216 | @pytest.mark.parametrize("model", models) |
| 217 | @pytest.mark.parametrize("dtype", [target_dtype]) |
| 218 | @pytest.mark.parametrize("max_model_len", [25600]) |
| 219 | @pytest.mark.parametrize("max_tokens", [128]) |
| 220 | @pytest.mark.parametrize("num_logprobs", [10]) |
| 221 | def test_multi_images_models( |
| 222 | hf_runner, |
| 223 | vllm_runner, |
| 224 | image_assets, |
| 225 | model, |
| 226 | dtype: str, |
| 227 | max_model_len: int, |
| 228 | max_tokens: int, |
| 229 | num_logprobs: int, |
| 230 | ) -> None: |
| 231 | images = [asset.pil_image for asset in image_assets] |
| 232 | |
| 233 | inputs_per_case = [ |
| 234 | ( |
| 235 | [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors], |
| 236 | [ |
| 237 | [rescale_image_size(image, factor) for image in images] |
| 238 | for factor in size_factors |
| 239 | ], |
| 240 | None, |
| 241 | ) |
| 242 | for size_factors in IMAGE_SIZE_FACTOR_GROUPS |
| 243 | ] |
| 244 | |
| 245 | run_test( |
| 246 | hf_runner, |
| 247 | vllm_runner, |
| 248 | inputs_per_case, |
| 249 | model, |
| 250 | dtype=dtype, |
| 251 | max_model_len=max_model_len, |
| 252 | max_tokens=max_tokens, |
| 253 | num_logprobs=num_logprobs, |
| 254 | mm_limit=2, |
| 255 | tensor_parallel_size=1, |
| 256 | ) |
| 257 | |
| 258 | |
| 259 | @pytest.mark.parametrize("model", models) |
| 260 | @pytest.mark.parametrize("dtype", [target_dtype]) |
| 261 | @pytest.mark.parametrize("max_model_len", [12800]) |
| 262 | @pytest.mark.parametrize("max_tokens", [128]) |
| 263 | @pytest.mark.parametrize("num_logprobs", [10]) |
| 264 | def test_vision_speech_models( |
| 265 | hf_runner, |
| 266 | vllm_runner, |
| 267 | model, |
| 268 | dtype: str, |
| 269 | max_model_len: int, |
| 270 | max_tokens: int, |
| 271 | num_logprobs: int, |
| 272 | ) -> None: |
| 273 | # use the example speech question so that the model outputs are reasonable |
| 274 | audio = load_audio(speech_question, sr=None) |
| 275 | image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB") |
| 276 | |
| 277 | inputs_vision_speech = [ |
| 278 | ( |
| 279 | ["<|user|><|image_1|><|audio_1|><|end|><|assistant|>"], |
| 280 | [image], |
| 281 | [audio], |
| 282 | ), |
| 283 | ] |
| 284 | |
| 285 | run_test( |
| 286 | hf_runner, |
| 287 | vllm_runner, |
| 288 | inputs_vision_speech, |
| 289 | model, |
| 290 | dtype=dtype, |
| 291 | max_model_len=max_model_len, |
| 292 | max_tokens=max_tokens, |
| 293 | num_logprobs=num_logprobs, |
| 294 | mm_limit=1, |
| 295 | tensor_parallel_size=1, |
| 296 | ) |