A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Bugfix][VLM] Fix failing Phi-4-MM multi-images tests and add vision-speech test (#16424)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Apr 11, 2025 at 12:57 UTC
93195146eaa48d102c272d24d7c4e0e148759602
5 files changed
+119
-46
examples/offline_inference/audio_language.py
+9
-9
@@ -199,13 +199,6 @@ def main(args):
199
engine_args = asdict(req_data.engine_args) | {"seed": args.seed}
200
llm = LLM(**engine_args)
201
202
- # To maintain code compatibility in this script, we add LoRA here.
203
- # You can also add LoRA using:
204
- # llm.generate(prompts, lora_request=lora_request,...)
205
- if req_data.lora_requests:
206
- for lora_request in req_data.lora_requests:
207
- llm.llm_engine.add_lora(lora_request=lora_request)
208
-
202
# We set temperature to 0.2 so that outputs can be different
203
# even when all prompts are identical when running batch inference.
204
sampling_params = SamplingParams(temperature=0.2,
@@ -226,8 +219,15 @@ def main(args):
219
if args.num_prompts > 1:
220
# Batch inference
221
inputs = [inputs] * args.num_prompts
229
-
230
- outputs = llm.generate(inputs, sampling_params=sampling_params)
222
+ # Add LoRA request if applicable
223
+ lora_request = (req_data.lora_requests *
224
+ args.num_prompts if req_data.lora_requests else None)
225
+
226
+ outputs = llm.generate(
227
+ inputs,
228
+ sampling_params=sampling_params,
229
+ lora_request=lora_request,
230
+ )
231
232
for o in outputs:
233
generated_text = o.outputs[0].text
examples/offline_inference/vision_language.py
+24
-10
@@ -8,6 +8,7 @@ on HuggingFace model repository.
8
"""
9
import os
10
import random
11
+from contextlib import contextmanager
12
from dataclasses import asdict
13
from typing import NamedTuple, Optional
14
@@ -1055,6 +1056,20 @@ def apply_image_repeat(image_repeat_prob, num_prompts, data,
1056
return inputs
1057
1058
1059
+@contextmanager
1060
+def time_counter(enable: bool):
1061
+ if enable:
1062
+ import time
1063
+ start_time = time.time()
1064
+ yield
1065
+ elapsed_time = time.time() - start_time
1066
+ print("-" * 50)
1067
+ print("-- generate time = {}".format(elapsed_time))
1068
+ print("-" * 50)
1069
+ else:
1070
+ yield
1071
+
1072
+
1073
def main(args):
1074
model = args.model_type
1075
if model not in model_example_map:
@@ -1113,17 +1128,16 @@ def main(args):
1128
},
1129
} for i in range(args.num_prompts)]
1130
1116
- if args.time_generate:
1117
- import time
1118
- start_time = time.time()
1119
- outputs = llm.generate(inputs, sampling_params=sampling_params)
1120
- elapsed_time = time.time() - start_time
1121
- print("-" * 50)
1122
- print("-- generate time = {}".format(elapsed_time))
1123
- print("-" * 50)
1131
+ # Add LoRA request if applicable
1132
+ lora_request = (req_data.lora_requests *
1133
+ args.num_prompts if req_data.lora_requests else None)
1134
1125
- else:
1126
- outputs = llm.generate(inputs, sampling_params=sampling_params)
1135
+ with time_counter(args.time_generate):
1136
+ outputs = llm.generate(
1137
+ inputs,
1138
+ sampling_params=sampling_params,
1139
+ lora_request=lora_request,
1140
+ )
1141
1142
print("-" * 50)
1143
for o in outputs:
examples/offline_inference/vision_language_multi_image.py
+4
-8
@@ -661,13 +661,6 @@ def run_generate(model, question: str, image_urls: list[str],
661
engine_args = asdict(req_data.engine_args) | {"seed": args.seed}
662
llm = LLM(**engine_args)
663
664
- # To maintain code compatibility in this script, we add LoRA here.
665
- # You can also add LoRA using:
666
- # llm.generate(prompts, lora_request=lora_request,...)
667
- if req_data.lora_requests:
668
- for lora_request in req_data.lora_requests:
669
- llm.llm_engine.add_lora(lora_request=lora_request)
670
-
664
sampling_params = SamplingParams(temperature=0.0,
665
max_tokens=256,
666
stop_token_ids=req_data.stop_token_ids)
@@ -679,7 +672,9 @@ def run_generate(model, question: str, image_urls: list[str],
672
"image": req_data.image_data
673
},
674
},
682
- sampling_params=sampling_params)
675
+ sampling_params=sampling_params,
676
+ lora_request=req_data.lora_requests,
677
+ )
678
679
print("-" * 50)
680
for o in outputs:
@@ -724,6 +719,7 @@ def run_chat(model: str, question: str, image_urls: list[str],
719
}],
720
sampling_params=sampling_params,
721
chat_template=req_data.chat_template,
722
+ lora_request=req_data.lora_requests,
723
)
724
725
print("-" * 50)
tests/models/decoder_only/vision_language/test_models.py
+2
-2
@@ -433,8 +433,8 @@ VLM_TEST_SETTINGS = {
433
max_model_len=4096,
434
max_num_seqs=2,
435
task="generate",
436
- # use eager mode for hf runner since phi3v didn't work with flash_attn
437
- hf_model_kwargs={"_attn_implementation": "eager"},
436
+ # use sdpa mode for hf runner since phi3v didn't work with flash_attn
437
+ hf_model_kwargs={"_attn_implementation": "sdpa"},
438
use_tokenizer_eos=True,
439
vllm_output_post_proc=model_utils.phi3v_vllm_to_hf_output,
440
num_logprobs=10,
tests/models/decoder_only/vision_language/test_phi4mm.py
+80
-17
@@ -2,18 +2,22 @@
2
3
import os
4
import re
5
+from collections.abc import Sequence
6
from typing import Optional
7
8
+import librosa
9
import pytest
10
from huggingface_hub import snapshot_download
11
from transformers import AutoTokenizer
12
13
+from vllm.assets.image import ImageAsset
14
from vllm.lora.request import LoRARequest
15
from vllm.multimodal.image import rescale_image_size
16
from vllm.platforms import current_platform
17
from vllm.sequence import SampleLogprobs
18
16
-from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
19
+from ....conftest import (IMAGE_ASSETS, HfRunner, PromptAudioInput,
20
+ PromptImageInput, VllmRunner)
21
from ....utils import large_gpu_test
22
from ...utils import check_logprobs_close
23
@@ -29,6 +33,8 @@ model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
33
# Since the vision-lora and speech-lora co-exist with the base model,
34
# we have to manually specify the path of the lora weights.
35
vision_lora_path = os.path.join(model_path, "vision-lora")
36
+speech_question = os.path.join(model_path, "examples",
37
+ "what_is_shown_in_this_image.wav")
38
models = [model_path]
39
40
@@ -64,7 +70,8 @@ if current_platform.is_rocm():
70
def run_test(
71
hf_runner: type[HfRunner],
72
vllm_runner: type[VllmRunner],
67
- inputs: list[tuple[list[str], PromptImageInput]],
73
+ inputs: Sequence[tuple[list[str], PromptImageInput,
74
+ Optional[PromptAudioInput]]],
75
model: str,
76
*,
77
max_model_len: int,
@@ -104,28 +111,49 @@ def run_test(
111
enforce_eager=True,
112
) as vllm_model:
113
lora_request = LoRARequest("vision", 1, vision_lora_path)
107
- vllm_model.model.llm_engine.add_lora(lora_request=lora_request)
114
vllm_outputs_per_case = [
115
vllm_model.generate_greedy_logprobs(prompts,
116
max_tokens,
117
num_logprobs=num_logprobs,
112
- images=images)
113
- for prompts, images in inputs
118
+ images=images,
119
+ audios=audios,
120
+ lora_request=lora_request)
121
+ for prompts, images, audios in inputs
122
]
123
116
- # use eager mode for hf runner, since phi3_v didn't work with flash_attn
117
- hf_model_kwargs = {"_attn_implementation": "eager"}
124
+ hf_model_kwargs = {"_attn_implementation": "sdpa"}
125
with hf_runner(model, dtype=dtype,
126
model_kwargs=hf_model_kwargs) as hf_model:
120
- eos_token_id = hf_model.processor.tokenizer.eos_token_id
127
+
128
+ hf_processor = hf_model.processor
129
+ eos_token_id = hf_processor.tokenizer.eos_token_id
130
+
131
+ def patch_hf_processor(*args,
132
+ text="",
133
+ images=None,
134
+ audio=None,
135
+ sampling_rate=None,
136
+ **kwargs):
137
+ audios = None
138
+ if audio is not None and sampling_rate is not None:
139
+ audios = [(audio, sampling_rate)]
140
+ return hf_processor(*args,
141
+ text=text,
142
+ images=images,
143
+ audios=audios,
144
+ **kwargs)
145
+
146
+ hf_model.processor = patch_hf_processor
147
+
148
hf_outputs_per_case = [
149
hf_model.generate_greedy_logprobs_limit(prompts,
150
max_tokens,
151
num_logprobs=num_logprobs,
152
images=images,
153
+ audios=audios,
154
eos_token_id=eos_token_id,
155
num_logits_to_keep=0)
128
- for prompts, images in inputs
156
+ for prompts, images, audios in inputs
157
]
158
159
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
@@ -138,8 +166,6 @@ def run_test(
166
)
167
168
141
-# Since we use _attn_implementation="eager" for hf_runner, there is more
142
-# significant numerical difference. The basic `logprobs=5` fails to pass.
169
@pytest.mark.parametrize("model", models)
170
@pytest.mark.parametrize(
171
"size_factors",
@@ -151,7 +177,7 @@ def run_test(
177
# Single-scale, batched
178
[1.0, 1.0, 1.0],
179
# Multi-scale
154
- [0.7, 0.75, 1.0],
180
+ [0.25, 0.5, 1.0],
181
],
182
)
183
@pytest.mark.parametrize("dtype", [target_dtype])
@@ -166,6 +192,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
192
inputs_per_image = [(
193
[prompt for _ in size_factors],
194
[rescale_image_size(image, factor) for factor in size_factors],
195
+ None,
196
) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
197
198
run_test(
@@ -201,17 +228,18 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
228
@pytest.mark.parametrize("max_model_len", [10000])
229
@pytest.mark.parametrize("max_tokens", [128])
230
@pytest.mark.parametrize("num_logprobs", [10])
204
-@pytest.mark.xfail(
205
- reason="Phi-4-MM multi-image inference is divergent with hf model.")
231
def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
232
size_factors, dtype: str, max_model_len: int,
233
max_tokens: int, num_logprobs: int) -> None:
234
images = [asset.pil_image for asset in image_assets]
235
236
inputs_per_case = [
212
- ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
213
- [[rescale_image_size(image, factor) for image in images]
214
- for factor in size_factors])
237
+ (
238
+ [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
239
+ [[rescale_image_size(image, factor) for image in images]
240
+ for factor in size_factors],
241
+ None,
242
+ ),
243
]
244
245
run_test(
@@ -226,3 +254,38 @@ def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
254
mm_limit=2,
255
tensor_parallel_size=1,
256
)
257
+
258
+
259
+@pytest.mark.parametrize("model", models)
260
+@pytest.mark.parametrize("dtype", [target_dtype])
261
+@pytest.mark.parametrize("max_model_len", [10000])
262
+@pytest.mark.parametrize("max_tokens", [128])
263
+@pytest.mark.parametrize("num_logprobs", [10])
264
+def test_vision_speech_models(hf_runner, vllm_runner, model, dtype: str,
265
+ max_model_len: int, max_tokens: int,
266
+ num_logprobs: int) -> None:
267
+
268
+ # use the example speech question so that the model outputs are reasonable
269
+ audio = librosa.load(speech_question, sr=None)
270
+ image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
271
+
272
+ inputs_vision_speech = [
273
+ (
274
+ ["<|user|><|image_1|><|audio_1|><|end|><|assistant|>"],
275
+ [image],
276
+ [audio],
277
+ ),
278
+ ]
279
+
280
+ run_test(
281
+ hf_runner,
282
+ vllm_runner,
283
+ inputs_vision_speech,
284
+ model,
285
+ dtype=dtype,
286
+ max_model_len=max_model_len,
287
+ max_tokens=max_tokens,
288
+ num_logprobs=num_logprobs,
289
+ mm_limit=1,
290
+ tensor_parallel_size=1,
291
+ )