A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Misc] Consolidate Audio tests into multimodal common generation tests (#18214)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
May 16, 2025 at 17:18 UTC
390ec88905fa2c7dfbcfc5e772891e48f228bf43
9 files changed
+282
-215
tests/models/multimodal/generation/test_common.py
+60
-4
@@ -8,14 +8,14 @@ from collections import defaultdict
8
from pathlib import PosixPath
9
10
import pytest
11
-from transformers import (AutoModelForImageTextToText,
11
+from transformers import (AutoModel, AutoModelForImageTextToText,
12
AutoModelForTextToWaveform, AutoModelForVision2Seq)
13
14
from vllm.platforms import current_platform
15
from vllm.utils import identity
16
17
-from ....conftest import (IMAGE_ASSETS, HfRunner, ImageTestAssets,
18
- VideoTestAssets, VllmRunner)
17
+from ....conftest import (IMAGE_ASSETS, AudioTestAssets, HfRunner,
18
+ ImageTestAssets, VideoTestAssets, VllmRunner)
19
from ....utils import (create_new_process_for_each_test, large_gpu_mark,
20
multi_gpu_marks)
21
from ...utils import check_outputs_equal
@@ -158,6 +158,17 @@ VLM_TEST_SETTINGS = {
158
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
159
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
160
),
161
+ "ultravox": VLMTestInfo(
162
+ models = ["fixie-ai/ultravox-v0_5-llama-3_2-1b"],
163
+ test_type=VLMTestType.AUDIO,
164
+ prompt_formatter=lambda audio_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{audio_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
165
+ audio_idx_to_prompt=lambda idx: "<|audio|>",
166
+ max_model_len=4096,
167
+ max_num_seqs=2,
168
+ auto_cls=AutoModel,
169
+ hf_output_post_proc=model_utils.ultravox_trunc_hf_output,
170
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
171
+ ),
172
#### Extended model tests
173
"aria": VLMTestInfo(
174
models=["rhymes-ai/Aria"],
@@ -393,7 +404,6 @@ VLM_TEST_SETTINGS = {
404
formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
405
),
406
limit_mm_per_prompt={"video": 4},
396
- runner_mm_key="videos",
407
)],
408
),
409
"llava_next_video": VLMTestInfo(
@@ -706,6 +716,7 @@ VLM_TEST_SETTINGS = _mark_splits(VLM_TEST_SETTINGS, num_groups=2)
716
# - multi-image
717
# - image embeddings
718
# - video
719
+# - audio
720
# - custom inputs
721
@pytest.mark.parametrize(
722
"model_type,test_case",
@@ -803,6 +814,28 @@ def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,
814
)
815
816
817
+@pytest.mark.parametrize(
818
+ "model_type,test_case",
819
+ get_parametrized_options(
820
+ VLM_TEST_SETTINGS,
821
+ test_type=VLMTestType.AUDIO,
822
+ create_new_process_for_each_test=False,
823
+ ))
824
+def test_audio_models(model_type: str, test_case: ExpandableVLMTestArgs,
825
+ hf_runner: type[HfRunner], vllm_runner: type[VllmRunner],
826
+ audio_assets: AudioTestAssets, monkeypatch):
827
+ if model_type in REQUIRES_V0_MODELS:
828
+ monkeypatch.setenv("VLLM_USE_V1", "0")
829
+ model_test_info = VLM_TEST_SETTINGS[model_type]
830
+ runners.run_audio_test(
831
+ model_test_info=model_test_info,
832
+ test_case=test_case,
833
+ hf_runner=hf_runner,
834
+ vllm_runner=vllm_runner,
835
+ audio_assets=audio_assets,
836
+ )
837
+
838
+
839
@pytest.mark.parametrize(
840
"model_type,test_case",
841
get_parametrized_options(
@@ -930,6 +963,29 @@ def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,
963
)
964
965
966
+@pytest.mark.parametrize(
967
+ "model_type,test_case",
968
+ get_parametrized_options(
969
+ VLM_TEST_SETTINGS,
970
+ test_type=VLMTestType.AUDIO,
971
+ create_new_process_for_each_test=True,
972
+ ))
973
+def test_audio_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,
974
+ hf_runner: type[HfRunner],
975
+ vllm_runner: type[VllmRunner],
976
+ audio_assets: AudioTestAssets, monkeypatch):
977
+ if model_type in REQUIRES_V0_MODELS:
978
+ monkeypatch.setenv("VLLM_USE_V1", "0")
979
+ model_test_info = VLM_TEST_SETTINGS[model_type]
980
+ runners.run_audio_test(
981
+ model_test_info=model_test_info,
982
+ test_case=test_case,
983
+ hf_runner=hf_runner,
984
+ vllm_runner=vllm_runner,
985
+ audio_assets=audio_assets,
986
+ )
987
+
988
+
989
@pytest.mark.parametrize(
990
"model_type,test_case",
991
get_parametrized_options(
tests/models/multimodal/generation/test_ultravox.py
+3
-109
@@ -1,20 +1,16 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-from typing import Any, Optional
4
+from typing import Any
5
6
import numpy as np
7
import pytest
8
import pytest_asyncio
9
-from transformers import AutoModel, AutoTokenizer
9
+from transformers import AutoTokenizer
10
11
-from vllm.multimodal.audio import resample_audio_librosa
12
-from vllm.sequence import SampleLogprobs
13
-
14
-from ....conftest import AUDIO_ASSETS, AudioTestAssets, HfRunner, VllmRunner
11
+from ....conftest import AUDIO_ASSETS, AudioTestAssets, VllmRunner
12
from ....utils import RemoteOpenAIServer
13
from ...registry import HF_EXAMPLE_MODELS
17
-from ...utils import check_logprobs_close
14
15
MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
16
@@ -88,79 +84,6 @@ def _get_prompt(audio_count, question, placeholder):
84
add_generation_prompt=True)
85
86
91
-def vllm_to_hf_output(vllm_output: tuple[list[int], str,
92
- Optional[SampleLogprobs]],
93
- model: str):
94
- """Sanitize vllm output to be comparable with hf output."""
95
- output_ids, output_str, out_logprobs = vllm_output
96
-
97
- tokenizer = AutoTokenizer.from_pretrained(model)
98
- eos_token_id = tokenizer.eos_token_id
99
-
100
- hf_output_ids = output_ids[:]
101
- hf_output_str = output_str
102
- if hf_output_ids[-1] == eos_token_id:
103
- hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
104
-
105
- return hf_output_ids, hf_output_str, out_logprobs
106
-
107
-
108
-def run_test(
109
- hf_runner: type[HfRunner],
110
- vllm_runner: type[VllmRunner],
111
- prompts_and_audios: list[tuple[str, str, AudioTuple]],
112
- model: str,
113
- *,
114
- dtype: str,
115
- max_tokens: int,
116
- num_logprobs: int,
117
- **kwargs,
118
-):
119
- """Inference result should be the same between hf and vllm."""
120
- model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
121
- model_info.check_available_online(on_fail="skip")
122
- model_info.check_transformers_version(on_fail="skip")
123
-
124
- # NOTE: take care of the order. run vLLM first, and then run HF.
125
- # vLLM needs a fresh new process without cuda initialization.
126
- # if we run HF first, the cuda initialization will be done and it
127
- # will hurt multiprocessing backend with fork method (the default method).
128
-
129
- with vllm_runner(model, dtype=dtype, enforce_eager=True,
130
- **kwargs) as vllm_model:
131
- vllm_outputs_per_audio = [
132
- vllm_model.generate_greedy_logprobs([vllm_prompt],
133
- max_tokens,
134
- num_logprobs=num_logprobs,
135
- audios=[audio])
136
- for vllm_prompt, _, audio in prompts_and_audios
137
- ]
138
-
139
- with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
140
- hf_outputs_per_audio = [
141
- hf_model.generate_greedy_logprobs_limit(
142
- [hf_prompt],
143
- max_tokens,
144
- num_logprobs=num_logprobs,
145
- audios=[(resample_audio_librosa(audio[0],
146
- orig_sr=audio[1],
147
- target_sr=16000), 16000)])
148
- for _, hf_prompt, audio in prompts_and_audios
149
- ]
150
-
151
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_audio,
152
- vllm_outputs_per_audio):
153
- check_logprobs_close(
154
- outputs_0_lst=hf_outputs,
155
- outputs_1_lst=[
156
- vllm_to_hf_output(vllm_output, model)
157
- for vllm_output in vllm_outputs
158
- ],
159
- name_0="hf",
160
- name_1="vllm",
161
- )
162
-
163
-
87
def run_multi_audio_test(
88
vllm_runner: type[VllmRunner],
89
prompts_and_audios: list[tuple[str, list[AudioTuple]]],
@@ -194,35 +117,6 @@ def run_multi_audio_test(
117
assert all(tokens for tokens, *_ in vllm_outputs)
118
119
197
-@pytest.mark.core_model
198
-@pytest.mark.parametrize("dtype", ["bfloat16"])
199
-@pytest.mark.parametrize("max_tokens", [128])
200
-@pytest.mark.parametrize("num_logprobs", [5])
201
-@pytest.mark.parametrize("vllm_kwargs", [
202
- pytest.param({}, marks=pytest.mark.cpu_model),
203
- pytest.param(CHUNKED_PREFILL_KWARGS),
204
-])
205
-def test_models(hf_runner, vllm_runner, audio_assets: AudioTestAssets,
206
- dtype: str, max_tokens: int, num_logprobs: int,
207
- vllm_kwargs: dict) -> None:
208
- audio_inputs = [(
209
- _get_prompt(1, audio, VLLM_PLACEHOLDER),
210
- _get_prompt(1, audio, HF_PLACEHOLDER),
211
- audio.audio_and_sample_rate,
212
- ) for audio in audio_assets]
213
-
214
- run_test(
215
- hf_runner,
216
- vllm_runner,
217
- audio_inputs,
218
- MODEL_NAME,
219
- dtype=dtype,
220
- max_tokens=max_tokens,
221
- num_logprobs=num_logprobs,
222
- **vllm_kwargs,
223
- )
224
-
225
-
120
@pytest.mark.core_model
121
@pytest.mark.parametrize("dtype", ["half"])
122
@pytest.mark.parametrize("max_tokens", [128])
tests/models/multimodal/generation/vlm_utils/builders.py
+96
-37
@@ -7,18 +7,21 @@ from typing import Callable, Optional, Union
7
8
import torch
9
10
+from vllm.multimodal.audio import AudioResampler
11
from vllm.multimodal.image import rescale_image_size
12
from vllm.multimodal.video import (rescale_video_size, resize_video,
13
sample_frames_from_video)
14
14
-from .....conftest import ImageTestAssets, VideoTestAssets
15
-from .types import (SINGLE_IMAGE_BASE_PROMPTS, TEST_IMG_PLACEHOLDER,
15
+from .....conftest import AudioTestAssets, ImageTestAssets, VideoTestAssets
16
+from .types import (SINGLE_AUDIO_BASE_PROMPT, SINGLE_IMAGE_BASE_PROMPTS,
17
+ TEST_AUDIO_PLACEHOLDER, TEST_IMG_PLACEHOLDER,
18
TEST_VIDEO_PLACEHOLDER, VIDEO_BASE_PROMPT,
17
- ImageSizeWrapper, SizeType, VLMTestInfo)
19
+ ImageSizeWrapper, PromptWithMultiModalInput, SizeType,
20
+ VLMTestInfo)
21
22
20
-def replace_test_placeholder(prompt: str, img_idx_to_prompt: Callable[[int],
21
- str],
23
+def replace_test_placeholder(prompt: str, mm_idx_to_prompt: Callable[[int],
24
+ str],
25
test_placeholder: str) -> str:
26
"""Given a prompt, replaces each test placeholder with the
27
model-specific tag.
@@ -26,7 +29,7 @@ def replace_test_placeholder(prompt: str, img_idx_to_prompt: Callable[[int],
29
prompt_segments = prompt.split(test_placeholder)
30
img_prompt = prompt_segments[0]
31
for placeholder_idx, next_seg in enumerate(prompt_segments[1:], start=1):
29
- img_prompt += img_idx_to_prompt(placeholder_idx)
32
+ img_prompt += mm_idx_to_prompt(placeholder_idx)
33
img_prompt += next_seg
34
return img_prompt
35
@@ -34,6 +37,7 @@ def replace_test_placeholder(prompt: str, img_idx_to_prompt: Callable[[int],
37
def get_model_prompts(base_prompts: Iterable[str],
38
img_idx_to_prompt: Optional[Callable[[int], str]],
39
video_idx_to_prompt: Optional[Callable[[int], str]],
40
+ audio_idx_to_prompt: Optional[Callable[[int], str]],
41
prompt_formatter: Callable[[str], str]) -> list[str]:
42
"""Given a model-agnostic base prompt and test configuration for a model(s)
43
to be tested, update the media placeholders and apply the prompt formatting
@@ -60,6 +64,11 @@ def get_model_prompts(base_prompts: Iterable[str],
64
video_idx_to_prompt,
65
TEST_VIDEO_PLACEHOLDER)
66
67
+ if audio_idx_to_prompt:
68
+ base_prompt = replace_test_placeholder(base_prompt,
69
+ audio_idx_to_prompt,
70
+ TEST_AUDIO_PLACEHOLDER)
71
+
72
# Apply the prompt formatter to wrap the base prompt with
73
# the correct media placeholders to get the model test prompt
74
model_prompt = prompt_formatter(base_prompt)
@@ -68,10 +77,11 @@ def get_model_prompts(base_prompts: Iterable[str],
77
78
79
def build_single_image_inputs_from_test_info(
71
- test_info: VLMTestInfo,
72
- image_assets: ImageTestAssets,
73
- size_wrapper: ImageSizeWrapper,
74
- tmp_path: Optional[PosixPath] = None):
80
+ test_info: VLMTestInfo,
81
+ image_assets: ImageTestAssets,
82
+ size_wrapper: ImageSizeWrapper,
83
+ tmp_path: Optional[PosixPath] = None,
84
+) -> list[PromptWithMultiModalInput]:
85
if test_info.prompt_formatter is None:
86
raise ValueError(
87
"Prompt formatter must be set to build single image inputs")
@@ -79,6 +89,7 @@ def build_single_image_inputs_from_test_info(
89
model_prompts = get_model_prompts(test_info.single_image_prompts,
90
test_info.img_idx_to_prompt,
91
test_info.video_idx_to_prompt,
92
+ test_info.audio_idx_to_prompt,
93
test_info.prompt_formatter)
94
95
# For models that require a local path / URL encoded in the image; export
@@ -97,28 +108,32 @@ def build_single_image_inputs_from_test_info(
108
return build_single_image_inputs(images, model_prompts, size_wrapper)
109
110
100
-def build_single_image_inputs(images, model_prompts,
101
- size_wrapper: ImageSizeWrapper):
111
+def build_single_image_inputs(
112
+ images, model_prompts,
113
+ size_wrapper: ImageSizeWrapper) -> list[PromptWithMultiModalInput]:
114
# For every image / prompt pair, get a pair containing two lists of
115
# length size_factors, where the first contains duplicates of the model
116
# prompt [str], and the second contains copies of the image after being
117
# scaled by one of the size factors.
118
#
119
# NOTE: rescaling preserves the image aspect ratio.
108
- return [(
109
- [prompt for _ in size_wrapper.data],
110
- [
111
- apply_image_size_scaling(image, size, size_wrapper.type)
112
- for size in size_wrapper.data
113
- ],
114
- ) for image, prompt in zip(images, model_prompts)]
120
+ return [
121
+ PromptWithMultiModalInput(
122
+ prompts=[prompt for _ in size_wrapper.data],
123
+ image_data=[
124
+ apply_image_size_scaling(image, size, size_wrapper.type)
125
+ for size in size_wrapper.data
126
+ ],
127
+ ) for image, prompt in zip(images, model_prompts)
128
+ ]
129
130
131
def build_multi_image_inputs_from_test_info(
118
- test_info: VLMTestInfo,
119
- image_assets: ImageTestAssets,
120
- size_wrapper: ImageSizeWrapper,
121
- tmp_path: Optional[PosixPath] = None):
132
+ test_info: VLMTestInfo,
133
+ image_assets: ImageTestAssets,
134
+ size_wrapper: ImageSizeWrapper,
135
+ tmp_path: Optional[PosixPath] = None,
136
+) -> list[PromptWithMultiModalInput]:
137
if test_info.prompt_formatter is None:
138
raise ValueError(
139
"Prompt formatter must be set to build multi image inputs")
@@ -126,6 +141,7 @@ def build_multi_image_inputs_from_test_info(
141
model_prompts = get_model_prompts([test_info.multi_image_prompt],
142
test_info.img_idx_to_prompt,
143
test_info.video_idx_to_prompt,
144
+ test_info.audio_idx_to_prompt,
145
test_info.prompt_formatter)
146
147
if test_info.prompt_path_encoder is not None:
@@ -146,15 +162,18 @@ def build_multi_image_inputs_from_test_info(
162
)
163
164
149
-def build_multi_image_inputs(image_lists, model_prompts,
150
- size_wrapper: ImageSizeWrapper):
151
- return [(
152
- [prompt for _ in size_wrapper.data],
153
- [[
154
- apply_image_size_scaling(image, size, size_wrapper.type)
155
- for image in images
156
- ] for size in size_wrapper.data],
157
- ) for images, prompt in zip(image_lists, model_prompts)]
165
+def build_multi_image_inputs(
166
+ image_lists, model_prompts,
167
+ size_wrapper: ImageSizeWrapper) -> list[PromptWithMultiModalInput]:
168
+ return [
169
+ PromptWithMultiModalInput(
170
+ prompts=[prompt for _ in size_wrapper.data],
171
+ image_data=[[
172
+ apply_image_size_scaling(image, size, size_wrapper.type)
173
+ for image in images
174
+ ] for size in size_wrapper.data],
175
+ ) for images, prompt in zip(image_lists, model_prompts)
176
+ ]
177
178
179
def build_embedding_inputs_from_test_info(
@@ -177,6 +196,7 @@ def build_embedding_inputs_from_test_info(
196
SINGLE_IMAGE_BASE_PROMPTS,
197
test_info.img_idx_to_prompt,
198
test_info.video_idx_to_prompt,
199
+ test_info.audio_idx_to_prompt,
200
test_info.prompt_formatter,
201
)
202
@@ -195,13 +215,14 @@ def build_video_inputs_from_test_info(
215
video_assets: VideoTestAssets,
216
size_wrapper: ImageSizeWrapper,
217
num_frames: int,
198
-):
218
+) -> list[PromptWithMultiModalInput]:
219
if test_info.prompt_formatter is None:
220
raise ValueError("Prompt formatter must be set to build video inputs")
221
model_prompts = get_model_prompts(
222
[VIDEO_BASE_PROMPT],
223
test_info.img_idx_to_prompt,
224
test_info.video_idx_to_prompt,
225
+ test_info.audio_idx_to_prompt,
226
test_info.prompt_formatter,
227
)
228
@@ -213,10 +234,14 @@ def build_video_inputs_from_test_info(
234
video_scaler = (resize_video if size_wrapper.type == SizeType.FIXED_SIZE
235
else rescale_video_size)
236
216
- return [(
217
- [prompt for _ in size_wrapper.data],
218
- [video_scaler(video, size) for size in size_wrapper.data],
219
- ) for video, prompt in zip(sampled_vids, model_prompts)]
237
+ return [
238
+ PromptWithMultiModalInput(
239
+ prompts=[prompt for _ in size_wrapper.data],
240
+ video_data=[
241
+ video_scaler(video, size) for size in size_wrapper.data
242
+ ],
243
+ ) for video, prompt in zip(sampled_vids, model_prompts)
244
+ ]
245
246
247
def apply_image_size_scaling(image, size: Union[float, tuple[int, int]],
@@ -236,3 +261,37 @@ def apply_image_size_scaling(image, size: Union[float, tuple[int, int]],
261
# We have a list of fixed sizes
262
return image.resize(size)
263
raise ValueError("ImageSizeWrapper type must be FIXED_SIZE or SIZE_FACTOR")
264
+
265
+
266
+def build_audio_inputs_from_test_info(
267
+ test_info: VLMTestInfo,
268
+ audio_assets: AudioTestAssets,
269
+) -> list[PromptWithMultiModalInput]:
270
+ if test_info.prompt_formatter is None:
271
+ raise ValueError("Prompt formatter must be set to build audio inputs")
272
+ model_prompts = get_model_prompts(
273
+ SINGLE_AUDIO_BASE_PROMPT,
274
+ test_info.img_idx_to_prompt,
275
+ test_info.video_idx_to_prompt,
276
+ test_info.audio_idx_to_prompt,
277
+ test_info.prompt_formatter,
278
+ )
279
+ resampler = AudioResampler(
280
+ target_sr=16000,
281
+ method="librosa",
282
+ )
283
+ audios = [asset.audio_and_sample_rate for asset in audio_assets]
284
+ resampled_audios = [(
285
+ resampler.resample(
286
+ audio,
287
+ orig_sr=sr,
288
+ ),
289
+ int(resampler.target_sr),
290
+ ) for audio, sr in audios]
291
+
292
+ return [
293
+ PromptWithMultiModalInput(
294
+ prompts=model_prompts,
295
+ audio_data=resampled_audios,
296
+ )
297
+ ]
tests/models/multimodal/generation/vlm_utils/case_filtering.py
+4
-4
@@ -83,7 +83,7 @@ def get_parametrized_options(test_settings: dict[str, VLMTestInfo],
83
test_info.num_video_frames)
84
85
# No sizes passed for custom inputs, since inputs are directly provided
86
- if test_type != VLMTestType.CUSTOM_INPUTS:
86
+ if test_type not in (VLMTestType.CUSTOM_INPUTS, VLMTestType.AUDIO):
87
wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
88
if wrapped_sizes is None:
89
raise ValueError(
@@ -91,7 +91,7 @@ def get_parametrized_options(test_settings: dict[str, VLMTestInfo],
91
iter_kwargs["size_wrapper"] = wrapped_sizes
92
93
#Otherwise expand the custom test options instead
94
- else:
94
+ elif test_type == VLMTestType.CUSTOM_INPUTS:
95
if test_info.custom_test_opts is None:
96
raise ValueError("Test has type CUSTOM_INPUTS, but none given")
97
iter_kwargs["custom_test_opts"] = test_info.custom_test_opts
@@ -136,8 +136,8 @@ def get_wrapped_test_sizes(
136
ImageSizeWrapper(type=SizeType.SIZE_FACTOR, data=factor)
137
for factor in EMBEDDING_SIZE_FACTORS
138
])
139
- # Custom inputs have preprocessed inputs
140
- elif test_type == VLMTestType.CUSTOM_INPUTS:
139
+ # Audio and Custom inputs have preprocessed inputs
140
+ elif test_type in (VLMTestType.AUDIO, VLMTestType.CUSTOM_INPUTS):
141
return tuple()
142
143
size_factors = test_info.image_size_factors \
tests/models/multimodal/generation/vlm_utils/core.py
+18
-11
@@ -1,9 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""Core test implementation to be shared across modalities."""
3
-from typing import Any, Callable, Optional, Union
3
+from typing import Any, Callable, Optional
4
5
import torch
6
-from PIL.Image import Image
6
from transformers.models.auto.auto_factory import _BaseAutoModelClass
7
8
from vllm.config import TaskOption
@@ -11,14 +10,14 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
10
11
from .....conftest import HfRunner, VllmRunner
12
from ....registry import HF_EXAMPLE_MODELS
14
-from .types import RunnerOutput
13
+from .types import PromptWithMultiModalInput, RunnerOutput
14
15
16
def run_test(
17
*,
18
hf_runner: type[HfRunner],
19
vllm_runner: type[VllmRunner],
21
- inputs: list[tuple[list[str], list[Union[list[Image], Image]]]],
20
+ inputs: list[PromptWithMultiModalInput],
21
model: str,
22
dtype: str,
23
max_tokens: int,
@@ -38,7 +37,6 @@ def run_test(
37
hf_model_kwargs: Optional[dict[str, Any]],
38
patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]],
39
task: TaskOption = "auto",
41
- runner_mm_key: str = "images",
40
distributed_executor_backend: Optional[str] = None,
41
tensor_parallel_size: int = 1,
42
vllm_embeddings: Optional[torch.Tensor] = None,
@@ -94,10 +92,16 @@ def run_test(
92
if stop_str:
93
vllm_kwargs["stop"] = stop_str
94
97
- for prompts, media in vllm_inputs:
98
- vllm_kwargs[runner_mm_key] = media
95
+ for prompts, image_data, video_data, audio_data in vllm_inputs:
96
+ mm_data = dict(images=image_data,
97
+ videos=video_data,
98
+ audios=audio_data)
99
+ vllm_kwargs_with_mm_data = vllm_kwargs | mm_data
100
vllm_output = vllm_model.generate_greedy_logprobs(
100
- prompts, max_tokens, num_logprobs=num_logprobs, **vllm_kwargs)
101
+ prompts,
102
+ max_tokens,
103
+ num_logprobs=num_logprobs,
104
+ **vllm_kwargs_with_mm_data)
105
vllm_outputs_per_mm.append(vllm_output)
106
107
hf_model = hf_runner(model,
@@ -122,14 +126,17 @@ def run_test(
126
if stop_str:
127
hf_kwargs["stop_strings"] = stop_str
128
125
- for prompts, media in inputs:
126
- hf_kwargs[runner_mm_key] = media
129
+ for prompts, image_data, video_data, audio_data in inputs:
130
+ mm_data = dict(images=image_data,
131
+ videos=video_data,
132
+ audios=audio_data)
133
+ hf_kwargs_with_mm_data = hf_kwargs | mm_data
134
hf_output = hf_model.generate_greedy_logprobs_limit(
135
prompts,
136
max_tokens,
137
num_logprobs=num_logprobs,
138
tokenizer=tokenizer,
132
- **hf_kwargs)
139
+ **hf_kwargs_with_mm_data)
140
hf_outputs_per_mm.append(hf_output)
141
142
# Apply output processing / sanitation to the vLLM and HF runner results
tests/models/multimodal/generation/vlm_utils/custom_inputs.py
+42
-34
@@ -12,7 +12,7 @@ from vllm.multimodal.video import (rescale_video_size, resize_video,
12
13
from .....conftest import IMAGE_ASSETS, VIDEO_ASSETS
14
from .builders import build_multi_image_inputs, build_single_image_inputs
15
-from .types import ImageSizeWrapper, SizeType
15
+from .types import ImageSizeWrapper, PromptWithMultiModalInput, SizeType
16
17
18
def multi_image_multi_aspect_ratio_inputs(formatter: Callable[[str], str]):
@@ -32,24 +32,28 @@ def multi_image_multi_aspect_ratio_inputs(formatter: Callable[[str], str]):
32
"<image>\nWhat is the season?",
33
]
34
formatted_prompts = [formatter(prompt) for prompt in img_prompts]
35
-
36
- return [(
37
- formatted_prompts,
35
+ aspect_ratio_images = [
36
+ [stop_sign, cherry_blossom],
37
+ # Images with different sizes and aspect-ratios
38
+ [
39
+ rescale_image_size(stop_sign, 0.1),
40
+ stop_sign,
41
+ ],
42
[
39
- [stop_sign, cherry_blossom],
40
- # Images with different sizes and aspect-ratios
41
- [
42
- rescale_image_size(stop_sign, 0.1),
43
- stop_sign,
44
- ],
45
- [
46
- stop_sign,
47
- rescale_image_size(stop_sign, 0.25),
48
- cherry_blossom.resize((183, 488)),
49
- cherry_blossom.resize((488, 183))
50
- ],
51
- cherry_blossom,
52
- ])]
43
+ stop_sign,
44
+ rescale_image_size(stop_sign, 0.25),
45
+ cherry_blossom.resize((183, 488)),
46
+ cherry_blossom.resize((488, 183))
47
+ ],
48
+ cherry_blossom,
49
+ ]
50
+
51
+ return [
52
+ PromptWithMultiModalInput(
53
+ prompts=formatted_prompts,
54
+ image_data=aspect_ratio_images,
55
+ )
56
+ ]
57
58
59
def multi_video_multi_aspect_ratio_inputs(formatter: Callable[[str], str],
@@ -68,24 +72,28 @@ def multi_video_multi_aspect_ratio_inputs(formatter: Callable[[str], str],
72
"<video>\nWhy is this video funny?",
73
]
74
formatted_prompts = [formatter(prompt) for prompt in video_prompts]
71
-
72
- return [(
73
- formatted_prompts,
75
+ aspect_ratio_videos = [
76
+ [video, video],
77
+ # Videos with different sizes and aspect-ratios
78
[
75
- [video, video],
76
- # Videos with different sizes and aspect-ratios
77
- [
78
- rescale_video_size(video, 0.1),
79
- video,
80
- ],
81
- [
82
- video,
83
- rescale_video_size(video, 0.25),
84
- resize_video(video, (183, 488)),
85
- resize_video(video, (488, 183))
86
- ],
79
+ rescale_video_size(video, 0.1),
80
video,
88
- ])]
81
+ ],
82
+ [
83
+ video,
84
+ rescale_video_size(video, 0.25),
85
+ resize_video(video, (183, 488)),
86
+ resize_video(video, (488, 183))
87
+ ],
88
+ video,
89
+ ]
90
+
91
+ return [
92
+ PromptWithMultiModalInput(
93
+ prompts=formatted_prompts,
94
+ video_data=aspect_ratio_videos,
95
+ )
96
+ ]
97
98
99
def different_patch_input_cases_internvl():
tests/models/multimodal/generation/vlm_utils/model_utils.py
+12
@@ -237,6 +237,18 @@ def minimax_vl_01_hf_output(hf_output: RunnerOutput,
237
return output_ids, output_str, out_logprobs
238
239
240
+def ultravox_trunc_hf_output(hf_output: RunnerOutput,
241
+ model: str) -> RunnerOutput:
242
+ output_ids, output_str, out_logprobs = hf_output
243
+
244
+ tokenizer = AutoTokenizer.from_pretrained(model)
245
+ eos_token_id = tokenizer.eos_token_id
246
+ eos_token = tokenizer.decode(eos_token_id)
247
+ if output_str.endswith(eos_token):
248
+ output_str = output_str.split(eos_token)[0]
249
+ return output_ids, output_str, out_logprobs
250
+
251
+
252
####### Functions for converting image assets to embeddings
253
def get_llava_embeddings(image_assets: ImageTestAssets):
254
return [asset.image_embeds for asset in image_assets]
tests/models/multimodal/generation/vlm_utils/runners.py
+27
-10
@@ -4,8 +4,8 @@ types / modalities.
4
"""
5
from pathlib import PosixPath
6
7
-from .....conftest import (HfRunner, ImageTestAssets, VideoTestAssets,
8
- VllmRunner)
7
+from .....conftest import (AudioTestAssets, HfRunner, ImageTestAssets,
8
+ VideoTestAssets, VllmRunner)
9
from . import builders, core
10
from .types import ExpandableVLMTestArgs, VLMTestInfo
11
@@ -30,7 +30,6 @@ def run_single_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,
30
num_logprobs=test_case.num_logprobs,
31
limit_mm_per_prompt={"image": 1},
32
distributed_executor_backend=test_case.distributed_executor_backend,
33
- runner_mm_key="images",
33
**model_test_info.get_non_parametrized_runner_kwargs())
34
35
@@ -53,7 +52,6 @@ def run_multi_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,
52
num_logprobs=test_case.num_logprobs,
53
limit_mm_per_prompt={"image": len(image_assets)},
54
distributed_executor_backend=test_case.distributed_executor_backend,
56
- runner_mm_key="images",
55
**model_test_info.get_non_parametrized_runner_kwargs())
56
57
@@ -77,7 +75,6 @@ def run_embedding_test(*, model_test_info: VLMTestInfo,
75
limit_mm_per_prompt={"image": 1},
76
vllm_embeddings=vllm_embeddings,
77
distributed_executor_backend=test_case.distributed_executor_backend,
80
- runner_mm_key="images",
78
**model_test_info.get_non_parametrized_runner_kwargs())
79
80
@@ -105,7 +102,30 @@ def run_video_test(
102
num_logprobs=test_case.num_logprobs,
103
limit_mm_per_prompt={"video": len(video_assets)},
104
distributed_executor_backend=test_case.distributed_executor_backend,
108
- runner_mm_key="videos",
105
+ **model_test_info.get_non_parametrized_runner_kwargs())
106
+
107
+
108
+def run_audio_test(
109
+ *,
110
+ model_test_info: VLMTestInfo,
111
+ test_case: ExpandableVLMTestArgs,
112
+ hf_runner: type[HfRunner],
113
+ vllm_runner: type[VllmRunner],
114
+ audio_assets: AudioTestAssets,
115
+):
116
+ inputs = builders.build_audio_inputs_from_test_info(
117
+ model_test_info, audio_assets)
118
+
119
+ core.run_test(
120
+ hf_runner=hf_runner,
121
+ vllm_runner=vllm_runner,
122
+ inputs=inputs,
123
+ model=test_case.model,
124
+ dtype=test_case.dtype,
125
+ max_tokens=test_case.max_tokens,
126
+ num_logprobs=test_case.num_logprobs,
127
+ limit_mm_per_prompt={"audio": 1},
128
+ distributed_executor_backend=test_case.distributed_executor_backend,
129
**model_test_info.get_non_parametrized_runner_kwargs())
130
131
@@ -120,11 +140,9 @@ def run_custom_inputs_test(*, model_test_info: VLMTestInfo,
140
141
inputs = test_case.custom_test_opts.inputs
142
limit_mm_per_prompt = test_case.custom_test_opts.limit_mm_per_prompt
123
- runner_mm_key = test_case.custom_test_opts.runner_mm_key
124
- # Inputs, limit_mm_per_prompt, and runner_mm_key should all be set
143
+ # Inputs and limit_mm_per_prompt should all be set
144
assert inputs is not None
145
assert limit_mm_per_prompt is not None
127
- assert runner_mm_key is not None
146
147
core.run_test(
148
hf_runner=hf_runner,
@@ -136,5 +154,4 @@ def run_custom_inputs_test(*, model_test_info: VLMTestInfo,
154
num_logprobs=test_case.num_logprobs,
155
limit_mm_per_prompt=limit_mm_per_prompt,
156
distributed_executor_backend=test_case.distributed_executor_backend,
139
- runner_mm_key=runner_mm_key,
157
**model_test_info.get_non_parametrized_runner_kwargs())
tests/models/multimodal/generation/vlm_utils/types.py
+20
-6
@@ -6,7 +6,6 @@ from pathlib import PosixPath
6
from typing import Any, Callable, NamedTuple, Optional, Union
7
8
import torch
9
-from PIL.Image import Image
9
from pytest import MarkDecorator
10
from transformers import AutoModelForCausalLM
11
from transformers.models.auto.auto_factory import _BaseAutoModelClass
@@ -15,18 +14,25 @@ from vllm.config import TaskOption
14
from vllm.sequence import SampleLogprobs
15
from vllm.transformers_utils.tokenizer import AnyTokenizer
16
18
-from .....conftest import IMAGE_ASSETS, HfRunner, ImageAsset, ImageTestAssets
17
+from .....conftest import (AUDIO_ASSETS, IMAGE_ASSETS, HfRunner, ImageAsset,
18
+ ImageTestAssets, PromptAudioInput, PromptImageInput,
19
+ PromptVideoInput)
20
from ....utils import check_logprobs_close
21
22
# meta image tag; will be replaced by the appropriate tag for the model
23
TEST_IMG_PLACEHOLDER = "<vlm_image>"
24
TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
25
+TEST_AUDIO_PLACEHOLDER = "<lmm_audio>"
26
27
# yapf: disable
28
SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts({
29
"stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
30
"cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
31
})
32
+SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts({
33
+ "mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
34
+ "winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
35
+})
36
37
MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
38
VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
@@ -38,12 +44,21 @@ RunnerOutput = tuple[list[int], str, Optional[SampleLogprobs]]
44
# yapf: enable
45
46
47
+class PromptWithMultiModalInput(NamedTuple):
48
+ """Holds the multimodal input for a single test case."""
49
+ prompts: list[str]
50
+ image_data: Optional[PromptImageInput] = None
51
+ video_data: Optional[PromptVideoInput] = None
52
+ audio_data: Optional[PromptAudioInput] = None
53
+
54
+
55
class VLMTestType(Enum):
56
IMAGE = 1
57
MULTI_IMAGE = 2
58
EMBEDDING = 3
59
VIDEO = 4
46
- CUSTOM_INPUTS = 5
60
+ AUDIO = 5
61
+ CUSTOM_INPUTS = 6
62
63
64
class SizeType(Enum):
@@ -52,10 +67,8 @@ class SizeType(Enum):
67
68
69
class CustomTestOptions(NamedTuple):
55
- inputs: list[tuple[list[str], list[Union[list[Image], Image]]]]
70
+ inputs: list[PromptWithMultiModalInput]
71
limit_mm_per_prompt: dict[str, int]
57
- # kwarg to pass multimodal data in as to vllm/hf runner instances.
58
- runner_mm_key: str = "images"
72
73
74
class ImageSizeWrapper(NamedTuple):
@@ -75,6 +88,7 @@ class VLMTestInfo(NamedTuple):
88
prompt_formatter: Optional[Callable[[str], str]] = None
89
img_idx_to_prompt: Callable[[int], str] = lambda idx: "<image>\n"
90
video_idx_to_prompt: Callable[[int], str] = lambda idx: "<video>\n"
91
+ audio_idx_to_prompt: Callable[[int], str] = lambda idx: "<audio>\n"
92
93
# Most models work on the single / multi-image prompts above, but in some
94
# cases the log prob check fails, e.g., for paligemma. We allow passing