A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 03:30 UTC
[CI/Build] Avoid duplicate empty inputs test for common multimodal generation tests (#29907)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Dec 3, 2025 at 18:27 UTC
cc4e296ea62226632de5285621fd0cd287621ddc
3 files changed
+69
-63
tests/models/multimodal/generation/test_common.py
+7
-7
@@ -137,7 +137,7 @@ VLM_TEST_SETTINGS = {
137
max_num_seqs=2,
138
auto_cls=AutoModelForImageTextToText,
139
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
140
- image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
140
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
141
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
142
),
143
"qwen2_5_omni": VLMTestInfo(
@@ -152,7 +152,7 @@ VLM_TEST_SETTINGS = {
152
auto_cls=AutoModelForTextToWaveform,
153
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
154
patch_hf_runner=model_utils.qwen2_5_omni_patch_hf_runner,
155
- image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
155
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
156
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
157
),
158
"qwen3_vl": VLMTestInfo(
@@ -173,7 +173,7 @@ VLM_TEST_SETTINGS = {
173
auto_cls=AutoModelForImageTextToText,
174
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
175
patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
176
- image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
176
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
177
marks=[
178
pytest.mark.core_model,
179
],
@@ -350,7 +350,7 @@ VLM_TEST_SETTINGS = {
350
patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
351
hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
352
stop_str=["<｜end▁of▁sentence｜>", "<｜begin▁of▁sentence｜>"],
353
- image_size_factors=[(), (1.0,), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
353
+ image_size_factors=[(1.0,), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
354
),
355
"fuyu": VLMTestInfo(
356
models=["adept/fuyu-8b"],
@@ -707,7 +707,7 @@ VLM_TEST_SETTINGS = {
707
max_model_len=8192,
708
max_num_seqs=2,
709
auto_cls=AutoModelForCausalLM,
710
- image_size_factors=[(), (0.25,)],
710
+ image_size_factors=[(0.25,)],
711
marks=[
712
pytest.mark.skipif(
713
Version(TRANSFORMERS_VERSION) == Version("4.57.3"),
@@ -760,7 +760,7 @@ VLM_TEST_SETTINGS = {
760
max_num_seqs=2,
761
auto_cls=AutoModelForImageTextToText,
762
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
763
- image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
763
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
764
marks=[pytest.mark.cpu_model],
765
),
766
"skywork_r1v": VLMTestInfo(
@@ -812,7 +812,7 @@ VLM_TEST_SETTINGS = {
812
max_model_len=4096,
813
max_num_seqs=2,
814
auto_cls=AutoModelForImageTextToText,
815
- image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
815
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
816
marks=[pytest.mark.skip("Model initialization hangs")],
817
),
818
### Tensor parallel / multi-gpu broadcast tests
tests/models/multimodal/generation/vlm_utils/case_filtering.py
+60
-54
@@ -62,6 +62,65 @@ def get_filtered_test_settings(
62
return matching_tests
63
64
65
+def get_model_type_cases(
66
+ model_type: str,
67
+ test_info: VLMTestInfo,
68
+ test_type: VLMTestType,
69
+):
70
+ # Ensure that something is wrapped as an iterable it's not already
71
+ ensure_wrapped = lambda e: e if isinstance(e, (list, tuple)) else (e,)
72
+
73
+ # This is essentially the same as nesting a bunch of mark.parametrize
74
+ # decorators, but we do it programmatically to allow overrides for on
75
+ # a per-model basis, while still being able to execute each of these
76
+ # as individual test cases in pytest.
77
+ iter_kwargs = OrderedDict(
78
+ [
79
+ ("model", ensure_wrapped(test_info.models)),
80
+ ("max_tokens", ensure_wrapped(test_info.max_tokens)),
81
+ ("num_logprobs", ensure_wrapped(test_info.num_logprobs)),
82
+ ("dtype", ensure_wrapped(test_info.dtype)),
83
+ (
84
+ "distributed_executor_backend",
85
+ ensure_wrapped(test_info.distributed_executor_backend),
86
+ ),
87
+ ]
88
+ )
89
+
90
+ # num_frames is video only
91
+ if test_type == VLMTestType.VIDEO:
92
+ iter_kwargs["num_video_frames"] = ensure_wrapped(test_info.num_video_frames)
93
+ iter_kwargs["needs_video_metadata"] = ensure_wrapped(
94
+ test_info.needs_video_metadata
95
+ )
96
+
97
+ # No sizes passed for custom inputs, since inputs are directly provided
98
+ if test_type not in (
99
+ VLMTestType.CUSTOM_INPUTS,
100
+ VLMTestType.AUDIO,
101
+ ):
102
+ wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
103
+ if wrapped_sizes is None:
104
+ raise ValueError(f"Sizes must be set for test type {test_type}")
105
+ iter_kwargs["size_wrapper"] = wrapped_sizes
106
+
107
+ # Otherwise expand the custom test options instead
108
+ elif test_type == VLMTestType.CUSTOM_INPUTS:
109
+ if test_info.custom_test_opts is None:
110
+ raise ValueError("Test has type CUSTOM_INPUTS, but none given")
111
+ iter_kwargs["custom_test_opts"] = test_info.custom_test_opts
112
+
113
+ # Wrap all model cases in a pytest parameter & pass marks through
114
+ return [
115
+ pytest.param(
116
+ model_type,
117
+ ExpandableVLMTestArgs(**{k: v for k, v in zip(iter_kwargs.keys(), case)}),
118
+ marks=test_info.marks if test_info.marks is not None else [],
119
+ )
120
+ for case in list(itertools.product(*iter_kwargs.values()))
121
+ ]
122
+
123
+
124
def get_parametrized_options(
125
test_settings: dict[str, VLMTestInfo],
126
test_type: VLMTestType,
@@ -76,64 +135,11 @@ def get_parametrized_options(
135
test_settings, test_type, create_new_process_for_each_test
136
)
137
79
- # Ensure that something is wrapped as an iterable it's not already
80
- ensure_wrapped = lambda e: e if isinstance(e, (list, tuple)) else (e,)
81
-
82
- def get_model_type_cases(model_type: str, test_info: VLMTestInfo):
83
- # This is essentially the same as nesting a bunch of mark.parametrize
84
- # decorators, but we do it programmatically to allow overrides for on
85
- # a per-model basis, while still being able to execute each of these
86
- # as individual test cases in pytest.
87
- iter_kwargs = OrderedDict(
88
- [
89
- ("model", ensure_wrapped(test_info.models)),
90
- ("max_tokens", ensure_wrapped(test_info.max_tokens)),
91
- ("num_logprobs", ensure_wrapped(test_info.num_logprobs)),
92
- ("dtype", ensure_wrapped(test_info.dtype)),
93
- (
94
- "distributed_executor_backend",
95
- ensure_wrapped(test_info.distributed_executor_backend),
96
- ),
97
- ]
98
- )
99
-
100
- # num_frames is video only
101
- if test_type == VLMTestType.VIDEO:
102
- iter_kwargs["num_video_frames"] = ensure_wrapped(test_info.num_video_frames)
103
- iter_kwargs["needs_video_metadata"] = ensure_wrapped(
104
- test_info.needs_video_metadata
105
- )
106
-
107
- # No sizes passed for custom inputs, since inputs are directly provided
108
- if test_type not in (VLMTestType.CUSTOM_INPUTS, VLMTestType.AUDIO):
109
- wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
110
- if wrapped_sizes is None:
111
- raise ValueError(f"Sizes must be set for test type {test_type}")
112
- iter_kwargs["size_wrapper"] = wrapped_sizes
113
-
114
- # Otherwise expand the custom test options instead
115
- elif test_type == VLMTestType.CUSTOM_INPUTS:
116
- if test_info.custom_test_opts is None:
117
- raise ValueError("Test has type CUSTOM_INPUTS, but none given")
118
- iter_kwargs["custom_test_opts"] = test_info.custom_test_opts
119
-
120
- # Wrap all model cases in a pytest parameter & pass marks through
121
- return [
122
- pytest.param(
123
- model_type,
124
- ExpandableVLMTestArgs(
125
- **{k: v for k, v in zip(iter_kwargs.keys(), case)}
126
- ),
127
- marks=test_info.marks if test_info.marks is not None else [],
128
- )
129
- for case in list(itertools.product(*iter_kwargs.values()))
130
- ]
131
-
138
# Get a list per model type, where each entry contains a tuple of all of
139
# that model type's cases, then flatten them into the top level so that
140
# we can consume them in one mark.parametrize call.
141
cases_by_model_type = [
136
- get_model_type_cases(model_type, test_info)
142
+ get_model_type_cases(model_type, test_info, test_type)
143
for model_type, test_info in matching_tests.items()
144
]
145
return list(itertools.chain(*cases_by_model_type))
tests/models/multimodal/generation/vlm_utils/types.py
+2
-2
@@ -50,8 +50,8 @@ MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PL
50
VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
51
52
53
-IMAGE_SIZE_FACTORS = [(), (1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
54
-EMBEDDING_SIZE_FACTORS = [(), (1.0,), (1.0, 1.0, 1.0)]
53
+IMAGE_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
54
+EMBEDDING_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0)]
55
RunnerOutput = tuple[list[int], str, SampleLogprobs | None]
56
57