[CI/Build] Avoid duplicate runner startup for multimodal test (#52417)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Isotr0py committed
Aug 17, 2026 at 06:17 UTC
e3c1cb54fcafe796bc2a0354bb0962edbb4abbb8
5 files changed
+77
-93
tests/models/multimodal/generation/test_phi4mm.py
+8
-27
@@ -66,6 +66,11 @@ def vllm_to_hf_output(
66
67
68
target_dtype = "half"
69
+IMAGE_SIZE_FACTOR_GROUPS = (
70
+ (1.0,),
71
+ (1.0, 1.0, 1.0),
72
+ (0.25, 0.5, 1.0),
73
+)
74
75
76
def run_test(
@@ -167,17 +172,6 @@ def run_test(
172
173
174
@pytest.mark.parametrize("model", models)
170
-@pytest.mark.parametrize(
171
- "size_factors",
172
- [
173
- # Single-scale
174
- [1.0],
175
- # Single-scale, batched
176
- [1.0, 1.0, 1.0],
177
- # Multi-scale
178
- [0.25, 0.5, 1.0],
179
- ],
180
-)
175
@pytest.mark.parametrize("dtype", [target_dtype])
176
@pytest.mark.parametrize("max_model_len", [12800])
177
@pytest.mark.parametrize("max_tokens", [128])
@@ -187,7 +181,6 @@ def test_models(
181
vllm_runner,
182
image_assets,
183
model,
190
- size_factors,
184
dtype: str,
185
max_model_len: int,
186
max_tokens: int,
@@ -201,6 +194,7 @@ def test_models(
194
[rescale_image_size(image, factor) for factor in size_factors],
195
None,
196
)
197
+ for size_factors in IMAGE_SIZE_FACTOR_GROUPS
198
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
199
]
200
@@ -220,19 +214,6 @@ def test_models(
214
215
@large_gpu_test(min_gb=48)
216
@pytest.mark.parametrize("model", models)
223
-@pytest.mark.parametrize(
224
- "size_factors",
225
- [
226
- # No image
227
- # [],
228
- # Single-scale
229
- [1.0],
230
- # Single-scale, batched
231
- [1.0, 1.0, 1.0],
232
- # Multi-scale
233
- [0.25, 0.5, 1.0],
234
- ],
235
-)
217
@pytest.mark.parametrize("dtype", [target_dtype])
218
@pytest.mark.parametrize("max_model_len", [25600])
219
@pytest.mark.parametrize("max_tokens", [128])
@@ -242,7 +223,6 @@ def test_multi_images_models(
223
vllm_runner,
224
image_assets,
225
model,
245
- size_factors,
226
dtype: str,
227
max_model_len: int,
228
max_tokens: int,
@@ -258,7 +238,8 @@ def test_multi_images_models(
238
for factor in size_factors
239
],
240
None,
261
- ),
241
+ )
242
+ for size_factors in IMAGE_SIZE_FACTOR_GROUPS
243
]
244
245
run_test(
tests/models/multimodal/generation/test_qwen2_vl.py
+13
-36
@@ -29,6 +29,16 @@ def enable_pickle(monkeypatch):
29
30
models = ["Qwen/Qwen2-VL-2B-Instruct"]
31
target_dtype = "half"
32
+IMAGE_SIZE_FACTOR_GROUPS = (
33
+ (0.5,),
34
+ (0.5, 0.5),
35
+ (0.25, 0.5, 0.5),
36
+)
37
+VIDEO_SIZE_FACTOR_GROUPS = (
38
+ (0.5,),
39
+ (0.5, 0.5),
40
+ (0.25, 0.25, 0.5),
41
+)
42
43
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
44
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
@@ -323,17 +333,6 @@ def run_embedding_input_test(
333
334
@pytest.mark.core_model
335
@pytest.mark.parametrize("model", models)
326
-@pytest.mark.parametrize(
327
- "size_factors",
328
- [
329
- # Single-scale
330
- [0.5],
331
- # Single-scale, batched
332
- [0.5, 0.5],
333
- # Multi-scale
334
- [0.25, 0.5, 0.5],
335
- ],
336
-)
336
@pytest.mark.parametrize("dtype", [target_dtype])
337
@pytest.mark.parametrize("max_tokens", [128])
338
@pytest.mark.parametrize("num_logprobs", [10])
@@ -341,7 +340,6 @@ def test_qwen2_vl_image_embeddings_input(
340
vllm_runner,
341
image_assets,
342
model,
344
- size_factors,
343
dtype,
344
max_tokens,
345
num_logprobs,
@@ -355,6 +353,7 @@ def test_qwen2_vl_image_embeddings_input(
353
[rescale_image_size(image, factor) for factor in size_factors],
354
[],
355
)
356
+ for size_factors in IMAGE_SIZE_FACTOR_GROUPS
357
for image, prompt in zip(images, IMAGE_PROMPTS)
358
]
359
@@ -372,17 +371,6 @@ def test_qwen2_vl_image_embeddings_input(
371
372
@pytest.mark.core_model
373
@pytest.mark.parametrize("model", models)
375
-@pytest.mark.parametrize(
376
- "size_factors",
377
- [
378
- # Single-scale
379
- [0.5],
380
- # Single-scale, batched
381
- [0.5, 0.5],
382
- # Multi-scale
383
- [0.25, 0.5, 0.5],
384
- ],
385
-)
374
@pytest.mark.parametrize("dtype", [target_dtype])
375
@pytest.mark.parametrize("max_tokens", [128])
376
@pytest.mark.parametrize("num_logprobs", [10])
@@ -390,7 +378,6 @@ def test_qwen2_vl_multiple_image_embeddings_input(
378
vllm_runner,
379
image_assets,
380
model,
393
- size_factors,
381
dtype: str,
382
max_tokens: int,
383
num_logprobs: int,
@@ -406,6 +393,7 @@ def test_qwen2_vl_multiple_image_embeddings_input(
393
],
394
[],
395
)
396
+ for size_factors in IMAGE_SIZE_FACTOR_GROUPS
397
]
398
399
run_embedding_input_test(
@@ -422,17 +410,6 @@ def test_qwen2_vl_multiple_image_embeddings_input(
410
411
@pytest.mark.core_model
412
@pytest.mark.parametrize("model", models)
425
-@pytest.mark.parametrize(
426
- "size_factors",
427
- [
428
- # Single-scale
429
- [0.5],
430
- # Single-scale, batched
431
- [0.5, 0.5],
432
- # Multi-scale
433
- [0.25, 0.25, 0.5],
434
- ],
435
-)
413
@pytest.mark.parametrize("dtype", [target_dtype])
414
@pytest.mark.parametrize("max_tokens", [128])
415
@pytest.mark.parametrize("num_logprobs", [10])
@@ -440,7 +417,6 @@ def test_qwen2_vl_video_embeddings_input(
417
vllm_runner,
418
video_assets,
419
model,
443
- size_factors,
420
dtype: str,
421
max_tokens: int,
422
num_logprobs: int,
@@ -457,6 +433,7 @@ def test_qwen2_vl_video_embeddings_input(
433
[],
434
[rescale_video_size(video, factor) for factor in size_factors],
435
)
436
+ for size_factors in VIDEO_SIZE_FACTOR_GROUPS
437
for video, prompt in zip(sampled_vids, VIDEO_PROMPTS)
438
]
439
tests/models/multimodal/generation/vlm_utils/case_filtering.py
+8
-7
@@ -94,7 +94,8 @@ def get_model_type_cases(
94
test_info.needs_video_metadata
95
)
96
97
- # No sizes passed for custom inputs, since inputs are directly provided
97
+ # Keep all size batches in one test case so they share the same model
98
+ # instances. No sizes are passed for preprocessed audio or custom inputs.
99
if test_type not in (
100
VLMTestType.CUSTOM_INPUTS,
101
VLMTestType.AUDIO,
@@ -102,7 +103,9 @@ def get_model_type_cases(
103
wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
104
if wrapped_sizes is None:
105
raise ValueError(f"Sizes must be set for test type {test_type}")
105
- iter_kwargs["size_wrapper"] = wrapped_sizes
106
+ if not wrapped_sizes:
107
+ return []
108
+ iter_kwargs["size_wrappers"] = (wrapped_sizes,)
109
110
# Otherwise expand the custom test options instead
111
elif test_type == VLMTestType.CUSTOM_INPUTS:
@@ -127,9 +130,8 @@ def get_parametrized_options(
130
create_new_process_for_each_test: bool,
131
):
132
"""Converts all of our VLMTestInfo into an expanded list of parameters.
130
- This is similar to nesting pytest parametrize calls, but done directly
131
- through an itertools product so that each test can set things like
132
- size factors etc, while still running in isolated test cases.
133
+ Runner configuration values are expanded through an itertools product.
134
+ Input size batches stay grouped so they can share model instances.
135
"""
136
matching_tests = get_filtered_test_settings(
137
test_settings, test_type, create_new_process_for_each_test
@@ -149,8 +151,7 @@ def get_wrapped_test_sizes(
151
test_info: VLMTestInfo, test_type: VLMTestType
152
) -> tuple[ImageSizeWrapper, ...]:
153
"""Given a test info which may have size factors or fixed sizes, wrap them
152
- and combine them into an iterable, each of which will be used in parameter
153
- expansion.
154
+ and combine them into an iterable of request batches.
155
156
Args:
157
test_info: Test configuration to be expanded.
tests/models/multimodal/generation/vlm_utils/runners.py
+43
-16
@@ -4,6 +4,7 @@
4
types / modalities.
5
"""
6
7
+import itertools
8
from pathlib import PosixPath
9
10
from .....conftest import (
@@ -27,9 +28,14 @@ def run_single_image_test(
28
vllm_runner: type[VllmRunner],
29
image_assets: ImageTestAssets,
30
):
30
- assert test_case.size_wrapper is not None
31
- inputs = builders.build_single_image_inputs_from_test_info(
32
- model_test_info, image_assets, test_case.size_wrapper, tmp_path
31
+ assert test_case.size_wrappers
32
+ inputs = list(
33
+ itertools.chain.from_iterable(
34
+ builders.build_single_image_inputs_from_test_info(
35
+ model_test_info, image_assets, size_wrapper, tmp_path
36
+ )
37
+ for size_wrapper in test_case.size_wrappers
38
+ )
39
)
40
41
core.run_test(
@@ -55,9 +61,14 @@ def run_multi_image_test(
61
vllm_runner: type[VllmRunner],
62
image_assets: ImageTestAssets,
63
):
58
- assert test_case.size_wrapper is not None
59
- inputs = builders.build_multi_image_inputs_from_test_info(
60
- model_test_info, image_assets, test_case.size_wrapper, tmp_path
64
+ assert test_case.size_wrappers
65
+ inputs = list(
66
+ itertools.chain.from_iterable(
67
+ builders.build_multi_image_inputs_from_test_info(
68
+ model_test_info, image_assets, size_wrapper, tmp_path
69
+ )
70
+ for size_wrapper in test_case.size_wrappers
71
+ )
72
)
73
74
core.run_test(
@@ -82,9 +93,20 @@ def run_embedding_test(
93
vllm_runner: type[VllmRunner],
94
image_assets: ImageTestAssets,
95
):
85
- assert test_case.size_wrapper is not None
86
- inputs, vllm_embeddings = builders.build_embedding_inputs_from_test_info(
87
- model_test_info, image_assets, test_case.size_wrapper
96
+ assert test_case.size_wrappers
97
+ inputs_and_embeddings = [
98
+ builders.build_embedding_inputs_from_test_info(
99
+ model_test_info, image_assets, size_wrapper
100
+ )
101
+ for size_wrapper in test_case.size_wrappers
102
+ ]
103
+ inputs = list(
104
+ itertools.chain.from_iterable(inputs for inputs, _ in inputs_and_embeddings)
105
+ )
106
+ vllm_embeddings = list(
107
+ itertools.chain.from_iterable(
108
+ embeddings for _, embeddings in inputs_and_embeddings
109
+ )
110
)
111
112
core.run_test(
@@ -110,14 +132,19 @@ def run_video_test(
132
vllm_runner: type[VllmRunner],
133
video_assets: VideoTestAssets,
134
):
113
- assert test_case.size_wrapper is not None
135
+ assert test_case.size_wrappers
136
assert test_case.num_video_frames is not None
115
- inputs = builders.build_video_inputs_from_test_info(
116
- model_test_info,
117
- video_assets,
118
- test_case.size_wrapper,
119
- test_case.num_video_frames,
120
- test_case.needs_video_metadata,
137
+ inputs = list(
138
+ itertools.chain.from_iterable(
139
+ builders.build_video_inputs_from_test_info(
140
+ model_test_info,
141
+ video_assets,
142
+ size_wrapper,
143
+ test_case.num_video_frames,
144
+ test_case.needs_video_metadata,
145
+ )
146
+ for size_wrapper in test_case.size_wrappers
147
+ )
148
)
149
150
core.run_test(
tests/models/multimodal/generation/vlm_utils/types.py
+5
-7
@@ -158,11 +158,9 @@ class VLMTestInfo(NamedTuple):
158
num_video_frames: int | tuple[int] = 16
159
needs_video_metadata: bool = False
160
161
- # Fixed image sizes / image size factors; most tests use image_size_factors
162
- # The values provided for these two fields will be stacked and expanded
163
- # such that each model will consider each image size factor / image size
164
- # once per tests (much like concatenating and wrapping in one parametrize
165
- # call)
161
+ # Fixed image sizes / image size factors; most tests use image_size_factors.
162
+ # Each inner iterable defines one request batch. All batches are run against
163
+ # the same model instance.
164
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
165
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
166
@@ -211,8 +209,8 @@ class ExpandableVLMTestArgs(NamedTuple):
209
num_logprobs: int
210
dtype: str
211
distributed_executor_backend: str | None
214
- # Sizes are used for everything except for custom input tests
215
- size_wrapper: ImageSizeWrapper | None = None
212
+ # Sizes are used for everything except audio and custom input tests.
213
+ size_wrappers: tuple[ImageSizeWrapper, ...] = ()
214
# Video only
215
num_video_frames: int | None = None
216
needs_video_metadata: bool = False