A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Misc] Update transformers version limits of multi-modal tests (#16381)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Apr 10, 2025 at 14:03 UTC
3d4c87758eba24dbed14b08992b6b577a8caa238
4 files changed
+20
-270
.buildkite/test-pipeline.yaml
+2
-5
@@ -429,7 +429,7 @@ steps:
429
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
430
- pytest -v -s models/multimodal
431
- pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
432
- - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
432
+ - pytest -v -s models/decoder_only/vision_language -m 'core_model or quant_model'
433
- pytest -v -s models/embedding/vision_language -m core_model
434
- pytest -v -s models/encoder_decoder/audio_language -m core_model
435
- pytest -v -s models/encoder_decoder/language -m core_model
@@ -448,10 +448,7 @@ steps:
448
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
449
- pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
450
- pytest -v -s models/decoder_only/vision_language/test_models.py -m 'split(group=0) and not core_model and not quant_model'
451
- # HACK - run phi3v tests separately to sidestep this transformers bug
452
- # https://github.com/huggingface/transformers/issues/34307
453
- - pytest -v -s models/decoder_only/vision_language/test_phi3v.py
454
- - pytest -v -s --ignore models/decoder_only/vision_language/test_models.py --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
451
+ - pytest -v -s --ignore models/decoder_only/vision_language/test_models.py models/decoder_only/vision_language -m 'not core_model and not quant_model'
452
- pytest -v -s models/embedding/vision_language -m 'not core_model'
453
- pytest -v -s models/encoder_decoder/language -m 'not core_model'
454
- pytest -v -s models/encoder_decoder/vision_language -m 'not core_model'
tests/models/decoder_only/vision_language/test_models.py
+14
-17
@@ -425,23 +425,20 @@ VLM_TEST_SETTINGS = {
425
max_num_seqs=2,
426
patch_hf_runner=model_utils.molmo_patch_hf_runner,
427
),
428
- # Tests for phi3v currently live in another file because of a bug in
429
- # transformers. Once this issue is fixed, we can enable them here instead.
430
- # https://github.com/huggingface/transformers/issues/34307
431
- # "phi3v": VLMTestInfo(
432
- # models=["microsoft/Phi-3.5-vision-instruct"],
433
- # test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
434
- # prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|end|>\n<|assistant|>\n", # noqa: E501
435
- # img_idx_to_prompt=lambda idx: f"<|image_{idx}|>\n",
436
- # max_model_len=4096,
437
- # max_num_seqs=2,
438
- # task="generate",
439
- # # use eager mode for hf runner since phi3v didn't work with flash_attn
440
- # hf_model_kwargs={"_attn_implementation": "eager"},
441
- # use_tokenizer_eos=True,
442
- # vllm_output_post_proc=model_utils.phi3v_vllm_to_hf_output,
443
- # num_logprobs=10,
444
- # ),
428
+ "phi3v": VLMTestInfo(
429
+ models=["microsoft/Phi-3.5-vision-instruct"],
430
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
431
+ prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|end|>\n<|assistant|>\n", # noqa: E501
432
+ img_idx_to_prompt=lambda idx: f"<|image_{idx}|>\n",
433
+ max_model_len=4096,
434
+ max_num_seqs=2,
435
+ task="generate",
436
+ # use eager mode for hf runner since phi3v didn't work with flash_attn
437
+ hf_model_kwargs={"_attn_implementation": "eager"},
438
+ use_tokenizer_eos=True,
439
+ vllm_output_post_proc=model_utils.phi3v_vllm_to_hf_output,
440
+ num_logprobs=10,
441
+ ),
442
"pixtral_hf": VLMTestInfo(
443
models=["nm-testing/pixtral-12b-FP8-dynamic"],
444
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
tests/models/decoder_only/vision_language/test_phi3v.py
deleted
-245
@@ -1,245 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
3
-import os
4
-import re
5
-from typing import Optional
6
-
7
-import pytest
8
-from packaging.version import Version
9
-from transformers import AutoTokenizer
10
-from transformers import __version__ as TRANSFORMERS_VERSION
11
-
12
-from vllm.multimodal.image import rescale_image_size
13
-from vllm.platforms import current_platform
14
-from vllm.sequence import SampleLogprobs
15
-
16
-from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
17
-from ...utils import check_logprobs_close
18
-
19
-HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20
- "stop_sign":
21
- "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
22
- "cherry_blossom":
23
- "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
24
-})
25
-HF_MULTIIMAGE_IMAGE_PROMPT = "<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
26
-
27
-models = ["microsoft/Phi-3.5-vision-instruct"]
28
-
29
-
30
-def vllm_to_hf_output(vllm_output: tuple[list[int], str,
31
- Optional[SampleLogprobs]],
32
- model: str):
33
- """Sanitize vllm output to be comparable with hf output."""
34
- _, output_str, out_logprobs = vllm_output
35
-
36
- output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
37
- assert output_str_without_image[0] == " "
38
- output_str_without_image = output_str_without_image[1:]
39
-
40
- hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
41
-
42
- tokenizer = AutoTokenizer.from_pretrained(model)
43
- hf_output_ids = tokenizer.encode(output_str_without_image)
44
- assert hf_output_ids[0] == 1
45
- hf_output_ids = hf_output_ids[1:]
46
-
47
- return hf_output_ids, hf_output_str, out_logprobs
48
-
49
-
50
-target_dtype = "half"
51
-
52
-# ROCm Triton FA can run into shared memory issues with these models,
53
-# use other backends in the meantime
54
-# FIXME (mattwong, gshtrasb, hongxiayan)
55
-if current_platform.is_rocm():
56
- os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
57
-
58
-
59
-def run_test(
60
- hf_runner: type[HfRunner],
61
- vllm_runner: type[VllmRunner],
62
- inputs: list[tuple[list[str], PromptImageInput]],
63
- model: str,
64
- *,
65
- dtype: str,
66
- max_tokens: int,
67
- num_logprobs: int,
68
- mm_limit: int,
69
- tensor_parallel_size: int,
70
- distributed_executor_backend: Optional[str] = None,
71
-):
72
- """Inference result should be the same between hf and vllm.
73
-
74
- All the image fixtures for the test are from IMAGE_ASSETS.
75
- For huggingface runner, we provide the PIL images as input.
76
- For vllm runner, we provide MultiModalDataDict objects
77
- and corresponding MultiModalConfig as input.
78
- Note, the text input is also adjusted to abide by vllm contract.
79
- The text output is sanitized to be able to compare with hf.
80
- """
81
- # HACK - this is an attempted workaround for the following bug
82
- # https://github.com/huggingface/transformers/issues/34307
83
- from transformers import AutoImageProcessor # noqa: F401
84
- from transformers import AutoProcessor # noqa: F401
85
-
86
- # Once the model repo is updated to 4.49, we should be able to run the
87
- # test in `test_models.py` without the above workaround
88
- if Version(TRANSFORMERS_VERSION) >= Version("4.49"):
89
- pytest.skip(f"`transformers=={TRANSFORMERS_VERSION}` installed, "
90
- "but `transformers<=4.49` is required to run this model. "
91
- "Reason: Cannot run HF implementation")
92
-
93
- # NOTE: take care of the order. run vLLM first, and then run HF.
94
- # vLLM needs a fresh new process without cuda initialization.
95
- # if we run HF first, the cuda initialization will be done and it
96
- # will hurt multiprocessing backend with fork method (the default method).
97
- # max_model_len should be greater than image_feature_size
98
- with vllm_runner(model,
99
- task="generate",
100
- max_model_len=4096,
101
- max_num_seqs=2,
102
- dtype=dtype,
103
- limit_mm_per_prompt={"image": mm_limit},
104
- tensor_parallel_size=tensor_parallel_size,
105
- distributed_executor_backend=distributed_executor_backend,
106
- enforce_eager=True) as vllm_model:
107
- vllm_outputs_per_case = [
108
- vllm_model.generate_greedy_logprobs(prompts,
109
- max_tokens,
110
- num_logprobs=num_logprobs,
111
- images=images)
112
- for prompts, images in inputs
113
- ]
114
-
115
- # use eager mode for hf runner, since phi3_v didn't work with flash_attn
116
- hf_model_kwargs = {"_attn_implementation": "eager"}
117
- with hf_runner(model, dtype=dtype,
118
- model_kwargs=hf_model_kwargs) as hf_model:
119
- eos_token_id = hf_model.processor.tokenizer.eos_token_id
120
- hf_outputs_per_case = [
121
- hf_model.generate_greedy_logprobs_limit(prompts,
122
- max_tokens,
123
- num_logprobs=num_logprobs,
124
- images=images,
125
- eos_token_id=eos_token_id)
126
- for prompts, images in inputs
127
- ]
128
-
129
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
130
- vllm_outputs_per_case):
131
- check_logprobs_close(
132
- outputs_0_lst=hf_outputs,
133
- outputs_1_lst=[
134
- vllm_to_hf_output(vllm_output, model)
135
- for vllm_output in vllm_outputs
136
- ],
137
- name_0="hf",
138
- name_1="vllm",
139
- )
140
-
141
-
142
-# Since we use _attn_implementation="eager" for hf_runner, there is more
143
-# significant numerical difference. The basic `logprobs=5` fails to pass.
144
-@pytest.mark.parametrize("model", models)
145
-@pytest.mark.parametrize(
146
- "size_factors",
147
- [
148
- # No image
149
- [],
150
- # Single-scale
151
- [1.0],
152
- # Single-scale, batched
153
- [1.0, 1.0, 1.0],
154
- # Multi-scale
155
- [0.25, 0.5, 1.0],
156
- ],
157
-)
158
-@pytest.mark.parametrize("dtype", [target_dtype])
159
-@pytest.mark.parametrize("max_tokens", [128])
160
-@pytest.mark.parametrize("num_logprobs", [10])
161
-def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
162
- dtype: str, max_tokens: int, num_logprobs: int) -> None:
163
- images = [asset.pil_image for asset in image_assets]
164
-
165
- inputs_per_image = [(
166
- [prompt for _ in size_factors],
167
- [rescale_image_size(image, factor) for factor in size_factors],
168
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
169
-
170
- run_test(
171
- hf_runner,
172
- vllm_runner,
173
- inputs_per_image,
174
- model,
175
- dtype=dtype,
176
- max_tokens=max_tokens,
177
- num_logprobs=num_logprobs,
178
- mm_limit=1,
179
- tensor_parallel_size=1,
180
- )
181
-
182
-
183
-@pytest.mark.parametrize("model", models)
184
-@pytest.mark.parametrize("dtype", [target_dtype])
185
-def test_regression_7840(hf_runner, vllm_runner, image_assets, model,
186
- dtype) -> None:
187
- images = [asset.pil_image for asset in image_assets]
188
-
189
- inputs_regresion_7840 = [
190
- ([prompt], [image]) for image, prompt in zip(images, HF_IMAGE_PROMPTS)
191
- ]
192
-
193
- # Regression test for #7840.
194
- run_test(
195
- hf_runner,
196
- vllm_runner,
197
- inputs_regresion_7840,
198
- model,
199
- dtype=dtype,
200
- max_tokens=128,
201
- num_logprobs=10,
202
- mm_limit=1,
203
- tensor_parallel_size=1,
204
- )
205
-
206
-
207
-@pytest.mark.parametrize("model", models)
208
-@pytest.mark.parametrize(
209
- "size_factors",
210
- [
211
- # No image
212
- [],
213
- # Single-scale
214
- [1.0],
215
- # Single-scale, batched
216
- [1.0, 1.0, 1.0],
217
- # Multi-scale
218
- [0.25, 0.5, 1.0],
219
- ],
220
-)
221
-@pytest.mark.parametrize("dtype", [target_dtype])
222
-@pytest.mark.parametrize("max_tokens", [128])
223
-@pytest.mark.parametrize("num_logprobs", [10])
224
-def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
225
- size_factors, dtype: str, max_tokens: int,
226
- num_logprobs: int) -> None:
227
- images = [asset.pil_image for asset in image_assets]
228
-
229
- inputs_per_case = [
230
- ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
231
- [[rescale_image_size(image, factor) for image in images]
232
- for factor in size_factors])
233
- ]
234
-
235
- run_test(
236
- hf_runner,
237
- vllm_runner,
238
- inputs_per_case,
239
- model,
240
- dtype=dtype,
241
- max_tokens=max_tokens,
242
- num_logprobs=num_logprobs,
243
- mm_limit=2,
244
- tensor_parallel_size=1,
245
- )
tests/models/registry.py
+4
-3
@@ -326,7 +326,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
326
extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"}), # noqa: E501
327
"MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
328
max_transformers_version="4.48",
329
- transformers_version_reason="Use of private method which no longer exists.", # noqa: E501
329
+ transformers_version_reason="Incorrectly-detected `tensorflow` import.", # noqa: E501
330
extras={"olmo": "allenai/Molmo-7B-O-0924"}, # noqa: E501
331
trust_remote_code=True),
332
"NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B",
@@ -335,6 +335,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
335
extras={"v2": "google/paligemma2-3b-ft-docci-448"}), # noqa: E501
336
"Phi3VForCausalLM": _HfExamplesInfo("microsoft/Phi-3-vision-128k-instruct",
337
trust_remote_code=True,
338
+ max_transformers_version="4.48",
339
+ transformers_version_reason="Use of deprecated imports which have been removed.", # noqa: E501
340
extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"}), # noqa: E501
341
"Phi4MMForCausalLM": _HfExamplesInfo("microsoft/Phi-4-multimodal-instruct",
342
trust_remote_code=True),
@@ -351,8 +353,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
353
"SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"),
354
"SmolVLMForConditionalGeneration": _HfExamplesInfo("HuggingFaceTB/SmolVLM2-2.2B-Instruct"), # noqa: E501
355
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501
354
- trust_remote_code=True,
355
- max_transformers_version="4.50"),
356
+ trust_remote_code=True),
357
# [Encoder-decoder]
358
# Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
359
# Therefore, we borrow the BartTokenizer from the original Bart model