A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[CI/Build] Use `AutoModelForImageTextToText` to load VLMs in tests (#14945)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 18, 2025 at 02:35 UTC
b89fb2a4a12458a1d2eca78a8e6cc4059f67e715
3 files changed
+19
-19
tests/models/decoder_only/vision_language/test_models.py
+14
-14
@@ -9,7 +9,7 @@ from pathlib import PosixPath
9
10
import pytest
11
from packaging.version import Version
12
-from transformers import AutoModelForPreTraining, AutoModelForVision2Seq
12
+from transformers import AutoModelForImageTextToText, AutoModelForVision2Seq
13
from transformers import __version__ as TRANSFORMERS_VERSION
14
15
from vllm.platforms import current_platform
@@ -101,7 +101,7 @@ VLM_TEST_SETTINGS = {
101
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
102
convert_assets_to_embeddings=model_utils.get_llava_embeddings,
103
max_model_len=4096,
104
- auto_cls=AutoModelForVision2Seq,
104
+ auto_cls=AutoModelForImageTextToText,
105
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
106
custom_test_opts=[CustomTestOptions(
107
inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
@@ -121,7 +121,7 @@ VLM_TEST_SETTINGS = {
121
"stop_sign": "caption es",
122
"cherry_blossom": "What is in the picture?",
123
}),
124
- auto_cls=AutoModelForVision2Seq,
124
+ auto_cls=AutoModelForImageTextToText,
125
postprocess_inputs=model_utils.cast_dtype_post_processor(
126
"pixel_values"
127
),
@@ -190,7 +190,7 @@ VLM_TEST_SETTINGS = {
190
test_type=VLMTestType.IMAGE,
191
prompt_formatter=lambda img_prompt: f"Question: {img_prompt} Answer:",
192
img_idx_to_prompt=lambda idx: "",
193
- auto_cls=AutoModelForVision2Seq,
193
+ auto_cls=AutoModelForImageTextToText,
194
vllm_output_post_proc=model_utils.blip2_vllm_to_hf_output,
195
),
196
"chameleon": VLMTestInfo(
@@ -199,7 +199,7 @@ VLM_TEST_SETTINGS = {
199
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
200
max_model_len=4096,
201
max_num_seqs=2,
202
- auto_cls=AutoModelForVision2Seq,
202
+ auto_cls=AutoModelForImageTextToText,
203
postprocess_inputs=model_utils.cast_dtype_post_processor(
204
"pixel_values"
205
),
@@ -240,6 +240,7 @@ VLM_TEST_SETTINGS = {
240
img_idx_to_prompt=lambda idx: "",
241
max_model_len=2048,
242
max_num_seqs=2,
243
+ auto_cls=AutoModelForImageTextToText,
244
use_tokenizer_eos=True,
245
vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
246
num_logprobs=10,
@@ -256,8 +257,7 @@ VLM_TEST_SETTINGS = {
257
multi_image_prompt="<start_of_image><start_of_image>Describe the two images in detail.", # noqa: E501
258
max_model_len=4096,
259
max_num_seqs=2,
259
- # TODO: Use AutoModelForVision2Seq once transformers supports this
260
- auto_cls=AutoModelForPreTraining,
260
+ auto_cls=AutoModelForImageTextToText,
261
dtype="bfloat16",
262
vllm_runner_kwargs={"mm_processor_kwargs": {"do_pan_and_scan": True}},
263
patch_hf_runner=model_utils.gemma3_patch_hf_runner,
@@ -307,7 +307,7 @@ VLM_TEST_SETTINGS = {
307
img_idx_to_prompt=lambda idx: "<image>",
308
max_model_len=8192,
309
max_num_seqs=2,
310
- auto_cls=AutoModelForVision2Seq,
310
+ auto_cls=AutoModelForImageTextToText,
311
hf_output_post_proc=model_utils.idefics3_trunc_hf_output,
312
),
313
"intern_vl": VLMTestInfo(
@@ -336,7 +336,7 @@ VLM_TEST_SETTINGS = {
336
test_type=(VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
337
prompt_formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]",
338
max_model_len=10240,
339
- auto_cls=AutoModelForVision2Seq,
339
+ auto_cls=AutoModelForImageTextToText,
340
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
341
custom_test_opts=[CustomTestOptions(
342
inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
@@ -382,7 +382,7 @@ VLM_TEST_SETTINGS = {
382
"pixel_values"
383
),
384
get_stop_token_ids=lambda tok: [128009],
385
- auto_cls=AutoModelForVision2Seq,
385
+ auto_cls=AutoModelForImageTextToText,
386
vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
387
patch_hf_runner=model_utils.mantis_patch_hf_runner,
388
marks=[
@@ -463,7 +463,7 @@ VLM_TEST_SETTINGS = {
463
img_idx_to_prompt=lambda idx: "[IMG]",
464
max_model_len=8192,
465
max_num_seqs=2,
466
- auto_cls=AutoModelForVision2Seq,
466
+ auto_cls=AutoModelForImageTextToText,
467
marks=[large_gpu_mark(min_gb=48)],
468
),
469
"qwen_vl": VLMTestInfo(
@@ -481,7 +481,7 @@ VLM_TEST_SETTINGS = {
481
models=["facebook/chameleon-7b"],
482
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
483
max_model_len=4096,
484
- auto_cls=AutoModelForVision2Seq,
484
+ auto_cls=AutoModelForImageTextToText,
485
postprocess_inputs=model_utils.cast_dtype_post_processor(
486
"pixel_values"
487
),
@@ -495,7 +495,7 @@ VLM_TEST_SETTINGS = {
495
models=["llava-hf/llava-1.5-7b-hf"],
496
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
497
max_model_len=4096,
498
- auto_cls=AutoModelForVision2Seq,
498
+ auto_cls=AutoModelForImageTextToText,
499
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
500
marks=multi_gpu_marks(num_gpus=2),
501
**COMMON_BROADCAST_SETTINGS # type: ignore
@@ -504,7 +504,7 @@ VLM_TEST_SETTINGS = {
504
models=["llava-hf/llava-v1.6-mistral-7b-hf"],
505
prompt_formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]",
506
max_model_len=10240,
507
- auto_cls=AutoModelForVision2Seq,
507
+ auto_cls=AutoModelForImageTextToText,
508
vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
509
marks=multi_gpu_marks(num_gpus=2),
510
**COMMON_BROADCAST_SETTINGS # type: ignore
tests/models/embedding/vision_language/test_llava_next.py
+2
-2
@@ -2,7 +2,7 @@
2
3
import pytest
4
import torch.nn.functional as F
5
-from transformers import AutoModelForVision2Seq
5
+from transformers import AutoModelForImageTextToText
6
7
from vllm.platforms import current_platform
8
@@ -70,7 +70,7 @@ def _run_test(
70
vllm_outputs = vllm_model.encode(input_texts, images=input_images)
71
72
with hf_runner(model, dtype=dtype,
73
- auto_cls=AutoModelForVision2Seq) as hf_model:
73
+ auto_cls=AutoModelForImageTextToText) as hf_model:
74
# Patch the issue where generation_config.json is missing
75
hf_model.processor.patch_size = \
76
hf_model.model.config.vision_config.patch_size
tests/models/encoder_decoder/vision_language/test_mllama.py
+3
-3
@@ -4,8 +4,8 @@ from typing import Optional, overload
4
5
import pytest
6
import torch
7
-from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
8
- BatchEncoding)
7
+from transformers import (AutoConfig, AutoModelForImageTextToText,
8
+ AutoTokenizer, BatchEncoding)
9
10
from vllm import LLM, SamplingParams
11
from vllm.attention.backends.flash_attn import FlashAttentionMetadata
@@ -234,7 +234,7 @@ def _run_test(
234
dtype=dtype,
235
model_kwargs={"device_map": "auto"},
236
postprocess_inputs=process,
237
- auto_cls=AutoModelForVision2Seq) as hf_model:
237
+ auto_cls=AutoModelForImageTextToText) as hf_model:
238
hf_outputs_per_image = [
239
hf_model.generate_greedy_logprobs_limit(prompts,
240
max_tokens,