A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[CI/Build] Add Model Tests for Qwen2-VL (#9846)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Alex Brooks committed
Oct 31, 2024 at 10:10 UTC
16b8f7a86f5a93d2b0dc4bd20709a47d34918b8f
9 files changed
+106
-52
.buildkite/test-pipeline.yaml
+14
-3
@@ -9,6 +9,7 @@
9
# label(str): the name of the test. emoji allowed.
10
# fast_check(bool): whether to run this on each commit on fastcheck pipeline.
11
# fast_check_only(bool): run this test on fastcheck pipeline only
12
+# nightly(bool): run this test in nightly pipeline only
13
# optional(bool): never run this test by default (i.e. need to unblock manually)
14
# command(str): the single command to run for tests. incompatible with commands.
15
# commands(list): the list of commands to run for test. incompatbile with command.
@@ -330,18 +331,28 @@ steps:
331
commands:
332
- pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py --ignore=models/decoder_only/language/test_big_models.py
333
333
-- label: Decoder-only Multi-Modal Models Test # 1h31min
334
+- label: Decoder-only Multi-Modal Models Test (Standard)
335
#mirror_hardwares: [amd]
336
source_file_dependencies:
337
- vllm/
338
- tests/models/decoder_only/audio_language
339
- tests/models/decoder_only/vision_language
340
commands:
340
- - pytest -v -s models/decoder_only/audio_language
341
+ - pytest -v -s models/decoder_only/audio_language -m core_model
342
+ - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m core_model
343
+
344
+- label: Decoder-only Multi-Modal Models Test (Extended)
345
+ nightly: true
346
+ source_file_dependencies:
347
+ - vllm/
348
+ - tests/models/decoder_only/audio_language
349
+ - tests/models/decoder_only/vision_language
350
+ commands:
351
+ - pytest -v -s models/decoder_only/audio_language -m 'not core_model'
352
# HACK - run phi3v tests separately to sidestep this transformers bug
353
# https://github.com/huggingface/transformers/issues/34307
354
- pytest -v -s models/decoder_only/vision_language/test_phi3v.py
344
- - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language
355
+ - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'not core_model'
356
357
- label: Other Models Test # 6min
358
#mirror_hardwares: [amd]
examples/offline_inference_vision_language.py
+1
-2
@@ -262,10 +262,9 @@ def run_qwen2_vl(question: str, modality: str):
262
263
model_name = "Qwen/Qwen2-VL-7B-Instruct"
264
265
- # Tested on L40
265
llm = LLM(
266
model=model_name,
268
- max_model_len=8192,
267
+ max_model_len=4096,
268
max_num_seqs=5,
269
# Note - mm_processor_kwargs can also be passed to generate/chat calls
270
mm_processor_kwargs={
tests/models/decoder_only/audio_language/test_ultravox.py
+2
@@ -158,6 +158,7 @@ def run_multi_audio_test(
158
assert all(tokens for tokens, *_ in vllm_outputs)
159
160
161
+@pytest.mark.core_model
162
@pytest.mark.parametrize("dtype", ["half"])
163
@pytest.mark.parametrize("max_tokens", [128])
164
@pytest.mark.parametrize("num_logprobs", [5])
@@ -178,6 +179,7 @@ def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
179
)
180
181
182
+@pytest.mark.core_model
183
@pytest.mark.parametrize("dtype", ["half"])
184
@pytest.mark.parametrize("max_tokens", [128])
185
@pytest.mark.parametrize("num_logprobs", [5])
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+1
-1
@@ -17,7 +17,7 @@ MAX_PIXELS = "max_pixels"
17
18
19
# Fixtures lazy import to avoid initializing CUDA during test collection
20
-# NOTE: Qwen2vl supports multiple input modalities, so it registers multiple
20
+# NOTE: Qwen2VL supports multiple input modalities, so it registers multiple
21
# input mappers.
22
@pytest.fixture()
23
def image_input_mapper_for_qwen2_vl():
tests/models/decoder_only/vision_language/test_models.py
+62
-39
@@ -75,6 +75,63 @@ COMMON_BROADCAST_SETTINGS = {
75
# this is a good idea for checking your command first, since tests are slow.
76
77
VLM_TEST_SETTINGS = {
78
+ #### Core tests to always run in the CI
79
+ "llava": VLMTestInfo(
80
+ models=["llava-hf/llava-1.5-7b-hf"],
81
+ test_type=(
82
+ VLMTestType.EMBEDDING,
83
+ VLMTestType.IMAGE,
84
+ VLMTestType.CUSTOM_INPUTS
85
+ ),
86
+ prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
87
+ convert_assets_to_embeddings=model_utils.get_llava_embeddings,
88
+ max_model_len=4096,
89
+ auto_cls=AutoModelForVision2Seq,
90
+ vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
91
+ custom_test_opts=[CustomTestOptions(
92
+ inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
93
+ formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
94
+ ),
95
+ limit_mm_per_prompt={"image": 4},
96
+ )],
97
+ marks=[pytest.mark.core_model],
98
+ ),
99
+ "paligemma": VLMTestInfo(
100
+ models=["google/paligemma-3b-mix-224"],
101
+ test_type=VLMTestType.IMAGE,
102
+ prompt_formatter=identity,
103
+ img_idx_to_prompt = lambda idx: "",
104
+ # Paligemma uses its own sample prompts because the default one fails
105
+ single_image_prompts=IMAGE_ASSETS.prompts({
106
+ "stop_sign": "caption es",
107
+ "cherry_blossom": "What is in the picture?",
108
+ }),
109
+ auto_cls=AutoModelForVision2Seq,
110
+ postprocess_inputs=model_utils.get_key_type_post_processor(
111
+ "pixel_values"
112
+ ),
113
+ vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
114
+ dtype="half" if current_platform.is_rocm() else ("half", "float"),
115
+ marks=[pytest.mark.core_model],
116
+ ),
117
+ "qwen2_vl": VLMTestInfo(
118
+ models=["Qwen/Qwen2-VL-2B-Instruct"],
119
+ test_type=(
120
+ VLMTestType.IMAGE,
121
+ VLMTestType.MULTI_IMAGE,
122
+ VLMTestType.VIDEO
123
+ ),
124
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
125
+ img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
126
+ video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
127
+ max_model_len=4096,
128
+ max_num_seqs=2,
129
+ auto_cls=AutoModelForVision2Seq,
130
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
131
+ marks=[pytest.mark.core_model],
132
+ image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
133
+ ),
134
+ #### Extended model tests
135
"blip2": VLMTestInfo(
136
models=["Salesforce/blip2-opt-2.7b"],
137
test_type=VLMTestType.IMAGE,
@@ -151,25 +208,6 @@ VLM_TEST_SETTINGS = {
208
use_tokenizer_eos=True,
209
patch_hf_runner=model_utils.internvl_patch_hf_runner,
210
),
154
- "llava": VLMTestInfo(
155
- models=["llava-hf/llava-1.5-7b-hf"],
156
- test_type=(
157
- VLMTestType.EMBEDDING,
158
- VLMTestType.IMAGE,
159
- VLMTestType.CUSTOM_INPUTS
160
- ),
161
- prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
162
- convert_assets_to_embeddings=model_utils.get_llava_embeddings,
163
- max_model_len=4096,
164
- auto_cls=AutoModelForVision2Seq,
165
- vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
166
- custom_test_opts=[CustomTestOptions(
167
- inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
168
- formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
169
- ),
170
- limit_mm_per_prompt={"image": 4},
171
- )],
172
- ),
211
"llava_next": VLMTestInfo(
212
models=["llava-hf/llava-v1.6-mistral-7b-hf"],
213
test_type=(VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
@@ -200,12 +238,12 @@ VLM_TEST_SETTINGS = {
238
vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
239
# Llava-one-vision tests fixed sizes & the default size factors
240
image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
203
- runner_mm_key="videos",
241
custom_test_opts=[CustomTestOptions(
242
inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
243
formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
244
),
245
limit_mm_per_prompt={"video": 4},
246
+ runner_mm_key="videos",
247
)],
248
),
249
# FIXME
@@ -218,9 +256,11 @@ VLM_TEST_SETTINGS = {
256
auto_cls=AutoModelForVision2Seq,
257
vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
258
image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
221
- runner_mm_key="videos",
259
marks=[
223
- pytest.mark.skip(reason="LLava next video tests currently fail.")
260
+ pytest.mark.skipif(
261
+ transformers.__version__.startswith("4.46"),
262
+ reason="Model broken with changes in transformers 4.46"
263
+ )
264
],
265
),
266
"minicpmv": VLMTestInfo(
@@ -234,23 +274,6 @@ VLM_TEST_SETTINGS = {
274
postprocess_inputs=model_utils.wrap_inputs_post_processor,
275
hf_output_post_proc=model_utils.minicmpv_trunc_hf_output,
276
),
237
- "paligemma": VLMTestInfo(
238
- models=["google/paligemma-3b-mix-224"],
239
- test_type=VLMTestType.IMAGE,
240
- prompt_formatter=identity,
241
- img_idx_to_prompt = lambda idx: "",
242
- # Paligemma uses its own sample prompts because the default one fails
243
- single_image_prompts=IMAGE_ASSETS.prompts({
244
- "stop_sign": "caption es",
245
- "cherry_blossom": "What is in the picture?",
246
- }),
247
- auto_cls=AutoModelForVision2Seq,
248
- postprocess_inputs=model_utils.get_key_type_post_processor(
249
- "pixel_values"
250
- ),
251
- vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
252
- dtype="half" if current_platform.is_rocm() else ("half", "float"),
253
- ),
277
# Tests for phi3v currently live in another file because of a bug in
278
# transformers. Once this issue is fixed, we can enable them here instead.
279
# https://github.com/huggingface/transformers/issues/34307
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+11
@@ -56,6 +56,17 @@ def qwen_vllm_to_hf_output(
56
return output_ids, hf_output_str, out_logprobs
57
58
59
+def qwen2_vllm_to_hf_output(
60
+ vllm_output: RunnerOutput,
61
+ model: str) -> Tuple[List[int], str, Optional[SampleLogprobs]]:
62
+ """Sanitize vllm output [qwen2 models] to be comparable with hf output."""
63
+ output_ids, output_str, out_logprobs = vllm_output
64
+
65
+ hf_output_str = output_str + "<|im_end|>"
66
+
67
+ return output_ids, hf_output_str, out_logprobs
68
+
69
+
70
def llava_image_vllm_to_hf_output(vllm_output: RunnerOutput,
71
model: str) -> RunnerOutput:
72
config = AutoConfig.from_pretrained(model)
tests/models/decoder_only/vision_language/vlm_utils/runners.py
+10
-1
@@ -29,6 +29,7 @@ def run_single_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,
29
num_logprobs=test_case.num_logprobs,
30
limit_mm_per_prompt={"image": 1},
31
distributed_executor_backend=test_case.distributed_executor_backend,
32
+ runner_mm_key="images",
33
**model_test_info.get_non_parametrized_runner_kwargs())
34
35
@@ -51,6 +52,7 @@ def run_multi_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,
52
num_logprobs=test_case.num_logprobs,
53
limit_mm_per_prompt={"image": len(image_assets)},
54
distributed_executor_backend=test_case.distributed_executor_backend,
55
+ runner_mm_key="images",
56
**model_test_info.get_non_parametrized_runner_kwargs())
57
58
@@ -74,6 +76,7 @@ def run_embedding_test(*, model_test_info: VLMTestInfo,
76
limit_mm_per_prompt={"image": 1},
77
vllm_embeddings=vllm_embeddings,
78
distributed_executor_backend=test_case.distributed_executor_backend,
79
+ runner_mm_key="images",
80
**model_test_info.get_non_parametrized_runner_kwargs())
81
82
@@ -101,6 +104,7 @@ def run_video_test(
104
num_logprobs=test_case.num_logprobs,
105
limit_mm_per_prompt={"video": len(video_assets)},
106
distributed_executor_backend=test_case.distributed_executor_backend,
107
+ runner_mm_key="videos",
108
**model_test_info.get_non_parametrized_runner_kwargs())
109
110
@@ -115,7 +119,11 @@ def run_custom_inputs_test(*, model_test_info: VLMTestInfo,
119
120
inputs = test_case.custom_test_opts.inputs
121
limit_mm_per_prompt = test_case.custom_test_opts.limit_mm_per_prompt
118
- assert inputs is not None and limit_mm_per_prompt is not None
122
+ runner_mm_key = test_case.custom_test_opts.runner_mm_key
123
+ # Inputs, limit_mm_per_prompt, and runner_mm_key should all be set
124
+ assert inputs is not None
125
+ assert limit_mm_per_prompt is not None
126
+ assert runner_mm_key is not None
127
128
core.run_test(
129
hf_runner=hf_runner,
@@ -127,4 +135,5 @@ def run_custom_inputs_test(*, model_test_info: VLMTestInfo,
135
num_logprobs=test_case.num_logprobs,
136
limit_mm_per_prompt=limit_mm_per_prompt,
137
distributed_executor_backend=test_case.distributed_executor_backend,
138
+ runner_mm_key=runner_mm_key,
139
**model_test_info.get_non_parametrized_runner_kwargs())
tests/models/decoder_only/vision_language/vlm_utils/types.py
+2
-4
@@ -52,6 +52,8 @@ class SizeType(Enum):
52
class CustomTestOptions(NamedTuple):
53
inputs: List[Tuple[List[str], List[Union[List[Image], Image]]]]
54
limit_mm_per_prompt: Dict[str, int]
55
+ # kwarg to pass multimodal data in as to vllm/hf runner instances.
56
+ runner_mm_key: str = "images"
57
58
59
class ImageSizeWrapper(NamedTuple):
@@ -141,9 +143,6 @@ class VLMTestInfo(NamedTuple):
143
Callable[[PosixPath, str, Union[List[ImageAsset], _ImageAssets]],
144
str]] = None # noqa: E501
145
144
- # kwarg to pass multimodal data in as to vllm/hf runner instances
145
- runner_mm_key: str = "images"
146
-
146
# Allows configuring a test to run with custom inputs
147
custom_test_opts: Optional[List[CustomTestOptions]] = None
148
@@ -168,7 +167,6 @@ class VLMTestInfo(NamedTuple):
167
"get_stop_token_ids": self.get_stop_token_ids,
168
"model_kwargs": self.model_kwargs,
169
"patch_hf_runner": self.patch_hf_runner,
171
- "runner_mm_key": self.runner_mm_key,
170
}
171
172
tests/models/embedding/vision_language/test_llava_next.py
+3
-2
@@ -2,6 +2,7 @@ from typing import List, Type
2
3
import pytest
4
import torch.nn.functional as F
5
+import transformers
6
from transformers import AutoModelForVision2Seq
7
8
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
@@ -85,8 +86,8 @@ def _run_test(
86
)
87
88
88
-# FIXME
89
-@pytest.mark.skip(reason="LLava next embedding tests currently fail")
89
+@pytest.mark.skipif(transformers.__version__.startswith("4.46"),
90
+ reason="Model broken with changes in transformers 4.46")
91
@pytest.mark.parametrize("model", MODELS)
92
@pytest.mark.parametrize("dtype", ["half"])
93
def test_models_text(