A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[CI/Build][VLM] Cleanup multiple images inputs model test (#7897)
Isotr0py committed
Aug 27, 2024 at 23:28 UTC
9db642138b54ef3df81873eac9fe7e15fc2da584
2 files changed
+74
-203
tests/models/test_minicpmv.py
+34
-102
@@ -1,14 +1,15 @@
1
-from typing import List, Optional, Tuple, Type
1
+from typing import List, Optional, Tuple, Type, Union
2
3
import pytest
4
import torch
5
import torch.types
6
+from PIL import Image
7
from transformers import BatchEncoding
8
9
from vllm.multimodal.utils import rescale_image_size
10
from vllm.sequence import SampleLogprobs
11
11
-from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
12
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner
13
from .utils import check_logprobs_close
14
15
pytestmark = pytest.mark.vlm
@@ -24,6 +25,11 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
25
"(<image>./</image>)\nWhat is the season?<|eot_id|>" \
26
"<|start_header_id|>assistant<|end_header_id|>\n\n",
27
})
28
+HF_MULTIIMAGE_IMAGE_PROMPT = \
29
+ "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
30
+ "(<image>./</image>)\n(<image>./</image>)\n" \
31
+ "Describe these images.<|eot_id|>" \
32
+ "<|start_header_id|>assistant<|end_header_id|>\n\n"
33
34
models = ["openbmb/MiniCPM-Llama3-V-2_5"]
35
@@ -46,13 +52,14 @@ target_dtype = "half"
52
def run_test(
53
hf_runner: Type[HfRunner],
54
vllm_runner: Type[VllmRunner],
49
- image_assets: _ImageAssets,
55
+ inputs: List[Tuple[List[str], Union[List[Image.Image],
56
+ List[List[Image.Image]]]]],
57
model: str,
58
*,
52
- size_factors: List[float],
59
dtype: str,
60
max_tokens: int,
61
num_logprobs: int,
62
+ mm_limit: int,
63
tensor_parallel_size: int,
64
distributed_executor_backend: Optional[str] = None,
65
):
@@ -65,12 +72,6 @@ def run_test(
72
Note, the text input is also adjusted to abide by vllm contract.
73
The text output is sanitized to be able to compare with hf.
74
"""
68
- images = [asset.pil_image for asset in image_assets]
69
-
70
- inputs_per_image = [(
71
- [prompt for _ in size_factors],
72
- [rescale_image_size(image, factor) for factor in size_factors],
73
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
75
76
# NOTE: take care of the order. run vLLM first, and then run HF.
77
# vLLM needs a fresh new process without cuda initialization.
@@ -82,6 +83,7 @@ def run_test(
83
max_model_len=4096,
84
max_num_seqs=1,
85
dtype=dtype,
86
+ limit_mm_per_prompt={"image": mm_limit},
87
tensor_parallel_size=tensor_parallel_size,
88
distributed_executor_backend=distributed_executor_backend,
89
enforce_eager=True) as vllm_model:
@@ -93,7 +95,7 @@ def run_test(
95
num_logprobs=num_logprobs,
96
images=images,
97
stop_token_ids=stop_token_ids)
96
- for prompts, images in inputs_per_image
98
+ for prompts, images in inputs
99
]
100
101
hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
@@ -104,7 +106,7 @@ def run_test(
106
num_logprobs=num_logprobs,
107
images=images,
108
tokenizer=tokenizer)
107
- for prompts, images in inputs_per_image
109
+ for prompts, images in inputs
110
]
111
112
for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -138,104 +140,26 @@ def run_test(
140
@pytest.mark.parametrize("num_logprobs", [5])
141
def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
142
dtype: str, max_tokens: int, num_logprobs: int) -> None:
143
+ images = [asset.pil_image for asset in image_assets]
144
+
145
+ inputs_per_image = [(
146
+ [prompt for _ in size_factors],
147
+ [rescale_image_size(image, factor) for factor in size_factors],
148
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
149
+
150
run_test(
151
hf_runner,
152
vllm_runner,
144
- image_assets,
153
+ inputs_per_image,
154
model,
146
- size_factors=size_factors,
155
dtype=dtype,
156
max_tokens=max_tokens,
157
num_logprobs=num_logprobs,
158
+ mm_limit=1,
159
tensor_parallel_size=1,
160
)
161
162
154
-HF_MULTIIMAGE_IMAGE_PROMPT = \
155
- "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
156
- "(<image>./</image>)\n(<image>./</image>)\n" \
157
- "Describe these images.<|eot_id|>" \
158
- "<|start_header_id|>assistant<|end_header_id|>\n\n"
159
-
160
-
161
-def run_multi_image_test(
162
- hf_runner: Type[HfRunner],
163
- vllm_runner: Type[VllmRunner],
164
- image_assets: _ImageAssets,
165
- model: str,
166
- *,
167
- size_factors: List[float],
168
- dtype: str,
169
- max_tokens: int,
170
- num_logprobs: int,
171
- tensor_parallel_size: int,
172
- distributed_executor_backend: Optional[str] = None,
173
-):
174
- """Inference result should be the same between hf and vllm.
175
-
176
- All the image fixtures for the test is under tests/images.
177
- For huggingface runner, we provide the PIL images as input.
178
- For vllm runner, we provide MultiModalDataDict objects
179
- and corresponding MultiModalConfig as input.
180
- Note, the text input is also adjusted to abide by vllm contract.
181
- The text output is sanitized to be able to compare with hf.
182
- """
183
- images = [asset.pil_image for asset in image_assets]
184
-
185
- inputs_per_case = [
186
- ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
187
- [[rescale_image_size(image, factor) for image in images]
188
- for factor in size_factors])
189
- ]
190
-
191
- # NOTE: take care of the order. run vLLM first, and then run HF.
192
- # vLLM needs a fresh new process without cuda initialization.
193
- # if we run HF first, the cuda initialization will be done and it
194
- # will hurt multiprocessing backend with fork method (the default method).
195
-
196
- # max_model_len should be greater than image_feature_size
197
- with vllm_runner(model,
198
- max_model_len=4096,
199
- max_num_seqs=1,
200
- limit_mm_per_prompt={"image": len(images)},
201
- dtype=dtype,
202
- tensor_parallel_size=tensor_parallel_size,
203
- distributed_executor_backend=distributed_executor_backend,
204
- enforce_eager=True) as vllm_model:
205
- tokenizer = vllm_model.model.get_tokenizer()
206
- stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
207
- vllm_outputs_per_case = [
208
- vllm_model.generate_greedy_logprobs(prompts,
209
- max_tokens,
210
- num_logprobs=num_logprobs,
211
- images=images,
212
- stop_token_ids=stop_token_ids)
213
- for prompts, images in inputs_per_case
214
- ]
215
-
216
- hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
217
- with hf_model, torch.no_grad():
218
- hf_outputs_per_case = [
219
- hf_model.generate_greedy_logprobs_limit(prompts,
220
- max_tokens,
221
- num_logprobs=num_logprobs,
222
- images=images,
223
- tokenizer=tokenizer)
224
- for prompts, images in inputs_per_case
225
- ]
226
-
227
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
228
- vllm_outputs_per_case):
229
- check_logprobs_close(
230
- outputs_0_lst=[
231
- trunc_hf_output(hf_output) for hf_output in hf_outputs
232
- ],
233
- outputs_1_lst=vllm_outputs,
234
- name_0="hf",
235
- name_1="vllm",
236
- )
237
-
238
-
163
@pytest.mark.parametrize("model", models)
164
@pytest.mark.parametrize(
165
"size_factors",
@@ -256,14 +180,22 @@ def run_multi_image_test(
180
def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
181
size_factors, dtype: str, max_tokens: int,
182
num_logprobs: int) -> None:
259
- run_multi_image_test(
183
+ images = [asset.pil_image for asset in image_assets]
184
+
185
+ inputs_per_case = [
186
+ ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
187
+ [[rescale_image_size(image, factor) for image in images]
188
+ for factor in size_factors])
189
+ ]
190
+
191
+ run_test(
192
hf_runner,
193
vllm_runner,
262
- image_assets,
194
+ inputs_per_case,
195
model,
264
- size_factors=size_factors,
196
dtype=dtype,
197
max_tokens=max_tokens,
198
num_logprobs=num_logprobs,
199
+ mm_limit=2,
200
tensor_parallel_size=1,
201
)
tests/models/test_phi3v.py
+40
-101
@@ -1,6 +1,6 @@
1
import os
2
import re
3
-from typing import List, Optional, Tuple, Type
3
+from typing import List, Optional, Tuple, Type, Union
4
5
import pytest
6
from PIL import Image
@@ -60,13 +60,14 @@ if is_hip():
60
def run_test(
61
hf_runner: Type[HfRunner],
62
vllm_runner: Type[VllmRunner],
63
- images: List[Image.Image],
63
+ inputs: List[Tuple[List[str], Union[List[Image.Image],
64
+ List[List[Image.Image]]]]],
65
model: str,
66
*,
66
- size_factors: List[float],
67
dtype: str,
68
max_tokens: int,
69
num_logprobs: int,
70
+ mm_limit: int,
71
tensor_parallel_size: int,
72
distributed_executor_backend: Optional[str] = None,
73
):
@@ -79,13 +80,6 @@ def run_test(
80
Note, the text input is also adjusted to abide by vllm contract.
81
The text output is sanitized to be able to compare with hf.
82
"""
82
- inputs_per_image = [(
83
- [prompt for _ in size_factors],
84
- [
85
- rescale_image_size(image, factor, transpose=idx)
86
- for idx, factor in enumerate(size_factors)
87
- ],
88
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
83
84
# NOTE: take care of the order. run vLLM first, and then run HF.
85
# vLLM needs a fresh new process without cuda initialization.
@@ -97,15 +91,16 @@ def run_test(
91
max_model_len=4096,
92
max_num_seqs=1,
93
dtype=dtype,
94
+ limit_mm_per_prompt={"image": mm_limit},
95
tensor_parallel_size=tensor_parallel_size,
96
distributed_executor_backend=distributed_executor_backend,
97
enforce_eager=True) as vllm_model:
103
- vllm_outputs_per_image = [
98
+ vllm_outputs_per_case = [
99
vllm_model.generate_greedy_logprobs(prompts,
100
max_tokens,
101
num_logprobs=num_logprobs,
102
images=images)
108
- for prompts, images in inputs_per_image
103
+ for prompts, images in inputs
104
]
105
106
# use eager mode for hf runner, since phi3_v didn't work with flash_attn
@@ -113,17 +108,17 @@ def run_test(
108
with hf_runner(model, dtype=dtype,
109
model_kwargs=hf_model_kwargs) as hf_model:
110
eos_token_id = hf_model.processor.tokenizer.eos_token_id
116
- hf_outputs_per_image = [
111
+ hf_outputs_per_case = [
112
hf_model.generate_greedy_logprobs_limit(prompts,
113
max_tokens,
114
num_logprobs=num_logprobs,
115
images=images,
116
eos_token_id=eos_token_id)
122
- for prompts, images in inputs_per_image
117
+ for prompts, images in inputs
118
]
119
125
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
126
- vllm_outputs_per_image):
120
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
121
+ vllm_outputs_per_case):
122
check_logprobs_close(
123
outputs_0_lst=hf_outputs,
124
outputs_1_lst=[
@@ -156,15 +151,22 @@ def run_test(
151
@pytest.mark.parametrize("num_logprobs", [10])
152
def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
153
dtype: str, max_tokens: int, num_logprobs: int) -> None:
154
+ images = [asset.pil_image for asset in image_assets]
155
+
156
+ inputs_per_image = [(
157
+ [prompt for _ in size_factors],
158
+ [rescale_image_size(image, factor) for factor in size_factors],
159
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
160
+
161
run_test(
162
hf_runner,
163
vllm_runner,
162
- [asset.pil_image for asset in image_assets],
164
+ inputs_per_image,
165
model,
164
- size_factors=size_factors,
166
dtype=dtype,
167
max_tokens=max_tokens,
168
num_logprobs=num_logprobs,
169
+ mm_limit=1,
170
tensor_parallel_size=1,
171
)
172
@@ -173,97 +175,26 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
175
@pytest.mark.parametrize("dtype", [target_dtype])
176
def test_regression_7840(hf_runner, vllm_runner, image_assets, model,
177
dtype) -> None:
178
+ images = [asset.pil_image for asset in image_assets]
179
+
180
+ inputs_regresion_7840 = [
181
+ ([prompt], [image]) for image, prompt in zip(images, HF_IMAGE_PROMPTS)
182
+ ]
183
+
184
# Regression test for #7840.
185
run_test(
186
hf_runner,
187
vllm_runner,
180
- [image_assets[0].pil_image.resize((465, 226))],
188
+ inputs_regresion_7840,
189
model,
182
- size_factors=[1.0],
190
dtype=dtype,
191
max_tokens=128,
192
num_logprobs=10,
193
+ mm_limit=1,
194
tensor_parallel_size=1,
195
)
196
197
190
-def run_multi_image_test(
191
- hf_runner: Type[HfRunner],
192
- vllm_runner: Type[VllmRunner],
193
- images: List[Image.Image],
194
- model: str,
195
- *,
196
- size_factors: List[float],
197
- dtype: str,
198
- max_tokens: int,
199
- num_logprobs: int,
200
- tensor_parallel_size: int,
201
- distributed_executor_backend: Optional[str] = None,
202
-):
203
- """Inference result should be the same between hf and vllm.
204
-
205
- All the image fixtures for the test is under tests/images.
206
- For huggingface runner, we provide the PIL images as input.
207
- For vllm runner, we provide MultiModalDataDict objects
208
- and corresponding MultiModalConfig as input.
209
- Note, the text input is also adjusted to abide by vllm contract.
210
- The text output is sanitized to be able to compare with hf.
211
- """
212
-
213
- inputs_per_case = [
214
- ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
215
- [[rescale_image_size(image, factor) for image in images]
216
- for factor in size_factors])
217
- ]
218
-
219
- # NOTE: take care of the order. run vLLM first, and then run HF.
220
- # vLLM needs a fresh new process without cuda initialization.
221
- # if we run HF first, the cuda initialization will be done and it
222
- # will hurt multiprocessing backend with fork method (the default method).
223
-
224
- # max_model_len should be greater than image_feature_size
225
- with vllm_runner(model,
226
- max_model_len=4096,
227
- max_num_seqs=1,
228
- limit_mm_per_prompt={"image": len(images)},
229
- dtype=dtype,
230
- tensor_parallel_size=tensor_parallel_size,
231
- distributed_executor_backend=distributed_executor_backend,
232
- enforce_eager=True) as vllm_model:
233
- vllm_outputs_per_case = [
234
- vllm_model.generate_greedy_logprobs(prompts,
235
- max_tokens,
236
- num_logprobs=num_logprobs,
237
- images=images)
238
- for prompts, images in inputs_per_case
239
- ]
240
-
241
- hf_model_kwargs = {"_attn_implementation": "eager"}
242
- with hf_runner(model, dtype=dtype,
243
- model_kwargs=hf_model_kwargs) as hf_model:
244
- eos_token_id = hf_model.processor.tokenizer.eos_token_id
245
- hf_outputs_per_case = [
246
- hf_model.generate_greedy_logprobs_limit(prompts,
247
- max_tokens,
248
- num_logprobs=num_logprobs,
249
- images=images,
250
- eos_token_id=eos_token_id)
251
- for prompts, images in inputs_per_case
252
- ]
253
-
254
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
255
- vllm_outputs_per_case):
256
- check_logprobs_close(
257
- outputs_0_lst=hf_outputs,
258
- outputs_1_lst=[
259
- vllm_to_hf_output(vllm_output, model)
260
- for vllm_output in vllm_outputs
261
- ],
262
- name_0="hf",
263
- name_1="vllm",
264
- )
265
-
266
-
198
@pytest.mark.parametrize("model", models)
199
@pytest.mark.parametrize(
200
"size_factors",
@@ -280,18 +211,26 @@ def run_multi_image_test(
211
)
212
@pytest.mark.parametrize("dtype", [target_dtype])
213
@pytest.mark.parametrize("max_tokens", [128])
283
-@pytest.mark.parametrize("num_logprobs", [5])
214
+@pytest.mark.parametrize("num_logprobs", [10])
215
def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
216
size_factors, dtype: str, max_tokens: int,
217
num_logprobs: int) -> None:
287
- run_multi_image_test(
218
+ images = [asset.pil_image for asset in image_assets]
219
+
220
+ inputs_per_case = [
221
+ ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
222
+ [[rescale_image_size(image, factor) for image in images]
223
+ for factor in size_factors])
224
+ ]
225
+
226
+ run_test(
227
hf_runner,
228
vllm_runner,
290
- [asset.pil_image for asset in image_assets],
229
+ inputs_per_case,
230
model,
292
- size_factors=size_factors,
231
dtype=dtype,
232
max_tokens=max_tokens,
233
num_logprobs=num_logprobs,
234
+ mm_limit=2,
235
tensor_parallel_size=1,
236
)