@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9db642138b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[CI/Build][VLM] Cleanup multiple images inputs model test (#7897)

Isotr0py committed Aug 27, 2024 at 23:28 UTC 9db642138b54ef3df81873eac9fe7e15fc2da584
2 files changed +74 -203
tests/models/test_minicpmv.py
+34 -102
@@ -1,14 +1,15 @@
1 -from typing import List, Optional, Tuple, Type
1 +from typing import List, Optional, Tuple, Type, Union
2
3 import pytest
4 import torch
5 import torch.types
6 +from PIL import Image
7 from transformers import BatchEncoding
8
9 from vllm.multimodal.utils import rescale_image_size
10 from vllm.sequence import SampleLogprobs
11
11 -from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
12 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner
13 from .utils import check_logprobs_close
14
15 pytestmark = pytest.mark.vlm
@@ -24,6 +25,11 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
25 "(<image>./</image>)\nWhat is the season?<|eot_id|>" \
26 "<|start_header_id|>assistant<|end_header_id|>\n\n",
27 })
28 +HF_MULTIIMAGE_IMAGE_PROMPT = \
29 + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
30 + "(<image>./</image>)\n(<image>./</image>)\n" \
31 + "Describe these images.<|eot_id|>" \
32 + "<|start_header_id|>assistant<|end_header_id|>\n\n"
33
34 models = ["openbmb/MiniCPM-Llama3-V-2_5"]
35
@@ -46,13 +52,14 @@ target_dtype = "half"
52 def run_test(
53 hf_runner: Type[HfRunner],
54 vllm_runner: Type[VllmRunner],
49 - image_assets: _ImageAssets,
55 + inputs: List[Tuple[List[str], Union[List[Image.Image],
56 + List[List[Image.Image]]]]],
57 model: str,
58 *,
52 - size_factors: List[float],
59 dtype: str,
60 max_tokens: int,
61 num_logprobs: int,
62 + mm_limit: int,
63 tensor_parallel_size: int,
64 distributed_executor_backend: Optional[str] = None,
65 ):
@@ -65,12 +72,6 @@ def run_test(
72 Note, the text input is also adjusted to abide by vllm contract.
73 The text output is sanitized to be able to compare with hf.
74 """
68 - images = [asset.pil_image for asset in image_assets]
69 -
70 - inputs_per_image = [(
71 - [prompt for _ in size_factors],
72 - [rescale_image_size(image, factor) for factor in size_factors],
73 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
75
76 # NOTE: take care of the order. run vLLM first, and then run HF.
77 # vLLM needs a fresh new process without cuda initialization.
@@ -82,6 +83,7 @@ def run_test(
83 max_model_len=4096,
84 max_num_seqs=1,
85 dtype=dtype,
86 + limit_mm_per_prompt={"image": mm_limit},
87 tensor_parallel_size=tensor_parallel_size,
88 distributed_executor_backend=distributed_executor_backend,
89 enforce_eager=True) as vllm_model:
@@ -93,7 +95,7 @@ def run_test(
95 num_logprobs=num_logprobs,
96 images=images,
97 stop_token_ids=stop_token_ids)
96 - for prompts, images in inputs_per_image
98 + for prompts, images in inputs
99 ]
100
101 hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
@@ -104,7 +106,7 @@ def run_test(
106 num_logprobs=num_logprobs,
107 images=images,
108 tokenizer=tokenizer)
107 - for prompts, images in inputs_per_image
109 + for prompts, images in inputs
110 ]
111
112 for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -138,104 +140,26 @@ def run_test(
140 @pytest.mark.parametrize("num_logprobs", [5])
141 def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
142 dtype: str, max_tokens: int, num_logprobs: int) -> None:
143 + images = [asset.pil_image for asset in image_assets]
144 +
145 + inputs_per_image = [(
146 + [prompt for _ in size_factors],
147 + [rescale_image_size(image, factor) for factor in size_factors],
148 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
149 +
150 run_test(
151 hf_runner,
152 vllm_runner,
144 - image_assets,
153 + inputs_per_image,
154 model,
146 - size_factors=size_factors,
155 dtype=dtype,
156 max_tokens=max_tokens,
157 num_logprobs=num_logprobs,
158 + mm_limit=1,
159 tensor_parallel_size=1,
160 )
161
162
154 -HF_MULTIIMAGE_IMAGE_PROMPT = \
155 - "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
156 - "(<image>./</image>)\n(<image>./</image>)\n" \
157 - "Describe these images.<|eot_id|>" \
158 - "<|start_header_id|>assistant<|end_header_id|>\n\n"
159 -
160 -
161 -def run_multi_image_test(
162 - hf_runner: Type[HfRunner],
163 - vllm_runner: Type[VllmRunner],
164 - image_assets: _ImageAssets,
165 - model: str,
166 - *,
167 - size_factors: List[float],
168 - dtype: str,
169 - max_tokens: int,
170 - num_logprobs: int,
171 - tensor_parallel_size: int,
172 - distributed_executor_backend: Optional[str] = None,
173 -):
174 - """Inference result should be the same between hf and vllm.
175 -
176 - All the image fixtures for the test is under tests/images.
177 - For huggingface runner, we provide the PIL images as input.
178 - For vllm runner, we provide MultiModalDataDict objects
179 - and corresponding MultiModalConfig as input.
180 - Note, the text input is also adjusted to abide by vllm contract.
181 - The text output is sanitized to be able to compare with hf.
182 - """
183 - images = [asset.pil_image for asset in image_assets]
184 -
185 - inputs_per_case = [
186 - ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
187 - [[rescale_image_size(image, factor) for image in images]
188 - for factor in size_factors])
189 - ]
190 -
191 - # NOTE: take care of the order. run vLLM first, and then run HF.
192 - # vLLM needs a fresh new process without cuda initialization.
193 - # if we run HF first, the cuda initialization will be done and it
194 - # will hurt multiprocessing backend with fork method (the default method).
195 -
196 - # max_model_len should be greater than image_feature_size
197 - with vllm_runner(model,
198 - max_model_len=4096,
199 - max_num_seqs=1,
200 - limit_mm_per_prompt={"image": len(images)},
201 - dtype=dtype,
202 - tensor_parallel_size=tensor_parallel_size,
203 - distributed_executor_backend=distributed_executor_backend,
204 - enforce_eager=True) as vllm_model:
205 - tokenizer = vllm_model.model.get_tokenizer()
206 - stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
207 - vllm_outputs_per_case = [
208 - vllm_model.generate_greedy_logprobs(prompts,
209 - max_tokens,
210 - num_logprobs=num_logprobs,
211 - images=images,
212 - stop_token_ids=stop_token_ids)
213 - for prompts, images in inputs_per_case
214 - ]
215 -
216 - hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
217 - with hf_model, torch.no_grad():
218 - hf_outputs_per_case = [
219 - hf_model.generate_greedy_logprobs_limit(prompts,
220 - max_tokens,
221 - num_logprobs=num_logprobs,
222 - images=images,
223 - tokenizer=tokenizer)
224 - for prompts, images in inputs_per_case
225 - ]
226 -
227 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
228 - vllm_outputs_per_case):
229 - check_logprobs_close(
230 - outputs_0_lst=[
231 - trunc_hf_output(hf_output) for hf_output in hf_outputs
232 - ],
233 - outputs_1_lst=vllm_outputs,
234 - name_0="hf",
235 - name_1="vllm",
236 - )
237 -
238 -
163 @pytest.mark.parametrize("model", models)
164 @pytest.mark.parametrize(
165 "size_factors",
@@ -256,14 +180,22 @@ def run_multi_image_test(
180 def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
181 size_factors, dtype: str, max_tokens: int,
182 num_logprobs: int) -> None:
259 - run_multi_image_test(
183 + images = [asset.pil_image for asset in image_assets]
184 +
185 + inputs_per_case = [
186 + ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
187 + [[rescale_image_size(image, factor) for image in images]
188 + for factor in size_factors])
189 + ]
190 +
191 + run_test(
192 hf_runner,
193 vllm_runner,
262 - image_assets,
194 + inputs_per_case,
195 model,
264 - size_factors=size_factors,
196 dtype=dtype,
197 max_tokens=max_tokens,
198 num_logprobs=num_logprobs,
199 + mm_limit=2,
200 tensor_parallel_size=1,
201 )
tests/models/test_phi3v.py
+40 -101
@@ -1,6 +1,6 @@
1 import os
2 import re
3 -from typing import List, Optional, Tuple, Type
3 +from typing import List, Optional, Tuple, Type, Union
4
5 import pytest
6 from PIL import Image
@@ -60,13 +60,14 @@ if is_hip():
60 def run_test(
61 hf_runner: Type[HfRunner],
62 vllm_runner: Type[VllmRunner],
63 - images: List[Image.Image],
63 + inputs: List[Tuple[List[str], Union[List[Image.Image],
64 + List[List[Image.Image]]]]],
65 model: str,
66 *,
66 - size_factors: List[float],
67 dtype: str,
68 max_tokens: int,
69 num_logprobs: int,
70 + mm_limit: int,
71 tensor_parallel_size: int,
72 distributed_executor_backend: Optional[str] = None,
73 ):
@@ -79,13 +80,6 @@ def run_test(
80 Note, the text input is also adjusted to abide by vllm contract.
81 The text output is sanitized to be able to compare with hf.
82 """
82 - inputs_per_image = [(
83 - [prompt for _ in size_factors],
84 - [
85 - rescale_image_size(image, factor, transpose=idx)
86 - for idx, factor in enumerate(size_factors)
87 - ],
88 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
83
84 # NOTE: take care of the order. run vLLM first, and then run HF.
85 # vLLM needs a fresh new process without cuda initialization.
@@ -97,15 +91,16 @@ def run_test(
91 max_model_len=4096,
92 max_num_seqs=1,
93 dtype=dtype,
94 + limit_mm_per_prompt={"image": mm_limit},
95 tensor_parallel_size=tensor_parallel_size,
96 distributed_executor_backend=distributed_executor_backend,
97 enforce_eager=True) as vllm_model:
103 - vllm_outputs_per_image = [
98 + vllm_outputs_per_case = [
99 vllm_model.generate_greedy_logprobs(prompts,
100 max_tokens,
101 num_logprobs=num_logprobs,
102 images=images)
108 - for prompts, images in inputs_per_image
103 + for prompts, images in inputs
104 ]
105
106 # use eager mode for hf runner, since phi3_v didn't work with flash_attn
@@ -113,17 +108,17 @@ def run_test(
108 with hf_runner(model, dtype=dtype,
109 model_kwargs=hf_model_kwargs) as hf_model:
110 eos_token_id = hf_model.processor.tokenizer.eos_token_id
116 - hf_outputs_per_image = [
111 + hf_outputs_per_case = [
112 hf_model.generate_greedy_logprobs_limit(prompts,
113 max_tokens,
114 num_logprobs=num_logprobs,
115 images=images,
116 eos_token_id=eos_token_id)
122 - for prompts, images in inputs_per_image
117 + for prompts, images in inputs
118 ]
119
125 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
126 - vllm_outputs_per_image):
120 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
121 + vllm_outputs_per_case):
122 check_logprobs_close(
123 outputs_0_lst=hf_outputs,
124 outputs_1_lst=[
@@ -156,15 +151,22 @@ def run_test(
151 @pytest.mark.parametrize("num_logprobs", [10])
152 def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
153 dtype: str, max_tokens: int, num_logprobs: int) -> None:
154 + images = [asset.pil_image for asset in image_assets]
155 +
156 + inputs_per_image = [(
157 + [prompt for _ in size_factors],
158 + [rescale_image_size(image, factor) for factor in size_factors],
159 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
160 +
161 run_test(
162 hf_runner,
163 vllm_runner,
162 - [asset.pil_image for asset in image_assets],
164 + inputs_per_image,
165 model,
164 - size_factors=size_factors,
166 dtype=dtype,
167 max_tokens=max_tokens,
168 num_logprobs=num_logprobs,
169 + mm_limit=1,
170 tensor_parallel_size=1,
171 )
172
@@ -173,97 +175,26 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
175 @pytest.mark.parametrize("dtype", [target_dtype])
176 def test_regression_7840(hf_runner, vllm_runner, image_assets, model,
177 dtype) -> None:
178 + images = [asset.pil_image for asset in image_assets]
179 +
180 + inputs_regresion_7840 = [
181 + ([prompt], [image]) for image, prompt in zip(images, HF_IMAGE_PROMPTS)
182 + ]
183 +
184 # Regression test for #7840.
185 run_test(
186 hf_runner,
187 vllm_runner,
180 - [image_assets[0].pil_image.resize((465, 226))],
188 + inputs_regresion_7840,
189 model,
182 - size_factors=[1.0],
190 dtype=dtype,
191 max_tokens=128,
192 num_logprobs=10,
193 + mm_limit=1,
194 tensor_parallel_size=1,
195 )
196
197
190 -def run_multi_image_test(
191 - hf_runner: Type[HfRunner],
192 - vllm_runner: Type[VllmRunner],
193 - images: List[Image.Image],
194 - model: str,
195 - *,
196 - size_factors: List[float],
197 - dtype: str,
198 - max_tokens: int,
199 - num_logprobs: int,
200 - tensor_parallel_size: int,
201 - distributed_executor_backend: Optional[str] = None,
202 -):
203 - """Inference result should be the same between hf and vllm.
204 -
205 - All the image fixtures for the test is under tests/images.
206 - For huggingface runner, we provide the PIL images as input.
207 - For vllm runner, we provide MultiModalDataDict objects
208 - and corresponding MultiModalConfig as input.
209 - Note, the text input is also adjusted to abide by vllm contract.
210 - The text output is sanitized to be able to compare with hf.
211 - """
212 -
213 - inputs_per_case = [
214 - ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
215 - [[rescale_image_size(image, factor) for image in images]
216 - for factor in size_factors])
217 - ]
218 -
219 - # NOTE: take care of the order. run vLLM first, and then run HF.
220 - # vLLM needs a fresh new process without cuda initialization.
221 - # if we run HF first, the cuda initialization will be done and it
222 - # will hurt multiprocessing backend with fork method (the default method).
223 -
224 - # max_model_len should be greater than image_feature_size
225 - with vllm_runner(model,
226 - max_model_len=4096,
227 - max_num_seqs=1,
228 - limit_mm_per_prompt={"image": len(images)},
229 - dtype=dtype,
230 - tensor_parallel_size=tensor_parallel_size,
231 - distributed_executor_backend=distributed_executor_backend,
232 - enforce_eager=True) as vllm_model:
233 - vllm_outputs_per_case = [
234 - vllm_model.generate_greedy_logprobs(prompts,
235 - max_tokens,
236 - num_logprobs=num_logprobs,
237 - images=images)
238 - for prompts, images in inputs_per_case
239 - ]
240 -
241 - hf_model_kwargs = {"_attn_implementation": "eager"}
242 - with hf_runner(model, dtype=dtype,
243 - model_kwargs=hf_model_kwargs) as hf_model:
244 - eos_token_id = hf_model.processor.tokenizer.eos_token_id
245 - hf_outputs_per_case = [
246 - hf_model.generate_greedy_logprobs_limit(prompts,
247 - max_tokens,
248 - num_logprobs=num_logprobs,
249 - images=images,
250 - eos_token_id=eos_token_id)
251 - for prompts, images in inputs_per_case
252 - ]
253 -
254 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
255 - vllm_outputs_per_case):
256 - check_logprobs_close(
257 - outputs_0_lst=hf_outputs,
258 - outputs_1_lst=[
259 - vllm_to_hf_output(vllm_output, model)
260 - for vllm_output in vllm_outputs
261 - ],
262 - name_0="hf",
263 - name_1="vllm",
264 - )
265 -
266 -
198 @pytest.mark.parametrize("model", models)
199 @pytest.mark.parametrize(
200 "size_factors",
@@ -280,18 +211,26 @@ def run_multi_image_test(
211 )
212 @pytest.mark.parametrize("dtype", [target_dtype])
213 @pytest.mark.parametrize("max_tokens", [128])
283 -@pytest.mark.parametrize("num_logprobs", [5])
214 +@pytest.mark.parametrize("num_logprobs", [10])
215 def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
216 size_factors, dtype: str, max_tokens: int,
217 num_logprobs: int) -> None:
287 - run_multi_image_test(
218 + images = [asset.pil_image for asset in image_assets]
219 +
220 + inputs_per_case = [
221 + ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
222 + [[rescale_image_size(image, factor) for image in images]
223 + for factor in size_factors])
224 + ]
225 +
226 + run_test(
227 hf_runner,
228 vllm_runner,
290 - [asset.pil_image for asset in image_assets],
229 + inputs_per_case,
230 model,
292 - size_factors=size_factors,
231 dtype=dtype,
232 max_tokens=max_tokens,
233 num_logprobs=num_logprobs,
234 + mm_limit=2,
235 tensor_parallel_size=1,
236 )