A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Model][VLM] Support multi-images inputs for Phi-3-vision models (#7783)
Isotr0py committed
Aug 25, 2024 at 19:51 UTC
8aaf3d5347ad536de25869caa67b90e43f1ccd5b
2 files changed
+168
-29
tests/models/test_phi3v.py
+111
@@ -21,6 +21,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21
"cherry_blossom":
22
"<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
23
})
24
+HF_MULTIIMAGE_IMAGE_PROMPT = "<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
25
26
models = ["microsoft/Phi-3.5-vision-instruct"]
27
@@ -184,3 +185,113 @@ def test_regression_7840(hf_runner, vllm_runner, image_assets, model,
185
num_logprobs=10,
186
tensor_parallel_size=1,
187
)
188
+
189
+
190
+def run_multi_image_test(
191
+ hf_runner: Type[HfRunner],
192
+ vllm_runner: Type[VllmRunner],
193
+ images: List[Image.Image],
194
+ model: str,
195
+ *,
196
+ size_factors: List[float],
197
+ dtype: str,
198
+ max_tokens: int,
199
+ num_logprobs: int,
200
+ tensor_parallel_size: int,
201
+ distributed_executor_backend: Optional[str] = None,
202
+):
203
+ """Inference result should be the same between hf and vllm.
204
+
205
+ All the image fixtures for the test is under tests/images.
206
+ For huggingface runner, we provide the PIL images as input.
207
+ For vllm runner, we provide MultiModalDataDict objects
208
+ and corresponding MultiModalConfig as input.
209
+ Note, the text input is also adjusted to abide by vllm contract.
210
+ The text output is sanitized to be able to compare with hf.
211
+ """
212
+
213
+ inputs_per_case = [
214
+ ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
215
+ [[rescale_image_size(image, factor) for image in images]
216
+ for factor in size_factors])
217
+ ]
218
+
219
+ # NOTE: take care of the order. run vLLM first, and then run HF.
220
+ # vLLM needs a fresh new process without cuda initialization.
221
+ # if we run HF first, the cuda initialization will be done and it
222
+ # will hurt multiprocessing backend with fork method (the default method).
223
+
224
+ # max_model_len should be greater than image_feature_size
225
+ with vllm_runner(model,
226
+ max_model_len=4096,
227
+ max_num_seqs=1,
228
+ limit_mm_per_prompt={"image": len(images)},
229
+ dtype=dtype,
230
+ tensor_parallel_size=tensor_parallel_size,
231
+ distributed_executor_backend=distributed_executor_backend,
232
+ enforce_eager=True) as vllm_model:
233
+ vllm_outputs_per_case = [
234
+ vllm_model.generate_greedy_logprobs(prompts,
235
+ max_tokens,
236
+ num_logprobs=num_logprobs,
237
+ images=images)
238
+ for prompts, images in inputs_per_case
239
+ ]
240
+
241
+ hf_model_kwargs = {"_attn_implementation": "eager"}
242
+ with hf_runner(model, dtype=dtype,
243
+ model_kwargs=hf_model_kwargs) as hf_model:
244
+ eos_token_id = hf_model.processor.tokenizer.eos_token_id
245
+ hf_outputs_per_case = [
246
+ hf_model.generate_greedy_logprobs_limit(prompts,
247
+ max_tokens,
248
+ num_logprobs=num_logprobs,
249
+ images=images,
250
+ eos_token_id=eos_token_id)
251
+ for prompts, images in inputs_per_case
252
+ ]
253
+
254
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
255
+ vllm_outputs_per_case):
256
+ check_logprobs_close(
257
+ outputs_0_lst=hf_outputs,
258
+ outputs_1_lst=[
259
+ vllm_to_hf_output(vllm_output, model)
260
+ for vllm_output in vllm_outputs
261
+ ],
262
+ name_0="hf",
263
+ name_1="vllm",
264
+ )
265
+
266
+
267
+@pytest.mark.parametrize("model", models)
268
+@pytest.mark.parametrize(
269
+ "size_factors",
270
+ [
271
+ # No image
272
+ [],
273
+ # Single-scale
274
+ [1.0],
275
+ # Single-scale, batched
276
+ [1.0, 1.0, 1.0],
277
+ # Multi-scale
278
+ [0.25, 0.5, 1.0],
279
+ ],
280
+)
281
+@pytest.mark.parametrize("dtype", [target_dtype])
282
+@pytest.mark.parametrize("max_tokens", [128])
283
+@pytest.mark.parametrize("num_logprobs", [5])
284
+def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
285
+ size_factors, dtype: str, max_tokens: int,
286
+ num_logprobs: int) -> None:
287
+ run_multi_image_test(
288
+ hf_runner,
289
+ vllm_runner,
290
+ [asset.pil_image for asset in image_assets],
291
+ model,
292
+ size_factors=size_factors,
293
+ dtype=dtype,
294
+ max_tokens=max_tokens,
295
+ num_logprobs=num_logprobs,
296
+ tensor_parallel_size=1,
297
+ )
vllm/model_executor/models/phi3v.py
+57
-29
@@ -13,6 +13,7 @@
13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
# See the License for the specific language governing permissions and
15
# limitations under the License.
16
+import itertools
17
import re
18
from functools import lru_cache
19
from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
@@ -37,11 +38,11 @@ from vllm.model_executor.models.clip import CLIPVisionModel
38
from vllm.model_executor.models.llama import LlamaModel
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.multimodal import MULTIMODAL_REGISTRY
40
-from vllm.multimodal.utils import cached_get_tokenizer
41
+from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
42
from vllm.sequence import IntermediateTensors, SamplerOutput
43
+from vllm.utils import is_list_of
44
43
-from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
44
- input_processor_for_clip)
45
+from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
46
from .interfaces import SupportsMultiModal
47
from .utils import merge_multimodal_embeddings
48
@@ -400,9 +401,20 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
401
image_data = multi_modal_data["image"]
402
if isinstance(image_data, Image.Image):
403
w, h = image_data.size
403
- image_feature_size = get_phi3v_image_feature_size(hf_config,
404
- input_width=w,
405
- input_height=h)
404
+ image_feature_size = [
405
+ get_phi3v_image_feature_size(hf_config,
406
+ input_width=w,
407
+ input_height=h)
408
+ ]
409
+ image_data = [image_data]
410
+ elif is_list_of(image_data, Image.Image):
411
+ image_feature_size = []
412
+ for image in image_data:
413
+ w, h = image.size
414
+ image_feature_size.append(
415
+ get_phi3v_image_feature_size(hf_config,
416
+ input_width=w,
417
+ input_height=h))
418
elif isinstance(image_data, torch.Tensor):
419
image_feature_size = image_data.shape[0]
420
else:
@@ -410,45 +422,61 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
422
423
prompt = llm_inputs.get("prompt")
424
if prompt is None:
425
+ image_idx = []
426
new_prompt = None
427
else:
428
+ image_idx = sorted(map(int, re.findall(r"<\|image_(\d+)\|>+", prompt)))
429
if prompt.count("<|image|>") > 0:
430
logger.warning("Please follow the prompt format that is "
431
"documented on HuggingFace which does not involve "
432
"repeating <|image|> tokens.")
419
- elif len(re.findall(r"(<\|image_\d+\|>)+", prompt)) > 1:
420
- logger.warning("Multiple image input is not supported yet, "
421
- "so any extra image tokens will be treated "
422
- "as plain text.")
423
-
433
+ elif (num_image_tags := len(image_idx)) > 1:
434
+ assert num_image_tags == len(
435
+ image_data), "The count of image_placeholder not match image's"
436
new_prompt = prompt
437
426
- prompt_token_ids = llm_inputs["prompt_token_ids"]
427
- image_1_token_ids = _get_image_placeholder_token_ids(model_config, idx=1)
438
+ prompt_token_ids = llm_inputs["prompt_token_ids"].copy()
439
+
440
+ # masked place_holder with image token id
441
+ for idx in image_idx:
442
+ image_token_ids = _get_image_placeholder_token_ids(model_config,
443
+ idx=idx)
444
+ for i in range(len(prompt_token_ids) - len(image_token_ids) + 1):
445
+ if prompt_token_ids[i:i + len(image_token_ids)] == image_token_ids:
446
+ prompt_token_ids[i:i + len(image_token_ids)] = [
447
+ _IMAGE_TOKEN_ID
448
+ ] * len(image_token_ids)
449
+ break
450
+
451
+ # merge consecutive tag ids
452
+ merged_token_ids: List[int] = []
453
+ for is_placeholder, token_ids in itertools.groupby(
454
+ prompt_token_ids, lambda x: x == _IMAGE_TOKEN_ID):
455
+ if is_placeholder:
456
+ merged_token_ids.append(_IMAGE_TOKEN_ID)
457
+ else:
458
+ merged_token_ids.extend(list(token_ids))
459
460
+ # TODO: Move this to utils or integrate with clip.
461
new_token_ids: List[int] = []
430
- for i in range(len(prompt_token_ids) - len(image_1_token_ids) + 1):
431
- if prompt_token_ids[i:i + len(image_1_token_ids)] == image_1_token_ids:
432
- new_token_ids.append(_IMAGE_TOKEN_ID)
433
-
434
- # No need to further scan the list since we only replace once
435
- new_token_ids.extend(prompt_token_ids[i + len(image_1_token_ids):])
436
- break
462
+ placeholder_idx = 0
463
+ while merged_token_ids:
464
+ token_id = merged_token_ids.pop(0)
465
+ if token_id == _IMAGE_TOKEN_ID:
466
+ new_token_ids.extend(
467
+ repeat_and_pad_token(
468
+ _IMAGE_TOKEN_ID,
469
+ repeat_count=image_feature_size[placeholder_idx],
470
+ ))
471
+ placeholder_idx += 1
472
else:
438
- new_token_ids.append(prompt_token_ids[i])
473
+ new_token_ids.append(token_id)
474
475
# NOTE: Create a defensive copy of the original inputs
476
llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
477
prompt=new_prompt,
478
multi_modal_data=multi_modal_data)
444
-
445
- return input_processor_for_clip(
446
- model_config,
447
- CLIP_VIT_LARGE_PATCH14_336_CONFIG,
448
- llm_inputs,
449
- image_token_id=_IMAGE_TOKEN_ID,
450
- image_feature_size_override=image_feature_size,
451
- )
479
+ return llm_inputs
480
481
482
@MULTIMODAL_REGISTRY.register_image_input_mapper()