A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Enable Online Multi-image Support for MLlama (#9393)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Alex Brooks committed
Oct 23, 2024 at 11:28 UTC
150b779081381124609a30383b5f87dbd6d110e5
2 files changed
+230
-37
tests/entrypoints/test_chat_utils.py
+176
@@ -8,11 +8,13 @@ from vllm.assets.image import ImageAsset
8
from vllm.config import ModelConfig
9
from vllm.entrypoints.chat_utils import (parse_chat_messages,
10
parse_chat_messages_futures)
11
+from vllm.entrypoints.llm import apply_hf_chat_template
12
from vllm.multimodal import MultiModalDataDict
13
from vllm.multimodal.utils import encode_image_base64
14
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
15
16
PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
17
+MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
18
19
20
@pytest.fixture(scope="module")
@@ -39,6 +41,30 @@ def phi3v_tokenizer():
41
)
42
43
44
+@pytest.fixture(scope="module")
45
+def mllama_model_config():
46
+ return ModelConfig(MLLAMA_MODEL_ID,
47
+ task="generate",
48
+ tokenizer=MLLAMA_MODEL_ID,
49
+ tokenizer_mode="auto",
50
+ trust_remote_code=True,
51
+ dtype="bfloat16",
52
+ seed=0,
53
+ limit_mm_per_prompt={
54
+ "image": 2,
55
+ })
56
+
57
+
58
+@pytest.fixture(scope="module")
59
+def mllama_tokenizer():
60
+ return TokenizerGroup(
61
+ MLLAMA_MODEL_ID,
62
+ enable_lora=False,
63
+ max_num_seqs=5,
64
+ max_input_length=None,
65
+ )
66
+
67
+
68
@pytest.fixture(scope="module")
69
def image_url():
70
image = ImageAsset('cherry_blossom')
@@ -414,3 +440,153 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
440
"<|image_1|>\n<|image_2|>\nWhat's in these images?"
441
}]
442
_assert_mm_data_is_image_input(mm_data, 2)
443
+
444
+
445
+### Mllama currently wraps images / texts as interleaved dictionaries
446
+def test_mllama_single_image(
447
+ mllama_model_config,
448
+ mllama_tokenizer,
449
+ image_url,
450
+):
451
+ """Ensures that a single image is parsed correctly mllama."""
452
+ conversation, mm_data = parse_chat_messages([{
453
+ "role":
454
+ "user",
455
+ "content": [{
456
+ 'type': 'text',
457
+ 'text': 'The content of this image is:'
458
+ }, {
459
+ "image_url": image_url
460
+ }]
461
+ }], mllama_model_config, mllama_tokenizer)
462
+ _assert_mm_data_is_image_input(mm_data, 1)
463
+ assert conversation == [{
464
+ 'role':
465
+ 'user',
466
+ 'content': [{
467
+ 'type': 'text',
468
+ 'text': 'The content of this image is:'
469
+ }, {
470
+ 'type': 'image'
471
+ }]
472
+ }]
473
+
474
+
475
+def test_mllama_interleaved_images(
476
+ mllama_model_config,
477
+ mllama_tokenizer,
478
+ image_url,
479
+):
480
+ """Ensures that multiple image are parsed as interleaved dicts."""
481
+ conversation, mm_data = parse_chat_messages([{
482
+ "role":
483
+ "user",
484
+ "content": [
485
+ {
486
+ 'type': 'text',
487
+ 'text': 'The content of the first image is:'
488
+ },
489
+ {
490
+ "image_url": image_url
491
+ },
492
+ {
493
+ 'type': 'text',
494
+ 'text': 'The content of the second image is:'
495
+ },
496
+ {
497
+ "image_url": image_url
498
+ },
499
+ ]
500
+ }], mllama_model_config, mllama_tokenizer)
501
+ _assert_mm_data_is_image_input(mm_data, 2)
502
+ assert conversation == [{
503
+ 'role':
504
+ 'user',
505
+ 'content': [{
506
+ 'type': 'text',
507
+ 'text': 'The content of the first image is:'
508
+ }, {
509
+ 'type': 'image'
510
+ }, {
511
+ 'type': 'text',
512
+ 'text': 'The content of the second image is:'
513
+ }, {
514
+ 'type': 'image'
515
+ }]
516
+ }]
517
+
518
+
519
+@pytest.mark.parametrize("model", [MLLAMA_MODEL_ID])
520
+def test_multimodal_image_parsing_matches_hf(model, image_url):
521
+ """Checks end to end hf alignment for multimodal [image] parsing."""
522
+
523
+ def get_conversation(is_hf: bool):
524
+ img_part = {"type": "image_url", "image_url": {"url": image_url}}
525
+ if is_hf:
526
+ img_part = {'type': 'image'}
527
+ return [{
528
+ 'role':
529
+ 'user',
530
+ 'content': [
531
+ {
532
+ 'type': 'text',
533
+ 'text': 'The content of the first image is:'
534
+ },
535
+ img_part,
536
+ {
537
+ 'type': 'text',
538
+ 'text': 'The content of the second image is:'
539
+ },
540
+ img_part,
541
+ {
542
+ 'type': 'text',
543
+ 'text': 'What animal is in the first image?'
544
+ },
545
+ ]
546
+ }]
547
+
548
+ # Build a config for the model
549
+ model_config = ModelConfig(model,
550
+ task="generate",
551
+ tokenizer=MLLAMA_MODEL_ID,
552
+ tokenizer_mode="auto",
553
+ trust_remote_code=True,
554
+ dtype="bfloat16",
555
+ seed=0,
556
+ limit_mm_per_prompt={
557
+ "image": 2,
558
+ })
559
+
560
+ # Build the tokenizer group and grab the underlying tokenizer
561
+ tokenizer_group = TokenizerGroup(
562
+ MLLAMA_MODEL_ID,
563
+ enable_lora=False,
564
+ max_num_seqs=5,
565
+ max_input_length=None,
566
+ )
567
+ tokenizer = tokenizer_group.tokenizer
568
+
569
+ # Build and parse a conversation with {"type": "image"} using the tokenizer
570
+ hf_conversation = get_conversation(is_hf=True)
571
+ hf_result = tokenizer.apply_chat_template(
572
+ hf_conversation,
573
+ tokenize=False,
574
+ add_generation_prompt=True,
575
+ )
576
+
577
+ # Now parse with vLLMs chat utils & apply the template
578
+ vllm_conversation = get_conversation(is_hf=False)
579
+ conversation, _ = parse_chat_messages(
580
+ vllm_conversation,
581
+ model_config,
582
+ tokenizer_group,
583
+ )
584
+
585
+ vllm_result = apply_hf_chat_template(
586
+ tokenizer,
587
+ conversation=conversation,
588
+ chat_template=None,
589
+ add_generation_prompt=True,
590
+ )
591
+
592
+ assert hf_result == vllm_result
vllm/entrypoints/chat_utils.py
+54
-37
@@ -483,53 +483,70 @@ def _parse_chat_message_content_parts(
483
parts: Iterable[ChatCompletionContentPartParam],
484
mm_tracker: BaseMultiModalItemTracker,
485
) -> List[ConversationMessage]:
486
- texts: List[str] = []
486
+ content: List[Union[str, Dict[str, str]]] = []
487
488
mm_parser = mm_tracker.create_parser()
489
keep_multimodal_content = \
490
mm_tracker._model_config.hf_config.model_type in \
491
MODEL_KEEP_MULTI_MODAL_CONTENT
492
493
- has_image = False
493
for part in parts:
495
- if isinstance(part, str): # Handle plain text parts
496
- text = _TextParser(part)
497
- texts.append(text)
498
- else: # Handle structured dictionary parts
499
- part_type, content = _parse_chat_message_content_mm_part(part)
500
-
501
- # if part_type is text/refusal/image_url/audio_url but
502
- # content is empty, logg a warning and skip
503
- if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
504
- logger.warning("Skipping multimodal part "
505
- "with empty / unparsable content.")
506
- continue
507
-
508
- if part_type in ("text", "refusal"):
509
- texts.append(content)
510
- elif part_type == "image_url":
511
- mm_parser.parse_image(content)
512
- has_image = True
513
- elif part_type == "audio_url":
514
- mm_parser.parse_audio(content)
515
- else:
516
- raise NotImplementedError(f"Unknown part type: {part_type}")
494
+ parse_res = _parse_chat_message_content_part(
495
+ part, mm_parser, wrap_dicts=keep_multimodal_content)
496
+ if parse_res:
497
+ content.append(parse_res)
498
518
- text_prompt = "\n".join(texts)
499
if keep_multimodal_content:
520
- text_prompt = "\n".join(texts)
521
- role_content = [{'type': 'text', 'text': text_prompt}]
522
-
523
- if has_image:
524
- role_content = [{'type': 'image'}] + role_content
500
+ # Parsing wraps images and texts as interleaved dictionaries
501
return [ConversationMessage(role=role,
526
- content=role_content)] # type: ignore
527
- else:
528
- mm_placeholder_counts = mm_parser.mm_placeholder_counts()
529
- if mm_placeholder_counts:
530
- text_prompt = _get_full_multimodal_text_prompt(
531
- mm_placeholder_counts, text_prompt)
532
- return [ConversationMessage(role=role, content=text_prompt)]
502
+ content=content)] # type: ignore
503
+ texts = cast(List[str], content)
504
+ text_prompt = "\n".join(texts)
505
+ mm_placeholder_counts = mm_parser.mm_placeholder_counts()
506
+ if mm_placeholder_counts:
507
+ text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
508
+ text_prompt)
509
+ return [ConversationMessage(role=role, content=text_prompt)]
510
+
511
+
512
+def _parse_chat_message_content_part(
513
+ part: ChatCompletionContentPartParam,
514
+ mm_parser: BaseMultiModalContentParser,
515
+ wrap_dicts: bool) -> Optional[Union[str, Dict[str, str]]]:
516
+ """Parses a single part of a conversation. If wrap_dicts is True,
517
+ structured dictionary pieces for texts and images will be
518
+ wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
519
+ {"type": "image"}, respectively. Otherwise multimodal data will be
520
+ handled by mm_parser, and texts will be returned as strings to be joined
521
+ with multimodal placeholders.
522
+ """
523
+ if isinstance(part, str): # Handle plain text parts
524
+ text = _TextParser(part)
525
+ return text
526
+
527
+ # Handle structured dictionary parts
528
+ part_type, content = _parse_chat_message_content_mm_part(part)
529
+
530
+ # if part_type is text/refusal/image_url/audio_url but
531
+ # content is empty, log a warning and skip
532
+ if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
533
+ logger.warning(
534
+ "Skipping multimodal part (type: '%s')"
535
+ "with empty / unparsable content.", part_type)
536
+ return None
537
+
538
+ if part_type in ("text", "refusal"):
539
+ return {'type': 'text', 'text': content} if wrap_dicts else content
540
+
541
+ if part_type == "image_url":
542
+ mm_parser.parse_image(content)
543
+ return {'type': 'image'} if wrap_dicts else None
544
+
545
+ if part_type == "audio_url":
546
+ mm_parser.parse_audio(content)
547
+ return {'type': 'audio'} if wrap_dicts else None
548
+
549
+ raise NotImplementedError(f"Unknown part type: {part_type}")
550
551
552
# No need to validate using Pydantic again