@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 150b779081
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend] Enable Online Multi-image Support for MLlama (#9393)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Alex Brooks committed Oct 23, 2024 at 11:28 UTC 150b779081381124609a30383b5f87dbd6d110e5
2 files changed +230 -37
tests/entrypoints/test_chat_utils.py
+176
@@ -8,11 +8,13 @@ from vllm.assets.image import ImageAsset
8 from vllm.config import ModelConfig
9 from vllm.entrypoints.chat_utils import (parse_chat_messages,
10 parse_chat_messages_futures)
11 +from vllm.entrypoints.llm import apply_hf_chat_template
12 from vllm.multimodal import MultiModalDataDict
13 from vllm.multimodal.utils import encode_image_base64
14 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
15
16 PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
17 +MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
18
19
20 @pytest.fixture(scope="module")
@@ -39,6 +41,30 @@ def phi3v_tokenizer():
41 )
42
43
44 +@pytest.fixture(scope="module")
45 +def mllama_model_config():
46 + return ModelConfig(MLLAMA_MODEL_ID,
47 + task="generate",
48 + tokenizer=MLLAMA_MODEL_ID,
49 + tokenizer_mode="auto",
50 + trust_remote_code=True,
51 + dtype="bfloat16",
52 + seed=0,
53 + limit_mm_per_prompt={
54 + "image": 2,
55 + })
56 +
57 +
58 +@pytest.fixture(scope="module")
59 +def mllama_tokenizer():
60 + return TokenizerGroup(
61 + MLLAMA_MODEL_ID,
62 + enable_lora=False,
63 + max_num_seqs=5,
64 + max_input_length=None,
65 + )
66 +
67 +
68 @pytest.fixture(scope="module")
69 def image_url():
70 image = ImageAsset('cherry_blossom')
@@ -414,3 +440,153 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
440 "<|image_1|>\n<|image_2|>\nWhat's in these images?"
441 }]
442 _assert_mm_data_is_image_input(mm_data, 2)
443 +
444 +
445 +### Mllama currently wraps images / texts as interleaved dictionaries
446 +def test_mllama_single_image(
447 + mllama_model_config,
448 + mllama_tokenizer,
449 + image_url,
450 +):
451 + """Ensures that a single image is parsed correctly mllama."""
452 + conversation, mm_data = parse_chat_messages([{
453 + "role":
454 + "user",
455 + "content": [{
456 + 'type': 'text',
457 + 'text': 'The content of this image is:'
458 + }, {
459 + "image_url": image_url
460 + }]
461 + }], mllama_model_config, mllama_tokenizer)
462 + _assert_mm_data_is_image_input(mm_data, 1)
463 + assert conversation == [{
464 + 'role':
465 + 'user',
466 + 'content': [{
467 + 'type': 'text',
468 + 'text': 'The content of this image is:'
469 + }, {
470 + 'type': 'image'
471 + }]
472 + }]
473 +
474 +
475 +def test_mllama_interleaved_images(
476 + mllama_model_config,
477 + mllama_tokenizer,
478 + image_url,
479 +):
480 + """Ensures that multiple image are parsed as interleaved dicts."""
481 + conversation, mm_data = parse_chat_messages([{
482 + "role":
483 + "user",
484 + "content": [
485 + {
486 + 'type': 'text',
487 + 'text': 'The content of the first image is:'
488 + },
489 + {
490 + "image_url": image_url
491 + },
492 + {
493 + 'type': 'text',
494 + 'text': 'The content of the second image is:'
495 + },
496 + {
497 + "image_url": image_url
498 + },
499 + ]
500 + }], mllama_model_config, mllama_tokenizer)
501 + _assert_mm_data_is_image_input(mm_data, 2)
502 + assert conversation == [{
503 + 'role':
504 + 'user',
505 + 'content': [{
506 + 'type': 'text',
507 + 'text': 'The content of the first image is:'
508 + }, {
509 + 'type': 'image'
510 + }, {
511 + 'type': 'text',
512 + 'text': 'The content of the second image is:'
513 + }, {
514 + 'type': 'image'
515 + }]
516 + }]
517 +
518 +
519 +@pytest.mark.parametrize("model", [MLLAMA_MODEL_ID])
520 +def test_multimodal_image_parsing_matches_hf(model, image_url):
521 + """Checks end to end hf alignment for multimodal [image] parsing."""
522 +
523 + def get_conversation(is_hf: bool):
524 + img_part = {"type": "image_url", "image_url": {"url": image_url}}
525 + if is_hf:
526 + img_part = {'type': 'image'}
527 + return [{
528 + 'role':
529 + 'user',
530 + 'content': [
531 + {
532 + 'type': 'text',
533 + 'text': 'The content of the first image is:'
534 + },
535 + img_part,
536 + {
537 + 'type': 'text',
538 + 'text': 'The content of the second image is:'
539 + },
540 + img_part,
541 + {
542 + 'type': 'text',
543 + 'text': 'What animal is in the first image?'
544 + },
545 + ]
546 + }]
547 +
548 + # Build a config for the model
549 + model_config = ModelConfig(model,
550 + task="generate",
551 + tokenizer=MLLAMA_MODEL_ID,
552 + tokenizer_mode="auto",
553 + trust_remote_code=True,
554 + dtype="bfloat16",
555 + seed=0,
556 + limit_mm_per_prompt={
557 + "image": 2,
558 + })
559 +
560 + # Build the tokenizer group and grab the underlying tokenizer
561 + tokenizer_group = TokenizerGroup(
562 + MLLAMA_MODEL_ID,
563 + enable_lora=False,
564 + max_num_seqs=5,
565 + max_input_length=None,
566 + )
567 + tokenizer = tokenizer_group.tokenizer
568 +
569 + # Build and parse a conversation with {"type": "image"} using the tokenizer
570 + hf_conversation = get_conversation(is_hf=True)
571 + hf_result = tokenizer.apply_chat_template(
572 + hf_conversation,
573 + tokenize=False,
574 + add_generation_prompt=True,
575 + )
576 +
577 + # Now parse with vLLMs chat utils & apply the template
578 + vllm_conversation = get_conversation(is_hf=False)
579 + conversation, _ = parse_chat_messages(
580 + vllm_conversation,
581 + model_config,
582 + tokenizer_group,
583 + )
584 +
585 + vllm_result = apply_hf_chat_template(
586 + tokenizer,
587 + conversation=conversation,
588 + chat_template=None,
589 + add_generation_prompt=True,
590 + )
591 +
592 + assert hf_result == vllm_result
vllm/entrypoints/chat_utils.py
+54 -37
@@ -483,53 +483,70 @@ def _parse_chat_message_content_parts(
483 parts: Iterable[ChatCompletionContentPartParam],
484 mm_tracker: BaseMultiModalItemTracker,
485 ) -> List[ConversationMessage]:
486 - texts: List[str] = []
486 + content: List[Union[str, Dict[str, str]]] = []
487
488 mm_parser = mm_tracker.create_parser()
489 keep_multimodal_content = \
490 mm_tracker._model_config.hf_config.model_type in \
491 MODEL_KEEP_MULTI_MODAL_CONTENT
492
493 - has_image = False
493 for part in parts:
495 - if isinstance(part, str): # Handle plain text parts
496 - text = _TextParser(part)
497 - texts.append(text)
498 - else: # Handle structured dictionary parts
499 - part_type, content = _parse_chat_message_content_mm_part(part)
500 -
501 - # if part_type is text/refusal/image_url/audio_url but
502 - # content is empty, logg a warning and skip
503 - if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
504 - logger.warning("Skipping multimodal part "
505 - "with empty / unparsable content.")
506 - continue
507 -
508 - if part_type in ("text", "refusal"):
509 - texts.append(content)
510 - elif part_type == "image_url":
511 - mm_parser.parse_image(content)
512 - has_image = True
513 - elif part_type == "audio_url":
514 - mm_parser.parse_audio(content)
515 - else:
516 - raise NotImplementedError(f"Unknown part type: {part_type}")
494 + parse_res = _parse_chat_message_content_part(
495 + part, mm_parser, wrap_dicts=keep_multimodal_content)
496 + if parse_res:
497 + content.append(parse_res)
498
518 - text_prompt = "\n".join(texts)
499 if keep_multimodal_content:
520 - text_prompt = "\n".join(texts)
521 - role_content = [{'type': 'text', 'text': text_prompt}]
522 -
523 - if has_image:
524 - role_content = [{'type': 'image'}] + role_content
500 + # Parsing wraps images and texts as interleaved dictionaries
501 return [ConversationMessage(role=role,
526 - content=role_content)] # type: ignore
527 - else:
528 - mm_placeholder_counts = mm_parser.mm_placeholder_counts()
529 - if mm_placeholder_counts:
530 - text_prompt = _get_full_multimodal_text_prompt(
531 - mm_placeholder_counts, text_prompt)
532 - return [ConversationMessage(role=role, content=text_prompt)]
502 + content=content)] # type: ignore
503 + texts = cast(List[str], content)
504 + text_prompt = "\n".join(texts)
505 + mm_placeholder_counts = mm_parser.mm_placeholder_counts()
506 + if mm_placeholder_counts:
507 + text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
508 + text_prompt)
509 + return [ConversationMessage(role=role, content=text_prompt)]
510 +
511 +
512 +def _parse_chat_message_content_part(
513 + part: ChatCompletionContentPartParam,
514 + mm_parser: BaseMultiModalContentParser,
515 + wrap_dicts: bool) -> Optional[Union[str, Dict[str, str]]]:
516 + """Parses a single part of a conversation. If wrap_dicts is True,
517 + structured dictionary pieces for texts and images will be
518 + wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
519 + {"type": "image"}, respectively. Otherwise multimodal data will be
520 + handled by mm_parser, and texts will be returned as strings to be joined
521 + with multimodal placeholders.
522 + """
523 + if isinstance(part, str): # Handle plain text parts
524 + text = _TextParser(part)
525 + return text
526 +
527 + # Handle structured dictionary parts
528 + part_type, content = _parse_chat_message_content_mm_part(part)
529 +
530 + # if part_type is text/refusal/image_url/audio_url but
531 + # content is empty, log a warning and skip
532 + if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
533 + logger.warning(
534 + "Skipping multimodal part (type: '%s')"
535 + "with empty / unparsable content.", part_type)
536 + return None
537 +
538 + if part_type in ("text", "refusal"):
539 + return {'type': 'text', 'text': content} if wrap_dicts else content
540 +
541 + if part_type == "image_url":
542 + mm_parser.parse_image(content)
543 + return {'type': 'image'} if wrap_dicts else None
544 +
545 + if part_type == "audio_url":
546 + mm_parser.parse_audio(content)
547 + return {'type': 'audio'} if wrap_dicts else None
548 +
549 + raise NotImplementedError(f"Unknown part type: {part_type}")
550
551
552 # No need to validate using Pydantic again