@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e601efcb10
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

[Misc] Add fully interleaved support for multimodal 'string' content format (#14047)

Signed-off-by: drobyshev.anton <drobyshev.anton@wb.ru> Co-authored-by: drobyshev.anton <drobyshev.anton@wb.ru>

Anton committed Jul 7, 2025 at 22:43 UTC e601efcb10ed982e15e522f7c29c8531677678ca
4 files changed +479 -44
tests/entrypoints/test_chat_utils.py
+350 -2
@@ -2,11 +2,14 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import warnings
5 -from typing import Optional
5 +from collections.abc import Mapping
6 +from typing import Literal, Optional
7
8 import pytest
9
10 +from vllm.assets.audio import AudioAsset
11 from vllm.assets.image import ImageAsset
12 +from vllm.assets.video import VideoAsset
13 from vllm.config import ModelConfig
14 from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,
15 parse_chat_messages,
@@ -15,7 +18,8 @@ from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,
18 resolve_hf_chat_template)
19 from vllm.entrypoints.llm import apply_hf_chat_template
20 from vllm.multimodal import MultiModalDataDict
18 -from vllm.multimodal.utils import encode_image_base64
21 +from vllm.multimodal.utils import (encode_audio_base64, encode_image_base64,
22 + encode_video_base64)
23 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
24
25 from ..models.registry import HF_EXAMPLE_MODELS
@@ -28,6 +32,7 @@ ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
32 QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
33 QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
34 QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
35 +QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
36 MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
37 LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
38 HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
@@ -48,6 +53,21 @@ def phi3v_model_config():
53 })
54
55
56 +@pytest.fixture(scope="function")
57 +def phi3v_model_config_mm_interleaved():
58 + return ModelConfig(PHI3V_MODEL_ID,
59 + task="generate",
60 + tokenizer=PHI3V_MODEL_ID,
61 + tokenizer_mode="auto",
62 + trust_remote_code=True,
63 + dtype="auto",
64 + seed=0,
65 + interleave_mm_strings=True,
66 + limit_mm_per_prompt={
67 + "image": 2,
68 + })
69 +
70 +
71 @pytest.fixture(scope="module")
72 def phi3v_tokenizer():
73 return TokenizerGroup(
@@ -58,6 +78,32 @@ def phi3v_tokenizer():
78 )
79
80
81 +@pytest.fixture(scope="function")
82 +def qwen25omni_model_config_mm_interleaved():
83 + return ModelConfig(QWEN25OMNI_MODEL_ID,
84 + task="generate",
85 + tokenizer=QWEN25OMNI_MODEL_ID,
86 + tokenizer_mode="auto",
87 + dtype="auto",
88 + seed=0,
89 + interleave_mm_strings=True,
90 + limit_mm_per_prompt={
91 + "image": 2,
92 + "audio": 1,
93 + "video": 1,
94 + })
95 +
96 +
97 +@pytest.fixture(scope="module")
98 +def qwen25omni_tokenizer():
99 + return TokenizerGroup(
100 + tokenizer_id=QWEN25OMNI_MODEL_ID,
101 + enable_lora=False,
102 + max_num_seqs=5,
103 + max_input_length=None,
104 + )
105 +
106 +
107 @pytest.fixture(scope="module")
108 def mllama_model_config():
109 return ModelConfig(MLLAMA_MODEL_ID,
@@ -113,6 +159,20 @@ def image_url():
159 return f"data:image/jpeg;base64,{base64}"
160
161
162 +@pytest.fixture(scope="module")
163 +def video_url():
164 + video = VideoAsset('baby_reading', 1)
165 + base64 = encode_video_base64(video.np_ndarrays)
166 + return f"data:video/jpeg;base64,{base64}"
167 +
168 +
169 +@pytest.fixture(scope="module")
170 +def audio_url():
171 + audio = AudioAsset('mary_had_lamb')
172 + base64 = encode_audio_base64(*audio.audio_and_sample_rate)
173 + return f"data:audio/ogg;base64,{base64}"
174 +
175 +
176 def _assert_mm_data_is_image_input(
177 mm_data: Optional[MultiModalDataDict],
178 image_count: int,
@@ -126,6 +186,23 @@ def _assert_mm_data_is_image_input(
186 assert isinstance(image_data, list) and len(image_data) == image_count
187
188
189 +ModalityType = Literal["image", "video", "audio"]
190 +MultiModalDataCounts = Mapping[ModalityType, int]
191 +
192 +
193 +def _assert_mm_data_inputs(
194 + mm_data: Optional[MultiModalDataDict],
195 + data_count: MultiModalDataCounts,
196 +) -> None:
197 + assert mm_data is not None
198 + assert set(data_count.keys()) == (set(mm_data.keys()))
199 +
200 + for modality, n in data_count.items():
201 + modality_data = mm_data.get(modality)
202 + assert modality_data is not None
203 + assert isinstance(modality_data, list) and len(modality_data) == n
204 +
205 +
206 def test_parse_chat_messages_single_image(
207 phi3v_model_config,
208 phi3v_tokenizer,
@@ -637,6 +714,277 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
714 _assert_mm_data_is_image_input(mm_data, 2)
715
716
717 +def test_parse_chat_messages_multiple_images_interleave(
718 + phi3v_model_config_mm_interleaved,
719 + phi3v_tokenizer,
720 + image_url,
721 +):
722 + conversation, mm_data = parse_chat_messages(
723 + [{
724 + "role":
725 + "user",
726 + "content": [{
727 + "type": "text",
728 + "text": "I need you to compare this image"
729 + }, {
730 + "type": "image_url",
731 + "image_url": {
732 + "url": image_url
733 + }
734 + }, {
735 + "type": "text",
736 + "text": "and this one"
737 + }, {
738 + "type": "image_url",
739 + "image_url": {
740 + "url": image_url
741 + }
742 + }, {
743 + "type": "text",
744 + "text": "Do they have differences?"
745 + }]
746 + }],
747 + phi3v_model_config_mm_interleaved,
748 + phi3v_tokenizer,
749 + content_format="string",
750 + )
751 +
752 + assert conversation == [{
753 + "role":
754 + "user",
755 + "content":
756 + "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n" # noqa: E501
757 + "Do they have differences?"
758 + }]
759 + _assert_mm_data_is_image_input(mm_data, 2)
760 +
761 +
762 +@pytest.mark.asyncio
763 +async def test_parse_chat_messages_multiple_images_interleave_async(
764 + phi3v_model_config_mm_interleaved,
765 + phi3v_tokenizer,
766 + image_url,
767 +):
768 + conversation, mm_data = parse_chat_messages_futures(
769 + [{
770 + "role":
771 + "user",
772 + "content": [{
773 + "type": "text",
774 + "text": "I need you to compare this image"
775 + }, {
776 + "type": "image_url",
777 + "image_url": {
778 + "url": image_url
779 + }
780 + }, {
781 + "type": "text",
782 + "text": "and this one"
783 + }, {
784 + "type": "image_url",
785 + "image_url": {
786 + "url": image_url
787 + }
788 + }, {
789 + "type": "text",
790 + "text": "Do they have differences?"
791 + }]
792 + }],
793 + phi3v_model_config_mm_interleaved,
794 + phi3v_tokenizer,
795 + content_format="string",
796 + )
797 +
798 + assert conversation == [{
799 + "role":
800 + "user",
801 + "content":
802 + "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n" # noqa: E501
803 + "Do they have differences?"
804 + }]
805 + _assert_mm_data_is_image_input(await mm_data, 2)
806 +
807 +
808 +def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
809 + phi3v_model_config_mm_interleaved,
810 + phi3v_tokenizer,
811 + image_url,
812 +):
813 + conversation, mm_data = parse_chat_messages(
814 + [{
815 + "role":
816 + "user",
817 + "content": [
818 + {
819 + "type": "text",
820 + "text": "What's on this image?"
821 + },
822 + {
823 + "type": "image_url",
824 + "image_url": {
825 + "url": image_url
826 + }
827 + },
828 + {
829 + "type": "text",
830 + "text": "Be accurate."
831 + },
832 + ]
833 + }, {
834 + "role": "assistant",
835 + "content": "Some stuff."
836 + }, {
837 + "role":
838 + "user",
839 + "content": [{
840 + "type": "text",
841 + "text": "What's on this image?"
842 + }, {
843 + "type": "image_url",
844 + "image_url": {
845 + "url": image_url
846 + }
847 + }]
848 + }],
849 + phi3v_model_config_mm_interleaved,
850 + phi3v_tokenizer,
851 + content_format="string",
852 + )
853 +
854 + assert conversation == [{
855 + "role":
856 + "user",
857 + "content":
858 + "What's on this image?\n<|image_1|>\nBe accurate."
859 + }, {
860 + "role": "assistant",
861 + "content": "Some stuff."
862 + }, {
863 + "role": "user",
864 + "content": "What's on this image?\n<|image_2|>"
865 + }]
866 + _assert_mm_data_is_image_input(mm_data, 2)
867 +
868 +
869 +def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
870 + qwen25omni_model_config_mm_interleaved, qwen25omni_tokenizer,
871 + image_url, video_url, audio_url):
872 + conversation, mm_data = parse_chat_messages(
873 + [{
874 + "role":
875 + "user",
876 + "content": [
877 + {
878 + "type": "text",
879 + "text": "What's on this image?"
880 + },
881 + {
882 + "type": "image_url",
883 + "image_url": {
884 + "url": image_url
885 + }
886 + },
887 + {
888 + "type": "text",
889 + "text": "Now listen to this audio"
890 + },
891 + {
892 + "type": "audio_url",
893 + "audio_url": {
894 + "url": audio_url
895 + }
896 + },
897 + ]
898 + }, {
899 + "role": "assistant",
900 + "content": "Some stuff."
901 + }, {
902 + "role":
903 + "user",
904 + "content": [{
905 + "type": "text",
906 + "text": "What's on this image?"
907 + }, {
908 + "type": "image_url",
909 + "image_url": {
910 + "url": image_url
911 + }
912 + }, {
913 + "type": "text",
914 + "text": "And what's in the video?"
915 + }, {
916 + "type": "video_url",
917 + "video_url": {
918 + "url": video_url
919 + }
920 + }]
921 + }],
922 + qwen25omni_model_config_mm_interleaved,
923 + qwen25omni_tokenizer,
924 + content_format="string",
925 + )
926 +
927 + assert conversation == [{
928 + "role":
929 + "user",
930 + "content":
931 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
932 + "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>"
933 + }, {
934 + "role": "assistant",
935 + "content": "Some stuff."
936 + }, {
937 + "role":
938 + "user",
939 + "content":
940 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
941 + "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>"
942 + }]
943 +
944 + _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
945 +
946 +
947 +def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
948 + phi3v_model_config_mm_interleaved,
949 + phi3v_tokenizer,
950 + image_url,
951 +):
952 + with pytest.raises(
953 + ValueError,
954 + match=r"Found more '<|image_1|>' placeholders in input prompt "
955 + "than actual multimodal data items."):
956 + parse_chat_messages(
957 + [{
958 + "role":
959 + "user",
960 + "content": [
961 + {
962 + "type": "image_url",
963 + "image_url": {
964 + "url": image_url
965 + }
966 + },
967 + {
968 + "type": "image_url",
969 + "image_url": {
970 + "url": image_url
971 + }
972 + },
973 + {
974 + "type":
975 + "text",
976 + "text":
977 + "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n" # noqa: E501
978 + "Do they have differences?"
979 + },
980 + ]
981 + }],
982 + phi3v_model_config_mm_interleaved,
983 + phi3v_tokenizer,
984 + content_format="string",
985 + )
986 +
987 +
988 ### Mllama currently wraps images / texts as interleaved dictionaries
989 def test_mllama_single_image(
990 mllama_model_config,
vllm/config.py
+13 -1
@@ -346,6 +346,9 @@ class ModelConfig:
346 limit_mm_per_prompt: dict[str, int] = field(default_factory=dict)
347 """Maximum number of data items per modality per prompt. Only applicable
348 for multimodal models."""
349 + interleave_mm_strings: bool = False
350 + """Enable fully interleaved support for multimodal prompts, while using
351 + --chat-template-content-format=string. Defaults to False."""
352 media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
353 """Additional args passed to process media inputs, keyed by modalities.
354 For example, to set num_frames for video, set
@@ -702,7 +705,8 @@ class ModelConfig:
705 media_io_kwargs=self.media_io_kwargs,
706 mm_processor_kwargs=self.mm_processor_kwargs,
707 disable_mm_preprocessor_cache=self.
705 - disable_mm_preprocessor_cache)
708 + disable_mm_preprocessor_cache,
709 + interleave_mm_strings=self.interleave_mm_strings)
710
711 if self.limit_mm_per_prompt:
712 raise ValueError("`limit_mm_per_prompt` is only supported for "
@@ -713,6 +717,9 @@ class ModelConfig:
717 if self.disable_mm_preprocessor_cache:
718 raise ValueError("`disable_mm_preprocessor_cache` is only "
719 "supported for multimodal models.")
720 + if self.interleave_mm_strings:
721 + raise ValueError("`interleave_mm_strings` is only "
722 + "supported for multimodal models.")
723
724 return None
725
@@ -3126,6 +3133,11 @@ class MultiModalConfig:
3133 If `True`, disable caching of the processed multi-modal inputs.
3134 """
3135
3136 + interleave_mm_strings: bool = False
3137 + """
3138 + Enable fully interleaved support for multimodal prompts.
3139 + """
3140 +
3141 def compute_hash(self) -> str:
3142 """
3143 WARNING: Whenever a new field is added to this config,
vllm/engine/arg_utils.py
+5
@@ -370,6 +370,7 @@ class EngineArgs:
370 get_field(TokenizerPoolConfig, "extra_config")
371 limit_mm_per_prompt: dict[str, int] = \
372 get_field(MultiModalConfig, "limit_per_prompt")
373 + interleave_mm_strings: bool = MultiModalConfig.interleave_mm_strings
374 media_io_kwargs: dict[str, dict[str,
375 Any]] = get_field(MultiModalConfig,
376 "media_io_kwargs")
@@ -763,6 +764,9 @@ class EngineArgs:
764 multimodal_group.add_argument(
765 "--disable-mm-preprocessor-cache",
766 **multimodal_kwargs["disable_mm_preprocessor_cache"])
767 + multimodal_group.add_argument(
768 + "--interleave-mm-strings",
769 + **multimodal_kwargs["interleave_mm_strings"])
770
771 # LoRA related configs
772 lora_kwargs = get_kwargs(LoRAConfig)
@@ -981,6 +985,7 @@ class EngineArgs:
985 enable_prompt_embeds=self.enable_prompt_embeds,
986 served_model_name=self.served_model_name,
987 limit_mm_per_prompt=self.limit_mm_per_prompt,
988 + interleave_mm_strings=self.interleave_mm_strings,
989 media_io_kwargs=self.media_io_kwargs,
990 use_async_output_proc=not self.disable_async_output_proc,
991 config_format=self.config_format,
vllm/entrypoints/chat_utils.py
+111 -41
@@ -4,7 +4,7 @@
4 import asyncio
5 import json
6 from abc import ABC, abstractmethod
7 -from collections import defaultdict, deque
7 +from collections import Counter, defaultdict, deque
8 from collections.abc import Awaitable, Iterable
9 from functools import cached_property, lru_cache, partial
10 from pathlib import Path
@@ -52,6 +52,12 @@ from vllm.utils import deprecate_kwargs, random_uuid
52
53 logger = init_logger(__name__)
54
55 +MODALITY_PLACEHOLDERS_MAP = {
56 + "image": "<##IMAGE##>",
57 + "audio": "<##AUDIO##>",
58 + "video": "<##VIDEO##>",
59 +}
60 +
61
62 class AudioURL(TypedDict, total=False):
63 url: Required[str]
@@ -354,6 +360,7 @@ def resolve_mistral_chat_template(
360 "so it will be ignored.")
361 return None
362
363 +
364 @deprecate_kwargs(
365 "trust_remote_code",
366 additional_message="Please use `model_config.trust_remote_code` instead.",
@@ -633,15 +640,22 @@ class BaseMultiModalContentParser(ABC):
640 def __init__(self) -> None:
641 super().__init__()
642
636 - # multimodal placeholder_string : count
637 - self._placeholder_counts: dict[str, int] = defaultdict(lambda: 0)
638 -
639 - def _add_placeholder(self, placeholder: Optional[str]):
643 + # stores model placehodlers list with corresponding
644 + # general MM placeholder:
645 + # {
646 + # "<##IMAGE##>": ["<image>", "<image>", "<image>"],
647 + # "<##AUDIO##>": ["<audio>", "<audio>"]
648 + # }
649 + self._placeholder_storage: dict[str, list] = defaultdict(list)
650 +
651 + def _add_placeholder(self, modality: ModalityStr,
652 + placeholder: Optional[str]):
653 + mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality]
654 if placeholder:
641 - self._placeholder_counts[placeholder] += 1
655 + self._placeholder_storage[mod_placeholder].append(placeholder)
656
643 - def mm_placeholder_counts(self) -> dict[str, int]:
644 - return dict(self._placeholder_counts)
657 + def mm_placeholder_storage(self) -> dict[str, list]:
658 + return dict(self._placeholder_storage)
659
660 @abstractmethod
661 def parse_image(self, image_url: str) -> None:
@@ -685,7 +699,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
699 image = self._connector.fetch_image(image_url)
700
701 placeholder = self._tracker.add("image", image)
688 - self._add_placeholder(placeholder)
702 + self._add_placeholder("image", placeholder)
703
704 def parse_image_embeds(self,
705 image_embeds: Union[str, dict[str, str]]) -> None:
@@ -700,17 +714,17 @@ class MultiModalContentParser(BaseMultiModalContentParser):
714 embedding = self._connector.fetch_image_embedding(image_embeds)
715 placeholder = self._tracker.add("image_embeds", embedding)
716
703 - self._add_placeholder(placeholder)
717 + self._add_placeholder("image", placeholder)
718
719 def parse_image_pil(self, image_pil: Image.Image) -> None:
720 placeholder = self._tracker.add("image", image_pil)
707 - self._add_placeholder(placeholder)
721 + self._add_placeholder("image", placeholder)
722
723 def parse_audio(self, audio_url: str) -> None:
724 audio = self._connector.fetch_audio(audio_url)
725
726 placeholder = self._tracker.add("audio", audio)
713 - self._add_placeholder(placeholder)
727 + self._add_placeholder("audio", placeholder)
728
729 def parse_input_audio(self, input_audio: InputAudio) -> None:
730 audio_data = input_audio.get("data", "")
@@ -723,7 +737,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
737 video = self._connector.fetch_video(video_url=video_url)
738
739 placeholder = self._tracker.add("video", video)
726 - self._add_placeholder(placeholder)
740 + self._add_placeholder("video", placeholder)
741
742
743 class AsyncMultiModalContentParser(BaseMultiModalContentParser):
@@ -741,7 +755,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
755 image_coro = self._connector.fetch_image_async(image_url)
756
757 placeholder = self._tracker.add("image", image_coro)
744 - self._add_placeholder(placeholder)
758 + self._add_placeholder("image", placeholder)
759
760 def parse_image_embeds(self,
761 image_embeds: Union[str, dict[str, str]]) -> None:
@@ -760,20 +774,20 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
774 future.set_result(embedding)
775
776 placeholder = self._tracker.add("image_embeds", future)
763 - self._add_placeholder(placeholder)
777 + self._add_placeholder("image", placeholder)
778
779 def parse_image_pil(self, image_pil: Image.Image) -> None:
780 future: asyncio.Future[Image.Image] = asyncio.Future()
781 future.set_result(image_pil)
782
783 placeholder = self._tracker.add("image", future)
770 - self._add_placeholder(placeholder)
784 + self._add_placeholder("image", placeholder)
785
786 def parse_audio(self, audio_url: str) -> None:
787 audio_coro = self._connector.fetch_audio_async(audio_url)
788
789 placeholder = self._tracker.add("audio", audio_coro)
776 - self._add_placeholder(placeholder)
790 + self._add_placeholder("audio", placeholder)
791
792 def parse_input_audio(self, input_audio: InputAudio) -> None:
793 audio_data = input_audio.get("data", "")
@@ -786,7 +800,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
800 video = self._connector.fetch_video_async(video_url=video_url)
801
802 placeholder = self._tracker.add("video", video)
789 - self._add_placeholder(placeholder)
803 + self._add_placeholder("video", placeholder)
804
805
806 def validate_chat_template(chat_template: Optional[Union[Path, str]]):
@@ -856,12 +870,40 @@ def load_chat_template(
870 return _cached_load_chat_template(chat_template, is_literal=is_literal)
871
872
873 +def _get_interleaved_text_prompt(placeholder_storage: dict[str, list],
874 + texts: list[str]) -> str:
875 + for idx, elem in enumerate(texts):
876 + if elem in placeholder_storage:
877 + texts[idx] = placeholder_storage[elem].pop(0)
878 +
879 + return "\n".join(texts)
880 +
881 +
882 # TODO: Let user specify how to insert multimodal tokens into prompt
883 # (similar to chat template)
861 -def _get_full_multimodal_text_prompt(placeholder_counts: dict[str, int],
862 - text_prompt: str) -> str:
884 +def _get_full_multimodal_text_prompt(placeholder_storage: dict[str, list],
885 + texts: list[str],
886 + interleave_strings: bool
887 + ) -> str:
888 """Combine multimodal prompts for a multimodal language model."""
889
890 + # flatten storage to make it looks like
891 + # {
892 + # "<|image|>": 2,
893 + # "<|audio|>": 1
894 + # }
895 + placeholder_counts = Counter(
896 + [v for elem in placeholder_storage.values() for v in elem]
897 + )
898 +
899 + if interleave_strings:
900 + text_prompt = _get_interleaved_text_prompt(placeholder_storage, texts)
901 + else:
902 + text_prompt = "\n".join(texts)
903 +
904 + # Pass interleaved text further in case the user used image placeholders
905 + # himself, but forgot to disable the 'interleave_strings' flag
906 +
907 # Look through the text prompt to check for missing placeholders
908 missing_placeholders: list[str] = []
909 for placeholder in placeholder_counts:
@@ -870,6 +912,13 @@ def _get_full_multimodal_text_prompt(placeholder_counts: dict[str, int],
912 placeholder_counts[placeholder] -= text_prompt.count(placeholder)
913
914 if placeholder_counts[placeholder] < 0:
915 + logger.error(
916 + "Placeholder count is negative! "
917 + "Ensure that the 'interleave_strings' flag is disabled "
918 + "(current value: %s) "
919 + "when manually placing image placeholders.", interleave_strings
920 + )
921 + logger.debug("Input prompt: %s", text_prompt)
922 raise ValueError(
923 f"Found more '{placeholder}' placeholders in input prompt than "
924 "actual multimodal data items.")
@@ -877,8 +926,8 @@ def _get_full_multimodal_text_prompt(placeholder_counts: dict[str, int],
926 missing_placeholders.extend([placeholder] *
927 placeholder_counts[placeholder])
928
880 - # NOTE: For now we always add missing placeholders at the front of
881 - # the prompt. This may change to be customizable in the future.
929 + # NOTE: Default behaviour: we always add missing placeholders
930 + # at the front of the prompt, if interleave_strings=False
931 return "\n".join(missing_placeholders + [text_prompt])
932
933
@@ -988,6 +1037,7 @@ def _parse_chat_message_content_parts(
1037 mm_tracker: BaseMultiModalItemTracker,
1038 *,
1039 wrap_dicts: bool,
1040 + interleave_strings: bool,
1041 ) -> list[ConversationMessage]:
1042 content = list[_ContentPart]()
1043
@@ -998,6 +1048,7 @@ def _parse_chat_message_content_parts(
1048 part,
1049 mm_parser,
1050 wrap_dicts=wrap_dicts,
1051 + interleave_strings=interleave_strings
1052 )
1053 if parse_res:
1054 content.append(parse_res)
@@ -1007,11 +1058,14 @@ def _parse_chat_message_content_parts(
1058 return [ConversationMessage(role=role,
1059 content=content)] # type: ignore
1060 texts = cast(list[str], content)
1010 - text_prompt = "\n".join(texts)
1011 - mm_placeholder_counts = mm_parser.mm_placeholder_counts()
1012 - if mm_placeholder_counts:
1013 - text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
1014 - text_prompt)
1061 + mm_placeholder_storage = mm_parser.mm_placeholder_storage()
1062 + if mm_placeholder_storage:
1063 + text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_storage,
1064 + texts,
1065 + interleave_strings)
1066 + else:
1067 + text_prompt = "\n".join(texts)
1068 +
1069 return [ConversationMessage(role=role, content=text_prompt)]
1070
1071
@@ -1020,6 +1074,7 @@ def _parse_chat_message_content_part(
1074 mm_parser: BaseMultiModalContentParser,
1075 *,
1076 wrap_dicts: bool,
1077 + interleave_strings: bool,
1078 ) -> Optional[_ContentPart]:
1079 """Parses a single part of a conversation. If wrap_dicts is True,
1080 structured dictionary pieces for texts and images will be
@@ -1049,34 +1104,37 @@ def _parse_chat_message_content_part(
1104 else:
1105 return str_content
1106
1107 + modality = None
1108 if part_type == "image_pil":
1109 image_content = cast(Image.Image, content)
1110 mm_parser.parse_image_pil(image_content)
1055 - return {'type': 'image'} if wrap_dicts else None
1056 - if part_type == "image_url":
1111 + modality = "image"
1112 + elif part_type == "image_url":
1113 str_content = cast(str, content)
1114 mm_parser.parse_image(str_content)
1059 - return {'type': 'image'} if wrap_dicts else None
1060 - if part_type == "image_embeds":
1115 + modality = "image"
1116 + elif part_type == "image_embeds":
1117 content = cast(Union[str, dict[str, str]], content)
1118 mm_parser.parse_image_embeds(content)
1063 - return {'type': 'image'} if wrap_dicts else None
1064 - if part_type == "audio_url":
1119 + modality = "image"
1120 + elif part_type == "audio_url":
1121 str_content = cast(str, content)
1122 mm_parser.parse_audio(str_content)
1067 - return {'type': 'audio'} if wrap_dicts else None
1068 -
1069 - if part_type == "input_audio":
1123 + modality = "audio"
1124 + elif part_type == "input_audio":
1125 dict_content = cast(InputAudio, content)
1126 mm_parser.parse_input_audio(dict_content)
1072 - return {'type': 'audio'} if wrap_dicts else None
1073 -
1074 - if part_type == "video_url":
1127 + modality = "audio"
1128 + elif part_type == "video_url":
1129 str_content = cast(str, content)
1130 mm_parser.parse_video(str_content)
1077 - return {'type': 'video'} if wrap_dicts else None
1131 + modality = "video"
1132 + else:
1133 + raise NotImplementedError(f"Unknown part type: {part_type}")
1134
1079 - raise NotImplementedError(f"Unknown part type: {part_type}")
1135 + return {'type': modality} if wrap_dicts else (
1136 + MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None
1137 + )
1138
1139
1140 # No need to validate using Pydantic again
@@ -1088,6 +1146,7 @@ def _parse_chat_message_content(
1146 message: ChatCompletionMessageParam,
1147 mm_tracker: BaseMultiModalItemTracker,
1148 content_format: _ChatTemplateContentFormat,
1149 + interleave_strings: bool,
1150 ) -> list[ConversationMessage]:
1151 role = message["role"]
1152 content = message.get("content")
@@ -1103,6 +1162,7 @@ def _parse_chat_message_content(
1162 content, # type: ignore
1163 mm_tracker,
1164 wrap_dicts=(content_format == "openai"),
1165 + interleave_strings=interleave_strings,
1166 )
1167
1168 for result_msg in result:
@@ -1155,6 +1215,11 @@ def parse_chat_messages(
1215 msg,
1216 mm_tracker,
1217 content_format,
1218 + interleave_strings=(
1219 + content_format == "string"
1220 + and model_config.multimodal_config is not None
1221 + and model_config.multimodal_config.interleave_mm_strings
1222 + )
1223 )
1224
1225 conversation.extend(sub_messages)
@@ -1178,6 +1243,11 @@ def parse_chat_messages_futures(
1243 msg,
1244 mm_tracker,
1245 content_format,
1246 + interleave_strings=(
1247 + content_format == "string"
1248 + and model_config.multimodal_config is not None
1249 + and model_config.multimodal_config.interleave_mm_strings
1250 + )
1251 )
1252
1253 conversation.extend(sub_messages)