A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Bugfix] Do not skip "empty" parts of chats that are parsable (#16219)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Apr 7, 2025 at 23:14 UTC
b99733d0929aba5ec4f523885d9b417d50b90fc2
2 files changed
+98
-13
tests/entrypoints/test_chat_utils.py
+87
-2
@@ -30,6 +30,7 @@ QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
30
MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
31
LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
32
HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
33
+MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
34
35
36
@pytest.fixture(scope="function")
@@ -80,6 +81,30 @@ def mllama_tokenizer():
81
)
82
83
84
+@pytest.fixture(scope="function")
85
+def mistral_model_config():
86
+ return ModelConfig(MISTRAL_MODEL_ID,
87
+ task="generate",
88
+ tokenizer=MISTRAL_MODEL_ID,
89
+ tokenizer_mode="auto",
90
+ trust_remote_code=True,
91
+ dtype="auto",
92
+ seed=0,
93
+ limit_mm_per_prompt={
94
+ "image": 2,
95
+ })
96
+
97
+
98
+@pytest.fixture(scope="module")
99
+def mistral_tokenizer():
100
+ return TokenizerGroup(
101
+ tokenizer_id=MISTRAL_MODEL_ID,
102
+ enable_lora=False,
103
+ max_num_seqs=5,
104
+ max_input_length=None,
105
+ )
106
+
107
+
108
@pytest.fixture(scope="module")
109
def image_url():
110
image = ImageAsset('cherry_blossom')
@@ -131,6 +156,66 @@ def test_parse_chat_messages_single_image(
156
_assert_mm_data_is_image_input(mm_data, 1)
157
158
159
+def test_parse_chat_messages_empty_system(
160
+ mistral_model_config,
161
+ mistral_tokenizer,
162
+):
163
+ # Test string format
164
+ conversation, _ = parse_chat_messages(
165
+ [{
166
+ "role": "system",
167
+ "content": ""
168
+ }, {
169
+ "role": "user",
170
+ "content": [{
171
+ "type": "text",
172
+ "text": "Who are you?"
173
+ }]
174
+ }],
175
+ mistral_model_config,
176
+ mistral_tokenizer,
177
+ content_format="string",
178
+ )
179
+ assert conversation == [{
180
+ "role": "system",
181
+ "content": ""
182
+ }, {
183
+ "role": "user",
184
+ "content": "Who are you?"
185
+ }]
186
+
187
+ # Test openai format
188
+ conversation, _ = parse_chat_messages(
189
+ [{
190
+ "role": "system",
191
+ "content": ""
192
+ }, {
193
+ "role": "user",
194
+ "content": [{
195
+ "type": "text",
196
+ "text": "Who are you?"
197
+ }]
198
+ }],
199
+ mistral_model_config,
200
+ mistral_tokenizer,
201
+ content_format="openai",
202
+ )
203
+ assert conversation == [{
204
+ "role": "system",
205
+ "content": [{
206
+ "type": "text",
207
+ "text": ""
208
+ }]
209
+ }, {
210
+ "role":
211
+ "user",
212
+ "content": [{
213
+ "type": "text",
214
+ "text": "Who are you?"
215
+ }]
216
+ }]
217
+
218
+
219
@pytest.mark.asyncio
220
async def test_parse_chat_messages_single_image_async(
221
phi3v_model_config,
@@ -671,7 +756,7 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
756
# Build a config for the model
757
model_config = ModelConfig(model,
758
task="generate",
674
- tokenizer=MLLAMA_MODEL_ID,
759
+ tokenizer=model,
760
tokenizer_mode="auto",
761
trust_remote_code=True,
762
dtype="auto",
@@ -682,7 +767,7 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
767
768
# Build the tokenizer group and grab the underlying tokenizer
769
tokenizer_group = TokenizerGroup(
685
- MLLAMA_MODEL_ID,
770
+ model,
771
enable_lora=False,
772
max_num_seqs=5,
773
max_input_length=None,
vllm/entrypoints/chat_utils.py
+11
-11
@@ -872,19 +872,19 @@ MM_PARSER_MAP: dict[
872
Callable[[ChatCompletionContentPartParam], _ContentPart],
873
] = {
874
"text":
875
- lambda part: _TextParser(part).get("text", ""),
875
+ lambda part: _TextParser(part).get("text", None),
876
"image_url":
877
- lambda part: _ImageParser(part).get("image_url", {}).get("url", ""),
877
+ lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
878
"image_embeds":
879
- lambda part: _ImageEmbedsParser(part).get("image_embeds", {}),
879
+ lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
880
"audio_url":
881
- lambda part: _AudioParser(part).get("audio_url", {}).get("url", ""),
881
+ lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
882
"input_audio":
883
- lambda part: _InputAudioParser(part).get("input_audio", {}),
883
+ lambda part: _InputAudioParser(part).get("input_audio", None),
884
"refusal":
885
- lambda part: _RefusalParser(part).get("refusal", ""),
885
+ lambda part: _RefusalParser(part).get("refusal", None),
886
"video_url":
887
- lambda part: _VideoParser(part).get("video_url", {}).get("url", ""),
887
+ lambda part: _VideoParser(part).get("video_url", {}).get("url", None),
888
}
889
890
@@ -1003,11 +1003,11 @@ def _parse_chat_message_content_part(
1003
part_type, content = _parse_chat_message_content_mm_part(part)
1004
1005
# if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
1006
- # content is empty, log a warning and skip
1007
- if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
1006
+ # content is None, log a warning and skip
1007
+ if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and content is None:
1008
logger.warning(
1009
- "Skipping multimodal part (type: '%s') "
1010
- "with empty / unparsable content.", part_type)
1009
+ "Skipping multimodal part '%s' (type: '%s') "
1010
+ "with empty / unparsable content.", part, part_type)
1011
return None
1012
1013
if part_type in ("text", "refusal"):