A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Support simpler image input format (#9478)
Yue Zhang committed
Oct 18, 2024 at 23:17 UTC
c5eea3c8ba7586e54f87b53a104cf2ac0f75069c
2 files changed
+140
-25
tests/entrypoints/test_chat_utils.py
+26
@@ -388,3 +388,29 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
388
"text": "What about these two?"
389
}]
390
}], phi3v_model_config, phi3v_tokenizer)
391
+
392
+
393
+def test_parse_chat_messages_multiple_images_uncommon_input(
394
+ phi3v_model_config,
395
+ phi3v_tokenizer,
396
+ image_url,
397
+):
398
+ conversation, mm_data = parse_chat_messages([{
399
+ "role":
400
+ "user",
401
+ "content": [
402
+ "What's in these images?", {
403
+ "image_url": image_url
404
+ }, {
405
+ "image_url": image_url
406
+ }
407
+ ]
408
+ }], phi3v_model_config, phi3v_tokenizer)
409
+
410
+ assert conversation == [{
411
+ "role":
412
+ "user",
413
+ "content":
414
+ "<|image_1|>\n<|image_2|>\nWhat's in these images?"
415
+ }]
416
+ _assert_mm_data_is_image_input(mm_data, 2)
vllm/entrypoints/chat_utils.py
+114
-25
@@ -5,8 +5,8 @@ from abc import ABC, abstractmethod
5
from collections import defaultdict
6
from functools import lru_cache, partial
7
from pathlib import Path
8
-from typing import (Any, Awaitable, Dict, Generic, Iterable, List, Literal,
9
- Mapping, Optional, Tuple, TypeVar, Union, cast)
8
+from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,
9
+ Literal, Mapping, Optional, Tuple, TypeVar, Union, cast)
10
11
# yapf conflicts with isort for this block
12
# yapf: disable
@@ -59,10 +59,35 @@ class CustomChatCompletionContentPartParam(TypedDict, total=False):
59
"""The type of the content part."""
60
61
62
+class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
63
+ """A simpler version of the param that only accepts a plain image_url.
64
+ This is supported by OpenAI API, although it is not documented.
65
+
66
+ Example:
67
+ {
68
+ "image_url": "https://example.com/image.jpg"
69
+ }
70
+ """
71
+ image_url: Required[str]
72
+
73
+
74
+class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
75
+ """A simpler version of the param that only accepts a plain audio_url.
76
+
77
+ Example:
78
+ {
79
+ "audio_url": "https://example.com/audio.mp3"
80
+ }
81
+ """
82
+ audio_url: Required[str]
83
+
84
+
85
ChatCompletionContentPartParam: TypeAlias = Union[
86
OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
87
ChatCompletionContentPartRefusalParam,
65
- CustomChatCompletionContentPartParam]
88
+ CustomChatCompletionContentPartParam,
89
+ CustomChatCompletionContentSimpleImageParam,
90
+ CustomChatCompletionContentSimpleAudioParam, str]
91
92
93
class CustomChatCompletionMessageParam(TypedDict, total=False):
@@ -387,6 +412,71 @@ _AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
412
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
413
MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
414
415
+# Define a mapping from part types to their corresponding parsing functions.
416
+MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
417
+ "text":
418
+ lambda part: _TextParser(part).get("text", ""),
419
+ "image_url":
420
+ lambda part: _ImageParser(part).get("image_url", {}).get("url", ""),
421
+ "audio_url":
422
+ lambda part: _AudioParser(part).get("audio_url", {}).get("url", ""),
423
+ "refusal":
424
+ lambda part: _RefusalParser(part).get("refusal", ""),
425
+}
426
+
427
+
428
+def _parse_chat_message_content_mm_part(
429
+ part: ChatCompletionContentPartParam) -> Tuple[str, str]:
430
+ """
431
+ Parses a given multi modal content part based on its type.
432
+
433
+ Args:
434
+ part: A dict containing the content part, with a potential 'type' field.
435
+
436
+ Returns:
437
+ A tuple (part_type, content) where:
438
+ - part_type: Type of the part (e.g., 'text', 'image_url').
439
+ - content: Parsed content (e.g., text, image URL).
440
+
441
+ Raises:
442
+ ValueError: If the 'type' field is missing and no direct URL is found.
443
+ """
444
+ assert isinstance(
445
+ part, dict) # This is needed to avoid mypy errors: part.get() from str
446
+ part_type = part.get("type", None)
447
+
448
+ if isinstance(part_type, str) and part_type in MM_PARSER_MAP:
449
+ content = MM_PARSER_MAP[part_type](part)
450
+
451
+ # Special case for 'image_url.detail'
452
+ if part_type == "image_url" and part.get("detail") != "auto":
453
+ logger.warning("'image_url.detail' is currently not supported "
454
+ "and will be ignored.")
455
+
456
+ return part_type, content
457
+
458
+ # Handle missing 'type' but provided direct URL fields.
459
+ if part_type is None:
460
+ if part.get("image_url") is not None:
461
+ image_params = cast(CustomChatCompletionContentSimpleImageParam,
462
+ part)
463
+ return "image_url", image_params.get("image_url", "")
464
+ if part.get("audio_url") is not None:
465
+ audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
466
+ part)
467
+ return "audio_url", audio_params.get("audio_url", "")
468
+
469
+ # Raise an error if no 'type' or direct URL is found.
470
+ raise ValueError("Missing 'type' field in multimodal part.")
471
+
472
+ if not isinstance(part_type, str):
473
+ raise ValueError("Invalid 'type' field in multimodal part.")
474
+ return part_type, "unknown part_type content"
475
+
476
+
477
+VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
478
+ "audio_url")
479
+
480
481
def _parse_chat_message_content_parts(
482
role: str,
@@ -402,29 +492,28 @@ def _parse_chat_message_content_parts(
492
493
has_image = False
494
for part in parts:
405
- part_type = part["type"]
406
- if part_type == "text":
407
- text = _TextParser(part)["text"]
495
+ if isinstance(part, str): # Handle plain text parts
496
+ text = _TextParser(part)
497
texts.append(text)
409
- elif part_type == "image_url":
410
- image_url = _ImageParser(part)["image_url"]
411
-
412
- if image_url.get("detail", "auto") != "auto":
413
- logger.warning(
414
- "'image_url.detail' is currently not supported and "
415
- "will be ignored.")
416
-
417
- mm_parser.parse_image(image_url["url"])
418
- has_image = True
419
- elif part_type == "audio_url":
420
- audio_url = _AudioParser(part)["audio_url"]
421
-
422
- mm_parser.parse_audio(audio_url["url"])
423
- elif part_type == "refusal":
424
- text = _RefusalParser(part)["refusal"]
425
- texts.append(text)
426
- else:
427
- raise NotImplementedError(f"Unknown part type: {part_type}")
498
+ else: # Handle structured dictionary parts
499
+ part_type, content = _parse_chat_message_content_mm_part(part)
500
+
501
+ # if part_type is text/refusal/image_url/audio_url but
502
+ # content is empty, logg a warning and skip
503
+ if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
504
+ logger.warning("Skipping multimodal part "
505
+ "with empty / unparsable content.")
506
+ continue
507
+
508
+ if part_type in ("text", "refusal"):
509
+ texts.append(content)
510
+ elif part_type == "image_url":
511
+ mm_parser.parse_image(content)
512
+ has_image = True
513
+ elif part_type == "audio_url":
514
+ mm_parser.parse_audio(content)
515
+ else:
516
+ raise NotImplementedError(f"Unknown part type: {part_type}")
517
518
text_prompt = "\n".join(texts)
519
if keep_multimodal_content: