@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c5eea3c8ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend] Support simpler image input format (#9478)

Yue Zhang committed Oct 18, 2024 at 23:17 UTC c5eea3c8ba7586e54f87b53a104cf2ac0f75069c
2 files changed +140 -25
tests/entrypoints/test_chat_utils.py
+26
@@ -388,3 +388,29 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
388 "text": "What about these two?"
389 }]
390 }], phi3v_model_config, phi3v_tokenizer)
391 +
392 +
393 +def test_parse_chat_messages_multiple_images_uncommon_input(
394 + phi3v_model_config,
395 + phi3v_tokenizer,
396 + image_url,
397 +):
398 + conversation, mm_data = parse_chat_messages([{
399 + "role":
400 + "user",
401 + "content": [
402 + "What's in these images?", {
403 + "image_url": image_url
404 + }, {
405 + "image_url": image_url
406 + }
407 + ]
408 + }], phi3v_model_config, phi3v_tokenizer)
409 +
410 + assert conversation == [{
411 + "role":
412 + "user",
413 + "content":
414 + "<|image_1|>\n<|image_2|>\nWhat's in these images?"
415 + }]
416 + _assert_mm_data_is_image_input(mm_data, 2)
vllm/entrypoints/chat_utils.py
+114 -25
@@ -5,8 +5,8 @@ from abc import ABC, abstractmethod
5 from collections import defaultdict
6 from functools import lru_cache, partial
7 from pathlib import Path
8 -from typing import (Any, Awaitable, Dict, Generic, Iterable, List, Literal,
9 - Mapping, Optional, Tuple, TypeVar, Union, cast)
8 +from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,
9 + Literal, Mapping, Optional, Tuple, TypeVar, Union, cast)
10
11 # yapf conflicts with isort for this block
12 # yapf: disable
@@ -59,10 +59,35 @@ class CustomChatCompletionContentPartParam(TypedDict, total=False):
59 """The type of the content part."""
60
61
62 +class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
63 + """A simpler version of the param that only accepts a plain image_url.
64 + This is supported by OpenAI API, although it is not documented.
65 +
66 + Example:
67 + {
68 + "image_url": "https://example.com/image.jpg"
69 + }
70 + """
71 + image_url: Required[str]
72 +
73 +
74 +class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
75 + """A simpler version of the param that only accepts a plain audio_url.
76 +
77 + Example:
78 + {
79 + "audio_url": "https://example.com/audio.mp3"
80 + }
81 + """
82 + audio_url: Required[str]
83 +
84 +
85 ChatCompletionContentPartParam: TypeAlias = Union[
86 OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
87 ChatCompletionContentPartRefusalParam,
65 - CustomChatCompletionContentPartParam]
88 + CustomChatCompletionContentPartParam,
89 + CustomChatCompletionContentSimpleImageParam,
90 + CustomChatCompletionContentSimpleAudioParam, str]
91
92
93 class CustomChatCompletionMessageParam(TypedDict, total=False):
@@ -387,6 +412,71 @@ _AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
412 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
413 MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
414
415 +# Define a mapping from part types to their corresponding parsing functions.
416 +MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
417 + "text":
418 + lambda part: _TextParser(part).get("text", ""),
419 + "image_url":
420 + lambda part: _ImageParser(part).get("image_url", {}).get("url", ""),
421 + "audio_url":
422 + lambda part: _AudioParser(part).get("audio_url", {}).get("url", ""),
423 + "refusal":
424 + lambda part: _RefusalParser(part).get("refusal", ""),
425 +}
426 +
427 +
428 +def _parse_chat_message_content_mm_part(
429 + part: ChatCompletionContentPartParam) -> Tuple[str, str]:
430 + """
431 + Parses a given multi modal content part based on its type.
432 +
433 + Args:
434 + part: A dict containing the content part, with a potential 'type' field.
435 +
436 + Returns:
437 + A tuple (part_type, content) where:
438 + - part_type: Type of the part (e.g., 'text', 'image_url').
439 + - content: Parsed content (e.g., text, image URL).
440 +
441 + Raises:
442 + ValueError: If the 'type' field is missing and no direct URL is found.
443 + """
444 + assert isinstance(
445 + part, dict) # This is needed to avoid mypy errors: part.get() from str
446 + part_type = part.get("type", None)
447 +
448 + if isinstance(part_type, str) and part_type in MM_PARSER_MAP:
449 + content = MM_PARSER_MAP[part_type](part)
450 +
451 + # Special case for 'image_url.detail'
452 + if part_type == "image_url" and part.get("detail") != "auto":
453 + logger.warning("'image_url.detail' is currently not supported "
454 + "and will be ignored.")
455 +
456 + return part_type, content
457 +
458 + # Handle missing 'type' but provided direct URL fields.
459 + if part_type is None:
460 + if part.get("image_url") is not None:
461 + image_params = cast(CustomChatCompletionContentSimpleImageParam,
462 + part)
463 + return "image_url", image_params.get("image_url", "")
464 + if part.get("audio_url") is not None:
465 + audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
466 + part)
467 + return "audio_url", audio_params.get("audio_url", "")
468 +
469 + # Raise an error if no 'type' or direct URL is found.
470 + raise ValueError("Missing 'type' field in multimodal part.")
471 +
472 + if not isinstance(part_type, str):
473 + raise ValueError("Invalid 'type' field in multimodal part.")
474 + return part_type, "unknown part_type content"
475 +
476 +
477 +VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
478 + "audio_url")
479 +
480
481 def _parse_chat_message_content_parts(
482 role: str,
@@ -402,29 +492,28 @@ def _parse_chat_message_content_parts(
492
493 has_image = False
494 for part in parts:
405 - part_type = part["type"]
406 - if part_type == "text":
407 - text = _TextParser(part)["text"]
495 + if isinstance(part, str): # Handle plain text parts
496 + text = _TextParser(part)
497 texts.append(text)
409 - elif part_type == "image_url":
410 - image_url = _ImageParser(part)["image_url"]
411 -
412 - if image_url.get("detail", "auto") != "auto":
413 - logger.warning(
414 - "'image_url.detail' is currently not supported and "
415 - "will be ignored.")
416 -
417 - mm_parser.parse_image(image_url["url"])
418 - has_image = True
419 - elif part_type == "audio_url":
420 - audio_url = _AudioParser(part)["audio_url"]
421 -
422 - mm_parser.parse_audio(audio_url["url"])
423 - elif part_type == "refusal":
424 - text = _RefusalParser(part)["refusal"]
425 - texts.append(text)
426 - else:
427 - raise NotImplementedError(f"Unknown part type: {part_type}")
498 + else: # Handle structured dictionary parts
499 + part_type, content = _parse_chat_message_content_mm_part(part)
500 +
501 + # if part_type is text/refusal/image_url/audio_url but
502 + # content is empty, logg a warning and skip
503 + if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
504 + logger.warning("Skipping multimodal part "
505 + "with empty / unparsable content.")
506 + continue
507 +
508 + if part_type in ("text", "refusal"):
509 + texts.append(content)
510 + elif part_type == "image_url":
511 + mm_parser.parse_image(content)
512 + has_image = True
513 + elif part_type == "audio_url":
514 + mm_parser.parse_audio(content)
515 + else:
516 + raise NotImplementedError(f"Unknown part type: {part_type}")
517
518 text_prompt = "\n".join(texts)
519 if keep_multimodal_content: