@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7af553ea30
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Misc] Abstract the logic for reading and writing media content (#11527)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 27, 2024 at 19:21 UTC 7af553ea30031446b4c1c74ad83187f9fd3de4e7
10 files changed +493 -387
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -33,6 +33,7 @@ class MockModelConfig:
33 hf_config = MockHFConfig()
34 logits_processor_pattern = None
35 diff_sampling_param: Optional[dict] = None
36 + allowed_local_media_path: str = ""
37
38 def get_diff_sampling_param(self):
39 return self.diff_sampling_param or {}
tests/entrypoints/test_chat_utils.py
+1 -5
@@ -2,7 +2,6 @@ import warnings
2 from typing import Optional
3
4 import pytest
5 -from PIL import Image
5
6 from vllm.assets.image import ImageAsset
7 from vllm.config import ModelConfig
@@ -91,10 +90,7 @@ def _assert_mm_data_is_image_input(
90 image_data = mm_data.get("image")
91 assert image_data is not None
92
94 - if image_count == 1:
95 - assert isinstance(image_data, Image.Image)
96 - else:
97 - assert isinstance(image_data, list) and len(image_data) == image_count
93 + assert isinstance(image_data, list) and len(image_data) == image_count
94
95
96 def test_parse_chat_messages_single_image(
tests/multimodal/test_utils.py
+34 -25
@@ -9,7 +9,7 @@ import pytest
9 from PIL import Image, ImageChops
10 from transformers import AutoConfig, AutoTokenizer
11
12 -from vllm.multimodal.utils import (async_fetch_image, fetch_image,
12 +from vllm.multimodal.utils import (MediaConnector,
13 repeat_and_pad_placeholder_tokens)
14
15 # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
@@ -23,7 +23,12 @@ TEST_IMAGE_URLS = [
23
24 @pytest.fixture(scope="module")
25 def url_images() -> Dict[str, Image.Image]:
26 - return {image_url: fetch_image(image_url) for image_url in TEST_IMAGE_URLS}
26 + connector = MediaConnector()
27 +
28 + return {
29 + image_url: connector.fetch_image(image_url)
30 + for image_url in TEST_IMAGE_URLS
31 + }
32
33
34 def get_supported_suffixes() -> Tuple[str, ...]:
@@ -43,8 +48,10 @@ def _image_equals(a: Image.Image, b: Image.Image) -> bool:
48 @pytest.mark.asyncio
49 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
50 async def test_fetch_image_http(image_url: str):
46 - image_sync = fetch_image(image_url)
47 - image_async = await async_fetch_image(image_url)
51 + connector = MediaConnector()
52 +
53 + image_sync = connector.fetch_image(image_url)
54 + image_async = await connector.fetch_image_async(image_url)
55 assert _image_equals(image_sync, image_async)
56
57
@@ -53,6 +60,7 @@ async def test_fetch_image_http(image_url: str):
60 @pytest.mark.parametrize("suffix", get_supported_suffixes())
61 async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
62 image_url: str, suffix: str):
63 + connector = MediaConnector()
64 url_image = url_images[image_url]
65
66 try:
@@ -75,48 +83,49 @@ async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
83 base64_image = base64.b64encode(f.read()).decode("utf-8")
84 data_url = f"data:{mime_type};base64,{base64_image}"
85
78 - data_image_sync = fetch_image(data_url)
86 + data_image_sync = connector.fetch_image(data_url)
87 if _image_equals(url_image, Image.open(f)):
88 assert _image_equals(url_image, data_image_sync)
89 else:
90 pass # Lossy format; only check that image can be opened
91
84 - data_image_async = await async_fetch_image(data_url)
92 + data_image_async = await connector.fetch_image_async(data_url)
93 assert _image_equals(data_image_sync, data_image_async)
94
95
96 @pytest.mark.asyncio
97 @pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
98 async def test_fetch_image_local_files(image_url: str):
99 + connector = MediaConnector()
100 +
101 with TemporaryDirectory() as temp_dir:
92 - origin_image = fetch_image(image_url)
102 + local_connector = MediaConnector(allowed_local_media_path=temp_dir)
103 +
104 + origin_image = connector.fetch_image(image_url)
105 origin_image.save(os.path.join(temp_dir, os.path.basename(image_url)),
106 quality=100,
107 icc_profile=origin_image.info.get('icc_profile'))
108
97 - image_async = await async_fetch_image(
98 - f"file://{temp_dir}/{os.path.basename(image_url)}",
99 - allowed_local_media_path=temp_dir)
100 -
101 - image_sync = fetch_image(
102 - f"file://{temp_dir}/{os.path.basename(image_url)}",
103 - allowed_local_media_path=temp_dir)
109 + image_async = await local_connector.fetch_image_async(
110 + f"file://{temp_dir}/{os.path.basename(image_url)}")
111 + image_sync = local_connector.fetch_image(
112 + f"file://{temp_dir}/{os.path.basename(image_url)}")
113 # Check that the images are equal
114 assert not ImageChops.difference(image_sync, image_async).getbbox()
115
107 - with pytest.raises(ValueError):
108 - await async_fetch_image(
109 - f"file://{temp_dir}/../{os.path.basename(image_url)}",
110 - allowed_local_media_path=temp_dir)
111 - with pytest.raises(ValueError):
112 - await async_fetch_image(
116 + with pytest.raises(ValueError, match="must be a subpath"):
117 + await local_connector.fetch_image_async(
118 + f"file://{temp_dir}/../{os.path.basename(image_url)}")
119 + with pytest.raises(RuntimeError, match="Cannot load local files"):
120 + await connector.fetch_image_async(
121 f"file://{temp_dir}/../{os.path.basename(image_url)}")
122
115 - with pytest.raises(ValueError):
116 - fetch_image(f"file://{temp_dir}/../{os.path.basename(image_url)}",
117 - allowed_local_media_path=temp_dir)
118 - with pytest.raises(ValueError):
119 - fetch_image(f"file://{temp_dir}/../{os.path.basename(image_url)}")
123 + with pytest.raises(ValueError, match="must be a subpath"):
124 + local_connector.fetch_image(
125 + f"file://{temp_dir}/../{os.path.basename(image_url)}")
126 + with pytest.raises(RuntimeError, match="Cannot load local files"):
127 + connector.fetch_image(
128 + f"file://{temp_dir}/../{os.path.basename(image_url)}")
129
130
131 @pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-mistral-7b-hf"])
vllm/assets/audio.py
+2 -4
@@ -21,12 +21,10 @@ class AudioAsset:
21 name: Literal["winning_call", "mary_had_lamb"]
22
23 @property
24 - def audio_and_sample_rate(self) -> tuple[npt.NDArray, int]:
24 + def audio_and_sample_rate(self) -> tuple[npt.NDArray, float]:
25 audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg",
26 s3_prefix=ASSET_DIR)
27 - y, sr = librosa.load(audio_path, sr=None)
28 - assert isinstance(sr, int)
29 - return y, sr
27 + return librosa.load(audio_path, sr=None)
28
29 @property
30 def url(self) -> str:
vllm/entrypoints/chat_utils.py
+57 -72
@@ -6,7 +6,7 @@ from collections import defaultdict, deque
6 from functools import lru_cache, partial
7 from pathlib import Path
8 from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,
9 - Literal, Mapping, Optional, Tuple, TypeVar, Union, cast)
9 + Literal, Optional, Tuple, TypeVar, Union, cast)
10
11 import jinja2.nodes
12 import transformers.utils.chat_template_utils as hf_chat_utils
@@ -23,6 +23,8 @@ from openai.types.chat import (
23 ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
24 from openai.types.chat import (ChatCompletionMessageToolCallParam,
25 ChatCompletionToolMessageParam)
26 +from openai.types.chat.chat_completion_content_part_input_audio_param import (
27 + InputAudio)
28 # yapf: enable
29 # pydantic needs the TypedDict from typing_extensions
30 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
@@ -31,11 +33,7 @@ from typing_extensions import Required, TypeAlias, TypedDict
33 from vllm.config import ModelConfig
34 from vllm.logger import init_logger
35 from vllm.multimodal import MultiModalDataDict
34 -from vllm.multimodal.utils import (async_get_and_parse_audio,
35 - async_get_and_parse_image,
36 - async_get_and_parse_video,
37 - get_and_parse_audio, get_and_parse_image,
38 - get_and_parse_video)
36 +from vllm.multimodal.utils import MediaConnector
37 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
38 from vllm.utils import print_warning_once
39
@@ -368,14 +366,17 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
366 self._tokenizer = tokenizer
367 self._allowed_items = (model_config.multimodal_config.limit_per_prompt
368 if model_config.multimodal_config else {})
371 - self._consumed_items = {k: 0 for k in self._allowed_items}
369
373 - self._items: List[_T] = []
370 + self._items_by_modality = defaultdict[str, list[_T]](list)
371
372 @property
373 def model_config(self) -> ModelConfig:
374 return self._model_config
375
376 + @property
377 + def allowed_local_media_path(self):
378 + return self._model_config.allowed_local_media_path
379 +
380 @staticmethod
381 @lru_cache(maxsize=None)
382 def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
@@ -435,38 +436,19 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
436 else:
437 raise TypeError(f"Unknown modality: {modality}")
438
438 - @staticmethod
439 - def _combine(items: List[MultiModalDataDict]) -> MultiModalDataDict:
440 - mm_lists: Mapping[str, List[object]] = defaultdict(list)
441 -
442 - # Merge all the multi-modal items
443 - for single_mm_data in items:
444 - for mm_key, mm_item in single_mm_data.items():
445 - if isinstance(mm_item, list):
446 - mm_lists[mm_key].extend(mm_item)
447 - else:
448 - mm_lists[mm_key].append(mm_item)
449 -
450 - # Unpack any single item lists for models that don't expect multiple.
451 - return {
452 - mm_key: mm_list[0] if len(mm_list) == 1 else mm_list
453 - for mm_key, mm_list in mm_lists.items()
454 - }
455 -
439 def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
440 """
441 Add a multi-modal item to the current prompt and returns the
442 placeholder string to use, if any.
443 """
444 allowed_count = self._allowed_items.get(modality, 1)
462 - current_count = self._consumed_items.get(modality, 0) + 1
445 + current_count = len(self._items_by_modality[modality]) + 1
446 if current_count > allowed_count:
447 raise ValueError(
448 f"At most {allowed_count} {modality}(s) may be provided in "
449 "one request.")
450
468 - self._consumed_items[modality] = current_count
469 - self._items.append(item)
451 + self._items_by_modality[modality].append(item)
452
453 return self._placeholder_str(modality, current_count)
454
@@ -475,22 +457,26 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
457 raise NotImplementedError
458
459
478 -class MultiModalItemTracker(BaseMultiModalItemTracker[MultiModalDataDict]):
460 +class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
461
462 def all_mm_data(self) -> Optional[MultiModalDataDict]:
481 - return self._combine(self._items) if self._items else None
463 + if self._items_by_modality:
464 + return dict(self._items_by_modality)
465 +
466 + return None
467
468 def create_parser(self) -> "BaseMultiModalContentParser":
469 return MultiModalContentParser(self)
470
471
487 -class AsyncMultiModalItemTracker(
488 - BaseMultiModalItemTracker[Awaitable[MultiModalDataDict]]):
472 +class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
473
474 async def all_mm_data(self) -> Optional[MultiModalDataDict]:
491 - if self._items:
492 - items = await asyncio.gather(*self._items)
493 - return self._combine(items)
475 + if self._items_by_modality:
476 + return {
477 + modality: await asyncio.gather(*items)
478 + for modality, items in self._items_by_modality.items()
479 + }
480
481 return None
482
@@ -522,7 +508,7 @@ class BaseMultiModalContentParser(ABC):
508 raise NotImplementedError
509
510 @abstractmethod
525 - def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
511 + def parse_input_audio(self, input_audio: InputAudio) -> None:
512 raise NotImplementedError
513
514 @abstractmethod
@@ -537,31 +523,31 @@ class MultiModalContentParser(BaseMultiModalContentParser):
523
524 self._tracker = tracker
525
526 + self._connector = MediaConnector(
527 + allowed_local_media_path=tracker.allowed_local_media_path,
528 + )
529 +
530 def parse_image(self, image_url: str) -> None:
541 - image = get_and_parse_image(image_url,
542 - allowed_local_media_path=self._tracker.
543 - _model_config.allowed_local_media_path)
531 + image = self._connector.fetch_image(image_url)
532
533 placeholder = self._tracker.add("image", image)
534 self._add_placeholder(placeholder)
535
536 def parse_audio(self, audio_url: str) -> None:
549 - audio = get_and_parse_audio(audio_url)
537 + audio = self._connector.fetch_audio(audio_url)
538
539 placeholder = self._tracker.add("audio", audio)
540 self._add_placeholder(placeholder)
541
554 - def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
555 - input_audio_data = input_audio.get("data","")
556 - input_audio_format = input_audio.get("format","")
557 - audio_url = f"data:audio/{input_audio_format};base64,{input_audio_data}"
558 - audio = get_and_parse_audio(audio_url)
542 + def parse_input_audio(self, input_audio: InputAudio) -> None:
543 + audio_data = input_audio.get("data", "")
544 + audio_format = input_audio.get("format", "")
545 + audio_url = f"data:audio/{audio_format};base64,{audio_data}"
546
560 - placeholder = self._tracker.add("audio", audio)
561 - self._add_placeholder(placeholder)
547 + return self.parse_audio(audio_url)
548
549 def parse_video(self, video_url: str) -> None:
564 - video = get_and_parse_video(video_url)
550 + video = self._connector.fetch_video(video_url)
551
552 placeholder = self._tracker.add("video", video)
553 self._add_placeholder(placeholder)
@@ -573,33 +559,31 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
559 super().__init__()
560
561 self._tracker = tracker
562 + self._connector = MediaConnector(
563 + allowed_local_media_path=tracker.allowed_local_media_path,
564 + )
565
566 def parse_image(self, image_url: str) -> None:
578 - image_coro = async_get_and_parse_image(
579 - image_url,
580 - allowed_local_media_path=self._tracker._model_config.
581 - allowed_local_media_path)
567 + image_coro = self._connector.fetch_image_async(image_url)
568
569 placeholder = self._tracker.add("image", image_coro)
570 self._add_placeholder(placeholder)
571
572 def parse_audio(self, audio_url: str) -> None:
587 - audio_coro = async_get_and_parse_audio(audio_url)
573 + audio_coro = self._connector.fetch_audio_async(audio_url)
574
575 placeholder = self._tracker.add("audio", audio_coro)
576 self._add_placeholder(placeholder)
577
592 - def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
593 - input_audio_data = input_audio.get("data","")
594 - input_audio_format = input_audio.get("format","")
595 - audio_url = f"data:audio/{input_audio_format};base64,{input_audio_data}"
596 - audio_coro = async_get_and_parse_audio(audio_url)
578 + def parse_input_audio(self, input_audio: InputAudio) -> None:
579 + audio_data = input_audio.get("data", "")
580 + audio_format = input_audio.get("format", "")
581 + audio_url = f"data:audio/{audio_format};base64,{audio_data}"
582
598 - placeholder = self._tracker.add("audio", audio_coro)
599 - self._add_placeholder(placeholder)
583 + return self.parse_audio(audio_url)
584
585 def parse_video(self, video_url: str) -> None:
602 - video = async_get_and_parse_video(video_url)
586 + video = self._connector.fetch_video_async(video_url)
587
588 placeholder = self._tracker.add("video", video)
589 self._add_placeholder(placeholder)
@@ -695,10 +679,13 @@ _InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
679 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
680 _VideoParser = partial(cast, ChatCompletionContentPartVideoParam)
681
682 +_ContentPart: TypeAlias = Union[str, Dict[str, str], InputAudio]
683 +
684 # Define a mapping from part types to their corresponding parsing functions.
699 -MM_PARSER_MAP: Dict[str,
700 - Callable[[ChatCompletionContentPartParam],
701 - Union[str, Dict[str,str]]]] = {
685 +MM_PARSER_MAP: Dict[
686 + str,
687 + Callable[[ChatCompletionContentPartParam], _ContentPart],
688 +] = {
689 "text":
690 lambda part: _TextParser(part).get("text", ""),
691 "image_url":
@@ -715,8 +702,7 @@ MM_PARSER_MAP: Dict[str,
702
703
704 def _parse_chat_message_content_mm_part(
718 - part: ChatCompletionContentPartParam) -> Tuple[str,
719 - Union[str, Dict[str, str]]]:
705 + part: ChatCompletionContentPartParam) -> tuple[str, _ContentPart]:
706 """
707 Parses a given multi-modal content part based on its type.
708
@@ -783,7 +769,7 @@ def _parse_chat_message_content_parts(
769 *,
770 wrap_dicts: bool,
771 ) -> List[ConversationMessage]:
786 - content: List[Union[str, Dict[str, str]]] = []
772 + content = list[_ContentPart]()
773
774 mm_parser = mm_tracker.create_parser()
775
@@ -814,7 +800,7 @@ def _parse_chat_message_content_part(
800 mm_parser: BaseMultiModalContentParser,
801 *,
802 wrap_dicts: bool,
817 -) -> Optional[Union[str, Dict[str, str]]]:
803 +) -> Optional[_ContentPart]:
804 """Parses a single part of a conversation. If wrap_dicts is True,
805 structured dictionary pieces for texts and images will be
806 wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
@@ -823,8 +809,7 @@ def _parse_chat_message_content_part(
809 with multimodal placeholders.
810 """
811 if isinstance(part, str): # Handle plain text parts
826 - text = _TextParser(part)
827 - return text
812 + return part
813
814 # Handle structured dictionary parts
815 part_type, content = _parse_chat_message_content_mm_part(part)
@@ -855,7 +840,7 @@ def _parse_chat_message_content_part(
840 return {'type': 'audio'} if wrap_dicts else None
841
842 if part_type == "input_audio":
858 - dict_content = cast(Dict[str, str], content)
843 + dict_content = cast(InputAudio, content)
844 mm_parser.parse_input_audio(dict_content)
845 return {'type': 'audio'} if wrap_dicts else None
846
vllm/multimodal/audio.py
+35 -1
@@ -1,10 +1,14 @@
1 +import base64
2 +from io import BytesIO
3 +from pathlib import Path
4 +
5 import numpy as np
6 import numpy.typing as npt
7
8 from vllm.inputs.registry import InputContext
9 from vllm.utils import PlaceholderModule
10
7 -from .base import MultiModalPlugin
11 +from .base import MediaIO, MultiModalPlugin
12 from .inputs import AudioItem, MultiModalData, MultiModalKwargs
13
14 try:
@@ -12,6 +16,11 @@ try:
16 except ImportError:
17 librosa = PlaceholderModule("librosa") # type: ignore[assignment]
18
19 +try:
20 + import soundfile
21 +except ImportError:
22 + soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
23 +
24
25 class AudioPlugin(MultiModalPlugin):
26 """Plugin for audio data."""
@@ -39,3 +48,28 @@ def resample_audio(
48 target_sr: float,
49 ) -> npt.NDArray[np.floating]:
50 return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
51 +
52 +
53 +class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
54 +
55 + def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
56 + return librosa.load(BytesIO(data), sr=None)
57 +
58 + def load_base64(
59 + self,
60 + media_type: str,
61 + data: str,
62 + ) -> tuple[npt.NDArray, float]:
63 + return self.load_bytes(base64.b64decode(data))
64 +
65 + def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
66 + return librosa.load(filepath, sr=None)
67 +
68 + def encode_base64(self, media: tuple[npt.NDArray, float]) -> str:
69 + audio, sr = media
70 +
71 + with BytesIO() as buffer:
72 + soundfile.write(buffer, audio, sr, format="WAV")
73 + data = buffer.getvalue()
74 +
75 + return base64.b64encode(data).decode('utf-8')
vllm/multimodal/base.py
+29 -9
@@ -1,6 +1,7 @@
1 from abc import ABC, abstractmethod
2 from collections import defaultdict
3 -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple,
3 +from pathlib import Path
4 +from typing import (TYPE_CHECKING, Any, Callable, Generic, NamedTuple,
5 Optional, Sequence, Tuple, Type, TypeVar, Union)
6
7 from torch import nn
@@ -118,7 +119,7 @@ class MultiModalPlugin(ABC):
119 self,
120 model_config: "ModelConfig",
121 data: MultiModalData[Any],
121 - mm_processor_kwargs: Optional[Dict[str, Any]],
122 + mm_processor_kwargs: Optional[dict[str, Any]],
123 ) -> MultiModalKwargs:
124 """
125 Transform the data into a dictionary of model inputs using the
@@ -254,10 +255,10 @@ class MultiModalPlaceholderMap:
255 """
256
257 class IndexMap(NamedTuple):
257 - src: List[int]
258 - dest: List[int]
258 + src: list[int]
259 + dest: list[int]
260
260 - src_ranges: List[range]
261 + src_ranges: list[range]
262 """
263 The indices of the multi-modal embeddings that will replace the
264 corresponding placeholder embeddings pointed to by ``dest_ranges``.
@@ -268,7 +269,7 @@ class MultiModalPlaceholderMap:
269 The total number of flattened multi-modal embeddings.
270 """
271
271 - dest_ranges: List[range]
272 + dest_ranges: list[range]
273 """
274 The indices of the placeholder embeddings that will be replaced by the
275 multimodal embeddings.
@@ -288,7 +289,7 @@ class MultiModalPlaceholderMap:
289 @classmethod
290 def from_seq_group(
291 cls, seq_group: "SequenceGroupMetadata", positions: range
291 - ) -> Tuple[Optional[MultiModalDataDict], Dict[str,
292 + ) -> Tuple[Optional[MultiModalDataDict], dict[str,
293 "MultiModalPlaceholderMap"]]:
294 """
295 Returns the multi-modal items that intersect with the portion of a
@@ -376,9 +377,9 @@ class MultiModalPlaceholderMap:
377 def append_items_from_seq_group(
378 self,
379 positions: range,
379 - multi_modal_items: List[_T],
380 + multi_modal_items: list[_T],
381 multi_modal_placeholders: Sequence[PlaceholderRange],
381 - ) -> List[_T]:
382 + ) -> list[_T]:
383 """
384 Adds the multi-modal items that intersect ```positions`` to this
385 placeholder map and returns the intersecting items.
@@ -454,3 +455,22 @@ class MultiModalPlaceholderMap:
455
456 return MultiModalPlaceholderMap.IndexMap(src=src_indices,
457 dest=dest_indices)
458 +
459 +
460 +class MediaIO(ABC, Generic[_T]):
461 +
462 + @abstractmethod
463 + def load_bytes(self, data: bytes) -> _T:
464 + raise NotImplementedError
465 +
466 + @abstractmethod
467 + def load_base64(self, media_type: str, data: str) -> _T:
468 + """
469 + List of media types:
470 + https://www.iana.org/assignments/media-types/media-types.xhtml
471 + """
472 + raise NotImplementedError
473 +
474 + @abstractmethod
475 + def load_file(self, filepath: Path) -> _T:
476 + raise NotImplementedError
vllm/multimodal/image.py
+40 -1
@@ -1,4 +1,7 @@
1 +import base64
2 from functools import lru_cache
3 +from io import BytesIO
4 +from pathlib import Path
5 from typing import TYPE_CHECKING, Any, Dict, Optional
6
7 import torch
@@ -9,7 +12,7 @@ from vllm.logger import init_logger
12 from vllm.transformers_utils.processor import get_image_processor
13 from vllm.utils import is_list_of
14
12 -from .base import MultiModalPlugin
15 +from .base import MediaIO, MultiModalPlugin
16 from .inputs import ImageItem, MultiModalData, MultiModalKwargs
17
18 if TYPE_CHECKING:
@@ -96,3 +99,39 @@ def rescale_image_size(image: Image.Image,
99 if transpose >= 0:
100 image = image.transpose(Image.Transpose(transpose))
101 return image
102 +
103 +
104 +class ImageMediaIO(MediaIO[Image.Image]):
105 +
106 + def __init__(self, *, image_mode: str = "RGB") -> None:
107 + super().__init__()
108 +
109 + self.image_mode = image_mode
110 +
111 + def load_bytes(self, data: bytes) -> Image.Image:
112 + image = Image.open(BytesIO(data))
113 + image.load()
114 + return image.convert(self.image_mode)
115 +
116 + def load_base64(self, media_type: str, data: str) -> Image.Image:
117 + return self.load_bytes(base64.b64decode(data))
118 +
119 + def load_file(self, filepath: Path) -> Image.Image:
120 + image = Image.open(filepath)
121 + image.load()
122 + return image.convert(self.image_mode)
123 +
124 + def encode_base64(
125 + self,
126 + media: Image.Image,
127 + *,
128 + image_format: str = "JPEG",
129 + ) -> str:
130 + image = media
131 +
132 + with BytesIO() as buffer:
133 + image = image.convert(self.image_mode)
134 + image.save(buffer, image_format)
135 + data = buffer.getvalue()
136 +
137 + return base64.b64encode(data).decode('utf-8')
vllm/multimodal/utils.py
+211 -266
@@ -1,8 +1,7 @@
1 -import base64
2 -import os
1 from functools import lru_cache
4 -from io import BytesIO
5 -from typing import List, Optional, Tuple, TypeVar, Union
2 +from pathlib import Path
3 +from typing import Optional, TypeVar, Union
4 +from urllib.parse import ParseResult, urlparse
5
6 import numpy as np
7 import numpy.typing as npt
@@ -10,283 +9,246 @@ import torch
9 from PIL import Image
10
11 import vllm.envs as envs
13 -from vllm.connections import global_http_connection
12 +from vllm.connections import HTTPConnection, global_http_connection
13 from vllm.logger import init_logger
14 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
16 -from vllm.utils import PlaceholderModule
15
18 -from .inputs import MultiModalDataDict, PlaceholderRange
19 -
20 -try:
21 - import decord
22 -except ImportError:
23 - decord = PlaceholderModule("decord") # type: ignore[assignment]
24 -
25 -try:
26 - import librosa
27 -except ImportError:
28 - librosa = PlaceholderModule("librosa") # type: ignore[assignment]
29 -
30 -try:
31 - import soundfile
32 -except ImportError:
33 - soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
16 +from .audio import AudioMediaIO
17 +from .base import MediaIO
18 +from .image import ImageMediaIO
19 +from .inputs import PlaceholderRange
20 +from .video import VideoMediaIO
21
22 logger = init_logger(__name__)
23
24 cached_get_tokenizer = lru_cache(get_tokenizer)
25
26 +_M = TypeVar("_M")
27
40 -def _load_image_from_bytes(b: bytes) -> Image.Image:
41 - image = Image.open(BytesIO(b))
42 - image.load()
43 - return image
44 -
45 -
46 -def _is_subpath(image_path: str, allowed_local_media_path: str) -> bool:
47 - # Get the common path
48 - common_path = os.path.commonpath([
49 - os.path.abspath(image_path),
50 - os.path.abspath(allowed_local_media_path)
51 - ])
52 - # Check if the common path is the same as allowed_local_media_path
53 - return common_path == os.path.abspath(allowed_local_media_path)
28
29 +class MediaConnector:
30
56 -def _load_image_from_file(image_url: str,
57 - allowed_local_media_path: str) -> Image.Image:
58 - if not allowed_local_media_path:
59 - raise ValueError("Invalid 'image_url': Cannot load local files without"
60 - "'--allowed-local-media-path'.")
61 - if allowed_local_media_path:
62 - if not os.path.exists(allowed_local_media_path):
63 - raise ValueError(
64 - "Invalid '--allowed-local-media-path': "
65 - f"The path {allowed_local_media_path} does not exist.")
66 - if not os.path.isdir(allowed_local_media_path):
31 + def __init__(
32 + self,
33 + connection: HTTPConnection = global_http_connection,
34 + *,
35 + allowed_local_media_path: str = "",
36 + ) -> None:
37 + super().__init__()
38 +
39 + self.connection = connection
40 +
41 + if allowed_local_media_path:
42 + allowed_local_media_path_ = Path(allowed_local_media_path)
43 +
44 + if not allowed_local_media_path_.exists():
45 + raise ValueError(
46 + "Invalid `--allowed-local-media-path`: The path "
47 + f"{allowed_local_media_path_} does not exist.")
48 + if not allowed_local_media_path_.is_dir():
49 + raise ValueError(
50 + "Invalid `--allowed-local-media-path`: The path "
51 + f"{allowed_local_media_path_} must be a directory.")
52 + else:
53 + allowed_local_media_path_ = None
54 +
55 + self.allowed_local_media_path = allowed_local_media_path_
56 +
57 + def _load_data_url(
58 + self,
59 + url_spec: ParseResult,
60 + media_io: MediaIO[_M],
61 + ) -> _M:
62 + data_spec, data = url_spec.path.split(",", 1)
63 + media_type, data_type = data_spec.split(";", 1)
64 +
65 + if data_type != "base64":
66 + msg = "Only base64 data URLs are supported for now."
67 + raise NotImplementedError(msg)
68 +
69 + return media_io.load_base64(media_type, data)
70 +
71 + def _load_file_url(
72 + self,
73 + url_spec: ParseResult,
74 + media_io: MediaIO[_M],
75 + ) -> _M:
76 + allowed_local_media_path = self.allowed_local_media_path
77 + if allowed_local_media_path is None:
78 + raise RuntimeError("Cannot load local files without "
79 + "`--allowed-local-media-path`.")
80 +
81 + filepath = Path(url_spec.path)
82 + if allowed_local_media_path not in filepath.resolve().parents:
83 raise ValueError(
68 - "Invalid '--allowed-local-media-path': "
69 - f"The path {allowed_local_media_path} must be a directory.")
70 -
71 - # Only split once and assume the second part is the image path
72 - _, image_path = image_url.split("file://", 1)
73 - if not _is_subpath(image_path, allowed_local_media_path):
74 - raise ValueError(
75 - f"Invalid 'image_url': The file path {image_path} must"
76 - " be a subpath of '--allowed-local-media-path'"
77 - f" '{allowed_local_media_path}'.")
78 -
79 - image = Image.open(image_path)
80 - image.load()
81 - return image
84 + f"The file path {filepath} must be a subpath "
85 + f"of `--allowed-local-media-path` {allowed_local_media_path}.")
86
87 + return media_io.load_file(filepath)
88
84 -def _load_image_from_data_url(image_url: str) -> Image.Image:
85 - # Only split once and assume the second part is the base64 encoded image
86 - _, image_base64 = image_url.split(",", 1)
87 - return load_image_from_base64(image_base64)
88 -
89 -
90 -def fetch_image(image_url: str,
91 - *,
92 - image_mode: str = "RGB",
93 - allowed_local_media_path: str = "") -> Image.Image:
94 - """
95 - Load a PIL image from a HTTP or base64 data URL.
96 -
97 - By default, the image is converted into RGB format.
98 - """
99 - if image_url.startswith('http'):
100 - image_raw = global_http_connection.get_bytes(
101 - image_url,
102 - timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
103 - )
104 - image = _load_image_from_bytes(image_raw)
105 -
106 - elif image_url.startswith('data:image'):
107 - image = _load_image_from_data_url(image_url)
108 - elif image_url.startswith('file://'):
109 - image = _load_image_from_file(image_url, allowed_local_media_path)
110 - else:
111 - raise ValueError("Invalid 'image_url': A valid 'image_url' must start "
112 - "with either 'data:image', 'file://' or 'http'.")
113 -
114 - return image.convert(image_mode)
115 -
116 -
117 -async def async_fetch_image(image_url: str,
118 - *,
119 - image_mode: str = "RGB",
120 - allowed_local_media_path: str = "") -> Image.Image:
121 - """
122 - Asynchronously load a PIL image from a HTTP or base64 data URL.
123 -
124 - By default, the image is converted into RGB format.
125 - """
126 - if image_url.startswith('http'):
127 - image_raw = await global_http_connection.async_get_bytes(
128 - image_url,
129 - timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
130 - )
131 - image = _load_image_from_bytes(image_raw)
132 -
133 - elif image_url.startswith('data:image'):
134 - image = _load_image_from_data_url(image_url)
135 - elif image_url.startswith('file://'):
136 - image = _load_image_from_file(image_url, allowed_local_media_path)
137 - else:
138 - raise ValueError("Invalid 'image_url': A valid 'image_url' must start "
139 - "with either 'data:image', 'file://' or 'http'.")
89 + def load_from_url(
90 + self,
91 + url: str,
92 + media_io: MediaIO[_M],
93 + *,
94 + fetch_timeout: Optional[int] = None,
95 + ) -> _M:
96 + url_spec = urlparse(url)
97
141 - return image.convert(image_mode)
98 + if url_spec.scheme.startswith("http"):
99 + connection = self.connection
100 + data = connection.get_bytes(url, timeout=fetch_timeout)
101
102 + return media_io.load_bytes(data)
103
144 -def _load_video_from_bytes(b: bytes, num_frames: int = 32) -> npt.NDArray:
145 - video_path = BytesIO(b)
146 - vr = decord.VideoReader(video_path, num_threads=1)
147 - total_frame_num = len(vr)
104 + if url_spec.scheme == "data":
105 + return self._load_data_url(url_spec, media_io)
106
149 - if total_frame_num > num_frames:
150 - uniform_sampled_frames = np.linspace(0,
151 - total_frame_num - 1,
152 - num_frames,
153 - dtype=int)
154 - frame_idx = uniform_sampled_frames.tolist()
155 - else:
156 - frame_idx = [i for i in range(0, total_frame_num)]
157 - frames = vr.get_batch(frame_idx).asnumpy()
107 + if url_spec.scheme == "file":
108 + return self._load_file_url(url_spec, media_io)
109
159 - return frames
110 + msg = "The URL must be either a HTTP, data or file URL."
111 + raise ValueError(msg)
112
113 + async def load_from_url_async(
114 + self,
115 + url: str,
116 + media_io: MediaIO[_M],
117 + *,
118 + fetch_timeout: Optional[int] = None,
119 + ) -> _M:
120 + url_spec = urlparse(url)
121
162 -def _load_video_from_data_url(video_url: str) -> npt.NDArray:
163 - # Only split once and assume the second part is the base64 encoded video
164 - _, video_base64 = video_url.split(",", 1)
122 + if url_spec.scheme.startswith("http"):
123 + connection = self.connection
124 + data = await connection.async_get_bytes(url, timeout=fetch_timeout)
125
166 - if video_url.startswith("data:video/jpeg;"):
167 - return np.stack([
168 - np.array(load_image_from_base64(frame_base64))
169 - for frame_base64 in video_base64.split(",")
170 - ])
126 + return media_io.load_bytes(data)
127
172 - return load_video_from_base64(video_base64)
128 + if url_spec.scheme == "data":
129 + return self._load_data_url(url_spec, media_io)
130
131 + if url_spec.scheme == "file":
132 + return self._load_file_url(url_spec, media_io)
133
175 -def fetch_video(video_url: str, *, num_frames: int = 32) -> npt.NDArray:
176 - """
177 - Load video from a HTTP or base64 data URL.
178 - """
179 - if video_url.startswith('http') or video_url.startswith('https'):
180 - video_raw = global_http_connection.get_bytes(
181 - video_url,
182 - timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
183 - )
184 - video = _load_video_from_bytes(video_raw, num_frames)
185 - elif video_url.startswith('data:video'):
186 - video = _load_video_from_data_url(video_url)
187 - else:
188 - raise ValueError("Invalid 'video_url': A valid 'video_url' must start "
189 - "with either 'data:video' or 'http'.")
190 - return video
134 + msg = "The URL must be either a HTTP, data or file URL."
135 + raise ValueError(msg)
136
137 + def fetch_audio(
138 + self,
139 + audio_url: str,
140 + ) -> tuple[np.ndarray, Union[int, float]]:
141 + """
142 + Load audio from a URL.
143 + """
144 + audio_io = AudioMediaIO()
145
193 -async def async_fetch_video(video_url: str,
194 - *,
195 - num_frames: int = 32) -> npt.NDArray:
196 - """
197 - Asynchronously load video from a HTTP or base64 data URL.
198 -
199 - By default, the image is converted into RGB format.
200 - """
201 - if video_url.startswith('http') or video_url.startswith('https'):
202 - video_raw = await global_http_connection.async_get_bytes(
203 - video_url,
204 - timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
205 - )
206 - video = _load_video_from_bytes(video_raw, num_frames)
207 - elif video_url.startswith('data:video'):
208 - video = _load_video_from_data_url(video_url)
209 - else:
210 - raise ValueError("Invalid 'video_url': A valid 'video_url' must start "
211 - "with either 'data:video' or 'http'.")
212 - return video
213 -
214 -
215 -def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
216 - """
217 - Load audio from a URL.
218 - """
219 - if audio_url.startswith("http"):
220 - audio_bytes = global_http_connection.get_bytes(
146 + return self.load_from_url(
147 audio_url,
222 - timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
148 + audio_io,
149 + fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
150 )
224 - elif audio_url.startswith("data:audio"):
225 - _, audio_base64 = audio_url.split(",", 1)
226 - audio_bytes = base64.b64decode(audio_base64)
227 - else:
228 - raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start "
229 - "with either 'data:audio' or 'http'.")
230 -
231 - return librosa.load(BytesIO(audio_bytes), sr=None)
151
152 + async def fetch_audio_async(
153 + self,
154 + audio_url: str,
155 + ) -> tuple[np.ndarray, Union[int, float]]:
156 + """
157 + Asynchronously fetch audio from a URL.
158 + """
159 + audio_io = AudioMediaIO()
160
234 -async def async_fetch_audio(
235 - audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
236 - """
237 - Asynchronously fetch audio from a URL.
238 - """
239 - if audio_url.startswith("http"):
240 - audio_bytes = await global_http_connection.async_get_bytes(
161 + return await self.load_from_url_async(
162 audio_url,
242 - timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
163 + audio_io,
164 + fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
165 )
244 - elif audio_url.startswith("data:audio"):
245 - _, audio_base64 = audio_url.split(",", 1)
246 - audio_bytes = base64.b64decode(audio_base64)
247 - else:
248 - raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start "
249 - "with either 'data:audio' or 'http'.")
250 -
251 - return librosa.load(BytesIO(audio_bytes), sr=None)
166
167 + def fetch_image(
168 + self,
169 + image_url: str,
170 + *,
171 + image_mode: str = "RGB",
172 + ) -> Image.Image:
173 + """
174 + Load a PIL image from a HTTP or base64 data URL.
175
254 -def get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
255 - audio, sr = fetch_audio(audio_url)
256 - return {"audio": (audio, sr)}
176 + By default, the image is converted into RGB format.
177 + """
178 + image_io = ImageMediaIO(image_mode=image_mode)
179
180 + return self.load_from_url(
181 + image_url,
182 + image_io,
183 + fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
184 + )
185
259 -def get_and_parse_image(
186 + async def fetch_image_async(
187 + self,
188 image_url: str,
189 *,
262 - allowed_local_media_path: str = "") -> MultiModalDataDict:
263 - image = fetch_image(image_url,
264 - allowed_local_media_path=allowed_local_media_path)
265 - return {"image": image}
266 -
190 + image_mode: str = "RGB",
191 + ) -> Image.Image:
192 + """
193 + Asynchronously load a PIL image from a HTTP or base64 data URL.
194
268 -def get_and_parse_video(video_url: str) -> MultiModalDataDict:
269 - video = fetch_video(video_url)
270 - return {"video": video}
195 + By default, the image is converted into RGB format.
196 + """
197 + image_io = ImageMediaIO(image_mode=image_mode)
198
199 + return await self.load_from_url_async(
200 + image_url,
201 + image_io,
202 + fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
203 + )
204
273 -async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
274 - audio, sr = await async_fetch_audio(audio_url)
275 - return {"audio": (audio, sr)}
276 -
205 + def fetch_video(
206 + self,
207 + video_url: str,
208 + *,
209 + image_mode: str = "RGB",
210 + num_frames: int = 32,
211 + ) -> npt.NDArray:
212 + """
213 + Load video from a HTTP or base64 data URL.
214 + """
215 + image_io = ImageMediaIO(image_mode=image_mode)
216 + video_io = VideoMediaIO(image_io, num_frames=num_frames)
217 +
218 + return self.load_from_url(
219 + video_url,
220 + video_io,
221 + fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
222 + )
223
278 -async def async_get_and_parse_image(
279 - image_url: str,
224 + async def fetch_video_async(
225 + self,
226 + video_url: str,
227 *,
281 - allowed_local_media_path: str = "") -> MultiModalDataDict:
282 - image = await async_fetch_image(
283 - image_url, allowed_local_media_path=allowed_local_media_path)
284 - return {"image": image}
228 + image_mode: str = "RGB",
229 + num_frames: int = 32,
230 + ) -> npt.NDArray:
231 + """
232 + Asynchronously load video from a HTTP or base64 data URL.
233 +
234 + By default, the image is converted into RGB format.
235 + """
236 + image_io = ImageMediaIO(image_mode=image_mode)
237 + video_io = VideoMediaIO(image_io, num_frames=num_frames)
238 +
239 + return await self.load_from_url_async(
240 + video_url,
241 + video_io,
242 + fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
243 + )
244
245
287 -async def async_get_and_parse_video(video_url: str) -> MultiModalDataDict:
288 - video = await async_fetch_video(video_url)
289 - return {"video": video}
246 +global_media_connector = MediaConnector()
247 +"""The global :class:`MediaConnector` instance used by vLLM."""
248 +
249 +fetch_audio = global_media_connector.fetch_audio
250 +fetch_image = global_media_connector.fetch_image
251 +fetch_video = global_media_connector.fetch_video
252
253
254 def encode_audio_base64(
@@ -294,10 +256,8 @@ def encode_audio_base64(
256 sampling_rate: int,
257 ) -> str:
258 """Encode audio as base64."""
297 - buffered = BytesIO()
298 - soundfile.write(buffered, audio, sampling_rate, format="WAV")
299 -
300 - return base64.b64encode(buffered.getvalue()).decode('utf-8')
259 + audio_io = AudioMediaIO()
260 + return audio_io.encode_base64((audio, sampling_rate))
261
262
263 def encode_image_base64(
@@ -311,29 +271,14 @@ def encode_image_base64(
271
272 By default, the image is converted into RGB format before being encoded.
273 """
314 - buffered = BytesIO()
315 - image = image.convert(image_mode)
316 - image.save(buffered, format)
317 - return base64.b64encode(buffered.getvalue()).decode('utf-8')
318 -
319 -
320 -def load_image_from_base64(image: Union[bytes, str]) -> Image.Image:
321 - """Load image from base64 format."""
322 - return _load_image_from_bytes(base64.b64decode(image))
274 + image_io = ImageMediaIO(image_mode=image_mode)
275 + return image_io.encode_base64(image, image_format=format)
276
277
278 def encode_video_base64(frames: npt.NDArray) -> str:
326 - base64_frames = []
327 - frames_list = [frames[i] for i in range(frames.shape[0])]
328 - for frame in frames_list:
329 - img_base64 = encode_image_base64(Image.fromarray(frame))
330 - base64_frames.append(img_base64)
331 - return ",".join(base64_frames)
332 -
333 -
334 -def load_video_from_base64(video: Union[bytes, str]) -> npt.NDArray:
335 - """Load video from base64 format."""
336 - return _load_video_from_bytes(base64.b64decode(video))
279 + image_io = ImageMediaIO()
280 + video_io = VideoMediaIO(image_io)
281 + return video_io.encode_base64(frames)
282
283
284 def resolve_visual_encoder_outputs(
@@ -389,7 +334,7 @@ def repeat_and_pad_token(
334 repeat_count: int = 1,
335 pad_token_left: Optional[_T] = None,
336 pad_token_right: Optional[_T] = None,
392 -) -> List[_T]:
337 +) -> list[_T]:
338 replacement = [token] * repeat_count
339 if pad_token_left is not None:
340 replacement = [pad_token_left] + replacement
@@ -402,13 +347,13 @@ def repeat_and_pad_token(
347 def repeat_and_pad_placeholder_tokens(
348 tokenizer: AnyTokenizer,
349 prompt: Optional[str],
405 - prompt_token_ids: List[int],
350 + prompt_token_ids: list[int],
351 *,
352 placeholder_token_id: int,
408 - repeat_count: Union[int, List[int]],
353 + repeat_count: Union[int, list[int]],
354 pad_token_left: Optional[int] = None,
355 pad_token_right: Optional[int] = None,
411 -) -> Tuple[Optional[str], List[int], List[PlaceholderRange]]:
356 +) -> tuple[Optional[str], list[int], list[PlaceholderRange]]:
357 if isinstance(repeat_count, int):
358 repeat_count = [repeat_count]
359
@@ -450,8 +395,8 @@ def repeat_and_pad_placeholder_tokens(
395 new_prompt += prompt_parts[i] + replacement_str
396 new_prompt += prompt_parts[-1]
397
453 - new_token_ids: List[int] = []
454 - placeholder_ranges: List[PlaceholderRange] = []
398 + new_token_ids = list[int]()
399 + placeholder_ranges = list[PlaceholderRange]()
400 placeholder_token_idx = 0
401 for i, token in enumerate(prompt_token_ids):
402 if token == placeholder_token_id:
@@ -481,7 +426,7 @@ def repeat_and_pad_placeholder_tokens(
426 def consecutive_placeholder_ranges(
427 num_items: int,
428 item_size: int,
484 - initial_offset: int = 0) -> List[PlaceholderRange]:
429 + initial_offset: int = 0) -> list[PlaceholderRange]:
430 """Returns a list of consecutive PlaceholderRanges of a fixed size"""
431
432 return [
vllm/multimodal/video.py
+83 -4
@@ -1,23 +1,32 @@
1 -from functools import lru_cache
1 +import base64
2 +from functools import lru_cache, partial
3 +from io import BytesIO
4 +from pathlib import Path
5 from typing import TYPE_CHECKING, Any, Dict, Optional
6
7 import cv2
8 import numpy as np
9 import numpy.typing as npt
10 +from PIL import Image
11
12 from vllm.inputs.registry import InputContext
13 from vllm.logger import init_logger
14 from vllm.transformers_utils.processor import get_video_processor
15 from vllm.transformers_utils.tokenizer import get_tokenizer
12 -from vllm.utils import is_list_of
16 +from vllm.utils import PlaceholderModule, is_list_of
17
14 -from .base import MultiModalData
15 -from .image import ImagePlugin
18 +from .base import MediaIO, MultiModalData
19 +from .image import ImageMediaIO, ImagePlugin
20 from .inputs import MultiModalKwargs, VideoItem
21
22 if TYPE_CHECKING:
23 from vllm.config import ModelConfig
24
25 +try:
26 + import decord
27 +except ImportError:
28 + decord = PlaceholderModule("decord") # type: ignore[assignment]
29 +
30 logger = init_logger(__name__)
31
32 cached_get_video_processor = lru_cache(get_video_processor)
@@ -107,3 +116,73 @@ def sample_frames_from_video(frames: npt.NDArray,
116 frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
117 sampled_frames = frames[frame_indices, ...]
118 return sampled_frames
119 +
120 +
121 +class VideoMediaIO(MediaIO[npt.NDArray]):
122 +
123 + def __init__(
124 + self,
125 + image_io: ImageMediaIO,
126 + *,
127 + num_frames: int = 32,
128 + ) -> None:
129 + super().__init__()
130 +
131 + self.image_io = image_io
132 + self.num_frames = num_frames
133 +
134 + def load_bytes(self, data: bytes) -> npt.NDArray:
135 + vr = decord.VideoReader(BytesIO(data), num_threads=1)
136 + total_frame_num = len(vr)
137 +
138 + num_frames = self.num_frames
139 + if total_frame_num > num_frames:
140 + uniform_sampled_frames = np.linspace(0,
141 + total_frame_num - 1,
142 + num_frames,
143 + dtype=int)
144 + frame_idx = uniform_sampled_frames.tolist()
145 + else:
146 + frame_idx = list(range(0, total_frame_num))
147 +
148 + return vr.get_batch(frame_idx).asnumpy()
149 +
150 + def load_base64(self, media_type: str, data: str) -> npt.NDArray:
151 + if media_type.lower() == "video/jpeg":
152 + load_frame = partial(
153 + self.image_io.load_base64,
154 + "image/jpeg",
155 + )
156 +
157 + return np.stack([
158 + np.array(load_frame(frame_data))
159 + for frame_data in data.split(",")
160 + ])
161 +
162 + return self.load_bytes(base64.b64decode(data))
163 +
164 + def load_file(self, filepath: Path) -> npt.NDArray:
165 + with filepath.open("rb") as f:
166 + data = f.read()
167 +
168 + return self.load_bytes(data)
169 +
170 + def encode_base64(
171 + self,
172 + media: npt.NDArray,
173 + *,
174 + video_format: str = "JPEG",
175 + ) -> str:
176 + video = media
177 +
178 + if video_format == "JPEG":
179 + encode_frame = partial(
180 + self.image_io.encode_base64,
181 + image_format=video_format,
182 + )
183 +
184 + return ",".join(
185 + encode_frame(Image.fromarray(frame)) for frame in video)
186 +
187 + msg = "Only JPEG format is supported for now."
188 + raise NotImplementedError(msg)