A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Misc] Abstract the logic for reading and writing media content (#11527)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 27, 2024 at 19:21 UTC
7af553ea30031446b4c1c74ad83187f9fd3de4e7
10 files changed
+493
-387
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -33,6 +33,7 @@ class MockModelConfig:
33
hf_config = MockHFConfig()
34
logits_processor_pattern = None
35
diff_sampling_param: Optional[dict] = None
36
+ allowed_local_media_path: str = ""
37
38
def get_diff_sampling_param(self):
39
return self.diff_sampling_param or {}
tests/entrypoints/test_chat_utils.py
+1
-5
@@ -2,7 +2,6 @@ import warnings
2
from typing import Optional
3
4
import pytest
5
-from PIL import Image
5
6
from vllm.assets.image import ImageAsset
7
from vllm.config import ModelConfig
@@ -91,10 +90,7 @@ def _assert_mm_data_is_image_input(
90
image_data = mm_data.get("image")
91
assert image_data is not None
92
94
- if image_count == 1:
95
- assert isinstance(image_data, Image.Image)
96
- else:
97
- assert isinstance(image_data, list) and len(image_data) == image_count
93
+ assert isinstance(image_data, list) and len(image_data) == image_count
94
95
96
def test_parse_chat_messages_single_image(
tests/multimodal/test_utils.py
+34
-25
@@ -9,7 +9,7 @@ import pytest
9
from PIL import Image, ImageChops
10
from transformers import AutoConfig, AutoTokenizer
11
12
-from vllm.multimodal.utils import (async_fetch_image, fetch_image,
12
+from vllm.multimodal.utils import (MediaConnector,
13
repeat_and_pad_placeholder_tokens)
14
15
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
@@ -23,7 +23,12 @@ TEST_IMAGE_URLS = [
23
24
@pytest.fixture(scope="module")
25
def url_images() -> Dict[str, Image.Image]:
26
- return {image_url: fetch_image(image_url) for image_url in TEST_IMAGE_URLS}
26
+ connector = MediaConnector()
27
+
28
+ return {
29
+ image_url: connector.fetch_image(image_url)
30
+ for image_url in TEST_IMAGE_URLS
31
+ }
32
33
34
def get_supported_suffixes() -> Tuple[str, ...]:
@@ -43,8 +48,10 @@ def _image_equals(a: Image.Image, b: Image.Image) -> bool:
48
@pytest.mark.asyncio
49
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
50
async def test_fetch_image_http(image_url: str):
46
- image_sync = fetch_image(image_url)
47
- image_async = await async_fetch_image(image_url)
51
+ connector = MediaConnector()
52
+
53
+ image_sync = connector.fetch_image(image_url)
54
+ image_async = await connector.fetch_image_async(image_url)
55
assert _image_equals(image_sync, image_async)
56
57
@@ -53,6 +60,7 @@ async def test_fetch_image_http(image_url: str):
60
@pytest.mark.parametrize("suffix", get_supported_suffixes())
61
async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
62
image_url: str, suffix: str):
63
+ connector = MediaConnector()
64
url_image = url_images[image_url]
65
66
try:
@@ -75,48 +83,49 @@ async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
83
base64_image = base64.b64encode(f.read()).decode("utf-8")
84
data_url = f"data:{mime_type};base64,{base64_image}"
85
78
- data_image_sync = fetch_image(data_url)
86
+ data_image_sync = connector.fetch_image(data_url)
87
if _image_equals(url_image, Image.open(f)):
88
assert _image_equals(url_image, data_image_sync)
89
else:
90
pass # Lossy format; only check that image can be opened
91
84
- data_image_async = await async_fetch_image(data_url)
92
+ data_image_async = await connector.fetch_image_async(data_url)
93
assert _image_equals(data_image_sync, data_image_async)
94
95
96
@pytest.mark.asyncio
97
@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
98
async def test_fetch_image_local_files(image_url: str):
99
+ connector = MediaConnector()
100
+
101
with TemporaryDirectory() as temp_dir:
92
- origin_image = fetch_image(image_url)
102
+ local_connector = MediaConnector(allowed_local_media_path=temp_dir)
103
+
104
+ origin_image = connector.fetch_image(image_url)
105
origin_image.save(os.path.join(temp_dir, os.path.basename(image_url)),
106
quality=100,
107
icc_profile=origin_image.info.get('icc_profile'))
108
97
- image_async = await async_fetch_image(
98
- f"file://{temp_dir}/{os.path.basename(image_url)}",
99
- allowed_local_media_path=temp_dir)
100
-
101
- image_sync = fetch_image(
102
- f"file://{temp_dir}/{os.path.basename(image_url)}",
103
- allowed_local_media_path=temp_dir)
109
+ image_async = await local_connector.fetch_image_async(
110
+ f"file://{temp_dir}/{os.path.basename(image_url)}")
111
+ image_sync = local_connector.fetch_image(
112
+ f"file://{temp_dir}/{os.path.basename(image_url)}")
113
# Check that the images are equal
114
assert not ImageChops.difference(image_sync, image_async).getbbox()
115
107
- with pytest.raises(ValueError):
108
- await async_fetch_image(
109
- f"file://{temp_dir}/../{os.path.basename(image_url)}",
110
- allowed_local_media_path=temp_dir)
111
- with pytest.raises(ValueError):
112
- await async_fetch_image(
116
+ with pytest.raises(ValueError, match="must be a subpath"):
117
+ await local_connector.fetch_image_async(
118
+ f"file://{temp_dir}/../{os.path.basename(image_url)}")
119
+ with pytest.raises(RuntimeError, match="Cannot load local files"):
120
+ await connector.fetch_image_async(
121
f"file://{temp_dir}/../{os.path.basename(image_url)}")
122
115
- with pytest.raises(ValueError):
116
- fetch_image(f"file://{temp_dir}/../{os.path.basename(image_url)}",
117
- allowed_local_media_path=temp_dir)
118
- with pytest.raises(ValueError):
119
- fetch_image(f"file://{temp_dir}/../{os.path.basename(image_url)}")
123
+ with pytest.raises(ValueError, match="must be a subpath"):
124
+ local_connector.fetch_image(
125
+ f"file://{temp_dir}/../{os.path.basename(image_url)}")
126
+ with pytest.raises(RuntimeError, match="Cannot load local files"):
127
+ connector.fetch_image(
128
+ f"file://{temp_dir}/../{os.path.basename(image_url)}")
129
130
131
@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-mistral-7b-hf"])
vllm/assets/audio.py
+2
-4
@@ -21,12 +21,10 @@ class AudioAsset:
21
name: Literal["winning_call", "mary_had_lamb"]
22
23
@property
24
- def audio_and_sample_rate(self) -> tuple[npt.NDArray, int]:
24
+ def audio_and_sample_rate(self) -> tuple[npt.NDArray, float]:
25
audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg",
26
s3_prefix=ASSET_DIR)
27
- y, sr = librosa.load(audio_path, sr=None)
28
- assert isinstance(sr, int)
29
- return y, sr
27
+ return librosa.load(audio_path, sr=None)
28
29
@property
30
def url(self) -> str:
vllm/entrypoints/chat_utils.py
+57
-72
@@ -6,7 +6,7 @@ from collections import defaultdict, deque
6
from functools import lru_cache, partial
7
from pathlib import Path
8
from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,
9
- Literal, Mapping, Optional, Tuple, TypeVar, Union, cast)
9
+ Literal, Optional, Tuple, TypeVar, Union, cast)
10
11
import jinja2.nodes
12
import transformers.utils.chat_template_utils as hf_chat_utils
@@ -23,6 +23,8 @@ from openai.types.chat import (
23
ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
24
from openai.types.chat import (ChatCompletionMessageToolCallParam,
25
ChatCompletionToolMessageParam)
26
+from openai.types.chat.chat_completion_content_part_input_audio_param import (
27
+ InputAudio)
28
# yapf: enable
29
# pydantic needs the TypedDict from typing_extensions
30
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
@@ -31,11 +33,7 @@ from typing_extensions import Required, TypeAlias, TypedDict
33
from vllm.config import ModelConfig
34
from vllm.logger import init_logger
35
from vllm.multimodal import MultiModalDataDict
34
-from vllm.multimodal.utils import (async_get_and_parse_audio,
35
- async_get_and_parse_image,
36
- async_get_and_parse_video,
37
- get_and_parse_audio, get_and_parse_image,
38
- get_and_parse_video)
36
+from vllm.multimodal.utils import MediaConnector
37
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
38
from vllm.utils import print_warning_once
39
@@ -368,14 +366,17 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
366
self._tokenizer = tokenizer
367
self._allowed_items = (model_config.multimodal_config.limit_per_prompt
368
if model_config.multimodal_config else {})
371
- self._consumed_items = {k: 0 for k in self._allowed_items}
369
373
- self._items: List[_T] = []
370
+ self._items_by_modality = defaultdict[str, list[_T]](list)
371
372
@property
373
def model_config(self) -> ModelConfig:
374
return self._model_config
375
376
+ @property
377
+ def allowed_local_media_path(self):
378
+ return self._model_config.allowed_local_media_path
379
+
380
@staticmethod
381
@lru_cache(maxsize=None)
382
def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
@@ -435,38 +436,19 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
436
else:
437
raise TypeError(f"Unknown modality: {modality}")
438
438
- @staticmethod
439
- def _combine(items: List[MultiModalDataDict]) -> MultiModalDataDict:
440
- mm_lists: Mapping[str, List[object]] = defaultdict(list)
441
-
442
- # Merge all the multi-modal items
443
- for single_mm_data in items:
444
- for mm_key, mm_item in single_mm_data.items():
445
- if isinstance(mm_item, list):
446
- mm_lists[mm_key].extend(mm_item)
447
- else:
448
- mm_lists[mm_key].append(mm_item)
449
-
450
- # Unpack any single item lists for models that don't expect multiple.
451
- return {
452
- mm_key: mm_list[0] if len(mm_list) == 1 else mm_list
453
- for mm_key, mm_list in mm_lists.items()
454
- }
455
-
439
def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
440
"""
441
Add a multi-modal item to the current prompt and returns the
442
placeholder string to use, if any.
443
"""
444
allowed_count = self._allowed_items.get(modality, 1)
462
- current_count = self._consumed_items.get(modality, 0) + 1
445
+ current_count = len(self._items_by_modality[modality]) + 1
446
if current_count > allowed_count:
447
raise ValueError(
448
f"At most {allowed_count} {modality}(s) may be provided in "
449
"one request.")
450
468
- self._consumed_items[modality] = current_count
469
- self._items.append(item)
451
+ self._items_by_modality[modality].append(item)
452
453
return self._placeholder_str(modality, current_count)
454
@@ -475,22 +457,26 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
457
raise NotImplementedError
458
459
478
-class MultiModalItemTracker(BaseMultiModalItemTracker[MultiModalDataDict]):
460
+class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
461
462
def all_mm_data(self) -> Optional[MultiModalDataDict]:
481
- return self._combine(self._items) if self._items else None
463
+ if self._items_by_modality:
464
+ return dict(self._items_by_modality)
465
+
466
+ return None
467
468
def create_parser(self) -> "BaseMultiModalContentParser":
469
return MultiModalContentParser(self)
470
471
487
-class AsyncMultiModalItemTracker(
488
- BaseMultiModalItemTracker[Awaitable[MultiModalDataDict]]):
472
+class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
473
474
async def all_mm_data(self) -> Optional[MultiModalDataDict]:
491
- if self._items:
492
- items = await asyncio.gather(*self._items)
493
- return self._combine(items)
475
+ if self._items_by_modality:
476
+ return {
477
+ modality: await asyncio.gather(*items)
478
+ for modality, items in self._items_by_modality.items()
479
+ }
480
481
return None
482
@@ -522,7 +508,7 @@ class BaseMultiModalContentParser(ABC):
508
raise NotImplementedError
509
510
@abstractmethod
525
- def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
511
+ def parse_input_audio(self, input_audio: InputAudio) -> None:
512
raise NotImplementedError
513
514
@abstractmethod
@@ -537,31 +523,31 @@ class MultiModalContentParser(BaseMultiModalContentParser):
523
524
self._tracker = tracker
525
526
+ self._connector = MediaConnector(
527
+ allowed_local_media_path=tracker.allowed_local_media_path,
528
+ )
529
+
530
def parse_image(self, image_url: str) -> None:
541
- image = get_and_parse_image(image_url,
542
- allowed_local_media_path=self._tracker.
543
- _model_config.allowed_local_media_path)
531
+ image = self._connector.fetch_image(image_url)
532
533
placeholder = self._tracker.add("image", image)
534
self._add_placeholder(placeholder)
535
536
def parse_audio(self, audio_url: str) -> None:
549
- audio = get_and_parse_audio(audio_url)
537
+ audio = self._connector.fetch_audio(audio_url)
538
539
placeholder = self._tracker.add("audio", audio)
540
self._add_placeholder(placeholder)
541
554
- def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
555
- input_audio_data = input_audio.get("data","")
556
- input_audio_format = input_audio.get("format","")
557
- audio_url = f"data:audio/{input_audio_format};base64,{input_audio_data}"
558
- audio = get_and_parse_audio(audio_url)
542
+ def parse_input_audio(self, input_audio: InputAudio) -> None:
543
+ audio_data = input_audio.get("data", "")
544
+ audio_format = input_audio.get("format", "")
545
+ audio_url = f"data:audio/{audio_format};base64,{audio_data}"
546
560
- placeholder = self._tracker.add("audio", audio)
561
- self._add_placeholder(placeholder)
547
+ return self.parse_audio(audio_url)
548
549
def parse_video(self, video_url: str) -> None:
564
- video = get_and_parse_video(video_url)
550
+ video = self._connector.fetch_video(video_url)
551
552
placeholder = self._tracker.add("video", video)
553
self._add_placeholder(placeholder)
@@ -573,33 +559,31 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
559
super().__init__()
560
561
self._tracker = tracker
562
+ self._connector = MediaConnector(
563
+ allowed_local_media_path=tracker.allowed_local_media_path,
564
+ )
565
566
def parse_image(self, image_url: str) -> None:
578
- image_coro = async_get_and_parse_image(
579
- image_url,
580
- allowed_local_media_path=self._tracker._model_config.
581
- allowed_local_media_path)
567
+ image_coro = self._connector.fetch_image_async(image_url)
568
569
placeholder = self._tracker.add("image", image_coro)
570
self._add_placeholder(placeholder)
571
572
def parse_audio(self, audio_url: str) -> None:
587
- audio_coro = async_get_and_parse_audio(audio_url)
573
+ audio_coro = self._connector.fetch_audio_async(audio_url)
574
575
placeholder = self._tracker.add("audio", audio_coro)
576
self._add_placeholder(placeholder)
577
592
- def parse_input_audio(self, input_audio: Dict[str, str]) -> None:
593
- input_audio_data = input_audio.get("data","")
594
- input_audio_format = input_audio.get("format","")
595
- audio_url = f"data:audio/{input_audio_format};base64,{input_audio_data}"
596
- audio_coro = async_get_and_parse_audio(audio_url)
578
+ def parse_input_audio(self, input_audio: InputAudio) -> None:
579
+ audio_data = input_audio.get("data", "")
580
+ audio_format = input_audio.get("format", "")
581
+ audio_url = f"data:audio/{audio_format};base64,{audio_data}"
582
598
- placeholder = self._tracker.add("audio", audio_coro)
599
- self._add_placeholder(placeholder)
583
+ return self.parse_audio(audio_url)
584
585
def parse_video(self, video_url: str) -> None:
602
- video = async_get_and_parse_video(video_url)
586
+ video = self._connector.fetch_video_async(video_url)
587
588
placeholder = self._tracker.add("video", video)
589
self._add_placeholder(placeholder)
@@ -695,10 +679,13 @@ _InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
679
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
680
_VideoParser = partial(cast, ChatCompletionContentPartVideoParam)
681
682
+_ContentPart: TypeAlias = Union[str, Dict[str, str], InputAudio]
683
+
684
# Define a mapping from part types to their corresponding parsing functions.
699
-MM_PARSER_MAP: Dict[str,
700
- Callable[[ChatCompletionContentPartParam],
701
- Union[str, Dict[str,str]]]] = {
685
+MM_PARSER_MAP: Dict[
686
+ str,
687
+ Callable[[ChatCompletionContentPartParam], _ContentPart],
688
+] = {
689
"text":
690
lambda part: _TextParser(part).get("text", ""),
691
"image_url":
@@ -715,8 +702,7 @@ MM_PARSER_MAP: Dict[str,
702
703
704
def _parse_chat_message_content_mm_part(
718
- part: ChatCompletionContentPartParam) -> Tuple[str,
719
- Union[str, Dict[str, str]]]:
705
+ part: ChatCompletionContentPartParam) -> tuple[str, _ContentPart]:
706
"""
707
Parses a given multi-modal content part based on its type.
708
@@ -783,7 +769,7 @@ def _parse_chat_message_content_parts(
769
*,
770
wrap_dicts: bool,
771
) -> List[ConversationMessage]:
786
- content: List[Union[str, Dict[str, str]]] = []
772
+ content = list[_ContentPart]()
773
774
mm_parser = mm_tracker.create_parser()
775
@@ -814,7 +800,7 @@ def _parse_chat_message_content_part(
800
mm_parser: BaseMultiModalContentParser,
801
*,
802
wrap_dicts: bool,
817
-) -> Optional[Union[str, Dict[str, str]]]:
803
+) -> Optional[_ContentPart]:
804
"""Parses a single part of a conversation. If wrap_dicts is True,
805
structured dictionary pieces for texts and images will be
806
wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
@@ -823,8 +809,7 @@ def _parse_chat_message_content_part(
809
with multimodal placeholders.
810
"""
811
if isinstance(part, str): # Handle plain text parts
826
- text = _TextParser(part)
827
- return text
812
+ return part
813
814
# Handle structured dictionary parts
815
part_type, content = _parse_chat_message_content_mm_part(part)
@@ -855,7 +840,7 @@ def _parse_chat_message_content_part(
840
return {'type': 'audio'} if wrap_dicts else None
841
842
if part_type == "input_audio":
858
- dict_content = cast(Dict[str, str], content)
843
+ dict_content = cast(InputAudio, content)
844
mm_parser.parse_input_audio(dict_content)
845
return {'type': 'audio'} if wrap_dicts else None
846
vllm/multimodal/audio.py
+35
-1
@@ -1,10 +1,14 @@
1
+import base64
2
+from io import BytesIO
3
+from pathlib import Path
4
+
5
import numpy as np
6
import numpy.typing as npt
7
8
from vllm.inputs.registry import InputContext
9
from vllm.utils import PlaceholderModule
10
7
-from .base import MultiModalPlugin
11
+from .base import MediaIO, MultiModalPlugin
12
from .inputs import AudioItem, MultiModalData, MultiModalKwargs
13
14
try:
@@ -12,6 +16,11 @@ try:
16
except ImportError:
17
librosa = PlaceholderModule("librosa") # type: ignore[assignment]
18
19
+try:
20
+ import soundfile
21
+except ImportError:
22
+ soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
23
+
24
25
class AudioPlugin(MultiModalPlugin):
26
"""Plugin for audio data."""
@@ -39,3 +48,28 @@ def resample_audio(
48
target_sr: float,
49
) -> npt.NDArray[np.floating]:
50
return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
51
+
52
+
53
+class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
54
+
55
+ def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
56
+ return librosa.load(BytesIO(data), sr=None)
57
+
58
+ def load_base64(
59
+ self,
60
+ media_type: str,
61
+ data: str,
62
+ ) -> tuple[npt.NDArray, float]:
63
+ return self.load_bytes(base64.b64decode(data))
64
+
65
+ def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
66
+ return librosa.load(filepath, sr=None)
67
+
68
+ def encode_base64(self, media: tuple[npt.NDArray, float]) -> str:
69
+ audio, sr = media
70
+
71
+ with BytesIO() as buffer:
72
+ soundfile.write(buffer, audio, sr, format="WAV")
73
+ data = buffer.getvalue()
74
+
75
+ return base64.b64encode(data).decode('utf-8')
vllm/multimodal/base.py
+29
-9
@@ -1,6 +1,7 @@
1
from abc import ABC, abstractmethod
2
from collections import defaultdict
3
-from typing import (TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple,
3
+from pathlib import Path
4
+from typing import (TYPE_CHECKING, Any, Callable, Generic, NamedTuple,
5
Optional, Sequence, Tuple, Type, TypeVar, Union)
6
7
from torch import nn
@@ -118,7 +119,7 @@ class MultiModalPlugin(ABC):
119
self,
120
model_config: "ModelConfig",
121
data: MultiModalData[Any],
121
- mm_processor_kwargs: Optional[Dict[str, Any]],
122
+ mm_processor_kwargs: Optional[dict[str, Any]],
123
) -> MultiModalKwargs:
124
"""
125
Transform the data into a dictionary of model inputs using the
@@ -254,10 +255,10 @@ class MultiModalPlaceholderMap:
255
"""
256
257
class IndexMap(NamedTuple):
257
- src: List[int]
258
- dest: List[int]
258
+ src: list[int]
259
+ dest: list[int]
260
260
- src_ranges: List[range]
261
+ src_ranges: list[range]
262
"""
263
The indices of the multi-modal embeddings that will replace the
264
corresponding placeholder embeddings pointed to by ``dest_ranges``.
@@ -268,7 +269,7 @@ class MultiModalPlaceholderMap:
269
The total number of flattened multi-modal embeddings.
270
"""
271
271
- dest_ranges: List[range]
272
+ dest_ranges: list[range]
273
"""
274
The indices of the placeholder embeddings that will be replaced by the
275
multimodal embeddings.
@@ -288,7 +289,7 @@ class MultiModalPlaceholderMap:
289
@classmethod
290
def from_seq_group(
291
cls, seq_group: "SequenceGroupMetadata", positions: range
291
- ) -> Tuple[Optional[MultiModalDataDict], Dict[str,
292
+ ) -> Tuple[Optional[MultiModalDataDict], dict[str,
293
"MultiModalPlaceholderMap"]]:
294
"""
295
Returns the multi-modal items that intersect with the portion of a
@@ -376,9 +377,9 @@ class MultiModalPlaceholderMap:
377
def append_items_from_seq_group(
378
self,
379
positions: range,
379
- multi_modal_items: List[_T],
380
+ multi_modal_items: list[_T],
381
multi_modal_placeholders: Sequence[PlaceholderRange],
381
- ) -> List[_T]:
382
+ ) -> list[_T]:
383
"""
384
Adds the multi-modal items that intersect ```positions`` to this
385
placeholder map and returns the intersecting items.
@@ -454,3 +455,22 @@ class MultiModalPlaceholderMap:
455
456
return MultiModalPlaceholderMap.IndexMap(src=src_indices,
457
dest=dest_indices)
458
+
459
+
460
+class MediaIO(ABC, Generic[_T]):
461
+
462
+ @abstractmethod
463
+ def load_bytes(self, data: bytes) -> _T:
464
+ raise NotImplementedError
465
+
466
+ @abstractmethod
467
+ def load_base64(self, media_type: str, data: str) -> _T:
468
+ """
469
+ List of media types:
470
+ https://www.iana.org/assignments/media-types/media-types.xhtml
471
+ """
472
+ raise NotImplementedError
473
+
474
+ @abstractmethod
475
+ def load_file(self, filepath: Path) -> _T:
476
+ raise NotImplementedError
vllm/multimodal/image.py
+40
-1
@@ -1,4 +1,7 @@
1
+import base64
2
from functools import lru_cache
3
+from io import BytesIO
4
+from pathlib import Path
5
from typing import TYPE_CHECKING, Any, Dict, Optional
6
7
import torch
@@ -9,7 +12,7 @@ from vllm.logger import init_logger
12
from vllm.transformers_utils.processor import get_image_processor
13
from vllm.utils import is_list_of
14
12
-from .base import MultiModalPlugin
15
+from .base import MediaIO, MultiModalPlugin
16
from .inputs import ImageItem, MultiModalData, MultiModalKwargs
17
18
if TYPE_CHECKING:
@@ -96,3 +99,39 @@ def rescale_image_size(image: Image.Image,
99
if transpose >= 0:
100
image = image.transpose(Image.Transpose(transpose))
101
return image
102
+
103
+
104
+class ImageMediaIO(MediaIO[Image.Image]):
105
+
106
+ def __init__(self, *, image_mode: str = "RGB") -> None:
107
+ super().__init__()
108
+
109
+ self.image_mode = image_mode
110
+
111
+ def load_bytes(self, data: bytes) -> Image.Image:
112
+ image = Image.open(BytesIO(data))
113
+ image.load()
114
+ return image.convert(self.image_mode)
115
+
116
+ def load_base64(self, media_type: str, data: str) -> Image.Image:
117
+ return self.load_bytes(base64.b64decode(data))
118
+
119
+ def load_file(self, filepath: Path) -> Image.Image:
120
+ image = Image.open(filepath)
121
+ image.load()
122
+ return image.convert(self.image_mode)
123
+
124
+ def encode_base64(
125
+ self,
126
+ media: Image.Image,
127
+ *,
128
+ image_format: str = "JPEG",
129
+ ) -> str:
130
+ image = media
131
+
132
+ with BytesIO() as buffer:
133
+ image = image.convert(self.image_mode)
134
+ image.save(buffer, image_format)
135
+ data = buffer.getvalue()
136
+
137
+ return base64.b64encode(data).decode('utf-8')
vllm/multimodal/utils.py
+211
-266
@@ -1,8 +1,7 @@
1
-import base64
2
-import os
1
from functools import lru_cache
4
-from io import BytesIO
5
-from typing import List, Optional, Tuple, TypeVar, Union
2
+from pathlib import Path
3
+from typing import Optional, TypeVar, Union
4
+from urllib.parse import ParseResult, urlparse
5
6
import numpy as np
7
import numpy.typing as npt
@@ -10,283 +9,246 @@ import torch
9
from PIL import Image
10
11
import vllm.envs as envs
13
-from vllm.connections import global_http_connection
12
+from vllm.connections import HTTPConnection, global_http_connection
13
from vllm.logger import init_logger
14
from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
16
-from vllm.utils import PlaceholderModule
15
18
-from .inputs import MultiModalDataDict, PlaceholderRange
19
-
20
-try:
21
- import decord
22
-except ImportError:
23
- decord = PlaceholderModule("decord") # type: ignore[assignment]
24
-
25
-try:
26
- import librosa
27
-except ImportError:
28
- librosa = PlaceholderModule("librosa") # type: ignore[assignment]
29
-
30
-try:
31
- import soundfile
32
-except ImportError:
33
- soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
16
+from .audio import AudioMediaIO
17
+from .base import MediaIO
18
+from .image import ImageMediaIO
19
+from .inputs import PlaceholderRange
20
+from .video import VideoMediaIO
21
22
logger = init_logger(__name__)
23
24
cached_get_tokenizer = lru_cache(get_tokenizer)
25
26
+_M = TypeVar("_M")
27
40
-def _load_image_from_bytes(b: bytes) -> Image.Image:
41
- image = Image.open(BytesIO(b))
42
- image.load()
43
- return image
44
-
45
-
46
-def _is_subpath(image_path: str, allowed_local_media_path: str) -> bool:
47
- # Get the common path
48
- common_path = os.path.commonpath([
49
- os.path.abspath(image_path),
50
- os.path.abspath(allowed_local_media_path)
51
- ])
52
- # Check if the common path is the same as allowed_local_media_path
53
- return common_path == os.path.abspath(allowed_local_media_path)
28
29
+class MediaConnector:
30
56
-def _load_image_from_file(image_url: str,
57
- allowed_local_media_path: str) -> Image.Image:
58
- if not allowed_local_media_path:
59
- raise ValueError("Invalid 'image_url': Cannot load local files without"
60
- "'--allowed-local-media-path'.")
61
- if allowed_local_media_path:
62
- if not os.path.exists(allowed_local_media_path):
63
- raise ValueError(
64
- "Invalid '--allowed-local-media-path': "
65
- f"The path {allowed_local_media_path} does not exist.")
66
- if not os.path.isdir(allowed_local_media_path):
31
+ def __init__(
32
+ self,
33
+ connection: HTTPConnection = global_http_connection,
34
+ *,
35
+ allowed_local_media_path: str = "",
36
+ ) -> None:
37
+ super().__init__()
38
+
39
+ self.connection = connection
40
+
41
+ if allowed_local_media_path:
42
+ allowed_local_media_path_ = Path(allowed_local_media_path)
43
+
44
+ if not allowed_local_media_path_.exists():
45
+ raise ValueError(
46
+ "Invalid `--allowed-local-media-path`: The path "
47
+ f"{allowed_local_media_path_} does not exist.")
48
+ if not allowed_local_media_path_.is_dir():
49
+ raise ValueError(
50
+ "Invalid `--allowed-local-media-path`: The path "
51
+ f"{allowed_local_media_path_} must be a directory.")
52
+ else:
53
+ allowed_local_media_path_ = None
54
+
55
+ self.allowed_local_media_path = allowed_local_media_path_
56
+
57
+ def _load_data_url(
58
+ self,
59
+ url_spec: ParseResult,
60
+ media_io: MediaIO[_M],
61
+ ) -> _M:
62
+ data_spec, data = url_spec.path.split(",", 1)
63
+ media_type, data_type = data_spec.split(";", 1)
64
+
65
+ if data_type != "base64":
66
+ msg = "Only base64 data URLs are supported for now."
67
+ raise NotImplementedError(msg)
68
+
69
+ return media_io.load_base64(media_type, data)
70
+
71
+ def _load_file_url(
72
+ self,
73
+ url_spec: ParseResult,
74
+ media_io: MediaIO[_M],
75
+ ) -> _M:
76
+ allowed_local_media_path = self.allowed_local_media_path
77
+ if allowed_local_media_path is None:
78
+ raise RuntimeError("Cannot load local files without "
79
+ "`--allowed-local-media-path`.")
80
+
81
+ filepath = Path(url_spec.path)
82
+ if allowed_local_media_path not in filepath.resolve().parents:
83
raise ValueError(
68
- "Invalid '--allowed-local-media-path': "
69
- f"The path {allowed_local_media_path} must be a directory.")
70
-
71
- # Only split once and assume the second part is the image path
72
- _, image_path = image_url.split("file://", 1)
73
- if not _is_subpath(image_path, allowed_local_media_path):
74
- raise ValueError(
75
- f"Invalid 'image_url': The file path {image_path} must"
76
- " be a subpath of '--allowed-local-media-path'"
77
- f" '{allowed_local_media_path}'.")
78
-
79
- image = Image.open(image_path)
80
- image.load()
81
- return image
84
+ f"The file path {filepath} must be a subpath "
85
+ f"of `--allowed-local-media-path` {allowed_local_media_path}.")
86
87
+ return media_io.load_file(filepath)
88
84
-def _load_image_from_data_url(image_url: str) -> Image.Image:
85
- # Only split once and assume the second part is the base64 encoded image
86
- _, image_base64 = image_url.split(",", 1)
87
- return load_image_from_base64(image_base64)
88
-
89
-
90
-def fetch_image(image_url: str,
91
- *,
92
- image_mode: str = "RGB",
93
- allowed_local_media_path: str = "") -> Image.Image:
94
- """
95
- Load a PIL image from a HTTP or base64 data URL.
96
-
97
- By default, the image is converted into RGB format.
98
- """
99
- if image_url.startswith('http'):
100
- image_raw = global_http_connection.get_bytes(
101
- image_url,
102
- timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
103
- )
104
- image = _load_image_from_bytes(image_raw)
105
-
106
- elif image_url.startswith('data:image'):
107
- image = _load_image_from_data_url(image_url)
108
- elif image_url.startswith('file://'):
109
- image = _load_image_from_file(image_url, allowed_local_media_path)
110
- else:
111
- raise ValueError("Invalid 'image_url': A valid 'image_url' must start "
112
- "with either 'data:image', 'file://' or 'http'.")
113
-
114
- return image.convert(image_mode)
115
-
116
-
117
-async def async_fetch_image(image_url: str,
118
- *,
119
- image_mode: str = "RGB",
120
- allowed_local_media_path: str = "") -> Image.Image:
121
- """
122
- Asynchronously load a PIL image from a HTTP or base64 data URL.
123
-
124
- By default, the image is converted into RGB format.
125
- """
126
- if image_url.startswith('http'):
127
- image_raw = await global_http_connection.async_get_bytes(
128
- image_url,
129
- timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
130
- )
131
- image = _load_image_from_bytes(image_raw)
132
-
133
- elif image_url.startswith('data:image'):
134
- image = _load_image_from_data_url(image_url)
135
- elif image_url.startswith('file://'):
136
- image = _load_image_from_file(image_url, allowed_local_media_path)
137
- else:
138
- raise ValueError("Invalid 'image_url': A valid 'image_url' must start "
139
- "with either 'data:image', 'file://' or 'http'.")
89
+ def load_from_url(
90
+ self,
91
+ url: str,
92
+ media_io: MediaIO[_M],
93
+ *,
94
+ fetch_timeout: Optional[int] = None,
95
+ ) -> _M:
96
+ url_spec = urlparse(url)
97
141
- return image.convert(image_mode)
98
+ if url_spec.scheme.startswith("http"):
99
+ connection = self.connection
100
+ data = connection.get_bytes(url, timeout=fetch_timeout)
101
102
+ return media_io.load_bytes(data)
103
144
-def _load_video_from_bytes(b: bytes, num_frames: int = 32) -> npt.NDArray:
145
- video_path = BytesIO(b)
146
- vr = decord.VideoReader(video_path, num_threads=1)
147
- total_frame_num = len(vr)
104
+ if url_spec.scheme == "data":
105
+ return self._load_data_url(url_spec, media_io)
106
149
- if total_frame_num > num_frames:
150
- uniform_sampled_frames = np.linspace(0,
151
- total_frame_num - 1,
152
- num_frames,
153
- dtype=int)
154
- frame_idx = uniform_sampled_frames.tolist()
155
- else:
156
- frame_idx = [i for i in range(0, total_frame_num)]
157
- frames = vr.get_batch(frame_idx).asnumpy()
107
+ if url_spec.scheme == "file":
108
+ return self._load_file_url(url_spec, media_io)
109
159
- return frames
110
+ msg = "The URL must be either a HTTP, data or file URL."
111
+ raise ValueError(msg)
112
113
+ async def load_from_url_async(
114
+ self,
115
+ url: str,
116
+ media_io: MediaIO[_M],
117
+ *,
118
+ fetch_timeout: Optional[int] = None,
119
+ ) -> _M:
120
+ url_spec = urlparse(url)
121
162
-def _load_video_from_data_url(video_url: str) -> npt.NDArray:
163
- # Only split once and assume the second part is the base64 encoded video
164
- _, video_base64 = video_url.split(",", 1)
122
+ if url_spec.scheme.startswith("http"):
123
+ connection = self.connection
124
+ data = await connection.async_get_bytes(url, timeout=fetch_timeout)
125
166
- if video_url.startswith("data:video/jpeg;"):
167
- return np.stack([
168
- np.array(load_image_from_base64(frame_base64))
169
- for frame_base64 in video_base64.split(",")
170
- ])
126
+ return media_io.load_bytes(data)
127
172
- return load_video_from_base64(video_base64)
128
+ if url_spec.scheme == "data":
129
+ return self._load_data_url(url_spec, media_io)
130
131
+ if url_spec.scheme == "file":
132
+ return self._load_file_url(url_spec, media_io)
133
175
-def fetch_video(video_url: str, *, num_frames: int = 32) -> npt.NDArray:
176
- """
177
- Load video from a HTTP or base64 data URL.
178
- """
179
- if video_url.startswith('http') or video_url.startswith('https'):
180
- video_raw = global_http_connection.get_bytes(
181
- video_url,
182
- timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
183
- )
184
- video = _load_video_from_bytes(video_raw, num_frames)
185
- elif video_url.startswith('data:video'):
186
- video = _load_video_from_data_url(video_url)
187
- else:
188
- raise ValueError("Invalid 'video_url': A valid 'video_url' must start "
189
- "with either 'data:video' or 'http'.")
190
- return video
134
+ msg = "The URL must be either a HTTP, data or file URL."
135
+ raise ValueError(msg)
136
137
+ def fetch_audio(
138
+ self,
139
+ audio_url: str,
140
+ ) -> tuple[np.ndarray, Union[int, float]]:
141
+ """
142
+ Load audio from a URL.
143
+ """
144
+ audio_io = AudioMediaIO()
145
193
-async def async_fetch_video(video_url: str,
194
- *,
195
- num_frames: int = 32) -> npt.NDArray:
196
- """
197
- Asynchronously load video from a HTTP or base64 data URL.
198
-
199
- By default, the image is converted into RGB format.
200
- """
201
- if video_url.startswith('http') or video_url.startswith('https'):
202
- video_raw = await global_http_connection.async_get_bytes(
203
- video_url,
204
- timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
205
- )
206
- video = _load_video_from_bytes(video_raw, num_frames)
207
- elif video_url.startswith('data:video'):
208
- video = _load_video_from_data_url(video_url)
209
- else:
210
- raise ValueError("Invalid 'video_url': A valid 'video_url' must start "
211
- "with either 'data:video' or 'http'.")
212
- return video
213
-
214
-
215
-def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
216
- """
217
- Load audio from a URL.
218
- """
219
- if audio_url.startswith("http"):
220
- audio_bytes = global_http_connection.get_bytes(
146
+ return self.load_from_url(
147
audio_url,
222
- timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
148
+ audio_io,
149
+ fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
150
)
224
- elif audio_url.startswith("data:audio"):
225
- _, audio_base64 = audio_url.split(",", 1)
226
- audio_bytes = base64.b64decode(audio_base64)
227
- else:
228
- raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start "
229
- "with either 'data:audio' or 'http'.")
230
-
231
- return librosa.load(BytesIO(audio_bytes), sr=None)
151
152
+ async def fetch_audio_async(
153
+ self,
154
+ audio_url: str,
155
+ ) -> tuple[np.ndarray, Union[int, float]]:
156
+ """
157
+ Asynchronously fetch audio from a URL.
158
+ """
159
+ audio_io = AudioMediaIO()
160
234
-async def async_fetch_audio(
235
- audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
236
- """
237
- Asynchronously fetch audio from a URL.
238
- """
239
- if audio_url.startswith("http"):
240
- audio_bytes = await global_http_connection.async_get_bytes(
161
+ return await self.load_from_url_async(
162
audio_url,
242
- timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
163
+ audio_io,
164
+ fetch_timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
165
)
244
- elif audio_url.startswith("data:audio"):
245
- _, audio_base64 = audio_url.split(",", 1)
246
- audio_bytes = base64.b64decode(audio_base64)
247
- else:
248
- raise ValueError("Invalid 'audio_url': A valid 'audio_url' must start "
249
- "with either 'data:audio' or 'http'.")
250
-
251
- return librosa.load(BytesIO(audio_bytes), sr=None)
166
167
+ def fetch_image(
168
+ self,
169
+ image_url: str,
170
+ *,
171
+ image_mode: str = "RGB",
172
+ ) -> Image.Image:
173
+ """
174
+ Load a PIL image from a HTTP or base64 data URL.
175
254
-def get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
255
- audio, sr = fetch_audio(audio_url)
256
- return {"audio": (audio, sr)}
176
+ By default, the image is converted into RGB format.
177
+ """
178
+ image_io = ImageMediaIO(image_mode=image_mode)
179
180
+ return self.load_from_url(
181
+ image_url,
182
+ image_io,
183
+ fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
184
+ )
185
259
-def get_and_parse_image(
186
+ async def fetch_image_async(
187
+ self,
188
image_url: str,
189
*,
262
- allowed_local_media_path: str = "") -> MultiModalDataDict:
263
- image = fetch_image(image_url,
264
- allowed_local_media_path=allowed_local_media_path)
265
- return {"image": image}
266
-
190
+ image_mode: str = "RGB",
191
+ ) -> Image.Image:
192
+ """
193
+ Asynchronously load a PIL image from a HTTP or base64 data URL.
194
268
-def get_and_parse_video(video_url: str) -> MultiModalDataDict:
269
- video = fetch_video(video_url)
270
- return {"video": video}
195
+ By default, the image is converted into RGB format.
196
+ """
197
+ image_io = ImageMediaIO(image_mode=image_mode)
198
199
+ return await self.load_from_url_async(
200
+ image_url,
201
+ image_io,
202
+ fetch_timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
203
+ )
204
273
-async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
274
- audio, sr = await async_fetch_audio(audio_url)
275
- return {"audio": (audio, sr)}
276
-
205
+ def fetch_video(
206
+ self,
207
+ video_url: str,
208
+ *,
209
+ image_mode: str = "RGB",
210
+ num_frames: int = 32,
211
+ ) -> npt.NDArray:
212
+ """
213
+ Load video from a HTTP or base64 data URL.
214
+ """
215
+ image_io = ImageMediaIO(image_mode=image_mode)
216
+ video_io = VideoMediaIO(image_io, num_frames=num_frames)
217
+
218
+ return self.load_from_url(
219
+ video_url,
220
+ video_io,
221
+ fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
222
+ )
223
278
-async def async_get_and_parse_image(
279
- image_url: str,
224
+ async def fetch_video_async(
225
+ self,
226
+ video_url: str,
227
*,
281
- allowed_local_media_path: str = "") -> MultiModalDataDict:
282
- image = await async_fetch_image(
283
- image_url, allowed_local_media_path=allowed_local_media_path)
284
- return {"image": image}
228
+ image_mode: str = "RGB",
229
+ num_frames: int = 32,
230
+ ) -> npt.NDArray:
231
+ """
232
+ Asynchronously load video from a HTTP or base64 data URL.
233
+
234
+ By default, the image is converted into RGB format.
235
+ """
236
+ image_io = ImageMediaIO(image_mode=image_mode)
237
+ video_io = VideoMediaIO(image_io, num_frames=num_frames)
238
+
239
+ return await self.load_from_url_async(
240
+ video_url,
241
+ video_io,
242
+ fetch_timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
243
+ )
244
245
287
-async def async_get_and_parse_video(video_url: str) -> MultiModalDataDict:
288
- video = await async_fetch_video(video_url)
289
- return {"video": video}
246
+global_media_connector = MediaConnector()
247
+"""The global :class:`MediaConnector` instance used by vLLM."""
248
+
249
+fetch_audio = global_media_connector.fetch_audio
250
+fetch_image = global_media_connector.fetch_image
251
+fetch_video = global_media_connector.fetch_video
252
253
254
def encode_audio_base64(
@@ -294,10 +256,8 @@ def encode_audio_base64(
256
sampling_rate: int,
257
) -> str:
258
"""Encode audio as base64."""
297
- buffered = BytesIO()
298
- soundfile.write(buffered, audio, sampling_rate, format="WAV")
299
-
300
- return base64.b64encode(buffered.getvalue()).decode('utf-8')
259
+ audio_io = AudioMediaIO()
260
+ return audio_io.encode_base64((audio, sampling_rate))
261
262
263
def encode_image_base64(
@@ -311,29 +271,14 @@ def encode_image_base64(
271
272
By default, the image is converted into RGB format before being encoded.
273
"""
314
- buffered = BytesIO()
315
- image = image.convert(image_mode)
316
- image.save(buffered, format)
317
- return base64.b64encode(buffered.getvalue()).decode('utf-8')
318
-
319
-
320
-def load_image_from_base64(image: Union[bytes, str]) -> Image.Image:
321
- """Load image from base64 format."""
322
- return _load_image_from_bytes(base64.b64decode(image))
274
+ image_io = ImageMediaIO(image_mode=image_mode)
275
+ return image_io.encode_base64(image, image_format=format)
276
277
278
def encode_video_base64(frames: npt.NDArray) -> str:
326
- base64_frames = []
327
- frames_list = [frames[i] for i in range(frames.shape[0])]
328
- for frame in frames_list:
329
- img_base64 = encode_image_base64(Image.fromarray(frame))
330
- base64_frames.append(img_base64)
331
- return ",".join(base64_frames)
332
-
333
-
334
-def load_video_from_base64(video: Union[bytes, str]) -> npt.NDArray:
335
- """Load video from base64 format."""
336
- return _load_video_from_bytes(base64.b64decode(video))
279
+ image_io = ImageMediaIO()
280
+ video_io = VideoMediaIO(image_io)
281
+ return video_io.encode_base64(frames)
282
283
284
def resolve_visual_encoder_outputs(
@@ -389,7 +334,7 @@ def repeat_and_pad_token(
334
repeat_count: int = 1,
335
pad_token_left: Optional[_T] = None,
336
pad_token_right: Optional[_T] = None,
392
-) -> List[_T]:
337
+) -> list[_T]:
338
replacement = [token] * repeat_count
339
if pad_token_left is not None:
340
replacement = [pad_token_left] + replacement
@@ -402,13 +347,13 @@ def repeat_and_pad_token(
347
def repeat_and_pad_placeholder_tokens(
348
tokenizer: AnyTokenizer,
349
prompt: Optional[str],
405
- prompt_token_ids: List[int],
350
+ prompt_token_ids: list[int],
351
*,
352
placeholder_token_id: int,
408
- repeat_count: Union[int, List[int]],
353
+ repeat_count: Union[int, list[int]],
354
pad_token_left: Optional[int] = None,
355
pad_token_right: Optional[int] = None,
411
-) -> Tuple[Optional[str], List[int], List[PlaceholderRange]]:
356
+) -> tuple[Optional[str], list[int], list[PlaceholderRange]]:
357
if isinstance(repeat_count, int):
358
repeat_count = [repeat_count]
359
@@ -450,8 +395,8 @@ def repeat_and_pad_placeholder_tokens(
395
new_prompt += prompt_parts[i] + replacement_str
396
new_prompt += prompt_parts[-1]
397
453
- new_token_ids: List[int] = []
454
- placeholder_ranges: List[PlaceholderRange] = []
398
+ new_token_ids = list[int]()
399
+ placeholder_ranges = list[PlaceholderRange]()
400
placeholder_token_idx = 0
401
for i, token in enumerate(prompt_token_ids):
402
if token == placeholder_token_id:
@@ -481,7 +426,7 @@ def repeat_and_pad_placeholder_tokens(
426
def consecutive_placeholder_ranges(
427
num_items: int,
428
item_size: int,
484
- initial_offset: int = 0) -> List[PlaceholderRange]:
429
+ initial_offset: int = 0) -> list[PlaceholderRange]:
430
"""Returns a list of consecutive PlaceholderRanges of a fixed size"""
431
432
return [
vllm/multimodal/video.py
+83
-4
@@ -1,23 +1,32 @@
1
-from functools import lru_cache
1
+import base64
2
+from functools import lru_cache, partial
3
+from io import BytesIO
4
+from pathlib import Path
5
from typing import TYPE_CHECKING, Any, Dict, Optional
6
7
import cv2
8
import numpy as np
9
import numpy.typing as npt
10
+from PIL import Image
11
12
from vllm.inputs.registry import InputContext
13
from vllm.logger import init_logger
14
from vllm.transformers_utils.processor import get_video_processor
15
from vllm.transformers_utils.tokenizer import get_tokenizer
12
-from vllm.utils import is_list_of
16
+from vllm.utils import PlaceholderModule, is_list_of
17
14
-from .base import MultiModalData
15
-from .image import ImagePlugin
18
+from .base import MediaIO, MultiModalData
19
+from .image import ImageMediaIO, ImagePlugin
20
from .inputs import MultiModalKwargs, VideoItem
21
22
if TYPE_CHECKING:
23
from vllm.config import ModelConfig
24
25
+try:
26
+ import decord
27
+except ImportError:
28
+ decord = PlaceholderModule("decord") # type: ignore[assignment]
29
+
30
logger = init_logger(__name__)
31
32
cached_get_video_processor = lru_cache(get_video_processor)
@@ -107,3 +116,73 @@ def sample_frames_from_video(frames: npt.NDArray,
116
frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
117
sampled_frames = frames[frame_indices, ...]
118
return sampled_frames
119
+
120
+
121
+class VideoMediaIO(MediaIO[npt.NDArray]):
122
+
123
+ def __init__(
124
+ self,
125
+ image_io: ImageMediaIO,
126
+ *,
127
+ num_frames: int = 32,
128
+ ) -> None:
129
+ super().__init__()
130
+
131
+ self.image_io = image_io
132
+ self.num_frames = num_frames
133
+
134
+ def load_bytes(self, data: bytes) -> npt.NDArray:
135
+ vr = decord.VideoReader(BytesIO(data), num_threads=1)
136
+ total_frame_num = len(vr)
137
+
138
+ num_frames = self.num_frames
139
+ if total_frame_num > num_frames:
140
+ uniform_sampled_frames = np.linspace(0,
141
+ total_frame_num - 1,
142
+ num_frames,
143
+ dtype=int)
144
+ frame_idx = uniform_sampled_frames.tolist()
145
+ else:
146
+ frame_idx = list(range(0, total_frame_num))
147
+
148
+ return vr.get_batch(frame_idx).asnumpy()
149
+
150
+ def load_base64(self, media_type: str, data: str) -> npt.NDArray:
151
+ if media_type.lower() == "video/jpeg":
152
+ load_frame = partial(
153
+ self.image_io.load_base64,
154
+ "image/jpeg",
155
+ )
156
+
157
+ return np.stack([
158
+ np.array(load_frame(frame_data))
159
+ for frame_data in data.split(",")
160
+ ])
161
+
162
+ return self.load_bytes(base64.b64decode(data))
163
+
164
+ def load_file(self, filepath: Path) -> npt.NDArray:
165
+ with filepath.open("rb") as f:
166
+ data = f.read()
167
+
168
+ return self.load_bytes(data)
169
+
170
+ def encode_base64(
171
+ self,
172
+ media: npt.NDArray,
173
+ *,
174
+ video_format: str = "JPEG",
175
+ ) -> str:
176
+ video = media
177
+
178
+ if video_format == "JPEG":
179
+ encode_frame = partial(
180
+ self.image_io.encode_base64,
181
+ image_format=video_format,
182
+ )
183
+
184
+ return ",".join(
185
+ encode_frame(Image.fromarray(frame)) for frame in video)
186
+
187
+ msg = "Only JPEG format is supported for now."
188
+ raise NotImplementedError(msg)