A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Doc] Expand Multimodal API Reference (#11852)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 9, 2025 at 01:14 UTC
5984499e473c387c17904aa9933b8ed080621ca6
9 files changed
+139
-71
docs/source/api/multimodal/index.md
+8
-53
@@ -2,10 +2,6 @@
2
3
# Multi-Modality
4
5
-```{eval-rst}
6
-.. currentmodule:: vllm.multimodal
7
-```
8
-
5
vLLM provides experimental support for multi-modal models through the {mod}`vllm.multimodal` package.
6
7
Multi-modal inputs can be passed alongside text and token prompts to [supported models](#supported-mm-models)
@@ -13,61 +9,20 @@ via the `multi_modal_data` field in {class}`vllm.inputs.PromptType`.
9
10
Looking to add your own multi-modal model? Please follow the instructions listed [here](#enabling-multimodal-inputs).
11
16
-
12
## Module Contents
13
19
-```{eval-rst}
20
-.. automodule:: vllm.multimodal
21
-```
22
-
23
-### Registry
24
-
14
```{eval-rst}
15
.. autodata:: vllm.multimodal.MULTIMODAL_REGISTRY
16
```
17
29
-```{eval-rst}
30
-.. autoclass:: vllm.multimodal.MultiModalRegistry
31
- :members:
32
- :show-inheritance:
33
-```
34
-
35
-### Base Classes
36
-
37
-```{eval-rst}
38
-.. automodule:: vllm.multimodal.base
39
- :members:
40
- :show-inheritance:
41
-```
18
+## Submodules
19
43
-### Input Classes
20
+```{toctree}
21
+:maxdepth: 1
22
45
-```{eval-rst}
46
-.. automodule:: vllm.multimodal.inputs
47
- :members:
48
- :show-inheritance:
49
-```
50
-
51
-### Audio Classes
52
-
53
-```{eval-rst}
54
-.. automodule:: vllm.multimodal.audio
55
- :members:
56
- :show-inheritance:
57
-```
58
-
59
-### Image Classes
60
-
61
-```{eval-rst}
62
-.. automodule:: vllm.multimodal.image
63
- :members:
64
- :show-inheritance:
65
-```
66
-
67
-### Video Classes
68
-
69
-```{eval-rst}
70
-.. automodule:: vllm.multimodal.video
71
- :members:
72
- :show-inheritance:
23
+inputs
24
+parse
25
+processing
26
+profiling
27
+registry
28
```
docs/source/api/multimodal/inputs.md
new
+49
@@ -0,0 +1,49 @@
1
+# Input Definitions
2
+
3
+## User-facing inputs
4
+
5
+```{eval-rst}
6
+.. autodata:: vllm.multimodal.MultiModalDataDict
7
+```
8
+
9
+## Internal data structures
10
+
11
+```{eval-rst}
12
+.. autoclass:: vllm.multimodal.inputs.PlaceholderRange
13
+ :members:
14
+ :show-inheritance:
15
+```
16
+
17
+```{eval-rst}
18
+.. autodata:: vllm.multimodal.inputs.NestedTensors
19
+```
20
+
21
+```{eval-rst}
22
+.. autoclass:: vllm.multimodal.inputs.MultiModalFieldElem
23
+ :members:
24
+ :show-inheritance:
25
+```
26
+
27
+```{eval-rst}
28
+.. autoclass:: vllm.multimodal.inputs.MultiModalFieldConfig
29
+ :members:
30
+ :show-inheritance:
31
+```
32
+
33
+```{eval-rst}
34
+.. autoclass:: vllm.multimodal.inputs.MultiModalKwargsItem
35
+ :members:
36
+ :show-inheritance:
37
+```
38
+
39
+```{eval-rst}
40
+.. autoclass:: vllm.multimodal.inputs.MultiModalKwargs
41
+ :members:
42
+ :show-inheritance:
43
+```
44
+
45
+```{eval-rst}
46
+.. autoclass:: vllm.multimodal.inputs.MultiModalInputsV2
47
+ :members:
48
+ :show-inheritance:
49
+```
docs/source/api/multimodal/parse.md
new
+9
@@ -0,0 +1,9 @@
1
+# Data Parsing
2
+
3
+## Module Contents
4
+
5
+```{eval-rst}
6
+.. automodule:: vllm.multimodal.parse
7
+ :members:
8
+ :member-order: bysource
9
+```
docs/source/api/multimodal/processing.md
new
+9
@@ -0,0 +1,9 @@
1
+# Data Processing
2
+
3
+## Module Contents
4
+
5
+```{eval-rst}
6
+.. automodule:: vllm.multimodal.processing
7
+ :members:
8
+ :member-order: bysource
9
+```
docs/source/api/multimodal/profiling.md
new
+9
@@ -0,0 +1,9 @@
1
+# Memory Profiling
2
+
3
+## Module Contents
4
+
5
+```{eval-rst}
6
+.. automodule:: vllm.multimodal.profiling
7
+ :members:
8
+ :member-order: bysource
9
+```
docs/source/api/multimodal/registry.md
new
+9
@@ -0,0 +1,9 @@
1
+# Registry
2
+
3
+## Module Contents
4
+
5
+```{eval-rst}
6
+.. automodule:: vllm.multimodal.registry
7
+ :members:
8
+ :member-order: bysource
9
+```
vllm/multimodal/parse.py
+21
-10
@@ -13,14 +13,16 @@ from vllm.utils import is_list_of
13
14
from .audio import resample_audio
15
from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
16
- ImageItem, ModalityData, MultiModalDataDict,
17
- NestedTensors, VideoItem)
16
+ ImageItem, ModalityData, MultiModalDataDict, VideoItem)
17
18
_T = TypeVar("_T")
19
_I = TypeVar("_I")
20
21
22
class ModalityDataItems(ABC, Generic[_T, _I]):
23
+ """
24
+ Represents data items for a modality in :class:`MultiModalDataItems`.
25
+ """
26
27
def __init__(self, data: _T, modality: str) -> None:
28
super().__init__()
@@ -69,6 +71,7 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
71
72
73
class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
74
+ """Base class for data items that are arranged in a list."""
75
76
def get_count(self) -> int:
77
return len(self.data)
@@ -83,7 +86,12 @@ class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
86
return {}
87
88
86
-class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
89
+class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
90
+ torch.Tensor]):
91
+ """
92
+ Base class for data items that are expressed as a batched embedding tensor,
93
+ or a list of embedding tensors (one per item).
94
+ """
95
96
def get_count(self) -> int:
97
return len(self.data)
@@ -109,7 +117,7 @@ class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
117
118
class AudioEmbeddingItems(EmbeddingItems):
119
112
- def __init__(self, data: NestedTensors) -> None:
120
+ def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
121
super().__init__(data, "audio")
122
123
@@ -137,7 +145,7 @@ class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
145
146
class ImageEmbeddingItems(EmbeddingItems):
147
140
- def __init__(self, data: NestedTensors) -> None:
148
+ def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
149
super().__init__(data, "image")
150
151
@@ -163,7 +171,7 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
171
172
class VideoEmbeddingItems(EmbeddingItems):
173
166
- def __init__(self, data: NestedTensors) -> None:
174
+ def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
175
super().__init__(data, "video")
176
177
@@ -172,8 +180,8 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
180
181
class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
182
"""
175
- As :class:`MultiModalDataDict`, but normalized such that each entry
176
- corresponds to a list.
183
+ As :data:`~vllm.multimodal.inputs.MultiModalDataDict`, but normalized
184
+ such that each entry corresponds to a list.
185
"""
186
187
def get_count(self, modality: str, *, strict: bool = True) -> int:
@@ -226,7 +234,8 @@ ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
234
235
class MultiModalDataParser:
236
"""
229
- Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
237
+ Parses :data:`~vllm.multimodal.inputs.MultiModalDataDict` into
238
+ :class:`MultiModalDataItems`.
239
240
Args:
241
target_sr (float, optional): Enables automatic resampling of audio
@@ -238,7 +247,9 @@ class MultiModalDataParser:
247
248
self.target_sr = target_sr
249
241
- def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
250
+ def _is_embeddings(
251
+ self, data: object
252
+ ) -> TypeGuard[Union[torch.Tensor, list[torch.Tensor]]]:
253
if isinstance(data, torch.Tensor):
254
return data.ndim == 3
255
if is_list_of(data, torch.Tensor):
vllm/multimodal/processing.py
+20
-6
@@ -33,20 +33,24 @@ _PromptSeq = Union[str, list[int]]
33
34
@dataclass
35
class PromptReplacement:
36
+ """
37
+ Defines how to replace portions of an input prompt with placeholder tokens.
38
+ """
39
+
40
modality: str
41
"""The modality for which the replacement is made."""
42
43
target: _PromptSeq
40
- """The text or token sequence to find and replace."""
44
+ """The token sequence (or text) to find and replace."""
45
46
replacement: Union[Callable[[int], _PromptSeq],
47
_PromptSeq] = field(repr=False)
48
"""
45
- Given the index of the processed item within :attr:`modality`, output the
46
- replacement text or token sequence.
49
+ Given the index of the processed item within :attr:`modality`,
50
+ output the replacement token sequence (or text).
51
48
- For convenience, you can pass in the replacement instead of a function
49
- if it does not depend on the input.
52
+ For convenience, you can directly pass in the replacement token sequence
53
+ (or text) instead of a function if it does not depend on the input.
54
"""
55
56
def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptReplacement":
@@ -132,6 +136,11 @@ class _BoundPromptSequence:
136
137
@dataclass
138
class BoundPromptReplacement:
139
+ """
140
+ A :class:`PromptReplacement` bound to a tokenizer to automatically
141
+ convert :attr:`target` and the result of :meth:`get_replacement` between
142
+ token sequence and text representations.
143
+ """
144
tokenizer: AnyTokenizer = field(repr=False)
145
modality: str
146
@@ -144,6 +153,7 @@ class BoundPromptReplacement:
153
154
@property
155
def target(self) -> _BoundPromptSequence:
156
+ """The token sequence (or text) to find and replace."""
157
target = self._target
158
159
return _BoundPromptSequence(
@@ -153,6 +163,10 @@ class BoundPromptReplacement:
163
)
164
165
def get_replacement(self, item_idx: int) -> _BoundPromptSequence:
166
+ """
167
+ Given the index of the processed item within :attr:`modality`,
168
+ output the replacement token sequence (or text).
169
+ """
170
replacement = self._replacement
171
if callable(replacement):
172
cache_key = item_idx
@@ -528,7 +542,7 @@ class ProcessingCache:
542
543
544
class BaseProcessingInfo:
531
- """Base class containing information to perform processing."""
545
+ """Base class to provide the information necessary for data processing."""
546
547
def __init__(self, ctx: InputProcessingContext) -> None:
548
super().__init__()
vllm/multimodal/profiling.py
+5
-2
@@ -19,7 +19,10 @@ logger = init_logger(__name__)
19
20
@dataclass
21
class ProcessorInputs:
22
- """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
22
+ """
23
+ Represents the keyword arguments to
24
+ :meth:`vllm.multimodal.processing.BaseMultiModalProcessor.apply`.
25
+ """
26
prompt_text: str
27
mm_data: MultiModalDataDict
28
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
@@ -47,7 +50,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
50
) -> ProcessorInputs:
51
"""
52
Build the input which, after processing, results in
50
- `self.info.get_mm_max_tokens_per_item()` placeholder tokens.
53
+ :code:`self.info.get_mm_max_tokens_per_item()` placeholder tokens.
54
"""
55
raise NotImplementedError
56