@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5984499e47
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Doc] Expand Multimodal API Reference (#11852)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 9, 2025 at 01:14 UTC 5984499e473c387c17904aa9933b8ed080621ca6
9 files changed +139 -71
docs/source/api/multimodal/index.md
+8 -53
@@ -2,10 +2,6 @@
2
3 # Multi-Modality
4
5 -```{eval-rst}
6 -.. currentmodule:: vllm.multimodal
7 -```
8 -
5 vLLM provides experimental support for multi-modal models through the {mod}`vllm.multimodal` package.
6
7 Multi-modal inputs can be passed alongside text and token prompts to [supported models](#supported-mm-models)
@@ -13,61 +9,20 @@ via the `multi_modal_data` field in {class}`vllm.inputs.PromptType`.
9
10 Looking to add your own multi-modal model? Please follow the instructions listed [here](#enabling-multimodal-inputs).
11
16 -
12 ## Module Contents
13
19 -```{eval-rst}
20 -.. automodule:: vllm.multimodal
21 -```
22 -
23 -### Registry
24 -
14 ```{eval-rst}
15 .. autodata:: vllm.multimodal.MULTIMODAL_REGISTRY
16 ```
17
29 -```{eval-rst}
30 -.. autoclass:: vllm.multimodal.MultiModalRegistry
31 - :members:
32 - :show-inheritance:
33 -```
34 -
35 -### Base Classes
36 -
37 -```{eval-rst}
38 -.. automodule:: vllm.multimodal.base
39 - :members:
40 - :show-inheritance:
41 -```
18 +## Submodules
19
43 -### Input Classes
20 +```{toctree}
21 +:maxdepth: 1
22
45 -```{eval-rst}
46 -.. automodule:: vllm.multimodal.inputs
47 - :members:
48 - :show-inheritance:
49 -```
50 -
51 -### Audio Classes
52 -
53 -```{eval-rst}
54 -.. automodule:: vllm.multimodal.audio
55 - :members:
56 - :show-inheritance:
57 -```
58 -
59 -### Image Classes
60 -
61 -```{eval-rst}
62 -.. automodule:: vllm.multimodal.image
63 - :members:
64 - :show-inheritance:
65 -```
66 -
67 -### Video Classes
68 -
69 -```{eval-rst}
70 -.. automodule:: vllm.multimodal.video
71 - :members:
72 - :show-inheritance:
23 +inputs
24 +parse
25 +processing
26 +profiling
27 +registry
28 ```
docs/source/api/multimodal/inputs.md new
+49
@@ -0,0 +1,49 @@
1 +# Input Definitions
2 +
3 +## User-facing inputs
4 +
5 +```{eval-rst}
6 +.. autodata:: vllm.multimodal.MultiModalDataDict
7 +```
8 +
9 +## Internal data structures
10 +
11 +```{eval-rst}
12 +.. autoclass:: vllm.multimodal.inputs.PlaceholderRange
13 + :members:
14 + :show-inheritance:
15 +```
16 +
17 +```{eval-rst}
18 +.. autodata:: vllm.multimodal.inputs.NestedTensors
19 +```
20 +
21 +```{eval-rst}
22 +.. autoclass:: vllm.multimodal.inputs.MultiModalFieldElem
23 + :members:
24 + :show-inheritance:
25 +```
26 +
27 +```{eval-rst}
28 +.. autoclass:: vllm.multimodal.inputs.MultiModalFieldConfig
29 + :members:
30 + :show-inheritance:
31 +```
32 +
33 +```{eval-rst}
34 +.. autoclass:: vllm.multimodal.inputs.MultiModalKwargsItem
35 + :members:
36 + :show-inheritance:
37 +```
38 +
39 +```{eval-rst}
40 +.. autoclass:: vllm.multimodal.inputs.MultiModalKwargs
41 + :members:
42 + :show-inheritance:
43 +```
44 +
45 +```{eval-rst}
46 +.. autoclass:: vllm.multimodal.inputs.MultiModalInputsV2
47 + :members:
48 + :show-inheritance:
49 +```
docs/source/api/multimodal/parse.md new
+9
@@ -0,0 +1,9 @@
1 +# Data Parsing
2 +
3 +## Module Contents
4 +
5 +```{eval-rst}
6 +.. automodule:: vllm.multimodal.parse
7 + :members:
8 + :member-order: bysource
9 +```
docs/source/api/multimodal/processing.md new
+9
@@ -0,0 +1,9 @@
1 +# Data Processing
2 +
3 +## Module Contents
4 +
5 +```{eval-rst}
6 +.. automodule:: vllm.multimodal.processing
7 + :members:
8 + :member-order: bysource
9 +```
docs/source/api/multimodal/profiling.md new
+9
@@ -0,0 +1,9 @@
1 +# Memory Profiling
2 +
3 +## Module Contents
4 +
5 +```{eval-rst}
6 +.. automodule:: vllm.multimodal.profiling
7 + :members:
8 + :member-order: bysource
9 +```
docs/source/api/multimodal/registry.md new
+9
@@ -0,0 +1,9 @@
1 +# Registry
2 +
3 +## Module Contents
4 +
5 +```{eval-rst}
6 +.. automodule:: vllm.multimodal.registry
7 + :members:
8 + :member-order: bysource
9 +```
vllm/multimodal/parse.py
+21 -10
@@ -13,14 +13,16 @@ from vllm.utils import is_list_of
13
14 from .audio import resample_audio
15 from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
16 - ImageItem, ModalityData, MultiModalDataDict,
17 - NestedTensors, VideoItem)
16 + ImageItem, ModalityData, MultiModalDataDict, VideoItem)
17
18 _T = TypeVar("_T")
19 _I = TypeVar("_I")
20
21
22 class ModalityDataItems(ABC, Generic[_T, _I]):
23 + """
24 + Represents data items for a modality in :class:`MultiModalDataItems`.
25 + """
26
27 def __init__(self, data: _T, modality: str) -> None:
28 super().__init__()
@@ -69,6 +71,7 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
71
72
73 class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
74 + """Base class for data items that are arranged in a list."""
75
76 def get_count(self) -> int:
77 return len(self.data)
@@ -83,7 +86,12 @@ class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
86 return {}
87
88
86 -class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
89 +class EmbeddingItems(ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]],
90 + torch.Tensor]):
91 + """
92 + Base class for data items that are expressed as a batched embedding tensor,
93 + or a list of embedding tensors (one per item).
94 + """
95
96 def get_count(self) -> int:
97 return len(self.data)
@@ -109,7 +117,7 @@ class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
117
118 class AudioEmbeddingItems(EmbeddingItems):
119
112 - def __init__(self, data: NestedTensors) -> None:
120 + def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
121 super().__init__(data, "audio")
122
123
@@ -137,7 +145,7 @@ class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
145
146 class ImageEmbeddingItems(EmbeddingItems):
147
140 - def __init__(self, data: NestedTensors) -> None:
148 + def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
149 super().__init__(data, "image")
150
151
@@ -163,7 +171,7 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
171
172 class VideoEmbeddingItems(EmbeddingItems):
173
166 - def __init__(self, data: NestedTensors) -> None:
174 + def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
175 super().__init__(data, "video")
176
177
@@ -172,8 +180,8 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
180
181 class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
182 """
175 - As :class:`MultiModalDataDict`, but normalized such that each entry
176 - corresponds to a list.
183 + As :data:`~vllm.multimodal.inputs.MultiModalDataDict`, but normalized
184 + such that each entry corresponds to a list.
185 """
186
187 def get_count(self, modality: str, *, strict: bool = True) -> int:
@@ -226,7 +234,8 @@ ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
234
235 class MultiModalDataParser:
236 """
229 - Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
237 + Parses :data:`~vllm.multimodal.inputs.MultiModalDataDict` into
238 + :class:`MultiModalDataItems`.
239
240 Args:
241 target_sr (float, optional): Enables automatic resampling of audio
@@ -238,7 +247,9 @@ class MultiModalDataParser:
247
248 self.target_sr = target_sr
249
241 - def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
250 + def _is_embeddings(
251 + self, data: object
252 + ) -> TypeGuard[Union[torch.Tensor, list[torch.Tensor]]]:
253 if isinstance(data, torch.Tensor):
254 return data.ndim == 3
255 if is_list_of(data, torch.Tensor):
vllm/multimodal/processing.py
+20 -6
@@ -33,20 +33,24 @@ _PromptSeq = Union[str, list[int]]
33
34 @dataclass
35 class PromptReplacement:
36 + """
37 + Defines how to replace portions of an input prompt with placeholder tokens.
38 + """
39 +
40 modality: str
41 """The modality for which the replacement is made."""
42
43 target: _PromptSeq
40 - """The text or token sequence to find and replace."""
44 + """The token sequence (or text) to find and replace."""
45
46 replacement: Union[Callable[[int], _PromptSeq],
47 _PromptSeq] = field(repr=False)
48 """
45 - Given the index of the processed item within :attr:`modality`, output the
46 - replacement text or token sequence.
49 + Given the index of the processed item within :attr:`modality`,
50 + output the replacement token sequence (or text).
51
48 - For convenience, you can pass in the replacement instead of a function
49 - if it does not depend on the input.
52 + For convenience, you can directly pass in the replacement token sequence
53 + (or text) instead of a function if it does not depend on the input.
54 """
55
56 def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptReplacement":
@@ -132,6 +136,11 @@ class _BoundPromptSequence:
136
137 @dataclass
138 class BoundPromptReplacement:
139 + """
140 + A :class:`PromptReplacement` bound to a tokenizer to automatically
141 + convert :attr:`target` and the result of :meth:`get_replacement` between
142 + token sequence and text representations.
143 + """
144 tokenizer: AnyTokenizer = field(repr=False)
145 modality: str
146
@@ -144,6 +153,7 @@ class BoundPromptReplacement:
153
154 @property
155 def target(self) -> _BoundPromptSequence:
156 + """The token sequence (or text) to find and replace."""
157 target = self._target
158
159 return _BoundPromptSequence(
@@ -153,6 +163,10 @@ class BoundPromptReplacement:
163 )
164
165 def get_replacement(self, item_idx: int) -> _BoundPromptSequence:
166 + """
167 + Given the index of the processed item within :attr:`modality`,
168 + output the replacement token sequence (or text).
169 + """
170 replacement = self._replacement
171 if callable(replacement):
172 cache_key = item_idx
@@ -528,7 +542,7 @@ class ProcessingCache:
542
543
544 class BaseProcessingInfo:
531 - """Base class containing information to perform processing."""
545 + """Base class to provide the information necessary for data processing."""
546
547 def __init__(self, ctx: InputProcessingContext) -> None:
548 super().__init__()
vllm/multimodal/profiling.py
+5 -2
@@ -19,7 +19,10 @@ logger = init_logger(__name__)
19
20 @dataclass
21 class ProcessorInputs:
22 - """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
22 + """
23 + Represents the keyword arguments to
24 + :meth:`vllm.multimodal.processing.BaseMultiModalProcessor.apply`.
25 + """
26 prompt_text: str
27 mm_data: MultiModalDataDict
28 hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
@@ -47,7 +50,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
50 ) -> ProcessorInputs:
51 """
52 Build the input which, after processing, results in
50 - `self.info.get_mm_max_tokens_per_item()` placeholder tokens.
53 + :code:`self.info.get_mm_max_tokens_per_item()` placeholder tokens.
54 """
55 raise NotImplementedError
56