A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Doc] Guide for adding multi-modal plugins (#6205)
Cyrus Leung committed
Jul 10, 2024 at 14:55 UTC
8a924d2248dedb620eb9a32ca5c9f97ab525aaf5
7 files changed
+64
-23
docs/source/_templates/sections/header.html
+1
@@ -5,6 +5,7 @@
5
justify-content: center;
6
align-items: center;
7
font-size: 16px;
8
+ padding: 0 6px 0 6px;
9
}
10
.notification-bar p {
11
margin: 0;
docs/source/dev/multimodal/adding_multimodal_plugin.rst
new
+17
@@ -0,0 +1,17 @@
1
+.. _adding_multimodal_plugin:
2
+
3
+Adding a Multimodal Plugin
4
+==========================
5
+
6
+This document teaches you how to add a new modality to vLLM.
7
+
8
+Each modality in vLLM is represented by a :class:`~vllm.multimodal.MultiModalPlugin` and registered to :data:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
9
+For vLLM to recognize a new modality type, you have to create a new plugin and then pass it to :meth:`~vllm.multimodal.MultiModalRegistry.register_plugin`.
10
+
11
+The remainder of this document details how to define custom :class:`~vllm.multimodal.MultiModalPlugin` s.
12
+
13
+.. note::
14
+ This article is a work in progress.
15
+
16
+..
17
+ TODO: Add more instructions on how to add new plugins once embeddings is in.
docs/source/dev/multimodal/multimodal_index.rst
+16
-8
@@ -7,17 +7,21 @@ Multi-Modality
7
8
vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10
-Multi-modal input can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
10
+Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11
via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptStrictInputs`.
12
13
-.. note::
14
- ``multi_modal_data`` can accept keys and values beyond the builtin ones, as long as a customized plugin is registered through
15
- the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
13
+Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14
+by following :ref:`this guide <adding_multimodal_plugin>`.
15
17
-To implement a new multi-modal model in vLLM, please follow :ref:`this guide <enabling_multimodal_inputs>`.
16
+Looking to add your own multi-modal model? Please follow the instructions listed :ref:`here <enabling_multimodal_inputs>`.
17
19
-..
20
- TODO: Add more instructions on how to add new plugins once embeddings is in.
18
+Guides
19
+++++++
20
+
21
+.. toctree::
22
+ :maxdepth: 1
23
+
24
+ adding_multimodal_plugin
25
26
Module Contents
27
+++++++++++++++
@@ -36,10 +40,14 @@ Registry
40
Base Classes
41
------------
42
39
-.. autoclass:: vllm.multimodal.MultiModalDataDict
43
+.. autodata:: vllm.multimodal.BatchedTensors
44
+
45
+.. autoclass:: vllm.multimodal.MultiModalDataBuiltins
46
:members:
47
:show-inheritance:
48
49
+.. autodata:: vllm.multimodal.MultiModalDataDict
50
+
51
.. autoclass:: vllm.multimodal.MultiModalInputs
52
:members:
53
:show-inheritance:
vllm/multimodal/__init__.py
+3
-2
@@ -1,5 +1,5 @@
1
-from .base import (BatchedTensors, MultiModalDataDict, MultiModalInputs,
2
- MultiModalPlugin)
1
+from .base import (BatchedTensors, MultiModalDataBuiltins, MultiModalDataDict,
2
+ MultiModalInputs, MultiModalPlugin)
3
from .registry import MultiModalRegistry
4
5
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -13,6 +13,7 @@ See also:
13
14
__all__ = [
15
"BatchedTensors",
16
+ "MultiModalDataBuiltins",
17
"MultiModalDataDict",
18
"MultiModalInputs",
19
"MultiModalPlugin",
vllm/multimodal/base.py
+13
-8
@@ -43,9 +43,6 @@ class MultiModalInputs(_MultiModalInputsBase):
43
*,
44
device: torch.types.Device,
45
) -> BatchedTensors:
46
- # Avoid initializing CUDA too early
47
- import torch
48
-
46
unbatched_shape = tensors[0].shape[1:]
47
48
for tensor in tensors:
@@ -84,16 +81,21 @@ class MultiModalInputs(_MultiModalInputsBase):
81
82
83
class MultiModalDataBuiltins(TypedDict, total=False):
84
+ """Modality types that are predefined by vLLM."""
85
+
86
image: Image.Image
87
+ """The input image."""
88
89
90
MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
91
"""
92
A dictionary containing an item for each modality type to input.
93
94
-The data belonging to each modality is converted into keyword arguments
95
-to the model by the corresponding mapper. By default, the mapper of
96
-the corresponding plugin with the same modality key is applied.
94
+Note:
95
+ This dictionary also accepts modality keys defined outside
96
+ :class:`MultiModalDataBuiltins` as long as a customized plugin is registered
97
+ through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
98
+ Read more on that :ref:`here <adding_multimodal_plugin>`.
99
"""
100
101
MultiModalInputMapper = Callable[[InputContext, object], MultiModalInputs]
@@ -123,6 +125,9 @@ class MultiModalPlugin(ABC):
125
process the same data differently). This registry is in turn used by
126
:class:`~MultiModalRegistry` which acts at a higher level
127
(i.e., the modality of the data).
128
+
129
+ See also:
130
+ :ref:`adding_multimodal_plugin`
131
"""
132
133
def __init__(self) -> None:
@@ -183,8 +188,8 @@ class MultiModalPlugin(ABC):
188
def map_input(self, model_config: ModelConfig,
189
data: object) -> MultiModalInputs:
190
"""
186
- Apply an input mapper to a data passed
187
- to the model, transforming the data into a dictionary of model inputs.
191
+ Transform the data into a dictionary of model inputs using the
192
+ input mapper registered for that model.
193
194
The model is identified by ``model_config``.
195
vllm/multimodal/image.py
+1
@@ -100,6 +100,7 @@ def repeat_and_pad_image_tokens(
100
101
102
class ImagePlugin(MultiModalPlugin):
103
+ """Plugin for image data."""
104
105
def get_data_key(self) -> str:
106
return "image"
vllm/multimodal/registry.py
+13
-5
@@ -15,10 +15,8 @@ logger = init_logger(__name__)
15
16
class MultiModalRegistry:
17
"""
18
- A registry to dispatch data processing
19
- according to its modality and the target model.
20
-
21
- The registry handles both external and internal data input.
18
+ A registry that dispatches data processing to the
19
+ :class:`~vllm.multimodal.MultiModalPlugin` for each modality.
20
"""
21
22
DEFAULT_PLUGINS = (ImagePlugin(), )
@@ -30,6 +28,12 @@ class MultiModalRegistry:
28
self._plugins = {p.get_data_key(): p for p in plugins}
29
30
def register_plugin(self, plugin: MultiModalPlugin) -> None:
31
+ """
32
+ Register a multi-modal plugin so it can be recognized by vLLM.
33
+
34
+ See also:
35
+ :ref:`adding_multimodal_plugin`
36
+ """
37
data_type_key = plugin.get_data_key()
38
39
if data_type_key in self._plugins:
@@ -75,7 +79,11 @@ class MultiModalRegistry:
79
data: MultiModalDataDict) -> MultiModalInputs:
80
"""
81
Apply an input mapper to the data passed to the model.
78
-
82
+
83
+ The data belonging to each modality is passed to the corresponding
84
+ plugin which in turn converts the data into into keyword arguments
85
+ via the input mapper registered for that model.
86
+
87
See :meth:`MultiModalPlugin.map_input` for more details.
88
"""
89
merged_dict: Dict[str, torch.Tensor] = {}