@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a924d2248
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Doc] Guide for adding multi-modal plugins (#6205)

Cyrus Leung committed Jul 10, 2024 at 14:55 UTC 8a924d2248dedb620eb9a32ca5c9f97ab525aaf5
7 files changed +64 -23
docs/source/_templates/sections/header.html
+1
@@ -5,6 +5,7 @@
5 justify-content: center;
6 align-items: center;
7 font-size: 16px;
8 + padding: 0 6px 0 6px;
9 }
10 .notification-bar p {
11 margin: 0;
docs/source/dev/multimodal/adding_multimodal_plugin.rst new
+17
@@ -0,0 +1,17 @@
1 +.. _adding_multimodal_plugin:
2 +
3 +Adding a Multimodal Plugin
4 +==========================
5 +
6 +This document teaches you how to add a new modality to vLLM.
7 +
8 +Each modality in vLLM is represented by a :class:`~vllm.multimodal.MultiModalPlugin` and registered to :data:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
9 +For vLLM to recognize a new modality type, you have to create a new plugin and then pass it to :meth:`~vllm.multimodal.MultiModalRegistry.register_plugin`.
10 +
11 +The remainder of this document details how to define custom :class:`~vllm.multimodal.MultiModalPlugin` s.
12 +
13 +.. note::
14 + This article is a work in progress.
15 +
16 +..
17 + TODO: Add more instructions on how to add new plugins once embeddings is in.
docs/source/dev/multimodal/multimodal_index.rst
+16 -8
@@ -7,17 +7,21 @@ Multi-Modality
7
8 vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10 -Multi-modal input can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
10 +Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11 via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptStrictInputs`.
12
13 -.. note::
14 - ``multi_modal_data`` can accept keys and values beyond the builtin ones, as long as a customized plugin is registered through
15 - the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
13 +Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14 +by following :ref:`this guide <adding_multimodal_plugin>`.
15
17 -To implement a new multi-modal model in vLLM, please follow :ref:`this guide <enabling_multimodal_inputs>`.
16 +Looking to add your own multi-modal model? Please follow the instructions listed :ref:`here <enabling_multimodal_inputs>`.
17
19 -..
20 - TODO: Add more instructions on how to add new plugins once embeddings is in.
18 +Guides
19 +++++++
20 +
21 +.. toctree::
22 + :maxdepth: 1
23 +
24 + adding_multimodal_plugin
25
26 Module Contents
27 +++++++++++++++
@@ -36,10 +40,14 @@ Registry
40 Base Classes
41 ------------
42
39 -.. autoclass:: vllm.multimodal.MultiModalDataDict
43 +.. autodata:: vllm.multimodal.BatchedTensors
44 +
45 +.. autoclass:: vllm.multimodal.MultiModalDataBuiltins
46 :members:
47 :show-inheritance:
48
49 +.. autodata:: vllm.multimodal.MultiModalDataDict
50 +
51 .. autoclass:: vllm.multimodal.MultiModalInputs
52 :members:
53 :show-inheritance:
vllm/multimodal/__init__.py
+3 -2
@@ -1,5 +1,5 @@
1 -from .base import (BatchedTensors, MultiModalDataDict, MultiModalInputs,
2 - MultiModalPlugin)
1 +from .base import (BatchedTensors, MultiModalDataBuiltins, MultiModalDataDict,
2 + MultiModalInputs, MultiModalPlugin)
3 from .registry import MultiModalRegistry
4
5 MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -13,6 +13,7 @@ See also:
13
14 __all__ = [
15 "BatchedTensors",
16 + "MultiModalDataBuiltins",
17 "MultiModalDataDict",
18 "MultiModalInputs",
19 "MultiModalPlugin",
vllm/multimodal/base.py
+13 -8
@@ -43,9 +43,6 @@ class MultiModalInputs(_MultiModalInputsBase):
43 *,
44 device: torch.types.Device,
45 ) -> BatchedTensors:
46 - # Avoid initializing CUDA too early
47 - import torch
48 -
46 unbatched_shape = tensors[0].shape[1:]
47
48 for tensor in tensors:
@@ -84,16 +81,21 @@ class MultiModalInputs(_MultiModalInputsBase):
81
82
83 class MultiModalDataBuiltins(TypedDict, total=False):
84 + """Modality types that are predefined by vLLM."""
85 +
86 image: Image.Image
87 + """The input image."""
88
89
90 MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
91 """
92 A dictionary containing an item for each modality type to input.
93
94 -The data belonging to each modality is converted into keyword arguments
95 -to the model by the corresponding mapper. By default, the mapper of
96 -the corresponding plugin with the same modality key is applied.
94 +Note:
95 + This dictionary also accepts modality keys defined outside
96 + :class:`MultiModalDataBuiltins` as long as a customized plugin is registered
97 + through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
98 + Read more on that :ref:`here <adding_multimodal_plugin>`.
99 """
100
101 MultiModalInputMapper = Callable[[InputContext, object], MultiModalInputs]
@@ -123,6 +125,9 @@ class MultiModalPlugin(ABC):
125 process the same data differently). This registry is in turn used by
126 :class:`~MultiModalRegistry` which acts at a higher level
127 (i.e., the modality of the data).
128 +
129 + See also:
130 + :ref:`adding_multimodal_plugin`
131 """
132
133 def __init__(self) -> None:
@@ -183,8 +188,8 @@ class MultiModalPlugin(ABC):
188 def map_input(self, model_config: ModelConfig,
189 data: object) -> MultiModalInputs:
190 """
186 - Apply an input mapper to a data passed
187 - to the model, transforming the data into a dictionary of model inputs.
191 + Transform the data into a dictionary of model inputs using the
192 + input mapper registered for that model.
193
194 The model is identified by ``model_config``.
195
vllm/multimodal/image.py
+1
@@ -100,6 +100,7 @@ def repeat_and_pad_image_tokens(
100
101
102 class ImagePlugin(MultiModalPlugin):
103 + """Plugin for image data."""
104
105 def get_data_key(self) -> str:
106 return "image"
vllm/multimodal/registry.py
+13 -5
@@ -15,10 +15,8 @@ logger = init_logger(__name__)
15
16 class MultiModalRegistry:
17 """
18 - A registry to dispatch data processing
19 - according to its modality and the target model.
20 -
21 - The registry handles both external and internal data input.
18 + A registry that dispatches data processing to the
19 + :class:`~vllm.multimodal.MultiModalPlugin` for each modality.
20 """
21
22 DEFAULT_PLUGINS = (ImagePlugin(), )
@@ -30,6 +28,12 @@ class MultiModalRegistry:
28 self._plugins = {p.get_data_key(): p for p in plugins}
29
30 def register_plugin(self, plugin: MultiModalPlugin) -> None:
31 + """
32 + Register a multi-modal plugin so it can be recognized by vLLM.
33 +
34 + See also:
35 + :ref:`adding_multimodal_plugin`
36 + """
37 data_type_key = plugin.get_data_key()
38
39 if data_type_key in self._plugins:
@@ -75,7 +79,11 @@ class MultiModalRegistry:
79 data: MultiModalDataDict) -> MultiModalInputs:
80 """
81 Apply an input mapper to the data passed to the model.
78 -
82 +
83 + The data belonging to each modality is passed to the corresponding
84 + plugin which in turn converts the data into into keyword arguments
85 + via the input mapper registered for that model.
86 +
87 See :meth:`MultiModalPlugin.map_input` for more details.
88 """
89 merged_dict: Dict[str, torch.Tensor] = {}