A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Core] Registry for processing model inputs (#5214)
Co-authored-by: ywang96 <ywang@roblox.com>
Cyrus Leung committed
Jun 28, 2024 at 20:09 UTC
5cbe8d155c33a75934782019c06860467a3efa97
26 files changed
+778
-392
docs/source/dev/input_processing/input_processing_pipeline.rst
new
+20
@@ -0,0 +1,20 @@
1
+.. _input_processing_pipeline:
2
+
3
+Input Processing Pipeline
4
+=========================
5
+
6
+1. Input data is passed to :class:`~vllm.LLMEngine` (or :class:`~vllm.AsyncLLMEngine`).
7
+
8
+2. Tokenize the data if necessary.
9
+
10
+3. Process the inputs using :meth:`INPUT_REGISTRY.process_input <vllm.inputs.registry.InputRegistry.process_input>`.
11
+
12
+ - For example, add placeholder tokens to reserve KV cache for multi-modal embeddings.
13
+
14
+4. Send the processed inputs to :class:`~vllm.executor.executor_base.ExecutorBase`.
15
+
16
+5. Distribute the inputs via :class:`~vllm.worker.worker_base.WorkerBase` to :class:`~vllm.worker.model_runner_base.ModelRunnerBase`.
17
+
18
+6. If the data contains multi-modal data, convert it into keyword arguments using :meth:`MULTIMODAL_REGISTRY.map_input <vllm.multimodal.MultiModalRegistry.map_input>`.
19
+
20
+ - For example, convert a :class:`PIL.Image.Image` input to its pixel values for a vision language model.
docs/source/dev/input_processing/model_inputs_index.rst
new
+39
@@ -0,0 +1,39 @@
1
+.. _input_processing:
2
+
3
+Input Processing
4
+================
5
+
6
+.. currentmodule:: vllm.inputs
7
+
8
+vLLM provides a mechanism for defining input processors for each model so that the inputs are processed
9
+in :class:`~vllm.LLMEngine` before they are passed to model executors.
10
+
11
+Currently, this mechanism is only utilized in **multi-modal models** for preprocessing multi-modal input
12
+data in addition to input prompt, but it can be extended to text-only language models when needed.
13
+
14
+Guides
15
+++++++
16
+
17
+.. toctree::
18
+ :maxdepth: 1
19
+
20
+ input_processing_pipeline
21
+
22
+Module Contents
23
++++++++++++++++
24
+
25
+LLM Engine Inputs
26
+-----------------
27
+
28
+.. autoclass:: vllm.inputs.LLMInputs
29
+ :members:
30
+ :show-inheritance:
31
+
32
+Registry
33
+--------
34
+
35
+.. autodata:: vllm.inputs.INPUT_REGISTRY
36
+
37
+.. automodule:: vllm.inputs.registry
38
+ :members:
39
+ :show-inheritance:
docs/source/dev/multimodal/multimodal_index.rst
+1
-7
@@ -12,10 +12,6 @@ By default, vLLM models do not support multi-modal inputs. To enable multi-modal
12
you must decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_dummy_data <MultiModalRegistry.register_dummy_data>`,
13
as well as :meth:`MULTIMODAL_REGISTRY.register_input <MultiModalRegistry.register_input>` for each modality type to support.
14
15
-.. contents::
16
- :local:
17
- :backlinks: none
18
-
15
Module Contents
16
+++++++++++++++
17
@@ -24,9 +20,7 @@ Module Contents
20
Registry
21
--------
22
27
-.. data:: vllm.multimodal.MULTIMODAL_REGISTRY
28
-
29
- The global :class:`MultiModalRegistry` which is used by model runners.
23
+.. autodata:: vllm.multimodal.MULTIMODAL_REGISTRY
24
25
.. autoclass:: vllm.multimodal.MultiModalRegistry
26
:members:
docs/source/index.rst
+1
@@ -120,6 +120,7 @@ Documentation
120
dev/offline_inference/offline_index
121
dev/engine/engine_index
122
dev/kernel/paged_attention
123
+ dev/input_processing/model_inputs_index
124
dev/multimodal/multimodal_index
125
dev/dockerfile/dockerfile
126
docs/source/models/adding_model.rst
+2
-2
@@ -37,7 +37,7 @@ For instance, vLLM's `OPT model <https://github.com/vllm-project/vllm/blob/main/
37
2. Rewrite the :code:`forward` methods
38
--------------------------------------
39
40
-Next, you need to rewrite the :code:`forward` methods of your model by following these steps:
40
+Next, you need to rewrite the :meth:`~torch.nn.Module.forward` method of your model by following these steps:
41
42
1. Remove any unnecessary code, such as the code only used for training.
43
2. Change the input parameters:
@@ -75,7 +75,7 @@ Next, you need to rewrite the :code:`forward` methods of your model by following
75
76
If your model is too large to fit into a single GPU, you can use tensor parallelism to manage it.
77
To do this, substitute your model's linear and embedding layers with their tensor-parallel versions.
78
-For the embedding layer, you can simply replace :code:`nn.Embedding` with :code:`VocabParallelEmbedding`. For the output LM head, you can use :code:`ParallelLMHead`.
78
+For the embedding layer, you can simply replace :class:`torch.nn.Embedding` with :code:`VocabParallelEmbedding`. For the output LM head, you can use :code:`ParallelLMHead`.
79
When it comes to the linear layers, we provide the following options to parallelize them:
80
81
* :code:`ReplicatedLinear`: Replicates the inputs and weights across multiple GPUs. No memory saving.
examples/phi3v_example.py
+2
-1
@@ -11,7 +11,7 @@ def run_phi3v():
11
model_path = "microsoft/Phi-3-vision-128k-instruct"
12
13
# Note: The model has 128k context length by default which may cause OOM
14
- # If that's the case, override `max_model_len` with a smaller value via args
14
+ # In this example, we override max_model_len to 2048.
15
llm = LLM(
16
model=model_path,
17
trust_remote_code=True,
@@ -19,6 +19,7 @@ def run_phi3v():
19
image_token_id=32044,
20
image_input_shape="1,3,1008,1344",
21
image_feature_size=1921,
22
+ max_model_len=2048,
23
)
24
25
image = Image.open("images/cherry_blossom.jpg")
tests/multimodal/test_mapper.py
renamed
+32
-37
@@ -25,14 +25,14 @@ def test_clip_image_processor(image_assets, dtype):
25
seed=0,
26
dtype=dtype,
27
revision=None,
28
- )
29
- vlm_config = VisionLanguageConfig(
30
- image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
31
- image_token_id=32000,
32
- image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
33
- image_feature_size=576,
34
- image_processor=MODEL_NAME,
35
- image_processor_revision=None,
28
+ multimodal_config=VisionLanguageConfig(
29
+ image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
30
+ image_token_id=32000,
31
+ image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
32
+ image_feature_size=576,
33
+ image_processor=MODEL_NAME,
34
+ image_processor_revision=None,
35
+ ),
36
)
37
38
for asset in image_assets:
@@ -40,10 +40,9 @@ def test_clip_image_processor(image_assets, dtype):
40
asset.pil_image,
41
return_tensors="pt",
42
).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
43
- vllm_result = MULTIMODAL_REGISTRY.process_input(
43
+ vllm_result = MULTIMODAL_REGISTRY.map_input(
44
+ model_config,
45
ImagePixelData(asset.pil_image),
45
- model_config=model_config,
46
- vlm_config=vlm_config,
46
)
47
48
assert hf_result.keys() == vllm_result.keys()
@@ -74,14 +73,14 @@ def test_llava_next_image_processor(image_assets, dtype):
73
seed=0,
74
dtype=dtype,
75
revision=None,
77
- )
78
- vlm_config = VisionLanguageConfig(
79
- image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
80
- image_token_id=64000,
81
- image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
82
- image_feature_size=2928,
83
- image_processor=MODEL_NAME,
84
- image_processor_revision=None,
76
+ multimodal_config=VisionLanguageConfig(
77
+ image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
78
+ image_token_id=64000,
79
+ image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
80
+ image_feature_size=2928,
81
+ image_processor=MODEL_NAME,
82
+ image_processor_revision=None,
83
+ ),
84
)
85
86
for asset in image_assets:
@@ -89,10 +88,9 @@ def test_llava_next_image_processor(image_assets, dtype):
88
asset.pil_image,
89
return_tensors="pt",
90
).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
92
- vllm_result = MULTIMODAL_REGISTRY.process_input(
91
+ vllm_result = MULTIMODAL_REGISTRY.map_input(
92
+ model_config,
93
ImagePixelData(asset.pil_image),
94
- model_config=model_config,
95
- vlm_config=vlm_config,
94
)
95
96
assert hf_result.keys() == vllm_result.keys()
@@ -119,26 +117,23 @@ def test_image_pixel_types(image_assets, dtype):
117
seed=0,
118
dtype=dtype,
119
revision=None,
122
- )
123
- vlm_config = VisionLanguageConfig(
124
- image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
125
- image_token_id=32000,
126
- image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
127
- image_feature_size=576,
128
- image_processor=MODEL_NAME,
129
- image_processor_revision=None,
130
- )
120
+ multimodal_config=VisionLanguageConfig(
121
+ image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
122
+ image_token_id=32000,
123
+ image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
124
+ image_feature_size=576,
125
+ image_processor=MODEL_NAME,
126
+ image_processor_revision=None,
127
+ ))
128
129
for asset in image_assets:
133
- image_result = MULTIMODAL_REGISTRY.process_input(
130
+ image_result = MULTIMODAL_REGISTRY.map_input(
131
+ model_config,
132
ImagePixelData(asset.pil_image),
135
- model_config=model_config,
136
- vlm_config=vlm_config,
133
)
138
- tensor_result = MULTIMODAL_REGISTRY.process_input(
134
+ tensor_result = MULTIMODAL_REGISTRY.map_input(
135
+ model_config,
136
ImagePixelData(asset.pixel_values),
140
- model_config=model_config,
141
- vlm_config=vlm_config,
137
)
138
139
assert image_result.keys() == tensor_result.keys()
vllm/config.py
+3
@@ -109,6 +109,7 @@ class ModelConfig:
109
disable_sliding_window: bool = False,
110
skip_tokenizer_init: bool = False,
111
served_model_name: Optional[Union[str, List[str]]] = None,
112
+ multimodal_config: Optional["VisionLanguageConfig"] = None,
113
) -> None:
114
self.model = model
115
self.tokenizer = tokenizer
@@ -159,6 +160,8 @@ class ModelConfig:
160
sliding_window_len=self.get_hf_config_sliding_window())
161
self.served_model_name = get_served_model_name(model,
162
served_model_name)
163
+ self.multimodal_config = multimodal_config
164
+
165
if not self.skip_tokenizer_init:
166
self._verify_tokenizer_mode()
167
self._verify_embedding_mode()
vllm/engine/arg_utils.py
+32
-32
@@ -643,6 +643,36 @@ class EngineArgs:
643
raise ValueError(
644
"BitsAndBytes load format and QLoRA adapter only support "
645
f"'bitsandbytes' quantization, but got {self.quantization}")
646
+ if self.image_input_type:
647
+ if (not self.image_token_id or not self.image_input_shape
648
+ or not self.image_feature_size):
649
+ raise ValueError(
650
+ 'Specify `image_token_id`, `image_input_shape` and '
651
+ '`image_feature_size` together with `image_input_type`.')
652
+
653
+ if self.image_processor is None:
654
+ self.image_processor = self.model
655
+ if self.disable_image_processor:
656
+ if self.image_processor != self.model:
657
+ warnings.warn(
658
+ "You've specified an image processor "
659
+ f"({self.image_processor}) but also disabled "
660
+ "it via `--disable-image-processor`.",
661
+ stacklevel=2)
662
+
663
+ self.image_processor = None
664
+
665
+ vision_language_config = VisionLanguageConfig(
666
+ image_input_type=VisionLanguageConfig.
667
+ get_image_input_enum_type(self.image_input_type),
668
+ image_token_id=self.image_token_id,
669
+ image_input_shape=str_to_int_tuple(self.image_input_shape),
670
+ image_feature_size=self.image_feature_size,
671
+ image_processor=self.image_processor,
672
+ image_processor_revision=self.image_processor_revision,
673
+ )
674
+ else:
675
+ vision_language_config = None
676
677
device_config = DeviceConfig(device=self.device)
678
model_config = ModelConfig(
@@ -666,7 +696,8 @@ class EngineArgs:
696
max_logprobs=self.max_logprobs,
697
disable_sliding_window=self.disable_sliding_window,
698
skip_tokenizer_init=self.skip_tokenizer_init,
669
- served_model_name=self.served_model_name)
699
+ served_model_name=self.served_model_name,
700
+ multimodal_config=vision_language_config)
701
cache_config = CacheConfig(
702
block_size=self.block_size,
703
gpu_memory_utilization=self.gpu_memory_utilization,
@@ -742,37 +773,6 @@ class EngineArgs:
773
model_loader_extra_config=self.model_loader_extra_config,
774
)
775
745
- if self.image_input_type:
746
- if (not self.image_token_id or not self.image_input_shape
747
- or not self.image_feature_size):
748
- raise ValueError(
749
- 'Specify `image_token_id`, `image_input_shape` and '
750
- '`image_feature_size` together with `image_input_type`.')
751
-
752
- if self.image_processor is None:
753
- self.image_processor = self.model
754
- if self.disable_image_processor:
755
- if self.image_processor != self.model:
756
- warnings.warn(
757
- "You've specified an image processor "
758
- f"({self.image_processor}) but also disabled "
759
- "it via `--disable-image-processor`.",
760
- stacklevel=2)
761
-
762
- self.image_processor = None
763
-
764
- vision_language_config = VisionLanguageConfig(
765
- image_input_type=VisionLanguageConfig.
766
- get_image_input_enum_type(self.image_input_type),
767
- image_token_id=self.image_token_id,
768
- image_input_shape=str_to_int_tuple(self.image_input_shape),
769
- image_feature_size=self.image_feature_size,
770
- image_processor=self.image_processor,
771
- image_processor_revision=self.image_processor_revision,
772
- )
773
- else:
774
- vision_language_config = None
775
-
776
decoding_config = DecodingConfig(
777
guided_decoding_backend=self.guided_decoding_backend)
778
vllm/engine/async_llm_engine.py
+5
-3
@@ -278,9 +278,11 @@ class _AsyncLLMEngine(LLMEngine):
278
else:
279
prompt_token_ids = inputs["prompt_token_ids"]
280
281
- return LLMInputs(prompt_token_ids=prompt_token_ids,
282
- prompt=inputs.get("prompt"),
283
- multi_modal_data=inputs.get("multi_modal_data"))
281
+ llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
282
+ prompt=inputs.get("prompt"),
283
+ multi_modal_data=inputs.get("multi_modal_data"))
284
+
285
+ return self.input_processor(llm_inputs)
286
287
async def add_request_async(
288
self,
vllm/engine/llm_engine.py
+9
-4
@@ -20,7 +20,7 @@ from vllm.engine.output_processor.stop_checker import StopChecker
20
from vllm.engine.output_processor.util import create_output_by_sequence_group
21
from vllm.executor.executor_base import ExecutorBase
22
from vllm.executor.ray_utils import initialize_ray_cluster
23
-from vllm.inputs import LLMInputs, PromptInputs
23
+from vllm.inputs import INPUT_REGISTRY, LLMInputs, PromptInputs
24
from vllm.logger import init_logger
25
from vllm.lora.request import LoRARequest
26
from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
@@ -227,6 +227,9 @@ class LLMEngine:
227
self.generation_config_fields = _load_generation_config_dict(
228
model_config)
229
230
+ self.input_processor = INPUT_REGISTRY.create_input_processor(
231
+ self.model_config)
232
+
233
self.model_executor = executor_class(
234
model_config=model_config,
235
cache_config=cache_config,
@@ -511,9 +514,11 @@ class LLMEngine:
514
else:
515
prompt_token_ids = inputs["prompt_token_ids"]
516
514
- return LLMInputs(prompt_token_ids=prompt_token_ids,
515
- prompt=inputs.get("prompt"),
516
- multi_modal_data=inputs.get("multi_modal_data"))
517
+ llm_inputs = LLMInputs(prompt_token_ids=prompt_token_ids,
518
+ prompt=inputs.get("prompt"),
519
+ multi_modal_data=inputs.get("multi_modal_data"))
520
+
521
+ return self.input_processor(llm_inputs)
522
523
def add_request(
524
self,
vllm/inputs/__init__.py
new
+19
@@ -0,0 +1,19 @@
1
+from .data import (LLMInputs, ParsedText, ParsedTokens, PromptInputs,
2
+ PromptStrictInputs, TextPrompt, TextTokensPrompt,
3
+ TokensPrompt, parse_and_batch_prompt)
4
+from .registry import InputContext, InputRegistry
5
+
6
+INPUT_REGISTRY = InputRegistry()
7
+"""
8
+The global :class:`~InputRegistry` which is used by :class:`~vllm.LLMEngine`
9
+to dispatch data processing according to the target model.
10
+
11
+See also:
12
+ :ref:`input_processing_pipeline`
13
+"""
14
+
15
+__all__ = [
16
+ "ParsedText", "ParsedTokens", "parse_and_batch_prompt", "TextPrompt",
17
+ "TokensPrompt", "TextTokensPrompt", "PromptStrictInputs", "PromptInputs",
18
+ "LLMInputs", "INPUT_REGISTRY", "InputContext", "InputRegistry"
19
+]
vllm/inputs/data.py
renamed
+16
-2
@@ -101,8 +101,7 @@ class TextTokensPrompt(TypedDict):
101
"""The prompt text."""
102
103
prompt_token_ids: List[int]
104
- """The token IDs of the prompt. If None, we use the
105
- tokenizer to convert the prompts to token IDs."""
104
+ """The token IDs of the prompt."""
105
106
multi_modal_data: NotRequired["MultiModalData"]
107
"""
@@ -125,6 +124,21 @@ PromptInputs = Union[str, TextPrompt, TokensPrompt, TextTokensPrompt]
124
125
126
class LLMInputs(TypedDict):
127
+ """
128
+ The inputs in :class:`~vllm.LLMEngine` before they are
129
+ passed to the model executor.
130
+ """
131
+
132
prompt_token_ids: List[int]
133
+ """The token IDs of the prompt."""
134
+
135
prompt: NotRequired[Optional[str]]
136
+ """
137
+ The original prompt text corresponding to the token IDs, if available.
138
+ """
139
+
140
multi_modal_data: NotRequired[Optional["MultiModalData"]]
141
+ """
142
+ Optional multi-modal data to pass to the model,
143
+ if the model supports it.
144
+ """
vllm/inputs/registry.py
new
+207
@@ -0,0 +1,207 @@
1
+import functools
2
+from dataclasses import dataclass
3
+from typing import (TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type,
4
+ TypeVar)
5
+
6
+from torch import nn
7
+from transformers import PretrainedConfig
8
+
9
+from vllm.logger import init_logger
10
+
11
+from .data import LLMInputs
12
+
13
+if TYPE_CHECKING:
14
+ from vllm.config import ModelConfig, VisionLanguageConfig
15
+ from vllm.multimodal import MultiModalData
16
+ from vllm.sequence import SequenceData
17
+
18
+logger = init_logger(__name__)
19
+
20
+C = TypeVar("C", bound=PretrainedConfig)
21
+
22
+
23
+@dataclass(frozen=True)
24
+class InputContext:
25
+ """
26
+ Contains information about the model which may be used to
27
+ modify the inputs.
28
+ """
29
+
30
+ model_config: "ModelConfig"
31
+ """The configuration of the model."""
32
+
33
+ def get_multimodal_config(self) -> "VisionLanguageConfig":
34
+ """
35
+ Get the multimodal configuration of the model.
36
+
37
+ Raises:
38
+ ValueError: If the model is not multimodal.
39
+ """
40
+
41
+ multimodal_config = self.model_config.multimodal_config
42
+ if multimodal_config is None:
43
+ raise ValueError("No multimodal config found")
44
+
45
+ return multimodal_config
46
+
47
+ def get_hf_config(self, hf_config_type: Type[C]) -> C:
48
+ """
49
+ Get the HuggingFace configuration
50
+ (:class:`transformers.PretrainedConfig`) of the model,
51
+ additionally checking its type.
52
+
53
+ Raises:
54
+ ValueError: If the model is not of the specified type.
55
+ """
56
+
57
+ hf_config = self.model_config.hf_config
58
+ if not isinstance(hf_config, hf_config_type):
59
+ raise TypeError("Invalid type of HuggingFace config. "
60
+ f"Expected type: {hf_config_type}, but "
61
+ f"found type: {type(hf_config)}")
62
+
63
+ return hf_config
64
+
65
+
66
+N = TypeVar("N", bound=Type[nn.Module])
67
+
68
+DummyDataFactory = Callable[[InputContext, int],
69
+ Tuple["SequenceData", Optional["MultiModalData"]]]
70
+"""
71
+Create dummy data to be inputted into the model.
72
+
73
+Note:
74
+ :data:`InputProcessor` is not applied to the dummy data.
75
+"""
76
+
77
+InputProcessor = Callable[[InputContext, LLMInputs], LLMInputs]
78
+"""Preprocess the inputs to the model."""
79
+
80
+
81
+class InputRegistry:
82
+ """
83
+ A registry to dispatch data processing
84
+ according to the target model.
85
+ """
86
+
87
+ def __init__(self) -> None:
88
+ self._dummy_factories_by_model_type: Dict[Type[nn.Module],
89
+ DummyDataFactory] = {}
90
+ self._input_processors_by_model_type: Dict[Type[nn.Module],
91
+ InputProcessor] = {}
92
+
93
+ def _default_dummy_data_factory(
94
+ self,
95
+ ctx: InputContext,
96
+ seq_len: int,
97
+ ) -> Tuple["SequenceData", Optional["MultiModalData"]]:
98
+ """
99
+ The default dummy data factory represents the longest possible text
100
+ that can be inputted to the model.
101
+
102
+ Note:
103
+ :data:`InputProcessor` is not applied to the dummy data.
104
+ """
105
+ # Avoid circular import
106
+ from vllm.sequence import SequenceData
107
+
108
+ dummy_seq_data = SequenceData([0] * seq_len)
109
+ dummy_multi_modal_data = None
110
+
111
+ return dummy_seq_data, dummy_multi_modal_data
112
+
113
+ def register_dummy_data(self, factory: DummyDataFactory):
114
+ """
115
+ Register a dummy data factory to a model class.
116
+
117
+ During memory profiling, the provided function is invoked to create
118
+ dummy data to be inputted into the model. The resulting memory usage
119
+ should be an upper bound of what the model would use at inference time.
120
+ """
121
+
122
+ def wrapper(model_cls: N) -> N:
123
+ if model_cls in self._dummy_factories_by_model_type:
124
+ logger.warning(
125
+ "Model class %s already has dummy data "
126
+ "registered to %s. It is overwritten by the new one.",
127
+ model_cls, self)
128
+
129
+ self._dummy_factories_by_model_type[model_cls] = factory
130
+
131
+ return model_cls
132
+
133
+ return wrapper
134
+
135
+ def dummy_data_for_profiling(self, model_config: "ModelConfig",
136
+ seq_len: int):
137
+ """
138
+ Create dummy data for profiling the memory usage of a model.
139
+
140
+ The model is identified by ``model_config``.
141
+
142
+ TODO: Add guide [ref: PR #5276]
143
+ """
144
+ # Avoid circular import
145
+ from vllm.model_executor.model_loader import get_model_architecture
146
+
147
+ model_cls, _ = get_model_architecture(model_config)
148
+ dummy_factory = self._dummy_factories_by_model_type \
149
+ .get(model_cls, self._default_dummy_data_factory)
150
+
151
+ return dummy_factory(InputContext(model_config), seq_len)
152
+
153
+ def _default_input_processor(self, ctx: InputContext,
154
+ inputs: LLMInputs) -> LLMInputs:
155
+ """The default input processor is a no-op."""
156
+ return inputs
157
+
158
+ def register_input_processor(self, processor: InputProcessor):
159
+ """
160
+ Register an input processor to a model class.
161
+
162
+ The provided function is invoked on each input to the model. This
163
+ happens before :meth:`~vllm.multimodal.MultiModalRegistry.map_input`.
164
+
165
+ See also:
166
+ :ref:`input_processing_pipeline`
167
+ """
168
+
169
+ def wrapper(model_cls: N) -> N:
170
+ if model_cls in self._input_processors_by_model_type:
171
+ logger.warning(
172
+ "Model class %s already has input processor "
173
+ "registered to %s. It is overwritten by the new one.",
174
+ model_cls, self)
175
+
176
+ self._input_processors_by_model_type[model_cls] = processor
177
+
178
+ return model_cls
179
+
180
+ return wrapper
181
+
182
+ def process_input(self, model_config: "ModelConfig",
183
+ inputs: LLMInputs) -> LLMInputs:
184
+ """
185
+ Apply an input processor to an instance of model inputs.
186
+
187
+ The model is identified by ``model_config``.
188
+
189
+ See also:
190
+ :ref:`input_processing_pipeline`
191
+ """
192
+ # Avoid circular import
193
+ from vllm.model_executor.model_loader import get_model_architecture
194
+
195
+ model_cls, _ = get_model_architecture(model_config)
196
+
197
+ processor = self._input_processors_by_model_type \
198
+ .get(model_cls, self._default_input_processor)
199
+
200
+ return processor(InputContext(model_config), inputs)
201
+
202
+ def create_input_processor(self, model_config: "ModelConfig"):
203
+ """
204
+ Create an input processor (see :meth:`process_input`) for a
205
+ specific model.
206
+ """
207
+ return functools.partial(self.process_input, model_config)
vllm/model_executor/models/clip.py
+69
-8
@@ -1,22 +1,83 @@
1
"""Minimal implementation of CLIPVisionModel intended to be only used
2
within a vision language model."""
3
-from typing import Optional, Tuple
3
+from typing import Optional
4
5
import torch
6
import torch.nn as nn
7
+from PIL import Image
8
from transformers import CLIPVisionConfig
9
from transformers.models.clip.modeling_clip import CLIPAttention
10
11
from vllm.model_executor.layers.activation import get_act_fn
12
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13
RowParallelLinear)
13
-from vllm.model_executor.layers.quantization.base_config import (
14
- QuantizationConfig)
14
+from vllm.model_executor.layers.quantization import QuantizationConfig
15
+from vllm.multimodal.image import ImageFeatureData, ImagePixelData
16
+from vllm.sequence import SequenceData
17
18
17
-def get_clip_num_patches(image_size: int, patch_size: int) -> int:
19
+def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
20
assert image_size % patch_size == 0
19
- return (image_size // patch_size)**2
21
+ return image_size // patch_size
22
+
23
+
24
+def get_clip_num_patches(*, image_size: int, patch_size: int) -> int:
25
+ grid_length = get_clip_patch_grid_length(image_size=image_size,
26
+ patch_size=patch_size)
27
+ return grid_length * grid_length
28
+
29
+
30
+def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int:
31
+ return get_clip_num_patches(image_size=hf_config.image_size,
32
+ patch_size=hf_config.patch_size)
33
+
34
+
35
+def dummy_seq_data_for_clip(
36
+ hf_config: CLIPVisionConfig,
37
+ seq_len: int,
38
+ *,
39
+ image_token_id: int,
40
+ image_feature_size_override: Optional[int] = None,
41
+):
42
+ if image_feature_size_override is None:
43
+ image_feature_size = get_clip_image_feature_size(hf_config)
44
+ else:
45
+ image_feature_size = image_feature_size_override
46
+
47
+ token_ids = [image_token_id] * image_feature_size
48
+ token_ids += [0] * (seq_len - image_feature_size)
49
+ return SequenceData(token_ids)
50
+
51
+
52
+def dummy_pixel_data_for_clip(
53
+ hf_config: CLIPVisionConfig,
54
+ *,
55
+ image_width_override: Optional[int] = None,
56
+ image_height_override: Optional[int] = None,
57
+):
58
+ width = height = hf_config.image_size
59
+ if image_width_override is not None:
60
+ width = image_width_override
61
+ if image_height_override is not None:
62
+ height = image_height_override
63
+
64
+ image = Image.new("RGB", (width, height), color=0)
65
+ return ImagePixelData(image)
66
+
67
+
68
+def dummy_feature_data_for_clip(
69
+ hf_config: CLIPVisionConfig,
70
+ *,
71
+ image_feature_size_override: Optional[int] = None,
72
+):
73
+ if image_feature_size_override is None:
74
+ image_feature_size = get_clip_image_feature_size(hf_config)
75
+ else:
76
+ image_feature_size = image_feature_size_override
77
+
78
+ values = torch.zeros((1, image_feature_size, hf_config.hidden_size),
79
+ dtype=torch.float16)
80
+ return ImageFeatureData(values)
81
82
83
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
@@ -39,8 +100,8 @@ class CLIPVisionEmbeddings(nn.Module):
100
bias=False,
101
)
102
42
- self.num_patches = get_clip_num_patches(self.image_size,
43
- self.patch_size)
103
+ self.num_patches = get_clip_num_patches(image_size=self.image_size,
104
+ patch_size=self.patch_size)
105
self.num_positions = self.num_patches + 1
106
self.position_embedding = nn.Embedding(self.num_positions,
107
self.embed_dim)
@@ -101,7 +162,7 @@ class CLIPEncoderLayer(nn.Module):
162
self.layer_norm2 = nn.LayerNorm(config.hidden_size,
163
eps=config.layer_norm_eps)
164
104
- def forward(self, hidden_states: torch.Tensor) -> Tuple[torch.Tensor]:
165
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
166
167
residual = hidden_states
168
vllm/model_executor/models/llava.py
+34
-6
@@ -2,10 +2,11 @@ from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3
import torch
4
import torch.nn as nn
5
-from transformers import LlavaConfig
5
+from transformers import CLIPVisionConfig, LlavaConfig
6
7
from vllm.attention import AttentionMetadata
8
from vllm.config import CacheConfig, VisionLanguageConfig
9
+from vllm.inputs import INPUT_REGISTRY, InputContext
10
from vllm.model_executor.layers.activation import get_act_fn
11
from vllm.model_executor.layers.logits_processor import LogitsProcessor
12
from vllm.model_executor.layers.quantization.base_config import (
@@ -16,10 +17,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.model_executor.models.clip import CLIPVisionModel
18
from vllm.model_executor.models.llama import LlamaModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
-from vllm.multimodal import MULTIMODAL_REGISTRY
20
-from vllm.multimodal.image import get_dummy_image_data
20
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
21
from vllm.sequence import SamplerOutput
22
23
+from .clip import (dummy_feature_data_for_clip, dummy_pixel_data_for_clip,
24
+ dummy_seq_data_for_clip)
25
from .interfaces import SupportsVision
26
27
_KEYS_TO_MODIFY_MAPPING = {
@@ -83,9 +85,35 @@ class LlavaImageFeatureInputs(TypedDict):
85
LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
86
87
86
-@MULTIMODAL_REGISTRY.register_image_feature_input()
87
-@MULTIMODAL_REGISTRY.register_image_pixel_input()
88
-@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
88
+def dummy_data_for_llava(ctx: InputContext, seq_len: int):
89
+ multimodal_config = ctx.get_multimodal_config()
90
+ hf_config = ctx.get_hf_config(LlavaConfig)
91
+ vision_config = hf_config.vision_config
92
+
93
+ if isinstance(vision_config, CLIPVisionConfig):
94
+ seq_data = dummy_seq_data_for_clip(
95
+ vision_config,
96
+ seq_len,
97
+ image_token_id=hf_config.image_token_index,
98
+ )
99
+
100
+ image_input_type = multimodal_config.image_input_type
101
+ ImageInputType = VisionLanguageConfig.ImageInputType
102
+ mm_data: MultiModalData
103
+ if image_input_type == ImageInputType.PIXEL_VALUES:
104
+ mm_data = dummy_pixel_data_for_clip(vision_config)
105
+ elif image_input_type == ImageInputType.IMAGE_FEATURES:
106
+ mm_data = dummy_feature_data_for_clip(vision_config)
107
+
108
+ return seq_data, mm_data
109
+
110
+ msg = f"Unsupported vision config: {type(vision_config)}"
111
+ raise NotImplementedError(msg)
112
+
113
+
114
+@MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
115
+@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
116
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
117
class LlavaForConditionalGeneration(nn.Module, SupportsVision):
118
119
def __init__(self,
vllm/model_executor/models/llava_next.py
+109
-29
@@ -3,14 +3,14 @@ from typing import (Dict, Iterable, List, Literal, Optional, Tuple, TypedDict,
3
4
import torch
5
import torch.nn as nn
6
-from PIL import Image
7
-from transformers import LlavaNextConfig
6
+from transformers import CLIPVisionConfig, LlavaNextConfig
7
from transformers.models.llava_next.modeling_llava_next import (
8
get_anyres_image_grid_shape, unpad_image)
9
from typing_extensions import NotRequired
10
11
from vllm.attention import AttentionMetadata
13
-from vllm.config import CacheConfig, ModelConfig, VisionLanguageConfig
12
+from vllm.config import CacheConfig, VisionLanguageConfig
13
+from vllm.inputs import INPUT_REGISTRY, InputContext
14
from vllm.logger import init_logger
15
from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
from vllm.model_executor.layers.quantization.base_config import (
@@ -22,9 +22,11 @@ from vllm.model_executor.models.clip import CLIPVisionModel
22
from vllm.model_executor.models.llama import LlamaModel
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
25
-from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
26
-from vllm.sequence import SamplerOutput, SequenceData
25
+from vllm.multimodal.image import ImagePixelData
26
+from vllm.sequence import SamplerOutput
27
28
+from .clip import (dummy_feature_data_for_clip, dummy_pixel_data_for_clip,
29
+ dummy_seq_data_for_clip, get_clip_patch_grid_length)
30
from .interfaces import SupportsVision
31
from .llava import LlavaMultiModalProjector, merge_vision_embeddings
32
@@ -58,41 +60,118 @@ LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
60
LlavaNextImageFeatureInputs]
61
62
61
-def _get_dummy_image_data(
62
- seq_len: int,
63
- model_config: ModelConfig,
64
- vlm_config: VisionLanguageConfig,
65
-) -> Tuple[SequenceData, MultiModalData]:
66
- seq_data, fake_mm_data = get_dummy_image_data(seq_len, model_config,
67
- vlm_config)
63
+def _get_llava_next_num_unpadded_features(
64
+ height: int,
65
+ width: int,
66
+ npatches: int,
67
+ num_patch_height: int,
68
+ num_patch_width: int,
69
+) -> Tuple[int, int]:
70
+ # Taken from: https://github.com/huggingface/text-generation-inference/blob/799a193b109662743bed1b18a09af1fdcd508c8b/server/text_generation_server/models/vlm_causal_lm.py#L111
71
+ current_height = npatches * num_patch_height
72
+ current_width = npatches * num_patch_width
73
+
74
+ aspect_ratio: float = width / height
75
+ current_aspect_ratio: float = current_width / current_height
76
+ if aspect_ratio > current_aspect_ratio:
77
+ new_height = (height * current_width) // width
78
+ current_height = new_height
79
+ else:
80
+ new_width = (width * current_height) // height
81
+ current_width = new_width
82
+
83
+ unpadded_features = current_height * current_width
84
+ newline_features = current_height
85
+ return (unpadded_features, newline_features)
86
+
87
+
88
+def _get_llava_next_image_feature_size(
89
+ hf_config: LlavaNextConfig,
90
+ *,
91
+ input_height: int,
92
+ input_width: int,
93
+) -> int:
94
+ vision_config = hf_config.vision_config
95
+
96
+ if isinstance(vision_config, CLIPVisionConfig):
97
+ num_patches = get_clip_patch_grid_length(
98
+ image_size=vision_config.image_size,
99
+ patch_size=vision_config.patch_size,
100
+ )
101
+ base_feature_size = num_patches * num_patches
102
+
103
+ num_patch_height, num_patch_width = get_anyres_image_grid_shape(
104
+ image_size=(input_height, input_width),
105
+ grid_pinpoints=hf_config.image_grid_pinpoints,
106
+ patch_size=vision_config.image_size,
107
+ )
108
+
109
+ (
110
+ unpadded_feature_size,
111
+ newline_feature_size,
112
+ ) = _get_llava_next_num_unpadded_features(input_height, input_width,
113
+ num_patches,
114
+ num_patch_height,
115
+ num_patch_width)
116
+
117
+ return unpadded_feature_size + newline_feature_size + base_feature_size
118
+
119
+ msg = f"Unsupported vision config: {type(vision_config)}"
120
+ raise NotImplementedError(msg)
121
+
122
69
- config_input_type = vlm_config.image_input_type
70
- ImageInputType = VisionLanguageConfig.ImageInputType
123
+def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
124
+ multimodal_config = ctx.get_multimodal_config()
125
+ hf_config = ctx.get_hf_config(LlavaNextConfig)
126
+ vision_config = hf_config.vision_config
127
+
128
+ #TODO: change the logic for dummy data to support dynamic shape
129
+ _, _, dummy_height, dummy_width = multimodal_config.image_input_shape
130
+ image_feature_size = _get_llava_next_image_feature_size(
131
+ hf_config, input_height=dummy_height, input_width=dummy_width)
132
+
133
+ if isinstance(vision_config, CLIPVisionConfig):
134
+ seq_data = dummy_seq_data_for_clip(
135
+ vision_config,
136
+ seq_len,
137
+ image_token_id=hf_config.image_token_index,
138
+ image_feature_size_override=image_feature_size,
139
+ )
140
+
141
+ image_input_type = multimodal_config.image_input_type
142
+ ImageInputType = VisionLanguageConfig.ImageInputType
143
+ mm_data: MultiModalData
144
+ if image_input_type == ImageInputType.PIXEL_VALUES:
145
+ mm_data = dummy_pixel_data_for_clip(
146
+ vision_config,
147
+ image_width_override=dummy_width,
148
+ image_height_override=dummy_height,
149
+ )
150
+ elif image_input_type == ImageInputType.IMAGE_FEATURES:
151
+ mm_data = dummy_feature_data_for_clip(
152
+ vision_config,
153
+ image_feature_size_override=image_feature_size,
154
+ )
155
72
- if config_input_type == ImageInputType.PIXEL_VALUES:
73
- _, c, h, w = vlm_config.image_input_shape
74
- mode = {1: "L", 3: "RGB"}[c]
75
- fake_mm_data = ImagePixelData(Image.new(mode, (w, h), color=0))
156
+ return seq_data, mm_data
157
77
- return seq_data, fake_mm_data
158
+ msg = f"Unsupported vision config: {type(vision_config)}"
159
+ raise NotImplementedError(msg)
160
161
80
-def _image_pixel_processor(
81
- data: ImagePixelData,
82
- model_config: ModelConfig,
83
- vlm_config: VisionLanguageConfig,
84
-) -> Dict[str, torch.Tensor]:
162
+def _pixel_mapper(ctx: InputContext,
163
+ data: ImagePixelData) -> Dict[str, torch.Tensor]:
164
image = data.image
165
166
if isinstance(image, torch.Tensor):
88
- pixel_values = image.to(model_config.dtype)
167
+ pixel_values = image.to(ctx.model_config.dtype)
168
batch_size, _, _, h, w = pixel_values.shape
169
image_sizes = torch.tensor([(w, h) for _ in range(batch_size)])
170
171
return {"pixel_values": pixel_values, "image_sizes": image_sizes}
172
173
# Temporary patch before dynamic number of image tokens is supported
95
- _, _, h, w = vlm_config.image_input_shape
174
+ _, _, h, w = ctx.get_multimodal_config().image_input_shape
175
if (w, h) != (image.width, image.height):
176
logger.warning(
177
"Dynamic image shape is currently not supported. "
@@ -101,11 +180,12 @@ def _image_pixel_processor(
180
data.image = image.resize((w, h))
181
182
return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
104
- ._default_input_processor(data, model_config, vlm_config)
183
+ ._default_input_mapper(ctx, data)
184
185
107
-@MULTIMODAL_REGISTRY.register_image_pixel_input(_image_pixel_processor)
108
-@MULTIMODAL_REGISTRY.register_dummy_data(_get_dummy_image_data)
186
+@MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
187
+@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper(_pixel_mapper)
188
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
189
class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
190
191
def __init__(self,
vllm/model_executor/models/phi3v.py
+54
-20
@@ -22,7 +22,8 @@ from PIL import Image
22
from transformers import CLIPVisionConfig, PretrainedConfig
23
24
from vllm.attention import AttentionMetadata
25
-from vllm.config import CacheConfig, ModelConfig, VisionLanguageConfig
25
+from vllm.config import CacheConfig, VisionLanguageConfig
26
+from vllm.inputs import INPUT_REGISTRY, InputContext
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.logits_processor import LogitsProcessor
29
from vllm.model_executor.layers.quantization.base_config import (
@@ -34,9 +35,10 @@ from vllm.model_executor.models.clip import CLIPVisionModel
35
from vllm.model_executor.models.llama import LlamaModel
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
37
from vllm.multimodal import MULTIMODAL_REGISTRY
37
-from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
38
+from vllm.multimodal.image import ImagePixelData
39
from vllm.sequence import SamplerOutput
40
41
+from .clip import dummy_pixel_data_for_clip, dummy_seq_data_for_clip
42
from .interfaces import SupportsVision
43
44
logger = init_logger(__name__)
@@ -107,7 +109,6 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
109
self.num_img_tokens = config.img_processor['num_img_tokens']
110
111
self.image_dim_out = image_dim_out
110
- self.img_sizes = None
112
113
# global_gn and sub_gn for hd transform, serves as line separator
114
self.use_hd_transform = config.embd_layer.get('use_hd_transform',
@@ -134,7 +135,6 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
135
self.img_projection = nn.Sequential(*layers)
136
137
self.vocab_size = config.vocab_size
137
- self.img_features = None
138
139
self.layer_idx = config.img_processor.get('layer_idx', -2)
140
self.type_feature = config.img_processor.get('type_feature', 'patch')
@@ -260,9 +260,44 @@ class Phi3VImagePixelInputs(TypedDict):
260
"""Shape: (batch_size, 2)"""
261
262
263
-# FIXME(Isotr0py): Remove these after dynamic num_img_tokens is supported
264
-# copied from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py
265
-def calc_padded_size(width, height, padding_unit=336):
263
+def _get_phi3v_image_feature_size(
264
+ *,
265
+ input_height: int,
266
+ input_width: int,
267
+) -> int:
268
+ h, w = input_height, input_width
269
+
270
+ # https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L178
271
+ return (h // 336 * w // 336 + 1) * 144 + 1 + (h // 336 + 1) * 12
272
+
273
+
274
+def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
275
+ multimodal_config = ctx.get_multimodal_config()
276
+
277
+ #TODO: change the logic for dummy data to support dynamic shape
278
+ _, _, dummy_height, dummy_width = multimodal_config.image_input_shape
279
+ image_feature_size = _get_phi3v_image_feature_size(
280
+ input_height=dummy_height,
281
+ input_width=dummy_width,
282
+ )
283
+
284
+ seq_data = dummy_seq_data_for_clip(
285
+ CLIP_VIT_LARGE_PATCH14_336_CONFIG,
286
+ seq_len,
287
+ image_token_id=32044,
288
+ image_feature_size_override=image_feature_size,
289
+ )
290
+ mm_data = dummy_pixel_data_for_clip(
291
+ CLIP_VIT_LARGE_PATCH14_336_CONFIG,
292
+ image_width_override=dummy_width,
293
+ image_height_override=dummy_height,
294
+ )
295
+
296
+ return seq_data, mm_data
297
+
298
+
299
+# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py
300
+def _calc_padded_size(*, width: int, height: int, padding_unit: int = 336):
301
target_height = int(np.ceil(height / padding_unit) * padding_unit)
302
top_padding = int((target_height - height) / 2)
303
bottom_padding = target_height - height - top_padding
@@ -271,8 +306,8 @@ def calc_padded_size(width, height, padding_unit=336):
306
return padded_width, padded_height
307
308
274
-# copied from https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py
275
-def calc_hd_transform_size(width, height, hd_num=16):
309
+# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py
310
+def _calc_hd_transform_size(*, width: int, height: int, hd_num: int = 16):
311
transposed = False
312
if width < height:
313
width, height = height, width
@@ -287,7 +322,8 @@ def calc_hd_transform_size(width, height, hd_num=16):
322
new_width = int(scale * 336)
323
new_height = int(new_width / ratio)
324
290
- padded_width, padded_height = calc_padded_size(new_width, new_height)
325
+ padded_width, padded_height = _calc_padded_size(width=new_width,
326
+ height=new_height)
327
328
if transposed:
329
padded_width, padded_height = padded_height, padded_width
@@ -295,17 +331,15 @@ def calc_hd_transform_size(width, height, hd_num=16):
331
return padded_width, padded_height
332
333
298
-def _image_processor(
299
- data: ImagePixelData,
300
- model_config: ModelConfig,
301
- vlm_config: VisionLanguageConfig,
302
-) -> Dict[str, torch.Tensor]:
334
+def _image_processor(ctx: InputContext,
335
+ data: ImagePixelData) -> Dict[str, torch.Tensor]:
336
image = data.image
337
338
if isinstance(image, Image.Image):
339
# Temporary patch before dynamic number of image tokens is supported
307
- _, _, h, w = vlm_config.image_input_shape
308
- if (w, h) != calc_hd_transform_size(image.width, image.height):
340
+ _, _, h, w = ctx.get_multimodal_config().image_input_shape
341
+ if (w, h) != _calc_hd_transform_size(width=image.width,
342
+ height=image.height):
343
logger.warning(
344
"Dynamic image shape is currently not supported. "
345
"Resizing input image to (%d, %d).", w, h)
@@ -313,11 +347,11 @@ def _image_processor(
347
data.image = image.resize((w, h))
348
349
return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
316
- ._default_input_processor(data, model_config, vlm_config)
350
+ ._default_input_mapper(ctx, data)
351
352
319
-@MULTIMODAL_REGISTRY.register_image_pixel_input(_image_processor)
320
-@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
353
+@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper(_image_processor)
354
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
355
class Phi3VForCausalLM(nn.Module, SupportsVision):
356
357
def __init__(self,
vllm/multimodal/__init__.py
+10
-1
@@ -1,5 +1,14 @@
1
from .base import MultiModalData, MultiModalPlugin
2
-from .registry import MULTIMODAL_REGISTRY, MultiModalRegistry
2
+from .registry import MultiModalRegistry
3
+
4
+MULTIMODAL_REGISTRY = MultiModalRegistry()
5
+"""
6
+The global :class:`~MultiModalRegistry` is used by model runners to
7
+dispatch data processing according to its modality and the target model.
8
+
9
+See also:
10
+ :ref:`input_processing_pipeline`
11
+"""
12
13
__all__ = [
14
"MultiModalData", "MultiModalPlugin", "MULTIMODAL_REGISTRY",
vllm/multimodal/base.py
+41
-43
@@ -2,7 +2,8 @@ from abc import ABC, abstractmethod
2
from typing import (TYPE_CHECKING, Callable, Dict, Generic, Optional, Type,
3
TypeVar)
4
5
-from vllm.config import ModelConfig, VisionLanguageConfig
5
+from vllm.config import ModelConfig
6
+from vllm.inputs import InputContext
7
from vllm.logger import init_logger
8
9
if TYPE_CHECKING:
@@ -23,7 +24,7 @@ class MultiModalData:
24
25
Finally, register the new plugin to
26
:const:`vllm.multimodal.MULTIMODAL_REGISTRY`.
26
- This enables models to call :meth:`MultiModalRegistry.register_input` for
27
+ This enables models to call :meth:`MultiModalRegistry.map_input` for
28
the new modality.
29
"""
30
pass
@@ -32,10 +33,9 @@ class MultiModalData:
33
D = TypeVar("D", bound=MultiModalData)
34
N = TypeVar("N", bound=Type["nn.Module"])
35
35
-MultiModalInputProcessor = Callable[[D, ModelConfig, VisionLanguageConfig],
36
- Dict[str, "torch.Tensor"]]
36
+MultiModalInputMapper = Callable[[InputContext, D], Dict[str, "torch.Tensor"]]
37
"""Return a dictionary to be passed as keyword arguments to
38
-:meth:`torch.nn.Module.forward`. This is similar in concept to tokenizers
38
+:meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
39
and processors in HuggingFace Transformers."""
40
41
@@ -50,16 +50,9 @@ class MultiModalPlugin(ABC, Generic[D]):
50
(i.e., the modality of the data).
51
"""
52
53
- @classmethod
54
- def get_model_cls(cls, model_config: ModelConfig) -> Type["nn.Module"]:
55
- # Avoid circular import
56
- from vllm.model_executor.model_loader import get_model_architecture
57
-
58
- return get_model_architecture(model_config)[0]
59
-
53
def __init__(self) -> None:
61
- self._input_processors: Dict[Type["nn.Module"],
62
- MultiModalInputProcessor[D]] = {}
54
+ self._input_mappers: Dict[Type["nn.Module"],
55
+ MultiModalInputMapper[D]] = {}
56
57
@abstractmethod
58
def get_data_type(self) -> Type[D]:
@@ -70,57 +63,62 @@ class MultiModalPlugin(ABC, Generic[D]):
63
raise NotImplementedError
64
65
@abstractmethod
73
- def _default_input_processor(
74
- self, data: D, model_config: ModelConfig,
75
- vlm_config: VisionLanguageConfig) -> Dict[str, "torch.Tensor"]:
66
+ def _default_input_mapper(self, ctx: InputContext,
67
+ data: D) -> Dict[str, "torch.Tensor"]:
68
"""Return a dictionary to be passed as keyword arguments to
77
- :meth:`torch.nn.Module.forward`. This is similar in concept to
69
+ :meth:`~torch.nn.Module.forward`. This is similar in concept to
70
tokenizers and processors in HuggingFace Transformers.
71
"""
72
raise NotImplementedError
73
82
- def register_input_processor(self,
83
- processor: Optional[
84
- MultiModalInputProcessor[D]] = None):
74
+ def register_input_mapper(
75
+ self,
76
+ mapper: Optional[MultiModalInputMapper[D]] = None,
77
+ ):
78
"""
86
- Register an input processor to a model class.
79
+ Register an input mapper to a model class.
80
81
When the model receives input data that matches the modality served by
89
- this plugin (see :meth:`get_data_type`), the provided input processor is
90
- applied to preprocess the data. If `None` is provided, then the default
91
- input processor is applied instead.
82
+ this plugin (see :meth:`get_data_type`), the provided function is
83
+ invoked to transform the data into a dictionary of model inputs.
84
+ If `None` is provided, then the default input mapper is used instead.
85
+
86
+ See also:
87
+ :ref:`input_processing_pipeline`
88
"""
89
90
def wrapper(model_cls: N) -> N:
95
- if model_cls in self._input_processors:
91
+ if model_cls in self._input_mappers:
92
logger.warning(
97
- "Model class %s already has an input processor "
93
+ "Model class %s already has an input mapper "
94
"registered to %s. It is overwritten by the new one.",
95
model_cls, self)
96
101
- self._input_processors[model_cls] = processor \
102
- or self._default_input_processor
97
+ self._input_mappers[model_cls] = mapper \
98
+ or self._default_input_mapper
99
100
return model_cls
101
102
return wrapper
103
108
- def process_input(
109
- self, data: D, model_config: ModelConfig,
110
- vlm_config: VisionLanguageConfig) -> Dict[str, "torch.Tensor"]:
104
+ def map_input(self, model_config: ModelConfig,
105
+ data: D) -> Dict[str, "torch.Tensor"]:
106
"""
112
- Apply an input processor to a :class:`~MultiModalData` instance passed
113
- to the model.
114
-
115
- The model is identified by ``model_config``. ``vlm_config`` is
116
- for compatibility purposes and may be merged into ``model_config``
117
- in the near future.
107
+ Apply an input mapper to a :class:`~MultiModalData` instance passed
108
+ to the model, transforming the data into a dictionary of model inputs.
109
+
110
+ The model is identified by ``model_config``.
111
+
112
+ TODO: Add guide [ref: PR #5276]
113
"""
119
- model_cls = self.get_model_cls(model_config)
114
+ # Avoid circular import
115
+ from vllm.model_executor.model_loader import get_model_architecture
116
+
117
+ model_cls, _ = get_model_architecture(model_config)
118
121
- processor = self._input_processors.get(model_cls)
122
- if processor is None:
123
- raise KeyError(f"No input processor in {self} is registered for "
119
+ mapper = self._input_mappers.get(model_cls)
120
+ if mapper is None:
121
+ raise KeyError(f"No input mapper in {self} is registered for "
122
f"model class {model_cls.__name__}.")
123
126
- return processor(data, model_config, vlm_config)
124
+ return mapper(InputContext(model_config), data)
vllm/multimodal/image.py
+18
-60
@@ -1,70 +1,28 @@
1
-from typing import Dict, Tuple, Type, Union
1
+from functools import lru_cache
2
+from typing import Dict, Type, Union
3
4
import torch
5
from PIL import Image
6
6
-from vllm.config import ModelConfig, VisionLanguageConfig
7
+from vllm.config import ModelConfig
8
+from vllm.inputs.registry import InputContext
9
from vllm.logger import init_logger
8
-from vllm.sequence import SequenceData
9
-from vllm.transformers_utils.image_processor import cached_get_image_processor
10
+from vllm.transformers_utils.image_processor import get_image_processor
11
12
from .base import MultiModalData, MultiModalPlugin
13
14
logger = init_logger(__name__)
15
15
-
16
-def _get_dummy_seq_data(seq_len: int,
17
- vlm_config: VisionLanguageConfig) -> SequenceData:
18
- # NOTE: We assume that <image> token is repeated `image_feature_size` times
19
- # and then concatenated with the text prompt
20
- # TODO: Enable other ways of inserting the image into the prompt
21
-
22
- token_ids = [vlm_config.image_token_id] * vlm_config.image_feature_size
23
- token_ids += [0] * (seq_len - vlm_config.image_feature_size)
24
-
25
- return SequenceData(token_ids)
26
-
27
-
28
-def _get_dummy_values(vlm_config: VisionLanguageConfig) -> torch.Tensor:
29
- if vlm_config.image_processor is None:
30
- values_dtype = torch.float16
31
- else:
32
- values_dtype = torch.uint8
33
-
34
- return torch.zeros(vlm_config.image_input_shape, dtype=values_dtype)
35
-
36
-
37
-def get_dummy_image_data(
38
- seq_len: int,
39
- model_config: ModelConfig,
40
- vlm_config: VisionLanguageConfig,
41
-) -> Tuple[SequenceData, MultiModalData]:
42
- """Standard dummy data factory for image data (to be used in
43
- :meth:`vlm.multimodal.MultiModalRegistry.register_dummy_data`)."""
44
- seq_data = _get_dummy_seq_data(seq_len, vlm_config)
45
- values = _get_dummy_values(vlm_config)
46
-
47
- config_input_type = vlm_config.image_input_type
48
- ImageInputType = VisionLanguageConfig.ImageInputType
49
-
50
- fake_mm_data: MultiModalData
51
- if config_input_type == ImageInputType.PIXEL_VALUES:
52
- fake_mm_data = ImagePixelData(values)
53
- elif config_input_type == ImageInputType.IMAGE_FEATURES:
54
- fake_mm_data = ImageFeatureData(values)
55
- else:
56
- raise NotImplementedError
57
-
58
- return seq_data, fake_mm_data
16
+cached_get_image_processor = lru_cache(get_image_processor)
17
18
19
class ImagePixelData(MultiModalData):
20
"""
21
The pixel data of an image. Can be one of:
22
65
- - :class:``PIL.Image``: An image object. Requires that a HuggingFace
23
+ - :class:`PIL.Image.Image`: An image object. Requires that a HuggingFace
24
processor is available to the model.
67
- - :class:``torch.Tensor``: The raw pixel data which is passed to the model
25
+ - :class:`torch.Tensor`: The raw pixel data which is passed to the model
26
without additional pre-processing.
27
"""
28
@@ -89,8 +47,8 @@ class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
47
def get_data_type(self) -> Type[ImagePixelData]:
48
return ImagePixelData
49
92
- def _get_hf_image_processor(self, model_config: ModelConfig,
93
- vlm_config: VisionLanguageConfig):
50
+ def _get_hf_image_processor(self, model_config: ModelConfig):
51
+ vlm_config = model_config.multimodal_config
52
if vlm_config is None or vlm_config.image_processor is None:
53
return None
54
@@ -100,14 +58,13 @@ class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
58
revision=vlm_config.image_processor_revision,
59
)
60
103
- def _default_input_processor(
104
- self, data: ImagePixelData, model_config: ModelConfig,
105
- vlm_config: VisionLanguageConfig) -> Dict[str, torch.Tensor]:
61
+ def _default_input_mapper(self, ctx: InputContext,
62
+ data: ImagePixelData) -> Dict[str, torch.Tensor]:
63
+ model_config = ctx.model_config
64
image = data.image
65
66
if isinstance(image, Image.Image):
109
- image_processor = self._get_hf_image_processor(
110
- model_config, vlm_config)
67
+ image_processor = self._get_hf_image_processor(model_config)
68
if image_processor is None:
69
raise RuntimeError("No HuggingFace processor is available"
70
"to process the image object")
@@ -147,9 +104,10 @@ class ImageFeaturePlugin(MultiModalPlugin[ImageFeatureData]):
104
def get_data_type(self) -> Type[ImageFeatureData]:
105
return ImageFeatureData
106
150
- def _default_input_processor(
151
- self, data: ImageFeatureData, model_config: ModelConfig,
152
- vlm_config: VisionLanguageConfig) -> Dict[str, torch.Tensor]:
107
+ def _default_input_mapper(
108
+ self, ctx: InputContext,
109
+ data: ImageFeatureData) -> Dict[str, torch.Tensor]:
110
+ model_config = ctx.model_config
111
image_features = data.image_features.to(model_config.dtype)
112
113
return {"image_features": image_features}
vllm/multimodal/registry.py
+40
-93
@@ -1,46 +1,35 @@
1
import functools
2
-from typing import (TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence,
3
- Tuple, Type, TypeVar)
2
+from typing import Any, Optional, Sequence, Type, TypeVar
3
5
-from vllm.config import ModelConfig, VisionLanguageConfig
4
+from torch import nn
5
+
6
+from vllm.config import ModelConfig
7
from vllm.logger import init_logger
8
8
-from .base import MultiModalData, MultiModalPlugin
9
+from .base import MultiModalData, MultiModalInputMapper, MultiModalPlugin
10
from .image import (ImageFeatureData, ImageFeaturePlugin, ImagePixelData,
11
ImagePixelPlugin)
12
12
-if TYPE_CHECKING:
13
- import torch
14
- from torch import nn
15
-
16
- from vllm.sequence import SequenceData
17
-
13
logger = init_logger(__name__)
14
15
D = TypeVar("D", bound=MultiModalData)
21
-N = TypeVar("N", bound=Type["nn.Module"])
22
-
23
-MultiModalInputProcessor = Callable[[D, ModelConfig, VisionLanguageConfig],
24
- Dict[str, "torch.Tensor"]]
25
-MultiModalDummyFactory = Callable[[int, ModelConfig, VisionLanguageConfig],
26
- Tuple["SequenceData", MultiModalData]]
16
+N = TypeVar("N", bound=Type[nn.Module])
17
18
19
class MultiModalRegistry:
20
"""
31
- This registry is used by model runners to dispatch data processing
21
+ A registry to dispatch data processing
22
according to its modality and the target model.
23
"""
24
25
DEFAULT_PLUGINS = (ImageFeaturePlugin(), ImagePixelPlugin())
26
37
- def __init__(self,
38
- *,
39
- plugins: Sequence[MultiModalPlugin[Any]] = DEFAULT_PLUGINS
40
- ) -> None:
27
+ def __init__(
28
+ self,
29
+ *,
30
+ plugins: Sequence[MultiModalPlugin[Any]] = DEFAULT_PLUGINS,
31
+ ) -> None:
32
self._plugins_by_data_type = {p.get_data_type(): p for p in plugins}
42
- self._dummy_factories_by_model_type: Dict[Type["nn.Module"],
43
- MultiModalDummyFactory] = {}
33
34
def register_plugin(self, plugin: MultiModalPlugin[Any]) -> None:
35
data_type = plugin.get_data_type()
@@ -62,95 +51,53 @@ class MultiModalRegistry:
51
msg = f"Unknown multi-modal data type: {data_type}"
52
raise NotImplementedError(msg)
53
65
- def register_dummy_data(self, factory: MultiModalDummyFactory):
66
- """
67
- Register a dummy data factory to a model class.
68
-
69
- During memory profiling, the provided function is invoked to create
70
- dummy data to be inputted into the model. The modality and shape of
71
- the dummy data should be an upper bound of what the model would receive
72
- at inference time.
73
- """
74
-
75
- def wrapper(model_cls: N) -> N:
76
- if model_cls in self._dummy_factories_by_model_type:
77
- logger.warning(
78
- "Model class %s already has dummy data "
79
- "registered to %s. It is overwritten by the new one.",
80
- model_cls, self)
81
-
82
- self._dummy_factories_by_model_type[model_cls] = factory
83
-
84
- return model_cls
85
-
86
- return wrapper
87
-
88
- def dummy_data_for_profiling(self, seq_len: int, model_config: ModelConfig,
89
- vlm_config: VisionLanguageConfig):
90
- """Create dummy data for memory profiling."""
91
- model_cls = MultiModalPlugin.get_model_cls(model_config)
92
- dummy_factory = self._dummy_factories_by_model_type.get(model_cls)
93
- if dummy_factory is None:
94
- msg = f"No dummy data defined for model class: {model_cls}"
95
- raise NotImplementedError(msg)
96
-
97
- return dummy_factory(seq_len, model_config, vlm_config)
98
-
99
- def register_input(
100
- self,
101
- data_type: Type[D],
102
- processor: Optional[MultiModalInputProcessor[D]] = None):
54
+ def register_input_mapper(
55
+ self,
56
+ data_type: Type[D],
57
+ mapper: Optional[MultiModalInputMapper[D]] = None,
58
+ ):
59
"""
104
- Register an input processor for a specific modality to a model class.
60
+ Register an input mapper for a specific modality to a model class.
61
106
- See :meth:`MultiModalPlugin.register_input_processor` for more details.
62
+ See :meth:`MultiModalPlugin.register_input_mapper` for more details.
63
"""
64
return self._get_plugin_for_data_type(data_type) \
109
- .register_input_processor(processor)
65
+ .register_input_mapper(mapper)
66
111
- def register_image_pixel_input(
112
- self,
113
- processor: Optional[
114
- MultiModalInputProcessor[ImagePixelData]] = None):
67
+ def register_image_pixel_input_mapper(
68
+ self,
69
+ mapper: Optional[MultiModalInputMapper[ImagePixelData]] = None,
70
+ ):
71
"""
116
- Register an input processor for image pixel data to a model class.
72
+ Register an input mapper for image pixel data to a model class.
73
118
- See :meth:`MultiModalPlugin.register_input_processor` for more details.
74
+ See :meth:`MultiModalPlugin.register_input_mapper` for more details.
75
"""
120
- return self.register_input(ImagePixelData, processor)
76
+ return self.register_input_mapper(ImagePixelData, mapper)
77
122
- def register_image_feature_input(
78
+ def register_image_feature_input_mapper(
79
self,
124
- processor: Optional[
125
- MultiModalInputProcessor[ImageFeatureData]] = None):
80
+ mapper: Optional[MultiModalInputMapper[ImageFeatureData]] = None,
81
+ ):
82
"""
127
- Register an input processor for image feature data to a model class.
83
+ Register an input mapper for image feature data to a model class.
84
129
- See :meth:`MultiModalPlugin.register_input_processor` for more details.
85
+ See :meth:`MultiModalPlugin.register_input_mapper` for more details.
86
"""
131
- return self.register_input(ImageFeatureData, processor)
87
+ return self.register_input_mapper(ImageFeatureData, mapper)
88
133
- def process_input(self, data: MultiModalData, model_config: ModelConfig,
134
- vlm_config: VisionLanguageConfig):
89
+ def map_input(self, model_config: ModelConfig, data: MultiModalData):
90
"""
136
- Apply an input processor to a :class:`~MultiModalData` instance passed
91
+ Apply an input mapper to a :class:`~MultiModalData` instance passed
92
to the model.
93
139
- See :meth:`MultiModalPlugin.process_input` for more details.
94
+ See :meth:`MultiModalPlugin.map_input` for more details.
95
"""
96
return self._get_plugin_for_data_type(type(data)) \
142
- .process_input(data, model_config, vlm_config)
97
+ .map_input(model_config, data)
98
144
- def create_input_processor(self, model_config: ModelConfig,
145
- vlm_config: VisionLanguageConfig):
99
+ def create_input_mapper(self, model_config: ModelConfig):
100
"""
147
- Create an input processor (see :meth:`process_input`) for a
148
- specific model.
101
+ Create an input mapper (see :meth:`map_input`) for a specific model.
102
"""
150
- return functools.partial(self.process_input,
151
- model_config=model_config,
152
- vlm_config=vlm_config)
153
-
154
-
155
-MULTIMODAL_REGISTRY = MultiModalRegistry()
156
-"""The global :class:`~MultiModalRegistry` which is used by model runners."""
103
+ return functools.partial(self.map_input, model_config)
vllm/sequence.py
+2
-2
@@ -8,12 +8,12 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
8
9
import torch
10
11
-from vllm.inputs import LLMInputs
11
from vllm.lora.request import LoRARequest
12
from vllm.pooling_params import PoolingParams
13
from vllm.sampling_params import SamplingParams
14
15
if TYPE_CHECKING:
16
+ from vllm.inputs import LLMInputs
17
from vllm.multimodal import MultiModalData
18
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
19
@@ -221,7 +221,7 @@ class Sequence:
221
def __init__(
222
self,
223
seq_id: int,
224
- inputs: LLMInputs,
224
+ inputs: "LLMInputs",
225
block_size: int,
226
eos_token_id: Optional[int] = None,
227
lora_request: Optional[LoRARequest] = None,
vllm/transformers_utils/image_processor.py
-4
@@ -1,4 +1,3 @@
1
-from functools import lru_cache
1
from typing import Optional
2
3
from transformers import AutoImageProcessor
@@ -40,6 +39,3 @@ def get_image_processor(
39
raise e
40
41
return processor
43
-
44
-
45
-cached_get_image_processor = lru_cache(get_image_processor)
vllm/worker/cpu_model_runner.py
+4
-16
@@ -110,15 +110,9 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
110
self.block_size,
111
)
112
113
- # Create processor for multi-modal data
114
- if self.vision_language_config is not None:
115
- self.multi_modal_input_processor = MULTIMODAL_REGISTRY \
116
- .create_input_processor(
117
- self.model_config,
118
- self.vision_language_config,
119
- )
120
- else:
121
- self.multi_modal_input_processor = None
113
+ # Multi-modal data support
114
+ self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
115
+ .create_input_mapper(self.model_config)
116
117
# Lazy initialization.
118
self.model: nn.Module # Set after init_Model
@@ -168,13 +162,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
162
163
mm_data = seq_group_metadata.multi_modal_data
164
if mm_data is not None:
171
- # Process multi-modal data
172
- if self.multi_modal_input_processor is None:
173
- raise ValueError(
174
- "Multi-modal inputs are only supported by "
175
- "vision language models.")
176
-
177
- mm_kwargs = self.multi_modal_input_processor(mm_data)
165
+ mm_kwargs = self.multi_modal_input_mapper(mm_data)
166
for k, v in mm_kwargs.items():
167
multi_modal_kwargs_list[k].append(v)
168
vllm/worker/model_runner.py
+9
-22
@@ -15,6 +15,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
ModelConfig, ParallelConfig, SchedulerConfig,
16
VisionLanguageConfig)
17
from vllm.distributed.parallel_state import graph_capture
18
+from vllm.inputs import INPUT_REGISTRY
19
from vllm.logger import init_logger
20
from vllm.lora.layers import LoRAMapping
21
from vllm.lora.request import LoRARequest
@@ -25,7 +26,7 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
26
from vllm.model_executor.models.interfaces import supports_lora
27
from vllm.multimodal import MULTIMODAL_REGISTRY
28
from vllm.sampling_params import SamplingParams
28
-from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
29
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
30
from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
31
is_pin_memory_available, make_tensor_with_pad)
32
from vllm.worker.model_runner_base import (
@@ -191,15 +192,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
192
self.block_size,
193
) if num_attn_heads else None
194
194
- # Create processor for multi-modal data
195
- if self.vision_language_config is not None:
196
- self.multi_modal_input_processor = MULTIMODAL_REGISTRY \
197
- .create_input_processor(
198
- self.model_config,
199
- self.vision_language_config,
200
- )
201
- else:
202
- self.multi_modal_input_processor = None
195
+ # Multi-modal data support
196
+ self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
197
+ .create_input_mapper(self.model_config)
198
199
# Lazy initialization
200
self.model: nn.Module # Set after load_model
@@ -506,12 +501,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
501
mm_data = seq_group_metadata.multi_modal_data
502
if mm_data is not None:
503
# Process multi-modal data
509
- if self.multi_modal_input_processor is None:
510
- raise ValueError(
511
- "Multi-modal inputs are only supported by "
512
- "vision language models.")
513
-
514
- mm_kwargs = self.multi_modal_input_processor(mm_data)
504
+ mm_kwargs = self.multi_modal_input_mapper(mm_data)
505
for k, v in mm_kwargs.items():
506
multi_modal_kwargs_list[k].append(v)
507
@@ -764,12 +754,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
754
seq_len = (max_num_batched_tokens // max_num_seqs +
755
(group_id < max_num_batched_tokens % max_num_seqs))
756
767
- if vlm_config is None:
768
- seq_data = SequenceData([0] * seq_len)
769
- dummy_multi_modal_data = None
770
- else:
771
- seq_data, dummy_multi_modal_data = MULTIMODAL_REGISTRY \
772
- .dummy_data_for_profiling(seq_len, model_config, vlm_config)
757
+ seq_data, dummy_multi_modal_data = INPUT_REGISTRY \
758
+ .dummy_data_for_profiling(model_config, seq_len)
759
+ assert len(seq_data.prompt_token_ids) == seq_len
760
761
seq = SequenceGroupMetadata(
762
request_id=str(group_id),