A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[VLM][Core] Support profiling with multiple multi-modal inputs per prompt (#7126)
Cyrus Leung committed
Aug 15, 2024 at 01:55 UTC
3f674a49b5033a6ed778ab960e86e03cfa64aa1f
38 files changed
+573
-217
docs/source/dev/input_processing/input_processing_pipeline.rst
+1
-1
@@ -17,4 +17,4 @@ Input Processing Pipeline
17
18
6. If the data contains multi-modal data, convert it into keyword arguments using :meth:`MULTIMODAL_REGISTRY.map_input <vllm.multimodal.MultiModalRegistry.map_input>`.
19
20
- - For example, convert a :class:`PIL.Image.Image` input to its pixel values for a vision language model.
20
+ - For example, convert a :class:`PIL.Image.Image` input to its pixel values for a vision model.
docs/source/dev/multimodal/multimodal_index.rst
+3
@@ -15,6 +15,9 @@ by following :ref:`this guide <adding_multimodal_plugin>`.
15
16
Looking to add your own multi-modal model? Please follow the instructions listed :ref:`here <enabling_multimodal_inputs>`.
17
18
+..
19
+ TODO: Add usage of --limit-mm-per-prompt when multi-image input is officially supported
20
+
21
Guides
22
++++++
23
docs/source/models/enabling_multimodal_inputs.rst
+1
-1
@@ -66,7 +66,7 @@ A default mapper is available for each modality in the core vLLM library. This i
66
3. Register maximum number of multi-modal tokens
67
------------------------------------------------
68
69
-For each modality type that the model accepts as input, calculate the maximum possible number of tokens
69
+For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data instance
70
and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
71
72
.. code-block:: diff
tests/engine/test_arg_utils.py
new
+24
@@ -0,0 +1,24 @@
1
+import pytest
2
+
3
+from vllm.engine.arg_utils import EngineArgs
4
+from vllm.utils import FlexibleArgumentParser
5
+
6
+
7
+@pytest.mark.parametrize(("arg", "expected"), [
8
+ (None, None),
9
+ ("image=16", {
10
+ "image": 16
11
+ }),
12
+ ("image=16,video=2", {
13
+ "image": 16,
14
+ "video": 2
15
+ }),
16
+])
17
+def test_limit_mm_per_prompt_parser(arg, expected):
18
+ parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
19
+ if arg is None:
20
+ args = parser.parse_args([])
21
+ else:
22
+ args = parser.parse_args(["--limit-mm-per-prompt", arg])
23
+
24
+ assert args.limit_mm_per_prompt == expected
tests/models/test_blip2.py
+1
-1
@@ -59,7 +59,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
59
All the image fixtures for the test is under tests/images.
60
For huggingface runner, we provide the PIL images as input.
61
For vllm runner, we provide MultiModalData objects and corresponding
62
- vision language config as input.
62
+ MultiModalConfig as input.
63
Note, the text input is also adjusted to abide by vllm contract.
64
The text output is sanitized to be able to compare with hf.
65
"""
tests/models/test_fuyu.py
+1
-1
@@ -49,7 +49,7 @@ def run_test(
49
All the image fixtures for the test is under tests/images.
50
For huggingface runner, we provide the PIL images as input.
51
For vllm runner, we provide MultiModalDataDict objects
52
- and corresponding vision language config as input.
52
+ and corresponding MultiModalConfig as input.
53
Note, the text input is also adjusted to abide by vllm contract.
54
The text output is sanitized to be able to compare with hf.
55
"""
tests/models/test_internvl.py
+1
-1
@@ -117,7 +117,7 @@ def run_test(
117
All the image fixtures for the test is under tests/images.
118
For huggingface runner, we provide the PIL images as input.
119
For vllm runner, we provide MultiModalDataDict objects
120
- and corresponding vision language config as input.
120
+ and corresponding MultiModalConfig as input.
121
Note, the text input is also adjusted to abide by vllm contract.
122
The text output is sanitized to be able to compare with hf.
123
"""
tests/models/test_llava.py
+1
-1
@@ -69,7 +69,7 @@ def run_test(
69
All the image fixtures for the test is under tests/images.
70
For huggingface runner, we provide the PIL images as input.
71
For vllm runner, we provide MultiModalDataDict objects
72
- and corresponding vision language config as input.
72
+ and corresponding MultiModalConfig as input.
73
Note, the text input is also adjusted to abide by vllm contract.
74
The text output is sanitized to be able to compare with hf.
75
"""
tests/models/test_llava_next.py
+1
-1
@@ -177,7 +177,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
177
All the image fixtures for the test is under tests/images.
178
For huggingface runner, we provide the PIL images as input.
179
For vllm runner, we provide MultiModalDataDict objects
180
- and corresponding vision language config as input.
180
+ and corresponding MultiModalConfig as input.
181
Note, the text input is also adjusted to abide by vllm contract.
182
The text output is sanitized to be able to compare with hf.
183
"""
tests/models/test_minicpmv.py
+3
-2
@@ -61,7 +61,7 @@ def run_test(
61
All the image fixtures for the test is under tests/images.
62
For huggingface runner, we provide the PIL images as input.
63
For vllm runner, we provide MultiModalDataDict objects
64
- and corresponding vision language config as input.
64
+ and corresponding MultiModalConfig as input.
65
Note, the text input is also adjusted to abide by vllm contract.
66
The text output is sanitized to be able to compare with hf.
67
"""
@@ -176,7 +176,7 @@ def run_multi_image_test(
176
All the image fixtures for the test is under tests/images.
177
For huggingface runner, we provide the PIL images as input.
178
For vllm runner, we provide MultiModalDataDict objects
179
- and corresponding vision language config as input.
179
+ and corresponding MultiModalConfig as input.
180
Note, the text input is also adjusted to abide by vllm contract.
181
The text output is sanitized to be able to compare with hf.
182
"""
@@ -197,6 +197,7 @@ def run_multi_image_test(
197
with vllm_runner(model,
198
max_model_len=4096,
199
max_num_seqs=1,
200
+ limit_mm_per_prompt={"image": len(images)},
201
dtype=dtype,
202
tensor_parallel_size=tensor_parallel_size,
203
distributed_executor_backend=distributed_executor_backend,
tests/models/test_paligemma.py
+1
-1
@@ -72,7 +72,7 @@ def run_test(
72
All the image fixtures for the test is under tests/images.
73
For huggingface runner, we provide the PIL images as input.
74
For vllm runner, we provide MultiModalDataDict objects
75
- and corresponding vision language config as input.
75
+ and corresponding MultiModalConfig as input.
76
Note, the text input is also adjusted to abide by vllm contract.
77
The text output is sanitized to be able to compare with hf.
78
"""
tests/models/test_phi3v.py
+1
-1
@@ -73,7 +73,7 @@ def run_test(
73
All the image fixtures for the test is under tests/images.
74
For huggingface runner, we provide the PIL images as input.
75
For vllm runner, we provide MultiModalDataDict objects
76
- and corresponding vision language config as input.
76
+ and corresponding MultiModalConfig as input.
77
Note, the text input is also adjusted to abide by vllm contract.
78
The text output is sanitized to be able to compare with hf.
79
"""
tests/multimodal/test_mapper.py
+78
-6
@@ -1,15 +1,22 @@
1
+from contextlib import nullcontext
2
+
3
import numpy as np
4
import pytest
5
from transformers import CLIPImageProcessor, LlavaNextImageProcessor
6
5
-from vllm.config import ModelConfig
6
-from vllm.multimodal import MULTIMODAL_REGISTRY
7
+from vllm.config import ModelConfig, MultiModalConfig
8
+from vllm.multimodal import MultiModalRegistry
9
from vllm.multimodal.utils import rescale_image_size
10
11
12
+@pytest.fixture
13
+def mm_registry():
14
+ return MultiModalRegistry()
15
+
16
+
17
@pytest.mark.parametrize("dtype", ["half", "float"])
18
@pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
12
-def test_clip_image_processor(image_assets, dtype, size_factor):
19
+def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
20
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
21
22
hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
@@ -24,6 +31,9 @@ def test_clip_image_processor(image_assets, dtype, size_factor):
31
dtype=dtype,
32
revision=None,
33
)
34
+ mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
35
+
36
+ mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
37
38
for asset in image_assets:
39
image = rescale_image_size(asset.pil_image, size_factor)
@@ -32,7 +42,7 @@ def test_clip_image_processor(image_assets, dtype, size_factor):
42
image,
43
return_tensors="pt",
44
)
35
- vllm_result = MULTIMODAL_REGISTRY.map_input(
45
+ vllm_result = mm_registry.map_input(
46
model_config,
47
{"image": image},
48
)
@@ -48,7 +58,8 @@ def test_clip_image_processor(image_assets, dtype, size_factor):
58
59
@pytest.mark.parametrize("dtype", ["half", "float"])
60
@pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
51
-def test_llava_next_image_processor(image_assets, dtype, size_factor):
61
+def test_llava_next_image_processor(image_assets, mm_registry, dtype,
62
+ size_factor):
63
MODEL_NAME = "llava-hf/llava-v1.6-vicuna-7b-hf"
64
65
hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
@@ -63,6 +74,9 @@ def test_llava_next_image_processor(image_assets, dtype, size_factor):
74
dtype=dtype,
75
revision=None,
76
)
77
+ mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
78
+
79
+ mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
80
81
for asset in image_assets:
82
image = rescale_image_size(asset.pil_image, size_factor)
@@ -71,7 +85,7 @@ def test_llava_next_image_processor(image_assets, dtype, size_factor):
85
image,
86
return_tensors="pt",
87
)
74
- vllm_result = MULTIMODAL_REGISTRY.map_input(
88
+ vllm_result = mm_registry.map_input(
89
model_config,
90
{"image": image},
91
)
@@ -83,3 +97,61 @@ def test_llava_next_image_processor(image_assets, dtype, size_factor):
97
98
assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
99
assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
100
+
101
+
102
+@pytest.mark.parametrize(
103
+ ("num_images", "limit", "is_valid"),
104
+ [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
105
+ (2, 1, False), (2, 2, True)],
106
+)
107
+def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
108
+ MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
109
+
110
+ model_config = ModelConfig(
111
+ model=MODEL_NAME,
112
+ tokenizer=MODEL_NAME,
113
+ tokenizer_mode="auto",
114
+ trust_remote_code=False,
115
+ seed=0,
116
+ dtype="half",
117
+ revision=None,
118
+ )
119
+ mm_config = MultiModalConfig(limit_per_prompt={"image": limit})
120
+
121
+ mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
122
+
123
+ image = image_assets[0].pil_image
124
+ if num_images == 0:
125
+ mm_inputs = {}
126
+ elif num_images == 1:
127
+ mm_inputs = {"image": image}
128
+ else:
129
+ mm_inputs = {"image": [image] * num_images}
130
+
131
+ with nullcontext() if is_valid else pytest.raises(ValueError):
132
+ mm_registry.map_input(model_config, mm_inputs)
133
+
134
+
135
+# NOTE: We don't test zero images since the HF processor doesn't support it
136
+@pytest.mark.parametrize("num_images", [1, 2])
137
+def test_image_mapper_multi(image_assets, mm_registry, num_images):
138
+ MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
139
+
140
+ model_config = ModelConfig(
141
+ model=MODEL_NAME,
142
+ tokenizer=MODEL_NAME,
143
+ tokenizer_mode="auto",
144
+ trust_remote_code=False,
145
+ seed=0,
146
+ dtype="half",
147
+ revision=None,
148
+ )
149
+ mm_config = MultiModalConfig(limit_per_prompt={"image": num_images})
150
+
151
+ mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
152
+
153
+ image = image_assets[0].pil_image
154
+ mm_inputs = {"image": [image] * num_images}
155
+
156
+ mapped_inputs = mm_registry.map_input(model_config, mm_inputs)
157
+ assert len(mapped_inputs["pixel_values"]) == num_images
vllm/config.py
+10
-4
@@ -1,7 +1,8 @@
1
import enum
2
import json
3
from dataclasses import dataclass, field, fields
4
-from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Type, Union
4
+from typing import (TYPE_CHECKING, ClassVar, List, Mapping, Optional, Tuple,
5
+ Type, Union)
6
7
import torch
8
from transformers import PretrainedConfig
@@ -1429,10 +1430,15 @@ class PromptAdapterConfig:
1430
1431
@dataclass
1432
class MultiModalConfig:
1432
- """Configs the input data format and how models should run for
1433
- multimodal models."""
1433
+ """Controls the behavior of multimodal models."""
1434
+
1435
+ limit_per_prompt: Mapping[str, int]
1436
+ """
1437
+ The maximum number of multi-modal input instances allowed per prompt
1438
+ for each :class:`~vllm.multimodal.MultiModalPlugin`.
1439
+ """
1440
+
1441
# TODO: Add configs to init vision tower or not.
1435
- pass
1442
1443
1444
_STR_DTYPE_TO_TORCH_DTYPE = {
vllm/engine/arg_utils.py
+43
-5
@@ -2,7 +2,8 @@ import argparse
2
import dataclasses
3
import json
4
from dataclasses import dataclass
5
-from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union
5
+from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Type,
6
+ Union)
7
8
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -15,8 +16,7 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
from vllm.utils import FlexibleArgumentParser
17
18
if TYPE_CHECKING:
18
- from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
19
- BaseTokenizerGroup)
19
+ from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
20
21
logger = init_logger(__name__)
22
@@ -29,11 +29,32 @@ def nullable_str(val: str):
29
return val
30
31
32
+def nullable_kvs(val: str) -> Optional[Mapping[str, int]]:
33
+ if len(val) == 0:
34
+ return None
35
+
36
+ out_dict: Dict[str, int] = {}
37
+ for item in val.split(","):
38
+ try:
39
+ key, value = item.split("=")
40
+ except TypeError as exc:
41
+ msg = "Each item should be in the form KEY=VALUE"
42
+ raise ValueError(msg) from exc
43
+
44
+ try:
45
+ out_dict[key] = int(value)
46
+ except ValueError as exc:
47
+ msg = f"Failed to parse value of item {key}={value}"
48
+ raise ValueError(msg) from exc
49
+
50
+ return out_dict
51
+
52
+
53
@dataclass
54
class EngineArgs:
55
"""Arguments for vLLM engine."""
56
model: str = 'facebook/opt-125m'
36
- served_model_name: Optional[Union[List[str]]] = None
57
+ served_model_name: Optional[Union[str, List[str]]] = None
58
tokenizer: Optional[str] = None
59
skip_tokenizer_init: bool = False
60
tokenizer_mode: str = 'auto'
@@ -81,6 +102,7 @@ class EngineArgs:
102
# notice.
103
tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]] = "ray"
104
tokenizer_pool_extra_config: Optional[dict] = None
105
+ limit_mm_per_prompt: Optional[Mapping[str, int]] = None
106
enable_lora: bool = False
107
max_loras: int = 1
108
max_lora_rank: int = 16
@@ -435,6 +457,21 @@ class EngineArgs:
457
'This should be a JSON string that will be '
458
'parsed into a dictionary. Ignored if '
459
'tokenizer_pool_size is 0.')
460
+
461
+ # Multimodal related configs
462
+ parser.add_argument(
463
+ '--limit-mm-per-prompt',
464
+ type=nullable_kvs,
465
+ default=EngineArgs.limit_mm_per_prompt,
466
+ # The default value is given in
467
+ # MultiModalRegistry.init_mm_limits_per_prompt
468
+ help=('For each multimodal plugin, limit how many '
469
+ 'input instances to allow for each prompt. '
470
+ 'Expects a comma-separated list of items, '
471
+ 'e.g.: `image=16,video=2` allows a maximum of 16 '
472
+ 'images and 2 videos per prompt. Defaults to 1 for '
473
+ 'each modality.'))
474
+
475
# LoRA related configs
476
parser.add_argument('--enable-lora',
477
action='store_true',
@@ -709,7 +746,8 @@ class EngineArgs:
746
"CPU offload space must be non-negative"
747
f", but got {self.cpu_offload_gb}")
748
712
- multimodal_config = MultiModalConfig()
749
+ multimodal_config = MultiModalConfig(
750
+ limit_per_prompt=self.limit_mm_per_prompt or {})
751
752
device_config = DeviceConfig(device=self.device)
753
model_config = ModelConfig(
vllm/engine/llm_engine.py
+7
-4
@@ -24,8 +24,9 @@ from vllm.engine.output_processor.stop_checker import StopChecker
24
from vllm.engine.output_processor.util import create_output_by_sequence_group
25
from vllm.executor.executor_base import ExecutorBase
26
from vllm.executor.ray_utils import initialize_ray_cluster
27
-from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs, LLMInputs,
28
- PromptInputs, SingletonPromptInputs)
27
+from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
28
+ InputRegistry, LLMInputs, PromptInputs,
29
+ SingletonPromptInputs)
30
from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
31
from vllm.logger import init_logger
32
from vllm.lora.request import LoRARequest
@@ -180,6 +181,7 @@ class LLMEngine:
181
log_stats: bool,
182
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
183
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
184
+ input_registry: InputRegistry = INPUT_REGISTRY,
185
) -> None:
186
logger.info(
187
"Initializing an LLM engine (v%s) with config: "
@@ -265,8 +267,9 @@ class LLMEngine:
267
self.generation_config_fields = _load_generation_config_dict(
268
model_config)
269
268
- self.input_processor = INPUT_REGISTRY.create_input_processor(
269
- self.model_config)
270
+ self.input_registry = input_registry
271
+ self.input_processor = input_registry.create_input_processor(
272
+ model_config)
273
274
self.model_executor = executor_class(
275
model_config=model_config,
vllm/inputs/registry.py
+69
-14
@@ -1,6 +1,8 @@
1
import functools
2
+from collections import UserDict
3
from dataclasses import dataclass
3
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type
4
+from typing import (TYPE_CHECKING, Callable, Dict, Mapping, Optional, Protocol,
5
+ Tuple, Type)
6
7
from torch import nn
8
from transformers import PretrainedConfig
@@ -12,7 +14,7 @@ from .data import LLMInputs
14
15
if TYPE_CHECKING:
16
from vllm.config import ModelConfig, MultiModalConfig
15
- from vllm.multimodal import MultiModalDataDict
17
+ from vllm.multimodal import MultiModalDataDict, MultiModalRegistry
18
from vllm.sequence import SequenceData
19
20
logger = init_logger(__name__)
@@ -65,15 +67,38 @@ class InputContext:
67
68
N = TypeVar("N", bound=Type[nn.Module])
69
68
-DummyDataFactory = Callable[[InputContext, int],
69
- Tuple["SequenceData",
70
- Optional["MultiModalDataDict"]]]
71
-"""
72
-Create dummy data to be inputted into the model.
70
74
-Note:
75
- :data:`InputProcessor` is not applied to the dummy data.
76
-"""
71
+class DummyDataFactory(Protocol):
72
+
73
+ def __call__(
74
+ self,
75
+ ctx: InputContext,
76
+ seq_len: int,
77
+ mm_counts: Mapping[str, int],
78
+ ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
79
+ """
80
+ Create dummy data to be inputted into the model.
81
+
82
+ Note:
83
+ :data:`InputProcessor` is not applied to the dummy data.
84
+ """
85
+ ...
86
+
87
+
88
+class _MultiModalCounts(UserDict):
89
+ """
90
+ Wraps `mm_counts` for a more informative error message
91
+ when attempting to access a plugin that does not exist.
92
+ """
93
+
94
+ def __getitem__(self, key: str) -> int:
95
+ try:
96
+ return super().__getitem__(key)
97
+ except KeyError as exc:
98
+ msg = (f"There is no multi-modal plugin with the key: {key}. "
99
+ f"Available keys: {set(self.keys())}")
100
+ raise KeyError(msg) from exc
101
+
102
103
InputProcessor = Callable[[InputContext, LLMInputs], LLMInputs]
104
"""Preprocess the inputs to the model."""
@@ -95,6 +120,7 @@ class InputRegistry:
120
self,
121
ctx: InputContext,
122
seq_len: int,
123
+ mm_counts: Mapping[str, int],
124
) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
125
"""
126
The default dummy data factory represents the longest possible text
@@ -133,8 +159,12 @@ class InputRegistry:
159
160
return wrapper
161
136
- def dummy_data_for_profiling(self, model_config: "ModelConfig",
137
- seq_len: int):
162
+ def dummy_data_for_profiling(
163
+ self,
164
+ model_config: "ModelConfig",
165
+ seq_len: int,
166
+ mm_registry: "MultiModalRegistry",
167
+ ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
168
"""
169
Create dummy data for profiling the memory usage of a model.
170
@@ -142,6 +172,10 @@ class InputRegistry:
172
173
See also:
174
:ref:`enabling_multimodal_inputs`
175
+
176
+ Note:
177
+ This should be called after
178
+ :meth:`~MultiModalRegistry.init_mm_limits_per_prompt`.
179
"""
180
# Avoid circular import
181
from vllm.model_executor.model_loader import get_model_architecture
@@ -149,8 +183,29 @@ class InputRegistry:
183
model_cls, _ = get_model_architecture(model_config)
184
dummy_factory = self._dummy_factories_by_model_type \
185
.get(model_cls, self._default_dummy_data_factory)
152
-
153
- return dummy_factory(InputContext(model_config), seq_len)
186
+ mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
187
+
188
+ seq_data, mm_data = dummy_factory(
189
+ InputContext(model_config),
190
+ seq_len,
191
+ _MultiModalCounts(mm_counts),
192
+ )
193
+
194
+ # Having more tokens is over-conservative but otherwise fine
195
+ num_tokens = seq_data.prompt_token_ids
196
+ assert len(num_tokens) >= seq_len, (
197
+ f"Expected at least {seq_len} dummy tokens for profiling, "
198
+ f"but found {len(num_tokens)} tokens instead.")
199
+
200
+ if mm_data is not None:
201
+ for k, v in mm_data.items():
202
+ num_items = len(v) if isinstance(v, list) else 1
203
+ num_expected = mm_counts[k]
204
+ assert num_items >= num_expected, (
205
+ f"Expected at least {num_expected} dummy '{k}' instances "
206
+ f"for profiling, but found {num_items} instances instead.")
207
+
208
+ return seq_data, mm_data
209
210
def _default_input_processor(self, ctx: InputContext,
211
inputs: LLMInputs) -> LLMInputs:
vllm/model_executor/model_loader/loader.py
+1
-1
@@ -133,7 +133,7 @@ def _get_model_initialization_kwargs(
133
134
if supports_multimodal(model_class):
135
if multimodal_config is None:
136
- raise ValueError("Provide vision related configurations "
136
+ raise ValueError("Provide multi-modal related configurations "
137
"through LLM entrypoint or engine arguments.")
138
139
extra_kwargs["multimodal_config"] = multimodal_config
vllm/model_executor/models/blip.py
+4
-3
@@ -31,13 +31,13 @@ def get_blip_num_patches(*, image_size: int, patch_size: int) -> int:
31
32
33
def get_blip_image_feature_size(
34
- hf_config: Union[BlipVisionConfig, Blip2VisionConfig], ) -> int:
34
+ hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int:
35
return get_blip_num_patches(image_size=hf_config.image_size,
36
patch_size=hf_config.patch_size)
37
38
39
def get_max_blip_image_tokens(
40
- hf_config: Union[BlipVisionConfig, Blip2VisionConfig], ) -> int:
40
+ hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int:
41
return get_blip_image_feature_size(hf_config)
42
43
@@ -60,6 +60,7 @@ def dummy_seq_data_for_blip(
60
61
def dummy_image_for_blip(
62
hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
63
+ num_images: int,
64
*,
65
image_width_override: Optional[int] = None,
66
image_height_override: Optional[int] = None,
@@ -71,7 +72,7 @@ def dummy_image_for_blip(
72
height = image_height_override
73
74
image = Image.new("RGB", (width, height), color=0)
74
- return {"image": image}
75
+ return {"image": image if num_images == 1 else [image] * num_images}
76
77
78
def input_processor_for_blip(
vllm/model_executor/models/blip2.py
+30
-7
@@ -1,4 +1,5 @@
1
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
1
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2
+ TypedDict, Union)
3
4
import torch
5
import torch.nn as nn
@@ -413,17 +414,39 @@ def get_max_blip2_image_tokens(ctx: InputContext):
414
raise NotImplementedError(msg)
415
416
416
-def dummy_data_for_blip2(ctx: InputContext, seq_len: int):
417
+def dummy_seq_data_for_blip2(
418
+ hf_config: Blip2Config,
419
+ seq_len: int,
420
+ num_images: int,
421
+ *,
422
+ image_token_id: int,
423
+ image_feature_size_override: Optional[int] = None,
424
+):
425
+ if image_feature_size_override is None:
426
+ image_feature_size = get_blip2_image_feature_size(hf_config)
427
+ else:
428
+ image_feature_size = image_feature_size_override
429
+
430
+ token_ids = [image_token_id] * image_feature_size * num_images
431
+ token_ids += [0] * (seq_len - image_feature_size * num_images)
432
+ return SequenceData(token_ids)
433
+
434
+
435
+def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
436
+ mm_counts: Mapping[str, int]):
437
hf_config = ctx.get_hf_config(Blip2Config)
438
vision_config = hf_config.vision_config
439
+ num_images = mm_counts["image"]
440
420
- image_feature_size = get_blip2_image_feature_size(hf_config)
421
- token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
422
- token_ids += [0] * (seq_len - image_feature_size)
423
- seq_data = SequenceData(token_ids)
441
+ seq_data = dummy_seq_data_for_blip2(
442
+ hf_config,
443
+ seq_len,
444
+ num_images,
445
+ image_token_id=BLIP2_IMAGE_TOKEN_ID,
446
+ )
447
448
if isinstance(vision_config, Blip2VisionConfig):
426
- mm_data = dummy_image_for_blip(vision_config)
449
+ mm_data = dummy_image_for_blip(vision_config, num_images)
450
451
return seq_data, mm_data
452
vllm/model_executor/models/chameleon.py
+14
-9
@@ -1,6 +1,6 @@
1
from functools import cached_property
2
-from typing import (Any, Dict, Iterable, List, Literal, Optional, Tuple,
3
- TypedDict)
2
+from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
3
+ Tuple, TypedDict)
4
5
import torch
6
import torch.nn.functional as F
@@ -19,8 +19,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
19
QKVParallelLinear,
20
RowParallelLinear)
21
from vllm.model_executor.layers.logits_processor import LogitsProcessor
22
-from vllm.model_executor.layers.quantization.base_config import (
23
- QuantizationConfig)
22
+from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.rotary_embedding import get_rope
24
from vllm.model_executor.layers.sampler import Sampler
25
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -61,6 +60,7 @@ def get_max_chameleon_image_tokens(ctx: InputContext):
60
61
def dummy_seq_data_for_chameleon(
62
seq_len: int,
63
+ num_images: int,
64
*,
65
image_token_id: int,
66
image_feature_size_override: Optional[int] = None,
@@ -70,12 +70,14 @@ def dummy_seq_data_for_chameleon(
70
else:
71
image_feature_size = image_feature_size_override
72
73
- token_ids = [image_token_id] * image_feature_size
74
- token_ids += [0] * (seq_len - image_feature_size)
73
+ token_ids = [image_token_id] * image_feature_size * num_images
74
+ token_ids += [0] * (seq_len - image_feature_size * num_images)
75
return SequenceData(token_ids)
76
77
78
def dummy_image_for_chameleon(
79
+ num_images: int,
80
+ *,
81
image_width_override: Optional[int] = None,
82
image_height_override: Optional[int] = None,
83
):
@@ -87,17 +89,20 @@ def dummy_image_for_chameleon(
89
height = image_height_override
90
91
image = Image.new("RGB", (width, height), color=0)
90
- return {"image": image}
92
+ return {"image": image if num_images == 1 else [image] * num_images}
93
94
93
-def dummy_data_for_chameleon(ctx: InputContext, seq_len: int):
95
+def dummy_data_for_chameleon(ctx: InputContext, seq_len: int,
96
+ mm_counts: Mapping[str, int]):
97
+ num_images = mm_counts["image"]
98
99
seq_data = dummy_seq_data_for_chameleon(
100
seq_len,
101
+ num_images,
102
image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
103
)
104
100
- mm_data = dummy_image_for_chameleon()
105
+ mm_data = dummy_image_for_chameleon(num_images)
106
return seq_data, mm_data
107
108
vllm/model_executor/models/clip.py
+5
-3
@@ -43,6 +43,7 @@ def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
43
def dummy_seq_data_for_clip(
44
hf_config: CLIPVisionConfig,
45
seq_len: int,
46
+ num_images: int,
47
*,
48
image_token_id: int,
49
image_feature_size_override: Optional[int] = None,
@@ -52,13 +53,14 @@ def dummy_seq_data_for_clip(
53
else:
54
image_feature_size = image_feature_size_override
55
55
- token_ids = [image_token_id] * image_feature_size
56
- token_ids += [0] * (seq_len - image_feature_size)
56
+ token_ids = [image_token_id] * image_feature_size * num_images
57
+ token_ids += [0] * (seq_len - image_feature_size * num_images)
58
return SequenceData(token_ids)
59
60
61
def dummy_image_for_clip(
62
hf_config: CLIPVisionConfig,
63
+ num_images: int,
64
*,
65
image_width_override: Optional[int] = None,
66
image_height_override: Optional[int] = None,
@@ -70,7 +72,7 @@ def dummy_image_for_clip(
72
height = image_height_override
73
74
image = Image.new("RGB", (width, height), color=0)
73
- return {"image": image}
75
+ return {"image": image if num_images == 1 else [image] * num_images}
76
77
78
def input_processor_for_clip(
vllm/model_executor/models/fuyu.py
+16
-11
@@ -16,7 +16,7 @@
16
# limitations under the License.
17
""" PyTorch Fuyu model."""
18
import math
19
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
19
+from typing import Iterable, List, Literal, Mapping, Optional, Tuple, TypedDict
20
21
import torch
22
import torch.nn as nn
@@ -29,8 +29,7 @@ from vllm.config import CacheConfig, MultiModalConfig
29
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30
from vllm.logger import init_logger
31
from vllm.model_executor.layers.linear import ColumnParallelLinear
32
-from vllm.model_executor.layers.quantization.base_config import (
33
- QuantizationConfig)
32
+from vllm.model_executor.layers.quantization import QuantizationConfig
33
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -94,27 +93,33 @@ def get_max_fuyu_image_tokens(ctx: InputContext):
93
return (ncol + 1) * nrow
94
95
97
-def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int):
96
+def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int):
97
ncol, nrow = get_max_fuyu_image_feature_size()
98
image_feature_size = get_max_fuyu_image_tokens(ctx)
99
101
- token_ids = ([_IMAGE_TOKEN_ID] * ncol + [_NEWLINE_TOKEN_ID]) * nrow
102
- token_ids += [0] * (seq_len - image_feature_size)
100
+ image_token_ids = ([_IMAGE_TOKEN_ID] * ncol + [_NEWLINE_TOKEN_ID]) * nrow
101
+ token_ids = image_token_ids * num_images
102
+ token_ids += [0] * (seq_len - image_feature_size * num_images)
103
return SequenceData(token_ids)
104
105
106
def dummy_image_for_fuyu(
107
+ num_images: int,
108
+ *,
109
image_width: int,
110
image_height: int,
111
):
112
image = Image.new("RGB", (image_width, image_height), color=0)
111
- return {"image": image}
113
+ return {"image": image if num_images == 1 else [image] * num_images}
114
115
114
-def dummy_data_for_fuyu(ctx: InputContext, seq_len: int):
115
- seq_data = dummy_seq_data_for_fuyu(ctx, seq_len)
116
- mm_data = dummy_image_for_fuyu(MAX_IMAGE_FEATURE_SIZE_WIDTH,
117
- MAX_IMAGE_FEATURE_SIZE_HEIGHT)
116
+def dummy_data_for_fuyu(ctx: InputContext, seq_len: int,
117
+ mm_counts: Mapping[str, int]):
118
+ num_images = mm_counts["image"]
119
+ seq_data = dummy_seq_data_for_fuyu(ctx, seq_len, num_images)
120
+ mm_data = dummy_image_for_fuyu(num_images,
121
+ image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
122
+ image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT)
123
return seq_data, mm_data
124
125
vllm/model_executor/models/interfaces.py
+2
-5
@@ -11,14 +11,11 @@ logger = init_logger(__name__)
11
12
@runtime_checkable
13
class SupportsMultiModal(Protocol):
14
- """
15
- The interface required for all multimodal (vision or audio) language
16
- models.
17
- """
14
+ """The interface required for all multi-modal models."""
15
16
supports_multimodal: ClassVar[Literal[True]] = True
17
"""
21
- A flag that indicates this model supports multimodal inputs.
18
+ A flag that indicates this model supports multi-modal inputs.
19
20
Note:
21
There is no need to redefine this flag if this class is in the
vllm/model_executor/models/internvl.py
+8
-3
@@ -5,7 +5,8 @@
5
# Licensed under The MIT License [see LICENSE for details]
6
# --------------------------------------------------------
7
import itertools
8
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
8
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9
+ TypedDict, Union)
10
11
import torch
12
import torch.nn as nn
@@ -230,7 +231,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
231
232
233
def input_mapper_for_internvl(ctx: InputContext, data: object):
233
- hf_config = ctx.get_hf_config(PretrainedConfig)
234
+ hf_config = ctx.get_hf_config()
235
236
use_thumbnail = hf_config.use_thumbnail
237
min_num = hf_config.min_dynamic_patch
@@ -256,7 +257,9 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
257
})
258
259
259
-def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
260
+def dummy_data_for_internvl(ctx: InputContext, seq_len: int,
261
+ mm_counts: Mapping[str, int]):
262
+ num_images = mm_counts["image"]
263
264
image_feature_size = get_max_internvl_image_tokens(ctx)
265
model_config = ctx.model_config
@@ -268,6 +271,7 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
271
seq_data = dummy_seq_data_for_clip(
272
vision_config,
273
seq_len,
274
+ num_images,
275
image_token_id=tokenizer.encode(IMG_CONTEXT,
276
add_special_tokens=False)[0],
277
image_feature_size_override=image_feature_size,
@@ -281,6 +285,7 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
285
286
mm_data = dummy_image_for_clip(
287
vision_config,
288
+ num_images,
289
image_width_override=max_image_width,
290
image_height_override=max_image_height,
291
)
vllm/model_executor/models/llava.py
+10
-6
@@ -1,5 +1,6 @@
1
import itertools
2
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
+ TypedDict, Union)
4
5
import torch
6
import torch.nn as nn
@@ -9,8 +10,7 @@ from vllm.attention import AttentionMetadata
10
from vllm.config import CacheConfig, MultiModalConfig
11
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12
from vllm.model_executor.layers.activation import get_act_fn
12
-from vllm.model_executor.layers.quantization.base_config import (
13
- QuantizationConfig)
13
+from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
16
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -88,9 +88,11 @@ def get_max_llava_image_tokens(ctx: InputContext):
88
raise ValueError(f"Unexpected select feature strategy: {strategy}")
89
90
91
-def dummy_data_for_llava(ctx: InputContext, seq_len: int):
91
+def dummy_data_for_llava(ctx: InputContext, seq_len: int,
92
+ mm_counts: Mapping[str, int]):
93
hf_config = ctx.get_hf_config(LlavaConfig)
94
vision_config = hf_config.vision_config
95
+ num_images = mm_counts["image"]
96
97
image_feature_size = get_max_llava_image_tokens(ctx)
98
@@ -98,21 +100,23 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int):
100
seq_data = dummy_seq_data_for_clip(
101
vision_config,
102
seq_len,
103
+ num_images,
104
image_token_id=hf_config.image_token_index,
105
image_feature_size_override=image_feature_size,
106
)
107
105
- mm_data = dummy_image_for_clip(vision_config)
108
+ mm_data = dummy_image_for_clip(vision_config, num_images)
109
return seq_data, mm_data
110
elif isinstance(vision_config, SiglipVisionConfig):
111
seq_data = dummy_seq_data_for_siglip(
112
vision_config,
113
seq_len,
114
+ num_images,
115
image_token_id=hf_config.image_token_index,
116
image_feature_size_override=image_feature_size,
117
)
118
115
- mm_data = dummy_image_for_siglip(vision_config)
119
+ mm_data = dummy_image_for_siglip(vision_config, num_images)
120
return seq_data, mm_data
121
122
msg = f"Unsupported vision config: {type(vision_config)}"
vllm/model_executor/models/llava_next.py
+10
-4
@@ -1,5 +1,6 @@
1
import itertools
2
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
+ TypedDict, Union)
4
5
import torch
6
import torch.nn as nn
@@ -13,8 +14,7 @@ from vllm.attention import AttentionMetadata
14
from vllm.config import CacheConfig, MultiModalConfig
15
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
16
from vllm.logger import init_logger
16
-from vllm.model_executor.layers.quantization.base_config import (
17
- QuantizationConfig)
17
+from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -158,9 +158,11 @@ def get_max_llava_next_image_tokens(ctx: InputContext):
158
)
159
160
161
-def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
161
+def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
162
+ mm_counts: Mapping[str, int]):
163
hf_config = ctx.get_hf_config(LlavaNextConfig)
164
vision_config = hf_config.vision_config
165
+ num_images = mm_counts["image"]
166
167
image_feature_size = get_max_llava_next_image_tokens(ctx)
168
@@ -168,12 +170,14 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
170
seq_data = dummy_seq_data_for_clip(
171
vision_config,
172
seq_len,
173
+ num_images,
174
image_token_id=hf_config.image_token_index,
175
image_feature_size_override=image_feature_size,
176
)
177
178
mm_data = dummy_image_for_clip(
179
vision_config,
180
+ num_images,
181
image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
182
image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
183
)
@@ -183,12 +187,14 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
187
seq_data = dummy_seq_data_for_siglip(
188
vision_config,
189
seq_len,
190
+ num_images,
191
image_token_id=hf_config.image_token_index,
192
image_feature_size_override=image_feature_size,
193
)
194
195
mm_data = dummy_image_for_siglip(
196
vision_config,
197
+ num_images,
198
image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
199
image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
200
)
vllm/model_executor/models/minicpmv.py
+11
-10
@@ -24,8 +24,8 @@
24
import math
25
import re
26
from functools import partial
27
-from typing import (Any, Callable, Iterable, List, Optional, Tuple, TypedDict,
28
- Union)
27
+from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
28
+ TypedDict, Union)
29
30
import numpy as np
31
import torch
@@ -42,8 +42,7 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
42
from vllm.logger import init_logger
43
from vllm.model_executor.layers.linear import ReplicatedLinear
44
from vllm.model_executor.layers.logits_processor import LogitsProcessor
45
-from vllm.model_executor.layers.quantization.base_config import (
46
- QuantizationConfig)
45
+from vllm.model_executor.layers.quantization import QuantizationConfig
46
from vllm.model_executor.layers.sampler import Sampler
47
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
48
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
@@ -408,22 +407,24 @@ def get_max_minicpmv_image_tokens(ctx: InputContext):
407
return getattr(hf_config, "query_num", 64)
408
409
411
-def dummy_seq_data_for_minicpmv(seq_len: int):
410
+def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
411
token_ids = [0] * seq_len
412
return SequenceData(token_ids)
413
414
416
-def dummy_image_for_minicpmv(hf_config: PretrainedConfig):
415
+def dummy_image_for_minicpmv(hf_config: PretrainedConfig, num_images: int):
416
width = height = hf_config.image_size
417
image = Image.new("RGB", (width, height), color=0)
419
- return {"image": image}
418
+ return {"image": image if num_images == 1 else [image] * num_images}
419
420
422
-def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int):
421
+def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
422
+ mm_counts: Mapping[str, int]):
423
hf_config = ctx.get_hf_config()
424
+ num_images = mm_counts["image"]
425
425
- seq_data = dummy_seq_data_for_minicpmv(seq_len)
426
- mm_data = dummy_image_for_minicpmv(hf_config)
426
+ seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images)
427
+ mm_data = dummy_image_for_minicpmv(hf_config, num_images)
428
429
return seq_data, mm_data
430
vllm/model_executor/models/paligemma.py
+8
-5
@@ -1,4 +1,5 @@
1
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
1
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2
+ TypedDict, Union)
3
4
import torch
5
from torch import nn
@@ -9,8 +10,7 @@ from vllm.config import CacheConfig, MultiModalConfig
10
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11
from vllm.logger import init_logger
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
12
-from vllm.model_executor.layers.quantization.base_config import (
13
- QuantizationConfig)
13
+from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.model_executor.layers.sampler import Sampler
15
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
from vllm.model_executor.models.gemma import GemmaModel
@@ -57,17 +57,20 @@ def get_max_paligemma_image_tokens(ctx: InputContext):
57
return get_max_siglip_image_tokens(vision_config)
58
59
60
-def dummy_data_for_paligemma(ctx: InputContext, seq_len: int):
60
+def dummy_data_for_paligemma(ctx: InputContext, seq_len: int,
61
+ mm_counts: Mapping[str, int]):
62
hf_config = ctx.get_hf_config(PaliGemmaConfig)
63
vision_config = hf_config.vision_config
64
+ num_images = mm_counts["image"]
65
66
seq_data = dummy_seq_data_for_siglip(
67
vision_config,
68
seq_len,
69
+ num_images,
70
image_token_id=hf_config.image_token_index,
71
)
72
70
- mm_data = dummy_image_for_siglip(vision_config)
73
+ mm_data = dummy_image_for_siglip(vision_config, num_images)
74
return seq_data, mm_data
75
76
vllm/model_executor/models/phi3v.py
+8
-4
@@ -15,7 +15,8 @@
15
# limitations under the License.
16
import re
17
from functools import lru_cache
18
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
18
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
19
+ TypedDict, Union)
20
21
import numpy as np
22
import torch
@@ -28,8 +29,7 @@ from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
29
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30
from vllm.logger import init_logger
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
31
-from vllm.model_executor.layers.quantization.base_config import (
32
- QuantizationConfig)
32
+from vllm.model_executor.layers.quantization import QuantizationConfig
33
from vllm.model_executor.layers.sampler import Sampler
34
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
35
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -347,18 +347,22 @@ def get_max_phi3v_image_tokens(ctx: InputContext):
347
)
348
349
350
-def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
350
+def dummy_data_for_phi3v(ctx: InputContext, seq_len: int,
351
+ mm_counts: Mapping[str, int]):
352
+ num_images = mm_counts["image"]
353
354
image_feature_size = get_max_phi3v_image_tokens(ctx)
355
356
seq_data = dummy_seq_data_for_clip(
357
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
358
seq_len,
359
+ num_images,
360
image_token_id=_IMAGE_TOKEN_ID,
361
image_feature_size_override=image_feature_size,
362
)
363
mm_data = dummy_image_for_clip(
364
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
365
+ num_images,
366
image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
367
image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
368
)
vllm/model_executor/models/siglip.py
+5
-3
@@ -52,6 +52,7 @@ def get_max_siglip_image_tokens(hf_config: SiglipVisionConfig) -> int:
52
def dummy_seq_data_for_siglip(
53
hf_config: SiglipVisionConfig,
54
seq_len: int,
55
+ num_images: int,
56
*,
57
image_token_id: int,
58
image_feature_size_override: Optional[int] = None,
@@ -61,13 +62,14 @@ def dummy_seq_data_for_siglip(
62
else:
63
image_feature_size = image_feature_size_override
64
64
- token_ids = [image_token_id] * image_feature_size
65
- token_ids += [0] * (seq_len - image_feature_size)
65
+ token_ids = [image_token_id] * image_feature_size * num_images
66
+ token_ids += [0] * (seq_len - image_feature_size * num_images)
67
return SequenceData(token_ids)
68
69
70
def dummy_image_for_siglip(
71
hf_config: SiglipVisionConfig,
72
+ num_images: int,
73
*,
74
image_width_override: Optional[int] = None,
75
image_height_override: Optional[int] = None,
@@ -79,7 +81,7 @@ def dummy_image_for_siglip(
81
height = image_height_override
82
83
image = Image.new("RGB", (width, height), color=0)
82
- return {"image": image}
84
+ return {"image": image if num_images == 1 else [image] * num_images}
85
86
87
def input_processor_for_siglip(
vllm/multimodal/base.py
+33
-15
@@ -1,9 +1,9 @@
1
import sys
2
from abc import ABC, abstractmethod
3
from collections import UserDict, defaultdict
4
-from typing import Any, Callable, Dict, List, Optional
4
+from typing import Callable, Dict, List, Mapping, Optional
5
from typing import Sequence as GenericSequence
6
-from typing import Tuple, Type, TypedDict, TypeVar, Union, cast
6
+from typing import Tuple, Type, TypedDict, TypeVar, Union, cast, final
7
8
import numpy as np
9
import torch
@@ -116,17 +116,30 @@ class MultiModalInputs(_MultiModalInputsBase):
116
batched_inputs)
117
118
119
+_T = TypeVar("_T")
120
+
121
+MultiModalData: TypeAlias = Union[_T, List[_T]]
122
+"""
123
+Either a single data instance, or a list of data instances.
124
+
125
+The number of data instances allowed per modality is restricted by
126
+`--limit-mm-per-prompt`.
127
+"""
128
+
129
+
130
+@final
131
class MultiModalDataBuiltins(TypedDict, total=False):
132
"""Modality types that are predefined by vLLM."""
133
122
- image: Image.Image
123
- """The input image."""
134
+ image: MultiModalData[Image.Image]
135
+ """The input image(s)."""
136
125
- audio: Tuple[np.ndarray, Union[int, float]]
126
- """The input audio and its sampling rate."""
137
+ audio: MultiModalData[Tuple[np.ndarray, Union[int, float]]]
138
+ """The input audio item(s) and corresponding sampling rate(s)."""
139
140
129
-MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
141
+MultiModalDataDict = Union[MultiModalDataBuiltins,
142
+ Mapping[str, MultiModalData[object]]]
143
"""
144
A dictionary containing an item for each modality type to input.
145
@@ -137,7 +150,8 @@ Note:
150
Read more on that :ref:`here <adding_multimodal_plugin>`.
151
"""
152
140
-MultiModalInputMapper = Callable[[InputContext, object], MultiModalInputs]
153
+MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
154
+ MultiModalInputs]
155
"""
156
Return a dictionary to be passed as keyword arguments to
157
:meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
@@ -181,8 +195,11 @@ class MultiModalPlugin(ABC):
195
raise NotImplementedError
196
197
@abstractmethod
184
- def _default_input_mapper(self, ctx: InputContext,
185
- data: object) -> MultiModalInputs:
198
+ def _default_input_mapper(
199
+ self,
200
+ ctx: InputContext,
201
+ data: MultiModalData[object],
202
+ ) -> MultiModalInputs:
203
"""
204
Return a dictionary to be passed as keyword arguments to
205
:meth:`~torch.nn.Module.forward`. This is similar in concept to
@@ -225,7 +242,7 @@ class MultiModalPlugin(ABC):
242
return wrapper
243
244
def map_input(self, model_config: ModelConfig,
228
- data: object) -> MultiModalInputs:
245
+ data: MultiModalData[object]) -> MultiModalInputs:
246
"""
247
Transform the data into a dictionary of model inputs using the
248
input mapper registered for that model.
@@ -254,8 +271,8 @@ class MultiModalPlugin(ABC):
271
@abstractmethod
272
def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
273
"""
257
- Calculate the maximum number of multimodal tokens input to the language
258
- model. This does not include tokens that correspond to the input text.
274
+ Calculate the maximum number of tokens, corresponding to a single
275
+ instance of multimodal data, that are passed to the language model.
276
"""
277
raise NotImplementedError
278
@@ -269,8 +286,9 @@ class MultiModalPlugin(ABC):
286
max_mm_tokens: Optional[MultiModalTokensCalc] = None,
287
):
288
"""
272
- Register the maximum number of multi-modal tokens input to the
273
- language model for a model class.
289
+ Register the maximum number of tokens, corresponding to a single
290
+ instance of multimodal data, that are passed to the language model
291
+ for a model class.
292
293
If `None` is provided, then the default calculation is used instead.
294
vllm/multimodal/image.py
+6
-3
@@ -11,7 +11,7 @@ from vllm.transformers_utils.image_processor import get_image_processor
11
from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
12
from vllm.utils import is_list_of
13
14
-from .base import MultiModalInputs, MultiModalPlugin
14
+from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
15
16
logger = init_logger(__name__)
17
@@ -110,8 +110,11 @@ class ImagePlugin(MultiModalPlugin):
110
model_config.model,
111
trust_remote_code=model_config.trust_remote_code)
112
113
- def _default_input_mapper(self, ctx: InputContext,
114
- data: object) -> MultiModalInputs:
113
+ def _default_input_mapper(
114
+ self,
115
+ ctx: InputContext,
116
+ data: MultiModalData[object],
117
+ ) -> MultiModalInputs:
118
model_config = ctx.model_config
119
120
# PIL image
vllm/multimodal/registry.py
+101
-16
@@ -1,19 +1,33 @@
1
import functools
2
-from typing import Dict, Optional, Sequence
2
+from collections import UserDict
3
+from typing import Dict, Mapping, Optional, Sequence
4
4
-import torch
5
-
6
-from vllm.config import ModelConfig
5
+from vllm.config import ModelConfig, MultiModalConfig
6
from vllm.logger import init_logger
7
8
from .audio import AudioPlugin
9
from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
11
- MultiModalPlugin, MultiModalTokensCalc)
10
+ MultiModalPlugin, MultiModalTokensCalc, NestedTensors)
11
from .image import ImagePlugin
12
13
logger = init_logger(__name__)
14
15
16
+class _MultiModalLimits(UserDict):
17
+ """
18
+ Wraps `_limits_by_model` for a more informative error message
19
+ when attempting to access a model that does not exist.
20
+ """
21
+
22
+ def __getitem__(self, key: ModelConfig) -> Dict[str, int]:
23
+ try:
24
+ return super().__getitem__(key)
25
+ except KeyError as exc:
26
+ msg = (f"Cannot find `mm_limits` for model={key.model}. Did you "
27
+ "forget to call `init_mm_limits_per_prompt`?")
28
+ raise KeyError(msg) from exc
29
+
30
+
31
class MultiModalRegistry:
32
"""
33
A registry that dispatches data processing to the
@@ -28,6 +42,11 @@ class MultiModalRegistry:
42
plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
43
self._plugins = {p.get_data_key(): p for p in plugins}
44
45
+ # This is used for non-multimodal models
46
+ self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
47
+
48
+ self._limits_by_model = _MultiModalLimits()
49
+
50
def register_plugin(self, plugin: MultiModalPlugin) -> None:
51
"""
52
Register a multi-modal plugin so it can be recognized by vLLM.
@@ -86,13 +105,24 @@ class MultiModalRegistry:
105
via the input mapper registered for that model.
106
107
See :meth:`MultiModalPlugin.map_input` for more details.
108
+
109
+ Note:
110
+ This should be called after :meth:`init_mm_limits_per_prompt`.
111
"""
90
- merged_dict: Dict[str, torch.Tensor] = {}
112
+ merged_dict: Dict[str, NestedTensors] = {}
113
114
for data_key, data_value in data.items():
93
- input_dict = self._get_plugin(data_key) \
94
- .map_input(model_config, data_value)
115
+ plugin = self._get_plugin(data_key)
116
117
+ num_items = len(data_value) if isinstance(data_value, list) else 1
118
+ max_items = self._limits_by_model[model_config][data_key]
119
+ if num_items > max_items:
120
+ raise ValueError(
121
+ f"You set {data_key}={max_items} (or defaulted to 1) in "
122
+ f"`--limit-mm-per-prompt`, but found {num_items} items "
123
+ "in the same prompt.")
124
+
125
+ input_dict = plugin.map_input(model_config, data_value)
126
for input_key, input_tensor in input_dict.items():
127
if input_key in merged_dict:
128
raise ValueError(f"The input mappers (keys={set(data)}) "
@@ -115,8 +145,9 @@ class MultiModalRegistry:
145
max_mm_tokens: Optional[MultiModalTokensCalc] = None,
146
):
147
"""
118
- Register the maximum number of tokens, belonging to a
119
- specific modality, input to the language model for a model class.
148
+ Register the maximum number of tokens, corresponding to a single
149
+ instance of multimodal data belonging to a specific modality, that are
150
+ passed to the language model for a model class.
151
"""
152
return self._get_plugin(data_type_key) \
153
.register_max_multimodal_tokens(max_mm_tokens)
@@ -126,8 +157,8 @@ class MultiModalRegistry:
157
max_mm_tokens: Optional[MultiModalTokensCalc] = None,
158
):
159
"""
129
- Register the maximum number of image tokens
130
- input to the language model for a model class.
160
+ Register the maximum number of image tokens, corresponding to a single
161
+ image, that are passed to the language model for a model class.
162
"""
163
return self.register_max_multimodal_tokens("image", max_mm_tokens)
164
@@ -135,9 +166,63 @@ class MultiModalRegistry:
166
"""
167
Get the maximum number of multi-modal tokens
168
for profiling the memory usage of a model.
138
-
169
+
170
See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
171
+
172
+ Note:
173
+ This should be called after :meth:`init_mm_limits_per_prompt`.
174
+ """
175
+ limits_per_plugin = self._limits_by_model[model_config]
176
+
177
+ return sum((limits_per_plugin[key] *
178
+ plugin.get_max_multimodal_tokens(model_config))
179
+ for key, plugin in self._plugins.items())
180
+
181
+ def init_mm_limits_per_prompt(
182
+ self,
183
+ model_config: ModelConfig,
184
+ multimodal_config: Optional[MultiModalConfig],
185
+ ) -> None:
186
+ """
187
+ Initialize the maximum number of multi-modal input instances for each
188
+ modality that are allowed per prompt for a model class.
189
+ """
190
+ if model_config in self._limits_by_model:
191
+ logger.warning(
192
+ "`mm_limits` has already been set for model=%s, and will "
193
+ "be overwritten by the new values.", model_config.model)
194
+
195
+ if multimodal_config is None:
196
+ limits_per_plugin = self._disabled_limits_per_plugin
197
+ else:
198
+ config_limits_per_plugin = multimodal_config.limit_per_prompt
199
+
200
+ extra_keys = config_limits_per_plugin.keys() - self._plugins.keys()
201
+ if extra_keys:
202
+ logger.warning(
203
+ "Detected extra keys in `--limit-mm-per-prompt` which "
204
+ "are not registered as multi-modal plugins: %s. "
205
+ "They will be ignored.", extra_keys)
206
+
207
+ # NOTE: Currently the default is set to 1 for each plugin
208
+ # TODO: Automatically determine the limits based on budget
209
+ # once more models support multi-image inputs
210
+ limits_per_plugin = {
211
+ key: config_limits_per_plugin.get(key, 1)
212
+ for key in self._plugins
213
+ }
214
+
215
+ self._limits_by_model[model_config] = limits_per_plugin
216
+
217
+ def get_mm_limits_per_prompt(
218
+ self,
219
+ model_config: ModelConfig,
220
+ ) -> Mapping[str, int]:
221
+ """
222
+ Get the maximum number of multi-modal input instances for each modality
223
+ that are allowed per prompt for a model class.
224
+
225
+ Note:
226
+ This should be called after :meth:`init_mm_limits_per_prompt`.
227
"""
141
- return sum(
142
- plugin.get_max_multimodal_tokens(model_config)
143
- for plugin in self._plugins.values())
228
+ return self._limits_by_model[model_config]
vllm/utils.py
-28
@@ -13,7 +13,6 @@ import threading
13
import uuid
14
import warnings
15
from asyncio import FIRST_COMPLETED, ensure_future
16
-from collections import defaultdict
16
from functools import lru_cache, partial, wraps
17
from platform import uname
18
from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
@@ -760,16 +759,6 @@ class CudaMemoryProfiler:
759
gc.collect()
760
761
763
-def str_to_int_tuple(s: str) -> Tuple[int, ...]:
764
- """Convert a string to a tuple of integers."""
765
- try:
766
- return tuple(map(int, s.split(",")))
767
- except ValueError as e:
768
- raise ValueError(
769
- "String must be a series of integers separated by commas "
770
- f"(e.g., 1, 2, 3). Given input: {s}") from e
771
-
772
-
762
def make_ndarray_with_pad(
763
x: List[List[T]],
764
pad: T,
@@ -863,23 +852,6 @@ def is_list_of(
852
assert_never(check)
853
854
866
-def merge_dicts(dict1: Dict[K, List[T]],
867
- dict2: Dict[K, List[T]]) -> Dict[K, List[T]]:
868
- """Merge 2 dicts that have key -> List of items.
869
-
870
- When a key conflicts, the values in dict1 is prioritized.
871
- """
872
- merged_dict: Dict[K, List[T]] = defaultdict(list)
873
-
874
- for key, value in dict1.items():
875
- merged_dict[key].extend(value)
876
-
877
- for key, value in dict2.items():
878
- merged_dict[key].extend(value)
879
-
880
- return dict(merged_dict)
881
-
882
-
855
JSONTree = Union[Dict[str, "JSONTree[T]"], List["JSONTree[T]"],
856
Tuple["JSONTree[T]", ...], T]
857
"""A nested JSON structure where the leaves need not be JSON-serializable."""
vllm/worker/enc_dec_model_runner.py
+16
-3
@@ -12,9 +12,10 @@ from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
12
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13
ModelConfig, MultiModalConfig, ObservabilityConfig,
14
ParallelConfig, PromptAdapterConfig, SchedulerConfig)
15
-from vllm.inputs import INPUT_REGISTRY
15
+from vllm.inputs import INPUT_REGISTRY, InputRegistry
16
from vllm.logger import init_logger
17
from vllm.model_executor import SamplingMetadata
18
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
19
from vllm.sampling_params import SamplingParams
20
from vllm.sequence import (IntermediateTensors, PoolerOutput, SamplerOutput,
21
SequenceGroupMetadata)
@@ -83,6 +84,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
84
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
85
multimodal_config: Optional[MultiModalConfig] = None,
86
observability_config: Optional[ObservabilityConfig] = None,
87
+ input_registry: InputRegistry = INPUT_REGISTRY,
88
+ mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
89
):
90
'''
91
EncoderDecoderModelRunner constructor.
@@ -271,6 +274,16 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
274
seqs: List[SequenceGroupMetadata] = []
275
276
model_config = self.model_config
277
+ mm_config = self.multimodal_config
278
+
279
+ input_registry = self.input_registry
280
+ mm_registry = self.mm_registry
281
+ mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
282
+
283
+ max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
284
+ if max_mm_tokens > 0:
285
+ raise NotImplementedError(
286
+ "Multi-modal encoder-decoder models are not supported yet")
287
288
batch_size = 0
289
for group_id in range(max_num_seqs):
@@ -278,8 +291,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
291
(group_id < max_num_batched_tokens % max_num_seqs))
292
batch_size += seq_len
293
281
- seq_data, _ = INPUT_REGISTRY \
282
- .dummy_data_for_profiling(model_config, seq_len)
294
+ seq_data, _ = input_registry \
295
+ .dummy_data_for_profiling(model_config, seq_len, mm_registry)
296
297
# Having more tokens is over-conservative but otherwise fine
298
assert len(seq_data.prompt_token_ids) >= seq_len, (
vllm/worker/model_runner.py
+20
-17
@@ -31,7 +31,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
31
ParallelConfig, PromptAdapterConfig, SchedulerConfig)
32
from vllm.distributed import get_pp_group
33
from vllm.distributed.parallel_state import graph_capture
34
-from vllm.inputs import INPUT_REGISTRY
34
+from vllm.inputs import INPUT_REGISTRY, InputRegistry
35
from vllm.logger import init_logger
36
from vllm.lora.layers import LoRAMapping
37
from vllm.lora.request import LoRARequest
@@ -43,7 +43,7 @@ from vllm.model_executor.models.interfaces import (supports_lora,
43
supports_multimodal)
44
from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
45
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
46
- MultiModalInputs)
46
+ MultiModalInputs, MultiModalRegistry)
47
from vllm.prompt_adapter.layers import PromptAdapterMapping
48
from vllm.prompt_adapter.request import PromptAdapterRequest
49
from vllm.prompt_adapter.worker_manager import (
@@ -807,6 +807,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
807
multimodal_config: Optional[MultiModalConfig] = None,
808
return_hidden_states: bool = False,
809
observability_config: Optional[ObservabilityConfig] = None,
810
+ input_registry: InputRegistry = INPUT_REGISTRY,
811
+ mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
812
):
813
self.model_config = model_config
814
self.parallel_config = parallel_config
@@ -860,8 +862,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
862
) if num_attn_heads else None
863
864
# Multi-modal data support
863
- self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
864
- .create_input_mapper(self.model_config)
865
+ self.input_registry = input_registry
866
+ self.mm_registry = mm_registry
867
+ self.multi_modal_input_mapper = mm_registry \
868
+ .create_input_mapper(model_config)
869
870
# Lazy initialization
871
self.model: nn.Module # Set after load_model
@@ -902,7 +906,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
906
assert supports_lora(self.model), "Model does not support LoRA"
907
assert not supports_multimodal(
908
self.model
905
- ), "To be tested: multimodal language model with LoRA settings."
909
+ ), "To be tested: Multi-modal model with LoRA settings."
910
911
self.lora_manager = LRUCacheWorkerLoRAManager(
912
self.scheduler_config.max_num_seqs,
@@ -1046,17 +1050,21 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1050
# Profile memory usage with max_num_sequences sequences and the total
1051
# number of tokens equal to max_num_batched_tokens.
1052
seqs: List[SequenceGroupMetadata] = []
1049
- # Additional GPU memory may be needed for vision encoding, which needs
1050
- # to be accounted for when calculating the GPU blocks for
1053
+ # Additional GPU memory may be needed for multi-modal encoding, which
1054
+ # needs to be accounted for when calculating the GPU blocks for
1055
# vLLM blocker manager.
1056
# To exercise the worst scenario for GPU memory consumption,
1057
# the number of seqs (batch_size) is chosen to maximize the number
1058
# of images processed.
1059
model_config = self.model_config
1060
+ mm_config = self.multimodal_config
1061
1057
- if supports_multimodal(self.model):
1058
- max_mm_tokens = MULTIMODAL_REGISTRY \
1059
- .get_max_multimodal_tokens(model_config)
1062
+ input_registry = self.input_registry
1063
+ mm_registry = self.mm_registry
1064
+ mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
1065
+
1066
+ max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
1067
+ if max_mm_tokens > 0:
1068
max_num_seqs_orig = max_num_seqs
1069
max_num_seqs = min(max_num_seqs,
1070
max_num_batched_tokens // max_mm_tokens)
@@ -1074,13 +1082,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1082
(group_id < max_num_batched_tokens % max_num_seqs))
1083
batch_size += seq_len
1084
1077
- seq_data, dummy_multi_modal_data = INPUT_REGISTRY \
1078
- .dummy_data_for_profiling(model_config, seq_len)
1079
-
1080
- # Having more tokens is over-conservative but otherwise fine
1081
- assert len(seq_data.prompt_token_ids) >= seq_len, (
1082
- f"Expected at least {seq_len} dummy tokens for profiling, "
1083
- f"but got: {len(seq_data.prompt_token_ids)}")
1085
+ seq_data, dummy_multi_modal_data = input_registry \
1086
+ .dummy_data_for_profiling(model_config, seq_len, mm_registry)
1087
1088
seq = SequenceGroupMetadata(
1089
request_id=str(group_id),
vllm/worker/xpu_model_runner.py
+19
-17
@@ -9,12 +9,11 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
ModelConfig, MultiModalConfig, ParallelConfig,
10
PromptAdapterConfig, SchedulerConfig)
11
from vllm.distributed import broadcast_tensor_dict
12
-from vllm.inputs import INPUT_REGISTRY
12
+from vllm.inputs import INPUT_REGISTRY, InputRegistry
13
from vllm.logger import init_logger
14
from vllm.model_executor.model_loader import get_model
15
-from vllm.model_executor.models.interfaces import supports_multimodal
15
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17
- MultiModalInputs)
16
+ MultiModalInputs, MultiModalRegistry)
17
from vllm.sampling_params import SamplingParams
18
from vllm.sequence import (IntermediateTensors, SamplerOutput,
19
SequenceGroupMetadata)
@@ -89,6 +88,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
88
kv_cache_dtype: Optional[str] = "auto",
89
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
90
is_driver_worker: bool = False,
91
+ input_registry: InputRegistry = INPUT_REGISTRY,
92
+ mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
93
*args,
94
**kwargs,
95
):
@@ -120,8 +121,10 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
121
)
122
123
# Multi-modal data support
123
- self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
124
- .create_input_mapper(self.model_config)
124
+ self.input_registry = input_registry
125
+ self.mm_registry = mm_registry
126
+ self.multi_modal_input_mapper = mm_registry \
127
+ .create_input_mapper(model_config)
128
129
# Lazy initialization.
130
self.model: nn.Module # Set after init_Model
@@ -157,17 +160,21 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
160
# Profile memory usage with max_num_sequences sequences and the total
161
# number of tokens equal to max_num_batched_tokens.
162
seqs: List[SequenceGroupMetadata] = []
160
- # Additional GPU memory may be needed for vision encoding, which needs
161
- # to be accounted for when calculating the GPU blocks for
163
+ # Additional GPU memory may be needed for multi-modal encoding, which
164
+ # needs to be accounted for when calculating the GPU blocks for
165
# vLLM blocker manager.
166
# To exercise the worst scenario for GPU memory consumption,
167
# the number of seqs (batch_size) is chosen to maximize the number
168
# of images processed.
169
model_config = self.model_config
170
+ mm_config = self.multimodal_config
171
168
- if supports_multimodal(self.model):
169
- max_mm_tokens = MULTIMODAL_REGISTRY \
170
- .get_max_multimodal_tokens(model_config)
172
+ input_registry = self.input_registry
173
+ mm_registry = self.mm_registry
174
+ mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
175
+
176
+ max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
177
+ if max_mm_tokens > 0:
178
max_num_seqs_orig = max_num_seqs
179
max_num_seqs = min(max_num_seqs,
180
max_num_batched_tokens // max_mm_tokens)
@@ -183,13 +190,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
190
seq_len = (max_num_batched_tokens // max_num_seqs +
191
(group_id < max_num_batched_tokens % max_num_seqs))
192
186
- seq_data, dummy_multi_modal_data = INPUT_REGISTRY \
187
- .dummy_data_for_profiling(model_config, seq_len)
188
-
189
- # Having more tokens is over-conservative but otherwise fine
190
- assert len(seq_data.prompt_token_ids) >= seq_len, (
191
- f"Expected at least {seq_len} dummy tokens for profiling, "
192
- f"but got: {len(seq_data.prompt_token_ids)}")
193
+ seq_data, dummy_multi_modal_data = input_registry \
194
+ .dummy_data_for_profiling(model_config, seq_len, mm_registry)
195
196
seq = SequenceGroupMetadata(
197
request_id=str(group_id),