@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f674a49b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[VLM][Core] Support profiling with multiple multi-modal inputs per prompt (#7126)

Cyrus Leung committed Aug 15, 2024 at 01:55 UTC 3f674a49b5033a6ed778ab960e86e03cfa64aa1f
38 files changed +573 -217
docs/source/dev/input_processing/input_processing_pipeline.rst
+1 -1
@@ -17,4 +17,4 @@ Input Processing Pipeline
17
18 6. If the data contains multi-modal data, convert it into keyword arguments using :meth:`MULTIMODAL_REGISTRY.map_input <vllm.multimodal.MultiModalRegistry.map_input>`.
19
20 - - For example, convert a :class:`PIL.Image.Image` input to its pixel values for a vision language model.
20 + - For example, convert a :class:`PIL.Image.Image` input to its pixel values for a vision model.
docs/source/dev/multimodal/multimodal_index.rst
+3
@@ -15,6 +15,9 @@ by following :ref:`this guide <adding_multimodal_plugin>`.
15
16 Looking to add your own multi-modal model? Please follow the instructions listed :ref:`here <enabling_multimodal_inputs>`.
17
18 +..
19 + TODO: Add usage of --limit-mm-per-prompt when multi-image input is officially supported
20 +
21 Guides
22 ++++++
23
docs/source/models/enabling_multimodal_inputs.rst
+1 -1
@@ -66,7 +66,7 @@ A default mapper is available for each modality in the core vLLM library. This i
66 3. Register maximum number of multi-modal tokens
67 ------------------------------------------------
68
69 -For each modality type that the model accepts as input, calculate the maximum possible number of tokens
69 +For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data instance
70 and register it via :meth:`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
71
72 .. code-block:: diff
tests/engine/test_arg_utils.py new
+24
@@ -0,0 +1,24 @@
1 +import pytest
2 +
3 +from vllm.engine.arg_utils import EngineArgs
4 +from vllm.utils import FlexibleArgumentParser
5 +
6 +
7 +@pytest.mark.parametrize(("arg", "expected"), [
8 + (None, None),
9 + ("image=16", {
10 + "image": 16
11 + }),
12 + ("image=16,video=2", {
13 + "image": 16,
14 + "video": 2
15 + }),
16 +])
17 +def test_limit_mm_per_prompt_parser(arg, expected):
18 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
19 + if arg is None:
20 + args = parser.parse_args([])
21 + else:
22 + args = parser.parse_args(["--limit-mm-per-prompt", arg])
23 +
24 + assert args.limit_mm_per_prompt == expected
tests/models/test_blip2.py
+1 -1
@@ -59,7 +59,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
59 All the image fixtures for the test is under tests/images.
60 For huggingface runner, we provide the PIL images as input.
61 For vllm runner, we provide MultiModalData objects and corresponding
62 - vision language config as input.
62 + MultiModalConfig as input.
63 Note, the text input is also adjusted to abide by vllm contract.
64 The text output is sanitized to be able to compare with hf.
65 """
tests/models/test_fuyu.py
+1 -1
@@ -49,7 +49,7 @@ def run_test(
49 All the image fixtures for the test is under tests/images.
50 For huggingface runner, we provide the PIL images as input.
51 For vllm runner, we provide MultiModalDataDict objects
52 - and corresponding vision language config as input.
52 + and corresponding MultiModalConfig as input.
53 Note, the text input is also adjusted to abide by vllm contract.
54 The text output is sanitized to be able to compare with hf.
55 """
tests/models/test_internvl.py
+1 -1
@@ -117,7 +117,7 @@ def run_test(
117 All the image fixtures for the test is under tests/images.
118 For huggingface runner, we provide the PIL images as input.
119 For vllm runner, we provide MultiModalDataDict objects
120 - and corresponding vision language config as input.
120 + and corresponding MultiModalConfig as input.
121 Note, the text input is also adjusted to abide by vllm contract.
122 The text output is sanitized to be able to compare with hf.
123 """
tests/models/test_llava.py
+1 -1
@@ -69,7 +69,7 @@ def run_test(
69 All the image fixtures for the test is under tests/images.
70 For huggingface runner, we provide the PIL images as input.
71 For vllm runner, we provide MultiModalDataDict objects
72 - and corresponding vision language config as input.
72 + and corresponding MultiModalConfig as input.
73 Note, the text input is also adjusted to abide by vllm contract.
74 The text output is sanitized to be able to compare with hf.
75 """
tests/models/test_llava_next.py
+1 -1
@@ -177,7 +177,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
177 All the image fixtures for the test is under tests/images.
178 For huggingface runner, we provide the PIL images as input.
179 For vllm runner, we provide MultiModalDataDict objects
180 - and corresponding vision language config as input.
180 + and corresponding MultiModalConfig as input.
181 Note, the text input is also adjusted to abide by vllm contract.
182 The text output is sanitized to be able to compare with hf.
183 """
tests/models/test_minicpmv.py
+3 -2
@@ -61,7 +61,7 @@ def run_test(
61 All the image fixtures for the test is under tests/images.
62 For huggingface runner, we provide the PIL images as input.
63 For vllm runner, we provide MultiModalDataDict objects
64 - and corresponding vision language config as input.
64 + and corresponding MultiModalConfig as input.
65 Note, the text input is also adjusted to abide by vllm contract.
66 The text output is sanitized to be able to compare with hf.
67 """
@@ -176,7 +176,7 @@ def run_multi_image_test(
176 All the image fixtures for the test is under tests/images.
177 For huggingface runner, we provide the PIL images as input.
178 For vllm runner, we provide MultiModalDataDict objects
179 - and corresponding vision language config as input.
179 + and corresponding MultiModalConfig as input.
180 Note, the text input is also adjusted to abide by vllm contract.
181 The text output is sanitized to be able to compare with hf.
182 """
@@ -197,6 +197,7 @@ def run_multi_image_test(
197 with vllm_runner(model,
198 max_model_len=4096,
199 max_num_seqs=1,
200 + limit_mm_per_prompt={"image": len(images)},
201 dtype=dtype,
202 tensor_parallel_size=tensor_parallel_size,
203 distributed_executor_backend=distributed_executor_backend,
tests/models/test_paligemma.py
+1 -1
@@ -72,7 +72,7 @@ def run_test(
72 All the image fixtures for the test is under tests/images.
73 For huggingface runner, we provide the PIL images as input.
74 For vllm runner, we provide MultiModalDataDict objects
75 - and corresponding vision language config as input.
75 + and corresponding MultiModalConfig as input.
76 Note, the text input is also adjusted to abide by vllm contract.
77 The text output is sanitized to be able to compare with hf.
78 """
tests/models/test_phi3v.py
+1 -1
@@ -73,7 +73,7 @@ def run_test(
73 All the image fixtures for the test is under tests/images.
74 For huggingface runner, we provide the PIL images as input.
75 For vllm runner, we provide MultiModalDataDict objects
76 - and corresponding vision language config as input.
76 + and corresponding MultiModalConfig as input.
77 Note, the text input is also adjusted to abide by vllm contract.
78 The text output is sanitized to be able to compare with hf.
79 """
tests/multimodal/test_mapper.py
+78 -6
@@ -1,15 +1,22 @@
1 +from contextlib import nullcontext
2 +
3 import numpy as np
4 import pytest
5 from transformers import CLIPImageProcessor, LlavaNextImageProcessor
6
5 -from vllm.config import ModelConfig
6 -from vllm.multimodal import MULTIMODAL_REGISTRY
7 +from vllm.config import ModelConfig, MultiModalConfig
8 +from vllm.multimodal import MultiModalRegistry
9 from vllm.multimodal.utils import rescale_image_size
10
11
12 +@pytest.fixture
13 +def mm_registry():
14 + return MultiModalRegistry()
15 +
16 +
17 @pytest.mark.parametrize("dtype", ["half", "float"])
18 @pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
12 -def test_clip_image_processor(image_assets, dtype, size_factor):
19 +def test_clip_image_processor(image_assets, mm_registry, dtype, size_factor):
20 MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
21
22 hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
@@ -24,6 +31,9 @@ def test_clip_image_processor(image_assets, dtype, size_factor):
31 dtype=dtype,
32 revision=None,
33 )
34 + mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
35 +
36 + mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
37
38 for asset in image_assets:
39 image = rescale_image_size(asset.pil_image, size_factor)
@@ -32,7 +42,7 @@ def test_clip_image_processor(image_assets, dtype, size_factor):
42 image,
43 return_tensors="pt",
44 )
35 - vllm_result = MULTIMODAL_REGISTRY.map_input(
45 + vllm_result = mm_registry.map_input(
46 model_config,
47 {"image": image},
48 )
@@ -48,7 +58,8 @@ def test_clip_image_processor(image_assets, dtype, size_factor):
58
59 @pytest.mark.parametrize("dtype", ["half", "float"])
60 @pytest.mark.parametrize("size_factor", [0.25, 0.5, 1.0])
51 -def test_llava_next_image_processor(image_assets, dtype, size_factor):
61 +def test_llava_next_image_processor(image_assets, mm_registry, dtype,
62 + size_factor):
63 MODEL_NAME = "llava-hf/llava-v1.6-vicuna-7b-hf"
64
65 hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
@@ -63,6 +74,9 @@ def test_llava_next_image_processor(image_assets, dtype, size_factor):
74 dtype=dtype,
75 revision=None,
76 )
77 + mm_config = MultiModalConfig(limit_per_prompt={"image": 1})
78 +
79 + mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
80
81 for asset in image_assets:
82 image = rescale_image_size(asset.pil_image, size_factor)
@@ -71,7 +85,7 @@ def test_llava_next_image_processor(image_assets, dtype, size_factor):
85 image,
86 return_tensors="pt",
87 )
74 - vllm_result = MULTIMODAL_REGISTRY.map_input(
88 + vllm_result = mm_registry.map_input(
89 model_config,
90 {"image": image},
91 )
@@ -83,3 +97,61 @@ def test_llava_next_image_processor(image_assets, dtype, size_factor):
97
98 assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
99 assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
100 +
101 +
102 +@pytest.mark.parametrize(
103 + ("num_images", "limit", "is_valid"),
104 + [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
105 + (2, 1, False), (2, 2, True)],
106 +)
107 +def test_mm_limits(image_assets, mm_registry, num_images, limit, is_valid):
108 + MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
109 +
110 + model_config = ModelConfig(
111 + model=MODEL_NAME,
112 + tokenizer=MODEL_NAME,
113 + tokenizer_mode="auto",
114 + trust_remote_code=False,
115 + seed=0,
116 + dtype="half",
117 + revision=None,
118 + )
119 + mm_config = MultiModalConfig(limit_per_prompt={"image": limit})
120 +
121 + mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
122 +
123 + image = image_assets[0].pil_image
124 + if num_images == 0:
125 + mm_inputs = {}
126 + elif num_images == 1:
127 + mm_inputs = {"image": image}
128 + else:
129 + mm_inputs = {"image": [image] * num_images}
130 +
131 + with nullcontext() if is_valid else pytest.raises(ValueError):
132 + mm_registry.map_input(model_config, mm_inputs)
133 +
134 +
135 +# NOTE: We don't test zero images since the HF processor doesn't support it
136 +@pytest.mark.parametrize("num_images", [1, 2])
137 +def test_image_mapper_multi(image_assets, mm_registry, num_images):
138 + MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
139 +
140 + model_config = ModelConfig(
141 + model=MODEL_NAME,
142 + tokenizer=MODEL_NAME,
143 + tokenizer_mode="auto",
144 + trust_remote_code=False,
145 + seed=0,
146 + dtype="half",
147 + revision=None,
148 + )
149 + mm_config = MultiModalConfig(limit_per_prompt={"image": num_images})
150 +
151 + mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
152 +
153 + image = image_assets[0].pil_image
154 + mm_inputs = {"image": [image] * num_images}
155 +
156 + mapped_inputs = mm_registry.map_input(model_config, mm_inputs)
157 + assert len(mapped_inputs["pixel_values"]) == num_images
vllm/config.py
+10 -4
@@ -1,7 +1,8 @@
1 import enum
2 import json
3 from dataclasses import dataclass, field, fields
4 -from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Type, Union
4 +from typing import (TYPE_CHECKING, ClassVar, List, Mapping, Optional, Tuple,
5 + Type, Union)
6
7 import torch
8 from transformers import PretrainedConfig
@@ -1429,10 +1430,15 @@ class PromptAdapterConfig:
1430
1431 @dataclass
1432 class MultiModalConfig:
1432 - """Configs the input data format and how models should run for
1433 - multimodal models."""
1433 + """Controls the behavior of multimodal models."""
1434 +
1435 + limit_per_prompt: Mapping[str, int]
1436 + """
1437 + The maximum number of multi-modal input instances allowed per prompt
1438 + for each :class:`~vllm.multimodal.MultiModalPlugin`.
1439 + """
1440 +
1441 # TODO: Add configs to init vision tower or not.
1435 - pass
1442
1443
1444 _STR_DTYPE_TO_TORCH_DTYPE = {
vllm/engine/arg_utils.py
+43 -5
@@ -2,7 +2,8 @@ import argparse
2 import dataclasses
3 import json
4 from dataclasses import dataclass
5 -from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union
5 +from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Type,
6 + Union)
7
8 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
9 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -15,8 +16,7 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 from vllm.utils import FlexibleArgumentParser
17
18 if TYPE_CHECKING:
18 - from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
19 - BaseTokenizerGroup)
19 + from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
20
21 logger = init_logger(__name__)
22
@@ -29,11 +29,32 @@ def nullable_str(val: str):
29 return val
30
31
32 +def nullable_kvs(val: str) -> Optional[Mapping[str, int]]:
33 + if len(val) == 0:
34 + return None
35 +
36 + out_dict: Dict[str, int] = {}
37 + for item in val.split(","):
38 + try:
39 + key, value = item.split("=")
40 + except TypeError as exc:
41 + msg = "Each item should be in the form KEY=VALUE"
42 + raise ValueError(msg) from exc
43 +
44 + try:
45 + out_dict[key] = int(value)
46 + except ValueError as exc:
47 + msg = f"Failed to parse value of item {key}={value}"
48 + raise ValueError(msg) from exc
49 +
50 + return out_dict
51 +
52 +
53 @dataclass
54 class EngineArgs:
55 """Arguments for vLLM engine."""
56 model: str = 'facebook/opt-125m'
36 - served_model_name: Optional[Union[List[str]]] = None
57 + served_model_name: Optional[Union[str, List[str]]] = None
58 tokenizer: Optional[str] = None
59 skip_tokenizer_init: bool = False
60 tokenizer_mode: str = 'auto'
@@ -81,6 +102,7 @@ class EngineArgs:
102 # notice.
103 tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]] = "ray"
104 tokenizer_pool_extra_config: Optional[dict] = None
105 + limit_mm_per_prompt: Optional[Mapping[str, int]] = None
106 enable_lora: bool = False
107 max_loras: int = 1
108 max_lora_rank: int = 16
@@ -435,6 +457,21 @@ class EngineArgs:
457 'This should be a JSON string that will be '
458 'parsed into a dictionary. Ignored if '
459 'tokenizer_pool_size is 0.')
460 +
461 + # Multimodal related configs
462 + parser.add_argument(
463 + '--limit-mm-per-prompt',
464 + type=nullable_kvs,
465 + default=EngineArgs.limit_mm_per_prompt,
466 + # The default value is given in
467 + # MultiModalRegistry.init_mm_limits_per_prompt
468 + help=('For each multimodal plugin, limit how many '
469 + 'input instances to allow for each prompt. '
470 + 'Expects a comma-separated list of items, '
471 + 'e.g.: `image=16,video=2` allows a maximum of 16 '
472 + 'images and 2 videos per prompt. Defaults to 1 for '
473 + 'each modality.'))
474 +
475 # LoRA related configs
476 parser.add_argument('--enable-lora',
477 action='store_true',
@@ -709,7 +746,8 @@ class EngineArgs:
746 "CPU offload space must be non-negative"
747 f", but got {self.cpu_offload_gb}")
748
712 - multimodal_config = MultiModalConfig()
749 + multimodal_config = MultiModalConfig(
750 + limit_per_prompt=self.limit_mm_per_prompt or {})
751
752 device_config = DeviceConfig(device=self.device)
753 model_config = ModelConfig(
vllm/engine/llm_engine.py
+7 -4
@@ -24,8 +24,9 @@ from vllm.engine.output_processor.stop_checker import StopChecker
24 from vllm.engine.output_processor.util import create_output_by_sequence_group
25 from vllm.executor.executor_base import ExecutorBase
26 from vllm.executor.ray_utils import initialize_ray_cluster
27 -from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs, LLMInputs,
28 - PromptInputs, SingletonPromptInputs)
27 +from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
28 + InputRegistry, LLMInputs, PromptInputs,
29 + SingletonPromptInputs)
30 from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
31 from vllm.logger import init_logger
32 from vllm.lora.request import LoRARequest
@@ -180,6 +181,7 @@ class LLMEngine:
181 log_stats: bool,
182 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
183 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
184 + input_registry: InputRegistry = INPUT_REGISTRY,
185 ) -> None:
186 logger.info(
187 "Initializing an LLM engine (v%s) with config: "
@@ -265,8 +267,9 @@ class LLMEngine:
267 self.generation_config_fields = _load_generation_config_dict(
268 model_config)
269
268 - self.input_processor = INPUT_REGISTRY.create_input_processor(
269 - self.model_config)
270 + self.input_registry = input_registry
271 + self.input_processor = input_registry.create_input_processor(
272 + model_config)
273
274 self.model_executor = executor_class(
275 model_config=model_config,
vllm/inputs/registry.py
+69 -14
@@ -1,6 +1,8 @@
1 import functools
2 +from collections import UserDict
3 from dataclasses import dataclass
3 -from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type
4 +from typing import (TYPE_CHECKING, Callable, Dict, Mapping, Optional, Protocol,
5 + Tuple, Type)
6
7 from torch import nn
8 from transformers import PretrainedConfig
@@ -12,7 +14,7 @@ from .data import LLMInputs
14
15 if TYPE_CHECKING:
16 from vllm.config import ModelConfig, MultiModalConfig
15 - from vllm.multimodal import MultiModalDataDict
17 + from vllm.multimodal import MultiModalDataDict, MultiModalRegistry
18 from vllm.sequence import SequenceData
19
20 logger = init_logger(__name__)
@@ -65,15 +67,38 @@ class InputContext:
67
68 N = TypeVar("N", bound=Type[nn.Module])
69
68 -DummyDataFactory = Callable[[InputContext, int],
69 - Tuple["SequenceData",
70 - Optional["MultiModalDataDict"]]]
71 -"""
72 -Create dummy data to be inputted into the model.
70
74 -Note:
75 - :data:`InputProcessor` is not applied to the dummy data.
76 -"""
71 +class DummyDataFactory(Protocol):
72 +
73 + def __call__(
74 + self,
75 + ctx: InputContext,
76 + seq_len: int,
77 + mm_counts: Mapping[str, int],
78 + ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
79 + """
80 + Create dummy data to be inputted into the model.
81 +
82 + Note:
83 + :data:`InputProcessor` is not applied to the dummy data.
84 + """
85 + ...
86 +
87 +
88 +class _MultiModalCounts(UserDict):
89 + """
90 + Wraps `mm_counts` for a more informative error message
91 + when attempting to access a plugin that does not exist.
92 + """
93 +
94 + def __getitem__(self, key: str) -> int:
95 + try:
96 + return super().__getitem__(key)
97 + except KeyError as exc:
98 + msg = (f"There is no multi-modal plugin with the key: {key}. "
99 + f"Available keys: {set(self.keys())}")
100 + raise KeyError(msg) from exc
101 +
102
103 InputProcessor = Callable[[InputContext, LLMInputs], LLMInputs]
104 """Preprocess the inputs to the model."""
@@ -95,6 +120,7 @@ class InputRegistry:
120 self,
121 ctx: InputContext,
122 seq_len: int,
123 + mm_counts: Mapping[str, int],
124 ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
125 """
126 The default dummy data factory represents the longest possible text
@@ -133,8 +159,12 @@ class InputRegistry:
159
160 return wrapper
161
136 - def dummy_data_for_profiling(self, model_config: "ModelConfig",
137 - seq_len: int):
162 + def dummy_data_for_profiling(
163 + self,
164 + model_config: "ModelConfig",
165 + seq_len: int,
166 + mm_registry: "MultiModalRegistry",
167 + ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
168 """
169 Create dummy data for profiling the memory usage of a model.
170
@@ -142,6 +172,10 @@ class InputRegistry:
172
173 See also:
174 :ref:`enabling_multimodal_inputs`
175 +
176 + Note:
177 + This should be called after
178 + :meth:`~MultiModalRegistry.init_mm_limits_per_prompt`.
179 """
180 # Avoid circular import
181 from vllm.model_executor.model_loader import get_model_architecture
@@ -149,8 +183,29 @@ class InputRegistry:
183 model_cls, _ = get_model_architecture(model_config)
184 dummy_factory = self._dummy_factories_by_model_type \
185 .get(model_cls, self._default_dummy_data_factory)
152 -
153 - return dummy_factory(InputContext(model_config), seq_len)
186 + mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
187 +
188 + seq_data, mm_data = dummy_factory(
189 + InputContext(model_config),
190 + seq_len,
191 + _MultiModalCounts(mm_counts),
192 + )
193 +
194 + # Having more tokens is over-conservative but otherwise fine
195 + num_tokens = seq_data.prompt_token_ids
196 + assert len(num_tokens) >= seq_len, (
197 + f"Expected at least {seq_len} dummy tokens for profiling, "
198 + f"but found {len(num_tokens)} tokens instead.")
199 +
200 + if mm_data is not None:
201 + for k, v in mm_data.items():
202 + num_items = len(v) if isinstance(v, list) else 1
203 + num_expected = mm_counts[k]
204 + assert num_items >= num_expected, (
205 + f"Expected at least {num_expected} dummy '{k}' instances "
206 + f"for profiling, but found {num_items} instances instead.")
207 +
208 + return seq_data, mm_data
209
210 def _default_input_processor(self, ctx: InputContext,
211 inputs: LLMInputs) -> LLMInputs:
vllm/model_executor/model_loader/loader.py
+1 -1
@@ -133,7 +133,7 @@ def _get_model_initialization_kwargs(
133
134 if supports_multimodal(model_class):
135 if multimodal_config is None:
136 - raise ValueError("Provide vision related configurations "
136 + raise ValueError("Provide multi-modal related configurations "
137 "through LLM entrypoint or engine arguments.")
138
139 extra_kwargs["multimodal_config"] = multimodal_config
vllm/model_executor/models/blip.py
+4 -3
@@ -31,13 +31,13 @@ def get_blip_num_patches(*, image_size: int, patch_size: int) -> int:
31
32
33 def get_blip_image_feature_size(
34 - hf_config: Union[BlipVisionConfig, Blip2VisionConfig], ) -> int:
34 + hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int:
35 return get_blip_num_patches(image_size=hf_config.image_size,
36 patch_size=hf_config.patch_size)
37
38
39 def get_max_blip_image_tokens(
40 - hf_config: Union[BlipVisionConfig, Blip2VisionConfig], ) -> int:
40 + hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int:
41 return get_blip_image_feature_size(hf_config)
42
43
@@ -60,6 +60,7 @@ def dummy_seq_data_for_blip(
60
61 def dummy_image_for_blip(
62 hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
63 + num_images: int,
64 *,
65 image_width_override: Optional[int] = None,
66 image_height_override: Optional[int] = None,
@@ -71,7 +72,7 @@ def dummy_image_for_blip(
72 height = image_height_override
73
74 image = Image.new("RGB", (width, height), color=0)
74 - return {"image": image}
75 + return {"image": image if num_images == 1 else [image] * num_images}
76
77
78 def input_processor_for_blip(
vllm/model_executor/models/blip2.py
+30 -7
@@ -1,4 +1,5 @@
1 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
1 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2 + TypedDict, Union)
3
4 import torch
5 import torch.nn as nn
@@ -413,17 +414,39 @@ def get_max_blip2_image_tokens(ctx: InputContext):
414 raise NotImplementedError(msg)
415
416
416 -def dummy_data_for_blip2(ctx: InputContext, seq_len: int):
417 +def dummy_seq_data_for_blip2(
418 + hf_config: Blip2Config,
419 + seq_len: int,
420 + num_images: int,
421 + *,
422 + image_token_id: int,
423 + image_feature_size_override: Optional[int] = None,
424 +):
425 + if image_feature_size_override is None:
426 + image_feature_size = get_blip2_image_feature_size(hf_config)
427 + else:
428 + image_feature_size = image_feature_size_override
429 +
430 + token_ids = [image_token_id] * image_feature_size * num_images
431 + token_ids += [0] * (seq_len - image_feature_size * num_images)
432 + return SequenceData(token_ids)
433 +
434 +
435 +def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
436 + mm_counts: Mapping[str, int]):
437 hf_config = ctx.get_hf_config(Blip2Config)
438 vision_config = hf_config.vision_config
439 + num_images = mm_counts["image"]
440
420 - image_feature_size = get_blip2_image_feature_size(hf_config)
421 - token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
422 - token_ids += [0] * (seq_len - image_feature_size)
423 - seq_data = SequenceData(token_ids)
441 + seq_data = dummy_seq_data_for_blip2(
442 + hf_config,
443 + seq_len,
444 + num_images,
445 + image_token_id=BLIP2_IMAGE_TOKEN_ID,
446 + )
447
448 if isinstance(vision_config, Blip2VisionConfig):
426 - mm_data = dummy_image_for_blip(vision_config)
449 + mm_data = dummy_image_for_blip(vision_config, num_images)
450
451 return seq_data, mm_data
452
vllm/model_executor/models/chameleon.py
+14 -9
@@ -1,6 +1,6 @@
1 from functools import cached_property
2 -from typing import (Any, Dict, Iterable, List, Literal, Optional, Tuple,
3 - TypedDict)
2 +from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
3 + Tuple, TypedDict)
4
5 import torch
6 import torch.nn.functional as F
@@ -19,8 +19,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
19 QKVParallelLinear,
20 RowParallelLinear)
21 from vllm.model_executor.layers.logits_processor import LogitsProcessor
22 -from vllm.model_executor.layers.quantization.base_config import (
23 - QuantizationConfig)
22 +from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.rotary_embedding import get_rope
24 from vllm.model_executor.layers.sampler import Sampler
25 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -61,6 +60,7 @@ def get_max_chameleon_image_tokens(ctx: InputContext):
60
61 def dummy_seq_data_for_chameleon(
62 seq_len: int,
63 + num_images: int,
64 *,
65 image_token_id: int,
66 image_feature_size_override: Optional[int] = None,
@@ -70,12 +70,14 @@ def dummy_seq_data_for_chameleon(
70 else:
71 image_feature_size = image_feature_size_override
72
73 - token_ids = [image_token_id] * image_feature_size
74 - token_ids += [0] * (seq_len - image_feature_size)
73 + token_ids = [image_token_id] * image_feature_size * num_images
74 + token_ids += [0] * (seq_len - image_feature_size * num_images)
75 return SequenceData(token_ids)
76
77
78 def dummy_image_for_chameleon(
79 + num_images: int,
80 + *,
81 image_width_override: Optional[int] = None,
82 image_height_override: Optional[int] = None,
83 ):
@@ -87,17 +89,20 @@ def dummy_image_for_chameleon(
89 height = image_height_override
90
91 image = Image.new("RGB", (width, height), color=0)
90 - return {"image": image}
92 + return {"image": image if num_images == 1 else [image] * num_images}
93
94
93 -def dummy_data_for_chameleon(ctx: InputContext, seq_len: int):
95 +def dummy_data_for_chameleon(ctx: InputContext, seq_len: int,
96 + mm_counts: Mapping[str, int]):
97 + num_images = mm_counts["image"]
98
99 seq_data = dummy_seq_data_for_chameleon(
100 seq_len,
101 + num_images,
102 image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
103 )
104
100 - mm_data = dummy_image_for_chameleon()
105 + mm_data = dummy_image_for_chameleon(num_images)
106 return seq_data, mm_data
107
108
vllm/model_executor/models/clip.py
+5 -3
@@ -43,6 +43,7 @@ def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
43 def dummy_seq_data_for_clip(
44 hf_config: CLIPVisionConfig,
45 seq_len: int,
46 + num_images: int,
47 *,
48 image_token_id: int,
49 image_feature_size_override: Optional[int] = None,
@@ -52,13 +53,14 @@ def dummy_seq_data_for_clip(
53 else:
54 image_feature_size = image_feature_size_override
55
55 - token_ids = [image_token_id] * image_feature_size
56 - token_ids += [0] * (seq_len - image_feature_size)
56 + token_ids = [image_token_id] * image_feature_size * num_images
57 + token_ids += [0] * (seq_len - image_feature_size * num_images)
58 return SequenceData(token_ids)
59
60
61 def dummy_image_for_clip(
62 hf_config: CLIPVisionConfig,
63 + num_images: int,
64 *,
65 image_width_override: Optional[int] = None,
66 image_height_override: Optional[int] = None,
@@ -70,7 +72,7 @@ def dummy_image_for_clip(
72 height = image_height_override
73
74 image = Image.new("RGB", (width, height), color=0)
73 - return {"image": image}
75 + return {"image": image if num_images == 1 else [image] * num_images}
76
77
78 def input_processor_for_clip(
vllm/model_executor/models/fuyu.py
+16 -11
@@ -16,7 +16,7 @@
16 # limitations under the License.
17 """ PyTorch Fuyu model."""
18 import math
19 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
19 +from typing import Iterable, List, Literal, Mapping, Optional, Tuple, TypedDict
20
21 import torch
22 import torch.nn as nn
@@ -29,8 +29,7 @@ from vllm.config import CacheConfig, MultiModalConfig
29 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30 from vllm.logger import init_logger
31 from vllm.model_executor.layers.linear import ColumnParallelLinear
32 -from vllm.model_executor.layers.quantization.base_config import (
33 - QuantizationConfig)
32 +from vllm.model_executor.layers.quantization import QuantizationConfig
33 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
35 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -94,27 +93,33 @@ def get_max_fuyu_image_tokens(ctx: InputContext):
93 return (ncol + 1) * nrow
94
95
97 -def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int):
96 +def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int):
97 ncol, nrow = get_max_fuyu_image_feature_size()
98 image_feature_size = get_max_fuyu_image_tokens(ctx)
99
101 - token_ids = ([_IMAGE_TOKEN_ID] * ncol + [_NEWLINE_TOKEN_ID]) * nrow
102 - token_ids += [0] * (seq_len - image_feature_size)
100 + image_token_ids = ([_IMAGE_TOKEN_ID] * ncol + [_NEWLINE_TOKEN_ID]) * nrow
101 + token_ids = image_token_ids * num_images
102 + token_ids += [0] * (seq_len - image_feature_size * num_images)
103 return SequenceData(token_ids)
104
105
106 def dummy_image_for_fuyu(
107 + num_images: int,
108 + *,
109 image_width: int,
110 image_height: int,
111 ):
112 image = Image.new("RGB", (image_width, image_height), color=0)
111 - return {"image": image}
113 + return {"image": image if num_images == 1 else [image] * num_images}
114
115
114 -def dummy_data_for_fuyu(ctx: InputContext, seq_len: int):
115 - seq_data = dummy_seq_data_for_fuyu(ctx, seq_len)
116 - mm_data = dummy_image_for_fuyu(MAX_IMAGE_FEATURE_SIZE_WIDTH,
117 - MAX_IMAGE_FEATURE_SIZE_HEIGHT)
116 +def dummy_data_for_fuyu(ctx: InputContext, seq_len: int,
117 + mm_counts: Mapping[str, int]):
118 + num_images = mm_counts["image"]
119 + seq_data = dummy_seq_data_for_fuyu(ctx, seq_len, num_images)
120 + mm_data = dummy_image_for_fuyu(num_images,
121 + image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
122 + image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT)
123 return seq_data, mm_data
124
125
vllm/model_executor/models/interfaces.py
+2 -5
@@ -11,14 +11,11 @@ logger = init_logger(__name__)
11
12 @runtime_checkable
13 class SupportsMultiModal(Protocol):
14 - """
15 - The interface required for all multimodal (vision or audio) language
16 - models.
17 - """
14 + """The interface required for all multi-modal models."""
15
16 supports_multimodal: ClassVar[Literal[True]] = True
17 """
21 - A flag that indicates this model supports multimodal inputs.
18 + A flag that indicates this model supports multi-modal inputs.
19
20 Note:
21 There is no need to redefine this flag if this class is in the
vllm/model_executor/models/internvl.py
+8 -3
@@ -5,7 +5,8 @@
5 # Licensed under The MIT License [see LICENSE for details]
6 # --------------------------------------------------------
7 import itertools
8 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
8 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
9 + TypedDict, Union)
10
11 import torch
12 import torch.nn as nn
@@ -230,7 +231,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
231
232
233 def input_mapper_for_internvl(ctx: InputContext, data: object):
233 - hf_config = ctx.get_hf_config(PretrainedConfig)
234 + hf_config = ctx.get_hf_config()
235
236 use_thumbnail = hf_config.use_thumbnail
237 min_num = hf_config.min_dynamic_patch
@@ -256,7 +257,9 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
257 })
258
259
259 -def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
260 +def dummy_data_for_internvl(ctx: InputContext, seq_len: int,
261 + mm_counts: Mapping[str, int]):
262 + num_images = mm_counts["image"]
263
264 image_feature_size = get_max_internvl_image_tokens(ctx)
265 model_config = ctx.model_config
@@ -268,6 +271,7 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
271 seq_data = dummy_seq_data_for_clip(
272 vision_config,
273 seq_len,
274 + num_images,
275 image_token_id=tokenizer.encode(IMG_CONTEXT,
276 add_special_tokens=False)[0],
277 image_feature_size_override=image_feature_size,
@@ -281,6 +285,7 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
285
286 mm_data = dummy_image_for_clip(
287 vision_config,
288 + num_images,
289 image_width_override=max_image_width,
290 image_height_override=max_image_height,
291 )
vllm/model_executor/models/llava.py
+10 -6
@@ -1,5 +1,6 @@
1 import itertools
2 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 + TypedDict, Union)
4
5 import torch
6 import torch.nn as nn
@@ -9,8 +10,7 @@ from vllm.attention import AttentionMetadata
10 from vllm.config import CacheConfig, MultiModalConfig
11 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12 from vllm.model_executor.layers.activation import get_act_fn
12 -from vllm.model_executor.layers.quantization.base_config import (
13 - QuantizationConfig)
13 +from vllm.model_executor.layers.quantization import QuantizationConfig
14 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
15 from vllm.model_executor.sampling_metadata import SamplingMetadata
16 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -88,9 +88,11 @@ def get_max_llava_image_tokens(ctx: InputContext):
88 raise ValueError(f"Unexpected select feature strategy: {strategy}")
89
90
91 -def dummy_data_for_llava(ctx: InputContext, seq_len: int):
91 +def dummy_data_for_llava(ctx: InputContext, seq_len: int,
92 + mm_counts: Mapping[str, int]):
93 hf_config = ctx.get_hf_config(LlavaConfig)
94 vision_config = hf_config.vision_config
95 + num_images = mm_counts["image"]
96
97 image_feature_size = get_max_llava_image_tokens(ctx)
98
@@ -98,21 +100,23 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int):
100 seq_data = dummy_seq_data_for_clip(
101 vision_config,
102 seq_len,
103 + num_images,
104 image_token_id=hf_config.image_token_index,
105 image_feature_size_override=image_feature_size,
106 )
107
105 - mm_data = dummy_image_for_clip(vision_config)
108 + mm_data = dummy_image_for_clip(vision_config, num_images)
109 return seq_data, mm_data
110 elif isinstance(vision_config, SiglipVisionConfig):
111 seq_data = dummy_seq_data_for_siglip(
112 vision_config,
113 seq_len,
114 + num_images,
115 image_token_id=hf_config.image_token_index,
116 image_feature_size_override=image_feature_size,
117 )
118
115 - mm_data = dummy_image_for_siglip(vision_config)
119 + mm_data = dummy_image_for_siglip(vision_config, num_images)
120 return seq_data, mm_data
121
122 msg = f"Unsupported vision config: {type(vision_config)}"
vllm/model_executor/models/llava_next.py
+10 -4
@@ -1,5 +1,6 @@
1 import itertools
2 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 + TypedDict, Union)
4
5 import torch
6 import torch.nn as nn
@@ -13,8 +14,7 @@ from vllm.attention import AttentionMetadata
14 from vllm.config import CacheConfig, MultiModalConfig
15 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
16 from vllm.logger import init_logger
16 -from vllm.model_executor.layers.quantization.base_config import (
17 - QuantizationConfig)
17 +from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -158,9 +158,11 @@ def get_max_llava_next_image_tokens(ctx: InputContext):
158 )
159
160
161 -def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
161 +def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
162 + mm_counts: Mapping[str, int]):
163 hf_config = ctx.get_hf_config(LlavaNextConfig)
164 vision_config = hf_config.vision_config
165 + num_images = mm_counts["image"]
166
167 image_feature_size = get_max_llava_next_image_tokens(ctx)
168
@@ -168,12 +170,14 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
170 seq_data = dummy_seq_data_for_clip(
171 vision_config,
172 seq_len,
173 + num_images,
174 image_token_id=hf_config.image_token_index,
175 image_feature_size_override=image_feature_size,
176 )
177
178 mm_data = dummy_image_for_clip(
179 vision_config,
180 + num_images,
181 image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
182 image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
183 )
@@ -183,12 +187,14 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
187 seq_data = dummy_seq_data_for_siglip(
188 vision_config,
189 seq_len,
190 + num_images,
191 image_token_id=hf_config.image_token_index,
192 image_feature_size_override=image_feature_size,
193 )
194
195 mm_data = dummy_image_for_siglip(
196 vision_config,
197 + num_images,
198 image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
199 image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
200 )
vllm/model_executor/models/minicpmv.py
+11 -10
@@ -24,8 +24,8 @@
24 import math
25 import re
26 from functools import partial
27 -from typing import (Any, Callable, Iterable, List, Optional, Tuple, TypedDict,
28 - Union)
27 +from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
28 + TypedDict, Union)
29
30 import numpy as np
31 import torch
@@ -42,8 +42,7 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
42 from vllm.logger import init_logger
43 from vllm.model_executor.layers.linear import ReplicatedLinear
44 from vllm.model_executor.layers.logits_processor import LogitsProcessor
45 -from vllm.model_executor.layers.quantization.base_config import (
46 - QuantizationConfig)
45 +from vllm.model_executor.layers.quantization import QuantizationConfig
46 from vllm.model_executor.layers.sampler import Sampler
47 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
48 from vllm.model_executor.model_loader.utils import set_default_torch_dtype
@@ -408,22 +407,24 @@ def get_max_minicpmv_image_tokens(ctx: InputContext):
407 return getattr(hf_config, "query_num", 64)
408
409
411 -def dummy_seq_data_for_minicpmv(seq_len: int):
410 +def dummy_seq_data_for_minicpmv(seq_len: int, num_images: int):
411 token_ids = [0] * seq_len
412 return SequenceData(token_ids)
413
414
416 -def dummy_image_for_minicpmv(hf_config: PretrainedConfig):
415 +def dummy_image_for_minicpmv(hf_config: PretrainedConfig, num_images: int):
416 width = height = hf_config.image_size
417 image = Image.new("RGB", (width, height), color=0)
419 - return {"image": image}
418 + return {"image": image if num_images == 1 else [image] * num_images}
419
420
422 -def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int):
421 +def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
422 + mm_counts: Mapping[str, int]):
423 hf_config = ctx.get_hf_config()
424 + num_images = mm_counts["image"]
425
425 - seq_data = dummy_seq_data_for_minicpmv(seq_len)
426 - mm_data = dummy_image_for_minicpmv(hf_config)
426 + seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images)
427 + mm_data = dummy_image_for_minicpmv(hf_config, num_images)
428
429 return seq_data, mm_data
430
vllm/model_executor/models/paligemma.py
+8 -5
@@ -1,4 +1,5 @@
1 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
1 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
2 + TypedDict, Union)
3
4 import torch
5 from torch import nn
@@ -9,8 +10,7 @@ from vllm.config import CacheConfig, MultiModalConfig
10 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11 from vllm.logger import init_logger
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
12 -from vllm.model_executor.layers.quantization.base_config import (
13 - QuantizationConfig)
13 +from vllm.model_executor.layers.quantization import QuantizationConfig
14 from vllm.model_executor.layers.sampler import Sampler
15 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 from vllm.model_executor.models.gemma import GemmaModel
@@ -57,17 +57,20 @@ def get_max_paligemma_image_tokens(ctx: InputContext):
57 return get_max_siglip_image_tokens(vision_config)
58
59
60 -def dummy_data_for_paligemma(ctx: InputContext, seq_len: int):
60 +def dummy_data_for_paligemma(ctx: InputContext, seq_len: int,
61 + mm_counts: Mapping[str, int]):
62 hf_config = ctx.get_hf_config(PaliGemmaConfig)
63 vision_config = hf_config.vision_config
64 + num_images = mm_counts["image"]
65
66 seq_data = dummy_seq_data_for_siglip(
67 vision_config,
68 seq_len,
69 + num_images,
70 image_token_id=hf_config.image_token_index,
71 )
72
70 - mm_data = dummy_image_for_siglip(vision_config)
73 + mm_data = dummy_image_for_siglip(vision_config, num_images)
74 return seq_data, mm_data
75
76
vllm/model_executor/models/phi3v.py
+8 -4
@@ -15,7 +15,8 @@
15 # limitations under the License.
16 import re
17 from functools import lru_cache
18 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
18 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
19 + TypedDict, Union)
20
21 import numpy as np
22 import torch
@@ -28,8 +29,7 @@ from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
29 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
30 from vllm.logger import init_logger
31 from vllm.model_executor.layers.logits_processor import LogitsProcessor
31 -from vllm.model_executor.layers.quantization.base_config import (
32 - QuantizationConfig)
32 +from vllm.model_executor.layers.quantization import QuantizationConfig
33 from vllm.model_executor.layers.sampler import Sampler
34 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
35 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -347,18 +347,22 @@ def get_max_phi3v_image_tokens(ctx: InputContext):
347 )
348
349
350 -def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
350 +def dummy_data_for_phi3v(ctx: InputContext, seq_len: int,
351 + mm_counts: Mapping[str, int]):
352 + num_images = mm_counts["image"]
353
354 image_feature_size = get_max_phi3v_image_tokens(ctx)
355
356 seq_data = dummy_seq_data_for_clip(
357 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
358 seq_len,
359 + num_images,
360 image_token_id=_IMAGE_TOKEN_ID,
361 image_feature_size_override=image_feature_size,
362 )
363 mm_data = dummy_image_for_clip(
364 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
365 + num_images,
366 image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
367 image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
368 )
vllm/model_executor/models/siglip.py
+5 -3
@@ -52,6 +52,7 @@ def get_max_siglip_image_tokens(hf_config: SiglipVisionConfig) -> int:
52 def dummy_seq_data_for_siglip(
53 hf_config: SiglipVisionConfig,
54 seq_len: int,
55 + num_images: int,
56 *,
57 image_token_id: int,
58 image_feature_size_override: Optional[int] = None,
@@ -61,13 +62,14 @@ def dummy_seq_data_for_siglip(
62 else:
63 image_feature_size = image_feature_size_override
64
64 - token_ids = [image_token_id] * image_feature_size
65 - token_ids += [0] * (seq_len - image_feature_size)
65 + token_ids = [image_token_id] * image_feature_size * num_images
66 + token_ids += [0] * (seq_len - image_feature_size * num_images)
67 return SequenceData(token_ids)
68
69
70 def dummy_image_for_siglip(
71 hf_config: SiglipVisionConfig,
72 + num_images: int,
73 *,
74 image_width_override: Optional[int] = None,
75 image_height_override: Optional[int] = None,
@@ -79,7 +81,7 @@ def dummy_image_for_siglip(
81 height = image_height_override
82
83 image = Image.new("RGB", (width, height), color=0)
82 - return {"image": image}
84 + return {"image": image if num_images == 1 else [image] * num_images}
85
86
87 def input_processor_for_siglip(
vllm/multimodal/base.py
+33 -15
@@ -1,9 +1,9 @@
1 import sys
2 from abc import ABC, abstractmethod
3 from collections import UserDict, defaultdict
4 -from typing import Any, Callable, Dict, List, Optional
4 +from typing import Callable, Dict, List, Mapping, Optional
5 from typing import Sequence as GenericSequence
6 -from typing import Tuple, Type, TypedDict, TypeVar, Union, cast
6 +from typing import Tuple, Type, TypedDict, TypeVar, Union, cast, final
7
8 import numpy as np
9 import torch
@@ -116,17 +116,30 @@ class MultiModalInputs(_MultiModalInputsBase):
116 batched_inputs)
117
118
119 +_T = TypeVar("_T")
120 +
121 +MultiModalData: TypeAlias = Union[_T, List[_T]]
122 +"""
123 +Either a single data instance, or a list of data instances.
124 +
125 +The number of data instances allowed per modality is restricted by
126 +`--limit-mm-per-prompt`.
127 +"""
128 +
129 +
130 +@final
131 class MultiModalDataBuiltins(TypedDict, total=False):
132 """Modality types that are predefined by vLLM."""
133
122 - image: Image.Image
123 - """The input image."""
134 + image: MultiModalData[Image.Image]
135 + """The input image(s)."""
136
125 - audio: Tuple[np.ndarray, Union[int, float]]
126 - """The input audio and its sampling rate."""
137 + audio: MultiModalData[Tuple[np.ndarray, Union[int, float]]]
138 + """The input audio item(s) and corresponding sampling rate(s)."""
139
140
129 -MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
141 +MultiModalDataDict = Union[MultiModalDataBuiltins,
142 + Mapping[str, MultiModalData[object]]]
143 """
144 A dictionary containing an item for each modality type to input.
145
@@ -137,7 +150,8 @@ Note:
150 Read more on that :ref:`here <adding_multimodal_plugin>`.
151 """
152
140 -MultiModalInputMapper = Callable[[InputContext, object], MultiModalInputs]
153 +MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
154 + MultiModalInputs]
155 """
156 Return a dictionary to be passed as keyword arguments to
157 :meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
@@ -181,8 +195,11 @@ class MultiModalPlugin(ABC):
195 raise NotImplementedError
196
197 @abstractmethod
184 - def _default_input_mapper(self, ctx: InputContext,
185 - data: object) -> MultiModalInputs:
198 + def _default_input_mapper(
199 + self,
200 + ctx: InputContext,
201 + data: MultiModalData[object],
202 + ) -> MultiModalInputs:
203 """
204 Return a dictionary to be passed as keyword arguments to
205 :meth:`~torch.nn.Module.forward`. This is similar in concept to
@@ -225,7 +242,7 @@ class MultiModalPlugin(ABC):
242 return wrapper
243
244 def map_input(self, model_config: ModelConfig,
228 - data: object) -> MultiModalInputs:
245 + data: MultiModalData[object]) -> MultiModalInputs:
246 """
247 Transform the data into a dictionary of model inputs using the
248 input mapper registered for that model.
@@ -254,8 +271,8 @@ class MultiModalPlugin(ABC):
271 @abstractmethod
272 def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
273 """
257 - Calculate the maximum number of multimodal tokens input to the language
258 - model. This does not include tokens that correspond to the input text.
274 + Calculate the maximum number of tokens, corresponding to a single
275 + instance of multimodal data, that are passed to the language model.
276 """
277 raise NotImplementedError
278
@@ -269,8 +286,9 @@ class MultiModalPlugin(ABC):
286 max_mm_tokens: Optional[MultiModalTokensCalc] = None,
287 ):
288 """
272 - Register the maximum number of multi-modal tokens input to the
273 - language model for a model class.
289 + Register the maximum number of tokens, corresponding to a single
290 + instance of multimodal data, that are passed to the language model
291 + for a model class.
292
293 If `None` is provided, then the default calculation is used instead.
294
vllm/multimodal/image.py
+6 -3
@@ -11,7 +11,7 @@ from vllm.transformers_utils.image_processor import get_image_processor
11 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
12 from vllm.utils import is_list_of
13
14 -from .base import MultiModalInputs, MultiModalPlugin
14 +from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
15
16 logger = init_logger(__name__)
17
@@ -110,8 +110,11 @@ class ImagePlugin(MultiModalPlugin):
110 model_config.model,
111 trust_remote_code=model_config.trust_remote_code)
112
113 - def _default_input_mapper(self, ctx: InputContext,
114 - data: object) -> MultiModalInputs:
113 + def _default_input_mapper(
114 + self,
115 + ctx: InputContext,
116 + data: MultiModalData[object],
117 + ) -> MultiModalInputs:
118 model_config = ctx.model_config
119
120 # PIL image
vllm/multimodal/registry.py
+101 -16
@@ -1,19 +1,33 @@
1 import functools
2 -from typing import Dict, Optional, Sequence
2 +from collections import UserDict
3 +from typing import Dict, Mapping, Optional, Sequence
4
4 -import torch
5 -
6 -from vllm.config import ModelConfig
5 +from vllm.config import ModelConfig, MultiModalConfig
6 from vllm.logger import init_logger
7
8 from .audio import AudioPlugin
9 from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
11 - MultiModalPlugin, MultiModalTokensCalc)
10 + MultiModalPlugin, MultiModalTokensCalc, NestedTensors)
11 from .image import ImagePlugin
12
13 logger = init_logger(__name__)
14
15
16 +class _MultiModalLimits(UserDict):
17 + """
18 + Wraps `_limits_by_model` for a more informative error message
19 + when attempting to access a model that does not exist.
20 + """
21 +
22 + def __getitem__(self, key: ModelConfig) -> Dict[str, int]:
23 + try:
24 + return super().__getitem__(key)
25 + except KeyError as exc:
26 + msg = (f"Cannot find `mm_limits` for model={key.model}. Did you "
27 + "forget to call `init_mm_limits_per_prompt`?")
28 + raise KeyError(msg) from exc
29 +
30 +
31 class MultiModalRegistry:
32 """
33 A registry that dispatches data processing to the
@@ -28,6 +42,11 @@ class MultiModalRegistry:
42 plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
43 self._plugins = {p.get_data_key(): p for p in plugins}
44
45 + # This is used for non-multimodal models
46 + self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
47 +
48 + self._limits_by_model = _MultiModalLimits()
49 +
50 def register_plugin(self, plugin: MultiModalPlugin) -> None:
51 """
52 Register a multi-modal plugin so it can be recognized by vLLM.
@@ -86,13 +105,24 @@ class MultiModalRegistry:
105 via the input mapper registered for that model.
106
107 See :meth:`MultiModalPlugin.map_input` for more details.
108 +
109 + Note:
110 + This should be called after :meth:`init_mm_limits_per_prompt`.
111 """
90 - merged_dict: Dict[str, torch.Tensor] = {}
112 + merged_dict: Dict[str, NestedTensors] = {}
113
114 for data_key, data_value in data.items():
93 - input_dict = self._get_plugin(data_key) \
94 - .map_input(model_config, data_value)
115 + plugin = self._get_plugin(data_key)
116
117 + num_items = len(data_value) if isinstance(data_value, list) else 1
118 + max_items = self._limits_by_model[model_config][data_key]
119 + if num_items > max_items:
120 + raise ValueError(
121 + f"You set {data_key}={max_items} (or defaulted to 1) in "
122 + f"`--limit-mm-per-prompt`, but found {num_items} items "
123 + "in the same prompt.")
124 +
125 + input_dict = plugin.map_input(model_config, data_value)
126 for input_key, input_tensor in input_dict.items():
127 if input_key in merged_dict:
128 raise ValueError(f"The input mappers (keys={set(data)}) "
@@ -115,8 +145,9 @@ class MultiModalRegistry:
145 max_mm_tokens: Optional[MultiModalTokensCalc] = None,
146 ):
147 """
118 - Register the maximum number of tokens, belonging to a
119 - specific modality, input to the language model for a model class.
148 + Register the maximum number of tokens, corresponding to a single
149 + instance of multimodal data belonging to a specific modality, that are
150 + passed to the language model for a model class.
151 """
152 return self._get_plugin(data_type_key) \
153 .register_max_multimodal_tokens(max_mm_tokens)
@@ -126,8 +157,8 @@ class MultiModalRegistry:
157 max_mm_tokens: Optional[MultiModalTokensCalc] = None,
158 ):
159 """
129 - Register the maximum number of image tokens
130 - input to the language model for a model class.
160 + Register the maximum number of image tokens, corresponding to a single
161 + image, that are passed to the language model for a model class.
162 """
163 return self.register_max_multimodal_tokens("image", max_mm_tokens)
164
@@ -135,9 +166,63 @@ class MultiModalRegistry:
166 """
167 Get the maximum number of multi-modal tokens
168 for profiling the memory usage of a model.
138 -
169 +
170 See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
171 +
172 + Note:
173 + This should be called after :meth:`init_mm_limits_per_prompt`.
174 + """
175 + limits_per_plugin = self._limits_by_model[model_config]
176 +
177 + return sum((limits_per_plugin[key] *
178 + plugin.get_max_multimodal_tokens(model_config))
179 + for key, plugin in self._plugins.items())
180 +
181 + def init_mm_limits_per_prompt(
182 + self,
183 + model_config: ModelConfig,
184 + multimodal_config: Optional[MultiModalConfig],
185 + ) -> None:
186 + """
187 + Initialize the maximum number of multi-modal input instances for each
188 + modality that are allowed per prompt for a model class.
189 + """
190 + if model_config in self._limits_by_model:
191 + logger.warning(
192 + "`mm_limits` has already been set for model=%s, and will "
193 + "be overwritten by the new values.", model_config.model)
194 +
195 + if multimodal_config is None:
196 + limits_per_plugin = self._disabled_limits_per_plugin
197 + else:
198 + config_limits_per_plugin = multimodal_config.limit_per_prompt
199 +
200 + extra_keys = config_limits_per_plugin.keys() - self._plugins.keys()
201 + if extra_keys:
202 + logger.warning(
203 + "Detected extra keys in `--limit-mm-per-prompt` which "
204 + "are not registered as multi-modal plugins: %s. "
205 + "They will be ignored.", extra_keys)
206 +
207 + # NOTE: Currently the default is set to 1 for each plugin
208 + # TODO: Automatically determine the limits based on budget
209 + # once more models support multi-image inputs
210 + limits_per_plugin = {
211 + key: config_limits_per_plugin.get(key, 1)
212 + for key in self._plugins
213 + }
214 +
215 + self._limits_by_model[model_config] = limits_per_plugin
216 +
217 + def get_mm_limits_per_prompt(
218 + self,
219 + model_config: ModelConfig,
220 + ) -> Mapping[str, int]:
221 + """
222 + Get the maximum number of multi-modal input instances for each modality
223 + that are allowed per prompt for a model class.
224 +
225 + Note:
226 + This should be called after :meth:`init_mm_limits_per_prompt`.
227 """
141 - return sum(
142 - plugin.get_max_multimodal_tokens(model_config)
143 - for plugin in self._plugins.values())
228 + return self._limits_by_model[model_config]
vllm/utils.py
-28
@@ -13,7 +13,6 @@ import threading
13 import uuid
14 import warnings
15 from asyncio import FIRST_COMPLETED, ensure_future
16 -from collections import defaultdict
16 from functools import lru_cache, partial, wraps
17 from platform import uname
18 from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
@@ -760,16 +759,6 @@ class CudaMemoryProfiler:
759 gc.collect()
760
761
763 -def str_to_int_tuple(s: str) -> Tuple[int, ...]:
764 - """Convert a string to a tuple of integers."""
765 - try:
766 - return tuple(map(int, s.split(",")))
767 - except ValueError as e:
768 - raise ValueError(
769 - "String must be a series of integers separated by commas "
770 - f"(e.g., 1, 2, 3). Given input: {s}") from e
771 -
772 -
762 def make_ndarray_with_pad(
763 x: List[List[T]],
764 pad: T,
@@ -863,23 +852,6 @@ def is_list_of(
852 assert_never(check)
853
854
866 -def merge_dicts(dict1: Dict[K, List[T]],
867 - dict2: Dict[K, List[T]]) -> Dict[K, List[T]]:
868 - """Merge 2 dicts that have key -> List of items.
869 -
870 - When a key conflicts, the values in dict1 is prioritized.
871 - """
872 - merged_dict: Dict[K, List[T]] = defaultdict(list)
873 -
874 - for key, value in dict1.items():
875 - merged_dict[key].extend(value)
876 -
877 - for key, value in dict2.items():
878 - merged_dict[key].extend(value)
879 -
880 - return dict(merged_dict)
881 -
882 -
855 JSONTree = Union[Dict[str, "JSONTree[T]"], List["JSONTree[T]"],
856 Tuple["JSONTree[T]", ...], T]
857 """A nested JSON structure where the leaves need not be JSON-serializable."""
vllm/worker/enc_dec_model_runner.py
+16 -3
@@ -12,9 +12,10 @@ from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
12 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13 ModelConfig, MultiModalConfig, ObservabilityConfig,
14 ParallelConfig, PromptAdapterConfig, SchedulerConfig)
15 -from vllm.inputs import INPUT_REGISTRY
15 +from vllm.inputs import INPUT_REGISTRY, InputRegistry
16 from vllm.logger import init_logger
17 from vllm.model_executor import SamplingMetadata
18 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
19 from vllm.sampling_params import SamplingParams
20 from vllm.sequence import (IntermediateTensors, PoolerOutput, SamplerOutput,
21 SequenceGroupMetadata)
@@ -83,6 +84,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
84 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
85 multimodal_config: Optional[MultiModalConfig] = None,
86 observability_config: Optional[ObservabilityConfig] = None,
87 + input_registry: InputRegistry = INPUT_REGISTRY,
88 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
89 ):
90 '''
91 EncoderDecoderModelRunner constructor.
@@ -271,6 +274,16 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
274 seqs: List[SequenceGroupMetadata] = []
275
276 model_config = self.model_config
277 + mm_config = self.multimodal_config
278 +
279 + input_registry = self.input_registry
280 + mm_registry = self.mm_registry
281 + mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
282 +
283 + max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
284 + if max_mm_tokens > 0:
285 + raise NotImplementedError(
286 + "Multi-modal encoder-decoder models are not supported yet")
287
288 batch_size = 0
289 for group_id in range(max_num_seqs):
@@ -278,8 +291,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
291 (group_id < max_num_batched_tokens % max_num_seqs))
292 batch_size += seq_len
293
281 - seq_data, _ = INPUT_REGISTRY \
282 - .dummy_data_for_profiling(model_config, seq_len)
294 + seq_data, _ = input_registry \
295 + .dummy_data_for_profiling(model_config, seq_len, mm_registry)
296
297 # Having more tokens is over-conservative but otherwise fine
298 assert len(seq_data.prompt_token_ids) >= seq_len, (
vllm/worker/model_runner.py
+20 -17
@@ -31,7 +31,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
31 ParallelConfig, PromptAdapterConfig, SchedulerConfig)
32 from vllm.distributed import get_pp_group
33 from vllm.distributed.parallel_state import graph_capture
34 -from vllm.inputs import INPUT_REGISTRY
34 +from vllm.inputs import INPUT_REGISTRY, InputRegistry
35 from vllm.logger import init_logger
36 from vllm.lora.layers import LoRAMapping
37 from vllm.lora.request import LoRARequest
@@ -43,7 +43,7 @@ from vllm.model_executor.models.interfaces import (supports_lora,
43 supports_multimodal)
44 from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
45 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
46 - MultiModalInputs)
46 + MultiModalInputs, MultiModalRegistry)
47 from vllm.prompt_adapter.layers import PromptAdapterMapping
48 from vllm.prompt_adapter.request import PromptAdapterRequest
49 from vllm.prompt_adapter.worker_manager import (
@@ -807,6 +807,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
807 multimodal_config: Optional[MultiModalConfig] = None,
808 return_hidden_states: bool = False,
809 observability_config: Optional[ObservabilityConfig] = None,
810 + input_registry: InputRegistry = INPUT_REGISTRY,
811 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
812 ):
813 self.model_config = model_config
814 self.parallel_config = parallel_config
@@ -860,8 +862,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
862 ) if num_attn_heads else None
863
864 # Multi-modal data support
863 - self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
864 - .create_input_mapper(self.model_config)
865 + self.input_registry = input_registry
866 + self.mm_registry = mm_registry
867 + self.multi_modal_input_mapper = mm_registry \
868 + .create_input_mapper(model_config)
869
870 # Lazy initialization
871 self.model: nn.Module # Set after load_model
@@ -902,7 +906,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
906 assert supports_lora(self.model), "Model does not support LoRA"
907 assert not supports_multimodal(
908 self.model
905 - ), "To be tested: multimodal language model with LoRA settings."
909 + ), "To be tested: Multi-modal model with LoRA settings."
910
911 self.lora_manager = LRUCacheWorkerLoRAManager(
912 self.scheduler_config.max_num_seqs,
@@ -1046,17 +1050,21 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1050 # Profile memory usage with max_num_sequences sequences and the total
1051 # number of tokens equal to max_num_batched_tokens.
1052 seqs: List[SequenceGroupMetadata] = []
1049 - # Additional GPU memory may be needed for vision encoding, which needs
1050 - # to be accounted for when calculating the GPU blocks for
1053 + # Additional GPU memory may be needed for multi-modal encoding, which
1054 + # needs to be accounted for when calculating the GPU blocks for
1055 # vLLM blocker manager.
1056 # To exercise the worst scenario for GPU memory consumption,
1057 # the number of seqs (batch_size) is chosen to maximize the number
1058 # of images processed.
1059 model_config = self.model_config
1060 + mm_config = self.multimodal_config
1061
1057 - if supports_multimodal(self.model):
1058 - max_mm_tokens = MULTIMODAL_REGISTRY \
1059 - .get_max_multimodal_tokens(model_config)
1062 + input_registry = self.input_registry
1063 + mm_registry = self.mm_registry
1064 + mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
1065 +
1066 + max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
1067 + if max_mm_tokens > 0:
1068 max_num_seqs_orig = max_num_seqs
1069 max_num_seqs = min(max_num_seqs,
1070 max_num_batched_tokens // max_mm_tokens)
@@ -1074,13 +1082,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1082 (group_id < max_num_batched_tokens % max_num_seqs))
1083 batch_size += seq_len
1084
1077 - seq_data, dummy_multi_modal_data = INPUT_REGISTRY \
1078 - .dummy_data_for_profiling(model_config, seq_len)
1079 -
1080 - # Having more tokens is over-conservative but otherwise fine
1081 - assert len(seq_data.prompt_token_ids) >= seq_len, (
1082 - f"Expected at least {seq_len} dummy tokens for profiling, "
1083 - f"but got: {len(seq_data.prompt_token_ids)}")
1085 + seq_data, dummy_multi_modal_data = input_registry \
1086 + .dummy_data_for_profiling(model_config, seq_len, mm_registry)
1087
1088 seq = SequenceGroupMetadata(
1089 request_id=str(group_id),
vllm/worker/xpu_model_runner.py
+19 -17
@@ -9,12 +9,11 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 ModelConfig, MultiModalConfig, ParallelConfig,
10 PromptAdapterConfig, SchedulerConfig)
11 from vllm.distributed import broadcast_tensor_dict
12 -from vllm.inputs import INPUT_REGISTRY
12 +from vllm.inputs import INPUT_REGISTRY, InputRegistry
13 from vllm.logger import init_logger
14 from vllm.model_executor.model_loader import get_model
15 -from vllm.model_executor.models.interfaces import supports_multimodal
15 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17 - MultiModalInputs)
16 + MultiModalInputs, MultiModalRegistry)
17 from vllm.sampling_params import SamplingParams
18 from vllm.sequence import (IntermediateTensors, SamplerOutput,
19 SequenceGroupMetadata)
@@ -89,6 +88,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
88 kv_cache_dtype: Optional[str] = "auto",
89 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
90 is_driver_worker: bool = False,
91 + input_registry: InputRegistry = INPUT_REGISTRY,
92 + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
93 *args,
94 **kwargs,
95 ):
@@ -120,8 +121,10 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
121 )
122
123 # Multi-modal data support
123 - self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
124 - .create_input_mapper(self.model_config)
124 + self.input_registry = input_registry
125 + self.mm_registry = mm_registry
126 + self.multi_modal_input_mapper = mm_registry \
127 + .create_input_mapper(model_config)
128
129 # Lazy initialization.
130 self.model: nn.Module # Set after init_Model
@@ -157,17 +160,21 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
160 # Profile memory usage with max_num_sequences sequences and the total
161 # number of tokens equal to max_num_batched_tokens.
162 seqs: List[SequenceGroupMetadata] = []
160 - # Additional GPU memory may be needed for vision encoding, which needs
161 - # to be accounted for when calculating the GPU blocks for
163 + # Additional GPU memory may be needed for multi-modal encoding, which
164 + # needs to be accounted for when calculating the GPU blocks for
165 # vLLM blocker manager.
166 # To exercise the worst scenario for GPU memory consumption,
167 # the number of seqs (batch_size) is chosen to maximize the number
168 # of images processed.
169 model_config = self.model_config
170 + mm_config = self.multimodal_config
171
168 - if supports_multimodal(self.model):
169 - max_mm_tokens = MULTIMODAL_REGISTRY \
170 - .get_max_multimodal_tokens(model_config)
172 + input_registry = self.input_registry
173 + mm_registry = self.mm_registry
174 + mm_registry.init_mm_limits_per_prompt(model_config, mm_config)
175 +
176 + max_mm_tokens = mm_registry.get_max_multimodal_tokens(model_config)
177 + if max_mm_tokens > 0:
178 max_num_seqs_orig = max_num_seqs
179 max_num_seqs = min(max_num_seqs,
180 max_num_batched_tokens // max_mm_tokens)
@@ -183,13 +190,8 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
190 seq_len = (max_num_batched_tokens // max_num_seqs +
191 (group_id < max_num_batched_tokens % max_num_seqs))
192
186 - seq_data, dummy_multi_modal_data = INPUT_REGISTRY \
187 - .dummy_data_for_profiling(model_config, seq_len)
188 -
189 - # Having more tokens is over-conservative but otherwise fine
190 - assert len(seq_data.prompt_token_ids) >= seq_len, (
191 - f"Expected at least {seq_len} dummy tokens for profiling, "
192 - f"but got: {len(seq_data.prompt_token_ids)}")
193 + seq_data, dummy_multi_modal_data = input_registry \
194 + .dummy_data_for_profiling(model_config, seq_len, mm_registry)
195
196 seq = SequenceGroupMetadata(
197 request_id=str(group_id),