A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Move supported limits and max tokens to merged multi-modal processor (#11669)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
Cyrus Leung committed
Jan 1, 2025 at 23:44 UTC
a115ac46b5be22289dec975c2c06653b22cd6315
16 files changed
+340
-350
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py
+2
-37
@@ -4,7 +4,7 @@ from typing import Optional
4
import pytest
5
from transformers import AutoTokenizer
6
7
-from vllm.inputs import InputContext, InputProcessingContext
7
+from vllm.inputs import InputProcessingContext
8
from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
9
10
from .....conftest import _ImageAssets
@@ -20,42 +20,6 @@ def processor_for_phi3v():
20
return Phi3VMultiModalProcessor
21
22
23
-@pytest.fixture()
24
-def get_max_phi3v_image_tokens():
25
- from vllm.model_executor.models.phi3v import get_max_phi3v_image_tokens
26
- return get_max_phi3v_image_tokens
27
-
28
-
29
-@pytest.mark.parametrize("model", models)
30
-@pytest.mark.parametrize("num_crops,expected_max_tokens", [
31
- (4, 781),
32
- (16, 2653),
33
-])
34
-def test_max_tokens_override(get_max_phi3v_image_tokens, model: str,
35
- num_crops: int, expected_max_tokens: int):
36
- """Ensure get_max_phi3v_image_tokens handles num_crops properly."""
37
- # NOTE: mm_processor_kwargs on the context in this test is unused, since
38
- # this is testing the mapper directly. In practice, the processor kwargs
39
- # are wrapped in a closure when calling the max tokens func. We explicitly
40
- # do NOT use the mm_processor_kwargs in the model context here to ensure
41
- # that the max image tokens implementation is referencing a mix of the
42
- # kwargs to the function and the original mm_processor_kwargs in case
43
- # values are somehow updated and end up in a bad state.
44
- ctx = build_model_context(
45
- model_name=model,
46
- tokenizer_name=model,
47
- trust_remote_code=True,
48
- mm_processor_kwargs=None,
49
- )
50
-
51
- actual_max_tokens = get_max_phi3v_image_tokens(
52
- InputContext(ctx.model_config),
53
- num_crops=num_crops,
54
- )
55
-
56
- assert expected_max_tokens == actual_max_tokens
57
-
58
-
23
@pytest.mark.parametrize("model", models)
24
@pytest.mark.parametrize(
25
"num_crops,expected_toks_per_img",
@@ -77,6 +41,7 @@ def test_processor_override(processor_for_phi3v, image_assets: _ImageAssets,
41
model_name=model,
42
tokenizer_name=model,
43
trust_remote_code=True,
44
+ limit_mm_per_prompt={"image": num_imgs},
45
)
46
tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
47
ctx = InputProcessingContext(ctx.model_config, tokenizer)
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+2
-34
@@ -3,7 +3,7 @@ from typing import Any, Dict, Tuple
3
import pytest
4
from transformers import AutoTokenizer
5
6
-from vllm.inputs import InputContext, InputProcessingContext
6
+from vllm.inputs import InputProcessingContext
7
8
from .....conftest import _ImageAssets
9
from ....utils import build_model_context
@@ -22,39 +22,6 @@ def processor_for_qwen2_vl():
22
return Qwen2VLMultiModalProcessor
23
24
25
-@pytest.fixture()
26
-def get_max_qwen2_vl_image_tokens():
27
- from vllm.model_executor.models.qwen2_vl import (
28
- get_max_qwen2_vl_image_tokens)
29
- return get_max_qwen2_vl_image_tokens
30
-
31
-
32
-@pytest.mark.parametrize("mm_processor_kwargs,expected_max_tokens", [
33
- ({}, 16384),
34
- ({
35
- MIN_PIXELS: 64**2,
36
- MAX_PIXELS: 512**2
37
- }, 324),
38
-])
39
-@pytest.mark.parametrize("model", [MODEL])
40
-def test_qwen2_vl_max_image_tokens(
41
- get_max_qwen2_vl_image_tokens,
42
- model: str,
43
- mm_processor_kwargs: Dict[str, Any],
44
- expected_max_tokens: int,
45
-):
46
- """Ensure that the max token calc handles min/max pixels properly."""
47
- ctx = build_model_context(
48
- model_name=model,
49
- tokenizer_name=model,
50
- mm_processor_kwargs=None,
51
- )
52
-
53
- actual_max_tokens = get_max_qwen2_vl_image_tokens(
54
- InputContext(ctx.model_config), **mm_processor_kwargs)
55
- assert actual_max_tokens == expected_max_tokens
56
-
57
-
25
@pytest.mark.parametrize(
26
"mm_processor_kwargs, expected_toks_per_img, expected_pixels_shape", [
27
({}, 1426, (5704, 1176)),
@@ -82,6 +49,7 @@ def test_processor_override(
49
model_name=model,
50
tokenizer_name=model,
51
mm_processor_kwargs=None,
52
+ limit_mm_per_prompt={"image": num_imgs},
53
)
54
tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
55
ctx = InputProcessingContext(ctx.model_config, tokenizer)
tests/multimodal/test_processing.py
+8
-6
@@ -538,6 +538,11 @@ def _test_processing_cache_correctness(
538
else:
539
hf_overrides = {}
540
541
+ limit_mm_per_prompt = {
542
+ modality: 3 if supports_multi else 1
543
+ for modality, supports_multi in modalities.items()
544
+ }
545
+
546
model_config = ModelConfig(
547
model_id,
548
task="auto",
@@ -548,6 +553,7 @@ def _test_processing_cache_correctness(
553
dtype="float16",
554
revision=None,
555
hf_overrides=hf_overrides,
556
+ limit_mm_per_prompt=limit_mm_per_prompt,
557
)
558
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
559
@@ -580,18 +586,14 @@ def _test_processing_cache_correctness(
586
min_wh=128,
587
max_wh=256),
588
"audio":
583
- partial(_rand_audio, rng, min_len=256, max_len=512, sr=16000),
584
- }
585
- input_max_count = {
586
- modality: 3 if supports_multi else 1
587
- for modality, supports_multi in modalities.items()
589
+ partial(_rand_audio, rng, min_len=512, max_len=1024, sr=16000),
590
}
591
592
for batch_idx in range(num_batches):
593
mm_data = {
594
k:
595
[(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
594
- for _ in range(rng.randint(input_max_count[k]))]
596
+ for _ in range(rng.randint(limit_mm_per_prompt[k]))]
597
for k in modalities
598
}
599
vllm/inputs/registry.py
+1
-7
@@ -331,13 +331,7 @@ class InputRegistry:
331
trust_remote_code=model_config.trust_remote_code,
332
)
333
processor = mm_registry.create_processor(model_config, tokenizer)
334
-
335
- mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
336
- mm_max_tokens = mm_registry.get_max_tokens_by_modality(
337
- model_config)
338
-
339
- dummy_data = processor.get_dummy_data(seq_len, mm_counts,
340
- mm_max_tokens)
334
+ dummy_data = processor.get_dummy_data(seq_len)
335
else:
336
model_cls, _ = get_model_architecture(model_config)
337
if is_encoder_data:
vllm/model_executor/models/aria.py
+42
-33
@@ -1,5 +1,5 @@
1
-from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
2
- Union)
1
+from typing import (Callable, Iterable, List, Mapping, Optional, Set, Tuple,
2
+ TypedDict, Union)
3
4
import torch
5
import torch.nn as nn
@@ -9,7 +9,6 @@ from transformers import BatchFeature, PretrainedConfig
9
from vllm.attention import AttentionMetadata
10
from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
11
from vllm.distributed import get_tensor_model_parallel_rank
12
-from vllm.inputs import InputContext
12
from vllm.model_executor.layers.activation import get_act_fn
13
from vllm.model_executor.layers.fused_moe import FusedMoE
14
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -87,8 +86,8 @@ class AriaVisionModel(nn.Module):
86
def forward(
87
self,
88
pixel_values: torch.Tensor,
90
- pixel_mask: Optional[torch.BoolTensor] = None,
91
- ) -> Tuple[torch.Tensor, Optional[torch.BoolTensor]]:
89
+ pixel_mask: Optional[torch.Tensor] = None,
90
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
91
patch_attention_mask = self._create_patch_attention_mask(pixel_mask)
92
93
vit_oup = self.vision_model(
@@ -100,7 +99,8 @@ class AriaVisionModel(nn.Module):
99
100
return vit_oup, image_atts
101
103
- def _create_patch_attention_mask(self, pixel_mask):
102
+ def _create_patch_attention_mask(
103
+ self, pixel_mask: Optional[torch.Tensor]) -> torch.Tensor:
104
if pixel_mask is None:
105
return None
106
@@ -115,7 +115,8 @@ class AriaVisionModel(nn.Module):
115
)
116
return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
117
118
- def _create_image_attention_mask(self, patch_attention_mask):
118
+ def _create_image_attention_mask(
119
+ self, patch_attention_mask: torch.Tensor) -> torch.Tensor:
120
if patch_attention_mask is None:
121
return None
122
@@ -125,13 +126,13 @@ class AriaVisionModel(nn.Module):
126
127
class FFN(nn.Module):
128
128
- def __init__(self, embed_dim, ff_dim, output_dim):
129
+ def __init__(self, embed_dim: int, ff_dim: int, output_dim: int) -> None:
130
super().__init__()
131
self.linear_in = ColumnParallelLinear(embed_dim, ff_dim, bias=False)
132
self.linear_out = RowParallelLinear(ff_dim, output_dim, bias=False)
133
self.act = get_act_fn("gelu_new")
134
134
- def forward(self, hidden_states):
135
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
136
hidden_states, _ = self.linear_in(hidden_states)
137
hidden_states = self.act(hidden_states)
138
hidden_states, _ = self.linear_out(hidden_states)
@@ -140,7 +141,7 @@ class FFN(nn.Module):
141
142
class CrossAttention(nn.Module):
143
143
- def __init__(self, kv_dim, embed_dim, num_heads, drop_out_rate=0):
144
+ def __init__(self, kv_dim: int, embed_dim: int, num_heads: int) -> None:
145
super().__init__()
146
self.num_heads = num_heads
147
self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False)
@@ -149,12 +150,16 @@ class CrossAttention(nn.Module):
150
151
self.multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)
152
self.linear = nn.Linear(embed_dim, embed_dim)
152
- self.dropout = nn.Dropout(drop_out_rate)
153
154
self.layer_norm = nn.LayerNorm(embed_dim)
155
self.ln_kv = nn.LayerNorm(kv_dim)
156
157
- def forward(self, x, hidden_states, attn_mask=None, add_residual=False):
157
+ def forward(
158
+ self,
159
+ x: torch.Tensor,
160
+ hidden_states: torch.Tensor,
161
+ attn_mask: Optional[torch.Tensor] = None,
162
+ ) -> torch.Tensor:
163
normed_hidden_states = self.layer_norm(hidden_states)
164
query = self.q_proj(normed_hidden_states).permute(1, 0, 2)
165
@@ -169,11 +174,7 @@ class CrossAttention(nn.Module):
174
175
attn_output = attn_output.permute(1, 0, 2)
176
172
- if add_residual:
173
- attn_output = hidden_states + self.dropout(
174
- self.linear(attn_output))
175
- else:
176
- attn_output = self.dropout(self.linear(attn_output))
177
+ attn_output = self.linear(attn_output)
178
179
return attn_output
180
@@ -201,14 +202,14 @@ class AriaProjector(nn.Module):
202
203
def __init__(
204
self,
204
- patch_to_query_dict,
205
- embed_dim,
206
- num_heads,
207
- kv_dim,
208
- ff_dim,
209
- output_dim,
210
- norm_layer=nn.LayerNorm,
211
- ):
205
+ patch_to_query_dict: dict[int, int],
206
+ embed_dim: int,
207
+ num_heads: int,
208
+ kv_dim: int,
209
+ ff_dim: int,
210
+ output_dim: int,
211
+ norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
212
+ ) -> None:
213
super().__init__()
214
self.patch_to_query_dict = patch_to_query_dict
215
self.embed_dim = embed_dim
@@ -224,7 +225,11 @@ class AriaProjector(nn.Module):
225
self.ln_ffn = norm_layer(embed_dim)
226
self.ffn = FFN(embed_dim, ff_dim, output_dim)
227
227
- def forward(self, x, attn_mask=None):
228
+ def forward(
229
+ self,
230
+ x: torch.Tensor,
231
+ attn_mask: Optional[torch.Tensor] = None,
232
+ ) -> torch.Tensor:
233
bs = x.shape[0]
234
queries = self.query.unsqueeze(0).repeat(bs, 1, 1)
235
@@ -442,12 +447,17 @@ def build_mm_projector(config: PretrainedConfig):
447
)
448
449
445
-def get_max_aria_image_tokens(ctx: InputContext):
446
- hf_config = ctx.get_hf_config()
447
- return max(hf_config.projector_patch_to_query_dict.values())
450
+class AriaMultiModalProcessor(BaseMultiModalProcessor):
451
+
452
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
453
+ return {"image": None}
454
455
+ def _get_num_image_tokens(self) -> int:
456
+ hf_config = self.ctx.get_hf_config()
457
+ return max(hf_config.projector_patch_to_query_dict.values())
458
450
-class AriaMultiModalProcessor(BaseMultiModalProcessor):
459
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
460
+ return {"image": self._get_num_image_tokens()}
461
462
def _get_mm_fields_config(
463
self,
@@ -468,13 +478,13 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
478
hf_config = self.ctx.get_hf_config()
479
image_token_id = hf_config.image_token_index
480
471
- max_image_tokens = get_max_aria_image_tokens(self.ctx)
481
+ num_image_tokens = self._get_num_image_tokens()
482
483
return [
484
PromptReplacement(
485
modality="image",
486
target=[image_token_id],
477
- replacement=[image_token_id] * max_image_tokens,
487
+ replacement=[image_token_id] * num_image_tokens,
488
)
489
]
490
@@ -504,7 +514,6 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
514
)
515
516
507
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_aria_image_tokens)
517
@MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor)
518
class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
519
"""
vllm/model_executor/models/blip2.py
+9
-10
@@ -9,7 +9,6 @@ from transformers import (BatchFeature, Blip2Config, Blip2Processor,
9
10
from vllm.attention import AttentionMetadata
11
from vllm.config import CacheConfig, VllmConfig
12
-from vllm.inputs import InputContext
12
from vllm.model_executor.layers.activation import get_act_fn
13
from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -18,7 +17,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
17
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18
MultiModalInputsV2, MultiModalKwargs,
19
NestedTensors, PlaceholderRange)
21
-from vllm.multimodal.parse import MultiModalDataParser
20
from vllm.multimodal.processing import (BaseMultiModalProcessor,
21
MultiModalDataItems, ProcessorInputs,
22
PromptReplacement)
@@ -398,15 +396,17 @@ class Blip2QFormerModel(nn.Module):
396
return sequence_output
397
398
401
-def get_max_blip2_image_tokens(ctx: InputContext):
402
- hf_config = ctx.get_hf_config(Blip2Config)
403
- return hf_config.num_query_tokens
399
+class Blip2MultiModalProcessor(BaseMultiModalProcessor):
400
401
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
402
+ return {"image": 1}
403
406
-class Blip2MultiModalProcessor(BaseMultiModalProcessor):
404
+ def _get_num_image_tokens(self) -> int:
405
+ hf_config = self.ctx.get_hf_config(Blip2Config)
406
+ return hf_config.num_query_tokens
407
408
- def _get_data_parser(self) -> MultiModalDataParser:
409
- return MultiModalDataParser(max_mm_counts={"image": 1})
408
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
409
+ return {"image": self._get_num_image_tokens()}
410
411
def _get_hf_processor(self) -> Blip2Processor:
412
return self.ctx.get_hf_processor(Blip2Processor)
@@ -427,7 +427,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
427
hf_processor_mm_kwargs: Mapping[str, object],
428
out_mm_kwargs: MultiModalKwargs,
429
) -> list[PromptReplacement]:
430
- max_image_tokens = get_max_blip2_image_tokens(self.ctx)
430
+ max_image_tokens = self._get_num_image_tokens()
431
432
return [
433
PromptReplacement(
@@ -480,7 +480,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
480
)
481
482
483
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens)
483
@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor)
484
class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
485
vllm/model_executor/models/chameleon.py
+16
-19
@@ -11,7 +11,6 @@ from transformers import (BatchFeature, ChameleonConfig, ChameleonProcessor,
11
from vllm.attention import Attention, AttentionMetadata
12
from vllm.config import CacheConfig, VllmConfig
13
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14
-from vllm.inputs import InputContext
14
from vllm.model_executor.layers.activation import SiluAndMul
15
from vllm.model_executor.layers.layernorm import RMSNorm
16
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -31,7 +30,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
30
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31
MultiModalInputsV2, MultiModalKwargs,
32
NestedTensors, PlaceholderRange)
34
-from vllm.multimodal.parse import MultiModalDataParser
33
from vllm.multimodal.processing import (BaseMultiModalProcessor,
34
MultiModalDataItems, ProcessorInputs,
35
PromptReplacement)
@@ -43,11 +41,6 @@ from .utils import (is_pp_missing_parameter,
41
make_empty_intermediate_tensors_factory, make_layers,
42
maybe_prefix, merge_multimodal_embeddings)
43
46
-# These configs are not part of the model config but the preprocessor
47
-# and processor files, so we hardcode them in the model file for now.
48
-CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512
49
-CHAMELEON_IMAGE_SEQ_LENGTH = 1024
50
-
44
45
class ChameleonImagePixelInputs(TypedDict):
46
type: Literal["pixel_values"]
@@ -55,14 +48,17 @@ class ChameleonImagePixelInputs(TypedDict):
48
"""Shape: `(batch_size * num_images, num_channels, height, width)`"""
49
50
58
-def get_max_chameleon_image_tokens(ctx: InputContext):
59
- return CHAMELEON_IMAGE_SEQ_LENGTH
51
+class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
52
53
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
54
+ return {"image": 1}
55
62
-class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
56
+ def _get_num_image_tokens(self) -> int:
57
+ processor = self._get_hf_processor()
58
+ return processor.image_seq_length
59
64
- def _get_data_parser(self) -> MultiModalDataParser:
65
- return MultiModalDataParser(max_mm_counts={"image": 1})
60
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
61
+ return {"image": self._get_num_image_tokens()}
62
63
def _get_hf_processor(self) -> ChameleonProcessor:
64
return self.ctx.get_hf_processor(ChameleonProcessor)
@@ -88,7 +84,7 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
84
target="<image>",
85
replacement="".join([
86
processor.image_start_token,
91
- processor.image_token * CHAMELEON_IMAGE_SEQ_LENGTH,
87
+ processor.image_token * self._get_num_image_tokens(),
88
processor.image_end_token,
89
]),
90
)
@@ -98,12 +94,15 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
94
self,
95
mm_counts: Mapping[str, int],
96
) -> ProcessorInputs:
97
+ config = self.ctx.get_hf_config(ChameleonConfig)
98
+
99
+ width = height = config.vq_config.resolution
100
num_images = mm_counts.get("image", 0)
101
102
mm_data = {
103
"image":
105
- self._get_dummy_images(width=CHAMELEON_CROP_SIZE_WIDTH,
106
- height=CHAMELEON_CROP_SIZE_HEIGHT,
104
+ self._get_dummy_images(width=width,
105
+ height=height,
106
num_images=num_images)
107
}
108
@@ -902,7 +901,6 @@ class ChameleonModel(nn.Module):
901
return hidden_states
902
903
905
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens)
904
@MULTIMODAL_REGISTRY.register_processor(ChameleonMultiModalProcessor)
905
class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
906
SupportsPP):
@@ -931,9 +929,8 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
929
self.model.make_empty_intermediate_tensors)
930
931
def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
934
-
935
- expected_dims = (3, CHAMELEON_CROP_SIZE_HEIGHT,
936
- CHAMELEON_CROP_SIZE_WIDTH)
932
+ vq_config: ChameleonVQVAEConfig = self.config.vq_config
933
+ expected_dims = (3, vq_config.resolution, vq_config.resolution)
934
actual_dims = tuple(data.shape[1:])
935
936
if actual_dims != expected_dims:
vllm/model_executor/models/fuyu.py
+46
-59
@@ -25,7 +25,6 @@ from transformers import (BatchFeature, FuyuConfig, FuyuImageProcessor,
25
26
from vllm.attention import AttentionMetadata
27
from vllm.config import VllmConfig
28
-from vllm.inputs import InputContext
28
from vllm.model_executor.layers.linear import ColumnParallelLinear
29
from vllm.model_executor.layers.sampler import SamplerOutput
30
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
@@ -34,7 +33,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
MultiModalInputsV2, MultiModalKwargs,
35
NestedTensors, PlaceholderRange)
37
-from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataParser
36
+from vllm.multimodal.parse import ImageProcessorItems, ImageSize
37
from vllm.multimodal.processing import (BaseMultiModalProcessor,
38
MultiModalDataItems, ProcessorInputs,
39
PromptReplacement)
@@ -48,9 +47,6 @@ from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
47
_IMAGE_TOKEN_ID = 71011
48
_NEWLINE_TOKEN_ID = 71019
49
51
-MAX_IMAGE_FEATURE_SIZE_HEIGHT = 1080
52
-MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920
53
-
50
51
class FuyuImagePatchInputs(TypedDict):
52
type: Literal["image_patches"]
@@ -67,43 +63,49 @@ class FuyuImagePatchInputs(TypedDict):
63
"""
64
65
70
-def _get_fuyu_num_image_tokens(
71
- image_height: int,
72
- image_width: int,
73
-) -> Tuple[int, int]:
74
- """
75
- Calculate the number of image tokens needed for a given image size.
76
-
77
- The expected Fuyu image prompts can be expressed as:
78
-
79
- .. code-block::
80
- (image_token * ncols + newline_token) * nrows
81
-
82
- Args:
83
- image_size: Tuple[int, int] - `(width, height)` of the image
84
-
85
- Returns:
86
- ncols: int - number of image tokens in `x` direction
87
- nrows: int - number of image tokens in `y` direction
88
- """
89
- ncols = math.ceil(image_width / 30)
90
- nrows = math.ceil(image_height / 30)
91
- return ncols, nrows
92
-
66
+class FuyuMultiModalProcessor(BaseMultiModalProcessor):
67
94
-def get_max_fuyu_image_tokens(ctx: InputContext):
95
- ncols, nrows = _get_fuyu_num_image_tokens(
96
- image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
97
- image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
98
- )
68
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
69
+ return {"image": 1}
70
100
- return (ncols + 1) * nrows
71
+ def _get_image_target_size(self) -> ImageSize:
72
+ processor = self._get_hf_processor()
73
+ image_processor: FuyuImageProcessor = processor.image_processor
74
75
+ target_size = image_processor.size
76
+ return ImageSize(width=target_size["width"],
77
+ height=target_size["height"])
78
103
-class FuyuMultiModalProcessor(BaseMultiModalProcessor):
79
+ def _get_image_grid_size(
80
+ self,
81
+ *,
82
+ image_width: int,
83
+ image_height: int,
84
+ ) -> tuple[int, int]:
85
+ target_width, target_height = self._get_image_target_size()
86
+
87
+ if not (image_width <= target_width and image_height <= target_height):
88
+ height_scale_factor = target_height / image_height
89
+ width_scale_factor = target_width / image_width
90
+ optimal_scale_factor = min(height_scale_factor, width_scale_factor)
91
+
92
+ image_height = int(image_height * optimal_scale_factor)
93
+ image_width = int(image_width * optimal_scale_factor)
94
+
95
+ ncols = math.ceil(image_width / 30)
96
+ nrows = math.ceil(image_height / 30)
97
+ return ncols, nrows
98
+
99
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
100
+ target_width, target_height = self._get_image_target_size()
101
+
102
+ max_ncols, max_nrows = self._get_image_grid_size(
103
+ image_width=target_width,
104
+ image_height=target_height,
105
+ )
106
+ max_image_tokens = (max_ncols + 1) * max_nrows
107
105
- def _get_data_parser(self) -> MultiModalDataParser:
106
- return MultiModalDataParser(max_mm_counts={"image": 1})
108
+ return {"image": max_image_tokens}
109
110
def _get_hf_processor(self) -> FuyuProcessor:
111
return self.ctx.get_hf_processor(FuyuProcessor)
@@ -166,28 +168,13 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
168
eot_token_id = tokenizer.bos_token_id
169
assert isinstance(eot_token_id, int)
170
169
- hf_processor = self._get_hf_processor()
170
- image_processor: FuyuImageProcessor = hf_processor.image_processor
171
- target_size = image_processor.size
172
- target_height, target_width = (target_size["height"],
173
- target_size["width"])
174
-
171
def get_replacement_fuyu(item_idx: int):
172
images = mm_items.get_items("image", ImageProcessorItems)
173
image_size = images.get_image_size(item_idx)
178
- width, height = image_size.width, image_size.height
179
- if not (width <= target_width and height <= target_height):
180
- height_scale_factor = target_height / height
181
- width_scale_factor = target_width / width
182
- optimal_scale_factor = min(height_scale_factor,
183
- width_scale_factor)
184
-
185
- height = int(height * optimal_scale_factor)
186
- width = int(width * optimal_scale_factor)
187
-
188
- ncols, nrows = _get_fuyu_num_image_tokens(
189
- image_width=width,
190
- image_height=height,
174
+
175
+ ncols, nrows = self._get_image_grid_size(
176
+ image_width=image_size.width,
177
+ image_height=image_size.height,
178
)
179
180
return (([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows +
@@ -225,12 +212,13 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
212
self,
213
mm_counts: Mapping[str, int],
214
) -> ProcessorInputs:
215
+ target_width, target_height = self._get_image_target_size()
216
num_images = mm_counts.get("image", 0)
217
218
mm_data = {
219
"image":
232
- self._get_dummy_images(width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
233
- height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
220
+ self._get_dummy_images(width=target_width,
221
+ height=target_height,
222
num_images=num_images)
223
}
224
@@ -240,7 +228,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
228
)
229
230
243
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens)
231
@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor)
232
class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
233
vllm/model_executor/models/llava.py
+6
-2
@@ -119,6 +119,12 @@ def get_max_llava_image_tokens(ctx: InputContext):
119
120
class LlavaMultiModalProcessor(BaseMultiModalProcessor):
121
122
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
123
+ return {"image": None}
124
+
125
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
126
+ return {"image": get_max_llava_image_tokens(self.ctx)}
127
+
128
def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
129
return self.ctx.get_hf_processor((LlavaProcessor, PixtralProcessor))
130
@@ -324,7 +330,6 @@ def init_vision_tower_for_llava(
330
raise NotImplementedError(msg)
331
332
327
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
333
@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
334
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
335
# BitandBytes specific attributes
@@ -649,7 +654,6 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
654
655
# To use this model, please use
656
# `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
652
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
657
@MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor)
658
class MantisForConditionalGeneration(LlavaForConditionalGeneration):
659
pass
vllm/model_executor/models/phi3v.py
+25
-20
@@ -23,7 +23,6 @@ from transformers import (BatchFeature, CLIPVisionConfig, PretrainedConfig,
23
24
from vllm.attention import AttentionMetadata
25
from vllm.config import VllmConfig
26
-from vllm.inputs import InputContext
26
from vllm.logger import init_logger
27
from vllm.model_executor.layers.quantization import QuantizationConfig
28
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -306,24 +305,31 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
305
return image_features_hd_newline
306
307
309
-def get_max_phi3v_image_tokens(
310
- ctx: InputContext,
311
- *,
312
- num_crops: Optional[int] = None,
313
-) -> int:
314
- hf_processor_mm_kwargs = {}
315
- if num_crops:
316
- hf_processor_mm_kwargs["num_crops"] = num_crops
308
+class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
309
318
- processor = ctx.get_hf_processor(**hf_processor_mm_kwargs)
310
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
311
+ return {"image": None}
312
320
- return processor.calc_num_image_tokens_from_image_size(
321
- width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
322
- height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
323
- )
313
+ def _get_num_image_tokens(
314
+ self,
315
+ *,
316
+ image_width: int,
317
+ image_height: int,
318
+ ) -> int:
319
+ processor = self._get_hf_processor()
320
+
321
+ return processor.calc_num_image_tokens_from_image_size( # type: ignore
322
+ width=image_width,
323
+ height=image_height,
324
+ )
325
326
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
327
+ max_image_tokens = self._get_num_image_tokens(
328
+ image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
329
+ image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
330
+ )
331
326
-class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
332
+ return {"image": max_image_tokens}
333
334
def _get_hf_processor(
335
self,
@@ -332,6 +338,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
338
) -> ProcessorMixin:
339
if num_crops is not None:
340
return self.ctx.get_hf_processor(num_crops=num_crops)
341
+
342
return self.ctx.get_hf_processor()
343
344
def _call_hf_processor(
@@ -375,7 +382,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
382
) -> list[PromptReplacement]:
383
hf_processor = self._get_hf_processor()
384
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
378
- image_processor = hf_processor.image_processor # type: ignore
385
386
tokenizer = self._get_tokenizer()
387
bos_token_id = tokenizer.bos_token_id
@@ -385,9 +391,9 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
391
images = mm_items.get_items("image", ImageProcessorItems)
392
image_size = images.get_image_size(item_idx)
393
388
- num_tokens = image_processor.calc_num_image_tokens_from_image_size(
389
- width=image_size.width,
390
- height=image_size.height,
394
+ num_tokens = self._get_num_image_tokens(
395
+ image_width=image_size.width,
396
+ image_height=image_size.height,
397
)
398
399
return [_IMAGE_TOKEN_ID] * num_tokens + [bos_token_id]
@@ -467,7 +473,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
473
return result
474
475
470
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
476
@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
477
class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
478
hf_to_vllm_mapper = WeightsMapper(
vllm/model_executor/models/qwen2_audio.py
+30
-12
@@ -33,13 +33,12 @@ from transformers.models.whisper import WhisperFeatureExtractor
33
34
from vllm.attention import AttentionMetadata
35
from vllm.config import VllmConfig
36
-from vllm.inputs import InputContext
36
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY
39
from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40
NestedTensors)
42
-from vllm.multimodal.parse import MultiModalDataParser
41
+from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataParser
42
from vllm.multimodal.processing import (BaseMultiModalProcessor,
43
MultiModalDataItems, ProcessorInputs,
44
PromptReplacement)
@@ -80,14 +79,17 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
79
return feat_lengths, output_lengths
80
81
83
-def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
84
- hf_config = ctx.get_hf_config(Qwen2AudioConfig)
85
- max_source_position = hf_config.audio_config.max_source_positions
86
- output_lengths = (max_source_position - 2) // 2 + 1
87
- return output_lengths
82
+class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
83
84
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
85
+ return {"audio": None}
86
90
-class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
87
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
88
+ hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
89
+ max_source_positions = hf_config.audio_config.max_source_positions
90
+ max_output_lengths = (max_source_positions - 2) // 2 + 1
91
+
92
+ return {"audio": max_output_lengths}
93
94
def _get_hf_processor(
95
self,
@@ -157,11 +159,21 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
159
audio_output_lengths = []
160
else:
161
assert isinstance(feature_attention_mask, torch.Tensor)
160
- _, audio_output_lengths = _get_feat_extract_output_lengths(
162
+ _, audio_output_lens = _get_feat_extract_output_lengths(
163
feature_attention_mask.sum(-1))
164
165
+ audio_output_lengths = audio_output_lens.tolist()
166
+
167
def get_replacement_qwen2_audio(item_idx: int):
164
- return [placeholder] * audio_output_lengths[item_idx]
168
+ num_placeholders = audio_output_lengths[item_idx]
169
+ if num_placeholders == 0:
170
+ audios = mm_items.get_items("audio", AudioProcessorItems)
171
+ audio = audios.get(item_idx)
172
+ raise ValueError(
173
+ f"The audio {audio} (len={len(audio)}) is too short "
174
+ "to be represented inside the model")
175
+
176
+ return [placeholder] * num_placeholders
177
178
return [
179
PromptReplacement(
@@ -171,6 +183,14 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
183
)
184
]
185
186
+ def _always_apply_prompt_replacements(self) -> bool:
187
+ # HF never applies prompt replacements, so we have to do it ourselves
188
+ # _find_placeholders may incorrectly think that HF has already performed
189
+ # processing for multi-audio input when the input audios are short
190
+ # (the corresponding placeholders may take up fewer tokens than
191
+ # the number of audio items)
192
+ return True
193
+
194
def _get_dummy_mm_inputs(
195
self,
196
mm_counts: Mapping[str, int],
@@ -192,8 +212,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
212
)
213
214
195
-@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
196
- "audio", get_max_qwen2_audio_audio_tokens)
215
@MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)
216
class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
217
SupportsPP):
vllm/model_executor/models/qwen2_vl.py
+33
-42
@@ -40,7 +40,6 @@ from vllm.attention import AttentionMetadata
40
from vllm.config import VllmConfig
41
from vllm.distributed import parallel_state
42
from vllm.distributed import utils as dist_utils
43
-from vllm.inputs import InputContext
43
from vllm.logger import init_logger
44
from vllm.model_executor import SamplingMetadata
45
from vllm.model_executor.layers.activation import QuickGELU
@@ -650,8 +649,9 @@ def _get_vision_info(
649
width: int,
650
min_pixels: int,
651
max_pixels: int,
652
+ *,
653
do_resize: bool = True,
654
- data_type_key: str = "image",
654
+ modality: str = "image",
655
mm_count: int = 1,
656
):
657
"""Get information (resized height / width and number of vision tokens)
@@ -671,11 +671,12 @@ def _get_vision_info(
671
else:
672
resized_height, resized_width = height, width
673
674
- if data_type_key == "image":
674
+ if modality == "image":
675
grid_t = mm_count
676
- else:
677
- assert data_type_key == "video"
676
+ elif modality == "video":
677
grid_t = max(mm_count // temporal_patch_size, 1)
678
+ else:
679
+ raise ValueError(f"Modality {modality} is not supported")
680
681
grid_h = resized_height // patch_size
682
grid_w = resized_width // patch_size
@@ -691,41 +692,11 @@ def _get_image_processor(hf_processor: Qwen2VLProcessor):
692
return image_processor
693
694
694
-def get_max_qwen2_vl_mm_tokens(ctx: InputContext,
695
- data_type_key: str,
696
- *,
697
- min_pixels: Optional[int] = None,
698
- max_pixels: Optional[int] = None) -> int:
699
- hf_config = ctx.get_hf_config(Qwen2VLConfig)
700
- vision_config = hf_config.vision_config
701
-
702
- hf_processor = ctx.get_hf_processor(Qwen2VLProcessor)
703
- image_processor = _get_image_processor(hf_processor)
704
-
705
- _, _, max_llm_image_tokens = _get_vision_info(
706
- vision_config,
707
- height=9999999,
708
- width=9999999,
709
- min_pixels=min_pixels or image_processor.min_pixels,
710
- max_pixels=max_pixels or image_processor.max_pixels,
711
- data_type_key=data_type_key,
712
- )
713
- return max_llm_image_tokens
714
-
715
-
716
-get_max_qwen2_vl_image_tokens = partial(get_max_qwen2_vl_mm_tokens,
717
- data_type_key="image")
718
-get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
719
- data_type_key="video")
720
-
721
-
695
class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
696
dict[str, torch.Tensor]]):
697
698
def __init__(self, data: dict, modality: str) -> None:
726
- super().__init__(data)
727
-
728
- self.modality = modality
699
+ super().__init__(data, modality)
700
701
grid_thw = data[f"{modality}_grid_thw"]
702
slice_idxs = [0] + grid_thw.prod(-1).cumsum_(0).tolist()
@@ -734,9 +705,6 @@ class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
705
for i in range(len(grid_thw))
706
]
707
737
- def __repr__(self) -> str:
738
- return (f"{type(self).__name__}(modality={self.modality!r})")
739
-
708
def get_count(self) -> int:
709
return len(self.data[f"{self.modality}_grid_thw"])
710
@@ -792,6 +760,32 @@ class Qwen2MultiModalDataParser(MultiModalDataParser):
760
761
class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
762
763
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
764
+ return {"image": None, "video": None}
765
+
766
+ def _get_max_mm_tokens(self, modality: str) -> int:
767
+ hf_config = self.ctx.get_hf_config(Qwen2VLConfig)
768
+ vision_config = hf_config.vision_config
769
+
770
+ hf_processor = self._get_hf_processor()
771
+ image_processor = _get_image_processor(hf_processor)
772
+
773
+ _, _, max_llm_image_tokens = _get_vision_info(
774
+ vision_config,
775
+ height=9999999,
776
+ width=9999999,
777
+ min_pixels=image_processor.min_pixels,
778
+ max_pixels=image_processor.max_pixels,
779
+ modality=modality,
780
+ )
781
+ return max_llm_image_tokens
782
+
783
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
784
+ return {
785
+ "image": self._get_max_mm_tokens("image"),
786
+ "video": self._get_max_mm_tokens("video"),
787
+ }
788
+
789
def _get_data_parser(self) -> MultiModalDataParser:
790
return Qwen2MultiModalDataParser()
791
@@ -908,9 +902,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
902
)
903
904
911
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_qwen2_vl_image_tokens)
912
-@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
913
- "video", get_max_qwen2_vl_video_tokens)
905
@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor)
906
class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
907
SupportsLoRA, SupportsPP):
vllm/model_executor/models/ultravox.py
+9
-17
@@ -2,7 +2,7 @@
2
"""PyTorch Ultravox model."""
3
4
import math
5
-from functools import cached_property, lru_cache
5
+from functools import cached_property
6
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7
TypedDict, Union)
8
@@ -17,7 +17,6 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
17
18
from vllm.attention import AttentionMetadata
19
from vllm.config import VllmConfig
20
-from vllm.inputs import InputContext
20
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
21
from vllm.model_executor.layers.layernorm import RMSNorm
22
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -58,22 +57,17 @@ UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
57
UltravoxAudioEmbeddingInputs]
58
59
61
-@lru_cache
62
-def cached_feature_extractor(model_id: str) -> WhisperFeatureExtractor:
63
- return WhisperFeatureExtractor.from_pretrained(model_id)
64
-
65
-
66
-def whisper_feature_extractor(ctx: InputContext) -> WhisperFeatureExtractor:
67
- hf_config = ctx.get_hf_config(UltravoxConfig)
68
- return cached_feature_extractor(hf_config.audio_model_id)
69
-
60
+class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
61
71
-def get_ultravox_max_audio_tokens(ctx: InputContext):
72
- feature_extractor = whisper_feature_extractor(ctx)
73
- return math.ceil(feature_extractor.chunk_length * _AUDIO_TOKENS_PER_SECOND)
62
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
63
+ return {"audio": None}
64
65
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
66
+ feature_extractor = self._get_feature_extractor()
67
+ max_audio_tokens = math.ceil(feature_extractor.chunk_length *
68
+ _AUDIO_TOKENS_PER_SECOND)
69
76
-class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
70
+ return {"audio": max_audio_tokens}
71
72
def _get_hf_processor(
73
self,
@@ -322,8 +316,6 @@ class ModifiedWhisperEncoder(WhisperEncoder):
316
return hidden_states
317
318
325
-@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
326
- "audio", get_ultravox_max_audio_tokens)
319
@MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor)
320
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
321
vllm/multimodal/parse.py
+8
-39
@@ -21,10 +21,15 @@ _I = TypeVar("_I")
21
22
class ModalityDataItems(ABC, Generic[_T, _I]):
23
24
- def __init__(self, data: _T) -> None:
24
+ def __init__(self, data: _T, modality: str) -> None:
25
super().__init__()
26
27
self.data = data
28
+ self.modality = modality
29
+
30
+ def __repr__(self) -> str:
31
+ return (f"{type(self).__name__}(modality={self.modality!r}, "
32
+ f"len={len(self)})")
33
34
def __len__(self) -> int:
35
return self.get_count()
@@ -64,14 +69,6 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
69
70
class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
71
67
- def __init__(self, data: Sequence[_T], modality: str) -> None:
68
- super().__init__(data)
69
-
70
- self.modality = modality
71
-
72
- def __repr__(self) -> str:
73
- return (f"{type(self).__name__}(modality={self.modality!r})")
74
-
72
def get_count(self) -> int:
73
return len(self.data)
74
@@ -87,14 +84,6 @@ class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
84
85
class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
86
90
- def __init__(self, data: NestedTensors, modality: str) -> None:
91
- super().__init__(data)
92
-
93
- self.modality = modality
94
-
95
- def __repr__(self) -> str:
96
- return (f"{type(self).__name__}(modality={self.modality!r})")
97
-
87
def get_count(self) -> int:
88
return len(self.data)
89
@@ -222,22 +211,13 @@ class MultiModalDataParser:
211
Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
212
213
Args:
225
- max_mm_counts (Mapping[str, int]): The maximum allowed number of items
226
- belonging to each modality. This effectively sets a hard limit over
227
- `--limit-mm-per-prompt`.
214
target_sr (float, optional): Enables automatic resampling of audio
215
items to the model's expected sampling rate.
216
"""
217
232
- def __init__(
233
- self,
234
- *,
235
- max_mm_counts: Mapping[str, int] = {},
236
- target_sr: Optional[float] = None,
237
- ) -> None:
218
+ def __init__(self, *, target_sr: Optional[float] = None) -> None:
219
super().__init__()
220
240
- self.max_mm_counts = max_mm_counts
221
self.target_sr = target_sr
222
223
def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
@@ -345,7 +325,6 @@ class MultiModalDataParser:
325
326
def parse_mm_data(self,
327
mm_data: MultiModalDataDict) -> MultiModalDataItems:
348
- max_mm_counts = self.max_mm_counts
328
subparsers = self._get_subparsers()
329
330
mm_items = MultiModalDataItems()
@@ -353,16 +332,6 @@ class MultiModalDataParser:
332
if k not in subparsers:
333
raise ValueError(f"Unsupported modality: {k}")
334
356
- modality_items = subparsers[k](v)
357
-
358
- if k in max_mm_counts:
359
- max_count = max_mm_counts[k]
360
- if len(modality_items) > max_count:
361
- raise ValueError(
362
- f"This model supports at most {max_count} {k} items "
363
- f"per prompt, but {len(modality_items)} {k} items "
364
- "were given or set as its limit_mm_per_prompt.")
365
-
366
- mm_items[k] = modality_items
335
+ mm_items[k] = subparsers[k](v)
336
337
return mm_items
vllm/multimodal/processing.py
+98
-13
@@ -624,6 +624,29 @@ class BaseMultiModalProcessor(ABC):
624
) -> MultiModalInputsV2:
625
return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
626
627
+ @abstractmethod
628
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
629
+ """
630
+ Return the maximum supported number of items for each modality.
631
+
632
+ A value of `None` means unlimited number of items.
633
+
634
+ Omitting a modality from the returned dictionary means that
635
+ it is not supported at all.
636
+ """
637
+ raise NotImplementedError
638
+
639
+ @abstractmethod
640
+ def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
641
+ """
642
+ Get the maximum possible number of tokens per data item
643
+ for each modality.
644
+
645
+ The dictionary returned by this method should have the same
646
+ keys as that returned by :meth:`get_supported_mm_limits`.
647
+ """
648
+ raise NotImplementedError
649
+
650
def _get_data_parser(self) -> MultiModalDataParser:
651
"""
652
Construct a data parser to preprocess multi-modal data items
@@ -653,7 +676,18 @@ class BaseMultiModalProcessor(ABC):
676
before passing them to :meth:`_get_hf_mm_data`.
677
"""
678
parser = self._get_data_parser()
656
- return parser.parse_mm_data(mm_data)
679
+ mm_items = parser.parse_mm_data(mm_data)
680
+
681
+ mm_limits = self.ctx.get_mm_config().limit_per_prompt
682
+ for modality, items in mm_items.items():
683
+ limit = mm_limits.get(modality, 1)
684
+ if len(items) > limit:
685
+ raise ValueError(
686
+ f"You set {modality}={limit} (or defaulted to 1) in "
687
+ f"`--limit-mm-per-prompt`, but passed {len(items)} "
688
+ f"{modality} items in the same prompt.")
689
+
690
+ return mm_items
691
692
@abstractmethod
693
def _get_mm_fields_config(
@@ -901,6 +935,17 @@ class BaseMultiModalProcessor(ABC):
935
936
return [prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls]
937
938
+ def _always_apply_prompt_replacements(self) -> bool:
939
+ """
940
+ A flag which can be overridden so that
941
+ :meth:`_apply_prompt_replacements` is always called even if we
942
+ detect that HF has performed processing via :meth:`_find_placeholders`.
943
+
944
+ This is useful in cases where :meth:`_find_placeholders` cannot be
945
+ reliably used to detect whether HF has performed processing or not.
946
+ """
947
+ return False
948
+
949
def _apply_prompt_replacements(
950
self,
951
token_ids: list[int],
@@ -995,7 +1040,7 @@ class BaseMultiModalProcessor(ABC):
1040
all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
1041
mm_item_counts)
1042
998
- if all_placeholders:
1043
+ if all_placeholders and not self._always_apply_prompt_replacements():
1044
tokenizer = self._get_tokenizer()
1045
prompt_text = _decode(tokenizer, prompt_ids)
1046
else:
@@ -1009,10 +1054,27 @@ class BaseMultiModalProcessor(ABC):
1054
mm_item_counts,
1055
)
1056
1012
- mm_placeholders = {
1013
- modality: [item.to_range() for item in items]
1014
- for modality, items in full_groupby_modality(all_placeholders)
1015
- }
1057
+ mm_placeholders = dict[str, list[PlaceholderRange]]()
1058
+ err_suffix = ("This suggests a problem with your implementation of "
1059
+ "the merged multi-modal processor for this model, "
1060
+ "particularly in the `_get_prompt_replacements` method.")
1061
+
1062
+ for modality, placeholders in full_groupby_modality(all_placeholders):
1063
+ if modality not in mm_items:
1064
+ raise AssertionError(
1065
+ f"Expected no placeholders for {modality=}, "
1066
+ f"but found {placeholders=}. Input items: {mm_items}"
1067
+ f"\n{err_suffix}")
1068
+
1069
+ if len(placeholders) != len(mm_items[modality]):
1070
+ raise AssertionError(
1071
+ f"Expected length of {placeholders=} for {modality=} "
1072
+ f"to equal that of input items: {mm_items[modality]}"
1073
+ f"\n{err_suffix}")
1074
+
1075
+ mm_placeholders[modality] = [
1076
+ item.to_range() for item in placeholders
1077
+ ]
1078
1079
return MultiModalInputsV2(
1080
type="multimodal",
@@ -1063,15 +1125,38 @@ class BaseMultiModalProcessor(ABC):
1125
"""
1126
raise NotImplementedError
1127
1066
- def get_dummy_data(
1067
- self,
1068
- seq_len: int,
1069
- mm_counts: Mapping[str, int],
1070
- mm_max_tokens: Mapping[str, int],
1071
- ) -> DummyData:
1128
+ def _get_and_validate_dummy_mm_counts(self) -> Mapping[str, int]:
1129
+ mm_limit_per_prompt = self.ctx.get_mm_config().limit_per_prompt
1130
+ supported_mm_limits = self.get_supported_mm_limits()
1131
+
1132
+ mm_limits = {
1133
+ modality: mm_limit_per_prompt.get(modality, 1)
1134
+ for modality in supported_mm_limits
1135
+ }
1136
+
1137
+ for modality, supported_limit in supported_mm_limits.items():
1138
+ limit = mm_limits[modality]
1139
+ if supported_limit is not None and supported_limit < limit:
1140
+ raise ValueError(
1141
+ f"You set {modality}={limit} (or defaulted to 1) in "
1142
+ f"`--limit-mm-per-prompt`, but this model only supports "
1143
+ f"at most {supported_limit} {modality} items.")
1144
+
1145
+ return mm_limits
1146
+
1147
+ def get_dummy_data(self, seq_len: int) -> DummyData:
1148
# Avoid circular import
1149
from vllm.sequence import SequenceData
1150
1151
+ mm_counts = self._get_and_validate_dummy_mm_counts()
1152
+ mm_max_tokens_per_item = self.get_mm_max_tokens_per_item()
1153
+ if mm_counts.keys() != mm_max_tokens_per_item.keys():
1154
+ raise AssertionError(
1155
+ "The keys returned by `get_supported_mm_limits`"
1156
+ f"({set(mm_counts.keys())}) should be the same as those "
1157
+ "returned by `get_mm_max_tokens_per_item` "
1158
+ f"({set(mm_max_tokens_per_item.keys())})")
1159
+
1160
processor_inputs = self._get_dummy_mm_inputs(mm_counts)
1161
mm_inputs = self.apply(
1162
prompt_text=processor_inputs.prompt_text,
@@ -1087,7 +1172,7 @@ class BaseMultiModalProcessor(ABC):
1172
for modality, placeholders in placeholders_by_modality.items()
1173
}
1174
expected_placeholders_by_modality = {
1090
- modality: mm_max_tokens[modality]
1175
+ modality: mm_max_tokens_per_item[modality] * mm_counts[modality]
1176
for modality in placeholders_by_modality
1177
}
1178
if total_placeholders_by_modality != expected_placeholders_by_modality:
vllm/multimodal/registry.py
+5
@@ -15,6 +15,7 @@ from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
15
from .image import ImagePlugin
16
from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
17
from .processing import BaseMultiModalProcessor, ProcessingCache
18
+from .utils import cached_get_tokenizer
19
from .video import VideoPlugin
20
21
if TYPE_CHECKING:
@@ -219,6 +220,10 @@ class MultiModalRegistry:
220
Note:
221
This is currently directly used only in V1.
222
"""
223
+ if self.has_processor(model_config):
224
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
225
+ processor = self.create_processor(model_config, tokenizer)
226
+ return processor.get_mm_max_tokens_per_item()
227
228
return {
229
key: plugin.get_max_multimodal_tokens(model_config)