@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a115ac46b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM] Move supported limits and max tokens to merged multi-modal processor (#11669)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>

Cyrus Leung committed Jan 1, 2025 at 23:44 UTC a115ac46b5be22289dec975c2c06653b22cd6315
16 files changed +340 -350
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py
+2 -37
@@ -4,7 +4,7 @@ from typing import Optional
4 import pytest
5 from transformers import AutoTokenizer
6
7 -from vllm.inputs import InputContext, InputProcessingContext
7 +from vllm.inputs import InputProcessingContext
8 from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
9
10 from .....conftest import _ImageAssets
@@ -20,42 +20,6 @@ def processor_for_phi3v():
20 return Phi3VMultiModalProcessor
21
22
23 -@pytest.fixture()
24 -def get_max_phi3v_image_tokens():
25 - from vllm.model_executor.models.phi3v import get_max_phi3v_image_tokens
26 - return get_max_phi3v_image_tokens
27 -
28 -
29 -@pytest.mark.parametrize("model", models)
30 -@pytest.mark.parametrize("num_crops,expected_max_tokens", [
31 - (4, 781),
32 - (16, 2653),
33 -])
34 -def test_max_tokens_override(get_max_phi3v_image_tokens, model: str,
35 - num_crops: int, expected_max_tokens: int):
36 - """Ensure get_max_phi3v_image_tokens handles num_crops properly."""
37 - # NOTE: mm_processor_kwargs on the context in this test is unused, since
38 - # this is testing the mapper directly. In practice, the processor kwargs
39 - # are wrapped in a closure when calling the max tokens func. We explicitly
40 - # do NOT use the mm_processor_kwargs in the model context here to ensure
41 - # that the max image tokens implementation is referencing a mix of the
42 - # kwargs to the function and the original mm_processor_kwargs in case
43 - # values are somehow updated and end up in a bad state.
44 - ctx = build_model_context(
45 - model_name=model,
46 - tokenizer_name=model,
47 - trust_remote_code=True,
48 - mm_processor_kwargs=None,
49 - )
50 -
51 - actual_max_tokens = get_max_phi3v_image_tokens(
52 - InputContext(ctx.model_config),
53 - num_crops=num_crops,
54 - )
55 -
56 - assert expected_max_tokens == actual_max_tokens
57 -
58 -
23 @pytest.mark.parametrize("model", models)
24 @pytest.mark.parametrize(
25 "num_crops,expected_toks_per_img",
@@ -77,6 +41,7 @@ def test_processor_override(processor_for_phi3v, image_assets: _ImageAssets,
41 model_name=model,
42 tokenizer_name=model,
43 trust_remote_code=True,
44 + limit_mm_per_prompt={"image": num_imgs},
45 )
46 tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
47 ctx = InputProcessingContext(ctx.model_config, tokenizer)
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+2 -34
@@ -3,7 +3,7 @@ from typing import Any, Dict, Tuple
3 import pytest
4 from transformers import AutoTokenizer
5
6 -from vllm.inputs import InputContext, InputProcessingContext
6 +from vllm.inputs import InputProcessingContext
7
8 from .....conftest import _ImageAssets
9 from ....utils import build_model_context
@@ -22,39 +22,6 @@ def processor_for_qwen2_vl():
22 return Qwen2VLMultiModalProcessor
23
24
25 -@pytest.fixture()
26 -def get_max_qwen2_vl_image_tokens():
27 - from vllm.model_executor.models.qwen2_vl import (
28 - get_max_qwen2_vl_image_tokens)
29 - return get_max_qwen2_vl_image_tokens
30 -
31 -
32 -@pytest.mark.parametrize("mm_processor_kwargs,expected_max_tokens", [
33 - ({}, 16384),
34 - ({
35 - MIN_PIXELS: 64**2,
36 - MAX_PIXELS: 512**2
37 - }, 324),
38 -])
39 -@pytest.mark.parametrize("model", [MODEL])
40 -def test_qwen2_vl_max_image_tokens(
41 - get_max_qwen2_vl_image_tokens,
42 - model: str,
43 - mm_processor_kwargs: Dict[str, Any],
44 - expected_max_tokens: int,
45 -):
46 - """Ensure that the max token calc handles min/max pixels properly."""
47 - ctx = build_model_context(
48 - model_name=model,
49 - tokenizer_name=model,
50 - mm_processor_kwargs=None,
51 - )
52 -
53 - actual_max_tokens = get_max_qwen2_vl_image_tokens(
54 - InputContext(ctx.model_config), **mm_processor_kwargs)
55 - assert actual_max_tokens == expected_max_tokens
56 -
57 -
25 @pytest.mark.parametrize(
26 "mm_processor_kwargs, expected_toks_per_img, expected_pixels_shape", [
27 ({}, 1426, (5704, 1176)),
@@ -82,6 +49,7 @@ def test_processor_override(
49 model_name=model,
50 tokenizer_name=model,
51 mm_processor_kwargs=None,
52 + limit_mm_per_prompt={"image": num_imgs},
53 )
54 tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
55 ctx = InputProcessingContext(ctx.model_config, tokenizer)
tests/multimodal/test_processing.py
+8 -6
@@ -538,6 +538,11 @@ def _test_processing_cache_correctness(
538 else:
539 hf_overrides = {}
540
541 + limit_mm_per_prompt = {
542 + modality: 3 if supports_multi else 1
543 + for modality, supports_multi in modalities.items()
544 + }
545 +
546 model_config = ModelConfig(
547 model_id,
548 task="auto",
@@ -548,6 +553,7 @@ def _test_processing_cache_correctness(
553 dtype="float16",
554 revision=None,
555 hf_overrides=hf_overrides,
556 + limit_mm_per_prompt=limit_mm_per_prompt,
557 )
558 model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
559
@@ -580,18 +586,14 @@ def _test_processing_cache_correctness(
586 min_wh=128,
587 max_wh=256),
588 "audio":
583 - partial(_rand_audio, rng, min_len=256, max_len=512, sr=16000),
584 - }
585 - input_max_count = {
586 - modality: 3 if supports_multi else 1
587 - for modality, supports_multi in modalities.items()
589 + partial(_rand_audio, rng, min_len=512, max_len=1024, sr=16000),
590 }
591
592 for batch_idx in range(num_batches):
593 mm_data = {
594 k:
595 [(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
594 - for _ in range(rng.randint(input_max_count[k]))]
596 + for _ in range(rng.randint(limit_mm_per_prompt[k]))]
597 for k in modalities
598 }
599
vllm/inputs/registry.py
+1 -7
@@ -331,13 +331,7 @@ class InputRegistry:
331 trust_remote_code=model_config.trust_remote_code,
332 )
333 processor = mm_registry.create_processor(model_config, tokenizer)
334 -
335 - mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
336 - mm_max_tokens = mm_registry.get_max_tokens_by_modality(
337 - model_config)
338 -
339 - dummy_data = processor.get_dummy_data(seq_len, mm_counts,
340 - mm_max_tokens)
334 + dummy_data = processor.get_dummy_data(seq_len)
335 else:
336 model_cls, _ = get_model_architecture(model_config)
337 if is_encoder_data:
vllm/model_executor/models/aria.py
+42 -33
@@ -1,5 +1,5 @@
1 -from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
2 - Union)
1 +from typing import (Callable, Iterable, List, Mapping, Optional, Set, Tuple,
2 + TypedDict, Union)
3
4 import torch
5 import torch.nn as nn
@@ -9,7 +9,6 @@ from transformers import BatchFeature, PretrainedConfig
9 from vllm.attention import AttentionMetadata
10 from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
11 from vllm.distributed import get_tensor_model_parallel_rank
12 -from vllm.inputs import InputContext
12 from vllm.model_executor.layers.activation import get_act_fn
13 from vllm.model_executor.layers.fused_moe import FusedMoE
14 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -87,8 +86,8 @@ class AriaVisionModel(nn.Module):
86 def forward(
87 self,
88 pixel_values: torch.Tensor,
90 - pixel_mask: Optional[torch.BoolTensor] = None,
91 - ) -> Tuple[torch.Tensor, Optional[torch.BoolTensor]]:
89 + pixel_mask: Optional[torch.Tensor] = None,
90 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
91 patch_attention_mask = self._create_patch_attention_mask(pixel_mask)
92
93 vit_oup = self.vision_model(
@@ -100,7 +99,8 @@ class AriaVisionModel(nn.Module):
99
100 return vit_oup, image_atts
101
103 - def _create_patch_attention_mask(self, pixel_mask):
102 + def _create_patch_attention_mask(
103 + self, pixel_mask: Optional[torch.Tensor]) -> torch.Tensor:
104 if pixel_mask is None:
105 return None
106
@@ -115,7 +115,8 @@ class AriaVisionModel(nn.Module):
115 )
116 return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
117
118 - def _create_image_attention_mask(self, patch_attention_mask):
118 + def _create_image_attention_mask(
119 + self, patch_attention_mask: torch.Tensor) -> torch.Tensor:
120 if patch_attention_mask is None:
121 return None
122
@@ -125,13 +126,13 @@ class AriaVisionModel(nn.Module):
126
127 class FFN(nn.Module):
128
128 - def __init__(self, embed_dim, ff_dim, output_dim):
129 + def __init__(self, embed_dim: int, ff_dim: int, output_dim: int) -> None:
130 super().__init__()
131 self.linear_in = ColumnParallelLinear(embed_dim, ff_dim, bias=False)
132 self.linear_out = RowParallelLinear(ff_dim, output_dim, bias=False)
133 self.act = get_act_fn("gelu_new")
134
134 - def forward(self, hidden_states):
135 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
136 hidden_states, _ = self.linear_in(hidden_states)
137 hidden_states = self.act(hidden_states)
138 hidden_states, _ = self.linear_out(hidden_states)
@@ -140,7 +141,7 @@ class FFN(nn.Module):
141
142 class CrossAttention(nn.Module):
143
143 - def __init__(self, kv_dim, embed_dim, num_heads, drop_out_rate=0):
144 + def __init__(self, kv_dim: int, embed_dim: int, num_heads: int) -> None:
145 super().__init__()
146 self.num_heads = num_heads
147 self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False)
@@ -149,12 +150,16 @@ class CrossAttention(nn.Module):
150
151 self.multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)
152 self.linear = nn.Linear(embed_dim, embed_dim)
152 - self.dropout = nn.Dropout(drop_out_rate)
153
154 self.layer_norm = nn.LayerNorm(embed_dim)
155 self.ln_kv = nn.LayerNorm(kv_dim)
156
157 - def forward(self, x, hidden_states, attn_mask=None, add_residual=False):
157 + def forward(
158 + self,
159 + x: torch.Tensor,
160 + hidden_states: torch.Tensor,
161 + attn_mask: Optional[torch.Tensor] = None,
162 + ) -> torch.Tensor:
163 normed_hidden_states = self.layer_norm(hidden_states)
164 query = self.q_proj(normed_hidden_states).permute(1, 0, 2)
165
@@ -169,11 +174,7 @@ class CrossAttention(nn.Module):
174
175 attn_output = attn_output.permute(1, 0, 2)
176
172 - if add_residual:
173 - attn_output = hidden_states + self.dropout(
174 - self.linear(attn_output))
175 - else:
176 - attn_output = self.dropout(self.linear(attn_output))
177 + attn_output = self.linear(attn_output)
178
179 return attn_output
180
@@ -201,14 +202,14 @@ class AriaProjector(nn.Module):
202
203 def __init__(
204 self,
204 - patch_to_query_dict,
205 - embed_dim,
206 - num_heads,
207 - kv_dim,
208 - ff_dim,
209 - output_dim,
210 - norm_layer=nn.LayerNorm,
211 - ):
205 + patch_to_query_dict: dict[int, int],
206 + embed_dim: int,
207 + num_heads: int,
208 + kv_dim: int,
209 + ff_dim: int,
210 + output_dim: int,
211 + norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
212 + ) -> None:
213 super().__init__()
214 self.patch_to_query_dict = patch_to_query_dict
215 self.embed_dim = embed_dim
@@ -224,7 +225,11 @@ class AriaProjector(nn.Module):
225 self.ln_ffn = norm_layer(embed_dim)
226 self.ffn = FFN(embed_dim, ff_dim, output_dim)
227
227 - def forward(self, x, attn_mask=None):
228 + def forward(
229 + self,
230 + x: torch.Tensor,
231 + attn_mask: Optional[torch.Tensor] = None,
232 + ) -> torch.Tensor:
233 bs = x.shape[0]
234 queries = self.query.unsqueeze(0).repeat(bs, 1, 1)
235
@@ -442,12 +447,17 @@ def build_mm_projector(config: PretrainedConfig):
447 )
448
449
445 -def get_max_aria_image_tokens(ctx: InputContext):
446 - hf_config = ctx.get_hf_config()
447 - return max(hf_config.projector_patch_to_query_dict.values())
450 +class AriaMultiModalProcessor(BaseMultiModalProcessor):
451 +
452 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
453 + return {"image": None}
454
455 + def _get_num_image_tokens(self) -> int:
456 + hf_config = self.ctx.get_hf_config()
457 + return max(hf_config.projector_patch_to_query_dict.values())
458
450 -class AriaMultiModalProcessor(BaseMultiModalProcessor):
459 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
460 + return {"image": self._get_num_image_tokens()}
461
462 def _get_mm_fields_config(
463 self,
@@ -468,13 +478,13 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
478 hf_config = self.ctx.get_hf_config()
479 image_token_id = hf_config.image_token_index
480
471 - max_image_tokens = get_max_aria_image_tokens(self.ctx)
481 + num_image_tokens = self._get_num_image_tokens()
482
483 return [
484 PromptReplacement(
485 modality="image",
486 target=[image_token_id],
477 - replacement=[image_token_id] * max_image_tokens,
487 + replacement=[image_token_id] * num_image_tokens,
488 )
489 ]
490
@@ -504,7 +514,6 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
514 )
515
516
507 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_aria_image_tokens)
517 @MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor)
518 class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
519 """
vllm/model_executor/models/blip2.py
+9 -10
@@ -9,7 +9,6 @@ from transformers import (BatchFeature, Blip2Config, Blip2Processor,
9
10 from vllm.attention import AttentionMetadata
11 from vllm.config import CacheConfig, VllmConfig
12 -from vllm.inputs import InputContext
12 from vllm.model_executor.layers.activation import get_act_fn
13 from vllm.model_executor.layers.quantization import QuantizationConfig
14 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -18,7 +17,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
17 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18 MultiModalInputsV2, MultiModalKwargs,
19 NestedTensors, PlaceholderRange)
21 -from vllm.multimodal.parse import MultiModalDataParser
20 from vllm.multimodal.processing import (BaseMultiModalProcessor,
21 MultiModalDataItems, ProcessorInputs,
22 PromptReplacement)
@@ -398,15 +396,17 @@ class Blip2QFormerModel(nn.Module):
396 return sequence_output
397
398
401 -def get_max_blip2_image_tokens(ctx: InputContext):
402 - hf_config = ctx.get_hf_config(Blip2Config)
403 - return hf_config.num_query_tokens
399 +class Blip2MultiModalProcessor(BaseMultiModalProcessor):
400
401 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
402 + return {"image": 1}
403
406 -class Blip2MultiModalProcessor(BaseMultiModalProcessor):
404 + def _get_num_image_tokens(self) -> int:
405 + hf_config = self.ctx.get_hf_config(Blip2Config)
406 + return hf_config.num_query_tokens
407
408 - def _get_data_parser(self) -> MultiModalDataParser:
409 - return MultiModalDataParser(max_mm_counts={"image": 1})
408 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
409 + return {"image": self._get_num_image_tokens()}
410
411 def _get_hf_processor(self) -> Blip2Processor:
412 return self.ctx.get_hf_processor(Blip2Processor)
@@ -427,7 +427,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
427 hf_processor_mm_kwargs: Mapping[str, object],
428 out_mm_kwargs: MultiModalKwargs,
429 ) -> list[PromptReplacement]:
430 - max_image_tokens = get_max_blip2_image_tokens(self.ctx)
430 + max_image_tokens = self._get_num_image_tokens()
431
432 return [
433 PromptReplacement(
@@ -480,7 +480,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
480 )
481
482
483 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens)
483 @MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor)
484 class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
485
vllm/model_executor/models/chameleon.py
+16 -19
@@ -11,7 +11,6 @@ from transformers import (BatchFeature, ChameleonConfig, ChameleonProcessor,
11 from vllm.attention import Attention, AttentionMetadata
12 from vllm.config import CacheConfig, VllmConfig
13 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14 -from vllm.inputs import InputContext
14 from vllm.model_executor.layers.activation import SiluAndMul
15 from vllm.model_executor.layers.layernorm import RMSNorm
16 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -31,7 +30,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
30 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31 MultiModalInputsV2, MultiModalKwargs,
32 NestedTensors, PlaceholderRange)
34 -from vllm.multimodal.parse import MultiModalDataParser
33 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 MultiModalDataItems, ProcessorInputs,
35 PromptReplacement)
@@ -43,11 +41,6 @@ from .utils import (is_pp_missing_parameter,
41 make_empty_intermediate_tensors_factory, make_layers,
42 maybe_prefix, merge_multimodal_embeddings)
43
46 -# These configs are not part of the model config but the preprocessor
47 -# and processor files, so we hardcode them in the model file for now.
48 -CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512
49 -CHAMELEON_IMAGE_SEQ_LENGTH = 1024
50 -
44
45 class ChameleonImagePixelInputs(TypedDict):
46 type: Literal["pixel_values"]
@@ -55,14 +48,17 @@ class ChameleonImagePixelInputs(TypedDict):
48 """Shape: `(batch_size * num_images, num_channels, height, width)`"""
49
50
58 -def get_max_chameleon_image_tokens(ctx: InputContext):
59 - return CHAMELEON_IMAGE_SEQ_LENGTH
51 +class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
52
53 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
54 + return {"image": 1}
55
62 -class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
56 + def _get_num_image_tokens(self) -> int:
57 + processor = self._get_hf_processor()
58 + return processor.image_seq_length
59
64 - def _get_data_parser(self) -> MultiModalDataParser:
65 - return MultiModalDataParser(max_mm_counts={"image": 1})
60 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
61 + return {"image": self._get_num_image_tokens()}
62
63 def _get_hf_processor(self) -> ChameleonProcessor:
64 return self.ctx.get_hf_processor(ChameleonProcessor)
@@ -88,7 +84,7 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
84 target="<image>",
85 replacement="".join([
86 processor.image_start_token,
91 - processor.image_token * CHAMELEON_IMAGE_SEQ_LENGTH,
87 + processor.image_token * self._get_num_image_tokens(),
88 processor.image_end_token,
89 ]),
90 )
@@ -98,12 +94,15 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
94 self,
95 mm_counts: Mapping[str, int],
96 ) -> ProcessorInputs:
97 + config = self.ctx.get_hf_config(ChameleonConfig)
98 +
99 + width = height = config.vq_config.resolution
100 num_images = mm_counts.get("image", 0)
101
102 mm_data = {
103 "image":
105 - self._get_dummy_images(width=CHAMELEON_CROP_SIZE_WIDTH,
106 - height=CHAMELEON_CROP_SIZE_HEIGHT,
104 + self._get_dummy_images(width=width,
105 + height=height,
106 num_images=num_images)
107 }
108
@@ -902,7 +901,6 @@ class ChameleonModel(nn.Module):
901 return hidden_states
902
903
905 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens)
904 @MULTIMODAL_REGISTRY.register_processor(ChameleonMultiModalProcessor)
905 class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
906 SupportsPP):
@@ -931,9 +929,8 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
929 self.model.make_empty_intermediate_tensors)
930
931 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
934 -
935 - expected_dims = (3, CHAMELEON_CROP_SIZE_HEIGHT,
936 - CHAMELEON_CROP_SIZE_WIDTH)
932 + vq_config: ChameleonVQVAEConfig = self.config.vq_config
933 + expected_dims = (3, vq_config.resolution, vq_config.resolution)
934 actual_dims = tuple(data.shape[1:])
935
936 if actual_dims != expected_dims:
vllm/model_executor/models/fuyu.py
+46 -59
@@ -25,7 +25,6 @@ from transformers import (BatchFeature, FuyuConfig, FuyuImageProcessor,
25
26 from vllm.attention import AttentionMetadata
27 from vllm.config import VllmConfig
28 -from vllm.inputs import InputContext
28 from vllm.model_executor.layers.linear import ColumnParallelLinear
29 from vllm.model_executor.layers.sampler import SamplerOutput
30 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
@@ -34,7 +33,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 MultiModalInputsV2, MultiModalKwargs,
35 NestedTensors, PlaceholderRange)
37 -from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataParser
36 +from vllm.multimodal.parse import ImageProcessorItems, ImageSize
37 from vllm.multimodal.processing import (BaseMultiModalProcessor,
38 MultiModalDataItems, ProcessorInputs,
39 PromptReplacement)
@@ -48,9 +47,6 @@ from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
47 _IMAGE_TOKEN_ID = 71011
48 _NEWLINE_TOKEN_ID = 71019
49
51 -MAX_IMAGE_FEATURE_SIZE_HEIGHT = 1080
52 -MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920
53 -
50
51 class FuyuImagePatchInputs(TypedDict):
52 type: Literal["image_patches"]
@@ -67,43 +63,49 @@ class FuyuImagePatchInputs(TypedDict):
63 """
64
65
70 -def _get_fuyu_num_image_tokens(
71 - image_height: int,
72 - image_width: int,
73 -) -> Tuple[int, int]:
74 - """
75 - Calculate the number of image tokens needed for a given image size.
76 -
77 - The expected Fuyu image prompts can be expressed as:
78 -
79 - .. code-block::
80 - (image_token * ncols + newline_token) * nrows
81 -
82 - Args:
83 - image_size: Tuple[int, int] - `(width, height)` of the image
84 -
85 - Returns:
86 - ncols: int - number of image tokens in `x` direction
87 - nrows: int - number of image tokens in `y` direction
88 - """
89 - ncols = math.ceil(image_width / 30)
90 - nrows = math.ceil(image_height / 30)
91 - return ncols, nrows
92 -
66 +class FuyuMultiModalProcessor(BaseMultiModalProcessor):
67
94 -def get_max_fuyu_image_tokens(ctx: InputContext):
95 - ncols, nrows = _get_fuyu_num_image_tokens(
96 - image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
97 - image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
98 - )
68 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
69 + return {"image": 1}
70
100 - return (ncols + 1) * nrows
71 + def _get_image_target_size(self) -> ImageSize:
72 + processor = self._get_hf_processor()
73 + image_processor: FuyuImageProcessor = processor.image_processor
74
75 + target_size = image_processor.size
76 + return ImageSize(width=target_size["width"],
77 + height=target_size["height"])
78
103 -class FuyuMultiModalProcessor(BaseMultiModalProcessor):
79 + def _get_image_grid_size(
80 + self,
81 + *,
82 + image_width: int,
83 + image_height: int,
84 + ) -> tuple[int, int]:
85 + target_width, target_height = self._get_image_target_size()
86 +
87 + if not (image_width <= target_width and image_height <= target_height):
88 + height_scale_factor = target_height / image_height
89 + width_scale_factor = target_width / image_width
90 + optimal_scale_factor = min(height_scale_factor, width_scale_factor)
91 +
92 + image_height = int(image_height * optimal_scale_factor)
93 + image_width = int(image_width * optimal_scale_factor)
94 +
95 + ncols = math.ceil(image_width / 30)
96 + nrows = math.ceil(image_height / 30)
97 + return ncols, nrows
98 +
99 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
100 + target_width, target_height = self._get_image_target_size()
101 +
102 + max_ncols, max_nrows = self._get_image_grid_size(
103 + image_width=target_width,
104 + image_height=target_height,
105 + )
106 + max_image_tokens = (max_ncols + 1) * max_nrows
107
105 - def _get_data_parser(self) -> MultiModalDataParser:
106 - return MultiModalDataParser(max_mm_counts={"image": 1})
108 + return {"image": max_image_tokens}
109
110 def _get_hf_processor(self) -> FuyuProcessor:
111 return self.ctx.get_hf_processor(FuyuProcessor)
@@ -166,28 +168,13 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
168 eot_token_id = tokenizer.bos_token_id
169 assert isinstance(eot_token_id, int)
170
169 - hf_processor = self._get_hf_processor()
170 - image_processor: FuyuImageProcessor = hf_processor.image_processor
171 - target_size = image_processor.size
172 - target_height, target_width = (target_size["height"],
173 - target_size["width"])
174 -
171 def get_replacement_fuyu(item_idx: int):
172 images = mm_items.get_items("image", ImageProcessorItems)
173 image_size = images.get_image_size(item_idx)
178 - width, height = image_size.width, image_size.height
179 - if not (width <= target_width and height <= target_height):
180 - height_scale_factor = target_height / height
181 - width_scale_factor = target_width / width
182 - optimal_scale_factor = min(height_scale_factor,
183 - width_scale_factor)
184 -
185 - height = int(height * optimal_scale_factor)
186 - width = int(width * optimal_scale_factor)
187 -
188 - ncols, nrows = _get_fuyu_num_image_tokens(
189 - image_width=width,
190 - image_height=height,
174 +
175 + ncols, nrows = self._get_image_grid_size(
176 + image_width=image_size.width,
177 + image_height=image_size.height,
178 )
179
180 return (([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows +
@@ -225,12 +212,13 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
212 self,
213 mm_counts: Mapping[str, int],
214 ) -> ProcessorInputs:
215 + target_width, target_height = self._get_image_target_size()
216 num_images = mm_counts.get("image", 0)
217
218 mm_data = {
219 "image":
232 - self._get_dummy_images(width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
233 - height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
220 + self._get_dummy_images(width=target_width,
221 + height=target_height,
222 num_images=num_images)
223 }
224
@@ -240,7 +228,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
228 )
229
230
243 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens)
231 @MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor)
232 class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
233
vllm/model_executor/models/llava.py
+6 -2
@@ -119,6 +119,12 @@ def get_max_llava_image_tokens(ctx: InputContext):
119
120 class LlavaMultiModalProcessor(BaseMultiModalProcessor):
121
122 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
123 + return {"image": None}
124 +
125 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
126 + return {"image": get_max_llava_image_tokens(self.ctx)}
127 +
128 def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
129 return self.ctx.get_hf_processor((LlavaProcessor, PixtralProcessor))
130
@@ -324,7 +330,6 @@ def init_vision_tower_for_llava(
330 raise NotImplementedError(msg)
331
332
327 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
333 @MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
334 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
335 # BitandBytes specific attributes
@@ -649,7 +654,6 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
654
655 # To use this model, please use
656 # `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
652 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
657 @MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor)
658 class MantisForConditionalGeneration(LlavaForConditionalGeneration):
659 pass
vllm/model_executor/models/phi3v.py
+25 -20
@@ -23,7 +23,6 @@ from transformers import (BatchFeature, CLIPVisionConfig, PretrainedConfig,
23
24 from vllm.attention import AttentionMetadata
25 from vllm.config import VllmConfig
26 -from vllm.inputs import InputContext
26 from vllm.logger import init_logger
27 from vllm.model_executor.layers.quantization import QuantizationConfig
28 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -306,24 +305,31 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
305 return image_features_hd_newline
306
307
309 -def get_max_phi3v_image_tokens(
310 - ctx: InputContext,
311 - *,
312 - num_crops: Optional[int] = None,
313 -) -> int:
314 - hf_processor_mm_kwargs = {}
315 - if num_crops:
316 - hf_processor_mm_kwargs["num_crops"] = num_crops
308 +class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
309
318 - processor = ctx.get_hf_processor(**hf_processor_mm_kwargs)
310 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
311 + return {"image": None}
312
320 - return processor.calc_num_image_tokens_from_image_size(
321 - width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
322 - height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
323 - )
313 + def _get_num_image_tokens(
314 + self,
315 + *,
316 + image_width: int,
317 + image_height: int,
318 + ) -> int:
319 + processor = self._get_hf_processor()
320 +
321 + return processor.calc_num_image_tokens_from_image_size( # type: ignore
322 + width=image_width,
323 + height=image_height,
324 + )
325
326 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
327 + max_image_tokens = self._get_num_image_tokens(
328 + image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
329 + image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
330 + )
331
326 -class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
332 + return {"image": max_image_tokens}
333
334 def _get_hf_processor(
335 self,
@@ -332,6 +338,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
338 ) -> ProcessorMixin:
339 if num_crops is not None:
340 return self.ctx.get_hf_processor(num_crops=num_crops)
341 +
342 return self.ctx.get_hf_processor()
343
344 def _call_hf_processor(
@@ -375,7 +382,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
382 ) -> list[PromptReplacement]:
383 hf_processor = self._get_hf_processor()
384 image_tokens: list[str] = hf_processor.img_tokens # type: ignore
378 - image_processor = hf_processor.image_processor # type: ignore
385
386 tokenizer = self._get_tokenizer()
387 bos_token_id = tokenizer.bos_token_id
@@ -385,9 +391,9 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
391 images = mm_items.get_items("image", ImageProcessorItems)
392 image_size = images.get_image_size(item_idx)
393
388 - num_tokens = image_processor.calc_num_image_tokens_from_image_size(
389 - width=image_size.width,
390 - height=image_size.height,
394 + num_tokens = self._get_num_image_tokens(
395 + image_width=image_size.width,
396 + image_height=image_size.height,
397 )
398
399 return [_IMAGE_TOKEN_ID] * num_tokens + [bos_token_id]
@@ -467,7 +473,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
473 return result
474
475
470 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
476 @MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
477 class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
478 hf_to_vllm_mapper = WeightsMapper(
vllm/model_executor/models/qwen2_audio.py
+30 -12
@@ -33,13 +33,12 @@ from transformers.models.whisper import WhisperFeatureExtractor
33
34 from vllm.attention import AttentionMetadata
35 from vllm.config import VllmConfig
36 -from vllm.inputs import InputContext
36 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY
39 from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40 NestedTensors)
42 -from vllm.multimodal.parse import MultiModalDataParser
41 +from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataParser
42 from vllm.multimodal.processing import (BaseMultiModalProcessor,
43 MultiModalDataItems, ProcessorInputs,
44 PromptReplacement)
@@ -80,14 +79,17 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
79 return feat_lengths, output_lengths
80
81
83 -def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
84 - hf_config = ctx.get_hf_config(Qwen2AudioConfig)
85 - max_source_position = hf_config.audio_config.max_source_positions
86 - output_lengths = (max_source_position - 2) // 2 + 1
87 - return output_lengths
82 +class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
83
84 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
85 + return {"audio": None}
86
90 -class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
87 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
88 + hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
89 + max_source_positions = hf_config.audio_config.max_source_positions
90 + max_output_lengths = (max_source_positions - 2) // 2 + 1
91 +
92 + return {"audio": max_output_lengths}
93
94 def _get_hf_processor(
95 self,
@@ -157,11 +159,21 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
159 audio_output_lengths = []
160 else:
161 assert isinstance(feature_attention_mask, torch.Tensor)
160 - _, audio_output_lengths = _get_feat_extract_output_lengths(
162 + _, audio_output_lens = _get_feat_extract_output_lengths(
163 feature_attention_mask.sum(-1))
164
165 + audio_output_lengths = audio_output_lens.tolist()
166 +
167 def get_replacement_qwen2_audio(item_idx: int):
164 - return [placeholder] * audio_output_lengths[item_idx]
168 + num_placeholders = audio_output_lengths[item_idx]
169 + if num_placeholders == 0:
170 + audios = mm_items.get_items("audio", AudioProcessorItems)
171 + audio = audios.get(item_idx)
172 + raise ValueError(
173 + f"The audio {audio} (len={len(audio)}) is too short "
174 + "to be represented inside the model")
175 +
176 + return [placeholder] * num_placeholders
177
178 return [
179 PromptReplacement(
@@ -171,6 +183,14 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
183 )
184 ]
185
186 + def _always_apply_prompt_replacements(self) -> bool:
187 + # HF never applies prompt replacements, so we have to do it ourselves
188 + # _find_placeholders may incorrectly think that HF has already performed
189 + # processing for multi-audio input when the input audios are short
190 + # (the corresponding placeholders may take up fewer tokens than
191 + # the number of audio items)
192 + return True
193 +
194 def _get_dummy_mm_inputs(
195 self,
196 mm_counts: Mapping[str, int],
@@ -192,8 +212,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
212 )
213
214
195 -@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
196 - "audio", get_max_qwen2_audio_audio_tokens)
215 @MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)
216 class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
217 SupportsPP):
vllm/model_executor/models/qwen2_vl.py
+33 -42
@@ -40,7 +40,6 @@ from vllm.attention import AttentionMetadata
40 from vllm.config import VllmConfig
41 from vllm.distributed import parallel_state
42 from vllm.distributed import utils as dist_utils
43 -from vllm.inputs import InputContext
43 from vllm.logger import init_logger
44 from vllm.model_executor import SamplingMetadata
45 from vllm.model_executor.layers.activation import QuickGELU
@@ -650,8 +649,9 @@ def _get_vision_info(
649 width: int,
650 min_pixels: int,
651 max_pixels: int,
652 + *,
653 do_resize: bool = True,
654 - data_type_key: str = "image",
654 + modality: str = "image",
655 mm_count: int = 1,
656 ):
657 """Get information (resized height / width and number of vision tokens)
@@ -671,11 +671,12 @@ def _get_vision_info(
671 else:
672 resized_height, resized_width = height, width
673
674 - if data_type_key == "image":
674 + if modality == "image":
675 grid_t = mm_count
676 - else:
677 - assert data_type_key == "video"
676 + elif modality == "video":
677 grid_t = max(mm_count // temporal_patch_size, 1)
678 + else:
679 + raise ValueError(f"Modality {modality} is not supported")
680
681 grid_h = resized_height // patch_size
682 grid_w = resized_width // patch_size
@@ -691,41 +692,11 @@ def _get_image_processor(hf_processor: Qwen2VLProcessor):
692 return image_processor
693
694
694 -def get_max_qwen2_vl_mm_tokens(ctx: InputContext,
695 - data_type_key: str,
696 - *,
697 - min_pixels: Optional[int] = None,
698 - max_pixels: Optional[int] = None) -> int:
699 - hf_config = ctx.get_hf_config(Qwen2VLConfig)
700 - vision_config = hf_config.vision_config
701 -
702 - hf_processor = ctx.get_hf_processor(Qwen2VLProcessor)
703 - image_processor = _get_image_processor(hf_processor)
704 -
705 - _, _, max_llm_image_tokens = _get_vision_info(
706 - vision_config,
707 - height=9999999,
708 - width=9999999,
709 - min_pixels=min_pixels or image_processor.min_pixels,
710 - max_pixels=max_pixels or image_processor.max_pixels,
711 - data_type_key=data_type_key,
712 - )
713 - return max_llm_image_tokens
714 -
715 -
716 -get_max_qwen2_vl_image_tokens = partial(get_max_qwen2_vl_mm_tokens,
717 - data_type_key="image")
718 -get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
719 - data_type_key="video")
720 -
721 -
695 class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
696 dict[str, torch.Tensor]]):
697
698 def __init__(self, data: dict, modality: str) -> None:
726 - super().__init__(data)
727 -
728 - self.modality = modality
699 + super().__init__(data, modality)
700
701 grid_thw = data[f"{modality}_grid_thw"]
702 slice_idxs = [0] + grid_thw.prod(-1).cumsum_(0).tolist()
@@ -734,9 +705,6 @@ class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
705 for i in range(len(grid_thw))
706 ]
707
737 - def __repr__(self) -> str:
738 - return (f"{type(self).__name__}(modality={self.modality!r})")
739 -
708 def get_count(self) -> int:
709 return len(self.data[f"{self.modality}_grid_thw"])
710
@@ -792,6 +760,32 @@ class Qwen2MultiModalDataParser(MultiModalDataParser):
760
761 class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
762
763 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
764 + return {"image": None, "video": None}
765 +
766 + def _get_max_mm_tokens(self, modality: str) -> int:
767 + hf_config = self.ctx.get_hf_config(Qwen2VLConfig)
768 + vision_config = hf_config.vision_config
769 +
770 + hf_processor = self._get_hf_processor()
771 + image_processor = _get_image_processor(hf_processor)
772 +
773 + _, _, max_llm_image_tokens = _get_vision_info(
774 + vision_config,
775 + height=9999999,
776 + width=9999999,
777 + min_pixels=image_processor.min_pixels,
778 + max_pixels=image_processor.max_pixels,
779 + modality=modality,
780 + )
781 + return max_llm_image_tokens
782 +
783 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
784 + return {
785 + "image": self._get_max_mm_tokens("image"),
786 + "video": self._get_max_mm_tokens("video"),
787 + }
788 +
789 def _get_data_parser(self) -> MultiModalDataParser:
790 return Qwen2MultiModalDataParser()
791
@@ -908,9 +902,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
902 )
903
904
911 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_qwen2_vl_image_tokens)
912 -@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
913 - "video", get_max_qwen2_vl_video_tokens)
905 @MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor)
906 class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
907 SupportsLoRA, SupportsPP):
vllm/model_executor/models/ultravox.py
+9 -17
@@ -2,7 +2,7 @@
2 """PyTorch Ultravox model."""
3
4 import math
5 -from functools import cached_property, lru_cache
5 +from functools import cached_property
6 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7 TypedDict, Union)
8
@@ -17,7 +17,6 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
17
18 from vllm.attention import AttentionMetadata
19 from vllm.config import VllmConfig
20 -from vllm.inputs import InputContext
20 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
21 from vllm.model_executor.layers.layernorm import RMSNorm
22 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -58,22 +57,17 @@ UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
57 UltravoxAudioEmbeddingInputs]
58
59
61 -@lru_cache
62 -def cached_feature_extractor(model_id: str) -> WhisperFeatureExtractor:
63 - return WhisperFeatureExtractor.from_pretrained(model_id)
64 -
65 -
66 -def whisper_feature_extractor(ctx: InputContext) -> WhisperFeatureExtractor:
67 - hf_config = ctx.get_hf_config(UltravoxConfig)
68 - return cached_feature_extractor(hf_config.audio_model_id)
69 -
60 +class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
61
71 -def get_ultravox_max_audio_tokens(ctx: InputContext):
72 - feature_extractor = whisper_feature_extractor(ctx)
73 - return math.ceil(feature_extractor.chunk_length * _AUDIO_TOKENS_PER_SECOND)
62 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
63 + return {"audio": None}
64
65 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
66 + feature_extractor = self._get_feature_extractor()
67 + max_audio_tokens = math.ceil(feature_extractor.chunk_length *
68 + _AUDIO_TOKENS_PER_SECOND)
69
76 -class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
70 + return {"audio": max_audio_tokens}
71
72 def _get_hf_processor(
73 self,
@@ -322,8 +316,6 @@ class ModifiedWhisperEncoder(WhisperEncoder):
316 return hidden_states
317
318
325 -@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
326 - "audio", get_ultravox_max_audio_tokens)
319 @MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor)
320 class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
321
vllm/multimodal/parse.py
+8 -39
@@ -21,10 +21,15 @@ _I = TypeVar("_I")
21
22 class ModalityDataItems(ABC, Generic[_T, _I]):
23
24 - def __init__(self, data: _T) -> None:
24 + def __init__(self, data: _T, modality: str) -> None:
25 super().__init__()
26
27 self.data = data
28 + self.modality = modality
29 +
30 + def __repr__(self) -> str:
31 + return (f"{type(self).__name__}(modality={self.modality!r}, "
32 + f"len={len(self)})")
33
34 def __len__(self) -> int:
35 return self.get_count()
@@ -64,14 +69,6 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
69
70 class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
71
67 - def __init__(self, data: Sequence[_T], modality: str) -> None:
68 - super().__init__(data)
69 -
70 - self.modality = modality
71 -
72 - def __repr__(self) -> str:
73 - return (f"{type(self).__name__}(modality={self.modality!r})")
74 -
72 def get_count(self) -> int:
73 return len(self.data)
74
@@ -87,14 +84,6 @@ class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
84
85 class EmbeddingItems(ModalityDataItems[NestedTensors, torch.Tensor]):
86
90 - def __init__(self, data: NestedTensors, modality: str) -> None:
91 - super().__init__(data)
92 -
93 - self.modality = modality
94 -
95 - def __repr__(self) -> str:
96 - return (f"{type(self).__name__}(modality={self.modality!r})")
97 -
87 def get_count(self) -> int:
88 return len(self.data)
89
@@ -222,22 +211,13 @@ class MultiModalDataParser:
211 Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
212
213 Args:
225 - max_mm_counts (Mapping[str, int]): The maximum allowed number of items
226 - belonging to each modality. This effectively sets a hard limit over
227 - `--limit-mm-per-prompt`.
214 target_sr (float, optional): Enables automatic resampling of audio
215 items to the model's expected sampling rate.
216 """
217
232 - def __init__(
233 - self,
234 - *,
235 - max_mm_counts: Mapping[str, int] = {},
236 - target_sr: Optional[float] = None,
237 - ) -> None:
218 + def __init__(self, *, target_sr: Optional[float] = None) -> None:
219 super().__init__()
220
240 - self.max_mm_counts = max_mm_counts
221 self.target_sr = target_sr
222
223 def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
@@ -345,7 +325,6 @@ class MultiModalDataParser:
325
326 def parse_mm_data(self,
327 mm_data: MultiModalDataDict) -> MultiModalDataItems:
348 - max_mm_counts = self.max_mm_counts
328 subparsers = self._get_subparsers()
329
330 mm_items = MultiModalDataItems()
@@ -353,16 +332,6 @@ class MultiModalDataParser:
332 if k not in subparsers:
333 raise ValueError(f"Unsupported modality: {k}")
334
356 - modality_items = subparsers[k](v)
357 -
358 - if k in max_mm_counts:
359 - max_count = max_mm_counts[k]
360 - if len(modality_items) > max_count:
361 - raise ValueError(
362 - f"This model supports at most {max_count} {k} items "
363 - f"per prompt, but {len(modality_items)} {k} items "
364 - "were given or set as its limit_mm_per_prompt.")
365 -
366 - mm_items[k] = modality_items
335 + mm_items[k] = subparsers[k](v)
336
337 return mm_items
vllm/multimodal/processing.py
+98 -13
@@ -624,6 +624,29 @@ class BaseMultiModalProcessor(ABC):
624 ) -> MultiModalInputsV2:
625 return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
626
627 + @abstractmethod
628 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
629 + """
630 + Return the maximum supported number of items for each modality.
631 +
632 + A value of `None` means unlimited number of items.
633 +
634 + Omitting a modality from the returned dictionary means that
635 + it is not supported at all.
636 + """
637 + raise NotImplementedError
638 +
639 + @abstractmethod
640 + def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
641 + """
642 + Get the maximum possible number of tokens per data item
643 + for each modality.
644 +
645 + The dictionary returned by this method should have the same
646 + keys as that returned by :meth:`get_supported_mm_limits`.
647 + """
648 + raise NotImplementedError
649 +
650 def _get_data_parser(self) -> MultiModalDataParser:
651 """
652 Construct a data parser to preprocess multi-modal data items
@@ -653,7 +676,18 @@ class BaseMultiModalProcessor(ABC):
676 before passing them to :meth:`_get_hf_mm_data`.
677 """
678 parser = self._get_data_parser()
656 - return parser.parse_mm_data(mm_data)
679 + mm_items = parser.parse_mm_data(mm_data)
680 +
681 + mm_limits = self.ctx.get_mm_config().limit_per_prompt
682 + for modality, items in mm_items.items():
683 + limit = mm_limits.get(modality, 1)
684 + if len(items) > limit:
685 + raise ValueError(
686 + f"You set {modality}={limit} (or defaulted to 1) in "
687 + f"`--limit-mm-per-prompt`, but passed {len(items)} "
688 + f"{modality} items in the same prompt.")
689 +
690 + return mm_items
691
692 @abstractmethod
693 def _get_mm_fields_config(
@@ -901,6 +935,17 @@ class BaseMultiModalProcessor(ABC):
935
936 return [prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls]
937
938 + def _always_apply_prompt_replacements(self) -> bool:
939 + """
940 + A flag which can be overridden so that
941 + :meth:`_apply_prompt_replacements` is always called even if we
942 + detect that HF has performed processing via :meth:`_find_placeholders`.
943 +
944 + This is useful in cases where :meth:`_find_placeholders` cannot be
945 + reliably used to detect whether HF has performed processing or not.
946 + """
947 + return False
948 +
949 def _apply_prompt_replacements(
950 self,
951 token_ids: list[int],
@@ -995,7 +1040,7 @@ class BaseMultiModalProcessor(ABC):
1040 all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
1041 mm_item_counts)
1042
998 - if all_placeholders:
1043 + if all_placeholders and not self._always_apply_prompt_replacements():
1044 tokenizer = self._get_tokenizer()
1045 prompt_text = _decode(tokenizer, prompt_ids)
1046 else:
@@ -1009,10 +1054,27 @@ class BaseMultiModalProcessor(ABC):
1054 mm_item_counts,
1055 )
1056
1012 - mm_placeholders = {
1013 - modality: [item.to_range() for item in items]
1014 - for modality, items in full_groupby_modality(all_placeholders)
1015 - }
1057 + mm_placeholders = dict[str, list[PlaceholderRange]]()
1058 + err_suffix = ("This suggests a problem with your implementation of "
1059 + "the merged multi-modal processor for this model, "
1060 + "particularly in the `_get_prompt_replacements` method.")
1061 +
1062 + for modality, placeholders in full_groupby_modality(all_placeholders):
1063 + if modality not in mm_items:
1064 + raise AssertionError(
1065 + f"Expected no placeholders for {modality=}, "
1066 + f"but found {placeholders=}. Input items: {mm_items}"
1067 + f"\n{err_suffix}")
1068 +
1069 + if len(placeholders) != len(mm_items[modality]):
1070 + raise AssertionError(
1071 + f"Expected length of {placeholders=} for {modality=} "
1072 + f"to equal that of input items: {mm_items[modality]}"
1073 + f"\n{err_suffix}")
1074 +
1075 + mm_placeholders[modality] = [
1076 + item.to_range() for item in placeholders
1077 + ]
1078
1079 return MultiModalInputsV2(
1080 type="multimodal",
@@ -1063,15 +1125,38 @@ class BaseMultiModalProcessor(ABC):
1125 """
1126 raise NotImplementedError
1127
1066 - def get_dummy_data(
1067 - self,
1068 - seq_len: int,
1069 - mm_counts: Mapping[str, int],
1070 - mm_max_tokens: Mapping[str, int],
1071 - ) -> DummyData:
1128 + def _get_and_validate_dummy_mm_counts(self) -> Mapping[str, int]:
1129 + mm_limit_per_prompt = self.ctx.get_mm_config().limit_per_prompt
1130 + supported_mm_limits = self.get_supported_mm_limits()
1131 +
1132 + mm_limits = {
1133 + modality: mm_limit_per_prompt.get(modality, 1)
1134 + for modality in supported_mm_limits
1135 + }
1136 +
1137 + for modality, supported_limit in supported_mm_limits.items():
1138 + limit = mm_limits[modality]
1139 + if supported_limit is not None and supported_limit < limit:
1140 + raise ValueError(
1141 + f"You set {modality}={limit} (or defaulted to 1) in "
1142 + f"`--limit-mm-per-prompt`, but this model only supports "
1143 + f"at most {supported_limit} {modality} items.")
1144 +
1145 + return mm_limits
1146 +
1147 + def get_dummy_data(self, seq_len: int) -> DummyData:
1148 # Avoid circular import
1149 from vllm.sequence import SequenceData
1150
1151 + mm_counts = self._get_and_validate_dummy_mm_counts()
1152 + mm_max_tokens_per_item = self.get_mm_max_tokens_per_item()
1153 + if mm_counts.keys() != mm_max_tokens_per_item.keys():
1154 + raise AssertionError(
1155 + "The keys returned by `get_supported_mm_limits`"
1156 + f"({set(mm_counts.keys())}) should be the same as those "
1157 + "returned by `get_mm_max_tokens_per_item` "
1158 + f"({set(mm_max_tokens_per_item.keys())})")
1159 +
1160 processor_inputs = self._get_dummy_mm_inputs(mm_counts)
1161 mm_inputs = self.apply(
1162 prompt_text=processor_inputs.prompt_text,
@@ -1087,7 +1172,7 @@ class BaseMultiModalProcessor(ABC):
1172 for modality, placeholders in placeholders_by_modality.items()
1173 }
1174 expected_placeholders_by_modality = {
1090 - modality: mm_max_tokens[modality]
1175 + modality: mm_max_tokens_per_item[modality] * mm_counts[modality]
1176 for modality in placeholders_by_modality
1177 }
1178 if total_placeholders_by_modality != expected_placeholders_by_modality:
vllm/multimodal/registry.py
+5
@@ -15,6 +15,7 @@ from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
15 from .image import ImagePlugin
16 from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
17 from .processing import BaseMultiModalProcessor, ProcessingCache
18 +from .utils import cached_get_tokenizer
19 from .video import VideoPlugin
20
21 if TYPE_CHECKING:
@@ -219,6 +220,10 @@ class MultiModalRegistry:
220 Note:
221 This is currently directly used only in V1.
222 """
223 + if self.has_processor(model_config):
224 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
225 + processor = self.create_processor(model_config, tokenizer)
226 + return processor.get_mm_max_tokens_per_item()
227
228 return {
229 key: plugin.get_max_multimodal_tokens(model_config)