@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 56d4aefa33
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 14:45 UTC

[VLM] Avoid unnecessary dummy multimodal data during processing (#16416)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Apr 11, 2025 at 03:32 UTC 56d4aefa33f3f8ffaf74d02a8d7eef9523651864
33 files changed +434 -392
vllm/model_executor/models/aria.py
+14 -13
@@ -21,12 +21,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21 from vllm.model_executor.model_loader.weight_utils import (
22 default_weight_loader, maybe_remap_kv_scale_name)
23 from vllm.multimodal import MULTIMODAL_REGISTRY
24 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
24 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25 + MultiModalKwargs)
26 from vllm.multimodal.parse import MultiModalDataItems
27 from vllm.multimodal.processing import (BaseMultiModalProcessor,
28 BaseProcessingInfo, PromptReplacement,
29 PromptUpdate)
29 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
31 from vllm.sequence import IntermediateTensors
32
33 # yapf: disable
@@ -415,31 +416,31 @@ class AriaProcessingInfo(BaseProcessingInfo):
416
417 class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
418
418 - def get_dummy_processor_inputs(
419 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
420 + num_images = mm_counts.get("image", 0)
421 +
422 + processor = self.info.get_hf_processor()
423 + image_token: str = processor.tokenizer.image_token # type: ignore
424 +
425 + return image_token * num_images
426 +
427 + def get_dummy_mm_data(
428 self,
429 seq_len: int,
430 mm_counts: Mapping[str, int],
422 - ) -> ProcessorInputs:
431 + ) -> MultiModalDataDict:
432 vision_config = self.info.get_vision_config()
433
434 max_image_size = vision_config.image_size
435 num_images = mm_counts.get("image", 0)
436
428 - mm_data = {
437 + return {
438 "image":
439 self._get_dummy_images(width=max_image_size,
440 height=max_image_size,
441 num_images=num_images)
442 }
443
435 - hf_processor = self.info.get_hf_processor()
436 - image_token: str = hf_processor.tokenizer.image_token # type: ignore
437 -
438 - return ProcessorInputs(
439 - prompt_text=image_token * num_images,
440 - mm_data=mm_data,
441 - )
442 -
444
445 class AriaMultiModalProcessor(BaseMultiModalProcessor[AriaProcessingInfo]):
446
vllm/model_executor/models/aya_vision.py
+13 -12
@@ -20,7 +20,7 @@ from vllm.jsontree import json_map_leaves
20 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
21 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 from vllm.multimodal import MULTIMODAL_REGISTRY
23 -from vllm.multimodal.inputs import MultiModalKwargs
23 +from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargs
24 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
25 MultiModalDataItems)
26 from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -28,7 +28,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
28 MultiModalFieldConfig,
29 PromptReplacement, PromptUpdate,
30 PromptUpdateDetails)
31 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
32 from vllm.sequence import IntermediateTensors
33
34 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -146,28 +146,29 @@ class AyaVisionProcessingInfo(BaseProcessingInfo):
146 class AyaVisionDummyInputsBuilder(
147 BaseDummyInputsBuilder[AyaVisionProcessingInfo]):
148
149 - def get_dummy_processor_inputs(
150 - self,
151 - seq_len: int,
152 - mm_counts: Mapping[str, int],
153 - ) -> ProcessorInputs:
149 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
150 + num_images = mm_counts.get("image", 0)
151 +
152 processor = self.info.get_hf_processor()
153 image_token = processor.image_token
154
155 + return image_token * num_images
156 +
157 + def get_dummy_mm_data(
158 + self,
159 + seq_len: int,
160 + mm_counts: Mapping[str, int],
161 + ) -> MultiModalDataDict:
162 num_images = mm_counts.get("image", 0)
163 image_size = \
164 self.info.get_image_size_with_most_features()
165
161 - mm_data = {
166 + return {
167 "image":
168 self._get_dummy_images(width=image_size.width,
169 height=image_size.height,
170 num_images=num_images)
171 }
167 - return ProcessorInputs(
168 - prompt_text=image_token * num_images,
169 - mm_data=mm_data,
170 - )
172
173
174 class AyaVisionMultiModalProcessor(
vllm/model_executor/models/blip2.py
+9 -10
@@ -15,12 +15,13 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
15 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
18 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
19 + MultiModalKwargs)
20 from vllm.multimodal.parse import MultiModalDataItems
21 from vllm.multimodal.processing import (BaseMultiModalProcessor,
22 BaseProcessingInfo, PromptIndexTargets,
23 PromptInsertion, PromptUpdate)
23 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
24 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
25 from vllm.sequence import IntermediateTensors
26
27 from .blip import BlipVisionModel
@@ -413,29 +414,27 @@ class Blip2ProcessingInfo(BaseProcessingInfo):
414
415 class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
416
416 - def get_dummy_processor_inputs(
417 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
418 + return ""
419 +
420 + def get_dummy_mm_data(
421 self,
422 seq_len: int,
423 mm_counts: Mapping[str, int],
420 - ) -> ProcessorInputs:
424 + ) -> MultiModalDataDict:
425 hf_config = self.info.get_hf_config()
426 vision_config = hf_config.vision_config
427
428 max_image_size = vision_config.image_size
429 num_images = mm_counts.get("image", 0)
430
427 - mm_data = {
431 + return {
432 "image":
433 self._get_dummy_images(width=max_image_size,
434 height=max_image_size,
435 num_images=num_images)
436 }
437
434 - return ProcessorInputs(
435 - prompt_text="",
436 - mm_data=mm_data,
437 - )
438 -
438
439 class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
440
vllm/model_executor/models/chameleon.py
+14 -10
@@ -30,12 +30,13 @@ from vllm.model_executor.model_loader.weight_utils import (
30 from vllm.model_executor.sampling_metadata import SamplingMetadata
31 from vllm.model_executor.utils import set_weight_attrs
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
33 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 + MultiModalKwargs)
35 from vllm.multimodal.parse import MultiModalDataItems
36 from vllm.multimodal.processing import (BaseMultiModalProcessor,
37 BaseProcessingInfo, PromptReplacement,
38 PromptUpdate, PromptUpdateDetails)
38 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
39 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
40 from vllm.sequence import IntermediateTensors
41
42 from .interfaces import (MultiModalEmbeddings, SupportsMultiModal, SupportsPP,
@@ -72,28 +73,31 @@ class ChameleonProcessingInfo(BaseProcessingInfo):
73 class ChameleonDummyInputsBuilder(
74 BaseDummyInputsBuilder[ChameleonProcessingInfo]):
75
75 - def get_dummy_processor_inputs(
76 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
77 + num_images = mm_counts.get("image", 0)
78 +
79 + processor = self.info.get_hf_processor()
80 + image_token = processor.image_token
81 +
82 + return image_token * num_images
83 +
84 + def get_dummy_mm_data(
85 self,
86 seq_len: int,
87 mm_counts: Mapping[str, int],
79 - ) -> ProcessorInputs:
88 + ) -> MultiModalDataDict:
89 config = self.info.get_hf_config()
90
91 width = height = config.vq_config.resolution
92 num_images = mm_counts.get("image", 0)
93
85 - mm_data = {
94 + return {
95 "image":
96 self._get_dummy_images(width=width,
97 height=height,
98 num_images=num_images)
99 }
100
92 - return ProcessorInputs(
93 - prompt_text="<image>" * num_images,
94 - mm_data=mm_data,
95 - )
96 -
101
102 class ChameleonMultiModalProcessor(
103 BaseMultiModalProcessor[ChameleonProcessingInfo]):
vllm/model_executor/models/deepseek_vl2.py
+14 -13
@@ -19,14 +19,14 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
19 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20 from vllm.model_executor.model_loader.utils import set_default_torch_dtype
21 from vllm.multimodal import MULTIMODAL_REGISTRY
22 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
23 - NestedTensors)
22 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
23 + MultiModalKwargs, NestedTensors)
24 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
25 ImageSize, MultiModalDataItems)
26 from vllm.multimodal.processing import (BaseMultiModalProcessor,
27 BaseProcessingInfo, PromptReplacement,
28 PromptUpdate)
29 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
29 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
30 from vllm.sequence import IntermediateTensors
31 from vllm.transformers_utils.configs.deepseek_vl2 import (DeepseekVLV2Config,
32 MlpProjectorConfig,
@@ -172,29 +172,30 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
172 class DeepseekVL2DummyInputsBuilder(
173 BaseDummyInputsBuilder[DeepseekVL2ProcessingInfo]):
174
175 - def get_dummy_processor_inputs(
175 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
176 + num_images = mm_counts.get("image", 0)
177 +
178 + processor = self.info.get_hf_processor()
179 + image_token = processor.image_token
180 +
181 + return image_token * num_images
182 +
183 + def get_dummy_mm_data(
184 self,
185 seq_len: int,
186 mm_counts: Mapping[str, int],
179 - ) -> ProcessorInputs:
187 + ) -> MultiModalDataDict:
188 num_images = mm_counts.get("image", 0)
181 - hf_processor = self.info.get_hf_processor()
182 - image_token: str = hf_processor.image_token
189
190 max_image_size = self.info.get_image_size_with_most_features()
191
186 - mm_data = {
192 + return {
193 "image":
194 self._get_dummy_images(width=max_image_size.width,
195 height=max_image_size.height,
196 num_images=num_images)
197 }
198
193 - return ProcessorInputs(
194 - prompt_text=image_token * num_images,
195 - mm_data=mm_data,
196 - )
197 -
199
200 class DeepseekVL2MultiModalProcessor(
201 BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]):
vllm/model_executor/models/florence2.py
+10 -11
@@ -21,13 +21,14 @@ from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
21 BartScaledWordEmbedding)
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.multimodal import MULTIMODAL_REGISTRY
24 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
25 -from vllm.multimodal.parse import MultiModalDataDict, MultiModalDataItems
24 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25 + MultiModalKwargs)
26 +from vllm.multimodal.parse import MultiModalDataItems
27 from vllm.multimodal.processing import (BaseProcessingInfo,
28 EncDecMultiModalProcessor,
29 PromptIndexTargets, PromptInsertion,
30 PromptUpdate)
30 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
32 from vllm.sequence import IntermediateTensors
33
34 from .interfaces import (MultiModalEmbeddings, SupportsMultiModal,
@@ -772,27 +773,25 @@ class Florence2ProcessingInfo(BaseProcessingInfo):
773 class Florence2DummyInputsBuilder(
774 BaseDummyInputsBuilder[Florence2ProcessingInfo]):
775
775 - def get_dummy_processor_inputs(
776 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
777 + return ""
778 +
779 + def get_dummy_mm_data(
780 self,
781 seq_len: int,
782 mm_counts: Mapping[str, int],
779 - ) -> ProcessorInputs:
783 + ) -> MultiModalDataDict:
784 num_images = mm_counts.get("image", 0)
785
786 target_width = target_height = self.info.get_hf_config().projection_dim
787
784 - mm_data = {
788 + return {
789 "image":
790 self._get_dummy_images(width=target_width,
791 height=target_height,
792 num_images=num_images)
793 }
794
791 - return ProcessorInputs(
792 - prompt_text="",
793 - mm_data=mm_data,
794 - )
795 -
795
796 class Florence2MultiModalProcessor(
797 EncDecMultiModalProcessor[Florence2ProcessingInfo]):
vllm/model_executor/models/fuyu.py
+9 -10
@@ -31,13 +31,14 @@ from vllm.model_executor.layers.sampler import SamplerOutput
31 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
32 from vllm.model_executor.sampling_metadata import SamplingMetadata
33 from vllm.multimodal import MULTIMODAL_REGISTRY
34 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
34 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35 + MultiModalKwargs)
36 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37 MultiModalDataItems)
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 BaseProcessingInfo, PromptReplacement,
40 PromptUpdate, PromptUpdateDetails)
40 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
41 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
42 from vllm.sequence import IntermediateTensors
43
44 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -125,27 +126,25 @@ class FuyuProcessingInfo(BaseProcessingInfo):
126
127 class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
128
128 - def get_dummy_processor_inputs(
129 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
130 + return ""
131 +
132 + def get_dummy_mm_data(
133 self,
134 seq_len: int,
135 mm_counts: Mapping[str, int],
132 - ) -> ProcessorInputs:
136 + ) -> MultiModalDataDict:
137 target_width, target_height = \
138 self.info.get_image_size_with_most_features()
139 num_images = mm_counts.get("image", 0)
140
137 - mm_data = {
141 + return {
142 "image":
143 self._get_dummy_images(width=target_width,
144 height=target_height,
145 num_images=num_images)
146 }
147
144 - return ProcessorInputs(
145 - prompt_text="",
146 - mm_data=mm_data,
147 - )
148 -
148
149 class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
150
vllm/model_executor/models/gemma3_mm.py
+15 -14
@@ -15,8 +15,9 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm
15 from vllm.model_executor.layers.sampler import SamplerOutput
16 from vllm.model_executor.models.module_mapping import MultiModelKeys
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
19 -from vllm.multimodal.inputs import MultiModalFieldConfig
18 +from vllm.multimodal import MULTIMODAL_REGISTRY
19 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
20 + MultiModalKwargs)
21 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
22 MultiModalDataItems)
23 # yapf: disable
@@ -28,7 +29,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 find_mm_placeholders,
30 replace_token_matches)
31 # yapf: enable
31 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
33 from vllm.sequence import IntermediateTensors
34
35 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -224,31 +225,31 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
225
226 class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
227
227 - def get_dummy_processor_inputs(
228 - self,
229 - seq_len: int,
230 - mm_counts: Mapping[str, int],
231 - ) -> ProcessorInputs:
228 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
229 + num_images = mm_counts.get("image", 0)
230 +
231 processor = self.info.get_hf_processor()
232 image_token = processor.boi_token
233
234 + return image_token * num_images
235 +
236 + def get_dummy_mm_data(
237 + self,
238 + seq_len: int,
239 + mm_counts: Mapping[str, int],
240 + ) -> MultiModalDataDict:
241 num_images = mm_counts.get("image", 0)
242
243 target_width, target_height = \
244 self.info.get_image_size_with_most_features()
245
240 - mm_data = {
246 + return {
247 "image":
248 self._get_dummy_images(width=target_width,
249 height=target_height,
250 num_images=num_images)
251 }
252
247 - return ProcessorInputs(
248 - prompt_text=image_token * num_images,
249 - mm_data=mm_data,
250 - )
251 -
253
254 class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
255
vllm/model_executor/models/glm4v.py
+16 -16
@@ -12,7 +12,7 @@ from torch import nn
12 from torch.nn import LayerNorm
13 from torchvision import transforms
14 from torchvision.transforms import InterpolationMode
15 -from transformers import PreTrainedTokenizer, TensorType
15 +from transformers import BatchFeature, PreTrainedTokenizer, TensorType
16 from transformers.image_utils import ImageInput
17 from transformers.tokenization_utils_base import TextInput
18
@@ -28,13 +28,13 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28 from vllm.model_executor.layers.quantization import QuantizationConfig
29 from vllm.model_executor.models.module_mapping import MultiModelKeys
30 from vllm.multimodal import MULTIMODAL_REGISTRY
31 -from vllm.multimodal.inputs import MultiModalKwargs
31 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32 + MultiModalKwargs)
33 from vllm.multimodal.parse import MultiModalDataItems
34 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 - BaseProcessingInfo, BatchFeature,
35 - MultiModalFieldConfig,
36 - PromptReplacement, PromptUpdate)
37 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
35 + BaseProcessingInfo, PromptReplacement,
36 + PromptUpdate)
37 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
38 from vllm.sequence import IntermediateTensors
39 from vllm.transformers_utils.configs import ChatGLMConfig
40
@@ -447,31 +447,31 @@ class GLM4VProcessingInfo(BaseProcessingInfo):
447
448 class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
449
450 - def get_dummy_processor_inputs(
450 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
451 + num_images = mm_counts.get("image", 0)
452 +
453 + base_text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
454 +
455 + return base_text * num_images
456 +
457 + def get_dummy_mm_data(
458 self,
459 seq_len: int,
460 mm_counts: Mapping[str, int],
454 - ) -> ProcessorInputs:
461 + ) -> MultiModalDataDict:
462 hf_config = self.info.get_hf_config()
463 vision_config = hf_config.vision_config
464
465 target_width = target_height = vision_config["image_size"]
466 num_images = mm_counts.get("image", 0)
467
461 - mm_data = {
468 + return {
469 "image":
470 self._get_dummy_images(width=target_width,
471 height=target_height,
472 num_images=num_images)
473 }
474
468 - base_text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
469 -
470 - return ProcessorInputs(
471 - prompt_text=base_text * num_images,
472 - mm_data=mm_data,
473 - )
474 -
475
476 class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
477
vllm/model_executor/models/idefics3.py
+17 -15
@@ -32,18 +32,18 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
32 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
33 from vllm.model_executor.models.module_mapping import MultiModelKeys
34 from vllm.model_executor.sampling_metadata import SamplingMetadata
35 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
35 +from vllm.multimodal import MULTIMODAL_REGISTRY
36 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
37 + MultiModalKwargs)
38 from vllm.multimodal.parse import ImageProcessorItems, ImageSize
39 # yapf conflicts with isort for this block
40 # yapf: disable
41 from vllm.multimodal.processing import (BaseMultiModalProcessor,
42 BaseProcessingInfo,
41 - MultiModalDataItems,
42 - MultiModalFieldConfig,
43 - PromptReplacement, PromptUpdate,
44 - PromptUpdateDetails)
43 + MultiModalDataItems, PromptReplacement,
44 + PromptUpdate, PromptUpdateDetails)
45 # yapf: enable
46 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
46 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
47 from vllm.sequence import IntermediateTensors
48
49 # yapf: disable
@@ -284,29 +284,31 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
284 class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]
285 ):
286
287 - def get_dummy_processor_inputs(
287 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
288 + num_images = mm_counts.get("image", 0)
289 +
290 + processor = self.info.get_hf_processor()
291 + image_token, _, _ = self.info._get_image_token(processor)
292 +
293 + return image_token * num_images
294 +
295 + def get_dummy_mm_data(
296 self,
297 seq_len: int,
298 mm_counts: Mapping[str, int],
291 - ) -> ProcessorInputs:
299 + ) -> MultiModalDataDict:
300 num_images = mm_counts.get("image", 0)
301 hf_processor = self.info.get_hf_processor()
302 image_processor: Idefics3ImageProcessor = hf_processor.image_processor
303 longest_edge = image_processor.max_image_size['longest_edge']
296 - image_token, _, _ = self.info._get_image_token(hf_processor)
304
298 - mm_data = {
305 + return {
306 "image":
307 self._get_dummy_images(width=longest_edge,
308 height=longest_edge,
309 num_images=num_images)
310 }
311
305 - return ProcessorInputs(
306 - prompt_text=image_token * num_images,
307 - mm_data=mm_data,
308 - )
309 -
312
313 class Idefics3MultiModalProcessor(
314 BaseMultiModalProcessor[Idefics3ProcessingInfo]):
vllm/model_executor/models/internvl.py
+11 -11
@@ -25,14 +25,14 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,
25 InternVisionPatchModel)
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.multimodal import MULTIMODAL_REGISTRY
28 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
29 - NestedTensors)
28 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
29 + MultiModalKwargs, NestedTensors)
30 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
31 ImageSize, MultiModalDataItems)
32 from vllm.multimodal.processing import (BaseMultiModalProcessor,
33 BaseProcessingInfo, PromptReplacement,
34 PromptUpdate, PromptUpdateDetails)
35 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
35 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
36 from vllm.sequence import IntermediateTensors
37 from vllm.transformers_utils.tokenizer import AnyTokenizer
38
@@ -504,27 +504,27 @@ _I = TypeVar("_I", bound=BaseInternVLProcessingInfo)
504
505 class InternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
506
507 - def get_dummy_processor_inputs(
507 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
508 + num_images = mm_counts.get("image", 0)
509 +
510 + return "<image>" * num_images
511 +
512 + def get_dummy_mm_data(
513 self,
514 seq_len: int,
515 mm_counts: Mapping[str, int],
511 - ) -> ProcessorInputs:
516 + ) -> MultiModalDataDict:
517 target_width, target_height = \
518 self.info.get_image_size_with_most_features()
519 num_images = mm_counts.get("image", 0)
520
516 - mm_data = {
521 + return {
522 "image":
523 self._get_dummy_images(width=target_width,
524 height=target_height,
525 num_images=num_images)
526 }
527
523 - return ProcessorInputs(
524 - prompt_text="<image>" * num_images,
525 - mm_data=mm_data,
526 - )
527 -
528
529 class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
530
vllm/model_executor/models/llava.py
+12 -11
@@ -34,7 +34,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 BaseProcessingInfo, ProcessingCache,
35 PromptReplacement, PromptUpdate,
36 PromptUpdateDetails)
37 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
38 from vllm.sequence import IntermediateTensors
39
40 from .clip import CLIPVisionModel
@@ -186,30 +186,31 @@ _I = TypeVar("_I", bound=BaseLlavaProcessingInfo)
186
187 class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
188
189 - def get_dummy_processor_inputs(
189 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
190 + num_images = mm_counts.get("image", 0)
191 +
192 + processor = self.info.get_hf_processor()
193 + image_token = processor.image_token
194 +
195 + return image_token * num_images
196 +
197 + def get_dummy_mm_data(
198 self,
199 seq_len: int,
200 mm_counts: Mapping[str, int],
193 - ) -> ProcessorInputs:
201 + ) -> MultiModalDataDict:
202 num_images = mm_counts.get("image", 0)
203
196 - processor = self.info.get_hf_processor()
197 - image_token = processor.image_token
204 target_width, target_height = \
205 self.info.get_image_size_with_most_features()
206
201 - mm_data = {
207 + return {
208 "image":
209 self._get_dummy_images(width=target_width,
210 height=target_height,
211 num_images=num_images)
212 }
213
208 - return ProcessorInputs(
209 - prompt_text=image_token * num_images,
210 - mm_data=mm_data,
211 - )
212 -
214
215 class LlavaProcessingInfo(BaseLlavaProcessingInfo):
216
vllm/model_executor/models/llava_next_video.py
+14 -13
@@ -16,13 +16,14 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
16 from vllm.model_executor.models.clip import CLIPVisionModel
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
19 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
20 + MultiModalKwargs)
21 from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
22 VideoEmbeddingItems, VideoProcessorItems)
23 from vllm.multimodal.processing import (BaseMultiModalProcessor,
24 BaseProcessingInfo, PromptReplacement,
25 PromptUpdate)
25 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
26 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
27 from vllm.sequence import IntermediateTensors
28 from vllm.utils import is_list_of
29
@@ -130,22 +131,27 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
131 class LlavaNextVideoDummyInputsBuilder(
132 BaseDummyInputsBuilder[LlavaNextVideoProcessingInfo]):
133
133 - def get_dummy_processor_inputs(
134 - self,
135 - seq_len: int,
136 - mm_counts: Mapping[str, int],
137 - ) -> ProcessorInputs:
134 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
135 num_videos = mm_counts.get("video", 0)
136
137 processor = self.info.get_hf_processor()
138 video_token = processor.video_token
139
140 + return video_token * num_videos
141 +
142 + def get_dummy_mm_data(
143 + self,
144 + seq_len: int,
145 + mm_counts: Mapping[str, int],
146 + ) -> MultiModalDataDict:
147 + num_videos = mm_counts.get("video", 0)
148 +
149 target_width, target_height = \
150 self.info.get_image_size_with_most_features()
151 target_num_frames = \
152 self.info.get_num_frames_with_most_features(seq_len, mm_counts)
153
148 - mm_data = {
154 + return {
155 "video":
156 self._get_dummy_videos(
157 width=target_width,
@@ -155,11 +161,6 @@ class LlavaNextVideoDummyInputsBuilder(
161 )
162 }
163
158 - return ProcessorInputs(
159 - prompt_text=video_token * num_videos,
160 - mm_data=mm_data,
161 - )
162 -
164
165 class LlavaNextVideoMultiModalProcessor(
166 BaseMultiModalProcessor[LlavaNextVideoProcessingInfo]):
vllm/model_executor/models/llava_onevision.py
+14 -13
@@ -19,11 +19,11 @@ from vllm.model_executor.layers.activation import get_act_fn
19 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20 from vllm.model_executor.sampling_metadata import SamplingMetadata
21 from vllm.multimodal import MULTIMODAL_REGISTRY
22 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
22 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
23 + MultiModalKwargs)
24 from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
25 VideoEmbeddingItems, VideoProcessorItems)
26 from vllm.multimodal.processing import PromptReplacement, PromptUpdate
26 -from vllm.multimodal.profiling import ProcessorInputs
27 from vllm.sequence import IntermediateTensors
28
29 from .clip import CLIPVisionModel
@@ -226,11 +226,7 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
226 class LlavaOnevisionDummyInputsBuilder(
227 LlavaDummyInputsBuilder[LlavaOnevisionProcessingInfo]):
228
229 - def get_dummy_processor_inputs(
230 - self,
231 - seq_len: int,
232 - mm_counts: Mapping[str, int],
233 - ) -> ProcessorInputs:
229 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
230 num_images = mm_counts.get("image", 0)
231 num_videos = mm_counts.get("video", 0)
232
@@ -238,13 +234,23 @@ class LlavaOnevisionDummyInputsBuilder(
234 image_token = processor.image_token
235 video_token = processor.video_token
236
237 + return image_token * num_images + video_token * num_videos
238 +
239 + def get_dummy_mm_data(
240 + self,
241 + seq_len: int,
242 + mm_counts: Mapping[str, int],
243 + ) -> MultiModalDataDict:
244 + num_images = mm_counts.get("image", 0)
245 + num_videos = mm_counts.get("video", 0)
246 +
247 target_width, target_height = \
248 self.info.get_image_size_with_most_features()
249 target_num_frames = \
250 self.info.get_num_frames_with_most_features(seq_len,
251 mm_counts)
252
247 - mm_data = {
253 + return {
254 "image":
255 self._get_dummy_images(width=target_width,
256 height=target_height,
@@ -258,11 +264,6 @@ class LlavaOnevisionDummyInputsBuilder(
264 )
265 }
266
261 - return ProcessorInputs(
262 - prompt_text=image_token * num_images + video_token * num_videos,
263 - mm_data=mm_data,
264 - )
265 -
267
268 class LlavaOnevisionMultiModalProcessor(
269 BaseLlavaNextMultiModalProcessor[LlavaOnevisionProcessingInfo]):
vllm/model_executor/models/minicpmo.py
+18 -16
@@ -35,14 +35,14 @@ from transformers.models.whisper.modeling_whisper import (
35
36 from vllm.config import VllmConfig
37 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
38 -from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
38 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
39 + NestedTensors)
40 from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,
41 DictEmbeddingItems, ModalityData,
42 ModalityDataItems, MultiModalDataItems,
43 MultiModalDataParser)
44 from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,
45 PromptUpdateDetails)
45 -from vllm.multimodal.profiling import ProcessorInputs
46
47 from .minicpmv import (_MAX_FRAMES_PER_VIDEO, MiniCPMV2_6,
48 MiniCPMVDummyInputsBuilder,
@@ -206,29 +206,31 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
206 class MiniCPMODummyInputsBuilder(
207 MiniCPMVDummyInputsBuilder[MiniCPMOProcessingInfo]):
208
209 - def get_dummy_processor_inputs(
210 - self, seq_len: int, mm_counts: Mapping[str,
211 - int]) -> ProcessorInputs:
209 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
210 + num_audios = mm_counts.get("audio", 0)
211 +
212 + audio_prompt_texts = self.info.audio_pattern * num_audios
213 +
214 + return super().get_dummy_text(mm_counts) + audio_prompt_texts
215 +
216 + def get_dummy_mm_data(
217 + self,
218 + seq_len: int,
219 + mm_counts: Mapping[str, int],
220 + ) -> MultiModalDataDict:
221 num_audios = mm_counts.get("audio", 0)
222 audio_len = self.info.get_max_audio_chunks_with_most_features() * \
223 self.info.get_default_audio_sampling_rate()
224
216 - processor_inputs = super().get_dummy_processor_inputs(
217 - seq_len, mm_counts)
218 -
219 - audio_prompt_texts = self.info.audio_pattern * num_audios
225 audio_mm_data = {
226 "audio":
227 self._get_dummy_audios(length=audio_len, num_audios=num_audios)
228 }
229
225 - return ProcessorInputs(
226 - prompt_text=processor_inputs.prompt_text + audio_prompt_texts,
227 - mm_data={
228 - **processor_inputs.mm_data,
229 - **audio_mm_data,
230 - },
231 - )
230 + return {
231 + **super().get_dummy_mm_data(seq_len, mm_counts),
232 + **audio_mm_data,
233 + }
234
235
236 class MiniCPMOMultiModalProcessor(
vllm/model_executor/models/minicpmv.py
+15 -12
@@ -48,7 +48,8 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
48 from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
51 -from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
51 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
52 + NestedTensors)
53 from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
54 ImageProcessorItems, ImageSize,
55 ModalityData, ModalityDataItems,
@@ -57,7 +58,7 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
58 from vllm.multimodal.processing import (BaseMultiModalProcessor,
59 BaseProcessingInfo, PromptReplacement,
60 PromptUpdate, PromptUpdateDetails)
60 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
61 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
62 from vllm.platforms import current_platform
63 from vllm.sequence import IntermediateTensors
64 from vllm.utils import flatten_2d_lists
@@ -471,11 +472,20 @@ _I = TypeVar("_I",
472
473 class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
474
474 - def get_dummy_processor_inputs(
475 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
476 + num_images = mm_counts.get("image", 0)
477 + num_videos = mm_counts.get("video", 0)
478 +
479 + image_prompt_texts = self.info.image_pattern * num_images
480 + video_prompt_texts = self.info.video_pattern * num_videos
481 +
482 + return image_prompt_texts + video_prompt_texts
483 +
484 + def get_dummy_mm_data(
485 self,
486 seq_len: int,
487 mm_counts: Mapping[str, int],
478 - ) -> ProcessorInputs:
488 + ) -> MultiModalDataDict:
489 num_images = mm_counts.get("image", 0)
490 num_videos = mm_counts.get("video", 0)
491
@@ -486,7 +496,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
496 num_video_frames = \
497 self.info.get_num_frames_with_most_features(seq_len, mm_counts)
498
489 - mm_data = {
499 + return {
500 "image":
501 self._get_dummy_images(width=image_width,
502 height=image_height,
@@ -498,13 +508,6 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
508 ] * num_videos,
509 }
510
501 - image_prompt_texts = self.info.image_pattern * num_images
502 - video_prompt_texts = self.info.video_pattern * num_videos
503 -
504 - return ProcessorInputs(prompt_text=image_prompt_texts +
505 - video_prompt_texts,
506 - mm_data=mm_data)
507 -
511
512 class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
513
vllm/model_executor/models/mistral3.py
+14 -12
@@ -22,14 +22,15 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
25 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
26 + MultiModalKwargs)
27 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
28 MultiModalDataItems)
29 from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 BaseProcessingInfo, ProcessingCache,
31 PromptReplacement, PromptUpdate,
32 PromptUpdateDetails)
32 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
33 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
34 from vllm.sequence import IntermediateTensors
35
36 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -185,30 +186,31 @@ _I = TypeVar("_I", bound=BaseLlavaProcessingInfo)
186
187 class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[_I]):
188
188 - def get_dummy_processor_inputs(
189 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
190 + num_images = mm_counts.get("image", 0)
191 +
192 + processor = self.info.get_hf_processor()
193 + image_token = processor.image_token
194 +
195 + return image_token * num_images
196 +
197 + def get_dummy_mm_data(
198 self,
199 seq_len: int,
200 mm_counts: Mapping[str, int],
192 - ) -> ProcessorInputs:
201 + ) -> MultiModalDataDict:
202 num_images = mm_counts.get("image", 0)
203
195 - processor = self.info.get_hf_processor()
196 - image_token = processor.image_token
204 target_width, target_height = \
205 self.info.get_image_size_with_most_features()
206
200 - mm_data = {
207 + return {
208 "image":
209 self._get_dummy_images(width=target_width,
210 height=target_height,
211 num_images=num_images)
212 }
213
207 - return ProcessorInputs(
208 - prompt_text=image_token * num_images,
209 - mm_data=mm_data,
210 - )
211 -
214
215 class Mistral3ProcessingInfo(BaseLlavaProcessingInfo):
216
vllm/model_executor/models/mllama.py
+14 -14
@@ -54,14 +54,14 @@ from vllm.model_executor.model_loader.weight_utils import (
54 default_weight_loader, maybe_remap_kv_scale_name)
55 from vllm.model_executor.sampling_metadata import SamplingMetadata
56 from vllm.multimodal import MULTIMODAL_REGISTRY
57 -from vllm.multimodal.inputs import (MultiModalEncDecInputs,
57 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
58 MultiModalFieldConfig, MultiModalKwargs)
59 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
60 - MultiModalDataDict, MultiModalDataItems)
60 + MultiModalDataItems)
61 from vllm.multimodal.processing import (BaseProcessingInfo,
62 EncDecMultiModalProcessor,
63 PromptReplacement, PromptUpdate)
64 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
64 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
65
66 from .clip import CLIPMLP
67 from .interfaces import SupportsMultiModal, SupportsV0Only
@@ -131,31 +131,31 @@ class MllamaProcessingInfo(BaseProcessingInfo):
131
132 class MllamaDummyInputsBuilder(BaseDummyInputsBuilder[MllamaProcessingInfo]):
133
134 - def get_dummy_processor_inputs(
134 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
135 + num_images = mm_counts.get("image", 0)
136 +
137 + processor = self.info.get_hf_processor()
138 + image_token = processor.image_token
139 +
140 + return image_token * num_images
141 +
142 + def get_dummy_mm_data(
143 self,
144 seq_len: int,
145 mm_counts: Mapping[str, int],
138 - ) -> ProcessorInputs:
146 + ) -> MultiModalDataDict:
147 num_images = mm_counts.get("image", 0)
148
149 target_width, target_height = \
150 self.info.get_image_size_with_most_features()
151
144 - mm_data = {
152 + return {
153 "image":
154 self._get_dummy_images(width=target_width,
155 height=target_height,
156 num_images=num_images)
157 }
158
151 - hf_processor = self.info.get_hf_processor()
152 - image_token: str = hf_processor.image_token
153 -
154 - return ProcessorInputs(
155 - prompt_text=image_token * num_images,
156 - mm_data=mm_data,
157 - )
158 -
159
160 class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
161 ):
vllm/model_executor/models/mllama4.py
+14 -12
@@ -43,14 +43,14 @@ from vllm.model_executor.model_loader.loader import _initialize_model
43 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.multimodal import MULTIMODAL_REGISTRY
46 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
47 - NestedTensors)
46 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
47 + MultiModalKwargs, NestedTensors)
48 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
49 MultiModalDataItems)
50 from vllm.multimodal.processing import (BaseMultiModalProcessor,
51 BaseProcessingInfo, PromptReplacement,
52 PromptUpdate, PromptUpdateDetails)
53 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
53 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
54 from vllm.sequence import IntermediateTensors
55
56 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -619,29 +619,31 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]
619
620 class Mllama4DummyInputsBuilder(BaseDummyInputsBuilder[Mllama4ProcessingInfo]):
621
622 - def get_dummy_processor_inputs(
622 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
623 + num_images = mm_counts.get("image", 0)
624 +
625 + processor = self.info.get_hf_processor()
626 + image_token = processor.fake_image_token
627 +
628 + return image_token * num_images
629 +
630 + def get_dummy_mm_data(
631 self,
632 seq_len: int,
633 mm_counts: Mapping[str, int],
626 - ) -> ProcessorInputs:
634 + ) -> MultiModalDataDict:
635 num_images = mm_counts.get("image", 0)
636
637 (target_width,
638 target_height) = self.info.get_image_size_with_most_features()
639
632 - mm_data = {
640 + return {
641 "image":
642 self._get_dummy_images(width=target_width,
643 height=target_height,
644 num_images=num_images)
645 }
646
639 - image_token = self.info.get_hf_processor().fake_image_token
640 - return ProcessorInputs(
641 - prompt_text=image_token * num_images,
642 - mm_data=mm_data,
643 - )
644 -
647
648 @MULTIMODAL_REGISTRY.register_processor(
649 Mllama4MultiModalProcessor,
vllm/model_executor/models/molmo.py
+9 -10
@@ -41,14 +41,15 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 from vllm.model_executor.models.module_mapping import MultiModelKeys
43 from vllm.multimodal import MULTIMODAL_REGISTRY
44 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
44 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
45 + MultiModalKwargs)
46 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
47 MultiModalDataItems)
48 from vllm.multimodal.processing import (BaseMultiModalProcessor,
49 BaseProcessingInfo, PromptIndexTargets,
50 PromptInsertion, PromptUpdate,
51 PromptUpdateDetails)
51 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
52 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
53 from vllm.sequence import IntermediateTensors
54
55 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -1216,27 +1217,25 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1217
1218 class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
1219
1219 - def get_dummy_processor_inputs(
1220 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
1221 + return ""
1222 +
1223 + def get_dummy_mm_data(
1224 self,
1225 seq_len: int,
1226 mm_counts: Mapping[str, int],
1223 - ) -> ProcessorInputs:
1227 + ) -> MultiModalDataDict:
1228 target_width, target_height = \
1229 self.info.get_image_size_with_most_features()
1230 num_images = mm_counts.get("image", 0)
1231
1228 - mm_data = {
1232 + return {
1233 "image":
1234 self._get_dummy_images(width=target_width,
1235 height=target_height,
1236 num_images=num_images)
1237 }
1238
1235 - return ProcessorInputs(
1236 - prompt_text="",
1237 - mm_data=mm_data,
1238 - )
1239 -
1239
1240 class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1241
vllm/model_executor/models/nvlm_d.py
+11 -12
@@ -15,12 +15,11 @@ from transformers import PretrainedConfig
15
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 -from vllm.multimodal.inputs import MultiModalKwargs
18 +from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargs
19 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
20 MultiModalDataItems)
21 from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,
22 PromptUpdateDetails)
23 -from vllm.multimodal.profiling import ProcessorInputs
23
24 from .intern_vit import InternVisionModel
25 from .internvl import (BaseInternVLProcessingInfo, BaseInternVLProcessor,
@@ -87,29 +86,29 @@ class NVLMProcessingInfo(BaseInternVLProcessingInfo):
86
87 class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):
88
90 - def get_dummy_processor_inputs(
89 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
90 + num_images = mm_counts.get("image", 0)
91 +
92 + # The newline is necessary to separate ">" of the current item
93 + # and "<" of the next item
94 + return "<image>\n" * num_images
95 +
96 + def get_dummy_mm_data(
97 self,
98 seq_len: int,
99 mm_counts: Mapping[str, int],
94 - ) -> ProcessorInputs:
100 + ) -> MultiModalDataDict:
101 target_width, target_height = \
102 self.info.get_image_size_with_most_features()
103 num_images = mm_counts.get("image", 0)
104
99 - mm_data = {
105 + return {
106 "image":
107 self._get_dummy_images(width=target_width,
108 height=target_height,
109 num_images=num_images)
110 }
111
106 - return ProcessorInputs(
107 - # The newline is necessary to separate ">" of the current item
108 - # and "<" of the next item
109 - prompt_text="<image>\n" * num_images,
110 - mm_data=mm_data,
111 - )
112 -
112
113 class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):
114
vllm/model_executor/models/paligemma.py
+7 -9
@@ -19,7 +19,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
19 BaseProcessingInfo, PromptIndexTargets,
20 PromptInsertion, PromptUpdate,
21 PromptUpdateDetails)
22 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
22 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
23 from vllm.sequence import IntermediateTensors
24
25 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -90,29 +90,27 @@ class PaliGemmaProcessingInfo(BaseProcessingInfo):
90 class PaliGemmaDummyInputsBuilder(
91 BaseDummyInputsBuilder[PaliGemmaProcessingInfo]):
92
93 - def get_dummy_processor_inputs(
93 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
94 + return ""
95 +
96 + def get_dummy_mm_data(
97 self,
98 seq_len: int,
99 mm_counts: Mapping[str, int],
97 - ) -> ProcessorInputs:
100 + ) -> MultiModalDataDict:
101 hf_config = self.info.get_hf_config()
102 vision_config = hf_config.vision_config
103 max_image_size = vision_config.image_size
104
105 num_images = mm_counts.get("image", 0)
106
104 - mm_data = {
107 + return {
108 "image":
109 self._get_dummy_images(width=max_image_size,
110 height=max_image_size,
111 num_images=num_images)
112 }
113
111 - return ProcessorInputs(
112 - prompt_text="",
113 - mm_data=mm_data,
114 - )
115 -
114
115 class PaliGemmaMultiModalProcessor(
116 BaseMultiModalProcessor[PaliGemmaProcessingInfo]):
vllm/model_executor/models/phi3v.py
+14 -13
@@ -32,7 +32,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32 VocabParallelEmbedding)
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
35 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36 + MultiModalKwargs)
37 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
38 ImageSize, MultiModalDataItems)
39 # yapf conflicts with isort for this block
@@ -42,7 +43,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
43 PlaceholderFeaturesInfo,
44 PromptReplacement, PromptUpdate)
45 # yapf: enable
45 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
46 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
47 from vllm.sequence import IntermediateTensors
48 from vllm.utils import is_list_of
49
@@ -343,31 +344,31 @@ class Phi3VProcessingInfo(BaseProcessingInfo):
344
345 class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
346
346 - def get_dummy_processor_inputs(
347 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
348 + num_images = mm_counts.get("image", 0)
349 +
350 + hf_processor = self.info.get_hf_processor()
351 + image_tokens: list[str] = hf_processor.img_tokens # type: ignore
352 +
353 + return "".join(image_tokens[:num_images])
354 +
355 + def get_dummy_mm_data(
356 self,
357 seq_len: int,
358 mm_counts: Mapping[str, int],
350 - ) -> ProcessorInputs:
359 + ) -> MultiModalDataDict:
360 num_images = mm_counts.get("image", 0)
361
362 target_width, target_height = \
363 self.info.get_image_size_with_most_features()
364
356 - mm_data = {
365 + return {
366 "image":
367 self._get_dummy_images(width=target_width,
368 height=target_height,
369 num_images=num_images)
370 }
371
363 - hf_processor = self.info.get_hf_processor()
364 - image_tokens: list[str] = hf_processor.img_tokens # type: ignore
365 -
366 - return ProcessorInputs(
367 - prompt_text="".join(image_tokens[:num_images]),
368 - mm_data=mm_data,
369 - )
370 -
372
373 class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
374
vllm/model_executor/models/pixtral.py
+9 -10
@@ -32,13 +32,14 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
32 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
35 -from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
35 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36 + NestedTensors)
37 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
38 MultiModalDataItems)
39 from vllm.multimodal.processing import (BaseMultiModalProcessor,
40 BaseProcessingInfo, PromptReplacement,
41 PromptUpdate, PromptUpdateDetails)
41 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
42 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
43 from vllm.sequence import IntermediateTensors
44 from vllm.transformers_utils.tokenizer import (MistralTokenizer,
45 cached_tokenizer_from_config)
@@ -203,28 +204,26 @@ class PixtralProcessingInfo(BaseProcessingInfo):
204
205 class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
206
206 - def get_dummy_processor_inputs(
207 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
208 + return ""
209 +
210 + def get_dummy_mm_data(
211 self,
212 seq_len: int,
213 mm_counts: Mapping[str, int],
210 - ) -> ProcessorInputs:
214 + ) -> MultiModalDataDict:
215 num_images = mm_counts.get("image", 0)
216
217 target_width, target_height = \
218 self.info.get_image_size_with_most_features()
219
216 - mm_data = {
220 + return {
221 "image":
222 self._get_dummy_images(width=target_width,
223 height=target_height,
224 num_images=num_images)
225 }
226
223 - return ProcessorInputs(
224 - prompt_text="",
225 - mm_data=mm_data,
226 - )
227 -
227
228 class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
229 ):
vllm/model_executor/models/prithvi_geospatial_mae.py
+13 -12
@@ -35,7 +35,7 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35 from vllm.multimodal.parse import MultiModalDataItems
36 from vllm.multimodal.processing import (BaseMultiModalProcessor,
37 BaseProcessingInfo, PromptUpdate)
38 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
39 from vllm.sequence import (IntermediateTensors, PoolerOutput,
40 PoolingSequenceGroupOutput)
41
@@ -49,20 +49,21 @@ class PrithviGeoSpatialMAEProcessingInfo(BaseProcessingInfo):
49 class PrithviGeoSpatialMAEInputBuilder(
50 BaseDummyInputsBuilder[PrithviGeoSpatialMAEProcessingInfo]):
51
52 - def get_dummy_processor_inputs(
52 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
53 + return ""
54 +
55 + def get_dummy_mm_data(
56 self,
57 seq_len: int,
58 mm_counts: Mapping[str, int],
56 - ) -> ProcessorInputs:
57 - return ProcessorInputs(
58 - prompt_text="",
59 - # This model input is fixed and is in the form of a torch Tensor.
60 - # The size of pixel_values might change in the cases where we resize
61 - # the input but never exceeds the dimensions below.
62 - mm_data={
63 - "pixel_values": torch.full((1, 6, 512, 512), 1.0),
64 - "location_coords": torch.full((1, 2), 1.0)
65 - })
59 + ) -> MultiModalDataDict:
60 + # This model input is fixed and is in the form of a torch Tensor.
61 + # The size of pixel_values might change in the cases where we resize
62 + # the input but never exceeds the dimensions below.
63 + return {
64 + "pixel_values": torch.full((1, 6, 512, 512), 1.0),
65 + "location_coords": torch.full((1, 2), 1.0),
66 + }
67
68
69 class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
vllm/model_executor/models/qwen2_audio.py
+14 -10
@@ -37,13 +37,14 @@ from vllm.config import VllmConfig
37 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.multimodal import MULTIMODAL_REGISTRY
40 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
40 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
41 + MultiModalKwargs)
42 from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
43 MultiModalDataParser)
44 from vllm.multimodal.processing import (BaseMultiModalProcessor,
45 BaseProcessingInfo, PromptReplacement,
46 PromptUpdate, PromptUpdateDetails)
46 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
48 from vllm.sequence import IntermediateTensors
49
50 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -113,27 +114,30 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
114 class Qwen2AudioDummyInputsBuilder(
115 BaseDummyInputsBuilder[Qwen2AudioProcessingInfo]):
116
116 - def get_dummy_processor_inputs(
117 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
118 + num_audios = mm_counts.get("audio", 0)
119 +
120 + hf_processor = self.info.get_hf_processor()
121 + audio_token = hf_processor.audio_token
122 +
123 + return audio_token * num_audios
124 +
125 + def get_dummy_mm_data(
126 self,
127 seq_len: int,
128 mm_counts: Mapping[str, int],
120 - ) -> ProcessorInputs:
129 + ) -> MultiModalDataDict:
130 feature_extractor = self.info.get_feature_extractor()
131
132 sampling_rate = feature_extractor.sampling_rate
133 audio_len = feature_extractor.chunk_length * sampling_rate
134 num_audios = mm_counts.get("audio", 0)
135
127 - mm_data = {
136 + return {
137 "audio":
138 self._get_dummy_audios(length=audio_len, num_audios=num_audios)
139 }
140
132 - return ProcessorInputs(
133 - prompt_text="<|AUDIO|>" * num_audios,
134 - mm_data=mm_data,
135 - )
136 -
141
142 class Qwen2AudioMultiModalProcessor(
143 BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
vllm/model_executor/models/qwen2_vl.py
+15 -14
@@ -56,15 +56,15 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56 from vllm.model_executor.models.module_mapping import MultiModelKeys
57 from vllm.multimodal import MULTIMODAL_REGISTRY
58 from vllm.multimodal.inputs import (ImageItem, ModalityData,
59 - MultiModalFieldConfig, MultiModalKwargs,
60 - VideoItem)
59 + MultiModalDataDict, MultiModalFieldConfig,
60 + MultiModalKwargs, VideoItem)
61 from vllm.multimodal.parse import (DictEmbeddingItems, ImageSize,
62 ModalityDataItems, MultiModalDataItems,
63 MultiModalDataParser)
64 from vllm.multimodal.processing import (BaseMultiModalProcessor,
65 BaseProcessingInfo, PromptReplacement,
66 PromptUpdate)
67 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
67 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
68 from vllm.platforms import _Backend
69 from vllm.sequence import IntermediateTensors
70 from vllm.transformers_utils.config import uses_mrope
@@ -965,11 +965,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
965
966 class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
967
968 - def get_dummy_processor_inputs(
969 - self,
970 - seq_len: int,
971 - mm_counts: Mapping[str, int],
972 - ) -> ProcessorInputs:
968 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
969 num_images = mm_counts.get("image", 0)
970 num_videos = mm_counts.get("video", 0)
971
@@ -977,12 +973,22 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
973 image_token: str = hf_processor.image_token
974 video_token: str = hf_processor.video_token
975
976 + return image_token * num_images + video_token * num_videos
977 +
978 + def get_dummy_mm_data(
979 + self,
980 + seq_len: int,
981 + mm_counts: Mapping[str, int],
982 + ) -> MultiModalDataDict:
983 + num_images = mm_counts.get("image", 0)
984 + num_videos = mm_counts.get("video", 0)
985 +
986 target_width, target_height = \
987 self.info.get_image_size_with_most_features()
988 target_num_frames = \
989 self.info.get_num_frames_with_most_features(seq_len, mm_counts)
990
985 - mm_data = {
991 + return {
992 "image":
993 self._get_dummy_images(width=target_width,
994 height=target_height,
@@ -996,11 +1002,6 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
1002 )
1003 }
1004
999 - return ProcessorInputs(
1000 - prompt_text=image_token * num_images + video_token * num_videos,
1001 - mm_data=mm_data,
1002 - )
1003 -
1005
1006 class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
1007 ):
vllm/model_executor/models/qwen_vl.py
+16 -15
@@ -32,12 +32,13 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
32 from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
33 from vllm.model_executor.models.module_mapping import MultiModelKeys
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
35 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36 + MultiModalKwargs)
37 from vllm.multimodal.parse import MultiModalDataItems
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 BaseProcessingInfo, PromptReplacement,
40 PromptUpdate, PromptUpdateDetails)
40 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
41 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
42 from vllm.sequence import IntermediateTensors
43
44 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -542,34 +543,34 @@ class QwenVLProcessingInfo(BaseProcessingInfo):
543
544 class QwenVLDummyInputsBuilder(BaseDummyInputsBuilder[QwenVLProcessingInfo]):
545
545 - def get_dummy_processor_inputs(
546 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
547 + num_images = mm_counts.get("image", 0)
548 +
549 + hf_processor = self.info.get_hf_processor()
550 + img_start = hf_processor.image_start_tag
551 + img_end = hf_processor.image_end_tag
552 +
553 + return "".join(f"Picture {i}: {img_start}{img_end}\n"
554 + for i in range(1, num_images + 1))
555 +
556 + def get_dummy_mm_data(
557 self,
558 seq_len: int,
559 mm_counts: Mapping[str, int],
549 - ) -> ProcessorInputs:
560 + ) -> MultiModalDataDict:
561 hf_config = self.info.get_hf_config()
562 vision_config = hf_config.visual
563
553 - processor = self.info.get_hf_processor()
554 - img_start = processor.image_start_tag
555 - img_end = processor.image_end_tag
556 -
564 target_width = target_height = vision_config["image_size"]
565 num_images = mm_counts.get("image", 0)
566
560 - mm_data = {
567 + return {
568 "image":
569 self._get_dummy_images(width=target_width,
570 height=target_height,
571 num_images=num_images)
572 }
573
567 - return ProcessorInputs(
568 - prompt_text="".join(f"Picture {i}: {img_start}{img_end}\n"
569 - for i in range(1, num_images + 1)),
570 - mm_data=mm_data,
571 - )
572 -
574
575 class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
576
vllm/model_executor/models/skyworkr1v.py
+11 -11
@@ -26,14 +26,14 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.multimodal import MULTIMODAL_REGISTRY
29 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
30 - NestedTensors)
29 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
30 + MultiModalKwargs, NestedTensors)
31 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
32 ImageSize, MultiModalDataItems)
33 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 BaseProcessingInfo, PromptReplacement,
35 PromptUpdate, PromptUpdateDetails)
36 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
36 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
37 from vllm.sequence import IntermediateTensors
38 from vllm.transformers_utils.tokenizer import AnyTokenizer
39
@@ -505,27 +505,27 @@ _I = TypeVar("_I", bound=BaseSkyworkR1VProcessingInfo)
505
506 class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
507
508 - def get_dummy_processor_inputs(
508 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
509 + num_images = mm_counts.get("image", 0)
510 +
511 + return "<image>" * num_images
512 +
513 + def get_dummy_mm_data(
514 self,
515 seq_len: int,
516 mm_counts: Mapping[str, int],
512 - ) -> ProcessorInputs:
517 + ) -> MultiModalDataDict:
518 target_width, target_height = \
519 self.info.get_image_size_with_most_features()
520 num_images = mm_counts.get("image", 0)
521
517 - mm_data = {
522 + return {
523 "image":
524 self._get_dummy_images(width=target_width,
525 height=target_height,
526 num_images=num_images)
527 }
528
524 - return ProcessorInputs(
525 - prompt_text="<image>" * num_images,
526 - mm_data=mm_data,
527 - )
528 -
529
530 class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]):
531
vllm/model_executor/models/ultravox.py
+11 -11
@@ -23,13 +23,13 @@ from vllm.model_executor.model_loader.loader import DefaultModelLoader
23 from vllm.model_executor.models.module_mapping import MultiModelKeys
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 from vllm.multimodal import MULTIMODAL_REGISTRY
26 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
27 - NestedTensors)
26 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
27 + MultiModalKwargs, NestedTensors)
28 from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
29 from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 BaseProcessingInfo, PromptReplacement,
31 PromptUpdate)
32 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
33 from vllm.sequence import IntermediateTensors
34 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
35
@@ -110,11 +110,16 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
110 class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
111 ):
112
113 - def get_dummy_processor_inputs(
113 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
114 + num_audios = mm_counts.get("audio", 0)
115 +
116 + return "<|audio|>" * num_audios
117 +
118 + def get_dummy_mm_data(
119 self,
120 seq_len: int,
121 mm_counts: Mapping[str, int],
117 - ) -> ProcessorInputs:
122 + ) -> MultiModalDataDict:
123 feature_extractor = self.info.get_feature_extractor()
124
125 sampling_rate = feature_extractor.sampling_rate
@@ -122,16 +127,11 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
127 _MAX_ENCODER_BATCH_SIZE)
128 num_audios = mm_counts.get("audio", 0)
129
125 - mm_data = {
130 + return {
131 "audio":
132 self._get_dummy_audios(length=audio_len, num_audios=num_audios)
133 }
134
130 - return ProcessorInputs(
131 - prompt_text="<|audio|>" * num_audios,
132 - mm_data=mm_data,
133 - )
134 -
135
136 class UltravoxMultiModalProcessor(
137 BaseMultiModalProcessor[UltravoxProcessingInfo]):
vllm/model_executor/models/whisper.py
+12 -12
@@ -26,13 +26,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors
29 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
30 -from vllm.multimodal.parse import (MultiModalDataDict, MultiModalDataItems,
31 - MultiModalDataParser)
29 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
30 + MultiModalKwargs)
31 +from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
32 from vllm.multimodal.processing import (BaseProcessingInfo,
33 EncDecMultiModalProcessor,
34 PromptReplacement, PromptUpdate)
35 -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
35 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
36
37 from .interfaces import (MultiModalEmbeddings, SupportsMultiModal,
38 SupportsTranscription, SupportsV0Only)
@@ -544,27 +544,27 @@ class WhisperProcessingInfo(BaseProcessingInfo):
544
545 class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
546
547 - def get_dummy_processor_inputs(
547 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
548 + num_audios = mm_counts.get("audio", 0)
549 +
550 + return "<|startoftranscript|>" * num_audios
551 +
552 + def get_dummy_mm_data(
553 self,
554 seq_len: int,
555 mm_counts: Mapping[str, int],
551 - ) -> ProcessorInputs:
556 + ) -> MultiModalDataDict:
557 feature_extractor = self.info.get_feature_extractor()
558
559 sampling_rate = feature_extractor.sampling_rate
560 audio_len = feature_extractor.chunk_length * sampling_rate
561 num_audios = mm_counts.get("audio", 0)
562
558 - mm_data = {
563 + return {
564 "audio":
565 self._get_dummy_audios(length=audio_len, num_audios=num_audios)
566 }
567
563 - return ProcessorInputs(
564 - prompt_text="<|startoftranscript|>" * num_audios,
565 - mm_data=mm_data,
566 - )
567 -
568
569 class WhisperMultiModalProcessor(
570 EncDecMultiModalProcessor[WhisperProcessingInfo]):
vllm/multimodal/processing.py
+1 -12
@@ -1051,12 +1051,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1051 *,
1052 cache: Optional[ProcessingCache] = None,
1053 enable_sanity_checks: bool = True) -> None:
1054 - if get_repls := getattr(self, "_get_prompt_replacements", None):
1055 - logger.warning_once("`_get_prompt_replacements` has been renamed "
1056 - "to `_get_prompt_updates`. The old name will "
1057 - "be removed in an upcoming release.")
1058 - self._get_prompt_updates = get_repls # type: ignore[method-assign]
1059 -
1054 super().__init__()
1055
1056 self.info = info
@@ -1274,13 +1268,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1268 """
1269 mm_counts = mm_items.get_all_counts()
1270
1277 - dummy_inputs = self.dummy_inputs.get_dummy_processor_inputs(
1278 - self.info.ctx.model_config.max_model_len,
1279 - mm_counts,
1280 - )
1281 -
1271 _, mm_kwargs, _ = self._apply_hf_processor_text_mm(
1283 - prompt_text=dummy_inputs.prompt_text,
1272 + prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1273 mm_items=mm_items,
1274 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1275 )
vllm/multimodal/profiling.py
+34 -3
@@ -1,6 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from abc import ABC, abstractmethod
3 +from abc import ABC
4 from collections.abc import Mapping
5 from dataclasses import dataclass, field
6 from typing import Generic, NamedTuple, Optional, TypeVar, cast
@@ -60,7 +60,35 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
60
61 self.info = info
62
63 - @abstractmethod
63 + # TODO: @abstractmethod after transition
64 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
65 + """
66 + Build the text input corresponding to :code:`mm_counts`.
67 + """
68 + if (type(self).get_dummy_processor_inputs ==
69 + BaseDummyInputsBuilder.get_dummy_processor_inputs):
70 + raise NotImplementedError
71 +
72 + logger.warning_once("`get_dummy_processor_inputs` has been split up "
73 + "into `get_dummy_text` and `get_dummy_mm_data`. "
74 + "These two methods will be marked as abstract "
75 + "in an upcoming release.")
76 +
77 + seq_len = self.info.ctx.model_config.max_model_len
78 + return self.get_dummy_processor_inputs(seq_len, mm_counts).prompt_text
79 +
80 + # TODO: @abstractmethod after transition
81 + def get_dummy_mm_data(
82 + self,
83 + seq_len: int,
84 + mm_counts: Mapping[str, int],
85 + ) -> MultiModalDataDict:
86 + """
87 + Build the multimodal input which, after processing, results in
88 + the maximum possible number of placeholder tokens.
89 + """
90 + raise NotImplementedError
91 +
92 def get_dummy_processor_inputs(
93 self,
94 seq_len: int,
@@ -70,7 +98,10 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
98 Build the input which, after processing, results in
99 the maximum possible number of placeholder tokens.
100 """
73 - raise NotImplementedError
101 + dummy_text = self.get_dummy_text(mm_counts)
102 + dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)
103 +
104 + return ProcessorInputs(prompt_text=dummy_text, mm_data=dummy_mm_data)
105
106 def _get_dummy_audios(
107 self,