A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Separate out profiling-related logic (#11746)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 6, 2025 at 16:02 UTC
996357e4808ca5eab97d4c97c7d25b3073f46aab
17 files changed
+1015
-718
tests/multimodal/test_processing.py
+4
-3
@@ -586,9 +586,10 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
586
)
587
588
processor = processor_factory(ctx, cache=None)
589
+ profiler = processor.profiling_info
590
591
mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
591
- processor.get_supported_mm_limits = mock_supported_mm_limits
592
+ profiler.get_supported_mm_limits = mock_supported_mm_limits
593
594
if is_valid:
595
exc_ctx = nullcontext()
@@ -596,7 +597,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
597
exc_ctx = pytest.raises(ValueError, match="this model only supports")
598
599
with exc_ctx:
599
- processor._get_and_validate_dummy_mm_counts()
600
+ profiler.get_mm_limits()
601
602
603
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
@@ -723,7 +724,7 @@ def _test_processing_cache_correctness(
724
}
725
726
mm_counts = {k: len(vs) for k, vs in mm_data.items()}
726
- prompt = baseline_processor._get_dummy_processor_inputs(
727
+ prompt = baseline_processor.profiling_info.get_dummy_processor_inputs(
728
model_config.max_model_len,
729
mm_counts,
730
).prompt_text
vllm/model_executor/models/aria.py
+47
-32
@@ -24,8 +24,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
24
from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
25
NestedTensors)
26
from vllm.multimodal.processing import (BaseMultiModalProcessor,
27
- MultiModalDataItems, ProcessorInputs,
27
+ MultiModalDataItems, ProcessingMixin,
28
PromptReplacement)
29
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
30
from vllm.sequence import IntermediateTensors
31
from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
32
AriaVisionConfig)
@@ -444,18 +445,58 @@ def build_mm_projector(config: PretrainedConfig):
445
)
446
447
447
-class AriaMultiModalProcessor(BaseMultiModalProcessor):
448
+class AriaProcessingMixin(ProcessingMixin):
449
449
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
450
- return {"image": None}
450
+ def _get_hf_config(self):
451
+ return self.ctx.get_hf_config()
452
+
453
+ def _get_vision_config(self) -> AriaVisionConfig:
454
+ return self._get_hf_config().vision_config
455
456
def _get_num_image_tokens(self) -> int:
453
- hf_config = self.ctx.get_hf_config()
457
+ hf_config = self._get_hf_config()
458
return max(hf_config.projector_patch_to_query_dict.values())
459
460
+
461
+class AriaProfilingInfo(AriaProcessingMixin, BaseProfilingInfo):
462
+
463
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
464
+ return {"image": None}
465
+
466
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
467
return {"image": self._get_num_image_tokens()}
468
469
+ def get_dummy_processor_inputs(
470
+ self,
471
+ seq_len: int,
472
+ mm_counts: Mapping[str, int],
473
+ ) -> ProcessorInputs:
474
+ vision_config = self._get_vision_config()
475
+
476
+ max_image_size = vision_config.image_size
477
+ num_images = mm_counts.get("image", 0)
478
+
479
+ mm_data = {
480
+ "image":
481
+ self._get_dummy_images(width=max_image_size,
482
+ height=max_image_size,
483
+ num_images=num_images)
484
+ }
485
+
486
+ hf_processor = self._get_hf_processor()
487
+ image_token: str = hf_processor.image_token # type: ignore
488
+
489
+ return ProcessorInputs(
490
+ prompt_text=image_token * num_images,
491
+ mm_data=mm_data,
492
+ )
493
+
494
+
495
+class AriaMultiModalProcessor(AriaProcessingMixin, BaseMultiModalProcessor):
496
+
497
+ def _get_profiling_info(self) -> BaseProfilingInfo:
498
+ return AriaProfilingInfo(self.ctx)
499
+
500
def _get_mm_fields_config(
501
self,
502
hf_inputs: BatchFeature,
@@ -472,7 +513,7 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
513
hf_processor_mm_kwargs: Mapping[str, object],
514
out_mm_kwargs: MultiModalKwargs,
515
) -> list[PromptReplacement]:
475
- hf_config = self.ctx.get_hf_config()
516
+ hf_config = self._get_hf_config()
517
image_token_id = hf_config.image_token_index
518
519
num_image_tokens = self._get_num_image_tokens()
@@ -485,32 +526,6 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
526
)
527
]
528
488
- def _get_dummy_processor_inputs(
489
- self,
490
- seq_len: int,
491
- mm_counts: Mapping[str, int],
492
- ) -> ProcessorInputs:
493
- hf_config = self.ctx.get_hf_config()
494
- vision_config: AriaVisionConfig = hf_config.vision_config
495
-
496
- max_image_size = vision_config.image_size
497
- num_images = mm_counts.get("image", 0)
498
-
499
- mm_data = {
500
- "image":
501
- self._get_dummy_images(width=max_image_size,
502
- height=max_image_size,
503
- num_images=num_images)
504
- }
505
-
506
- hf_processor = self._get_hf_processor()
507
- image_token: str = hf_processor.image_token # type: ignore
508
-
509
- return ProcessorInputs(
510
- prompt_text=image_token * num_images,
511
- mm_data=mm_data,
512
- )
513
-
529
530
@MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor)
531
class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
vllm/model_executor/models/blip2.py
+44
-34
@@ -4,8 +4,8 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
5
import torch
6
import torch.nn as nn
7
-from transformers import (BatchFeature, Blip2Config, Blip2Processor,
8
- Blip2QFormerConfig, apply_chunking_to_forward)
7
+from transformers import (BatchFeature, Blip2Config, Blip2QFormerConfig,
8
+ apply_chunking_to_forward)
9
10
from vllm.attention import AttentionMetadata
11
from vllm.config import CacheConfig, VllmConfig
@@ -18,8 +18,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18
MultiModalInputsV2, MultiModalKwargs,
19
NestedTensors, PlaceholderRange)
20
from vllm.multimodal.processing import (BaseMultiModalProcessor,
21
- MultiModalDataItems, ProcessorInputs,
21
+ MultiModalDataItems, ProcessingMixin,
22
PromptReplacement)
23
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
24
from vllm.sequence import IntermediateTensors
25
26
from .blip import BlipVisionModel
@@ -396,20 +397,52 @@ class Blip2QFormerModel(nn.Module):
397
return sequence_output
398
399
399
-class Blip2MultiModalProcessor(BaseMultiModalProcessor):
400
+class Blip2ProcessingMixin(ProcessingMixin):
401
401
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
402
- return {"image": 1}
402
+ def _get_hf_config(self):
403
+ return self.ctx.get_hf_config(Blip2Config)
404
405
def _get_num_image_tokens(self) -> int:
405
- hf_config = self.ctx.get_hf_config(Blip2Config)
406
+ hf_config = self._get_hf_config()
407
return hf_config.num_query_tokens
408
409
+
410
+class Blip2ProfilingInfo(Blip2ProcessingMixin, BaseProfilingInfo):
411
+
412
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
413
+ return {"image": 1}
414
+
415
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
416
return {"image": self._get_num_image_tokens()}
417
411
- def _get_hf_processor(self) -> Blip2Processor:
412
- return self.ctx.get_hf_processor(Blip2Processor)
418
+ def get_dummy_processor_inputs(
419
+ self,
420
+ seq_len: int,
421
+ mm_counts: Mapping[str, int],
422
+ ) -> ProcessorInputs:
423
+ hf_config = self._get_hf_config()
424
+ vision_config = hf_config.vision_config
425
+
426
+ max_image_size = vision_config.image_size
427
+ num_images = mm_counts.get("image", 0)
428
+
429
+ mm_data = {
430
+ "image":
431
+ self._get_dummy_images(width=max_image_size,
432
+ height=max_image_size,
433
+ num_images=num_images)
434
+ }
435
+
436
+ return ProcessorInputs(
437
+ prompt_text="",
438
+ mm_data=mm_data,
439
+ )
440
+
441
+
442
+class Blip2MultiModalProcessor(Blip2ProcessingMixin, BaseMultiModalProcessor):
443
+
444
+ def _get_profiling_info(self) -> BaseProfilingInfo:
445
+ return Blip2ProfilingInfo(self.ctx)
446
447
def _get_mm_fields_config(
448
self,
@@ -427,13 +460,13 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
460
hf_processor_mm_kwargs: Mapping[str, object],
461
out_mm_kwargs: MultiModalKwargs,
462
) -> list[PromptReplacement]:
430
- max_image_tokens = self._get_num_image_tokens()
463
+ num_image_tokens = self._get_num_image_tokens()
464
465
return [
466
PromptReplacement(
467
modality="image",
468
target="</s>",
436
- replacement="<image>" * max_image_tokens + "</s>",
469
+ replacement="<image>" * num_image_tokens + "</s>",
470
)
471
]
472
@@ -457,29 +490,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
490
491
return result
492
460
- def _get_dummy_processor_inputs(
461
- self,
462
- seq_len: int,
463
- mm_counts: Mapping[str, int],
464
- ) -> ProcessorInputs:
465
- hf_config = self.ctx.get_hf_config(Blip2Config)
466
- vision_config = hf_config.vision_config
467
-
468
- max_image_size = vision_config.image_size
469
- num_images = mm_counts.get("image", 0)
470
-
471
- mm_data = {
472
- "image":
473
- self._get_dummy_images(width=max_image_size,
474
- height=max_image_size,
475
- num_images=num_images)
476
- }
477
-
478
- return ProcessorInputs(
479
- prompt_text="",
480
- mm_data=mm_data,
481
- )
482
-
493
494
@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor)
495
class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
vllm/model_executor/models/chameleon.py
+43
-29
@@ -31,8 +31,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31
MultiModalInputsV2, MultiModalKwargs,
32
NestedTensors, PlaceholderRange)
33
from vllm.multimodal.processing import (BaseMultiModalProcessor,
34
- MultiModalDataItems, ProcessorInputs,
34
+ MultiModalDataItems, ProcessingMixin,
35
PromptReplacement)
36
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
37
from vllm.sequence import IntermediateTensors
38
from vllm.utils import print_warning_once
39
@@ -48,20 +49,55 @@ class ChameleonImagePixelInputs(TypedDict):
49
"""Shape: `(batch_size * num_images, num_channels, height, width)`"""
50
51
51
-class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
52
+class ChameleonProcessingMixin(ProcessingMixin):
53
53
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
54
- return {"image": 1}
54
+ def _get_hf_config(self):
55
+ return self.ctx.get_hf_config(ChameleonConfig)
56
+
57
+ def _get_hf_processor(self):
58
+ return self.ctx.get_hf_processor(ChameleonProcessor)
59
60
def _get_num_image_tokens(self) -> int:
61
processor = self._get_hf_processor()
62
return processor.image_seq_length
63
64
+
65
+class ChameleonProfilingInfo(ChameleonProcessingMixin, BaseProfilingInfo):
66
+
67
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
68
+ return {"image": 1}
69
+
70
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
71
return {"image": self._get_num_image_tokens()}
72
63
- def _get_hf_processor(self) -> ChameleonProcessor:
64
- return self.ctx.get_hf_processor(ChameleonProcessor)
73
+ def get_dummy_processor_inputs(
74
+ self,
75
+ seq_len: int,
76
+ mm_counts: Mapping[str, int],
77
+ ) -> ProcessorInputs:
78
+ config = self._get_hf_config()
79
+
80
+ width = height = config.vq_config.resolution
81
+ num_images = mm_counts.get("image", 0)
82
+
83
+ mm_data = {
84
+ "image":
85
+ self._get_dummy_images(width=width,
86
+ height=height,
87
+ num_images=num_images)
88
+ }
89
+
90
+ return ProcessorInputs(
91
+ prompt_text="<image>" * num_images,
92
+ mm_data=mm_data,
93
+ )
94
+
95
+
96
+class ChameleonMultiModalProcessor(ChameleonProcessingMixin,
97
+ BaseMultiModalProcessor):
98
+
99
+ def _get_profiling_info(self) -> BaseProfilingInfo:
100
+ return ChameleonProfilingInfo(self.ctx)
101
102
def _get_mm_fields_config(
103
self,
@@ -76,7 +112,7 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
112
hf_processor_mm_kwargs: Mapping[str, object],
113
out_mm_kwargs: MultiModalKwargs,
114
) -> list[PromptReplacement]:
79
- processor = self._get_hf_processor()
115
+ processor = self._get_hf_processor(**hf_processor_mm_kwargs)
116
117
return [
118
PromptReplacement(
@@ -90,28 +126,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
126
)
127
]
128
93
- def _get_dummy_processor_inputs(
94
- self,
95
- seq_len: int,
96
- mm_counts: Mapping[str, int],
97
- ) -> ProcessorInputs:
98
- config = self.ctx.get_hf_config(ChameleonConfig)
99
-
100
- width = height = config.vq_config.resolution
101
- num_images = mm_counts.get("image", 0)
102
-
103
- mm_data = {
104
- "image":
105
- self._get_dummy_images(width=width,
106
- height=height,
107
- num_images=num_images)
108
- }
109
-
110
- return ProcessorInputs(
111
- prompt_text="<image>" * num_images,
112
- mm_data=mm_data,
113
- )
114
-
129
def apply(
130
self,
131
prompt_text: str,
vllm/model_executor/models/fuyu.py
+50
-35
@@ -35,8 +35,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35
NestedTensors, PlaceholderRange)
36
from vllm.multimodal.parse import ImageProcessorItems, ImageSize
37
from vllm.multimodal.processing import (BaseMultiModalProcessor,
38
- MultiModalDataItems, ProcessorInputs,
38
+ MultiModalDataItems, ProcessingMixin,
39
PromptReplacement)
40
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
41
from vllm.sequence import IntermediateTensors
42
43
from .interfaces import SupportsMultiModal, SupportsPP
@@ -63,18 +64,16 @@ class FuyuImagePatchInputs(TypedDict):
64
"""
65
66
66
-class FuyuMultiModalProcessor(BaseMultiModalProcessor):
67
+class FuyuProcessingMixin(ProcessingMixin):
68
68
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
69
- return {"image": 1}
69
+ def _get_hf_config(self):
70
+ return self.ctx.get_hf_config(FuyuConfig)
71
71
- def _get_image_target_size(self) -> ImageSize:
72
- processor = self._get_hf_processor()
73
- image_processor: FuyuImageProcessor = processor.image_processor
72
+ def _get_hf_processor(self):
73
+ return self.ctx.get_hf_processor(FuyuProcessor)
74
75
- target_size = image_processor.size
76
- return ImageSize(width=target_size["width"],
77
- height=target_size["height"])
75
+ def _get_image_processor(self) -> FuyuImageProcessor:
76
+ return self._get_hf_processor().image_processor
77
78
def _get_image_feature_grid_size(
79
self,
@@ -82,7 +81,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
81
image_width: int,
82
image_height: int,
83
) -> tuple[int, int]:
85
- target_width, target_height = self._get_image_target_size()
84
+ image_processor = self._get_image_processor()
85
+ target_width = image_processor.size["width"]
86
+ target_height = image_processor.size["height"]
87
88
if not (image_width <= target_width and image_height <= target_height):
89
height_scale_factor = target_height / image_height
@@ -96,8 +97,14 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
97
nrows = math.ceil(image_height / 30)
98
return ncols, nrows
99
100
+
101
+class FuyuProfilingInfo(FuyuProcessingMixin, BaseProfilingInfo):
102
+
103
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104
+ return {"image": 1}
105
+
106
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
100
- target_width, target_height = self._get_image_target_size()
107
+ target_width, target_height = self._get_image_size_with_most_features()
108
109
max_ncols, max_nrows = self._get_image_feature_grid_size(
110
image_width=target_width,
@@ -107,8 +114,36 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
114
115
return {"image": max_image_tokens}
116
110
- def _get_hf_processor(self) -> FuyuProcessor:
111
- return self.ctx.get_hf_processor(FuyuProcessor)
117
+ def _get_image_size_with_most_features(self) -> ImageSize:
118
+ image_processor = self._get_image_processor()
119
+ return ImageSize(width=image_processor.size["width"],
120
+ height=image_processor.size["height"])
121
+
122
+ def get_dummy_processor_inputs(
123
+ self,
124
+ seq_len: int,
125
+ mm_counts: Mapping[str, int],
126
+ ) -> ProcessorInputs:
127
+ target_width, target_height = self._get_image_size_with_most_features()
128
+ num_images = mm_counts.get("image", 0)
129
+
130
+ mm_data = {
131
+ "image":
132
+ self._get_dummy_images(width=target_width,
133
+ height=target_height,
134
+ num_images=num_images)
135
+ }
136
+
137
+ return ProcessorInputs(
138
+ prompt_text="",
139
+ mm_data=mm_data,
140
+ )
141
+
142
+
143
+class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
144
+
145
+ def _get_profiling_info(self) -> BaseProfilingInfo:
146
+ return FuyuProfilingInfo(self.ctx)
147
148
def _call_hf_processor(
149
self,
@@ -161,7 +196,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
196
hf_processor_mm_kwargs: Mapping[str, object],
197
out_mm_kwargs: MultiModalKwargs,
198
) -> list[PromptReplacement]:
164
- hf_config = self.ctx.get_hf_config(FuyuConfig)
199
+ hf_config = self._get_hf_config()
200
bos_token_id = hf_config.bos_token_id
201
202
tokenizer = self._get_tokenizer()
@@ -208,26 +243,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
243
244
return result
245
211
- def _get_dummy_processor_inputs(
212
- self,
213
- seq_len: int,
214
- mm_counts: Mapping[str, int],
215
- ) -> ProcessorInputs:
216
- target_width, target_height = self._get_image_target_size()
217
- num_images = mm_counts.get("image", 0)
218
-
219
- mm_data = {
220
- "image":
221
- self._get_dummy_images(width=target_width,
222
- height=target_height,
223
- num_images=num_images)
224
- }
225
-
226
- return ProcessorInputs(
227
- prompt_text="",
228
- mm_data=mm_data,
229
- )
230
-
246
247
@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor)
248
class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
vllm/model_executor/models/llava.py
+115
-66
@@ -1,4 +1,4 @@
1
-from abc import abstractmethod
1
+from abc import ABC, abstractmethod
2
from functools import cached_property
3
from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4
Protocol, Set, Tuple, TypedDict, Union)
@@ -13,6 +13,7 @@ from transformers.models.pixtral import PixtralProcessor
13
14
from vllm.attention import AttentionMetadata
15
from vllm.config import VllmConfig
16
+from vllm.inputs import InputProcessingContext
17
from vllm.model_executor.layers.activation import get_act_fn
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
RowParallelLinear)
@@ -25,9 +26,10 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
26
NestedTensors)
27
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
28
ImageSize)
28
-from vllm.multimodal.processing import (InputProcessingContext,
29
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
30
MultiModalDataItems, ProcessingCache,
30
- ProcessorInputs, PromptReplacement)
31
+ ProcessingMixin, PromptReplacement)
32
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
33
from vllm.sequence import IntermediateTensors
34
35
from .clip import CLIPVisionModel
@@ -37,7 +39,7 @@ from .pixtral import (PixtralHFVisionModel,
39
from .siglip import SiglipVisionModel
40
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
41
maybe_prefix, merge_multimodal_embeddings)
40
-from .vision import BaseVisionLanguageMultiModalProcessor
42
+from .vision import get_vision_encoder_info
43
44
45
class LlavaImagePixelInputs(TypedDict):
@@ -94,30 +96,42 @@ class LlavaMultiModalProjector(nn.Module):
96
97
class LlavaLikeConfig(Protocol):
98
vision_config: Final[PretrainedConfig]
99
+ image_token_index: Final[int]
100
vision_feature_select_strategy: Final[str]
98
- vision_feature_layer: Final[Union[int, List[int]]]
101
+ vision_feature_layer: Final[Union[int, list[int]]]
102
103
101
-class BaseLlavaMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
104
+class LlavaLikeProcessor(Protocol):
105
+ image_token: Final[str]
106
+
107
+
108
+class BaseLlavaProcessingMixin(ProcessingMixin, ABC):
109
103
- @abstractmethod
110
def _get_hf_config(self) -> LlavaLikeConfig:
105
- raise NotImplementedError
111
+ return self.ctx.get_hf_config(LlavaConfig)
112
107
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
108
- return {"image": None}
113
+ def _get_vision_encoder_info(self):
114
+ return get_vision_encoder_info(self._get_hf_config())
115
110
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
111
- return {"image": self._get_max_image_tokens()}
116
+ @abstractmethod
117
+ def _get_hf_processor(self) -> LlavaLikeProcessor:
118
+ raise NotImplementedError
119
113
- def _get_mm_fields_config(
120
+ def _get_num_image_tokens(
121
self,
115
- hf_inputs: BatchFeature,
116
- hf_processor_mm_kwargs: Mapping[str, object],
117
- ) -> Mapping[str, MultiModalFieldConfig]:
118
- return dict(
119
- pixel_values=MultiModalFieldConfig.batched("image"),
120
- image_embeds=MultiModalFieldConfig.batched("image"),
122
+ *,
123
+ image_width: int,
124
+ image_height: int,
125
+ ) -> int:
126
+ hf_config = self._get_hf_config()
127
+ vision_encoder_info = self._get_vision_encoder_info()
128
+
129
+ return self._apply_feature_select_strategy(
130
+ hf_config.vision_feature_select_strategy,
131
+ vision_encoder_info.get_num_image_tokens(
132
+ image_width=image_width,
133
+ image_height=image_height,
134
+ ),
135
)
136
137
def _apply_feature_select_strategy(
@@ -133,31 +147,38 @@ class BaseLlavaMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
147
msg = f"Unexpected feature select strategy: {strategy!r}"
148
raise NotImplementedError(msg)
149
136
- def _get_max_image_tokens(self) -> int:
137
- hf_config = self._get_hf_config()
150
139
- return self._apply_feature_select_strategy(
140
- hf_config.vision_feature_select_strategy,
141
- self._vision_encoder_info.get_max_image_tokens(),
142
- )
151
+class BaseLlavaProfilingInfo(BaseLlavaProcessingMixin, BaseProfilingInfo):
152
144
- def _get_dummy_image_size(self) -> ImageSize:
145
- image_size = self._vision_encoder_info.get_image_size()
146
- return ImageSize(image_size, image_size)
153
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
154
+ return {"image": None}
155
148
- @abstractmethod
149
- def _get_image_token(self) -> str:
150
- raise NotImplementedError
156
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
157
+ return {"image": self._get_max_image_tokens()}
158
+
159
+ def _get_image_size_with_most_features(self) -> ImageSize:
160
+ vision_encoder_info = self._get_vision_encoder_info()
161
+ width = height = vision_encoder_info.get_image_size()
162
+ return ImageSize(width=width, height=height)
163
152
- def _get_dummy_processor_inputs(
164
+ def _get_max_image_tokens(self) -> int:
165
+ target_width, target_height = self._get_image_size_with_most_features()
166
+
167
+ return self._get_num_image_tokens(
168
+ image_width=target_width,
169
+ image_height=target_height,
170
+ )
171
+
172
+ def get_dummy_processor_inputs(
173
self,
174
seq_len: int,
175
mm_counts: Mapping[str, int],
176
) -> ProcessorInputs:
177
num_images = mm_counts.get("image", 0)
178
159
- image_token = self._get_image_token()
160
- target_width, target_height = self._get_dummy_image_size()
179
+ processor = self._get_hf_processor()
180
+ image_token = processor.image_token
181
+ target_width, target_height = self._get_image_size_with_most_features()
182
183
mm_data = {
184
"image":
@@ -172,32 +193,32 @@ class BaseLlavaMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
193
)
194
195
175
-class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
176
-
177
- def _get_hf_config(self) -> LlavaConfig:
178
- return self.ctx.get_hf_config(LlavaConfig)
196
+class LlavaProcessingMixin(BaseLlavaProcessingMixin):
197
180
- def _get_hf_processor(self) -> LlavaProcessor:
198
+ def _get_hf_processor(self):
199
return self.ctx.get_hf_processor(LlavaProcessor)
200
183
- def _get_image_token(self) -> str:
184
- return self._get_hf_processor().image_token
201
186
- def _get_num_image_tokens(
187
- self,
188
- *,
189
- image_width: int,
190
- image_height: int,
191
- ) -> int:
192
- hf_config = self._get_hf_config()
202
+class LlavaProfilingInfo(LlavaProcessingMixin, BaseLlavaProfilingInfo):
203
+ pass
204
194
- return self._apply_feature_select_strategy(
195
- hf_config.vision_feature_select_strategy,
196
- self._vision_encoder_info.get_num_image_tokens(
197
- image_width=image_width,
198
- image_height=image_height,
199
- ),
200
- )
205
+
206
+class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
207
+ BaseMultiModalProcessor):
208
+
209
+ # Copied from BaseMultiModalProcessor
210
+ @abstractmethod
211
+ def _get_profiling_info(self) -> BaseProfilingInfo:
212
+ raise NotImplementedError
213
+
214
+ # Copied from BaseMultiModalProcessor
215
+ @abstractmethod
216
+ def _get_mm_fields_config(
217
+ self,
218
+ hf_inputs: BatchFeature,
219
+ hf_processor_mm_kwargs: Mapping[str, object],
220
+ ) -> Mapping[str, MultiModalFieldConfig]:
221
+ raise NotImplementedError
222
223
def _get_prompt_replacements(
224
self,
@@ -232,16 +253,37 @@ class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
253
]
254
255
235
-class PixtralHFMultiModalProcessor(BaseLlavaMultiModalProcessor):
256
+class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
257
+
258
+ def _get_profiling_info(self) -> BaseProfilingInfo:
259
+ return LlavaProfilingInfo(self.ctx)
260
+
261
+ def _get_mm_fields_config(
262
+ self,
263
+ hf_inputs: BatchFeature,
264
+ hf_processor_mm_kwargs: Mapping[str, object],
265
+ ) -> Mapping[str, MultiModalFieldConfig]:
266
+ return dict(
267
+ pixel_values=MultiModalFieldConfig.batched("image"),
268
+ image_embeds=MultiModalFieldConfig.batched("image"),
269
+ )
270
237
- def _get_hf_config(self) -> LlavaConfig:
238
- return self.ctx.get_hf_config(LlavaConfig)
271
240
- def _get_hf_processor(self) -> PixtralProcessor:
272
+class PixtralHFProcessingMixin(BaseLlavaProcessingMixin):
273
+
274
+ def _get_hf_processor(self):
275
return self.ctx.get_hf_processor(PixtralProcessor)
276
243
- def _get_image_token(self) -> str:
244
- return self._get_hf_processor().image_token
277
+
278
+class PixtralHFProfilingInfo(PixtralHFProcessingMixin, BaseLlavaProfilingInfo):
279
+ pass
280
+
281
+
282
+class PixtralHFMultiModalProcessor(PixtralHFProcessingMixin,
283
+ BaseMultiModalProcessor):
284
+
285
+ def _get_profiling_info(self) -> BaseProfilingInfo:
286
+ return PixtralHFProfilingInfo(self.ctx)
287
288
def _call_hf_processor(
289
self,
@@ -270,6 +312,16 @@ class PixtralHFMultiModalProcessor(BaseLlavaMultiModalProcessor):
312
313
return processed_outputs
314
315
+ def _get_mm_fields_config(
316
+ self,
317
+ hf_inputs: BatchFeature,
318
+ hf_processor_mm_kwargs: Mapping[str, object],
319
+ ) -> Mapping[str, MultiModalFieldConfig]:
320
+ return dict(
321
+ pixel_values=MultiModalFieldConfig.batched("image"),
322
+ image_embeds=MultiModalFieldConfig.batched("image"),
323
+ )
324
+
325
def _get_prompt_replacements(
326
self,
327
mm_items: MultiModalDataItems,
@@ -316,7 +368,7 @@ def _build_llava_or_pixtral_hf_processor(
368
*,
369
cache: Optional[ProcessingCache] = None,
370
enable_sanity_checks: bool = True,
319
-) -> BaseLlavaMultiModalProcessor:
371
+) -> BaseMultiModalProcessor:
372
hf_config = ctx.get_hf_config(LlavaConfig)
373
374
if isinstance(hf_config.vision_config, PixtralVisionConfig):
@@ -663,16 +715,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
715
716
class MantisMultiModalProcessor(LlavaMultiModalProcessor):
717
666
- def _get_hf_processor(self):
667
- return self.ctx.get_hf_processor(LlavaProcessor)
668
-
718
def apply(
719
self,
720
prompt_text: str,
721
mm_data: MultiModalDataDict,
722
hf_processor_mm_kwargs: Mapping[str, object],
723
) -> MultiModalInputsV2:
675
- hf_config = self.ctx.get_hf_config(LlavaConfig)
724
+ hf_config = self._get_hf_config()
725
image_token_id = hf_config.image_token_index
726
727
# Assume that it doesn't depend on the image size
vllm/model_executor/models/llava_next.py
+38
-37
@@ -1,6 +1,6 @@
1
from functools import cached_property
2
-from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
3
- TypedDict, Union)
2
+from typing import (Final, Iterable, List, Literal, Mapping, Optional,
3
+ Protocol, Set, Tuple, TypedDict, Union)
4
5
import numpy as np
6
import torch
@@ -17,12 +17,14 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
19
from vllm.multimodal.parse import ImageSize
20
+from vllm.multimodal.profiling import BaseProfilingInfo
21
from vllm.sequence import IntermediateTensors
22
23
from .clip import CLIPVisionModel
24
from .interfaces import SupportsMultiModal, SupportsPP
24
-from .llava import (LlavaMultiModalProcessor, LlavaMultiModalProjector,
25
- init_vision_tower_for_llava)
25
+from .llava import (BaseLlavaMultiModalProcessor, BaseLlavaProcessingMixin,
26
+ BaseLlavaProfilingInfo, LlavaLikeConfig,
27
+ LlavaMultiModalProjector, init_vision_tower_for_llava)
28
from .siglip import SiglipVisionModel
29
from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
30
init_vllm_registered_model, maybe_prefix)
@@ -60,35 +62,17 @@ LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
62
LlavaNextImageEmbeddingInputs]
63
64
63
-class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
65
+class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):
66
+ image_grid_pinpoints: Final[list[list[int]]]
67
65
- def _get_hf_config(self) -> LlavaNextConfig:
66
- return self.ctx.get_hf_config(LlavaNextConfig)
68
68
- def _get_hf_processor(self) -> LlavaNextProcessor:
69
- return self.ctx.get_hf_processor(LlavaNextProcessor)
69
+class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
70
71
- def _get_mm_fields_config(
72
- self,
73
- hf_inputs: BatchFeature,
74
- hf_processor_mm_kwargs: Mapping[str, object],
75
- ) -> Mapping[str, MultiModalFieldConfig]:
76
- return dict(
77
- pixel_values=MultiModalFieldConfig.batched("image"),
78
- image_sizes=MultiModalFieldConfig.batched("image"),
79
- image_embeds=MultiModalFieldConfig.batched("image"),
80
- )
81
-
82
- def _get_image_token(self) -> str:
83
- return self._get_hf_processor().image_token
84
-
85
- def _get_max_image_tokens(self) -> int:
86
- largest_feature_size, _ = self._get_pinpoint_with_most_features()
87
- return largest_feature_size
71
+ def _get_hf_config(self) -> LlavaNextLikeConfig:
72
+ return self.ctx.get_hf_config(LlavaNextConfig)
73
89
- def _get_dummy_image_size(self) -> ImageSize:
90
- _, pinpoint = self._get_pinpoint_with_most_features()
91
- return pinpoint
74
+ def _get_hf_processor(self):
75
+ return self.ctx.get_hf_processor(LlavaNextProcessor)
76
77
# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
78
def _get_num_image_tokens(
@@ -98,7 +82,7 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
82
image_height: int,
83
) -> int:
84
hf_config = self._get_hf_config()
101
- vision_encoder_info = self._vision_encoder_info
85
+ vision_encoder_info = self._get_vision_encoder_info()
86
87
base_feature_size = self._apply_feature_select_strategy(
88
hf_config.vision_feature_select_strategy,
@@ -140,7 +124,7 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
124
current_height = npatches * num_patch_height
125
current_width = npatches * num_patch_width
126
143
- # NOTE: HF resizes based on float32
127
+ # NOTE: Use float32 to remain consistent with HF output
128
original_aspect_ratio = np.array(original_width / original_height,
129
dtype=np.float32)
130
current_aspect_ratio = np.array(current_width / current_height,
@@ -164,11 +148,10 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
148
149
return (unpadded_features, newline_features)
150
167
- def _get_pinpoint_with_most_features(self) -> tuple[int, ImageSize]:
168
- """
169
- Get the grid pinpoint with the most features and
170
- the corresponding feature size.
171
- """
151
+
152
+class LlavaNextProfilingInfo(LlavaNextProcessingMixin, BaseLlavaProfilingInfo):
153
+
154
+ def _get_image_size_with_most_features(self) -> ImageSize:
155
hf_config = self._get_hf_config()
156
157
largest_feature_size, largest_feature_pinpoint = 0, None
@@ -183,7 +166,25 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
166
if largest_feature_size == 0 or largest_feature_pinpoint is None:
167
raise ValueError("Cannot have a largest feature size of 0!")
168
186
- return largest_feature_size, largest_feature_pinpoint
169
+ return largest_feature_pinpoint
170
+
171
+
172
+class LlavaNextMultiModalProcessor(LlavaNextProcessingMixin,
173
+ BaseLlavaMultiModalProcessor):
174
+
175
+ def _get_profiling_info(self) -> BaseProfilingInfo:
176
+ return LlavaNextProfilingInfo(self.ctx)
177
+
178
+ def _get_mm_fields_config(
179
+ self,
180
+ hf_inputs: BatchFeature,
181
+ hf_processor_mm_kwargs: Mapping[str, object],
182
+ ) -> Mapping[str, MultiModalFieldConfig]:
183
+ return dict(
184
+ pixel_values=MultiModalFieldConfig.batched("image"),
185
+ image_sizes=MultiModalFieldConfig.batched("image"),
186
+ image_embeds=MultiModalFieldConfig.batched("image"),
187
+ )
188
189
190
@MULTIMODAL_REGISTRY.register_processor(LlavaNextMultiModalProcessor)
vllm/model_executor/models/llava_next_video.py
+86
-62
@@ -15,11 +15,14 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15
from vllm.model_executor.models.clip import CLIPVisionModel
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
-from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
19
-from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
20
- VideoEmbeddingItems, VideoProcessorItems)
21
-from vllm.multimodal.processing import (MultiModalFieldConfig, ProcessorInputs,
18
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
19
+ NestedTensors)
20
+from vllm.multimodal.parse import (ImageSize, VideoEmbeddingItems,
21
+ VideoProcessorItems)
22
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
23
+ MultiModalDataItems, ProcessingMixin,
24
PromptReplacement)
25
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
26
from vllm.sequence import IntermediateTensors
27
from vllm.utils import is_list_of
28
@@ -28,7 +31,7 @@ from .llava import init_vision_tower_for_llava
31
from .siglip import SiglipVisionModel
32
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
33
maybe_prefix, merge_multimodal_embeddings)
31
-from .vision import BaseVisionLanguageMultiModalProcessor
34
+from .vision import get_vision_encoder_info
35
36
37
class LlavaNextVideoPixelInputs(TypedDict):
@@ -44,29 +47,16 @@ class LlavaNextVideoPixelInputs(TypedDict):
47
"""
48
49
47
-class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
50
+class LlavaNextVideoProcessingMixin(ProcessingMixin):
51
49
- def _get_hf_config(self) -> LlavaNextVideoConfig:
52
+ def _get_hf_config(self):
53
return self.ctx.get_hf_config(LlavaNextVideoConfig)
54
52
- def _get_hf_processor(self) -> LlavaNextVideoProcessor:
53
- return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
54
-
55
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
56
- return {"video": 1}
57
-
58
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
59
- num_frames = self._get_dummy_num_frames(seq_len)
60
- max_video_tokens = self._get_max_video_tokens(num_frames)
61
-
62
- return {"video": max_video_tokens}
55
+ def _get_vision_encoder_info(self):
56
+ return get_vision_encoder_info(self._get_hf_config())
57
64
- def _get_mm_fields_config(
65
- self,
66
- hf_inputs: BatchFeature,
67
- hf_processor_mm_kwargs: Mapping[str, object],
68
- ) -> Mapping[str, MultiModalFieldConfig]:
69
- return dict(pixel_values_videos=MultiModalFieldConfig.batched("video"))
58
+ def _get_hf_processor(self):
59
+ return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
60
61
def _get_num_frame_tokens(
62
self,
@@ -77,7 +67,8 @@ class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
67
hf_config = self._get_hf_config()
68
spatial_pool_stride = hf_config.spatial_pool_stride
69
80
- patch_grid_length = self._vision_encoder_info.get_patch_grid_length()
70
+ vision_encoder_info = self._get_vision_encoder_info()
71
+ patch_grid_length = vision_encoder_info.get_patch_grid_length()
72
pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
73
74
return pooled_grid_length * pooled_grid_length
@@ -96,18 +87,43 @@ class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
87
88
return num_frame_tokens * num_frames
89
99
- def _get_max_video_tokens(self, num_frames: int) -> int:
100
- return self._get_num_video_tokens(image_width=999999,
101
- image_height=999999,
102
- num_frames=num_frames)
90
+
91
+class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
92
+ BaseProfilingInfo):
93
+
94
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
95
+ return {"video": 1}
96
+
97
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
98
+ target_width, target_height = self._get_image_size_with_most_features()
99
+
100
+ max_video_tokens = self._get_num_video_tokens(
101
+ image_width=target_width,
102
+ image_height=target_height,
103
+ num_frames=self._get_dummy_num_frames(seq_len),
104
+ )
105
+
106
+ return {"video": max_video_tokens}
107
+
108
+ def _get_image_size_with_most_features(self) -> ImageSize:
109
+ vision_encoder_info = self._get_vision_encoder_info()
110
+ width = height = vision_encoder_info.get_image_size()
111
+ return ImageSize(width=width, height=height)
112
113
def _get_max_video_frames(self, max_tokens: int) -> int:
114
+ target_width, target_height = self._get_image_size_with_most_features()
115
+
116
num_frames = 0
117
118
while True:
119
next_num_frames = num_frames + 1
120
+ next_max_tokens = self._get_num_video_tokens(
121
+ image_width=target_width,
122
+ image_height=target_height,
123
+ num_frames=next_num_frames,
124
+ )
125
110
- if self._get_max_video_tokens(next_num_frames) > max_tokens:
126
+ if next_max_tokens > max_tokens:
127
break
128
129
num_frames = next_num_frames
@@ -122,12 +138,45 @@ class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
138
139
return max(max_total_frames // max(max_videos, 1), 1)
140
125
- def _get_dummy_image_size(self) -> ImageSize:
126
- image_size = self._vision_encoder_info.get_image_size()
127
- return ImageSize(image_size, image_size)
141
+ def get_dummy_processor_inputs(
142
+ self,
143
+ seq_len: int,
144
+ mm_counts: Mapping[str, int],
145
+ ) -> ProcessorInputs:
146
+ num_videos = mm_counts.get("video", 0)
147
+
148
+ processor = self._get_hf_processor()
149
+ video_token = processor.video_token
150
+ target_width, target_height = self._get_image_size_with_most_features()
151
+
152
+ mm_data = {
153
+ "video":
154
+ self._get_dummy_videos(
155
+ width=target_width,
156
+ height=target_height,
157
+ num_frames=self._get_dummy_num_frames(seq_len),
158
+ num_videos=num_videos,
159
+ )
160
+ }
161
+
162
+ return ProcessorInputs(
163
+ prompt_text=video_token * num_videos,
164
+ mm_data=mm_data,
165
+ )
166
+
167
129
- def _get_video_token(self) -> str:
130
- return self._get_hf_processor().video_token
168
+class LlavaNextVideoMultiModalProcessor(LlavaNextVideoProcessingMixin,
169
+ BaseMultiModalProcessor):
170
+
171
+ def _get_profiling_info(self) -> BaseProfilingInfo:
172
+ return LlavaNextVideoProfilingInfo(self.ctx)
173
+
174
+ def _get_mm_fields_config(
175
+ self,
176
+ hf_inputs: BatchFeature,
177
+ hf_processor_mm_kwargs: Mapping[str, object],
178
+ ) -> Mapping[str, MultiModalFieldConfig]:
179
+ return dict(pixel_values_videos=MultiModalFieldConfig.batched("video"))
180
181
def _get_prompt_replacements(
182
self,
@@ -162,36 +211,11 @@ class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
211
),
212
]
213
165
- def _get_dummy_processor_inputs(
166
- self,
167
- seq_len: int,
168
- mm_counts: Mapping[str, int],
169
- ) -> ProcessorInputs:
170
- num_videos = mm_counts.get("video", 0)
171
-
172
- video_token = self._get_video_token()
173
- target_width, target_height = self._get_dummy_image_size()
174
-
175
- mm_data = {
176
- "video":
177
- self._get_dummy_videos(
178
- width=target_width,
179
- height=target_height,
180
- num_frames=self._get_dummy_num_frames(seq_len),
181
- num_videos=num_videos,
182
- )
183
- }
184
-
185
- return ProcessorInputs(
186
- prompt_text=video_token * num_videos,
187
- mm_data=mm_data,
188
- )
189
-
214
215
# adopted from transformers modeling_llava_next_video.py
216
class LlavaNextVideoPooler(nn.Module):
217
194
- def __init__(self, config):
218
+ def __init__(self, config: LlavaNextVideoConfig):
219
super().__init__()
220
221
mode = config.spatial_pool_mode
@@ -209,7 +233,7 @@ class LlavaNextVideoPooler(nn.Module):
233
raise ValueError(
234
f"Unknown pooling mode: {mode}. Expected [`average`, `max`]")
235
212
- def forward(self, image_features):
236
+ def forward(self, image_features: torch.Tensor):
237
ori_width = int(
238
math.sqrt(image_features.shape[1] * self.image_size //
239
self.image_size))
vllm/model_executor/models/llava_onevision.py
+98
-76
@@ -1,7 +1,7 @@
1
import math
2
from functools import cached_property
3
-from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
- TypedDict, Union)
3
+from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4
+ Protocol, Set, Tuple, TypedDict, Union)
5
6
import numpy as np
7
import torch
@@ -21,15 +21,16 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
21
from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
22
from vllm.multimodal.parse import (MultiModalDataItems, VideoEmbeddingItems,
23
VideoProcessorItems)
24
-from vllm.multimodal.processing import (MultiModalFieldConfig, ProcessorInputs,
25
- PromptReplacement)
24
+from vllm.multimodal.processing import MultiModalFieldConfig, PromptReplacement
25
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
26
from vllm.sequence import IntermediateTensors
27
from vllm.utils import is_list_of
28
29
from .clip import CLIPVisionModel
30
from .interfaces import SupportsMultiModal, SupportsPP
31
-from .llava import init_vision_tower_for_llava
32
-from .llava_next import LlavaNextMultiModalProcessor
31
+from .llava import BaseLlavaProfilingInfo, init_vision_tower_for_llava
32
+from .llava_next import (LlavaNextLikeConfig, LlavaNextMultiModalProcessor,
33
+ LlavaNextProcessingMixin)
34
from .siglip import SiglipVisionModel
35
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36
maybe_prefix, merge_multimodal_embeddings)
@@ -82,39 +83,17 @@ LlavaOnevisionMultiInputs = Union[LlavaOnevisionImageInputs,
83
LlavaOnevisionVideoPixelInputs]
84
85
85
-class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
86
+class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
87
+ video_token_index: Final[int]
88
87
- def _get_hf_config(self) -> LlavaOnevisionConfig:
88
- return self.ctx.get_hf_config(LlavaOnevisionConfig)
89
-
90
- def _get_hf_processor(self) -> LlavaOnevisionProcessor:
91
- return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
89
93
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
94
- return {"image": None, "video": None}
95
-
96
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
97
- max_image_tokens = self._get_max_image_tokens()
90
+class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
91
99
- num_frames = self._get_dummy_num_frames(seq_len)
100
- max_video_tokens = self._get_max_video_tokens(num_frames)
101
-
102
- return {
103
- "image": max_image_tokens,
104
- "video": max_video_tokens,
105
- }
92
+ def _get_hf_config(self) -> LlavaOnevisionLikeConfig:
93
+ return self.ctx.get_hf_config(LlavaOnevisionConfig)
94
107
- def _get_mm_fields_config(
108
- self,
109
- hf_inputs: BatchFeature,
110
- hf_processor_mm_kwargs: Mapping[str, object],
111
- ) -> Mapping[str, MultiModalFieldConfig]:
112
- return dict(
113
- pixel_values=MultiModalFieldConfig.batched("image"),
114
- image_sizes=MultiModalFieldConfig.batched("image"),
115
- image_embeds=MultiModalFieldConfig.batched("image"),
116
- pixel_values_videos=MultiModalFieldConfig.batched("video"),
117
- )
95
+ def _get_hf_processor(self):
96
+ return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
97
98
def _get_num_unpadded_features(
99
self,
@@ -128,7 +107,7 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
107
current_height = npatches * num_patch_height
108
current_width = npatches * num_patch_width
109
131
- # NOTE: HF resizes based on float32
110
+ # NOTE: Use float32 to remain consistent with HF output
111
original_aspect_ratio = np.array(original_width / original_height,
112
dtype=np.float32)
113
current_aspect_ratio = np.array(current_width / current_height,
@@ -167,7 +146,8 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
146
hf_config = self._get_hf_config()
147
spatial_pool_stride = getattr(hf_config, "spatial_pool_stride", 2)
148
170
- patch_grid_length = self._vision_encoder_info.get_patch_grid_length()
149
+ vision_encoder_info = self._get_vision_encoder_info()
150
+ patch_grid_length = vision_encoder_info.get_patch_grid_length()
151
pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
152
153
return pooled_grid_length * pooled_grid_length
@@ -186,18 +166,33 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
166
167
return num_frame_tokens * num_frames + 1 # Newline token
168
189
- def _get_max_video_tokens(self, num_frames: int) -> int:
190
- return self._get_num_video_tokens(image_width=999999,
191
- image_height=999999,
192
- num_frames=num_frames)
169
+
170
+class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
171
+ BaseLlavaProfilingInfo):
172
+
173
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
174
+ return {"image": None, "video": None}
175
+
176
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
177
+ return {
178
+ "image": self._get_max_image_tokens(),
179
+ "video": self._get_max_video_tokens(seq_len),
180
+ }
181
182
def _get_max_video_frames(self, max_tokens: int) -> int:
183
+ target_width, target_height = self._get_image_size_with_most_features()
184
+
185
num_frames = 0
186
187
while True:
188
next_num_frames = num_frames + 1
189
+ next_max_tokens = self._get_num_video_tokens(
190
+ image_width=target_width,
191
+ image_height=target_height,
192
+ num_frames=next_num_frames,
193
+ )
194
200
- if self._get_max_video_tokens(next_num_frames) > max_tokens:
195
+ if next_max_tokens > max_tokens:
196
break
197
198
num_frames = next_num_frames
@@ -215,8 +210,65 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
210
211
return max(max_total_frames // max(max_videos, 1), 1)
212
218
- def _get_video_token(self) -> str:
219
- return self._get_hf_processor().video_token
213
+ def _get_max_video_tokens(self, seq_len: int) -> int:
214
+ target_width, target_height = self._get_image_size_with_most_features()
215
+
216
+ return self._get_num_video_tokens(
217
+ image_width=target_width,
218
+ image_height=target_height,
219
+ num_frames=self._get_dummy_num_frames(seq_len),
220
+ )
221
+
222
+ def get_dummy_processor_inputs(
223
+ self,
224
+ seq_len: int,
225
+ mm_counts: Mapping[str, int],
226
+ ) -> ProcessorInputs:
227
+ num_images = mm_counts.get("image", 0)
228
+ num_videos = mm_counts.get("video", 0)
229
+
230
+ processor = self._get_hf_processor()
231
+ image_token = processor.image_token
232
+ video_token = processor.video_token
233
+ target_width, target_height = self._get_image_size_with_most_features()
234
+
235
+ mm_data = {
236
+ "image":
237
+ self._get_dummy_images(width=target_width,
238
+ height=target_height,
239
+ num_images=num_images),
240
+ "video":
241
+ self._get_dummy_videos(
242
+ width=target_width,
243
+ height=target_height,
244
+ num_frames=self._get_dummy_num_frames(seq_len),
245
+ num_videos=num_videos,
246
+ )
247
+ }
248
+
249
+ return ProcessorInputs(
250
+ prompt_text=image_token * num_images + video_token * num_videos,
251
+ mm_data=mm_data,
252
+ )
253
+
254
+
255
+class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
256
+ LlavaNextMultiModalProcessor):
257
+
258
+ def _get_profiling_info(self) -> BaseProfilingInfo:
259
+ return LlavaOnevisionProfilingInfo(self.ctx)
260
+
261
+ def _get_mm_fields_config(
262
+ self,
263
+ hf_inputs: BatchFeature,
264
+ hf_processor_mm_kwargs: Mapping[str, object],
265
+ ) -> Mapping[str, MultiModalFieldConfig]:
266
+ return dict(
267
+ pixel_values=MultiModalFieldConfig.batched("image"),
268
+ image_sizes=MultiModalFieldConfig.batched("image"),
269
+ image_embeds=MultiModalFieldConfig.batched("image"),
270
+ pixel_values_videos=MultiModalFieldConfig.batched("video"),
271
+ )
272
273
def _call_hf_processor(
274
self,
@@ -235,7 +287,8 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
287
mm_kwargs=mm_kwargs,
288
)
289
238
- video_token = self._get_video_token()
290
+ processor = self._get_hf_processor()
291
+ video_token = processor.video_token
292
293
# LLaVA-OneVision processor doesn't support multiple videos
294
# with different sizes when converting back to tensors
@@ -303,37 +356,6 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
356
),
357
]
358
306
- def _get_dummy_processor_inputs(
307
- self,
308
- seq_len: int,
309
- mm_counts: Mapping[str, int],
310
- ) -> ProcessorInputs:
311
- num_images = mm_counts.get("image", 0)
312
- num_videos = mm_counts.get("video", 0)
313
-
314
- image_token = self._get_image_token()
315
- video_token = self._get_video_token()
316
- target_width, target_height = self._get_dummy_image_size()
317
-
318
- mm_data = {
319
- "image":
320
- self._get_dummy_images(width=target_width,
321
- height=target_height,
322
- num_images=num_images),
323
- "video":
324
- self._get_dummy_videos(
325
- width=target_width,
326
- height=target_height,
327
- num_frames=self._get_dummy_num_frames(seq_len),
328
- num_videos=num_videos,
329
- )
330
- }
331
-
332
- return ProcessorInputs(
333
- prompt_text=image_token * num_images + video_token * num_videos,
334
- mm_data=mm_data,
335
- )
336
-
359
360
class LlavaOnevisionMultiModalProjector(nn.Module):
361
vllm/model_executor/models/phi3v.py
+59
-45
@@ -14,7 +14,7 @@
14
# limitations under the License.
15
from collections.abc import Iterable, Mapping, Sequence
16
from functools import cached_property
17
-from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
17
+from typing import Any, List, Literal, Optional, Set, Tuple, TypedDict, Union
18
19
import torch
20
import torch.nn as nn
@@ -28,22 +28,23 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
28
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
29
from vllm.model_executor.layers.vocab_parallel_embedding import (
30
VocabParallelEmbedding)
31
-from vllm.model_executor.models.clip import CLIPVisionModel
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
MultiModalInputsV2, MultiModalKwargs,
35
NestedTensors, PlaceholderRange)
37
-from vllm.multimodal.parse import ImageEmbeddingItems, ImageProcessorItems
36
+from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
37
+ ImageSize)
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
- MultiModalDataItems, ProcessorInputs,
39
+ MultiModalDataItems, ProcessingMixin,
40
PromptReplacement,
41
_BoundPromptReplacement,
42
_PlaceholderInfo)
43
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
44
from vllm.sequence import IntermediateTensors
45
from vllm.utils import is_list_of
46
46
-from .clip import dummy_image_for_clip
47
+from .clip import CLIPVisionModel
48
from .interfaces import SupportsMultiModal, SupportsPP
49
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
50
init_vllm_registered_model, maybe_prefix,
@@ -54,10 +55,6 @@ logger = init_logger(__name__)
55
# Cannot find the following 2 numbers from hf config.
56
_IMAGE_TOKEN_ID = 32044
57
57
-# Result in the max possible feature size (h:w = 16:1)
58
-MAX_IMAGE_FEATURE_SIZE_HEIGHT = 8000
59
-MAX_IMAGE_FEATURE_SIZE_WIDTH = 50
60
-
58
CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
59
hidden_act="quick_gelu",
60
hidden_size=1024,
@@ -305,10 +302,17 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
302
return image_features_hd_newline
303
304
308
-class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
305
+class Phi3VProcessingMixin(ProcessingMixin):
306
310
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
311
- return {"image": None}
307
+ def _get_hf_processor(
308
+ self,
309
+ *,
310
+ num_crops: Optional[int] = None,
311
+ ) -> ProcessorMixin:
312
+ if num_crops is not None:
313
+ return self.ctx.get_hf_processor(num_crops=num_crops)
314
+
315
+ return self.ctx.get_hf_processor()
316
317
def _get_num_image_tokens(
318
self,
@@ -323,23 +327,55 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
327
height=image_height,
328
)
329
330
+
331
+class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
332
+
333
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
334
+ return {"image": None}
335
+
336
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
337
+ target_width, target_height = self._get_image_size_with_most_features()
338
+
339
max_image_tokens = self._get_num_image_tokens(
328
- image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
329
- image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
340
+ image_width=target_width,
341
+ image_height=target_height,
342
)
343
344
return {"image": max_image_tokens}
345
334
- def _get_hf_processor(
346
+ def _get_image_size_with_most_features(self) -> ImageSize:
347
+ # Result in the max possible feature size (h:w = 16:1)
348
+ return ImageSize(height=8000, width=50)
349
+
350
+ def get_dummy_processor_inputs(
351
self,
336
- *,
337
- num_crops: Optional[int] = None,
338
- ) -> ProcessorMixin:
339
- if num_crops is not None:
340
- return self.ctx.get_hf_processor(num_crops=num_crops)
352
+ seq_len: int,
353
+ mm_counts: Mapping[str, int],
354
+ ) -> ProcessorInputs:
355
+ num_images = mm_counts.get("image", 0)
356
342
- return self.ctx.get_hf_processor()
357
+ target_width, target_height = self._get_image_size_with_most_features()
358
+
359
+ mm_data = {
360
+ "image":
361
+ self._get_dummy_images(width=target_width,
362
+ height=target_height,
363
+ num_images=num_images)
364
+ }
365
+
366
+ hf_processor = self._get_hf_processor()
367
+ image_tokens: list[str] = hf_processor.img_tokens # type: ignore
368
+
369
+ return ProcessorInputs(
370
+ prompt_text="".join(image_tokens[:num_images]),
371
+ mm_data=mm_data,
372
+ )
373
+
374
+
375
+class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
376
+
377
+ def _get_profiling_info(self) -> BaseProfilingInfo:
378
+ return Phi3VProfilingInfo(self.ctx)
379
380
def _call_hf_processor(
381
self,
@@ -377,10 +413,10 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
413
def _get_prompt_replacements(
414
self,
415
mm_items: MultiModalDataItems,
380
- hf_processor_mm_kwargs: Mapping[str, object],
416
+ hf_processor_mm_kwargs: Mapping[str, Any],
417
out_mm_kwargs: MultiModalKwargs,
418
) -> list[PromptReplacement]:
383
- hf_processor = self._get_hf_processor()
419
+ hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
420
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
421
422
tokenizer = self._get_tokenizer()
@@ -442,28 +478,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
478
479
return token_ids, text, placeholders
480
445
- def _get_dummy_processor_inputs(
446
- self,
447
- seq_len: int,
448
- mm_counts: Mapping[str, int],
449
- ) -> ProcessorInputs:
450
- num_images = mm_counts.get("image", 0)
451
-
452
- data = dummy_image_for_clip(
453
- CLIP_VIT_LARGE_PATCH14_336_CONFIG,
454
- num_images,
455
- image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
456
- image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
457
- )
458
-
459
- hf_processor = self._get_hf_processor()
460
- image_tokens: list[str] = hf_processor.img_tokens # type: ignore
461
-
462
- return ProcessorInputs(
463
- prompt_text="".join(image_tokens[:num_images]),
464
- mm_data=data,
465
- )
466
-
481
def apply(
482
self,
483
prompt_text: str,
vllm/model_executor/models/qwen2_audio.py
+59
-37
@@ -20,8 +20,8 @@
20
# limitations under the License.
21
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22
from functools import cached_property
23
-from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24
- Union)
23
+from typing import (Any, Iterable, List, Mapping, Optional, Set, Tuple,
24
+ TypedDict, Union)
25
26
import torch
27
import torch.nn as nn
@@ -40,8 +40,9 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40
NestedTensors)
41
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataParser
42
from vllm.multimodal.processing import (BaseMultiModalProcessor,
43
- MultiModalDataItems, ProcessorInputs,
43
+ MultiModalDataItems, ProcessingMixin,
44
PromptReplacement)
45
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
46
from vllm.sequence import IntermediateTensors
47
48
from .interfaces import SupportsMultiModal, SupportsPP
@@ -79,28 +80,70 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
80
return feat_lengths, output_lengths
81
82
82
-class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
83
+class Qwen2AudioProcessingMixin(ProcessingMixin):
84
+
85
+ def _get_hf_config(self):
86
+ return self.ctx.get_hf_config(Qwen2AudioConfig)
87
+
88
+ def _get_hf_processor(
89
+ self,
90
+ *,
91
+ # Ignored in initialization
92
+ sampling_rate: Optional[int] = None,
93
+ ) -> Qwen2AudioProcessor:
94
+ return self.ctx.get_hf_processor(Qwen2AudioProcessor)
95
+
96
+ def _get_feature_extractor(
97
+ self,
98
+ *,
99
+ # Ignored in initialization
100
+ sampling_rate: Optional[int] = None,
101
+ ) -> WhisperFeatureExtractor:
102
+ hf_processor = self._get_hf_processor(sampling_rate=sampling_rate)
103
+ feature_extractor = hf_processor.feature_extractor # type: ignore
104
+ assert isinstance(feature_extractor, WhisperFeatureExtractor)
105
+ return feature_extractor
106
+
107
+
108
+class Qwen2AudioProfilingInfo(Qwen2AudioProcessingMixin, BaseProfilingInfo):
109
110
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
111
return {"audio": None}
112
113
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
88
- hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
114
+ hf_config = self._get_hf_config()
115
max_source_positions = hf_config.audio_config.max_source_positions
116
max_output_lengths = (max_source_positions - 2) // 2 + 1
117
118
return {"audio": max_output_lengths}
119
94
- def _get_hf_processor(
120
+ def get_dummy_processor_inputs(
121
self,
96
- *,
97
- # Ignored in initialization
98
- sampling_rate: Optional[int] = None,
99
- ) -> Qwen2AudioProcessor:
100
- return self.ctx.get_hf_processor(Qwen2AudioProcessor)
122
+ seq_len: int,
123
+ mm_counts: Mapping[str, int],
124
+ ) -> ProcessorInputs:
125
+ feature_extractor = self._get_feature_extractor()
126
+
127
+ sampling_rate = feature_extractor.sampling_rate
128
+ audio_len = feature_extractor.chunk_length * sampling_rate
129
+ num_audios = mm_counts.get("audio", 0)
130
+
131
+ mm_data = {
132
+ "audio":
133
+ self._get_dummy_audios(length=audio_len, num_audios=num_audios)
134
+ }
135
+
136
+ return ProcessorInputs(
137
+ prompt_text="<|AUDIO|>" * num_audios,
138
+ mm_data=mm_data,
139
+ )
140
+
141
102
- def _get_feature_extractor(self) -> WhisperFeatureExtractor:
103
- return self._get_hf_processor().feature_extractor # type: ignore
142
+class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
143
+ BaseMultiModalProcessor):
144
+
145
+ def _get_profiling_info(self) -> BaseProfilingInfo:
146
+ return Qwen2AudioProfilingInfo(self.ctx)
147
148
def _get_data_parser(self) -> MultiModalDataParser:
149
feature_extractor = self._get_feature_extractor()
@@ -110,7 +153,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
153
self,
154
prompt: str,
155
mm_data: Mapping[str, object],
113
- mm_kwargs: Mapping[str, object],
156
+ mm_kwargs: Mapping[str, Any],
157
) -> BatchFeature:
158
mm_data = dict(mm_data)
159
audios = mm_data.pop("audios", [])
@@ -118,7 +161,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
161
if audios:
162
mm_data["audios"] = audios
163
121
- feature_extractor = self._get_feature_extractor()
164
+ feature_extractor = self._get_feature_extractor(**mm_kwargs)
165
mm_kwargs = dict(
166
**mm_kwargs,
167
sampling_rate=feature_extractor.sampling_rate,
@@ -151,7 +194,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
194
hf_processor_mm_kwargs: Mapping[str, object],
195
out_mm_kwargs: MultiModalKwargs,
196
) -> list[PromptReplacement]:
154
- hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
197
+ hf_config = self._get_hf_config()
198
placeholder = hf_config.audio_token_index
199
200
feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
@@ -191,27 +234,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
234
# tokens than the number of audio items)
235
return True
236
194
- def _get_dummy_processor_inputs(
195
- self,
196
- seq_len: int,
197
- mm_counts: Mapping[str, int],
198
- ) -> ProcessorInputs:
199
- feature_extractor = self._get_feature_extractor()
200
-
201
- sampling_rate = feature_extractor.sampling_rate
202
- audio_len = feature_extractor.chunk_length * sampling_rate
203
- num_audios = mm_counts.get("audio", 0)
204
-
205
- mm_data = {
206
- "audio":
207
- self._get_dummy_audios(length=audio_len, num_audios=num_audios)
208
- }
209
-
210
- return ProcessorInputs(
211
- prompt_text="<|AUDIO|>" * num_audios,
212
- mm_data=mm_data,
213
- )
214
-
237
238
@MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)
239
class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
vllm/model_executor/models/qwen2_vl.py
+144
-87
@@ -59,8 +59,9 @@ from vllm.multimodal.inputs import (ImageItem, ModalityData,
59
from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
60
MultiModalDataParser)
61
from vllm.multimodal.processing import (BaseMultiModalProcessor,
62
- MultiModalDataItems, ProcessorInputs,
62
+ MultiModalDataItems, ProcessingMixin,
63
PromptReplacement)
64
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
65
from vllm.platforms import _Backend
66
from vllm.sequence import IntermediateTensors
67
from vllm.transformers_utils.config import uses_mrope
@@ -708,10 +709,44 @@ class Qwen2MultiModalDataParser(MultiModalDataParser):
709
return super()._parse_video_data(data)
710
711
711
-class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
712
+class Qwen2VLProcessingMixin(ProcessingMixin):
713
713
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
714
- return {"image": None, "video": None}
714
+ def _get_hf_config(self):
715
+ return self.ctx.get_hf_config(Qwen2VLConfig)
716
+
717
+ def _get_hf_processor(
718
+ self,
719
+ *,
720
+ min_pixels: Optional[int] = None,
721
+ max_pixels: Optional[int] = None,
722
+ ) -> Qwen2VLProcessor:
723
+ hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
724
+ image_processor = hf_processor.image_processor # type: ignore
725
+ assert isinstance(image_processor, Qwen2VLImageProcessor)
726
+
727
+ if min_pixels:
728
+ image_processor.min_pixels = min_pixels
729
+ if max_pixels:
730
+ image_processor.max_pixels = max_pixels
731
+ if max_pixels or min_pixels:
732
+ image_processor.size = {
733
+ "min_pixels": image_processor.min_pixels,
734
+ "max_pixels": image_processor.max_pixels,
735
+ }
736
+
737
+ return hf_processor
738
+
739
+ def _get_image_processor(
740
+ self,
741
+ *,
742
+ min_pixels: Optional[int] = None,
743
+ max_pixels: Optional[int] = None,
744
+ ):
745
+ hf_processor = self._get_hf_processor(min_pixels=min_pixels,
746
+ max_pixels=max_pixels)
747
+ image_processor = hf_processor.image_processor # type: ignore
748
+ assert isinstance(image_processor, Qwen2VLImageProcessor)
749
+ return image_processor
750
751
def _get_vision_info(
752
self,
@@ -721,14 +756,13 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
756
num_frames: int = 1,
757
do_resize: bool = True,
758
) -> tuple[ImageSize, int]:
724
- hf_config = self.ctx.get_hf_config(Qwen2VLConfig)
759
+ hf_config = self._get_hf_config()
760
vision_config = hf_config.vision_config
761
patch_size = vision_config.patch_size
762
merge_size = vision_config.spatial_merge_size
763
temporal_patch_size = vision_config.temporal_patch_size
764
730
- hf_processor = self._get_hf_processor()
731
- image_processor = self._get_image_processor(hf_processor)
765
+ image_processor = self._get_image_processor()
766
767
if do_resize:
768
resized_height, resized_width = smart_resize(
@@ -753,7 +787,45 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
787
788
return preprocessed_size, num_vision_tokens
789
756
- def _get_dummy_image_size(self) -> ImageSize:
790
+ def _get_num_image_tokens(
791
+ self,
792
+ *,
793
+ image_width: int,
794
+ image_height: int,
795
+ ) -> int:
796
+ _, num_image_tokens = self._get_vision_info(
797
+ image_width=image_width,
798
+ image_height=image_height,
799
+ )
800
+ return num_image_tokens
801
+
802
+ def _get_num_video_tokens(
803
+ self,
804
+ *,
805
+ image_width: int,
806
+ image_height: int,
807
+ num_frames: int,
808
+ ) -> int:
809
+ _, num_video_tokens = self._get_vision_info(
810
+ image_width=image_width,
811
+ image_height=image_height,
812
+ num_frames=num_frames,
813
+ )
814
+ return num_video_tokens
815
+
816
+
817
+class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
818
+
819
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
820
+ return {"image": None, "video": None}
821
+
822
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
823
+ return {
824
+ "image": self._get_max_image_tokens(),
825
+ "video": self._get_max_video_tokens(seq_len),
826
+ }
827
+
828
+ def _get_image_size_with_most_features(self) -> ImageSize:
829
max_image_size, _ = self._get_vision_info(
830
image_width=9999999,
831
image_height=9999999,
@@ -761,27 +833,27 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
833
return max_image_size
834
835
def _get_max_image_tokens(self) -> int:
764
- _, max_image_tokens = self._get_vision_info(
765
- image_width=9999999,
766
- image_height=9999999,
767
- )
768
- return max_image_tokens
836
+ target_width, target_height = self._get_image_size_with_most_features()
837
770
- def _get_max_video_tokens(self, num_frames: int) -> int:
771
- _, max_video_tokens = self._get_vision_info(
772
- image_width=9999999,
773
- image_height=9999999,
774
- num_frames=num_frames,
838
+ return self._get_num_image_tokens(
839
+ image_width=target_width,
840
+ image_height=target_height,
841
)
776
- return max_video_tokens
842
843
def _get_max_video_frames(self, max_tokens: int) -> int:
844
+ target_width, target_height = self._get_image_size_with_most_features()
845
+
846
num_frames = 0
847
848
while True:
849
next_num_frames = num_frames + 1
850
+ next_max_tokens = self._get_num_video_tokens(
851
+ image_width=target_width,
852
+ image_height=target_height,
853
+ num_frames=next_num_frames,
854
+ )
855
784
- if self._get_max_video_tokens(next_num_frames) > max_tokens:
856
+ if next_max_tokens > max_tokens:
857
break
858
859
num_frames = next_num_frames
@@ -797,56 +869,73 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
869
max_total_frames = self._get_max_video_frames(seq_len -
870
max_image_tokens)
871
800
- return max(max_total_frames // max(max_videos, 1), 1)
872
+ num_frames = max(max_total_frames // max(max_videos, 1), 1)
873
802
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
803
- max_image_tokens = self._get_max_image_tokens()
874
+ # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
875
+ if num_frames > 1 and num_frames % 2 == 1:
876
+ num_frames += 1
877
805
- num_frames = self._get_dummy_num_frames(seq_len)
806
- max_video_tokens = self._get_max_video_tokens(num_frames)
878
+ return num_frames
879
808
- return {
809
- "image": max_image_tokens,
810
- "video": max_video_tokens,
880
+ def _get_max_video_tokens(self, seq_len: int) -> int:
881
+ target_width, target_height = self._get_image_size_with_most_features()
882
+
883
+ return self._get_num_video_tokens(
884
+ image_width=target_width,
885
+ image_height=target_height,
886
+ num_frames=self._get_dummy_num_frames(seq_len),
887
+ )
888
+
889
+ def get_dummy_processor_inputs(
890
+ self,
891
+ seq_len: int,
892
+ mm_counts: Mapping[str, int],
893
+ ) -> ProcessorInputs:
894
+ num_images = mm_counts.get("image", 0)
895
+ num_videos = mm_counts.get("video", 0)
896
+
897
+ hf_processor = self._get_hf_processor()
898
+ image_token: str = hf_processor.image_token
899
+ video_token: str = hf_processor.video_token
900
+ target_width, target_height = self._get_image_size_with_most_features()
901
+
902
+ mm_data = {
903
+ "image":
904
+ self._get_dummy_images(width=target_width,
905
+ height=target_height,
906
+ num_images=num_images),
907
+ "video":
908
+ self._get_dummy_videos(
909
+ width=target_width,
910
+ height=target_height,
911
+ num_frames=self._get_dummy_num_frames(seq_len),
912
+ num_videos=num_videos,
913
+ )
914
}
915
813
- def _get_data_parser(self) -> MultiModalDataParser:
814
- return Qwen2MultiModalDataParser()
916
+ return ProcessorInputs(
917
+ prompt_text=image_token * num_images + video_token * num_videos,
918
+ mm_data=mm_data,
919
+ )
920
816
- def _get_image_processor(self, hf_processor: Qwen2VLProcessor):
817
- image_processor = hf_processor.image_processor # type: ignore
818
- assert isinstance(image_processor, Qwen2VLImageProcessor)
819
- return image_processor
921
821
- def _get_hf_processor(
822
- self,
823
- *,
824
- min_pixels: Optional[int] = None,
825
- max_pixels: Optional[int] = None,
826
- ) -> Qwen2VLProcessor:
827
- hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
828
- image_processor = self._get_image_processor(hf_processor)
922
+class Qwen2VLMultiModalProcessor(Qwen2VLProcessingMixin,
923
+ BaseMultiModalProcessor):
924
830
- if min_pixels:
831
- image_processor.min_pixels = min_pixels
832
- if max_pixels:
833
- image_processor.max_pixels = max_pixels
834
- if max_pixels or min_pixels:
835
- image_processor.size = {
836
- "min_pixels": image_processor.min_pixels,
837
- "max_pixels": image_processor.max_pixels,
838
- }
925
+ def _get_profiling_info(self) -> BaseProfilingInfo:
926
+ return Qwen2VLProfilingInfo(self.ctx)
927
840
- return hf_processor
928
+ def _get_data_parser(self) -> MultiModalDataParser:
929
+ return Qwen2MultiModalDataParser()
930
931
def _get_prompt_replacements(
932
self,
933
mm_items: MultiModalDataItems,
845
- hf_processor_mm_kwargs: Mapping[str, object],
934
+ hf_processor_mm_kwargs: Mapping[str, Any],
935
out_mm_kwargs: MultiModalKwargs,
936
) -> list[PromptReplacement]:
848
- hf_processor = self._get_hf_processor()
849
- image_processor = self._get_image_processor(hf_processor)
937
+ hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
938
+ image_processor = self._get_image_processor(**hf_processor_mm_kwargs)
939
940
# NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
941
# image_token and video_token registered
@@ -901,38 +990,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
990
video_grid_thw=MultiModalFieldConfig.batched("video"),
991
)
992
904
- def _get_dummy_processor_inputs(
905
- self,
906
- seq_len: int,
907
- mm_counts: Mapping[str, int],
908
- ) -> ProcessorInputs:
909
- num_images = mm_counts.get("image", 0)
910
- num_videos = mm_counts.get("video", 0)
911
-
912
- hf_processor = self._get_hf_processor()
913
- image_token: str = hf_processor.image_token
914
- video_token: str = hf_processor.video_token
915
- target_width, target_height = self._get_dummy_image_size()
916
-
917
- mm_data = {
918
- "image":
919
- self._get_dummy_images(width=target_width,
920
- height=target_height,
921
- num_images=num_images),
922
- "video":
923
- self._get_dummy_videos(
924
- width=target_width,
925
- height=target_height,
926
- num_frames=self._get_dummy_num_frames(seq_len),
927
- num_videos=num_videos,
928
- )
929
- }
930
-
931
- return ProcessorInputs(
932
- prompt_text=image_token * num_images + video_token * num_videos,
933
- mm_data=mm_data,
934
- )
935
-
993
994
@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor)
995
class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
vllm/model_executor/models/ultravox.py
+55
-36
@@ -3,8 +3,8 @@
3
4
import math
5
from functools import cached_property
6
-from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7
- TypedDict, Union)
6
+from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
7
+ Tuple, TypedDict, Union)
8
9
import torch
10
import torch.utils.checkpoint
@@ -26,8 +26,9 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
26
NestedTensors)
27
from vllm.multimodal.parse import MultiModalDataParser
28
from vllm.multimodal.processing import (BaseMultiModalProcessor,
29
- MultiModalDataItems, ProcessorInputs,
29
+ MultiModalDataItems, ProcessingMixin,
30
PromptReplacement)
31
+from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
32
from vllm.sequence import IntermediateTensors
33
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
34
@@ -55,7 +56,30 @@ UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
56
UltravoxAudioEmbeddingInputs]
57
58
58
-class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
59
+class UltravoxProcessingMixin(ProcessingMixin):
60
+
61
+ def _get_hf_processor(
62
+ self,
63
+ *,
64
+ # Ignored in initialization
65
+ sampling_rate: Optional[int] = None,
66
+ ) -> ProcessorMixin:
67
+ return self.ctx.get_hf_processor()
68
+
69
+ def _get_feature_extractor(
70
+ self,
71
+ *,
72
+ # Ignored in initialization
73
+ sampling_rate: Optional[int] = None,
74
+ ) -> WhisperFeatureExtractor:
75
+ hf_processor = self._get_hf_processor(sampling_rate=sampling_rate)
76
+ audio_processor = hf_processor.audio_processor # type: ignore
77
+ feature_extractor = audio_processor.feature_extractor # type: ignore
78
+ assert isinstance(feature_extractor, WhisperFeatureExtractor)
79
+ return feature_extractor
80
+
81
+
82
+class UltravoxProfilingInfo(UltravoxProcessingMixin, BaseProfilingInfo):
83
84
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
85
return {"audio": None}
@@ -67,17 +91,33 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
91
92
return {"audio": max_audio_tokens}
93
70
- def _get_hf_processor(
94
+ def get_dummy_processor_inputs(
95
self,
72
- *,
73
- # Ignored in initialization
74
- sampling_rate: Optional[int] = None,
75
- ) -> ProcessorMixin:
76
- return self.ctx.get_hf_processor()
96
+ seq_len: int,
97
+ mm_counts: Mapping[str, int],
98
+ ) -> ProcessorInputs:
99
+ feature_extractor = self._get_feature_extractor()
100
+
101
+ sampling_rate = feature_extractor.sampling_rate
102
+ audio_len = feature_extractor.chunk_length * sampling_rate
103
+ num_audios = mm_counts.get("audio", 0)
104
+
105
+ mm_data = {
106
+ "audio":
107
+ self._get_dummy_audios(length=audio_len, num_audios=num_audios)
108
+ }
109
+
110
+ return ProcessorInputs(
111
+ prompt_text="<|audio|>" * num_audios,
112
+ mm_data=mm_data,
113
+ )
114
78
- def _get_feature_extractor(self) -> WhisperFeatureExtractor:
79
- hf_processor = self._get_hf_processor()
80
- return hf_processor.audio_processor.feature_extractor # type: ignore
115
+
116
+class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
117
+ BaseMultiModalProcessor):
118
+
119
+ def _get_profiling_info(self) -> BaseProfilingInfo:
120
+ return UltravoxProfilingInfo(self.ctx)
121
122
def _get_data_parser(self) -> MultiModalDataParser:
123
feature_extractor = self._get_feature_extractor()
@@ -155,10 +195,10 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
195
def _get_prompt_replacements(
196
self,
197
mm_items: MultiModalDataItems,
158
- hf_processor_mm_kwargs: Mapping[str, object],
198
+ hf_processor_mm_kwargs: Mapping[str, Any],
199
out_mm_kwargs: MultiModalKwargs,
200
) -> list[PromptReplacement]:
161
- hf_processor = self._get_hf_processor()
201
+ hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
202
placeholder = hf_processor.audio_token_replacement # type: ignore
203
204
def get_replacement_ultravox(item_idx: int):
@@ -173,27 +213,6 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
213
)
214
]
215
176
- def _get_dummy_processor_inputs(
177
- self,
178
- seq_len: int,
179
- mm_counts: Mapping[str, int],
180
- ) -> ProcessorInputs:
181
- feature_extractor = self._get_feature_extractor()
182
-
183
- sampling_rate = feature_extractor.sampling_rate
184
- audio_len = feature_extractor.chunk_length * sampling_rate
185
- num_audios = mm_counts.get("audio", 0)
186
-
187
- mm_data = {
188
- "audio":
189
- self._get_dummy_audios(length=audio_len, num_audios=num_audios)
190
- }
191
-
192
- return ProcessorInputs(
193
- prompt_text="<|audio|>" * num_audios,
194
- mm_data=mm_data,
195
- )
196
-
216
217
class StackAudioFrames(nn.Module):
218
"""
vllm/model_executor/models/vision.py
+8
-29
@@ -1,12 +1,8 @@
1
from abc import ABC, abstractmethod
2
-from typing import Final, Generic, Optional, Protocol, TypeVar
2
+from typing import Final, Generic, Protocol, TypeVar
3
4
from transformers import PretrainedConfig
5
6
-from vllm.multimodal.processing import (BaseMultiModalProcessor,
7
- InputProcessingContext,
8
- ProcessingCache)
9
-
6
_C = TypeVar("_C", bound=PretrainedConfig)
7
8
@@ -43,12 +39,18 @@ class VisionEncoderInfo(ABC, Generic[_C]):
39
raise NotImplementedError
40
41
46
-def vision_encoder_info(vision_config: PretrainedConfig) -> VisionEncoderInfo:
42
+class VisionLanguageConfig(Protocol):
43
+ vision_config: Final[PretrainedConfig]
44
+
45
+
46
+def get_vision_encoder_info(
47
+ hf_config: VisionLanguageConfig) -> VisionEncoderInfo:
48
# Avoid circular imports
49
from .clip import CLIPEncoderInfo, CLIPVisionConfig
50
from .pixtral import PixtralHFEncoderInfo, PixtralVisionConfig
51
from .siglip import SiglipEncoderInfo, SiglipVisionConfig
52
53
+ vision_config = hf_config.vision_config
54
if isinstance(vision_config, CLIPVisionConfig):
55
return CLIPEncoderInfo(vision_config)
56
if isinstance(vision_config, PixtralVisionConfig):
@@ -58,26 +60,3 @@ def vision_encoder_info(vision_config: PretrainedConfig) -> VisionEncoderInfo:
60
61
msg = f"Unsupported vision config: {type(vision_config)}"
62
raise NotImplementedError(msg)
61
-
62
-
63
-class VisionLanguageConfig(Protocol):
64
- vision_config: Final[PretrainedConfig]
65
-
66
-
67
-class BaseVisionLanguageMultiModalProcessor(BaseMultiModalProcessor):
68
-
69
- def __init__(self,
70
- ctx: InputProcessingContext,
71
- *,
72
- cache: Optional[ProcessingCache] = None,
73
- enable_sanity_checks: bool = True) -> None:
74
- super().__init__(ctx,
75
- cache=cache,
76
- enable_sanity_checks=enable_sanity_checks)
77
-
78
- vision_config = self._get_hf_config().vision_config
79
- self._vision_encoder_info = vision_encoder_info(vision_config)
80
-
81
- @abstractmethod
82
- def _get_hf_config(self) -> VisionLanguageConfig:
83
- raise NotImplementedError
vllm/multimodal/processing.py
+43
-109
@@ -8,11 +8,10 @@ from functools import lru_cache
8
from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
9
10
import numpy as np
11
-import numpy.typing as npt
11
import torch
12
from blake3 import blake3
13
from PIL import Image
15
-from transformers import BatchFeature, ProcessorMixin
14
+from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
15
16
from vllm.inputs import DummyData, InputProcessingContext
17
from vllm.logger import init_logger
@@ -24,6 +23,7 @@ from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
23
MultiModalInputsV2, MultiModalKwargs,
24
MultiModalKwargsItem, PlaceholderRange)
25
from .parse import MultiModalDataItems, MultiModalDataParser
26
+from .profiling import BaseProfilingInfo
27
28
logger = init_logger(__name__)
29
@@ -466,14 +466,6 @@ def find_mm_placeholders(
466
return dict(full_groupby_modality(it))
467
468
469
-@dataclass
470
-class ProcessorInputs:
471
- """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
472
- prompt_text: str
473
- mm_data: MultiModalDataDict
474
- hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
475
-
476
-
469
class ProcessingCache:
470
471
def __init__(self, capacity: int) -> None:
@@ -585,9 +577,33 @@ class ProcessingCache:
577
self._cache.put(cache_key, output_kwargs)
578
579
588
-class BaseMultiModalProcessor(ABC):
580
+class ProcessingMixin:
581
+ """
582
+ Contains helper functions to perform processing.
583
+
584
+ Not to be confused with :class:`transformers.ProcessorMixin`.
585
+ """
586
+ ctx: InputProcessingContext
587
+
588
+ def _get_tokenizer(self) -> AnyTokenizer:
589
+ return self.ctx.tokenizer
590
+
591
+ def _get_hf_config(self) -> PretrainedConfig:
592
+ return self.ctx.get_hf_config()
593
+
594
+ def _get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
595
+ """
596
+ Subclasses can override this method to handle
597
+ specific kwargs from model config or user inputs.
598
+ """
599
+ return self.ctx.get_hf_processor(**kwargs)
600
+
601
+
602
+class BaseMultiModalProcessor(ProcessingMixin, ABC):
603
"""
604
Abstract base class to process multi-modal inputs to be used in vLLM.
605
+
606
+ Not to be confused with :class:`transformers.ProcessorMixin`.
607
"""
608
609
def __init__(self,
@@ -601,6 +617,9 @@ class BaseMultiModalProcessor(ABC):
617
self.cache = cache
618
self.enable_sanity_checks = enable_sanity_checks
619
620
+ self.data_parser = self._get_data_parser()
621
+ self.profiling_info = self._get_profiling_info()
622
+
623
def __call__(
624
self,
625
prompt: str,
@@ -609,32 +628,9 @@ class BaseMultiModalProcessor(ABC):
628
) -> MultiModalInputsV2:
629
return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
630
612
- @abstractmethod
613
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
614
- """
615
- Return the maximum supported number of items for each modality.
616
-
617
- A value of `None` means unlimited number of items.
618
-
619
- Omitting a modality from the returned dictionary means that
620
- it is not supported at all.
621
- """
622
- raise NotImplementedError
623
-
624
- @abstractmethod
625
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
626
- """
627
- Get the maximum possible number of tokens per data item
628
- for each modality.
629
-
630
- The dictionary returned by this method should have the same
631
- keys as that returned by :meth:`get_supported_mm_limits`.
632
- """
633
- raise NotImplementedError
634
-
631
def _get_data_parser(self) -> MultiModalDataParser:
632
"""
637
- Construct a data parser to preprocess multi-modal data items
633
+ Construct a parser to preprocess multi-modal data items
634
before passing them to :meth:`_get_hf_mm_data`.
635
636
You can support additional modalities by creating a subclass
@@ -642,15 +638,12 @@ class BaseMultiModalProcessor(ABC):
638
"""
639
return MultiModalDataParser()
640
645
- def _get_hf_processor(self) -> ProcessorMixin:
641
+ def _get_profiling_info(self) -> BaseProfilingInfo:
642
"""
647
- Subclasses can add keyword arguments to this method to accept
648
- additional kwargs from model config or user inputs.
643
+ Get the profiling information to find the worst-case memory usage of
644
+ the model.
645
"""
650
- return self.ctx.get_hf_processor()
651
-
652
- def _get_tokenizer(self) -> AnyTokenizer:
653
- return self.ctx.tokenizer
646
+ raise NotImplementedError
647
648
def _to_mm_items(
649
self,
@@ -660,8 +653,7 @@ class BaseMultiModalProcessor(ABC):
653
Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`
654
before passing them to :meth:`_get_hf_mm_data`.
655
"""
663
- parser = self._get_data_parser()
664
- mm_items = parser.parse_mm_data(mm_data)
656
+ mm_items = self.data_parser.parse_mm_data(mm_data)
657
658
mm_limits = self.ctx.get_mm_config().limit_per_prompt
659
for modality, items in mm_items.items():
@@ -799,7 +791,7 @@ class BaseMultiModalProcessor(ABC):
791
792
# Some HF processors (e.g. Qwen2-VL) expect corresponding
793
# multi-modal tokens to be in the prompt text
802
- dummy_inputs = self._get_dummy_processor_inputs(
794
+ dummy_inputs = self.profiling_info.get_dummy_processor_inputs(
795
self.ctx.model_config.max_model_len,
796
mm_missing_counts,
797
)
@@ -1133,73 +1125,14 @@ class BaseMultiModalProcessor(ABC):
1125
mm_placeholders=mm_placeholder_ranges,
1126
)
1127
1136
- def _get_dummy_audios(
1137
- self,
1138
- *,
1139
- length: int,
1140
- num_audios: int,
1141
- ) -> list[npt.NDArray]:
1142
- audio = np.zeros((length, ))
1143
- return [audio] * num_audios
1144
-
1145
- def _get_dummy_images(
1146
- self,
1147
- *,
1148
- width: int,
1149
- height: int,
1150
- num_images: int,
1151
- ) -> list[Image.Image]:
1152
- image = Image.new("RGB", (width, height), color=0)
1153
- return [image] * num_images
1154
-
1155
- def _get_dummy_videos(
1156
- self,
1157
- *,
1158
- width: int,
1159
- height: int,
1160
- num_frames: int,
1161
- num_videos: int,
1162
- ) -> list[npt.NDArray]:
1163
- video = np.zeros((num_frames, width, height, 3))
1164
- return [video] * num_videos
1165
-
1166
- @abstractmethod
1167
- def _get_dummy_processor_inputs(
1168
- self,
1169
- seq_len: int,
1170
- mm_counts: Mapping[str, int],
1171
- ) -> ProcessorInputs:
1172
- """
1173
- Build the multi-modal portion of the input which, after processing,
1174
- results in `mm_max_tokens` in :meth:`get_dummy_data`.
1175
- """
1176
- raise NotImplementedError
1177
-
1178
- def _get_and_validate_dummy_mm_counts(self) -> Mapping[str, int]:
1179
- mm_limit_per_prompt = self.ctx.get_mm_config().limit_per_prompt
1180
- supported_mm_limits = self.get_supported_mm_limits()
1181
-
1182
- mm_limits = {
1183
- modality: mm_limit_per_prompt.get(modality, 1)
1184
- for modality in supported_mm_limits
1185
- }
1186
-
1187
- for modality, supported_limit in supported_mm_limits.items():
1188
- limit = mm_limits[modality]
1189
- if supported_limit is not None and supported_limit < limit:
1190
- raise ValueError(
1191
- f"You set {modality}={limit} (or defaulted to 1) in "
1192
- f"`--limit-mm-per-prompt`, but this model only supports "
1193
- f"at most {supported_limit} {modality} items.")
1194
-
1195
- return mm_limits
1196
-
1128
def _get_dummy_mm_inputs(
1129
self,
1130
seq_len: int,
1131
mm_counts: Mapping[str, int],
1132
) -> MultiModalInputsV2:
1202
- processor_inputs = self._get_dummy_processor_inputs(seq_len, mm_counts)
1133
+ profiling = self.profiling_info
1134
+ processor_inputs = profiling.get_dummy_processor_inputs(
1135
+ seq_len, mm_counts)
1136
1137
return self.apply(
1138
prompt_text=processor_inputs.prompt_text,
@@ -1211,8 +1144,9 @@ class BaseMultiModalProcessor(ABC):
1144
# Avoid circular import
1145
from vllm.sequence import SequenceData
1146
1214
- mm_counts = self._get_and_validate_dummy_mm_counts()
1215
- mm_max_tokens_per_item = self.get_mm_max_tokens_per_item(seq_len)
1147
+ profiling = self.profiling_info
1148
+ mm_counts = profiling.get_mm_limits()
1149
+ mm_max_tokens_per_item = profiling.get_mm_max_tokens_per_item(seq_len)
1150
if mm_counts.keys() != mm_max_tokens_per_item.keys():
1151
raise AssertionError(
1152
"The keys returned by `get_supported_mm_limits`"
vllm/multimodal/profiling.py
new
+121
@@ -0,0 +1,121 @@
1
+from abc import ABC, abstractmethod
2
+from collections.abc import Mapping
3
+from dataclasses import dataclass, field
4
+from typing import Optional
5
+
6
+import numpy as np
7
+import numpy.typing as npt
8
+from PIL import Image
9
+
10
+from vllm.inputs import InputProcessingContext
11
+from vllm.logger import init_logger
12
+
13
+from .inputs import MultiModalDataDict
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+@dataclass
19
+class ProcessorInputs:
20
+ """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
21
+ prompt_text: str
22
+ mm_data: MultiModalDataDict
23
+ hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
24
+
25
+
26
+class BaseProfilingInfo(ABC):
27
+ """
28
+ Abstract base class that provides the information necessary to profile
29
+ multi-modal models.
30
+ """
31
+
32
+ def __init__(self, ctx: InputProcessingContext) -> None:
33
+ super().__init__()
34
+
35
+ self.ctx = ctx
36
+
37
+ @abstractmethod
38
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
39
+ """
40
+ Return the maximum supported number of items for each modality.
41
+
42
+ A value of `None` means unlimited number of items.
43
+
44
+ Omitting a modality from the returned dictionary means that
45
+ it is not supported at all.
46
+ """
47
+ raise NotImplementedError
48
+
49
+ @abstractmethod
50
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
51
+ """
52
+ Get the maximum possible number of tokens per data item
53
+ for each modality.
54
+
55
+ The dictionary returned by this method should have the same
56
+ keys as that returned by :meth:`get_supported_mm_limits`.
57
+ """
58
+ raise NotImplementedError
59
+
60
+ @abstractmethod
61
+ def get_dummy_processor_inputs(
62
+ self,
63
+ seq_len: int,
64
+ mm_counts: Mapping[str, int],
65
+ ) -> ProcessorInputs:
66
+ """
67
+ Build the multi-modal portion of the input which, after processing,
68
+ results in `mm_max_tokens` in :meth:`get_mm_max_tokens_per_item`.
69
+ """
70
+ raise NotImplementedError
71
+
72
+ def _get_dummy_audios(
73
+ self,
74
+ *,
75
+ length: int,
76
+ num_audios: int,
77
+ ) -> list[npt.NDArray]:
78
+ audio = np.zeros((length, ))
79
+ return [audio] * num_audios
80
+
81
+ def _get_dummy_images(
82
+ self,
83
+ *,
84
+ width: int,
85
+ height: int,
86
+ num_images: int,
87
+ ) -> list[Image.Image]:
88
+ image = Image.new("RGB", (width, height), color=0)
89
+ return [image] * num_images
90
+
91
+ def _get_dummy_videos(
92
+ self,
93
+ *,
94
+ width: int,
95
+ height: int,
96
+ num_frames: int,
97
+ num_videos: int,
98
+ ) -> list[npt.NDArray]:
99
+ video = np.zeros((num_frames, width, height, 3))
100
+ return [video] * num_videos
101
+
102
+ def get_mm_limits(self) -> Mapping[str, int]:
103
+ mm_config = self.ctx.get_mm_config()
104
+ mm_limit_per_prompt = mm_config.limit_per_prompt
105
+
106
+ supported_mm_limits = self.get_supported_mm_limits()
107
+
108
+ mm_limits = {
109
+ modality: mm_limit_per_prompt.get(modality, 1)
110
+ for modality in supported_mm_limits
111
+ }
112
+
113
+ for modality, supported_limit in supported_mm_limits.items():
114
+ limit = mm_limits[modality]
115
+ if supported_limit is not None and supported_limit < limit:
116
+ raise ValueError(
117
+ f"You set {modality}={limit} (or defaulted to 1) in "
118
+ f"`--limit-mm-per-prompt`, but this model only supports "
119
+ f"at most {supported_limit} {modality} items.")
120
+
121
+ return mm_limits
vllm/multimodal/registry.py
+1
-1
@@ -224,7 +224,7 @@ class MultiModalRegistry:
224
tokenizer = cached_get_tokenizer(model_config.tokenizer)
225
processor = self.create_processor(model_config, tokenizer)
226
seq_len = model_config.max_model_len
227
- return processor.get_mm_max_tokens_per_item(seq_len)
227
+ return processor.profiling_info.get_mm_max_tokens_per_item(seq_len)
228
229
return {
230
key: plugin.get_max_multimodal_tokens(model_config)